def UploadFileToHDFS(self,VodTsDir,VodStartTsNum,VodTsNum): PrePicFilePath = VodTsDir + "/" + self.PrePicFileName M3u8FilePath = VodTsDir + "/" + self.M3u8FileName #DebugLog = GetLog(self.cid+"_"+self.M3u8FileName.split(".")[0]) self.logger.info("Prepare upload File to hdfs,File in dir is : %s"%VodTsDir) if os.path.isfile(M3u8FilePath) and os.path.isfile(PrePicFilePath) and len(glob.glob(VodTsDir+"/*.ts")) != 0: WebHdfsObj = WebHadoop(self.HDFSInterfaces,self.HDFSInterfacePort,self.HDFSUser,self.logger) HdfsTsSaveDir = "/vod/%s/%s/%s"%(self.oid,self.cid,self.VodDate) if WebHdfsObj.mkdir(str(HdfsTsSaveDir)) == True: self.logger.info("Create dir %s in hdfs success!!!"%HdfsTsSaveDir) #上传TS文件到HDFS for i in range(VodTsNum): TsNum = VodStartTsNum + i VodTsName = self.VodDate+"%05d"%TsNum+".ts" VodTsPath = VodTsDir+"/"+VodTsName HdfsTsSavePath = "/vod/%s/%s/%s/%s"%(self.oid,self.cid,self.VodDate,VodTsName) WebHdfsObj.put_file(str(VodTsPath),str(HdfsTsSavePath),overwrite="true") #上传m3u8文件到HDFS HdfsM3u8SavePath = "/vod/%s/%s/%s/%s"%(self.oid,self.cid,self.VodDate,self.M3u8FileName) WebHdfsObj.put_file(str(M3u8FilePath),str(HdfsM3u8SavePath),overwrite="true") #上传预览图到HDFS HdfsPrePicSavePath = "/vod/%s/%s/%s/%s"%(self.oid,self.cid,self.VodDate,self.PrePicFileName) WebHdfsObj.put_file(str(PrePicFilePath),str(HdfsPrePicSavePath),overwrite="true") else: self.logger.error("Create dir %s in hdfs fail,quit!!!"%HdfsTsSaveDir) sys.exit() else: self.logger.error("Upload condiition is not satisfied,quit!!!") sys.exit()
def UploadFileToHDFS(self, VodTsDir, VodStartTsNum, VodTsNum): PrePicFilePath = VodTsDir + "/" + self.PrePicFileName M3u8FilePath = VodTsDir + "/" + self.M3u8FileName #DebugLog = GetLog(self.cid+"_"+self.M3u8FileName.split(".")[0]) self.logger.info("Prepare upload File to hdfs,File in dir is : %s" % VodTsDir) if os.path.isfile(M3u8FilePath) and os.path.isfile( PrePicFilePath) and len(glob.glob(VodTsDir + "/*.ts")) != 0: WebHdfsObj = WebHadoop(self.HDFSInterfaces, self.HDFSInterfacePort, self.HDFSUser, self.logger) HdfsTsSaveDir = "/vod/%s/%s/%s" % (self.oid, self.cid, self.VodDate) if WebHdfsObj.mkdir(str(HdfsTsSaveDir)) == True: self.logger.info("Create dir %s in hdfs success!!!" % HdfsTsSaveDir) #上传TS文件到HDFS for i in range(VodTsNum): TsNum = VodStartTsNum + i VodTsName = self.VodDate + "%05d" % TsNum + ".ts" VodTsPath = VodTsDir + "/" + VodTsName HdfsTsSavePath = "/vod/%s/%s/%s/%s" % ( self.oid, self.cid, self.VodDate, VodTsName) WebHdfsObj.put_file(str(VodTsPath), str(HdfsTsSavePath), overwrite="true") #上传m3u8文件到HDFS HdfsM3u8SavePath = "/vod/%s/%s/%s/%s" % ( self.oid, self.cid, self.VodDate, self.M3u8FileName) WebHdfsObj.put_file(str(M3u8FilePath), str(HdfsM3u8SavePath), overwrite="true") #上传预览图到HDFS HdfsPrePicSavePath = "/vod/%s/%s/%s/%s" % ( self.oid, self.cid, self.VodDate, self.PrePicFileName) WebHdfsObj.put_file(str(PrePicFilePath), str(HdfsPrePicSavePath), overwrite="true") else: self.logger.error("Create dir %s in hdfs fail,quit!!!" % HdfsTsSaveDir) sys.exit() else: self.logger.error("Upload condiition is not satisfied,quit!!!") sys.exit()
import logging.handlers def getLog(logflag,loglevel="debug"): logger = logging.Logger(logflag) logfile = "/var/log/%s.log"%logflag hdlr = logging.handlers.RotatingFileHandler(logfile, maxBytes = 5*1024*1024, backupCount = 5) formatter = logging.Formatter("%(asctime)s -- [ %(name)s ] -- %(levelname)s -- %(message)s") hdlr.setFormatter(formatter) logger.addHandler(hdlr) if loglevel == "debug": logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) return logger hosts = "10.2.0.8,10.2.0.10" port = 14000 username = "******" logger = getLog("GenVod") WebHadoopObj = WebHadoop(hosts,port,username,logger) #WebHadoopObj.put_file("/tmp/debug.txt","/vod/debug.txt") print WebHadoopObj.mkdir("/kqvod/wyehaha")
def getLog(logflag, loglevel="debug"): logger = logging.Logger(logflag) logfile = "/var/log/%s.log" % logflag hdlr = logging.handlers.RotatingFileHandler(logfile, maxBytes=5 * 1024 * 1024, backupCount=5) formatter = logging.Formatter( "%(asctime)s -- [ %(name)s ] -- %(levelname)s -- %(message)s") hdlr.setFormatter(formatter) logger.addHandler(hdlr) if loglevel == "debug": logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) return logger hosts = "10.2.0.8,10.2.0.10" port = 14000 username = "******" logger = getLog("GenVod") WebHadoopObj = WebHadoop(hosts, port, username, logger) #WebHadoopObj.put_file("/tmp/debug.txt","/vod/debug.txt") print WebHadoopObj.mkdir("/kqvod/wyehaha")