def upload(self, path): retans = RetransInfo() retans.filename = FileUtility.getfilename(path) if int(self.start_time) == int(time.time()): time.sleep(0.1) self.start_time = time.time() retans.start_time = self.start_time self.uploadfile_retranslist[retans.filename] = retans self.upload_file_list[FileUtility.getfilename(path)] = [] cmd = TencentDownloader.UPLOAD_COMMAND.format( file=path, url=self.upload_url, user_id=self.taskinfo.userid, task_name=self.taskinfo.taskname, task_id=self.taskinfo.taskid) if self.execute(cmd): return True secs = 10 for count in range(0, self.retrytimes): time.sleep(secs) secs *= 2 if self.execute(cmd): return True else: param = NotifyParam() param.code = NotifyParam.SPIDER_NOTIFY_UPLOAD_FAILED param.message = NotifyParam.SPIDER_NOTIFY_UPLOAD_FAILED_MESSAGE_FORMAT.format( file=FileUtility.getfilename(path), taskid=self.taskinfo.taskid) SpiderNotify.notify(param) return False
def download(self): files = [] if self.completed(): return files Logger.getlogging().debug(self.info.donepath) srclist = self.sshls(self.info.donepath) for donefile in srclist: donefile = donefile.strip() filename = FileUtility.getfilename(donefile) if donefile.endswith( 'done') and filename not in self.download_file_list: self.download_file_list.append(filename) for upfile in self.upload_file_list.keys(): if filename.startswith(upfile): FileUtility.mkdirs(self.info.localdonepath) self.sshdownload(donefile) dfile = self.info.localdonepath + FileUtility.getfilename( donefile) if self.info.jsonpath: dfile = self.bin2json(dfile) files.append(dfile) self.download_time = int(time.time()) self.upload_file_list.pop(upfile) self.uploadfile_retranslist.pop(upfile) if not FileUtility.exists(dfile): Logger.getlogging().error( 'no json file generate from done file:{done}'. format(done=dfile)) break return files
def upload(self, path): retans = RetransInfo() retans.filename = FileUtility.getfilename(path) retans.start_time = int(time.time()) self.uploadfile_retranslist[retans.filename] = retans self.upload_file_list[FileUtility.getfilename(path)] = [] if self.localupload(path): return True
def download(self): """ 平台上的下载分为两个步骤,而windows直接请求数据则只有step2:download() step1:从平台上下载数据到本地./data/platform step2:从./data/platform拷贝数据到./data/temp/done下,再存储解析后的json数据至./data/temp/json """ files = [] Logger.getlogging().debug('Get Valid PUC File From ' + self.download_path) #srclist = self.getvalidfiles(self.download_path) srclist = FileUtility.getfilelist(self.download_path, [])[0:self.maxfilenum] for donefile in srclist: try: if donefile.endswith('done'): Logger.getlogging().info('MOVE {file} TO {path}'.format( file=donefile, path=self.done_file)) FileUtility.move(donefile, self.done_file) binfile = os.path.join(self.done_file, FileUtility.getfilename(donefile)) #FileUtility.copy(donefile, self.cache_path) #binfile = self.cache_path+ FileUtility.getfilename(donefile) #if FileUtility.getfilesize(donefile) == FileUtility.getfilesize(binfile): ##备份当天的puc文件 #Logger.getlogging().info('MOVE {file} TO {path}'.format(file=donefile,path=self.pucbacktoday)) #FileUtility.move(donefile, self.pucbacktoday) #if FileUtility.exists(donefile): #Logger.getlogging().error('MOVE {file} failed'.format(file=donefile)) #else: #Logger.getlogging().error('File not equal {file}'.format(file=donefile)) jsonfile = self.bin2json(binfile) files.append(jsonfile) try: self.s3puc_dumpurls(jsonfile) time.sleep(0.5) Logger.getlogging().debug( 'Remove {f}'.format(f=jsonfile)) FileUtility.remove(jsonfile) donefile2 = os.path.join( self.done_file, FileUtility.getfilename(donefile)) Logger.getlogging().debug( 'Remove {f}'.format(f=donefile2)) FileUtility.remove(donefile2) except: Logger.printexception() Logger.getlogging().error( 'no json file generate from done file:{done}'. format(done=binfile)) os.mknod(jsonfile) except: Logger.printexception() return files
def upload(self, path): retans = RetransInfo() retans.filename = FileUtility.getfilename(path) retans.start_time = int(time.time()) self.uploadfile_retranslist[retans.filename] = retans self.upload_file_list[FileUtility.getfilename(path)] = [] if self.sshupload(path): return True else: param = NotifyParam() param.code = NotifyParam.SPIDER_NOTIFY_UPLOAD_FAILED param.message = NotifyParam.SPIDER_NOTIFY_UPLOAD_FAILED_MESSAGE_FORMAT.format( file=FileUtility.getfilename(path), taskid=self.info.ip) SpiderNotify.notify(param) return False
def retrydownload(self, jsonfile, urlset): Logger.getlogging().warning( 'upload failed urls {num}'.format(num=len(urlset))) context = URLFileManager.getinstance().geturlfilecontext( FileUtility.getfilename(jsonfile)) if context.retry >= 2: Logger.getlogging().error('do not upload for failed again') for key in urlset.keys(): Logger.getlogging().error( 'download {url} failed'.format(url=key)) else: urls = [] for key in urlset.keys(): Logger.getlogging().warning( 'retry download {url}'.format(url=key)) for i in range(0, urlset[key]): urls.append(key) newurlfile = URLFileManager.getinstance().generateurlfilepath( context.retry + 1) Logger.getlogging().warning( 'Retry download URL {file}'.format(file=newurlfile)) if constant.POST_FILE_SUFFIX in jsonfile: URLManager.getinstance().storeurls(urls, constant.REQUEST_TYPE_POST) elif constant.WEBKIT_FILE_SUFFIX in jsonfile: URLManager.getinstance().storeurls( urls, constant.REQUEST_TYPE_WEBKIT) else: URLManager.getinstance().storeurls( urls, constant.REQUEST_TYPE_COMMON)
def s2query(self): self.conf.setchannel(SPIDER_CHANNEL_S2) s2file = SpiderConfigure.getinstance().gets2file() file = FileUtility.getfilename(s2file) s2temppath = Storage.getstoragelocation(const.SPIDER_QUERY_TEMP_PATH) + file if FileUtility.exists(s2temppath): with open(s2temppath, 'r') as fp: querylist = [] firstline = True for strquery in fp.readlines(): if firstline: firstline = False if strquery[:3] == codecs.BOM_UTF8: Logger.getlogging().warning('Remove BOM from {file}!'.format(file=file)) strquery = strquery[3:] strquery = Common.strip(strquery) if not strquery: continue Logger.getlogging().info('S2 {query} start...'.format(query=strquery)) self.conf.setquery(strquery) URLStorage.updaterecycle() querylist.append(strquery) for site in self.factory.getall(): site.s2query(strquery.replace('&', ' ')) sitelist = [] for site in self.factory.getall(): if site.exists2(): sitelist.append(site) SpiderReport.loadquery(querylist) SpiderReport.loadsites(sitelist)
def renewfilename(self, file): """""" filename = FileUtility.getfilename(file) context = URLFileManager.getinstance().geturlfilecontext(filename) if not context: return False if self.filetime == int(time.time()): time.sleep(1) self.filetime = int(time.time()) newfilename = filename.replace( re.findall('\d+', filename)[-1], str(self.filetime)) urlsfile = self.tempurlpath + newfilename context.filename = urlsfile URLFileManager.getinstance().updateurlfilecontext( FileUtility.getfilename(urlsfile), context) return urlsfile
def __upload__(self, filepath): flag = True FileUtility.mkdirs(self.urlbackuppath) FileUtility.copy(filepath, self.urlbackuppath) self.upload_file_list[FileUtility.getfilename(filepath)] = [] # if filepath.endswith(constant.POST_FILE_SUFFIX) or FileUtility.getfilelines(filepath) <= constant.REMOTE_DOWNLOADER_MIN_LINES: # if self.limpls: # if self.limplsindex >= len(self.limpls): # self.limplsindex = 0 # flag = self.limpls[self.limplsindex].upload(filepath) # self.limplsindex += 1 if filepath.endswith(constant.WEBKIT_FILE_SUFFIX): if self.wimpls: if self.wimplsindoex >= len(self.wimpls): self.wimplsindoex = 0 self.wimpls[self.wimplsindoex].upload(filepath) self.wimplsindoex += 1 elif self.impls: if self.implsindex >= len(self.impls): self.implsindex = 0 flag = self.impls[self.implsindex].upload(filepath) self.implsindex += 1 else: flag = False Logger.getlogging().warning('No taskid or download platform!') return flag
def bin2json(self, file): filename = FileUtility.getfilename(file).replace('.done', '.json') cmd = PUCDownloader.PARSE_COMMAND.format(command=self.parse_tool, input=file, output=self.json_path, filename=filename) self.execute(cmd) return self.json_path + filename
def getqueryfromdb(self): #指定s2 query输出文件路径 s2file = SpiderConfigure.getinstance().gets2file() temppath = Storage.getstoragelocation( const.SPIDER_QUERY_TEMP_PATH) + FileUtility.getfilename(s2file) QueryStorage.getinstance().getlocalquerys( temppath, ETLController.LOCALMACHINEFLAG) if FileUtility.exists(temppath): return temppath
def gettiebaqueryfromdb(self): #指定s2 query输出文件路径 tiebafile = SpiderConfigure.getconfig(const.SPIDER_STORAGE_DOMAIN, const.SPIDER_S3_INPUT_FILE) temppath = Storage.getstoragelocation( const.SPIDER_TIEBA_TEMP_PATH) + FileUtility.getfilename(tiebafile) QueryStorage.getinstance().getlocalquerys_tieba( temppath, ETLController.LOCALMACHINEFLAG) if FileUtility.exists(temppath): return temppath
def upload(self, path): tencentplatform.postdownloader.PostDownloader.upload(self, path) filename = FileUtility.getfilename(path) FileUtility.mkdirs(self.download_path) FileUtility.copy( path, '{dir}/{filename}.txt.{ts}.done'.format(dir=self.download_path, filename=filename, ts=int(time.time()))) return True
def waibuetl(self): waibubackup = SpiderConfigure.getwaibubaup() if not FileUtility.exists(waibubackup): FileUtility.mkdirs(waibubackup) waibufile = self.etl.getqueryfromdb() if not FileUtility.exists(waibufile): Logger.getlogging().warning( '{waibufile} not generate!'.format(waibufile=waibufile)) return outtime = 0 self.wdownloader.upload(waibufile) continueflag = True while continueflag: downloadfiles = [] while True: Logger.getlogging().info( 'sleeping {sec}s......'.format(sec=self.waitingperiod)) #time.sleep(self.waitingperiod) outtime += self.waitingperiod if self.wdownloader.iscompleted(): continueflag = False break try: downloadfiles = self.wdownloader.download() if downloadfiles: break except: Logger.printexception() if outtime >= self.waibutimeout: Logger.getlogging().warning( 'Waibu Data Download Timeout! Spending {sec}s'.format( sec=outtime)) continueflag = False break for dfile in downloadfiles: starttime = TimeUtility.getcurrentdate( TimeUtility.TIME_FORMAT_DEFAULT) self.etl.wb_analysis(dfile) #if FileUtility.exists(waibubackup+FileUtility.getfilename(dfile)): #FileUtility.remove(waibubackup+FileUtility.getfilename(dfile)) FileUtility.move(dfile, waibubackup) logstring = 'PROCESSWAIBUFILE:\t{file}\t{start}\t{end}'.format( file=FileUtility.getfilename(dfile), start=starttime, end=TimeUtility.getcurrentdate()) Logger.getlogging().info(logstring) if outtime >= self.waibutimeout: Logger.getlogging().warning( 'Waibu Data Download Timeout! Spending {sec}s'.format( sec=outtime)) continueflag = False break
def bin2json(self, file): filename = FileUtility.getfilename(file).replace('.done', '.json') tool = self.parse_tool if constant.IMG_FILE_SUFFIX in filename: tool = self.parse_tool_img cmd = TencentDownloader.PARSE_COMMAND.format(command=tool, input=file, output=self.json_path, filename=filename) self.execute(cmd) return self.json_path + filename
def preprocess(self, filepath): result = False context = URLStorage.getfilecontext(FileUtility.getfilename(filepath)) if context: self.conf.setchannel(context.channel) if context.channel == SPIDER_CHANNEL_S2: self.conf.setquery(context.query) else: self.conf.setquery('') URLStorage.updaterecycle() result = True return result
def preprocess(self, filepath): result = False context = URLFileManager.getinstance().geturlfilecontext( FileUtility.getfilename(filepath)) if context: self.conf.setchannel(context.channel) if context.channel == SPIDER_CHANNEL_S2: self.conf.setquery(context.query) else: self.conf.setquery('') URLFileManager.getinstance().generateurlfilepath() result = True return result
def upload(self, path): tencentplatform.tencentdownloader.TencentDownloader.upload(self, path) filename = FileUtility.getfilename(path) ts = int(time.time()) FileUtility.mkdirs(self.download_path) Logger.getlogging( ).debug(path + '--->' + '{dir}/{filename}.txt.{ts}.done'.format( dir=self.download_path, filename=filename, ts=int(time.time()))) FileUtility.copy( path, '{dir}/{filename}.txt.{ts}.done'.format(dir=self.download_path, filename=filename, ts=int(time.time()))) return True
def backupfile(self, jsonfile): urlmap = {} splitkey = '.' if '_split' in jsonfile: splitkey = '_split' bkfile = self.urlbackuppath + '/' + FileUtility.getfilename(jsonfile).split(splitkey)[0] if FileUtility.exists(bkfile): with open(bkfile, 'r') as bkfh: for line in bkfh.readlines(): line = line.strip() if line in urlmap: urlmap[line] += 1 else: urlmap[line] = 1 return urlmap
def upload(self, upfiles): Logger.getlogging().debug('uploading ......') for file in upfiles: if self.emptyfile(file): Logger.getlogging().info('remove empty file: ' + file) FileUtility.remove(file) continue if not self.__upload__(file): Logger.log(FileUtility.getfilename(file), constant.ERRORCODE_FAIL_LOAD_UP) return False Logger.getlogging().info('remove uploadedfile' + file) FileUtility.remove(file) time.sleep(1) return True
def retrans(self): if len(self.retransmissionfiles) > 0: # 从backup中恢复数据到temp/urls for fl in self.retransmissionfiles.keys(): newfl = self.recoverfile(fl) newfilename = FileUtility.getfilename(newfl) self.all_retransmissionfiles[ newfilename] = self.all_retransmissionfiles[fl] Logger.getlogging().debug( 'download fail and transimission file {fl}:{num}th'.format( fl=newfilename, num=self.all_retransmissionfiles[newfilename]. retrans_num)) impl = self.all_retransmissionfiles[newfilename].taskinfo if not impl.taskstatusflag: if impl in self.wimpls: if len(self.wimpls) != 1: self.wimpls.remove(impl) Logger.getlogging().warning( 'download fail and delete taskid is :{impl}'. format(impl=impl.taskinfo.taskid)) else: Logger.getlogging().warning( 'only one taskid is :{impl}'.format( impl=impl.taskinfo.taskid)) if impl in self.impls: if len(self.impls) != 1: self.impls.remove(impl) Logger.getlogging().warning( 'download fail and delete taskid is :{impl}'. format(impl=impl.taskinfo.taskid)) else: Logger.getlogging().warning( 'only one taskid is :{impl}'.format( impl=impl.taskinfo.taskid)) if impl in self.limpls: if len(self.limpls) != 1: self.limpls.remove(impl) Logger.getlogging().warning( 'download fail and delete downloadplatform is :{impl}' .format(impl=impl.info.ip)) else: Logger.getlogging().warning( 'only one downloadplatform is :{impl}'.format( impl=impl.info.ip)) self.retransmissionfiles.pop(fl)
def loop(self): # 循环URL,包括S1以及S2 continueflag = True while continueflag: downloadfiles = [] while True: # check time out if self.istimeout(): param = NotifyParam() param.code = NotifyParam.SPIDER_NOTIFY_TIMEOUT param.message = 'Spider timeout for %s o\'clock, stop' % constant.SPIDER_RUN_TIMEOUT_HOUR SpiderNotify.notify(param) continueflag = False break if self.downloader.iscompleted(): continueflag = False break try: downloadfiles = self.downloader.download() self.upload() if len(downloadfiles) > 0: break else: Logger.getlogging().info('sleeping {0}s......'.format( self.waitingperiod)) time.sleep(self.waitingperiod) except: Logger.printexception() for dfile in downloadfiles: starttime = TimeUtility.getcurrentdate( TimeUtility.TIME_FORMAT_DEFAULT) self.etl.processfile(dfile) logstring = 'PROCESSFILE:\t{file}\t{start}\t{end}'.format( file=FileUtility.getfilename(dfile), start=starttime, end=TimeUtility.getcurrentdate()) Logger.getlogging().info(logstring) if self.istimeout(): param = NotifyParam() param.code = NotifyParam.SPIDER_NOTIFY_TIMEOUT param.message = 'Spider timeout for %s o\'clock, stop' % constant.SPIDER_RUN_TIMEOUT_HOUR SpiderNotify.notify(param) continueflag = False break self.upload()
def processfile(self, jsonfile): if not self.preprocess(jsonfile): return post = (constant.POST_FILE_SUFFIX in jsonfile) urls = self.backupfile(jsonfile) context = URLStorage.getfilecontext(FileUtility.getfilename(jsonfile)) with open(jsonfile, 'r') as fp: lines = fp.readlines() for line in lines: param = self.analysis(line, post) if param is None: continue url = param.url if context.retry >= 2: param.lastretry = True if post: url = json.dumps({'url': param.url, 'data': param.data}) else: Logger.getlogging().warning(url) info = None if URLStorage.hasurl(url): info = URLStorage.geturlcontext(url) param.originalurl = info.originalurl param.step = info.step param.customized = info.customized else: param.originalurl = param.url res = True if SiteS2Query.REFER_URL in param.customized: site = self.factory.getsite(param.customized[SiteS2Query.REFER_URL]) res = site.process(param) else: site = self.factory.getsite(param.originalurl) res = site.process(param) if not res: if info: URLStorage.seturlcontext(param.url, info) else: if url in urls: urls[url] -= 1 if urls[url] == 0: urls.pop(url) # upload failed urls if urls: self.retrydownload(jsonfile, urls)
def processfile(self, jsonfile): if not self.preprocess(jsonfile): return method = self.requesttype(jsonfile) urls = self.backupfile(jsonfile) context = URLFileManager.getinstance().geturlfilecontext( FileUtility.getfilename(jsonfile)) with open(jsonfile, 'r') as fp: lines = fp.readlines() for line in lines: param = self.analysis(line, method) if param is None: continue url = param.url if context.retry >= 2: param.lastretry = True if method == constant.REQUEST_TYPE_POST: url = json.dumps({'url': param.url, 'data': param.data}) info = None if URLManager.getinstance().exist(url): info = URLManager.getinstance().geturlcontext(url) param.originalurl = info.originalurl param.step = info.step param.type = info.type param.customized = info.customized else: param.originalurl = param.url param.type = URLContext.S1_MAIN_BODY if SiteS2Query.REFER_URL in param.customized: site = self.factory.getsite( param.customized[SiteS2Query.REFER_URL]) else: site = self.factory.getsite(param.originalurl) res = site.process(param) if not res: if info: URLManager.getinstance().seturlcontext(param.url, info) else: if url in urls: urls[url] -= 1 if urls[url] == 0: urls.pop(url) # upload failed urls if urls: self.retrydownload(jsonfile, urls)
def generateurlfilepath(self, retrytimes=0): context = URLFileContext() context.channel = SpiderConfigure.getinstance().getchannel() context.query = SpiderConfigure.getinstance().getquery() context.retry = retrytimes # 防止生成相同的URL文件,等待1秒后重新获取时间戳 if self.urlfiletimestamp == int(time.time()): time.sleep(1) self.urlfiletimestamp = int(time.time()) self.urlsfile = URLFileManager.URLS_FILE_PATTERN.format( path=self.tempurldir, channel=context.channel, query=Common.md5(context.query), ts=self.urlfiletimestamp) context.filename = self.urlsfile self.urlsfilemap[FileUtility.getfilename(self.urlsfile)] = context Logger.getlogging().info(self.urlsfile) return self.urlsfile
def retrydownload(self, jsonfile, urlset): Logger.getlogging().warning('upload failed urls {num}'.format(num=len(urlset))) context = URLStorage.getfilecontext(FileUtility.getfilename(jsonfile)) if context.retry >= 2: Logger.getlogging().error('do not upload for failed again') for key in urlset.keys(): Logger.getlogging().error('download {url} failed'.format(url=key)) else: urls = [] for key in urlset.keys(): Logger.getlogging().warning('retry download {url}'.format(url=key)) for i in range(0, urlset[key]): urls.append(key) StatisticsManager.updateall(-len(urls)) URLStorage.updaterecycle(context.retry + 1) if constant.POST_FILE_SUFFIX in jsonfile: URLStorage.storeurls(urls, constant.REQUEST_TYPE_POST) elif constant.WEBKIT_FILE_SUFFIX in jsonfile: URLStorage.storeurls(urls, constant.REQUEST_TYPE_WEBKIT) else: URLStorage.storeurls(urls, constant.REQUEST_TYPE_COMMON)
def __download__(self): cmd = self.DOWNLOADCMD.format(path=self.download_path, appId=self.appid, jobId=self.jobid, token=self.token) if constant.DEBUG_FLAG == constant.DEBUG_FLAG_WINDOWS: Logger.getlogging().debug(cmd) return True if self.execute2(cmd): return True secs = 5 for count in range(0, self.RETRYTIMES): time.sleep(secs) secs *= 2 if self.execute2(cmd): return True else: param = NotifyParam() param.code = NotifyParam.SPIDER_NOTIFY_UPLOAD_FAILED param.message = NotifyParam.SPIDER_NOTIFY_UPLOAD_FAILED_MESSAGE_FORMAT.format( file=FileUtility.getfilename(self.download_path), taskid=self.jobid) SpiderNotify.notify(param) return False
def initialize(self): files = ssh.sshls(self.info.ip, self.info.port, self.info.username, self.info.password, self.info.donepath) for fl in files: fl = fl.strip() self.download_file_list.append(FileUtility.getfilename(fl))
def bin2json(self, file): filename = FileUtility.getfilename(file).replace('.done', '.json') fullpath = self.info.jsonpath + filename FileUtility.copy(file, fullpath) return fullpath
def download(self): """ 平台上的下载分为两个步骤,而windows直接请求数据则只有step2:download() step1:从平台上下载数据到本地./data/platform step2:从./data/platform拷贝数据到./data/temp/done下,再存储解析后的json数据至./data/temp/json """ files = [] if self.completed(): return files Logger.getlogging().debug(self.download_path) srclist = FileUtility.getfilelist(self.download_path, []) for donefile in srclist: filename = FileUtility.getfilename(donefile) if donefile.endswith( 'done') and filename not in self.download_file_list: self.download_file_list.append(filename) self.download_time = time.time() for upfile in self.upload_file_list.keys(): if filename.startswith(upfile): FileUtility.copy(donefile, self.cache_path) binfile = self.cache_path + FileUtility.getfilename( donefile) if FileUtility.getfilesize( donefile) == FileUtility.getfilesize(binfile): Logger.getlogging().info( 'Remove {file}'.format(file=donefile)) FileUtility.remove(donefile) if FileUtility.exists(donefile): Logger.getlogging().error( 'Remove {file} failed'.format( file=donefile)) else: Logger.getlogging().error( 'File not equal {file}'.format(file=donefile)) jsonfile = self.bin2json(binfile) files.append(jsonfile) uploadtime = self.uploadfile_retranslist[ upfile].start_time if RegexUtility.match( TencentDownloader.DOWNLOAD_FORMAT1.format( file=upfile), filename): self.upload_file_list.pop(upfile) self.uploadfile_retranslist.pop(upfile) elif RegexUtility.match( TencentDownloader.DOWNLOAD_FORMAT2.format( file=upfile), filename): value = \ RegexUtility.parse(TencentDownloader.DOWNLOAD_FORMAT2.format(file=upfile), filename)[0] if value[0] == value[1]: self.upload_file_list.pop(upfile) self.uploadfile_retranslist.pop(upfile) if not FileUtility.exists(jsonfile): Logger.getlogging().error( 'no json file generate from done file:{done}'. format(done=binfile)) os.mknod(jsonfile) # update upload time keys = self.sortkeys() for fl in keys: if self.uploadfile_retranslist[ fl].start_time >= uploadtime: self.uploadfile_retranslist[ fl].start_time = time.time() time.sleep(0.1) break return files