def run(self): self.status = "waiting start" if not self.enabled: self.status = "disabled" return if self.bootwait > 0: time.sleep(self.bootwait) while not self.thread_stop: self.status = "checking" # print '%s start checking at %s ...\n' %(self.name,time.ctime()) if self.processHandle != None and self.processHandle.poll( ) != None: print "recycle %s" % (self.name) self.processHandle = None if not check_pid(self._pid): self.update_pid() if not check_pid(self._pid): self._pid = 0 if self._pid == 0: self.processHandle = None PLOG.warn("%s check failed!restarting ..." % (self.name)) if self.rebootwait > 0: self.status = "waiting restart" PLOG.info("%s restarting wait %d second..." % (self.name, self.rebootwait)) time.sleep(self.rebootwait) try: self.status = "starting" # 修改当前路径 if len(self.runpath) > 0: try: if not os.path.isdir(self.runpath): os.makedirs(self.runpath) if not os.path.isdir(self.runpath): self.enabled = False PLOG.error("%s run path invalid!" % (self.name)) break os.chdir(self.runpath) except Exception, e: PLOG.error( "%s restart failed!change current path failed!err=%s" % (self.name, e)) PLOG.info("%s execute command:'%s'" % (self.name, self.command)) self.processHandle = subprocess.Popen( self.command, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=True, cwd=self.runpath, env=None, universal_newlines=False, startupinfo=None, creationflags=0) self._pid = self.processHandle.pid except Exception, e: PLOG.error("%s restart failed!err=%s" % (self.name, e))
def addJsonInfo(jsonSourcefile,destJson): filedir = os.path.dirname(jsonSourcefile) parentDirName = os.path.split(filedir)[-1] primaryFilename = "" jsSourceFileInfo = None with open(jsonSourcefile,"r") as f: jsSourceFileInfo = json.load(f,'utf8') if jsSourceFileInfo !=None and isinstance(jsSourceFileInfo,dict): if jsSourceFileInfo.has_key("file"): primaryFilename = jsSourceFileInfo["file"] if primaryFilename != "": jsSourceFileInfo["id"] = str(uuid.uuid1()) if primaryFilename.startswith("https:") : # ios info file filetimestamp = time.localtime( os.path.getmtime(jsonSourcefile)) primaryFileTime = time.strftime('%Y-%m-%d %H:%M:%S',filetimestamp) jsSourceFileInfo["filetime"] = primaryFileTime if not jsSourceFileInfo.has_key("filesize") : jsSourceFileInfo["filesize"] = "0" #destJson["list"].append(jsSourceFileInfo) else: try: primaryFileSize = os.path.getsize(os.path.join(filedir,primaryFilename)) filetimestamp = time.localtime( os.path.getmtime(os.path.join(filedir,primaryFilename)) ) primaryFileTime = time.strftime('%Y-%m-%d %H:%M:%S',filetimestamp) jsSourceFileInfo["filesize"] = str(primaryFileSize) jsSourceFileInfo["filetime"] = primaryFileTime if jsSourceFileInfo.has_key("file") : jsSourceFileInfo["file"] = parentDirName +'/' + jsSourceFileInfo["file"] except: PLOG.info("generate file info of dir %s failed,primary File %s not find,skip it"% (filedir,primaryFilename)) return if jsSourceFileInfo.has_key("poster") : jsSourceFileInfo["poster"] = parentDirName +'/' + jsSourceFileInfo["poster"] if jsSourceFileInfo.has_key("thumbnail") : jsSourceFileInfo["thumbnail"] = parentDirName +'/' + jsSourceFileInfo["thumbnail"] if jsSourceFileInfo.has_key("extend") : jsextend = jsSourceFileInfo["extend"] if jsextend.has_key("screenshot") : jsscreenshottmp = [] for picture in jsextend["screenshot"] : picture = parentDirName +'/' + picture jsscreenshottmp.append(picture) jsextend["screenshot"] =jsscreenshottmp destJson["list"].append(jsSourceFileInfo) PLOG.debug('generate file info of dir "%s" success'%(filedir)) else: PLOG.debug("generate file info of dir %s failed,primary File name is empty"% (filedir)) else : PLOG.debug('not find "file" node in info file %s , skip it' %(jsonSourcefile)) else: PLOG.warn('js file %s is null,maybe path error! skip it' %(jsonSourcefile))
def login(self): try: FTP.connect(self,self.host,timeout=10) except: PLOG.warn('Can not connect to ftp server "%s"' % self.host) return False try: FTP.login(self,self.user,self.pwd) except: PLOG.warn('Login ftp server "%s" failed ,username or password error' % self.host) return False return True
def login(self): try: FTP.connect(self, self.host, timeout=10) except: PLOG.warn('Can not connect to ftp server "%s"' % self.host) return False try: FTP.login(self, self.user, self.pwd) except: PLOG.warn( 'Login ftp server "%s" failed ,username or password error' % self.host) return False return True
def run(self): self.status = "waiting start" if not self.enabled: self.status = "disabled" return if self.bootwait > 0:time.sleep(self.bootwait) while not self.thread_stop: self.status = "checking" # print '%s start checking at %s ...\n' %(self.name,time.ctime()) if self.processHandle!=None and self.processHandle.poll()!=None: print "recycle %s" % (self.name) self.processHandle=None if not check_pid(self._pid):self.update_pid() if not check_pid(self._pid):self._pid = 0 if self._pid == 0: self.processHandle=None PLOG.warn("%s check failed!restarting ..." % (self.name)) if self.rebootwait > 0: self.status = "waiting restart" PLOG.info("%s restarting wait %d second..." % (self.name, self.rebootwait)) time.sleep(self.rebootwait) try: self.status = "starting" # 修改当前路径 if len(self.runpath) > 0: try: if not os.path.isdir(self.runpath):os.makedirs(self.runpath) if not os.path.isdir(self.runpath): self.enabled = False PLOG.error("%s run path invalid!"%(self.name)) break os.chdir(self.runpath) except Exception, e: PLOG.error("%s restart failed!change current path failed!err=%s" % (self.name, e)) PLOG.info("%s execute command:'%s'"%(self.name,self.command)) self.processHandle=subprocess.Popen(self.command, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=True, cwd=self.runpath, env=None, universal_newlines=False, startupinfo=None, creationflags=0) self._pid=self.processHandle.pid except Exception, e: PLOG.error("%s restart failed!err=%s" % (self.name, e))
def run(): init() if conf.httpServerSite[-1] != '/' : conf.httpServerSite += '/' # 爬取文件,生成最终json文件 start if len(conf.movieDir) != 0 : scanFile(conf.movieDir,"movie") else: PLOG.warn("moviedir is empty,please check config file") if len(conf.androidDir) != 0 : scanFile(conf.androidDir,"android") else: PLOG.warn("androiddir is empty,please check config file") if len(conf.iosDir) != 0 : scanFile(conf.iosDir,"ios") else: PLOG.warn("iosdir is empty,please check config file")
def scanFtpServerFiles(self,root,filetype): PLOG.debug('Type["%s"] file start crawling...ftpserver = %s ,dir = %s ' %(filetype,self.host,root)) outputjsfilename = "" filesource = "" if filetype == "movie": outputjsfilename = conf.movieOutputFile filesource = conf.ftpServerMovieSource elif filetype == "app": outputjsfilename = conf.appOutputFile filesource = conf.ftpServerAppSource # 枚举工作目录下的所有目录 fileDir = self.listdir(root) # 所有电影 or APP信息json串 allJsonInfo = {} allJsonInfo["update"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") allJsonInfo["source"] = filesource allJsonInfo["list"] =[] for filedir in fileDir : PLOG.debug('start generate file info of dir "%s"...'%(root+filedir)) fileItems = self.listFile(root+filedir) primaryFilename = "" primaryFileSize = "" primaryFileTime = "" jsFileInfo = None for fileitem in fileItems: if fileitem[-5:] == ".json" : fileinfo = [] if fileitem.find("/") == -1 : fileitem = root+filedir+'/'+fileitem try: self.retrlines("RETR %s"%fileitem,fileinfo.append) except: PLOG.warn('retr %s except! skip it !' %fileitem) filedetailinfo = "" for linestr in fileinfo: filedetailinfo += linestr if filedetailinfo != "": try: filedetailinfo = filedetailinfo.decode("gbk") except: pass try: filedetailinfo = filedetailinfo.decode("gb2312") except: pass #PLOG.debug("decode failed! %s is not encoded by gbk") jsFileInfo = json.loads(filedetailinfo,'utf8') if jsFileInfo !=None : if jsFileInfo.has_key("file"): primaryFilename = jsFileInfo["file"] else : PLOG.debug('not find "file" node in info file %s , skip it' %(fileitem)) else: PLOG.error('js file %s is null,maybe path error! skip it' %(fileitem)) break if jsFileInfo != None and jsFileInfo != "" : if primaryFilename != "" : try: timestamp = [] self.retrlines("LIST %s"%root+filedir+'/'+primaryFilename,lambda x:timestamp.append(self.separateFileTime(x))) primaryFileSize = self.size(root+filedir+'/'+primaryFilename) primaryFileTime = timestamp.pop() jsFileInfo["filesize"] = primaryFileSize jsFileInfo["filetime"] = primaryFileTime jsFileInfo["id"] = str(uuid.uuid1()) filerelativedir = filedir + '/' if jsFileInfo.has_key("file") : jsFileInfo["file"] = filerelativedir +jsFileInfo["file"] if jsFileInfo.has_key("poster") : jsFileInfo["poster"] = filerelativedir +jsFileInfo["poster"] if jsFileInfo.has_key("thumbnail") : jsFileInfo["thumbnail"] = filerelativedir +jsFileInfo["thumbnail"] if jsFileInfo.has_key("extend") : jsextend = jsFileInfo["extend"] if jsextend.has_key("screenshot") : jsscreenshottmp = [] for picture in jsextend["screenshot"] : picture = filerelativedir + picture jsscreenshottmp.append(picture) jsextend["screenshot"] =jsscreenshottmp allJsonInfo["list"].append(jsFileInfo) PLOG.debug('generate file info of dir "%s" success'%(root+filedir)) except: PLOG.warn('retr %s except! skip it !' %(root+filedir+'/'+primaryFilename)) PLOG.debug("generate file info of dir %s failed,not find primary File %s" % (root+filedir,primaryFilename)) else: PLOG.debug("generate file info of dir %s failed,primary File name is empty"% (root+filedir)) else: PLOG.debug("generate file info of dir %s failed,not find js info file"% (root+filedir)) if(outputjsfilename == ""): PLOG.debug("unkown file type!") return 0 with open(outputjsfilename, "w") as f: json.dump(allJsonInfo, f,indent=4,ensure_ascii=False) # 将json文件传到ftpserver ttt = len(outputjsfilename) with open(outputjsfilename,"r") as f: try: outputdirtmp=conf.ftpJsonOutputPath.replace("ftp://","") outputdir = outputdirtmp[outputdirtmp.find("/")+1:] self.storlines("STOR %s"%outputdir+outputjsfilename,f) PLOG.debug('upload json file %s success !'%outputjsfilename) except: PLOG.warn('upload json file %s failed,exception !'%outputjsfilename) PLOG.debug('Type["%s"] file crawl dir %s finished' %(filetype,root))
def scanFtpServerFiles(self, root, filetype): PLOG.debug('Type["%s"] file start crawling...ftpserver = %s ,dir = %s ' % (filetype, self.host, root)) outputjsfilename = "" filesource = "" if filetype == "movie": outputjsfilename = conf.movieOutputFile filesource = conf.ftpServerMovieSource elif filetype == "app": outputjsfilename = conf.appOutputFile filesource = conf.ftpServerAppSource # 枚举工作目录下的所有目录 fileDir = self.listdir(root) # 所有电影 or APP信息json串 allJsonInfo = {} allJsonInfo["update"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") allJsonInfo["source"] = filesource allJsonInfo["list"] = [] for filedir in fileDir: PLOG.debug('start generate file info of dir "%s"...' % (root + filedir)) dictFileItems = self.listFile(root + filedir) primaryFilename = "" primaryFileSize = "" primaryFileTime = "" jsFileInfo = None for (k, v) in dictFileItems.items(): if v.ext == ".json": fileinfo = [] try: self.retrlines("RETR %s" % root + filedir + "/" + v.fname, fileinfo.append) except: PLOG.warn("retr %s except! skip it !" % v.fname) filedetailinfo = "" for linestr in fileinfo: filedetailinfo += linestr if filedetailinfo != "": filedetailinfo = filedetailinfo.decode("gbk") jsFileInfo = json.loads(filedetailinfo, "utf8") if jsFileInfo != None: if jsFileInfo.has_key("file"): primaryFilename = jsFileInfo["file"] else: PLOG.debug('not find "file" node in info file %s , skip it' % (v.fname)) else: PLOG.error("js file %s is null,maybe path error! skip it" % (v.fname)) break if jsFileInfo != "": if primaryFilename != "": if dictFileItems.has_key(primaryFilename): primaryFileItem = dictFileItems[primaryFilename] primaryFileSize = primaryFileItem.size time = primaryFileItem.time[: primaryFileItem.time.find(".")] primaryFileTime = datetime.datetime.strptime(time, "%Y%m%d%H%M%S").strftime("%Y-%m-%d %H:%M:%S") jsFileInfo["filesize"] = primaryFileSize jsFileInfo["filetime"] = primaryFileTime jsFileInfo["id"] = str(uuid.uuid1()) filerelativedir = filedir + "/" if jsFileInfo.has_key("file"): jsFileInfo["file"] = filerelativedir + jsFileInfo["file"] if jsFileInfo.has_key("poster"): jsFileInfo["poster"] = filerelativedir + jsFileInfo["poster"] if jsFileInfo.has_key("thumbnail"): jsFileInfo["thumbnail"] = filerelativedir + jsFileInfo["thumbnail"] if jsFileInfo.has_key("extend"): jsextend = jsFileInfo["extend"] if jsextend.has_key("screenshot"): jsscreenshottmp = [] for picture in jsextend["screenshot"]: picture = filerelativedir + picture jsscreenshottmp.append(picture) jsextend["screenshot"] = jsscreenshottmp allJsonInfo["list"].append(jsFileInfo) PLOG.debug('generate file info of dir "%s" success' % (root + filedir)) else: PLOG.debug( "generate file info of dir %s failed,not find primary File %s" % (root + filedir, primaryFilename) ) else: PLOG.debug("generate file info of dir %s failed,primary File name is empty" % (root + filedir)) else: PLOG.debug("generate file info of dir %s failed,not find js info file" % (root + filedir)) if outputjsfilename == "": PLOG.debug("unkown file type!") return 0 with open(outputjsfilename, "w") as f: json.dump(allJsonInfo, f, indent=4, ensure_ascii=False) # 将json文件传到ftpserver with open(outputjsfilename, "r") as f: try: outputdirtmp = conf.ftpJsonOutputPath.replace("ftp://", "") outputdir = outputdirtmp[outputdirtmp.find("/") + 1 :] self.storlines("STOR %s" % outputdir + outputjsfilename, f) PLOG.debug("upload json file %s success !" % outputjsfilename) except: PLOG.warn("upload json file %s failed,exception !" % outputjsfilename) PLOG.debug('Type["%s"] file crawl dir %s finished' % (filetype, root))
def scanFtpServerFiles(self, root, filetype): PLOG.debug( 'Type["%s"] file start crawling...ftpserver = %s ,dir = %s ' % (filetype, self.host, root)) outputjsfilename = "" filesource = "" if filetype == "movie": outputjsfilename = conf.movieOutputFile filesource = conf.ftpServerMovieSource elif filetype == "app": outputjsfilename = conf.appOutputFile filesource = conf.ftpServerAppSource # 枚举工作目录下的所有目录 fileDir = self.listdir(root) # 所有电影 or APP信息json串 allJsonInfo = {} allJsonInfo["update"] = datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") allJsonInfo["source"] = filesource allJsonInfo["list"] = [] for filedir in fileDir: PLOG.debug('start generate file info of dir "%s"...' % (root + filedir)) dictFileItems = self.listFile(root + filedir) primaryFilename = "" primaryFileSize = "" primaryFileTime = "" jsFileInfo = None for (k, v) in dictFileItems.items(): if v.ext == ".json": fileinfo = [] try: self.retrlines( "RETR %s" % root + filedir + '/' + v.fname, fileinfo.append) except: PLOG.warn('retr %s except! skip it !' % v.fname) filedetailinfo = "" for linestr in fileinfo: filedetailinfo += linestr if filedetailinfo != "": filedetailinfo = filedetailinfo.decode("gbk") jsFileInfo = json.loads(filedetailinfo, 'utf8') if jsFileInfo != None: if jsFileInfo.has_key("file"): primaryFilename = jsFileInfo["file"] else: PLOG.debug( 'not find "file" node in info file %s , skip it' % (v.fname)) else: PLOG.error( 'js file %s is null,maybe path error! skip it' % (v.fname)) break if jsFileInfo != "": if primaryFilename != "": if dictFileItems.has_key(primaryFilename): primaryFileItem = dictFileItems[primaryFilename] primaryFileSize = primaryFileItem.size time = primaryFileItem.time[:primaryFileItem.time. find('.')] primaryFileTime = datetime.datetime.strptime( time, '%Y%m%d%H%M%S').strftime("%Y-%m-%d %H:%M:%S") jsFileInfo["filesize"] = primaryFileSize jsFileInfo["filetime"] = primaryFileTime jsFileInfo["id"] = str(uuid.uuid1()) filerelativedir = filedir + '/' if jsFileInfo.has_key("file"): jsFileInfo[ "file"] = filerelativedir + jsFileInfo["file"] if jsFileInfo.has_key("poster"): jsFileInfo[ "poster"] = filerelativedir + jsFileInfo[ "poster"] if jsFileInfo.has_key("thumbnail"): jsFileInfo[ "thumbnail"] = filerelativedir + jsFileInfo[ "thumbnail"] if jsFileInfo.has_key("extend"): jsextend = jsFileInfo["extend"] if jsextend.has_key("screenshot"): jsscreenshottmp = [] for picture in jsextend["screenshot"]: picture = filerelativedir + picture jsscreenshottmp.append(picture) jsextend["screenshot"] = jsscreenshottmp allJsonInfo["list"].append(jsFileInfo) PLOG.debug('generate file info of dir "%s" success' % (root + filedir)) else: PLOG.debug( "generate file info of dir %s failed,not find primary File %s" % (root + filedir, primaryFilename)) else: PLOG.debug( "generate file info of dir %s failed,primary File name is empty" % (root + filedir)) else: PLOG.debug( "generate file info of dir %s failed,not find js info file" % (root + filedir)) if (outputjsfilename == ""): PLOG.debug("unkown file type!") return 0 with open(outputjsfilename, "w") as f: json.dump(allJsonInfo, f, indent=4, ensure_ascii=False) # 将json文件传到ftpserver with open(outputjsfilename, "r") as f: try: outputdirtmp = conf.ftpJsonOutputPath.replace("ftp://", "") outputdir = outputdirtmp[outputdirtmp.find("/") + 1:] self.storlines("STOR %s" % outputdir + outputjsfilename, f) PLOG.debug('upload json file %s success !' % outputjsfilename) except: PLOG.warn('upload json file %s failed,exception !' % outputjsfilename) PLOG.debug('Type["%s"] file crawl dir %s finished' % (filetype, root))
def statisticsCurrentDayData(daydate): nextday = daydate + datetime.timedelta(days=1) startquerytime = daydate endquerytime = daydate + datetime.timedelta( hours=SAPeakDataPublic.st.queryunit) while endquerytime <= nextday: acctquerysql = "select acctinputoctets,acctoutputoctets,acctstarttime,acctstoptime,regionid from %s where acctstarttime>='%s' and acctstarttime<'%s'"%\ (SAPeakDataPublic.sadb.tablename,startquerytime.strftime('%Y-%m-%d %H:%M:%S'),endquerytime.strftime('%Y-%m-%d %H:%M:%S')) PLOG.debug("sql=%s" % acctquerysql) startquerytime = endquerytime endquerytime = endquerytime + datetime.timedelta( hours=SAPeakDataPublic.st.queryunit) i = 0 while i < SAPeakDataPublic.st.queryrepeattimes: res = SAPeakDataPublic.querysql(acctquerysql) if res != None: break else: i = i + 1 if i == 3 or res == None: print( "%s statistics data failed! db query appear error %d consecutive times,please execute again later!" % (daydate.strftime('%Y-%m-%d'), SAPeakDataPublic.st.queryrepeattimes)) PLOG.info( "%s statistics data failed! db query appear error %d consecutive times,please execute again later!" % (daydate.strftime('%Y-%m-%d'), SAPeakDataPublic.st.queryrepeattimes)) return # 统计数据 PLOG.trace("start statistics...") for row in res: if row[2] == None or row[3] == None or row[4] == None: PLOG.warn("lack essential data!skip this data") continue regionid = row[4] totalflow = 0 if row[0] != None: totalflow += row[0] if row[1] != None: totalflow += row[1] if row[3].day > row[2].day: # 跨天 endMinute = 23 * 60 + 59 elif row[3].day < row[2].day: PLOG.info( "stoptime day less than starttime day,invalid data,skip") else: endMinute = row[3].hour * 60 + row[3].minute startMinute = row[2].hour * 60 + row[2].minute #startMinute = datetime.datetime.strptime(row[2],'%Y-%m-%d %H:%M:%S') #endMinute = datetime.datetime.strptime(row[3],'%Y-%m-%d %H:%M:%S') totalMinute = endMinute - startMinute + 1 if totalMinute <= 0: PLOG.info("stoptime less than starttime,invalid data,skip") continue if SAStopDefine.stopDc.stops.has_key(regionid): stop = SAStopDefine.stopDc.stops[regionid] startindex = startMinute endindex = endMinute flowOneMinute = float(totalflow) / totalMinute / 1024 / 1024 index = startindex while index <= endindex: stop.dayArray[index][0] += 1 stop.dayArray[index][1] += flowOneMinute if stop.dayArray[index][0] > stop.peakonlinenum: stop.peakonlinenum = stop.dayArray[index][0] stop.peakonlinetime = datetime.datetime( daydate.year, daydate.month, daydate.day, index / 60, index % 60) if stop.dayArray[index][0] > stop.peakbandwidth: stop.peakbandwidth = stop.dayArray[index][1] stop.peakbandwidthtime = datetime.datetime( daydate.year, daydate.month, daydate.day, index / 60, index % 60) index += 1 PLOG.trace("statistics end") # 数据处理结束,输出各站点峰值数据 for stopid, stop in stopsCentor.stops.items(): peakbandwidth = stop.peakbandwidth * 8 / 60 print("%s %s %d %.2f" % (daydate.strftime('%Y-%m-%d'), stop.name, stop.peakonlinenum, peakbandwidth)) PLOG.debug( "%s %s %d %.2f %s %s" % (daydate.strftime('%Y-%m-%d'), stop.name, stop.peakonlinenum, peakbandwidth, stop.peakonlinetime.strftime('%H:%M'), stop.peakbandwidthtime.strftime('%H:%M')))
def statisticsCurrentDayData(daydate) : nextday = daydate+datetime.timedelta(days=1) startquerytime = daydate endquerytime = daydate+datetime.timedelta(hours=SAPeakDataPublic.st.queryunit) while endquerytime<=nextday: acctquerysql = "select acctinputoctets,acctoutputoctets,acctstarttime,acctstoptime,regionid from %s where acctstarttime>='%s' and acctstarttime<'%s'"%\ (SAPeakDataPublic.sadb.tablename,startquerytime.strftime('%Y-%m-%d %H:%M:%S'),endquerytime.strftime('%Y-%m-%d %H:%M:%S')) PLOG.debug("sql=%s"%acctquerysql) startquerytime=endquerytime endquerytime=endquerytime+datetime.timedelta(hours=SAPeakDataPublic.st.queryunit) i = 0 while i<SAPeakDataPublic.st.queryrepeattimes: res = SAPeakDataPublic.querysql(acctquerysql) if res!=None: break else: i = i+1 if i==3 or res==None: print("%s statistics data failed! db query appear error %d consecutive times,please execute again later!"%(daydate.strftime('%Y-%m-%d'),SAPeakDataPublic.st.queryrepeattimes)) PLOG.info("%s statistics data failed! db query appear error %d consecutive times,please execute again later!"%(daydate.strftime('%Y-%m-%d'),SAPeakDataPublic.st.queryrepeattimes)) return # 统计数据 PLOG.trace("start statistics...") for row in res: if row[2] ==None or row[3] ==None or row[4] ==None: PLOG.warn("lack essential data!skip this data") continue regionid = row[4] totalflow = 0 if row[0]!=None: totalflow += row[0] if row[1]!=None: totalflow += row[1] if row[3].day > row[2].day: # 跨天 endMinute = 23*60+59 elif row[3].day < row[2].day: PLOG.info("stoptime day less than starttime day,invalid data,skip") else: endMinute = row[3].hour*60+row[3].minute startMinute = row[2].hour*60+row[2].minute #startMinute = datetime.datetime.strptime(row[2],'%Y-%m-%d %H:%M:%S') #endMinute = datetime.datetime.strptime(row[3],'%Y-%m-%d %H:%M:%S') totalMinute = endMinute-startMinute + 1 if totalMinute <=0: PLOG.info("stoptime less than starttime,invalid data,skip") continue if SAStopDefine.stopDc.stops.has_key(regionid): stop = SAStopDefine.stopDc.stops[regionid] startindex = startMinute endindex = endMinute flowOneMinute = float(totalflow)/totalMinute/1024/1024 index = startindex while index <= endindex: stop.dayArray[index][0] += 1 stop.dayArray[index][1] += flowOneMinute if stop.dayArray[index][0] > stop.peakonlinenum: stop.peakonlinenum = stop.dayArray[index][0] stop.peakonlinetime = datetime.datetime(daydate.year,daydate.month,daydate.day,index/60,index%60) if stop.dayArray[index][0] > stop.peakbandwidth: stop.peakbandwidth = stop.dayArray[index][1] stop.peakbandwidthtime = datetime.datetime(daydate.year,daydate.month,daydate.day,index/60,index%60) index += 1 PLOG.trace("statistics end") # 数据处理结束,输出各站点峰值数据 for stopid,stop in stopsCentor.stops.items(): peakbandwidth = stop.peakbandwidth*8/60 print("%s %s %d %.2f"%(daydate.strftime('%Y-%m-%d'),stop.name,stop.peakonlinenum,peakbandwidth)) PLOG.debug("%s %s %d %.2f %s %s"%(daydate.strftime('%Y-%m-%d'),stop.name,stop.peakonlinenum,peakbandwidth,stop.peakonlinetime.strftime('%H:%M'),stop.peakbandwidthtime.strftime('%H:%M')))