def run(): """ get work flow """ now = datetime.datetime.now() now.strftime('%Y-%m-%d 00:00:00') now_time = str(now).split(' ')[0] + "%" now = str(now).split(' ')[0] + " 00:00:00" operatedb = OperateDB("./mysql.conf") operatedb.connect() configure = ConfigParser.ConfigParser() configure.read('reason.conf') figures = configure.options('reason') for figure in figures: sql_cmd = "select sum(count) as count from work_flow_news_%s where time like \'%s\'"%(figure, now_time) cursor = operatedb.selectsql(sql_cmd) result = cursor.fetchall() print result if result[0][0] != None: reason = configure.get('reason', figure) sql_cmd = "replace into work_flow_news (time, reason, count) values (\'%s\', \'%s\', %s)"%(now, reason, int(result[0][0])) print sql_cmd operatedb.executesql(sql_cmd)
def main(): """ 任务调度 5分钟采集一次数据 """ global flow_funnel operatedb = OperateDB("./mysql.conf") operatedb.connect() pool = redis.ConnectionPool(host='localhost', port=6380, db=0) redis_data = redis.Redis(connection_pool=pool) keys = redis_data.keys() for key in keys: file_record[key] = redis_data.get(key) while True: rootdir = "../cpu_publish_image_api/" #now = datetime.datetime.now() #begin_time = now.replace(hour = 00, minute = 15) #end_time = now.replace(hour = 00, minute = 45) #if now >= begin_time and now <= end_time: # cmd = 'sh -x ./clear_api.sh' # os.system(cmd) # sys.exit() #else: filelist = get_file_list(rootdir) filelist = sorted(filelist) if len(filelist) == 0: continue else: for filename in filelist: flow_funnel = handle_file(filename) if len(flow_funnel.keys()) != 0: now = datetime.datetime.now() now = now.strftime('%Y-%M-%D %H:%M:%S') now = now.split(' ')[0] + "%" sql = "select time from Image_Flow_Funnel where Stage = 200 and time like \'%s\' order by time desc" % ( now) cursor = operatedb.selectsql(sql) timelist = [] for row in cursor: row = ''.join(row) timelist.append(row) datelist = sorted(flow_funnel.keys()) for i in range(0, len(datelist) - 1): if datelist[i] in timelist: sql = "update Image_Flow_Funnel\ set ImageNum = ImageNum + %s where Stage = 200 and time = \'%s\'" % ( flow_funnel[datelist[i]], datelist[i]) logging.info( "save to db success image number as %s%s" % (flow_funnel[datelist[i]], sql)) else: sql = "insert into Image_Flow_Funnel\ (time, Stage, ImageNum) values (\'%s\', \'%s\', %s)"\ %(datelist[i], '200', flow_funnel[datelist[i]]) logging.info( "save to db success image number as %s%s" % (flow_funnel[datelist[i]], sql)) operatedb.executesql(sql) flow_funnel.pop(datelist[i]) else: continue for filename in file_record.keys(): redis_data.set(filename, file_record[filename]) operatedb.disconnect()
def main(): """ 实时采集数据 """ logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S', filename='savetodb.log', filemode='w') operatedb = OperateDB("./mysql.conf") operatedb.connect() pool = redis.ConnectionPool(host='localhost', port=6381, db=0) redis_data = redis.Redis(connection_pool=pool) keys = redis_data.keys() for key in keys: file_record[key] = redis_data.get(key) while True: rootdir = "../cpu_publish_image_mq/" filelist = get_file_list(rootdir) filelist = sorted(filelist) if len(filelist) == 0: continue else: for filename in filelist: save_to_db = handle_file(filename) """ 策略过滤掉的图片数量以及对应的原因 """ """ 入库的图片数量以及对应的类目信息 """ now = datetime.datetime.now() begin_time = now.replace(hour = 00, minute = 15) end_time = now.replace(hour = 00, minute = 45) if now >= begin_time and now <= end_time: cmd = "sh -x ./clear_savedb.sh" os.system(cmd) sys.exit() else: now.strftime('%Y-%m-%d 00:00:00') now = str(now).split(' ')[0] + "%" sql = "select time from Image_Flow_Funnel where Stage = 400 and time like \'%s\' order by time desc"%(now) cursor = operatedb.selectsql(sql) timelist = [] for row in cursor: row = ''.join(row) timelist.append(row) if len(save_to_db.keys()) != 0: datelist = sorted(save_to_db.keys()) for i in range(0, len(datelist)): if datelist[i] in timelist: sql = "update Image_Flow_Funnel\ set ImageNum = ImageNum + %s where\ time = \'%s\' and Stage = 400"%(save_to_db[datelist[i]], datelist[i]) logging.info("save to db success image number as %s%s"%(save_to_db[datelist[i]], sql)) else: sql = "insert into Image_Flow_Funnel\ (time, Stage, ImageNum) values (\'%s\', \'400\', %s)"\ %(datelist[i], save_to_db[datelist[i]]) logging.info("save to db success image number as %s%s"%(save_to_db[datelist[i]], sql)) operatedb.executesql(sql) save_to_db.pop(datelist[i]) else: continue for filename in file_record.keys(): redis_data.set(filename, file_record[filename]) operatedb.disconnect()
def run(self): """ run the mian process """ logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S', filename='videoworkflow_%s.log'%(self.table), filemode='w') operatedb = OperateDB("./mysql.conf") pool = redis.ConnectionPool(host='localhost', port=6387, db=0) redis_data = redis.Redis(connection_pool=pool) self.file_record[reason] = redis_data.hgetall(reason) while True: rootdir = "../cpu_publish_news_mq/" filelist = self.get_file_list(rootdir) filelist = sorted(filelist) if len(filelist) == 0: continue else: for filename in filelist: logging.info("filename is %s"%(filename)) self.work_flow = self.handle_file(filename) now = datetime.datetime.now() now.strftime('%Y-%m-%d 00:00:00') now = str(now).split(' ')[0] + "%" if len(self.work_flow.keys()) != 0: sql = "select reason, time from work_flow_video where time like \'%s\' order by time desc"%(now) logging.info("sql is %s"%(sql)) try: operatedb.connect() cursor = operatedb.selectsql(sql) results = cursor.fetchall() except BaseException, e: logging.warning("get reason, time result failed %s"%(e)) timelist = [] if len(results) == 0: datelist = sorted(self.work_flow.keys()) for i in range(0, len(datelist)): sql = "insert into work_flow_video\ (time, count, reason) values (\'%s\', \'%s\', \'%s\')"\ %(datelist[i], self.work_flow[datelist[i]], self.reason) logging.info("sql is %s"%(sql)) operatedb.executesql(sql) self.work_flow.pop(datelist[i]) else: for result in results: timelist.append(result[1]) datelist = sorted(self.work_flow.keys()) for i in range(0, len(datelist)): if ( datelist[i] in timelist and result[0] == reason): sql = "update work_flow_video\ set count = count + %s where\ time = \'%s\' and reason = \'%s\'"%(self.work_flow[datelist[i]], datelist[i], reason) logging.info("sql is %s"%(sql)) else: sql = "insert into work_flow_video\ (time, count, reason) values (\'%s\', \'%s\', \'%s\')"\ %(datelist[i], self.work_flow[datelist[i]], reason) logging.info("sql is %s"%(sql)) operatedb.executesql(sql) logging.info("save to db success video number as %s"%(self.work_flow[datelist[i]])) self.work_flow.pop(datelist[i]) else: continue redis_data.hset(reason, filename, self.file_record[reason][filename]) operatedb.disconnect()