def stale_record_clean(a, b): """ 定时清除 6 个月之前的数据 """ app = create_app() with app.app_context(): # 获得 N 天前的日期时间 n_day_ago = (datetime.datetime.now() - datetime.timedelta(days=cons.CLEANING_RECORD_INTERVAL_TIME / cons.DAY_UNIT)) # 转换为时间字符串 clean_time = n_day_ago.strftime("%Y-%m-%d %H:%M:%S") # "wf_record" # 开始删除, 每次删除 DEL_INTERVAL del_count = 0 start_time = time.time() while True: del_record_list = db.session.query(WfRecord).filter(WfRecord.create_time < clean_time).limit(cons.DEL_INTERVAL).all() # if not del_record_list: break for record in del_record_list: db.session.delete(record) # db.session.commit() # 统计删除的条数 del_count + len(del_record_list) # clear_cost = round((time.time() - start_time) * 1000, 2) logging.info("clearing stale record complete, clear total:[{}], clear time:[{}ms]".format(del_count, clear_cost))
def yesterday_count(a, b): app = create_app() with app.app_context(): # 昨天日期 yesterday = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d") result = get_count(yesterday) insert_date_count(result)
def all_date_count(a, b): app = create_app(RUN_MODEL) with app.app_context(): record = db.session.query(Wfrecord).first() first_time = record.data_entry_time.strftime("%Y-%m-%d") next_day = first_time today = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d") print('+++++++++ all_date_count', today, next_day) while today != next_day: result = get_count(next_day) insert_date_count(result) next_day = (datetime.datetime.strptime(next_day, '%Y-%m-%d') + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
def all_date_count(a, b): app = create_app() with app.app_context(): # record = db.session.query(WfRecord).first() if record: # 第一条数据的录入时间 first_date = record.create_time.strftime("%Y-%m-%d") next_date = first_date # today = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d") current_date = to_day() print('+++++++++ all_date_count', next_date, current_date) while next_date < current_date: result = get_count(next_date) insert_date_count(result) next_date = ( datetime.datetime.strptime(next_date, '%Y-%m-%d') + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
def add_data(tmp, ex_tmp): app = create_app() with app.app_context(): # adds = [] # ex_adds = [] for t in tmp: w_id = t["id"] w = db.session.query(WfRecord).filter(WfRecord.id == w_id).first() w.sdk_car_plate_number = t["plate_number"] w.sdk_car_plate_type = t["plate_type"] w.sdk_reason_code = t["reason_code"] w.sdk_plate_rect = json.dumps(t["plate_rect"]) w.recog_status = 2 w.recog_data = t['recog_data'] w.plate_head_score = t['head_score'] w.sdk_recog_time = now_time() # db.session.bulk_save_objects(adds) # db.session.add_all(adds) for t in ex_tmp: w_id = t w = db.session.query(WfRecord).filter(WfRecord.id == w_id).first() w.recog_status = 3 db.session.commit()
def sdk_request(a, b): app = create_app() with app.app_context(): # 一天限制40w数量图片 global img_counts, today if today != time.strftime('%Y-%m-%d', time.localtime(time.time())): today = time.strftime('%Y-%m-%d', time.localtime(time.time())) img_counts = 0 if img_counts >= cons.MAX_IMAGE_NUM: logging.info("image processing number reached 40w") # scheduler.pause_job('job2') return # query = s.query(WfRecord.id, WfRecord.car_plate_number, WfRecord.car_num_pic_path, WfRecord.illegal_action)\ # .filter(WfRecord.recog_status.in_((0,))).filter(WfRecord.illegal_action.notin_(no_recog_action)) # TODO 不过滤违法编码 query = db.session.query(WfRecord.id, WfRecord.car_plate_number, WfRecord.img_path, WfRecord.illegal_code) \ .filter(WfRecord.recog_status.in_((0,))) if cons.LIMIT_NUM: records = query.limit(cons.LIMIT_NUM).all() else: return # records = query.all() if not records: # 没有查询到记录 logging.info('no records to recog') return else: tmp = [] ex_tmp = [] images = [] cal_params = [] queue = Queue(cons.LIMIT_NUM) thred_list = [] rs = [] # 创建线程 for i in range(int(cons.THREAD_NUM)): # t = Process(target=process, args=(queue,)) t = threading.Thread(target=process, args=(queue, )) thred_list.append(t) for r in records: try: image_path = os.path.join(cons.IMAGE_SAVE_PATH, r[2]) with open(image_path, 'rb') as img_file: img_data = img_file.read() b64_bytes = base64.b64encode(img_data) b64_str = b64_bytes.decode() cal_param = calc_param_patern.format(mode="0", srcplate=r[1].replace( ".", "")) except Exception as e: logging.exception('sdk_request{}'.format(e)) w_id = r[0] w = db.session.query(WfRecord).filter( WfRecord.id == w_id).first() # TODO 识别异常 打开图片失败 w.recog_status = 3 # db.session.add(w) db.session.commit() continue if not b64_str: logging.error("no b64_str") w_id = r[0] w = db.session.query(WfRecord).filter( WfRecord.id == w_id).first() # TODO 没有图片的情况 w.recog_status = 4 # db.session.add(w) db.session.commit() continue images.append(b64_str) cal_params.append(cal_param) rs.append(r) # 如果数据大于 8 条, 一次只放 8 条数据 data_len = len(images) if data_len >= 8: images_fake = copy.deepcopy(images) cal_fake = copy.deepcopy(cal_params) rs_fake = copy.deepcopy(rs) queue.put((images_fake, cal_fake, rs_fake)) logging.info( "inject data into queue, count:{}".format(data_len)) del rs[:] del images[:] del cal_params[:] img_counts += 1 # 一天限制40w数量图片 if img_counts >= cons.MAX_IMAGE_NUM: logging.info("image processing number reached 40w") # scheduler.pause_job('job2') break # 如果还有数据, 且没有超过授权数量, 继续放数据(用来放少于8条的数据) if images and img_counts < cons.MAX_IMAGE_NUM: queue.put((images, cal_params, rs)) logging.info( "inject data into queue, count:{}".format(data_len)) # 数据放完之后, 开启线程任务 for t in thred_list: t.start() for t in thred_list: t.join()
def recog_data_report(a, b): app = create_app() with app.app_context(): # 上报前 n 天的数据 # 获得 N 天前的日期时间, 默认 1 天 n_day_ago = ( datetime.datetime.now() - datetime.timedelta(days=cons.REPORT_INTERVAL_TIME / cons.DAY_UNIT)) # 转换为时间字符串 start_time = n_day_ago.strftime("%Y-%m-%d %H:%M:%S") # query data record_list = db.session.query(WfRecord) \ .filter(WfRecord.report_status.in_([0, 2]), WfRecord.create_time > start_time, WfRecord.recog_status == 2) \ .limit(cons.QUERY_INTERVAL).all() if not record_list: logging.info( "no data to report, report_start_time:{}".format(start_time)) return # process for record in record_list: if record.car_plate_number == record.sdk_car_plate_number: result = "true" else: result = "false" # data_dict = { "msgid": record.id, "sbbh": record.device_code, "wfxw": record.illegal_code, "wfsj": str(record.illegal_time), "hphm": record.car_plate_number, "result": result, "algLpn": record.sdk_car_plate_number } # 处理一条, 上报一条 # write_start = time.time() resp = http_post(cons.SERVICE_URL, param=data_dict) if resp is None: # call failed logging.error("api call timeout, url:[{}]".format( cons.SERVICE_URL)) break else: # call success cost_time = round((time.time() - write_start) * 1000, 2) # parse resp resp_dict = json.loads(resp.text, encoding="utf-8") code = resp_dict["code"] message = resp_dict["message"] if str(code) != "200": # report failed report_status = cons.REPORT_FAILED logging.error( 'api call failed, result:[code:{}, message:{}], data:[record_id:{}, illegal_code:{},' 'cost_time:[{}]ms'.format(code, message, record.record_id, record.illegal_code, cost_time)) else: report_status = cons.REPORT_SUCCESS logging.info( 'send to service done! result:True, record_id:[{}], illegal_code:[{}], cost_time:[{}]ms' .format(record.record_id, record.illegal_code, cost_time)) # update report status and time record.report_status = report_status record.report_time = now_time() db.session.commit()
# -*- coding:utf-8 -*- from business import create_app from flask_script import Manager # 程序运行模式 RUN_MODEL = "prod" # 从自定义方法中创建 app, 传入环境配置选项 app = create_app(RUN_MODEL) manager = Manager(app) if __name__ == "__main__": manager.run()
# -*- coding:utf-8 -*- import os import sys server_path = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(server_path, 'packages')) from business import create_app from flask_script import Manager # 程序运行模式 RUN_MODEL = "prod" # 从自定义方法中创建 app, 传入环境配置选项 app = create_app(config_name=RUN_MODEL) manager = Manager(app) if __name__ == "__main__": manager.run() # python3 manage.py runserver -h "0.0.0.0" -p 5000
def sdk_request(a, b): app = create_app(RUN_MODEL) with app.app_context(): # 一天限制40w数量图片 global img_counts, today if today != time.strftime('%Y-%m-%d', time.localtime(time.time())): today = time.strftime('%Y-%m-%d', time.localtime(time.time())) img_counts = 0 if img_counts >= cons.MAX_IMAGE_NUM: logging.info("image processing number reached 40w") print("image processing number reached 40w") # scheduler.pause_job('job2') return # query = s.query(Wfrecord.id, Wfrecord.src_car_plate_number, Wfrecord.car_num_pic_path, Wfrecord.src_illegal_action)\ # .filter(Wfrecord.check_status.in_((0,))).filter(Wfrecord.src_illegal_action.notin_(no_recog_action)) # TODO 不过滤违法编码 query = db.session.query(Wfrecord.id, Wfrecord.src_car_plate_number, Wfrecord.car_num_pic_path, Wfrecord.src_illegal_action) \ .filter(Wfrecord.check_status.in_((0,))) if cons.LIMIT_NUM: records = query.limit(cons.LIMIT_NUM).all() else: return # records = query.all() if not records: logging.info('no records to recog') print('no records to recog') else: tmp = [] ex_tmp = [] images = [] cal_params = [] queue = Queue() thred_list = [] rs = [] for i in range(int(cons.THREAD_NUM)): t = threading.Thread(target=process, args=(queue,)) thred_list.append(t) for r in records: try: # print '+++1',len(images) image_path = os.path.join(cons.IMAGE_PATH, r[2]) with open(image_path, 'rb') as img_file: img_data = img_file.read() b64img = base64.b64encode(img_data) cal_param = calc_param_patern.format(mode="0", srcplate=r[1].replace(".", "")) except Exception as e: logging.exception('sdk_request{}'.format(e)) w_id = r[0] w = db.session.query(Wfrecord).filter(Wfrecord.id == w_id).first() # TODO 调用算法异常 w.check_status = 3 # db.session.add(w) db.session.commit() continue else: if not b64img: w_id = r[0] w = db.session.query(Wfrecord).filter(Wfrecord.id == w_id).first() # TODO 没有图片的情况 w.check_status = 4 # db.session.add(w) db.session.commit() continue images.append(b64img) cal_params.append(cal_param) rs.append(r) if len(images) == 8: images_fake = copy.deepcopy(images) cal_fake = copy.deepcopy(cal_params) rs_fake = copy.deepcopy(rs) queue.put((images_fake, cal_fake, rs_fake)) del rs[:] del images[:] del cal_params[:] img_counts += 1 # 一天限制40w数量图片 if img_counts >= cons.MAX_IMAGE_NUM: logging.info("image processing number reached 40w") print("image processing number reached 40w") # scheduler.pause_job('job2') break # 开启线程任务 for t in thred_list: t.start() if images and img_counts < cons.MAX_IMAGE_NUM: queue.put((images, cal_params, rs)) for t in thred_list: t.join()