def stale_record_clean(a, b):
    """
    定时清除 6 个月之前的数据
    """
    app = create_app()
    with app.app_context():
        # 获得 N 天前的日期时间
        n_day_ago = (datetime.datetime.now() - datetime.timedelta(days=cons.CLEANING_RECORD_INTERVAL_TIME / cons.DAY_UNIT))
        # 转换为时间字符串
        clean_time = n_day_ago.strftime("%Y-%m-%d %H:%M:%S")

        # "wf_record"
        # 开始删除, 每次删除 DEL_INTERVAL
        del_count = 0
        start_time = time.time()
        while True:
            del_record_list = db.session.query(WfRecord).filter(WfRecord.create_time < clean_time).limit(cons.DEL_INTERVAL).all()
            #
            if not del_record_list:
                break
            for record in del_record_list:
                db.session.delete(record)
            #
            db.session.commit()
            # 统计删除的条数
            del_count + len(del_record_list)
        #
        clear_cost = round((time.time() - start_time) * 1000, 2)
        logging.info("clearing stale record complete, clear total:[{}], clear time:[{}ms]".format(del_count, clear_cost))
Exemple #2
0
def yesterday_count(a, b):
    app = create_app()
    with app.app_context():
        # 昨天日期
        yesterday = (datetime.datetime.now() -
                     datetime.timedelta(days=1)).strftime("%Y-%m-%d")
        result = get_count(yesterday)
        insert_date_count(result)
Exemple #3
0
def all_date_count(a, b):
    app = create_app(RUN_MODEL)
    with app.app_context():
        record = db.session.query(Wfrecord).first()
        first_time = record.data_entry_time.strftime("%Y-%m-%d")
        next_day = first_time
        today = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d")

        print('+++++++++ all_date_count', today, next_day)
        while today != next_day:
            result = get_count(next_day)
            insert_date_count(result)
            next_day = (datetime.datetime.strptime(next_day, '%Y-%m-%d') + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
Exemple #4
0
def all_date_count(a, b):
    app = create_app()
    with app.app_context():
        #
        record = db.session.query(WfRecord).first()
        if record:
            # 第一条数据的录入时间
            first_date = record.create_time.strftime("%Y-%m-%d")
            next_date = first_date
            # today = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
            current_date = to_day()

            print('+++++++++ all_date_count', next_date, current_date)
            while next_date < current_date:
                result = get_count(next_date)
                insert_date_count(result)
                next_date = (
                    datetime.datetime.strptime(next_date, '%Y-%m-%d') +
                    datetime.timedelta(days=1)).strftime("%Y-%m-%d")
Exemple #5
0
def add_data(tmp, ex_tmp):
    app = create_app()
    with app.app_context():
        # adds = []
        # ex_adds = []
        for t in tmp:
            w_id = t["id"]
            w = db.session.query(WfRecord).filter(WfRecord.id == w_id).first()
            w.sdk_car_plate_number = t["plate_number"]
            w.sdk_car_plate_type = t["plate_type"]
            w.sdk_reason_code = t["reason_code"]
            w.sdk_plate_rect = json.dumps(t["plate_rect"])
            w.recog_status = 2
            w.recog_data = t['recog_data']
            w.plate_head_score = t['head_score']
            w.sdk_recog_time = now_time()
        # db.session.bulk_save_objects(adds)
        # db.session.add_all(adds)

        for t in ex_tmp:
            w_id = t
            w = db.session.query(WfRecord).filter(WfRecord.id == w_id).first()
            w.recog_status = 3
        db.session.commit()
Exemple #6
0
def sdk_request(a, b):
    app = create_app()
    with app.app_context():
        # 一天限制40w数量图片
        global img_counts, today

        if today != time.strftime('%Y-%m-%d', time.localtime(time.time())):
            today = time.strftime('%Y-%m-%d', time.localtime(time.time()))
            img_counts = 0

        if img_counts >= cons.MAX_IMAGE_NUM:
            logging.info("image processing number reached 40w")
            # scheduler.pause_job('job2')
            return

        # query = s.query(WfRecord.id, WfRecord.car_plate_number, WfRecord.car_num_pic_path, WfRecord.illegal_action)\
        #            .filter(WfRecord.recog_status.in_((0,))).filter(WfRecord.illegal_action.notin_(no_recog_action))
        # TODO 不过滤违法编码
        query = db.session.query(WfRecord.id, WfRecord.car_plate_number, WfRecord.img_path,
                                 WfRecord.illegal_code) \
                          .filter(WfRecord.recog_status.in_((0,)))
        if cons.LIMIT_NUM:
            records = query.limit(cons.LIMIT_NUM).all()
        else:
            return
            # records = query.all()
        if not records:
            # 没有查询到记录
            logging.info('no records to recog')
            return
        else:
            tmp = []
            ex_tmp = []
            images = []
            cal_params = []

            queue = Queue(cons.LIMIT_NUM)
            thred_list = []
            rs = []

            # 创建线程
            for i in range(int(cons.THREAD_NUM)):
                # t = Process(target=process, args=(queue,))
                t = threading.Thread(target=process, args=(queue, ))
                thred_list.append(t)

            for r in records:
                try:
                    image_path = os.path.join(cons.IMAGE_SAVE_PATH, r[2])
                    with open(image_path, 'rb') as img_file:
                        img_data = img_file.read()
                    b64_bytes = base64.b64encode(img_data)
                    b64_str = b64_bytes.decode()
                    cal_param = calc_param_patern.format(mode="0",
                                                         srcplate=r[1].replace(
                                                             ".", ""))
                except Exception as e:
                    logging.exception('sdk_request{}'.format(e))
                    w_id = r[0]
                    w = db.session.query(WfRecord).filter(
                        WfRecord.id == w_id).first()
                    # TODO 识别异常 打开图片失败
                    w.recog_status = 3
                    # db.session.add(w)
                    db.session.commit()
                    continue

                if not b64_str:
                    logging.error("no b64_str")
                    w_id = r[0]
                    w = db.session.query(WfRecord).filter(
                        WfRecord.id == w_id).first()
                    # TODO 没有图片的情况
                    w.recog_status = 4
                    # db.session.add(w)
                    db.session.commit()
                    continue
                images.append(b64_str)
                cal_params.append(cal_param)
                rs.append(r)

                # 如果数据大于 8 条, 一次只放 8 条数据
                data_len = len(images)
                if data_len >= 8:
                    images_fake = copy.deepcopy(images)
                    cal_fake = copy.deepcopy(cal_params)
                    rs_fake = copy.deepcopy(rs)

                    queue.put((images_fake, cal_fake, rs_fake))
                    logging.info(
                        "inject data into queue, count:{}".format(data_len))

                    del rs[:]
                    del images[:]
                    del cal_params[:]

                    img_counts += 1
                    # 一天限制40w数量图片
                    if img_counts >= cons.MAX_IMAGE_NUM:
                        logging.info("image processing number reached 40w")
                        # scheduler.pause_job('job2')
                        break
            # 如果还有数据, 且没有超过授权数量, 继续放数据(用来放少于8条的数据)
            if images and img_counts < cons.MAX_IMAGE_NUM:
                queue.put((images, cal_params, rs))
                logging.info(
                    "inject data into queue, count:{}".format(data_len))

            # 数据放完之后, 开启线程任务
            for t in thred_list:
                t.start()

            for t in thred_list:
                t.join()
Exemple #7
0
def recog_data_report(a, b):
    app = create_app()
    with app.app_context():
        # 上报前 n 天的数据
        # 获得 N 天前的日期时间, 默认 1 天
        n_day_ago = (
            datetime.datetime.now() -
            datetime.timedelta(days=cons.REPORT_INTERVAL_TIME / cons.DAY_UNIT))
        # 转换为时间字符串
        start_time = n_day_ago.strftime("%Y-%m-%d %H:%M:%S")

        # query data
        record_list = db.session.query(WfRecord) \
            .filter(WfRecord.report_status.in_([0, 2]),
                    WfRecord.create_time > start_time,
                    WfRecord.recog_status == 2) \
            .limit(cons.QUERY_INTERVAL).all()
        if not record_list:
            logging.info(
                "no data to report, report_start_time:{}".format(start_time))
            return

        # process
        for record in record_list:
            if record.car_plate_number == record.sdk_car_plate_number:
                result = "true"
            else:
                result = "false"
            #
            data_dict = {
                "msgid": record.id,
                "sbbh": record.device_code,
                "wfxw": record.illegal_code,
                "wfsj": str(record.illegal_time),
                "hphm": record.car_plate_number,
                "result": result,
                "algLpn": record.sdk_car_plate_number
            }

            # 处理一条, 上报一条
            #
            write_start = time.time()
            resp = http_post(cons.SERVICE_URL, param=data_dict)
            if resp is None:
                # call failed
                logging.error("api call timeout, url:[{}]".format(
                    cons.SERVICE_URL))
                break
            else:
                # call success
                cost_time = round((time.time() - write_start) * 1000, 2)
                # parse resp
                resp_dict = json.loads(resp.text, encoding="utf-8")
                code = resp_dict["code"]
                message = resp_dict["message"]
                if str(code) != "200":
                    # report failed
                    report_status = cons.REPORT_FAILED
                    logging.error(
                        'api call failed, result:[code:{}, message:{}], data:[record_id:{}, illegal_code:{},'
                        'cost_time:[{}]ms'.format(code, message,
                                                  record.record_id,
                                                  record.illegal_code,
                                                  cost_time))
                else:
                    report_status = cons.REPORT_SUCCESS
                    logging.info(
                        'send to service done! result:True, record_id:[{}], illegal_code:[{}], cost_time:[{}]ms'
                        .format(record.record_id, record.illegal_code,
                                cost_time))
                # update report status and time
                record.report_status = report_status
                record.report_time = now_time()
                db.session.commit()
Exemple #8
0
# -*- coding:utf-8 -*-
from business import create_app
from flask_script import Manager

# 程序运行模式
RUN_MODEL = "prod"

# 从自定义方法中创建 app, 传入环境配置选项
app = create_app(RUN_MODEL)
manager = Manager(app)

if __name__ == "__main__":
    manager.run()
Exemple #9
0
# -*- coding:utf-8 -*-
import os
import sys

server_path = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(server_path, 'packages'))
from business import create_app
from flask_script import Manager

# 程序运行模式
RUN_MODEL = "prod"

# 从自定义方法中创建 app, 传入环境配置选项
app = create_app(config_name=RUN_MODEL)
manager = Manager(app)

if __name__ == "__main__":
    manager.run()
    # python3 manage.py runserver -h "0.0.0.0" -p 5000
Exemple #10
0
def sdk_request(a, b):
    app = create_app(RUN_MODEL)
    with app.app_context():
        # 一天限制40w数量图片
        global img_counts, today

        if today != time.strftime('%Y-%m-%d', time.localtime(time.time())):
            today = time.strftime('%Y-%m-%d', time.localtime(time.time()))
            img_counts = 0

        if img_counts >= cons.MAX_IMAGE_NUM:
            logging.info("image processing number reached 40w")
            print("image processing number reached 40w")
            # scheduler.pause_job('job2')
            return

        # query = s.query(Wfrecord.id, Wfrecord.src_car_plate_number, Wfrecord.car_num_pic_path, Wfrecord.src_illegal_action)\
        #            .filter(Wfrecord.check_status.in_((0,))).filter(Wfrecord.src_illegal_action.notin_(no_recog_action))
        # TODO 不过滤违法编码
        query = db.session.query(Wfrecord.id, Wfrecord.src_car_plate_number, Wfrecord.car_num_pic_path,
                                 Wfrecord.src_illegal_action) \
                          .filter(Wfrecord.check_status.in_((0,)))
        if cons.LIMIT_NUM:
            records = query.limit(cons.LIMIT_NUM).all()
        else:
            return
            # records = query.all()
        if not records:
            logging.info('no records to recog')
            print('no records to recog')
        else:
            tmp = []
            ex_tmp = []
            images = []
            cal_params = []

            queue = Queue()
            thred_list = []
            rs = []

            for i in range(int(cons.THREAD_NUM)):
                t = threading.Thread(target=process, args=(queue,))
                thred_list.append(t)

            for r in records:
                try:
                    # print '+++1',len(images)
                    image_path = os.path.join(cons.IMAGE_PATH, r[2])
                    with open(image_path, 'rb') as img_file:
                        img_data = img_file.read()
                    b64img = base64.b64encode(img_data)
                    cal_param = calc_param_patern.format(mode="0", srcplate=r[1].replace(".", ""))
                except Exception as e:
                    logging.exception('sdk_request{}'.format(e))

                    w_id = r[0]
                    w = db.session.query(Wfrecord).filter(Wfrecord.id == w_id).first()
                    # TODO 调用算法异常
                    w.check_status = 3
                    # db.session.add(w)
                    db.session.commit()
                    continue

                else:
                    if not b64img:
                        w_id = r[0]
                        w = db.session.query(Wfrecord).filter(Wfrecord.id == w_id).first()
                        # TODO 没有图片的情况
                        w.check_status = 4
                        # db.session.add(w)
                        db.session.commit()
                        continue
                    images.append(b64img)
                    cal_params.append(cal_param)
                    rs.append(r)

                if len(images) == 8:
                    images_fake = copy.deepcopy(images)
                    cal_fake = copy.deepcopy(cal_params)
                    rs_fake = copy.deepcopy(rs)

                    queue.put((images_fake, cal_fake, rs_fake))

                    del rs[:]
                    del images[:]
                    del cal_params[:]

                    img_counts += 1
                    # 一天限制40w数量图片
                    if img_counts >= cons.MAX_IMAGE_NUM:
                        logging.info("image processing number reached 40w")
                        print("image processing number reached 40w")
                        # scheduler.pause_job('job2')
                        break
            # 开启线程任务
            for t in thred_list:
                t.start()

            if images and img_counts < cons.MAX_IMAGE_NUM:
                queue.put((images, cal_params, rs))

            for t in thred_list:
                t.join()