def stale_record_clean(a, b):
"""
定时清除 6 个月之前的数据
"""
app = create_app()
with app.app_context():
# 获得 N 天前的日期时间
n_day_ago = (datetime.datetime.now() - datetime.timedelta(days=cons.CLEANING_RECORD_INTERVAL_TIME / cons.DAY_UNIT))
# 转换为时间字符串
clean_time = n_day_ago.strftime("%Y-%m-%d %H:%M:%S")
# "wf_record"
# 开始删除, 每次删除 DEL_INTERVAL
del_count = 0
start_time = time.time()
while True:
del_record_list = db.session.query(WfRecord).filter(WfRecord.create_time < clean_time).limit(cons.DEL_INTERVAL).all()
#
if not del_record_list:
break
for record in del_record_list:
db.session.delete(record)
#
db.session.commit()
# 统计删除的条数
del_count + len(del_record_list)
#
clear_cost = round((time.time() - start_time) * 1000, 2)
logging.info("clearing stale record complete, clear total:[{}], clear time:[{}ms]".format(del_count, clear_cost))
def yesterday_count(a, b):
app = create_app()
with app.app_context():
# 昨天日期
yesterday = (datetime.datetime.now() -
datetime.timedelta(days=1)).strftime("%Y-%m-%d")
result = get_count(yesterday)
insert_date_count(result)
def all_date_count(a, b):
app = create_app(RUN_MODEL)
with app.app_context():
record = db.session.query(Wfrecord).first()
first_time = record.data_entry_time.strftime("%Y-%m-%d")
next_day = first_time
today = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
print('+++++++++ all_date_count', today, next_day)
while today != next_day:
result = get_count(next_day)
insert_date_count(result)
next_day = (datetime.datetime.strptime(next_day, '%Y-%m-%d') + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
def all_date_count(a, b):
app = create_app()
with app.app_context():
#
record = db.session.query(WfRecord).first()
if record:
# 第一条数据的录入时间
first_date = record.create_time.strftime("%Y-%m-%d")
next_date = first_date
# today = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
current_date = to_day()
print('+++++++++ all_date_count', next_date, current_date)
while next_date < current_date:
result = get_count(next_date)
insert_date_count(result)
next_date = (
datetime.datetime.strptime(next_date, '%Y-%m-%d') +
datetime.timedelta(days=1)).strftime("%Y-%m-%d")
def add_data(tmp, ex_tmp):
app = create_app()
with app.app_context():
# adds = []
# ex_adds = []
for t in tmp:
w_id = t["id"]
w = db.session.query(WfRecord).filter(WfRecord.id == w_id).first()
w.sdk_car_plate_number = t["plate_number"]
w.sdk_car_plate_type = t["plate_type"]
w.sdk_reason_code = t["reason_code"]
w.sdk_plate_rect = json.dumps(t["plate_rect"])
w.recog_status = 2
w.recog_data = t['recog_data']
w.plate_head_score = t['head_score']
w.sdk_recog_time = now_time()
# db.session.bulk_save_objects(adds)
# db.session.add_all(adds)
for t in ex_tmp:
w_id = t
w = db.session.query(WfRecord).filter(WfRecord.id == w_id).first()
w.recog_status = 3
db.session.commit()
def sdk_request(a, b):
app = create_app()
with app.app_context():
# 一天限制40w数量图片
global img_counts, today
if today != time.strftime('%Y-%m-%d', time.localtime(time.time())):
today = time.strftime('%Y-%m-%d', time.localtime(time.time()))
img_counts = 0
if img_counts >= cons.MAX_IMAGE_NUM:
logging.info("image processing number reached 40w")
# scheduler.pause_job('job2')
return
# query = s.query(WfRecord.id, WfRecord.car_plate_number, WfRecord.car_num_pic_path, WfRecord.illegal_action)\
# .filter(WfRecord.recog_status.in_((0,))).filter(WfRecord.illegal_action.notin_(no_recog_action))
# TODO 不过滤违法编码
query = db.session.query(WfRecord.id, WfRecord.car_plate_number, WfRecord.img_path,
WfRecord.illegal_code) \
.filter(WfRecord.recog_status.in_((0,)))
if cons.LIMIT_NUM:
records = query.limit(cons.LIMIT_NUM).all()
else:
return
# records = query.all()
if not records:
# 没有查询到记录
logging.info('no records to recog')
return
else:
tmp = []
ex_tmp = []
images = []
cal_params = []
queue = Queue(cons.LIMIT_NUM)
thred_list = []
rs = []
# 创建线程
for i in range(int(cons.THREAD_NUM)):
# t = Process(target=process, args=(queue,))
t = threading.Thread(target=process, args=(queue, ))
thred_list.append(t)
for r in records:
try:
image_path = os.path.join(cons.IMAGE_SAVE_PATH, r[2])
with open(image_path, 'rb') as img_file:
img_data = img_file.read()
b64_bytes = base64.b64encode(img_data)
b64_str = b64_bytes.decode()
cal_param = calc_param_patern.format(mode="0",
srcplate=r[1].replace(
".", ""))
except Exception as e:
logging.exception('sdk_request{}'.format(e))
w_id = r[0]
w = db.session.query(WfRecord).filter(
WfRecord.id == w_id).first()
# TODO 识别异常 打开图片失败
w.recog_status = 3
# db.session.add(w)
db.session.commit()
continue
if not b64_str:
logging.error("no b64_str")
w_id = r[0]
w = db.session.query(WfRecord).filter(
WfRecord.id == w_id).first()
# TODO 没有图片的情况
w.recog_status = 4
# db.session.add(w)
db.session.commit()
continue
images.append(b64_str)
cal_params.append(cal_param)
rs.append(r)
# 如果数据大于 8 条, 一次只放 8 条数据
data_len = len(images)
if data_len >= 8:
images_fake = copy.deepcopy(images)
cal_fake = copy.deepcopy(cal_params)
rs_fake = copy.deepcopy(rs)
queue.put((images_fake, cal_fake, rs_fake))
logging.info(
"inject data into queue, count:{}".format(data_len))
del rs[:]
del images[:]
del cal_params[:]
img_counts += 1
# 一天限制40w数量图片
if img_counts >= cons.MAX_IMAGE_NUM:
logging.info("image processing number reached 40w")
# scheduler.pause_job('job2')
break
# 如果还有数据, 且没有超过授权数量, 继续放数据(用来放少于8条的数据)
if images and img_counts < cons.MAX_IMAGE_NUM:
queue.put((images, cal_params, rs))
logging.info(
"inject data into queue, count:{}".format(data_len))
# 数据放完之后, 开启线程任务
for t in thred_list:
t.start()
for t in thred_list:
t.join()
def recog_data_report(a, b):
app = create_app()
with app.app_context():
# 上报前 n 天的数据
# 获得 N 天前的日期时间, 默认 1 天
n_day_ago = (
datetime.datetime.now() -
datetime.timedelta(days=cons.REPORT_INTERVAL_TIME / cons.DAY_UNIT))
# 转换为时间字符串
start_time = n_day_ago.strftime("%Y-%m-%d %H:%M:%S")
# query data
record_list = db.session.query(WfRecord) \
.filter(WfRecord.report_status.in_([0, 2]),
WfRecord.create_time > start_time,
WfRecord.recog_status == 2) \
.limit(cons.QUERY_INTERVAL).all()
if not record_list:
logging.info(
"no data to report, report_start_time:{}".format(start_time))
return
# process
for record in record_list:
if record.car_plate_number == record.sdk_car_plate_number:
result = "true"
else:
result = "false"
#
data_dict = {
"msgid": record.id,
"sbbh": record.device_code,
"wfxw": record.illegal_code,
"wfsj": str(record.illegal_time),
"hphm": record.car_plate_number,
"result": result,
"algLpn": record.sdk_car_plate_number
}
# 处理一条, 上报一条
#
write_start = time.time()
resp = http_post(cons.SERVICE_URL, param=data_dict)
if resp is None:
# call failed
logging.error("api call timeout, url:[{}]".format(
cons.SERVICE_URL))
break
else:
# call success
cost_time = round((time.time() - write_start) * 1000, 2)
# parse resp
resp_dict = json.loads(resp.text, encoding="utf-8")
code = resp_dict["code"]
message = resp_dict["message"]
if str(code) != "200":
# report failed
report_status = cons.REPORT_FAILED
logging.error(
'api call failed, result:[code:{}, message:{}], data:[record_id:{}, illegal_code:{},'
'cost_time:[{}]ms'.format(code, message,
record.record_id,
record.illegal_code,
cost_time))
else:
report_status = cons.REPORT_SUCCESS
logging.info(
'send to service done! result:True, record_id:[{}], illegal_code:[{}], cost_time:[{}]ms'
.format(record.record_id, record.illegal_code,
cost_time))
# update report status and time
record.report_status = report_status
record.report_time = now_time()
db.session.commit()
# -*- coding:utf-8 -*-
from business import create_app
from flask_script import Manager
# 程序运行模式
RUN_MODEL = "prod"
# 从自定义方法中创建 app, 传入环境配置选项
app = create_app(RUN_MODEL)
manager = Manager(app)
if __name__ == "__main__":
manager.run()
# -*- coding:utf-8 -*-
import os
import sys
server_path = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(server_path, 'packages'))
from business import create_app
from flask_script import Manager
# 程序运行模式
RUN_MODEL = "prod"
# 从自定义方法中创建 app, 传入环境配置选项
app = create_app(config_name=RUN_MODEL)
manager = Manager(app)
if __name__ == "__main__":
manager.run()
# python3 manage.py runserver -h "0.0.0.0" -p 5000
def sdk_request(a, b):
app = create_app(RUN_MODEL)
with app.app_context():
# 一天限制40w数量图片
global img_counts, today
if today != time.strftime('%Y-%m-%d', time.localtime(time.time())):
today = time.strftime('%Y-%m-%d', time.localtime(time.time()))
img_counts = 0
if img_counts >= cons.MAX_IMAGE_NUM:
logging.info("image processing number reached 40w")
print("image processing number reached 40w")
# scheduler.pause_job('job2')
return
# query = s.query(Wfrecord.id, Wfrecord.src_car_plate_number, Wfrecord.car_num_pic_path, Wfrecord.src_illegal_action)\
# .filter(Wfrecord.check_status.in_((0,))).filter(Wfrecord.src_illegal_action.notin_(no_recog_action))
# TODO 不过滤违法编码
query = db.session.query(Wfrecord.id, Wfrecord.src_car_plate_number, Wfrecord.car_num_pic_path,
Wfrecord.src_illegal_action) \
.filter(Wfrecord.check_status.in_((0,)))
if cons.LIMIT_NUM:
records = query.limit(cons.LIMIT_NUM).all()
else:
return
# records = query.all()
if not records:
logging.info('no records to recog')
print('no records to recog')
else:
tmp = []
ex_tmp = []
images = []
cal_params = []
queue = Queue()
thred_list = []
rs = []
for i in range(int(cons.THREAD_NUM)):
t = threading.Thread(target=process, args=(queue,))
thred_list.append(t)
for r in records:
try:
# print '+++1',len(images)
image_path = os.path.join(cons.IMAGE_PATH, r[2])
with open(image_path, 'rb') as img_file:
img_data = img_file.read()
b64img = base64.b64encode(img_data)
cal_param = calc_param_patern.format(mode="0", srcplate=r[1].replace(".", ""))
except Exception as e:
logging.exception('sdk_request{}'.format(e))
w_id = r[0]
w = db.session.query(Wfrecord).filter(Wfrecord.id == w_id).first()
# TODO 调用算法异常
w.check_status = 3
# db.session.add(w)
db.session.commit()
continue
else:
if not b64img:
w_id = r[0]
w = db.session.query(Wfrecord).filter(Wfrecord.id == w_id).first()
# TODO 没有图片的情况
w.check_status = 4
# db.session.add(w)
db.session.commit()
continue
images.append(b64img)
cal_params.append(cal_param)
rs.append(r)
if len(images) == 8:
images_fake = copy.deepcopy(images)
cal_fake = copy.deepcopy(cal_params)
rs_fake = copy.deepcopy(rs)
queue.put((images_fake, cal_fake, rs_fake))
del rs[:]
del images[:]
del cal_params[:]
img_counts += 1
# 一天限制40w数量图片
if img_counts >= cons.MAX_IMAGE_NUM:
logging.info("image processing number reached 40w")
print("image processing number reached 40w")
# scheduler.pause_job('job2')
break
# 开启线程任务
for t in thred_list:
t.start()
if images and img_counts < cons.MAX_IMAGE_NUM:
queue.put((images, cal_params, rs))
for t in thred_list:
t.join()
