def business_process(inputs):
"""
不同业务，数据处理的方式可能不一样，统一在这里修改和控制
为了充分利用多线程并行的方式来处理数据，所以首先由生产者调用process_data来处理数据，并提交至任务队列
此处直接从任务队列中获取处理好的数据即可，因此该部分应该和process_data一起修改
:param inputs: 从任务队列中获取的预处理后的数据
:return:
"""
# 数据
if len(inputs) < 0:
logger.critical("must set a data")
exit(1)
start_time = time.time()
# 通过不同的model名称来调用模型级联任务中 不同阶段需要用到的模型
text = ""
try:
result = infer_with_executor(inputs)
period = time.time() - start_time
if len(result) == 0 or result is None:
text = ""
elif len(result) > 0:
return result
logger.info(
"executor infer num {} cost time: {}".format(str(10), "%2.2f sec" % period))
except Exception:
log.info("模型预测出现异常".format(Exception))
return text
def init_evn(cls, current_evn):
"""
init config environment, and only once
:param current_evn:
:return:
"""
if cls.env is None:
cls.env = current_evn
logger.info("current config section %s",
cls.env,
extra={'logid': 'react-main-loop-logid'})
async def open(self):
# buyer_id 就是买家的标识符， seller_id 是卖家的主页
logger.info('new websocket uri:{} coming'.format(self.request.uri))
if self.buyer_id:
self.chat_container[self.buyer_id].append(self)
is_new = self.is_new_user(self.seller_id, self.buyer_id)
if is_new:
# 拷贝机器人
# self.greet_by_robot()
self.copy_flow_to_status()
msg = self.build_msg(self.buyer_id, self.seller_id, 'test')
self.check_robot_reply(msg)
else:
self.chat_container[self.seller_id].append(self)
def send_msg(self, chat_msg):
logger.info('ready send messenger %s' % chat_msg)
waiter_set = set()
for send_key in [
chat_msg.get(CONST.MSG_FROM),
chat_msg.get(CONST.MSG_TO)
]:
for waiter in self.chat_container.get(send_key, []):
waiter_set.add(waiter)
for waiter in waiter_set:
_chat_msg = dict(chat_msg)
try:
waiter.write_message(_chat_msg)
except:
logger.error("Error sending message", exc_info=True)
async def on_message(self, message):
logger.info("got message %s", message)
try:
msg = tornado.escape.json_decode(message)
except Exception:
await self.write_message(
{CONST.REASON: "the message isn't a json"})
return
msg_from = self.client_key
msg_to = msg.get(CONST.MSG_TO)
msg_content = msg.get(CONST.MSG_CONTENT)
msg_type = msg.get(CONST.MSG_TYPE, 'text')
chat_msg = self.build_msg(msg_from, msg_to, msg_content, msg_type)
self.send_msg_and_to_db(chat_msg)
self.check_robot_reply(chat_msg)
def recognize():
"""
处理post请求,并返回处理后的结果
"""
if not request.json:
abort(400)
if 'orderNo' in request.json:
orderNo = request.json['orderNo']
logger.set_logid(str(orderNo))
else:
logger.set_auto_logid()
orderNo = logger.get_logid()
orderNo = int(orderNo)
result = ApiResult(orderNo=orderNo)
start_time = time.time()
try:
origin_data = request.json
log.debug('origin_data: %s' % repr(origin_data))
task_data = process_data(origin_data)
log.debug('task_data_orderNo: %s' % repr(task_data))
start_time = time.time()
log.debug("prepare start_time cost time: {}".format("%2.2f sec" %
start_time))
max_request_time = float(config.get('max_request_time'))
index = pivot.set_task_queue(task_data, max_request_time)
if index is not None:
text = pivot.get_result_list(index)
log.debug('text: %s' % repr(text))
pred = dict()
pred['result'] = text
pred['orderNo'] = orderNo
result.success(data=pred)
else:
result.error(message="busy, wait then retry")
except Exception as err:
logger.error("infer exception: {}".format(err))
result.error(message=str(err))
period = time.time() - start_time
logger.info("request cost:{}".format("%2.2f sec" % period))
return json.dumps(result, default=lambda o: o.__dict__)
def copy_flow_to_status(self, keyword='test'):
filter_dict = {
# CONST.SELLER_ID: self.seller_id,
'{}.{}'.format(CONST.KEYWORD_LIST, CONST.KEYWORD):
keyword
}
flow = mongo_op.find_one(self.auto_flow_coll, filter_dict)
logger.info('copy flow_id:{} to buyer_id:{} status'.format(
flow[CONST.ID], self.buyer_id))
if flow:
flow[CONST.BUYER_ID] = self.buyer_id
flow[CONST.STATUS] = 'processing'
flow[CONST.KEYWORD] = keyword
flow['origin_flow_id'] = str(flow.pop(CONST.ID, ''))
for action in flow.get(CONST.ACTION_LIST, []):
if action.get('action_type') == 'flow_start':
flow['current_action_id'] = action.get('action_id')
old_action_list = flow['action_list']
flow['action_list'] = {i['action_id']: i for i in old_action_list}
mongo_op.insert_one(self.auto_flow_coll_status, flow)
def check_robot_reply(self, msg):
logger.info(
'ready check_robot_reply, seller_id:{}, buyer_id:{}'.format(
self.seller_id, self.buyer_id))
if msg[CONST.MSG_FROM] != self.buyer_id:
return
filter_dict = {
CONST.BUYER_ID: self.buyer_id,
# CONST.SELLER_ID: self.seller_id,
CONST.STATUS: 'processing',
}
flow = mongo_op.find_one(self.auto_flow_coll_status,
filter_dict,
sort=[(CONST.ID, -1)])
if not flow:
logger.info("Can't find flow")
return
keyword = msg[CONST.MSG_CONTENT]
action_dict = flow[CONST.ACTION_LIST]
action = action_dict[flow['current_action_id']]
action_data = action['action_data']
action = action_dict[action['next_id_list'][0]]
# 判断关键词
# if action_data.get('is_reply') == 'T':
# for i in action_data.get('short_reply', []):
# if i[CONST.KEYWORD] == keyword:
# action = action_dict[i['action_id']]
# break
# # for...else... 表示 for循环正常执行，没有 Break 才会执行的语句
# else:
# return
while True:
# 先判断关键词
action_data = action['action_data']
logger.info(action_data)
if action['action_type'] == 'send_img':
robot_msg = self.build_msg(self.seller_id,
self.buyer_id,
[{
"url": action_data['file_url']
}],
msg_type='image')
self.reply_by_robot(robot_msg)
elif action['action_type'] == 'send_msg':
if action_data['is_reply'] == 'T':
robot_msg = self.build_msg(
self.seller_id,
self.buyer_id,
action_data['message'],
msg_type='reply',
reply_content=[
i[CONST.KEYWORD]
for i in action_data['short_reply']
])
else:
robot_msg = self.build_msg(self.seller_id, self.buyer_id,
action_data['message'])
self.reply_by_robot(robot_msg)
if action_data.get('send_continue',
'T') == 'T' and action['next_id_list']:
if action_data.get('is_reply') == 'T':
for i in action_data.get('short_reply'):
if i[CONST.KEYWORD] == keyword:
action = action_dict[i['action_id']]
else:
action = action_dict[action['next_id_list'][0]]
else:
update_dict = {'current_action_id': action['action_id']}
if not action['next_id_list']:
update_dict['status'] = 'done'
mongo_op.update_one(self.auto_flow_coll_status, filter_dict,
update_dict)
break
def on_close(self):
client_key = self.client_key
logger.info('on_close client_key: {}'.format(client_key))
self.chat_container.pop(client_key, '')
def state():
"""
访问state
"""
logger.info("visit state page")
return '0'
def index():
"""
访问root
"""
logger.info("visit root page")
return 'Index Page'
from twisted.web.resource import Resource
from twisted.web.server import Site
from twisted.web.wsgi import WSGIResource
from util.consumer import InferConsumer
from business_service import process_data
pivot = Pivot(int(config.get("max_task_num")))
infer_consumer = InferConsumer(pivot,
float(config.get("max_get_time")))
port = int(config.get("flask.server.port"))
work_thread_count = int(config.get('work.thread.count'))
site_root = str(config.get('server.root.site'))
reactor.suggestThreadPoolSize(int(work_thread_count))
flask_site = WSGIResource(reactor, reactor.getThreadPool(), app)
root = Resource()
root.putChild(
site_root if PYTHON_VERSION == 2 else bytes(site_root,
encoding='utf-8'),
flask_site)
logger.info(
"start app listen port:{} thread pool size:{} site root:{}".format(
port, work_thread_count, site_root))
reactor.listenTCP(port, Site(root))
# 把主线程设置为infer_consumer的守护线程，如果主线程结束了，infer_consumer线程就跟着结束
infer_consumer.setDaemon(True)
infer_consumer.start()
reactor.run()