def extract_data_ipc_file(file_path, logger):
"""Extracts data from a pickles file at file_path
Returns the data"""
num_tries = 10
file_path = file_path.decode('utf-8')
while True:
try:
time.sleep(0.1)
if os.path.isfile(file_path):
with open(file_path, "rb") as pickle_file:
data = pickle.load(pickle_file)
#print data
return data
else:
return pd.DataFrame()
except EOFError:
if num_tries == 0:
raise
else:
logger.info("EOFError on extracting: " + str(file_path) +
" trying again...")
num_tries = num_tries - 1
continue
except (OSError, IOError) as e:
print_error_message(
e, "Error 2.2b: Data exchange issues related to file: " +
file_path, logger)
def text_reply(self, msg):
logger.debug(u'msg dict : ' + json.dumps(msg, indent=True, ensure_ascii=False))
#from_user = itchat.search_friends(userName=msg['FromUserName'])['NickName'] if \
# itchat.search_friends(userName=msg['FromUserName']) else 'Somebody'
from_user = msg.get('ActualNickName', 'Somebody')
logger.info(u'Receive message [%s] from [%s]' % (msg['Text'], from_user))
#itchat.send_msg(u'自动回复收到-来自Brishen的机器人', toUserName=msg['FromUserName'])
if msg.get('isAt'):
logger.info(u'收到At你的消息，内容是[%s]' % msg.get('Text'))
itchat.send_msg(u'自动回复收到-来自Brishen的机器人', toUserName=msg.get('FromUserName'))
def object_detect(self):
logger.info('face++ object detect picurl=%s', self.picurl)
s = requests.session()
post_params = {
'api_key': self.api_key,
'api_secret': self.api_secret,
'image_url': self.picurl
}
r = s.post(self.object_detect_url, data=post_params)
object_result = r.content
logger.info('face++ object detect = %s', object_result)
return object_result
def prepare(self):
"""
overwrite setup
"""
logger.info('===== ITCHAT MESSAGE START =====')
itchat.auto_login(hotReload=True, enableCmdQR=True)
itchat.run()
#获取friends和chatroom信息
logger.debug('Friend list :' + json.dumps(
itchat.get_friends(update=True), indent=True, ensure_ascii=False))
logger.debug('Chatroom list :' + json.dumps(
itchat.get_chatrooms(
update=True), indent=True, ensure_ascii=False))
def txt_process(self):
txt_rst_str = get_random_err_msg()
try:
logger.info('txt_process = %s', self.text)
if self.text in [u'刘雨墨', u'刘雨墨墨娃']:
logger.info('Call Menu %s', SITE_IMG_URL)
txt_rst_str = u'<a href="%s">干嘛</a>' % SITE_IMG_URL
else:
turing_result = self.turing_bot.get_turing_result(self.text)
logger.info('turing_result = %s', turing_result)
turing_json = json.loads(turing_result)
text_strs = []
text_strs.append(turing_json.get('text'))
for each in turing_json.get('list', []):
text_strs.append(u'<a href="%s">%s</a>' % \
((each.get('detailurl', '')),
each.get('article', '')))
txt_rst_str = '\n'.join(text_strs)
except:
logger.error('txt_process exception = %s', traceback.format_exc())
txt_rst_str = get_random_err_msg()
finally:
self.msg_dic.update({'ResponseMsg': txt_rst_str})
logger.info('Write txt message to DB %s', \
json.dumps(self.msg_dic, ensure_ascii=False))
self.insert_msg(self.msg_dic)
return txt_rst_str
def load_tel_book(self, tel_book_file):
logger.info('Loading tel book file %s' % tel_book_file)
cnt = 0
for line in open(tel_book_file):
try:
tel_item = line.rstrip('\n').split('\t')
tel_name, tel_no, dept = tel_item[:3]
tel_sys = tel_item[3] if len(tel_item) == 4 else ''
tel_name = tel_name.strip()
tel_dic = {
"tel_name": tel_name,
"tel_no": tel_no,
"dept": dept,
"tel_sys": tel_sys
}
cnt += 1
if cnt % 100 == 0:
logger.info('%s tel loaded' % cnt)
#如已存在，则跳过
#if self.tel_book.find({"$and":[{"tel_name":tel_name}, {"tel_no":tel_no}]}):
# logger.info('tel info already loaded [%s-%s]' % (tel_name, tel_no))
# continue
self.tel_book.insert_one(tel_dic)
except:
logger.error('Error tel format in line %s %s' %
(line, traceback.format_exc()))
logger.info('All [%s] tel loaded' % cnt)
for t in self.tel_book.find({'tel_name': u"刘兵"}):
logger.info(t)
return cnt
def evt_process(self):
evt_rst_str = ''
try:
logger.info('evt_process = %s', self.event)
if self.event == 'subscribe':
evt_rst_str = SUB_WEL_MSG
except:
logger.error('evt_process exception = %s', traceback.format_exc())
evt_rst_str = get_random_err_msg()
finally:
self.msg_dic.update({'ResponseMsg': evt_rst_str})
logger.info('Write evt message to DB %s', self.msg_dic)
self.insert_msg(self.msg_dic)
return evt_rst_str
async def crawl(self, district_code, year):
logger.info("crawling report id for district {0} in year {1}".format(
district_code, year))
page_num = await self.get_page_num(district_code, year)
if page_num == 0:
return
tasks = [
self.get_report_id(district_code, page_id + 1, year)
for page_id in range(page_num)
]
await asyncio.wait(tasks)
logger.info(
"crawl report id for district {0} in year {1} successfully".format(
district_code, year))
def main(config, logger, test=False):
"""
This function loads all the models from memory and sets up the environment
for receiving requests
Note: all files in model_path need to be pickle files and all of them will be
loaded into the model. If the model names are not in the INI file then
they will not be used in prediction.
"""
# Load paths from config
home = os.environ[config["PATHS"]["HOME"]]
# Load paths from config
paths_dict = createPaths(home, config)
ceilinglookup_filename = paths_dict['input_path'] + config["DATA"][
"IWA_CEILING_PROD"]
svc_to_prod_filename = paths_dict['input_path'] + config["DATA"][
"SVC_MATCHING"]
strategic_overlay_filename = paths_dict['input_path'] + config["DATA"][
"STRATEGIC_OVERLAY"]
sic_to_industry_filename = paths_dict['input_path'] + config["DATA"][
"SIC_TO_INDUSTRY"]
eligible_accessorials_filename = paths_dict['input_path'] + config["DATA"][
"ELIGIBLE_ACCESSORIALS"]
accessorial_ceiling_filename = paths_dict['input_path'] + config["DATA"][
"ACCESSORIAL_CEILING"]
datatypes_filename = paths_dict['input_path'] + config["DATA"]["DATA_TYPE"]
cwt_filename = paths_dict['model_path'] + config["MODELS"]["CWT"] + ".p"
# Load models from model_path directory
model_objs = {}
logger.info("Model Path loaded from: " + paths_dict['model_path'])
for model in os.listdir(paths_dict['model_path']):
try:
if (model == "README.MD") or (model[-2:] != ".p") or (
model == "cwt_production.p"):
continue
with open(paths_dict['model_path'] + model, "rb") as model_pickle:
InfoLine = "Loading " + model + " model..."
logger.info(InfoLine)
modelName = model[:-2]
model_objs[modelName] = pickle.load(model_pickle)
except Exception, e:
print_error_message(e,
"Error 3.2a: Model cannot be loaded: " + model,
logger)
def get_tel_info_by_key(self, tel_key):
logger.info('Search mongodb tel_book by %s' % tel_key)
tel_infos = []
#TODO support other key
for it, t in enumerate(self.tel_book.find({"tel_name": tel_key})):
logger.info(t)
tel_info = []
tel_info.append(u'姓名: %s' % t.get('tel_name'))
tel_info.append(u'电话: %s' % t.get('tel_no'))
tel_info.append(u'部门: %s' % t.get('dept'))
#if t.get('tel_sys'):
# tel_info.append(u'负责系统: %s' % t.get('tel_sys'))
if it > 0:
break
tel_infos.append('\n'.join(tel_info))
return '\n'.join(tel_infos) if tel_infos else 'Not found'
def face_detect(self):
s = requests.session()
post_params = {
'api_key': self.api_key,
'api_secret': self.api_secret,
'image_url': self.picurl,
'image_file': self.local_pic_path,
'return_attributes': 'gender,age,smiling,glass'
}
logger.info('face++ face detect picurl=%s, image_file=%s, post_params=%s', \
self.picurl, self.local_pic_path, post_params)
r = s.post(self.face_detect_url, data=post_params, timeout=1)
face_result = r.content
#TODO 可能报CONCURRENCY_LIMIT_EXCEEDED错误
logger.info('face++ face detect = %s', face_result)
return face_result
def loc_process(self):
loc_rst_str = ''
try:
logging.warn('loc_process = %s', self.label)
if not self.label:
loc_rst_str = LOC_LABEL_NIL_MSG
else:
loc_rst_str = self.wea_reporter.\
get_real_time_weather(self.longitude, self.latitude)
except:
logging.error(traceback.format_exc())
loc_rst_str = get_random_err_msg()
finally:
self.msg_dic.update({'ResponseMsg': loc_rst_str})
logger.info('Write loc message to DB %s', self.msg_dic)
self.insert_msg(self.msg_dic)
return loc_rst_str
def get_objects(self):
object_result = self.object_detect()
try:
object_result_json = json.loads(object_result)
objects = []
for obj in object_result_json.get('objects'):
if float(obj.get('confidence')) > self.object_threshold:
obj_val = obj.get('value')
if obj_val == 'Person':
continue
objects.append('%s(%s)' % (obj_val, \
TRANS_OBJ.get_trans(obj_val)))
logger.info('face++ object result=%s', ','.join(objects))
obj_rst_str = u'我看到图片里有%s哦！' % ','.join(objects) \
if len(objects) > 0 else ''
return obj_rst_str
except:
logger.error(traceback.format_exc())
return ''
def post(self):
msg = MessageProcessor()
body = self.request.body
xml = etree.fromstring(body)
msg.load_message(xml)
to_user = msg.get_to_user()
from_user = msg.get_from_user()
msg_type = msg.get_msg_type()
create_time = int(time.time())
out_str = ''
if not to_user or not from_user or not msg_type:
logger.error('to_user/from_user/msg_type lost')
self.write(out_str)
logger.info('%s XML = \n%s\nMSG = %s\n', msg_type.upper(), body,
msg.dump_message())
if msg_type in ['text', 'voice']:
from txtProcessor import TxtProcessor
txt_processor = TxtProcessor(msg)
rst_str = txt_processor.txt_process()
out_str = self.reply_text(from_user, to_user, create_time, rst_str)
elif msg_type == 'image':
from imgProcessor import ImgProcessor
img_processor = ImgProcessor(msg)
rst_str = img_processor.img_process()
out_str = self.reply_text(from_user, to_user, create_time, rst_str)
elif msg_type == 'event':
from evtProcessor import EvtProcessor
evt_processor = EvtProcessor(msg)
rst_str = evt_processor.evt_process()
out_str = self.reply_text(from_user, to_user, create_time, rst_str)
elif msg_type == 'location':
from locProcessor import LocProcessor
loc_processor = LocProcessor(msg)
rst_str = loc_processor.loc_process()
out_str = self.reply_text(from_user, to_user, create_time, rst_str)
else:
out_str = self.reply_text(from_user, to_user, create_time, \
u'这种消息我还不会处理，等我长大哦！')
self.write(out_str)
def img_process(self):
img_rst_str = PIC_NO_REC_MSG
try:
local_pic_path, local_thumb_path = self.save_pic(
self.picurl, self.msg_id)
self.face_plus_plus.local_pic_path = local_pic_path
self.face_plus_plus.local_thumb_path = local_thumb_path
img_rst_str = self.face_plus_plus.parse_result()
logger.info('img_process = %s', img_rst_str)
img_rst_str = img_rst_str.strip()
except:
logger.error('img_process error %s', traceback.format_exc())
img_rst_str = PIC_NO_REC_MSG
finally:
self.msg_dic.update({'ResponseMsg': img_rst_str, \
'LocalPicUrl': self.face_plus_plus.local_pic_path, \
'LocalThumbUrl': self.face_plus_plus.local_thumb_path})
logger.info('Write img message to DB %s', self.msg_dic)
self.insert_msg(self.msg_dic)
return img_rst_str
def get_task_by_name(self, name):
logger.info('Search mongodb by %s' % name)
task_infos = []
for it, t in enumerate(
self.xhx_task.find({
"$or": [{
"task_owner": name
}, {
"task_checker": name
}]
}).sort("time_beg_timestramp")):
logger.info(t)
task_info = []
task_info.append(u'【Task %s】' % (it + 1))
task_info.append(u'任务书编号: %s' % t.get('task_id'))
task_info.append(u'任务书描述: %s' % t.get('task_desc'))
task_info.append(u'负责人: %s' % t.get('task_owner'))
task_info.append(u'监督人: %s' % t.get('task_checker'))
task_info.append(u'计划开始时间: %s' % t.get('task_beg_time'))
task_info.append(u'计划结束时间: %s\n' % t.get('task_end_time'))
task_infos.append('\n'.join(task_info))
return '\n'.join(task_infos) if task_infos else 'No task found'
def friend_reply(msg):
logger.info('Receive friend msg [%s] from [%s][%s] to [%s][%s]' %
(msg.text, msg.sender, msg.sender.puid, msg.receiver,
msg.receiver.puid))
msg.forward(bot.self,
prefix='Receive friend msg [',
suffix='] from %s' % msg.sender.name)
if msg.text.strip() in [u'?', u'？']:
msg.sender.send_msg(
u'【WARNING】\n本功能仅适用于826工程投产演练及上线流程，非法使用带来的生理或心理伤害，本人概不负责:)\n\n【当前功能】\n1. 回复“task 姓名”, 获取名下所有责任人和复核人任务\n2. 回复“add 姓名”，经本人注册后，可接收任务提醒（开始前10分钟，指令群@时，结束前10分钟，暂无法区分高威与单高威）\n3. 回复“tel 姓名”，查询目标姓名的联系方式\n\nPS: 自建乞丐服务器，CPU low，存储low，网络low，随时存在宕机风险，且用且珍惜！欢迎提bug，反正提了也不改\n祝各位826一切顺利！'
+ TARGET_SUFFIX)
elif msg.text.strip().lower().startswith('task'):
task_owners = msg.text.strip().split()[1:]
if len(task_owners) < 1:
logger.error('No task owner found')
for task_owner in task_owners:
msg.sender.send_msg(MONGO_OBJ.get_task_by_name(task_owner))
elif msg.text.strip().lower().startswith('tel'):
tel_keys = msg.text.strip().split()[1:]
if len(tel_keys) < 1:
logger.error('Not found')
for tel_key in tel_keys:
msg.sender.send_msg(MONGO_OBJ.get_tel_info_by_key(tel_key))
def get_faces(self):
try:
face_result = self.face_detect()
face_rst_json = json.loads(face_result)
#TODO get first result by default
sex = face_rst_json.get('faces')[0].get('attributes').get(
'gender').get('value')
age = face_rst_json.get('faces')[0].get('attributes').get(
'age').get('value')
glass = face_rst_json.get('faces')[0].get('attributes').get(
'glass').get('value')
is_smile = face_rst_json.get('faces')[0].get('attributes').get('smile').get('value') \
>= face_rst_json.get('faces')[0].get('attributes').get('smile').get('threshold')
reply_content = u'看起来是个%s, %s岁, %s戴眼镜, %s笑' % \
(u'男生' if sex == 'Male' else u'女生', \
age, \
u'没有' if glass == 'None' else u'', \
u'正在' if is_smile else u'没有')
logger.info('face++ result = %s', reply_content)
return reply_content
except:
logger.error(traceback.format_exc())
return ''
async def start(self):
from spider.listSpider import listSpider
from spider.reportSpider import reportSpider
print("Loading config...")
logger.info("Loading config")
self.load_config()
print("Load config successfully.")
logger.info("Loading config complete")
conn = aiohttp.TCPConnector(limit=self.settings['tcp_limit'])
async with aiohttp.ClientSession(connector=conn) as session:
await session.get(
'http://search.gjsy.gov.cn:9090/queryAll/searchFrame?districtCode=110000&checkYear=2016&sydwName=&selectPage=1'
)
if os.path.exists('.job') and \
input("You have unsave job, do you still want to continue the job last time? y/n:") != 'n':
with open('.job', 'r') as f:
self.report_list = set([(i.split(',')[0], i.split(',')[1])
for i in f.read().splitlines()])
os.remove('.job')
else:
print("Crawling report_id...")
logger.info("Crawling report id")
self.list_spider = listSpider(self.settings,
self.districts_json, session)
self.report_list = await self.list_spider.start()
print("Crawl report_id successfully.")
logger.info("Crawl report id successfully")
print("Crawling report...")
logger.info("Crawling report")
self.report_spider = reportSpider(self.settings, self.report_list,
self.xpath, self.districts_json,
session)
await self.report_spider.start()
def load_schedule(self, schedule_file):
logger.info('Loading schedule file %s' % schedule_file)
cnt = 0
for line in open(schedule_file):
if line and line.strip().startswith('ZK') and len(
line.strip().split('\t')) == 6:
task_id, task_desc, task_owner, task_checker, task_beg_time, task_end_time = line.strip(
).split('\t')[:6]
try:
#task_beg_timestramp = time.mktime(time.strptime(task_beg_time,'%Y-%m-%d %H:%M:%S'))
#task_end_timestramp = time.mktime(time.strptime(task_end_time,'%Y-%m-%d %H:%M:%S'))
task_beg_timestramp = time.mktime(
time.strptime(task_beg_time, '%Y-%m-%d %H:%M'))
task_end_timestramp = time.mktime(
time.strptime(task_end_time, '%Y-%m-%d %H:%M'))
except:
logger.error('Error time format in task %s %s' %
(task_id, traceback.format_exc()))
continue
task_dic = {
"task_id": task_id,
"task_desc": task_desc,
"task_owner": task_owner,
"task_checker": task_checker,
"task_beg_time": task_beg_time,
"task_end_time": task_end_time,
"task_beg_timestramp": task_beg_timestramp,
"task_end_timestramp": task_end_timestramp,
"task_raw_str": line.strip(),
"is_noticed": 0
}
cnt += 1
if cnt % 100 == 0:
logger.info('%s tasks loaded' % cnt)
self.xhx_task.insert_one(task_dic)
logger.info('All [%s] tasks loaded' % cnt)
for t in self.xhx_task.find({'task_owner': u"刘兵"}):
logger.info(t)
return cnt
def group_reply(msg):
#global ITCHAT_MODE
logger.debug(dump_json(u'group dict', msg))
chatroom_id = msg.get('FromUserName')
#chatroom_name = msg.get('User').get('NickName') if msg.get('User') else 'SomeGroup'
chatroom_name = itchat.search_chatrooms(chatroom_id).get(
'NickName') if itchat.search_chatrooms(
chatroom_id) else 'SomeGroup'
from_user = msg.get('ActualNickName', 'Somebody')
content = msg.get('Text')
if msg.get('isAt'):
msg_log_info = u'Receive message [%s] at you from [%s] in group [%s]' % (
content, from_user, chatroom_id)
msg_send_info = u'Receive message [%s] at you from [%s] in group [%s]' % (
content, from_user, chatroom_name)
logger.info(msg_log_info)
itchat.send_msg(msg_send_info)
else:
msg_log_info = u'Receive message [%s] from [%s] in group [%s]' % (
content, from_user, chatroom_id)
msg_send_info = u'Receive message [%s] from [%s] in group [%s]' % (
content, from_user, chatroom_name)
logger.info(msg_log_info)
itchat.send_msg(msg_send_info)
itchat_mode = get_itchat_mode()
if itchat_mode == 'GAI' and is_target_chatroom(
chatroom_id) and is_target_keyword(content):
logger.info('Get target chatroom %s' % chatroom_name)
#for friend in REDIS_OBJ.client.smembers('TARGET_FRIENDS'):
for friend in TARGET_FRIENDS:
#未@到的，直接跳过 TODO 根据displayName精细判断
if friend not in content:
continue
friend_info = itchat.search_friends(name=friend)
if friend_info:
itchat.send_msg(content + TURING_BOT.suffix,
friend_info[0].get('UserName'))
itchat.send_msg('Send notice [%s] to [%s] in group [%s]' %
(content, friend, chatroom_name))
#有@自己的消息，sleep后回复
if msg.get('isAt'):
time.sleep(10)
itchat.send_msg(TARGET_REPLY,
toUserName=msg.get('FromUserName'))
itchat.send_msg('Send auto reply [%s] in group [%s]' %
(TARGET_REPLY, chatroom_name))
def self_reply(msg):
#文本消息
if msg.type == 'Text':
logger.info('Receive self text msg [%s][%s][%s][%s][%s]' %
(msg.text, msg.type, msg.sender, msg.receiver,
msg.sender.puid))
content = msg.text
return_msg = ''
#发送者为本人，进入控制逻辑
content = content.strip().lower()
if content in [u'人工', u'人工模式', u'H', u'h']:
logger.info(u'Change mode to [%s]' % content)
set_wxpy_mode('HUMAN')
logger.info(u'Current mode is [%s]' % get_wxpy_mode())
return_msg = u'Current mode is [%s]' % get_wxpy_mode()
elif content in [u'机器', u'机器模式', u'A', u'a']:
logger.info(u'Change mode to [%s]' % content)
set_wxpy_mode('AI')
logger.info(u'Current mode is [%s]' % get_wxpy_mode())
return_msg = u'Current mode is [%s]' % get_wxpy_mode()
elif content in [u'群机器', u'群机器模式', u'GA', u'ga']:
logger.info(u'Change mode to [%s]' % content)
set_wxpy_mode('GAI')
logger.info(u'Current mode is [%s]' % get_wxpy_mode())
return_msg = u'Current mode is [%s]' % get_wxpy_mode()
elif content in [u'?', u'stat', u'状态']:
return_msg = 'Mode: %s\nMEM: %s' % (get_wxpy_mode(), 'TODO')
elif content.startswith('add') or content.startswith('del'):
#register_name
register_names = content.strip().split()[1:] if len(
content.strip().split()) > 1 else []
if content.startswith('add'):
for rn in register_names:
logger.info('Add register [%s]' % rn)
if rn == u'刘兵':
REDIS_OBJ.client.sadd('TARGET_FRIENDS', rn)
continue
friends = bot.friends().search(rn)
if len(friends) != 1:
logger.info('Add register failed [%s]' %
','.join(f.name for f in friends))
msg.forward(
bot.self,
prefix='Add register [%s] failed' % rn,
suffix='0 or more than 1 friends found')
continue
logger.info(
'Add register [%s][nick_name:%s, puid=%s] success'
% (rn, friends[0].nick_name, friends[0].puid))
REDIS_OBJ.client.sadd('TARGET_FRIENDS', rn)
bot.self.send_msg(
'Add register [%s][nick_name:%s, puid=%s] success'
% (rn, friends[0].nick_name, friends[0].puid))
logger.info(
dump_json(
'All registers',
list(REDIS_OBJ.client.smembers('TARGET_FRIENDS'))))
return_msg = 'All registers : [%s]' % ','.join(
list(REDIS_OBJ.client.smembers('TARGET_FRIENDS')))
elif content == 'del':
#无后续参数，删除全部register
logger.info('Delete all registers')
bot.self.send_msg('Delete all registers [%s]' % ','.join(
list(REDIS_OBJ.client.smembers('TARGET_FRIENDS'))))
REDIS_OBJ.client.delete('TARGET_FRIENDS')
elif content.strip().lower().startswith('task'):
task_owners = content.strip().split()[1:]
if len(task_owners) < 1:
logger.error('No task owner found')
for task_owner in task_owners:
bot.self.send_msg(MONGO_OBJ.get_task_by_name(task_owner))
elif content.strip().lower().startswith('tel'):
tel_keys = content.strip().split()[1:]
if len(tel_keys) < 1:
logger.error('Not found')
for tel_key in tel_keys:
bot.self.send_msg(MONGO_OBJ.get_tel_info_by_key(tel_key))
#return时return_msg将发送到自己微信
return return_msg
#文件消息
elif msg.type == 'Attachment':
logger.info('Receive self file msg [%s][%s][%s][%s][%s]' %
(msg.file_name, msg.type, msg.sender, msg.receiver,
msg.sender.puid))
if msg.file_name == os.path.split(PROJ_SCHEDULE_FILE)[1]:
msg.get_file(save_path=PROJ_SCHEDULE_FILE)
logger.info('Save received file %s to %s' %
(msg.file_name, PROJ_SCHEDULE_FILE))
MONGO_OBJ.xhx_task.remove()
cnt = MONGO_OBJ.load_schedule(PROJ_SCHEDULE_FILE)
bot.self.send_msg('%s tasks loaded' % cnt)
elif msg.file_name == os.path.split(TEL_BOOK_FILE)[1]:
msg.get_file(save_path=TEL_BOOK_FILE)
logger.info('Save received file %s to %s' %
(msg.file_name, TEL_BOOK_FILE))
MONGO_OBJ.tel_book.remove()
cnt = MONGO_OBJ.load_tel_book(TEL_BOOK_FILE)
bot.self.send_msg('%s tel loaded' % cnt)
def on_finish(self):
"""
overwrite teardown
"""
logger.info('===== WXPY MESSAGE END =====\n')
def run(home, c2p_path, p2c_path, model_objs, settings, init_path,
ceilinglookup_filename, svc_to_prod_filename,
strategic_overlay_filename, sic_to_industry_filename,
datatypes_filename, cwt_filename, accessorial_ceiling_filename,
eligible_accessorials_filename, test, logger):
"""
This function runs as continuous loop and receives and processes requests
using the models brought into memory using the setup() function
"""
modified_start = max([os.path.getctime(p2c_path + f) \
for f in os.listdir(p2c_path)])
# clean up IPC directories
for f in os.listdir(p2c_path):
check_files = os.path.join(p2c_path, f)
if f == 'init':
continue
cleanup(check_files)
for f in os.listdir(c2p_path):
check_files = os.path.join(c2p_path, f)
if f == 'init':
continue
cleanup(check_files)
logger.info("IPC folders cleaned")
try:
# read in datatypes and create a dtypes dict
datatypes_table = pd.read_csv(datatypes_filename, index_col='Feature')
data_type_dict = datatypes_table.T.to_dict(orient='record')[0]
ceilinglookup_table = pd.read_csv(ceilinglookup_filename,
dtype={
'Product': 'str',
'Min_List_Rev_Wkly': 'float64',
'Max_List_Rev_Wkly': 'float64',
'Off_Inc_Cap': 'float64'
})
svc_to_prod_table = pd.read_csv(svc_to_prod_filename, dtype=str)
strategic_overlay_table = pd.read_csv(strategic_overlay_filename)
sic_to_industry_table = pd.read_csv(sic_to_industry_filename,
dtype=str)
eligible_accessorials = pd.read_csv(eligible_accessorials_filename)
accessorial_ceiling = pd.read_csv(accessorial_ceiling_filename)
# cwt calibration tables
pd.options.mode.chained_assignment = None
model = OptimalIncentives(settings=settings,
model_objects=model_objs,
ceilinglookup_file=ceilinglookup_table,
svc_to_prod_file=svc_to_prod_table,
industry_name_lookup=sic_to_industry_table,
strategicOverlay=strategic_overlay_table,
accessorial_ceiling=accessorial_ceiling,
eligible_accessorials=eligible_accessorials,
isProduction=True)
model_cwt = cwt_production_class(cwt_filename, svc_to_prod_table,
settings)
except Exception, e:
print_error_message(e, "Error 3.2b: Model created error", logger)
raise
if (model == "README.MD") or (model[-2:] != ".p") or (
model == "cwt_production.p"):
continue
with open(paths_dict['model_path'] + model, "rb") as model_pickle:
InfoLine = "Loading " + model + " model..."
logger.info(InfoLine)
modelName = model[:-2]
model_objs[modelName] = pickle.load(model_pickle)
except Exception, e:
print_error_message(e,
"Error 3.2a: Model cannot be loaded: " + model,
logger)
logger.info("All models loaded")
# Load config variables
settings_dict = {}
for key, value in config.iteritems():
settings_dict.update(value)
settings = settings_from_init_file(settings_dict)
# Start run() which will check for requests
logger.debug('******************Exiting function main_Consumer')
run(home, paths_dict['c2p_path'], paths_dict['p2c_path'], model_objs,
settings, paths_dict['init_path'], ceilinglookup_filename,
svc_to_prod_filename, strategic_overlay_filename,
sic_to_industry_filename, datatypes_filename, cwt_filename,
accessorial_ceiling_filename, eligible_accessorials_filename, test,
logger)
def cur_itchat_mode():
itchat_mode = REDIS_OBJ.client.get('ITCHAT_MODE')
logger.info('Current mode: %s' % itchat_mode)
