async def process(self, data):
LOG.info('{} process {}'.format(self.job, data))
async with aiohttp.ClientSession(loop=self._loop) as session:
# 推送到启信宝易文斌
if data.get('last_update') and int(
data['last_update']) > 1477929600:
await self.update_biddings(session, data)
def stat_by_redis(self, ret):
redis_schema = self.config.CONFIG['GLOBAL']['JOB'][self.job].get(
'REDIS_SCHEMA', 'DEFAULT')
redis_instance = ConnectionFactory.get_redis_connection(
**self.config.CONFIG['GLOBAL']['REDIS'][redis_schema])
redis_conn = redis_instance.connection
if not redis_conn:
LOG.error("cannot connect to redis server")
return False
push_redis_key = self.config.CONFIG['GLOBAL']['JOB'][
self.job]['PUSH_REDIS_KEY']
stat_key = push_redis_key + '_stat_' + datetime.datetime.now(
).strftime("%Y-%m-%d")
is_existed = redis_conn.hgetall(stat_key)
if is_existed:
if ret:
redis_conn.hincrby(stat_key, "success", 1)
else:
redis_conn.hincrby(stat_key, "fail", 1)
redis_conn.hset(stat_key, "last_push", int(time.time()))
else:
stat_map = {
"project": self.config.NAME,
"task": self.job,
"success": 0,
"fail": 0,
"last_push": int(time.time())
}
if ret:
stat_map['success'] = 1
else:
stat_map['fail'] = 1
redis_conn.hmset(stat_key, stat_map)
return True
async def add_ccinfo_msg_target(self, session, document):
url = self.config.CONFIG['GLOBAL']['API']['YUNYING_PUSH_API'] + '/push/add_ccinfo_msg_target'
ret = await self._lcurl.post(session, url, data=json.dumps(document))
LOG.info('add ccinfo msg target by {}, result: {}'.format(document, ret))
if ret and str(ret['errno']) == '0':
return True
else:
return False
def __init__(self, job):
super().__init__(job=job)
if not self.config.CONFIG['DATA_MAP'].get(
'trans_company_increment_map', None):
LOG.error(
'for job[{}], trans_company_increment_map is invalid'.format(
self.job))
raise Exception('Initial error')
async def synchronizeWeb(self, session, data):
post = [{
'info_id': str(data['_id']),
'issue_time': data['last_update']
}]
ret = await self._lcurl.post(
session, self.config.CONFIG['GLOBAL']['API']['WEB_API'] +
"/seo/multiaddbidid", json.dumps(post))
LOG.info('web seo result: {}'.format(ret))
async def process(self, data):
LOG.info('{} process {}'.format(self.job, data))
async with aiohttp.ClientSession(loop=self._loop) as session:
# 推送web seo到林雨森
await self.synchronizeWeb(session, data)
# 推送到启信宝易文斌
await self.update_biddings(session, data)
# 组装数据推送ccpush信息流
target = await self.load_msg_target(session, data)
await self.add_ccinfo_msg_target(session, target)
async def getSummaryByName(self, session, name):
if not name:
return False
url = self.config.CONFIG['GLOBAL']['API'][
'CORP_QUERY_API'] + '/CCAppService/enterprise/getSummaryByName'
url_param = {'name': name}
ret = await self._lcurl.get(session, url, url_param)
LOG.info('get corp summary by name[{}]...result: {}'.format(name, ret))
if not ret:
return False
if str(ret['status']) == '1':
return ret['data']
else:
return False
async def upload_news(self, session, document):
if not document:
return False
url = self.config.CONFIG['GLOBAL']['API']['BUSINESS_TOPNEWS_API']
data = {"records": [{"value": document}]}
ret = await self._lcurl.post(
session=session,
url=url,
data=json.dumps(data),
headers={"Content-Type": "application/vnd.kafka.json.v1+json"},
do_log=False)
LOG.info('upload news by {}, result: {}'.format(document, ret))
if not ret:
return False
else:
return True
async def update_biddings(self, session, document):
if not document:
return False
channel = 'HR72FESQE5TZPWDLIAY8D8EX71Z3WGNW'
timestamp = int(time.time())
body_str = json.dumps(document)
sign = Tool.md5('{}{}{}'.format(body_str, channel, timestamp))
url = '{}?timestamp={}&sign={}'.format(self.config.CONFIG['GLOBAL']['API']['RM_UPDATE_BIDDINGS_API'], timestamp, sign)
ret = await self._lcurl.post(session=session, url=url, data=body_str, headers={"Content-Type":"application/json"})
LOG.info('update total bidding by {}, result: {}'.format(document, ret))
if not ret:
return False
if ret['status'] == 200:
return True
else:
return False
async def worker(self, message_queue):
LOG.info('Start worker for job {}'.format(self.job))
while True:
try:
data = message_queue.get(block=False)
except Empty:
await asyncio.sleep(1)
continue
try:
if data:
future = await self.process(data)
except Exception as e:
LOG.error('Error during data processing: %s' % e)
finally:
pass
def process_message(self, message_queue):
LOG.info('Start message consumer for job {}'.format(self.job))
task_config = self.config.CONFIG['GLOBAL']['JOB'][self.job]
processor_num = int(task_config.get('PROCESSOR_NUM', 1))
# 事件循环
self._loop = asyncio.new_event_loop()
try:
for i in range(processor_num):
asyncio.ensure_future(coro_or_future=self.worker(message_queue), loop=self._loop)
self._loop.run_forever()
# self._loop.run_until_complete(asyncio.gather(self.worker(redis_conn)))
except Exception as e:
print(asyncio.gather(*asyncio.Task.all_tasks()).cancel(), loop=self._loop)
# self._loop.run_until_complete(self._loop.shutdown_asyncgens())
finally:
self._loop.close()
async def fuzzySuggestCorpName(self, session, keyword):
if not keyword:
return False
url = self.config.CONFIG['GLOBAL']['API'][
'INTSIG_API'] + '/user/CCAppService/enterprise/advanceSearch'
url_param = {'keyword': keyword, 'start': 0}
ret = await self._lcurl.get(session, url, url_param)
if not ret:
return False
try:
if ret['status'] == '1' and ret['data']['total'] > 0:
return ret['data']['items'][0]
else:
return False
except Exception as e:
LOG.error('[fuzzySuggestCorpName ERROR] %s' % e)
return False
async def worker(self, redis_conn):
LOG.info('Start worker for job {}'.format(self.job))
push_redis_key = self.config.CONFIG['GLOBAL']['JOB'][
self.job]['PUSH_REDIS_KEY']
while True:
record = redis_conn.lpop(push_redis_key)
if record is None:
await asyncio.sleep(1)
continue
data = json.loads(record.decode('utf-8'))
try:
await self.process(data)
except Exception as e:
LOG.error('Error during data processing: %s' % e)
finally:
pass
async def upload_product_increment(self, session, document):
if not document:
return False
url = self.config.CONFIG['GLOBAL']['API']['COMPANY_INCREMENT_API']
data = {"topic": "ProductsIntsigInc", "document": document}
ret = await self._lcurl.post(
session=session,
url=url,
data=json.dumps(data),
headers={"Content-Type": "application/json"})
LOG.info('upload product increment by {}, result: {}'.format(
data, ret))
if not ret:
return False
if ret['code'] == 0:
return True
else:
return False
async def process(self, data):
LOG.info('company2data [{}] process {}'.format(self.job, data))
config = self.config
if config.ENV == 'DEV':
return True
async with aiohttp.ClientSession(loop=self._loop) as session:
if 'corp_category' in data and int(data['corp_category']) != 1:
LOG.warning(
'for job[{}], corp_category is invalid in {}'.format(
self.job, data))
return False
await self.pre_trans(session, data)
# 上传公司(必要）
company_increment_output = self.trans(
data, config.CONFIG['DATA_MAP']['trans_company_increment_map'],
None)
await self.upload_company_increment(session,
company_increment_output)
# 上传产品（非必要）
if not config.CONFIG['DATA_MAP'].get('trans_product_increment_map',
None):
LOG.warning(
'for job[{}], trans_product_increment_map is invalid'.
format(self.job))
return
product_increment_output = self.trans(
data, config.CONFIG['DATA_MAP']['trans_product_increment_map'],
None)
await self.upload_product_increment(session,
product_increment_output)
def trans(self, input, map, exception_default=''):
output = {}
func_service = FUNCBOX()
try:
for (k, (v, func_name, must_or_not)) in map.items():
kw = {
"value": v,
"new": input,
"instance": self,
"method": func_name
}
try:
col = getattr(func_service, func_name)(**kw)
if not col and int(must_or_not) == 1:
LOG.error('trans {} to column [{}] failed'.format(
v, k))
return False
output[k] = col
except Exception as e:
if int(must_or_not) == 1:
LOG.error('trans {} to column [{}] failed'.format(
v, k))
return False
output[k] = exception_default
return output
except Exception as e:
LOG.error(e)
return False
async def process(self, data):
LOG.info('Kr2data process {}'.format(data))
config = self.config
async with aiohttp.ClientSession(loop=self._loop) as session:
self.pre_trans(session, data)
# 上传公司
company_increment_output = self.trans(
data, config.CONFIG['DATA_MAP']['trans_company_increment_map'],
None)
await self.upload_company_increment(session,
company_increment_output)
# 上传产品
if not config.CONFIG['DATA_MAP'].get('trans_product_increment_map',
None):
LOG.warning(
'for job[{}], trans_product_increment_map is invalid'.
format(self.job))
return
product_increment_output = self.trans(
data, config.CONFIG['DATA_MAP']['trans_product_increment_map'],
None)
await self.upload_product_increment(session,
product_increment_output)
def get_message(self, message_queue):
LOG.info('Start message producer for job {}'.format(self.job))
task_config = self.config.CONFIG['GLOBAL']['JOB'][self.job]
redis_schema = task_config.get('REDIS_SCHEMA', 'DEFAULT')
redis_instance = ConnectionFactory.get_redis_connection(**self.config.CONFIG['GLOBAL']['REDIS'][redis_schema])
with redis_instance as redis_conn:
while True:
if message_queue.qsize() < 10:
record = redis_conn.blpop(task_config['PUSH_REDIS_KEY'])
if record:
LOG.info('put message into queue: {}'.format(record))
data = json.loads(record[1].decode('utf-8'))
message_queue.put(data)
else:
LOG.info('too busy, have a rest...')
time.sleep(1)
