def stat_by_redis(self, ret):
redis_schema = self.config.CONFIG['GLOBAL']['JOB'][self.job].get(
'REDIS_SCHEMA', 'DEFAULT')
redis_instance = ConnectionFactory.get_redis_connection(
**self.config.CONFIG['GLOBAL']['REDIS'][redis_schema])
redis_conn = redis_instance.connection
if not redis_conn:
LOG.error("cannot connect to redis server")
return False
push_redis_key = self.config.CONFIG['GLOBAL']['JOB'][
self.job]['PUSH_REDIS_KEY']
stat_key = push_redis_key + '_stat_' + datetime.datetime.now(
).strftime("%Y-%m-%d")
is_existed = redis_conn.hgetall(stat_key)
if is_existed:
if ret:
redis_conn.hincrby(stat_key, "success", 1)
else:
redis_conn.hincrby(stat_key, "fail", 1)
redis_conn.hset(stat_key, "last_push", int(time.time()))
else:
stat_map = {
"project": self.config.NAME,
"task": self.job,
"success": 0,
"fail": 0,
"last_push": int(time.time())
}
if ret:
stat_map['success'] = 1
else:
stat_map['fail'] = 1
redis_conn.hmset(stat_key, stat_map)
return True
def __init__(self, job):
super().__init__(job=job)
if not self.config.CONFIG['DATA_MAP'].get(
'trans_company_increment_map', None):
LOG.error(
'for job[{}], trans_company_increment_map is invalid'.format(
self.job))
raise Exception('Initial error')
async def fuzzySuggestCorpName(self, session, keyword):
if not keyword:
return False
url = self.config.CONFIG['GLOBAL']['API'][
'INTSIG_API'] + '/user/CCAppService/enterprise/advanceSearch'
url_param = {'keyword': keyword, 'start': 0}
ret = await self._lcurl.get(session, url, url_param)
if not ret:
return False
try:
if ret['status'] == '1' and ret['data']['total'] > 0:
return ret['data']['items'][0]
else:
return False
except Exception as e:
LOG.error('[fuzzySuggestCorpName ERROR] %s' % e)
return False
async def worker(self, redis_conn):
LOG.info('Start worker for job {}'.format(self.job))
push_redis_key = self.config.CONFIG['GLOBAL']['JOB'][
self.job]['PUSH_REDIS_KEY']
while True:
record = redis_conn.lpop(push_redis_key)
if record is None:
await asyncio.sleep(1)
continue
data = json.loads(record.decode('utf-8'))
try:
await self.process(data)
except Exception as e:
LOG.error('Error during data processing: %s' % e)
finally:
pass
async def worker(self, message_queue):
LOG.info('Start worker for job {}'.format(self.job))
while True:
try:
data = message_queue.get(block=False)
except Empty:
await asyncio.sleep(1)
continue
try:
if data:
future = await self.process(data)
except Exception as e:
LOG.error('Error during data processing: %s' % e)
finally:
pass
def trans(self, input, map, exception_default=''):
output = {}
func_service = FUNCBOX()
try:
for (k, (v, func_name, must_or_not)) in map.items():
kw = {
"value": v,
"new": input,
"instance": self,
"method": func_name
}
try:
col = getattr(func_service, func_name)(**kw)
if not col and int(must_or_not) == 1:
LOG.error('trans {} to column [{}] failed'.format(
v, k))
return False
output[k] = col
except Exception as e:
if int(must_or_not) == 1:
LOG.error('trans {} to column [{}] failed'.format(
v, k))
return False
output[k] = exception_default
return output
except Exception as e:
LOG.error(e)
return False