def stat_by_redis(self, ret): redis_schema = self.config.CONFIG['GLOBAL']['JOB'][self.job].get( 'REDIS_SCHEMA', 'DEFAULT') redis_instance = ConnectionFactory.get_redis_connection( **self.config.CONFIG['GLOBAL']['REDIS'][redis_schema]) redis_conn = redis_instance.connection if not redis_conn: LOG.error("cannot connect to redis server") return False push_redis_key = self.config.CONFIG['GLOBAL']['JOB'][ self.job]['PUSH_REDIS_KEY'] stat_key = push_redis_key + '_stat_' + datetime.datetime.now( ).strftime("%Y-%m-%d") is_existed = redis_conn.hgetall(stat_key) if is_existed: if ret: redis_conn.hincrby(stat_key, "success", 1) else: redis_conn.hincrby(stat_key, "fail", 1) redis_conn.hset(stat_key, "last_push", int(time.time())) else: stat_map = { "project": self.config.NAME, "task": self.job, "success": 0, "fail": 0, "last_push": int(time.time()) } if ret: stat_map['success'] = 1 else: stat_map['fail'] = 1 redis_conn.hmset(stat_key, stat_map) return True
def __init__(self, job): super().__init__(job=job) if not self.config.CONFIG['DATA_MAP'].get( 'trans_company_increment_map', None): LOG.error( 'for job[{}], trans_company_increment_map is invalid'.format( self.job)) raise Exception('Initial error')
async def fuzzySuggestCorpName(self, session, keyword): if not keyword: return False url = self.config.CONFIG['GLOBAL']['API'][ 'INTSIG_API'] + '/user/CCAppService/enterprise/advanceSearch' url_param = {'keyword': keyword, 'start': 0} ret = await self._lcurl.get(session, url, url_param) if not ret: return False try: if ret['status'] == '1' and ret['data']['total'] > 0: return ret['data']['items'][0] else: return False except Exception as e: LOG.error('[fuzzySuggestCorpName ERROR] %s' % e) return False
async def worker(self, redis_conn): LOG.info('Start worker for job {}'.format(self.job)) push_redis_key = self.config.CONFIG['GLOBAL']['JOB'][ self.job]['PUSH_REDIS_KEY'] while True: record = redis_conn.lpop(push_redis_key) if record is None: await asyncio.sleep(1) continue data = json.loads(record.decode('utf-8')) try: await self.process(data) except Exception as e: LOG.error('Error during data processing: %s' % e) finally: pass
async def worker(self, message_queue): LOG.info('Start worker for job {}'.format(self.job)) while True: try: data = message_queue.get(block=False) except Empty: await asyncio.sleep(1) continue try: if data: future = await self.process(data) except Exception as e: LOG.error('Error during data processing: %s' % e) finally: pass
def trans(self, input, map, exception_default=''): output = {} func_service = FUNCBOX() try: for (k, (v, func_name, must_or_not)) in map.items(): kw = { "value": v, "new": input, "instance": self, "method": func_name } try: col = getattr(func_service, func_name)(**kw) if not col and int(must_or_not) == 1: LOG.error('trans {} to column [{}] failed'.format( v, k)) return False output[k] = col except Exception as e: if int(must_or_not) == 1: LOG.error('trans {} to column [{}] failed'.format( v, k)) return False output[k] = exception_default return output except Exception as e: LOG.error(e) return False