def report_perform_res(self, id, resp_cnt, resp_status):
self._producer.send('agency.command.reporter', {
'id': id,
'resp_cnt': resp_cnt,
'resp_status': resp_status,
})
logger.info('Reporter results of performing commands successfully')
def wrapper(*args, **kwargs):
for msg in consumer:
try:
logger.info('Receive data from kafka for cunsumer [%s]' % client)
kwargs['message'] = msg
func(*args, **kwargs)
except Exception as e:
logger.error('Err when handle data')
logger.error(e)
pass
def report_cmd_res(self, id, resp_cnt, resp_status):
'''
报告命令执行结果
:param id:
:param resp_cnt:
:param resp_status:
:return:
'''
self._producer.send(AGENCY_COMMAND_REPORTER_TOPIC, {
'id': id,
'resp_cnt': resp_cnt,
'resp_status': resp_status,
})
logger.info('Reporter results of performing commands successfully')
def __init__(self):
kafka_server = '%s:%d' % (config.get('app.kafka.host'),
config.get('app.kafka.port'))
logger.info('Start to connect kafka [%s]' % kafka_server)
self._client_command_consumer = KafkaConsumer(
AGENCY_COMMAND_TOPIC,
client_id='agency_manager',
bootstrap_servers=kafka_server)
# run initial clients automatically
logger.info('Start to run default client...')
self._run_default_client()
logger.info('Start statistic...')
self.start_statistic()
self.handle_command()
def statistic(self):
while True:
while self._data_q.poll():
try:
data = str(self._data_q.recv_bytes(), encoding='utf-8')
resp = json.loads(data)
if resp['type'] == TYPE_CAMP_INFO:
'''
Report campaign info
'''
logger.info('Receive campaigns info')
self.report_camp_info(
resp['data']['account'],
json.loads(resp['data']['campaigns']))
logger.info('Send campaign data to kafka successfully')
elif resp['type'] == TYPE_ACTION_RES:
'''
Report action result
{
id: 1,
resp_cnt: 'success',
resp_status: 200
}
'''
logger.info('Receive action perform results')
data = resp['data']
self.report_cmd_res(data['id'], data['resp_cnt'],
data['resp_status'])
logger.info(
'Send action results to kafka successfully')
elif resp['type'] == TYPE_STATISTIC:
''''
Report statistic
{
data: [
{},
{}
],
account: 'myaccount',
'update_hour: '201804151005'
}
'''
resp = resp['data']
logger.info('Receive statistic info')
processed_data = []
update_at = pendulum.from_format(
resp['update_hour'],
'%Y%m%d%H%M').to_datetime_string()
for record in resp['data']:
record['update_time'] = update_at
record['account'] = resp['account']
processed_data.append(self.transformer(record))
self.report_statistic(resp['account'], {
'data': processed_data,
'update_time': update_at
})
logger.info('Send ad data to kafka successfully')
except Exception as e:
logger.error('Exception raised when send data to kafka')
logger.error(e)
time.sleep(5)
def _handle_command(self, commands, meta):
'''
:param commands:
{
"target": "client" // manager
"client": "wxext",
"commands": [
// details, parsed by the client
]
}
:param meta:
:return:
'''
try:
logger.info('Receive command: %r' % commands)
if commands['target'] == 'client':
if 'client' in commands and commands['client'] in self._clients:
self._clients[commands['client']].perform(
commands['commands'])
elif commands['target'] == 'manager':
for command in commands['commands']:
if command['type'] == 'operate_client':
if command['action'] == 'start':
if command['target'] in self._clients:
logger.notice('%s is already running...' %
command['target'])
elif command['target'] in _available_clients:
self._clients[
command['target']] = _available_clients[
command['target']]()
else:
raise UnknowCommandException()
elif command['action'] == 'stop':
if command['target'] in self._clients:
rtn = self._clients[command['target']].quit()
logger.info('Stop [%s] with status [%s]' %
(command['target'], rtn))
else:
logger.notice('[%s] is not running!' %
command['target'])
elif command['action'] == 'restart':
if command['target'] in self._clients:
rtn = self._clients[command['target']].quit()
logger.info('Stop [%s] with status [%s]' %
(command['target'], rtn))
self._clients[
command['target']] = _available_clients[
command['target']]()
else:
raise UnknowCommandException(command)
else:
raise UnknowCommandException(command)
pass
pass
except UnknowCommandException as e:
logger.error('Unknow command:')
logger.error(e)
except Exception as e:
logger.error(e)