def _consumer_campaign_info(self):
coll = 'campaigns'
for msg in self._campaign_info_consumer:
try:
'''
@:var data:
{
'agency': 'wxect',
'account': 'MyAccount',
'campaigns: [
{
cid: XX,
},
]
}
'''
logger.info('Receive campaign info from kafka')
data = json.loads(msg.value)
for campaign in data['campaigns']:
campaign.update({
'agency': data['agency'],
'account': data['account']
})
self._mongo[coll].replace_one(
{
'cid': campaign['cid'],
'agency': data['agency'],
'account': data['account']
}, campaign, True)
except Exception as e:
logger.error(e)
def __init__(self):
Bowler.__init__(self)
kafka_server = '%s:%d' % (config.get('app.kafka.host'), config.get('app.kafka.port'))
self._consumer = KafkaConsumer(AD_PROCESSED_TOPIC,
client_id='ad_statistic_catcher_reporter',
group_id='ad_statistic_catcher',
bootstrap_servers=kafka_server)
logger.info('Connect to kafka[%s] successfully' % kafka_server)
def _msg_generator(self):
while True:
while self._data_q.poll():
try:
logger.info('Receive ad data')
data = str(self._data_q.recv_bytes(), encoding='utf-8')
yield json.loads(data)
except Exception as e:
logger.error(e)
time.sleep(5)
def _consumer_statistic(self):
for msg in self._consumer:
try:
logger.info('Receive ad info from kafka')
data = json.loads(msg.value)
if cache.get('dc.catcher.reporter.%s.%s' %
(data['account'], data['update_time'])):
continue
account_name = data['account']
account = self._db.table('accounts').where(
'name', account_name).first()
if not account:
continue
order_data = fetch_order_info(
data['update_time'][:10] + ' 00:00:00',
data['update_time'])
records = connect_order(data['data'], order_data)
points = []
avai_fields = [
'total_cost', 'view_count', 'sy_cost', 'click_count',
'1day_action_step', '1day_action_reversion',
'1day_action_complete_order',
'1day_action_complete_order_amount'
]
avai_tags = ['account', 'cname']
with self._db.transaction():
for record in records:
fields = {}
tags = {}
record['account'] = account_name
record['account_id'] = account['id']
for key in avai_fields:
if key in record:
fields[key] = float(record[key])
for key in avai_tags:
if key in record:
tags[key] = str(record[key])
points.append({
'measurement':
record['agency'],
'tags':
tags,
'time':
pendulum.from_format(record['update_time'],
'%Y-%m-%d %H:%M:%S'),
'fields':
fields
})
self._db.table('points').insert(record)
self._producer.send(AD_PROCESSED_TOPIC, record)
self._influxdb.write_points(points)
except Exception as e:
logger.error(e)
def __init__(self):
Catcher.__init__(self)
kafka_server = '%s:%d' % (config.get('app.kafka.host'),
config.get('app.kafka.port'))
logger.info('Try to connect to kafka...')
self._consumer = KafkaConsumer(AGENCY_COMMAND_REPORTER_TOPIC,
client_id='commander_result_reporter',
group_id='commander_result_reporter',
bootstrap_servers=kafka_server)
logger.info('Connect to kafka[%s] successfully' %
AGENCY_COMMAND_REPORTER_TOPIC)
self._consumer_command_res()
def trim(self):
camp_model = Campaign(self._mongo)
for data in self._bowler:
try:
# Only active campaign need to trim
if data['status'] != ADSTATUS_NORMAL:
continue
strategies = Strategy.where('account_id',
data['account_id']).get()
# actions = []
campaign = camp_model.find(data['campaign_id'])
if not campaign:
logger.notice('Campaign info not found for [%d]' %
data['campaign_id'])
continue
wxcampaign = WXCampaign(campaign)
actions = self._filter(data, strategies, wxcampaign)
# for strategy in strategies:
# if not self._filter(data, strategy, wxcampaign):
# continue
# if not self._history_filter(data, strategy, wxcampaign):
# continue
# actions.append(self._parse_actions(data, strategy))
commands = []
for act in actions:
action = Action()
action.campaign_id = data['campaign_id']
action.action = act['action']
action.value = act['value']
action.triggered_point = json.dumps(data)
action.account_id = data['account_id']
action.save()
commands.append(action.serialize())
# if len(commands) == 0:
# commands.append(build_command(data, 'timeset_end', random.randint(6, 20)))
if len(commands) > 0:
self._commander.transmit({
'target': 'client',
'client': data['agency'],
'commands': commands
})
logger.info(
'Campaign [%d] should be trimmed, send action successfully'
% data['campaign_id'])
except Exception as e:
logger.error('Exception occurred when handle data [%s]' %
json.dumps(data))
logger.error(e)
def _consumer_command_res(self):
for msg in self._consumer:
try:
logger.info('Receive command results from kafka')
action = json.loads(msg.value)
Action.where('id', action['id']).update({
'resp_cnt':
action['resp_cnt'],
'resp_status_code':
action['resp_status']
})
except Exception as e:
logger.error(e)
def __init__(self):
logger.info('Init Reporter...')
Catcher.__init__(self)
self._mongo = db.get_mongo_client(config.get('app.db.mongo'))
kafka_server = '%s:%d' % (config.get('app.kafka.host'),
config.get('app.kafka.port'))
logger.info('Try to connect to kafka...')
self._consumer = KafkaConsumer(
AD_ORIGIN_STATISTIC_TOPIC,
client_id='ad_statistic_catcher_reporter',
group_id='ad_statistic_catcher',
bootstrap_servers=kafka_server)
logger.info('Connect to kafka[%s] successfully' %
AD_ORIGIN_STATISTIC_TOPIC)
self._producer = KafkaProducer(
value_serializer=lambda v: json.dumps(v).encode('utf-8'),
client_id='ad.statistic.worker',
compression_type='gzip',
bootstrap_servers=kafka_server,
retries=3)
self._campaign_info_consumer = KafkaConsumer(
AD_CAMPAIGN_INFO_TOPIC,
client_id='ad_campaign_catcher_reporter',
group_id='ad_campaign_catcher',
bootstrap_servers=kafka_server)
logger.info('Connect to kafka[%s] successfully' %
AD_CAMPAIGN_INFO_TOPIC)
t1 = threading.Thread(target=Reporter._consumer_statistic,
args=(self, ))
t2 = threading.Thread(target=Reporter._consumer_campaign_info,
args=(self, ))
t1.start()
t2.start()
t1.join()
t2.join()