def _msg_generator(self):
for msg in self._consumer:
try:
data = json.loads(msg.value)
yield data
except Exception as e:
logger.error(e)
def _attr_filter(self, data, strategy: Strategy, wxcampaign: WXCampaign):
if not strategy:
return False
# Check worktime
if strategy.worktime and (not pycron.is_now(strategy.worktime)):
return False
# Check according to the attr of data
for record in strategy.records:
key = data[record.key]
val = _cast_to(record.value, record.type)
if record.op == OP_EQ:
if not (key == val):
return False
elif record.op == OP_NEQ:
if not (key != val):
return False
elif record.op == OP_GT:
if not (key > val):
return False
elif record.op == OP_EGT:
if not (key >= val):
return False
elif record.op == OP_LT:
if not (key < val):
return False
elif record.op == OP_ELT:
if not (key <= val):
return False
else:
logger.error('Invalid op of filter[%d]' % record.id)
return False
return True
def _consumer_campaign_info(self):
coll = 'campaigns'
for msg in self._campaign_info_consumer:
try:
'''
@:var data:
{
'agency': 'wxect',
'account': 'MyAccount',
'campaigns: [
{
cid: XX,
},
]
}
'''
logger.info('Receive campaign info from kafka')
data = json.loads(msg.value)
for campaign in data['campaigns']:
campaign.update({
'agency': data['agency'],
'account': data['account']
})
self._mongo[coll].replace_one(
{
'cid': campaign['cid'],
'agency': data['agency'],
'account': data['account']
}, campaign, True)
except Exception as e:
logger.error(e)
def _msg_generator(self):
while True:
while self._data_q.poll():
try:
logger.info('Receive ad data')
data = str(self._data_q.recv_bytes(), encoding='utf-8')
yield json.loads(data)
except Exception as e:
logger.error(e)
time.sleep(5)
def _cast_to(val, tp):
if tp == TYPE_STR:
return str(val)
elif tp == TYPE_INT:
return int(val)
elif tp == TYPE_DOUBLE:
return float(val)
else:
logger.error('Invalid type when cast [%s] to [%s]' % (val, tp))
raise InvalidType
def _consumer_statistic(self):
for msg in self._consumer:
try:
logger.info('Receive ad info from kafka')
data = json.loads(msg.value)
if cache.get('dc.catcher.reporter.%s.%s' %
(data['account'], data['update_time'])):
continue
account_name = data['account']
account = self._db.table('accounts').where(
'name', account_name).first()
if not account:
continue
order_data = fetch_order_info(
data['update_time'][:10] + ' 00:00:00',
data['update_time'])
records = connect_order(data['data'], order_data)
points = []
avai_fields = [
'total_cost', 'view_count', 'sy_cost', 'click_count',
'1day_action_step', '1day_action_reversion',
'1day_action_complete_order',
'1day_action_complete_order_amount'
]
avai_tags = ['account', 'cname']
with self._db.transaction():
for record in records:
fields = {}
tags = {}
record['account'] = account_name
record['account_id'] = account['id']
for key in avai_fields:
if key in record:
fields[key] = float(record[key])
for key in avai_tags:
if key in record:
tags[key] = str(record[key])
points.append({
'measurement':
record['agency'],
'tags':
tags,
'time':
pendulum.from_format(record['update_time'],
'%Y-%m-%d %H:%M:%S'),
'fields':
fields
})
self._db.table('points').insert(record)
self._producer.send(AD_PROCESSED_TOPIC, record)
self._influxdb.write_points(points)
except Exception as e:
logger.error(e)
def trim(self):
camp_model = Campaign(self._mongo)
for data in self._bowler:
try:
# Only active campaign need to trim
if data['status'] != ADSTATUS_NORMAL:
continue
strategies = Strategy.where('account_id',
data['account_id']).get()
# actions = []
campaign = camp_model.find(data['campaign_id'])
if not campaign:
logger.notice('Campaign info not found for [%d]' %
data['campaign_id'])
continue
wxcampaign = WXCampaign(campaign)
actions = self._filter(data, strategies, wxcampaign)
# for strategy in strategies:
# if not self._filter(data, strategy, wxcampaign):
# continue
# if not self._history_filter(data, strategy, wxcampaign):
# continue
# actions.append(self._parse_actions(data, strategy))
commands = []
for act in actions:
action = Action()
action.campaign_id = data['campaign_id']
action.action = act['action']
action.value = act['value']
action.triggered_point = json.dumps(data)
action.account_id = data['account_id']
action.save()
commands.append(action.serialize())
# if len(commands) == 0:
# commands.append(build_command(data, 'timeset_end', random.randint(6, 20)))
if len(commands) > 0:
self._commander.transmit({
'target': 'client',
'client': data['agency'],
'commands': commands
})
logger.info(
'Campaign [%d] should be trimmed, send action successfully'
% data['campaign_id'])
except Exception as e:
logger.error('Exception occurred when handle data [%s]' %
json.dumps(data))
logger.error(e)
def _consumer_command_res(self):
for msg in self._consumer:
try:
logger.info('Receive command results from kafka')
action = json.loads(msg.value)
Action.where('id', action['id']).update({
'resp_cnt':
action['resp_cnt'],
'resp_status_code':
action['resp_status']
})
except Exception as e:
logger.error(e)
def bid_type(self):
try:
ad_group = self._campaign['target_groups'][0]['ad_groups'][0][
'ad_group']
if not ad_group:
return None, None
if 'strategy_opt' not in ad_group:
return AD_BID_TYPE.CPM, None
opt = json.loads(ad_group['strategy_opt'])
if opt['bid_action_type'] == AD_BID_TYPE_OCPM_OPT_MORE_ORDER:
return AD_BID_TYPE.OCPM, AD_BID_TYPE_OCPM_OPT_MORE_ORDER
elif opt['bid_action_type'] == AD_BID_TYPE_OCPM_OPT_MORE_CLICK:
return AD_BID_TYPE.OCPM, AD_BID_TYPE_OCPM_OPT_MORE_CLICK
except Exception as e:
logger.error(e)
return None, None
def is_beishang(self):
'''
定向城市是否为北京上海
:return:
'''
try:
beijing = 110000
shanghai = 310000
areas = json.loads(self._campaign['target_groups'][0]['ad_groups']
[0]['ad_target']['area'])
if beijing in areas and shanghai in areas:
return True
return False
except Exception as e:
logger.error(e)
return False
def _parse_actions(self, data, strategy: Strategy):
'''
目前支持修改出价／修改生效时间
:param data:
:param strategy:
:return:
'''
trims = strategy.trims
actions = []
for trim in trims:
if trim.action == 'modify_timeset':
if trim.key == 'end':
actions.append({
'action': 'timeset_end',
'value': trim.value,
})
elif trim.action == 'suspend_camp':
actions.append({'action': 'suspend', 'value': None})
else:
logger.error('Unknow trim [%s]' % json.dumps(trim))
return actions