def floodlight_rows(
config, task: dict, report_id: int
) -> Generator[list[str, str, str, str, str, str, int], None, None]:
""" Monitor a report for completion and return rows
Args:
report_id - the report created earlier for a specific floodlight id.
Returns:
A stream of rows, see FLOODLIGHT_* constants for definitions.
"""
# fetch report file if it exists
filename, report = report_file(
config,
task['auth'],
task['account'],
report_id,
None, # no name
10 # wait up to 10 minutes for report to complete
)
# clean up rows
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_header_trim(rows)
rows = rows_to_type(rows, column=6)
return rows
def floodlight_rows(report_id):
# fetch report file if it exists
filename, report = report_file(project.task['auth'],
project.task['account'], report_id, None,
10)
# clean up rows
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_header_trim(rows)
rows = rows_to_type(rows, column=6)
return rows
def floodlight_monitor():
if project.verbose: print("FLOODLIGHT MONITOR")
# make sure tab exists in sheet ( deprecated, use sheet task instead )
if 'template' in project.task['sheet']:
sheets_tab_copy(project.task['auth'],
project.task['sheet']['template']['sheet'],
project.task['sheet']['template']['tab'],
project.task['sheet']['sheet'],
project.task['sheet']['tab'])
# read peers from sheet
triggers = sheets_read(project.task['auth'],
project.task['sheet']['sheet'],
project.task['sheet']['tab'],
project.task['sheet']['range'])
# 0 - Floodlight Id
# 1 - email
if project.verbose and len(triggers) == 0:
print("FLOODLIGHT MONITOR: No floodlight ids specified in sheet.")
alerts = {}
day = None
for trigger in triggers:
# get report data for each floodlight
report = floodlight_report(trigger[0])
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_header_trim(rows)
rows = rows_to_type(rows, column=6)
# calculate outliers
last_day, rows = floodlight_analysis(rows)
if last_day:
# find last day report ran
day = last_day if day is None else max(day, last_day)
# group alerts by email
alerts.setdefault(trigger[1], [])
alerts[trigger[1]].extend(rows)
if alerts:
floodlight_email(day, alerts)
def sov_process_client(report_name):
sov_rows = {}
# Download DBM report: ['Advertiser', 'Advertiser ID', 'Advertiser Status', 'Advertiser Integration Code', 'Date', 'Month', 'Device Type', 'Creative Type', 'Region', 'Region ID', 'City', 'City ID', 'Impressions']
# 0 1 2 3 4 5 6 7 8 9 10 11 12
filename, report = report_file(project.task['auth'], None, report_name, 60,
DBM_CHUNKSIZE)
# if a report exists
if report:
if project.verbose: print 'CLIENT FILE', filename
# convert report to array
rows = report_to_rows(report)
rows = report_clean(rows, datastudio=True, nulls=True)
rows = rows_header_trim(rows)
# pull only needed fields ( see: SCHEMA )
for row in rows:
key = ''.join(row[4:-1]) # Everything except impressions
# if peer is in sov, then just add the impressions
if key in sov_rows:
sov_rows[key][7] += long(row[12])
# otherwise, create a new anonymous peer row
else:
sov_rows[key] = [
row[4], # 0 Report_Day
row[5], # 1 Year_Month
'Client', # 2 Advertiser_Type
row[6], # 3 Platform_Type
row[7], # 4 Creative_Type
RE_STATE.sub('', row[8]), # 5 State_Region
row[10], # 6 Designated_Market_Area
long(row[12]), # 7 Client_Impressions
0 # 8 Peer_Impressions
]
else:
if project.verbose: print 'SOV REPORT NOT READY YET'
# return only row values, hash key no longer necessary
return sov_rows.values()
def sov_process_peer(report_name):
sov_rows = {}
sov_mix = {}
mix_ratio_high = 50
warnings = []
errors = []
# Download DBM report: ['Advertiser', 'Advertiser ID', 'Advertiser Status', 'Advertiser Integration Code', 'Date', 'Month', 'Device Type', 'Creative Type', 'Region', 'Region ID', 'City', 'City ID', 'Impressions']
# 0 1 2 3 4 5 6 7 8 9 10 11 12
filename, report = report_file(project.task['auth'], None, report_name, 60,
DBM_CHUNKSIZE)
# if a report exists
if report:
if project.verbose: print 'CLIENT FILE', filename
# convert report to array
rows = report_to_rows(report)
rows = report_clean(rows, datastudio=True, nulls=True)
rows = rows_header_trim(rows)
for row in rows:
key = ''.join(row[4:-1]) # Everything except impressions
# track advertiser level mix
sov_mix[row[1]] = sov_mix.get(row[1], 0) + long(row[12])
# if peer is in sov, then just add the impressions
if key in sov_rows:
sov_rows[key][8] += long(row[12])
# otherwise, create a new anonymous peer row
else:
sov_rows[key] = [
row[4], # 0 Report_Day
row[5], # 1 Year_Month
'Peer', # 2 Advertiser_Type
row[6], # 3 Platform_Type
row[7], # 4 Creative_Type
RE_STATE.sub('', row[8]), # 5 State_Region
row[10], # 6 Designated_Market_Area
0, # 7 Client_Impressions
long(row[12]) # 8 Peer_Impressions
]
# CHECK: Mix must be right, make sure we've got obfuscated data, no peer has more than 50%
mix_total = sum(sov_mix.values())
for account, impressions in sov_mix.items():
percent = (100 * impressions) / mix_total
if project.verbose:
print 'EXPECTED MIX %d%% ACTUAL MIX: %s %d%%' % (
mix_ratio_high, account, percent)
if impressions == 0:
warnings.append('Warning advertiser %s has no impressions.' %
account)
elif percent > mix_ratio_high:
errors.append(
'Advertiser %s has too much weight %d%%, expected under %d%%, add other big peers!'
% (account, percent, mix_ratio_high))
if len(sov_mix.keys()) < 5:
errors.extend(warnings)
errors.append(
'Need at least 5 DBM advertisers with impressions to ensure anonymity!'
)
# raise all errors at once so user can clean up multiple errors at once
if errors: raise Exception('\n'.join(errors))
else:
if project.verbose: print 'SOV REPORT NOT READY YET'
# return only row values, hash key no longer necessary
return sov_rows.values()
