def _process_lifetime_campaign_report(campaign, report_id, queued_date):
"""
Processes report for the lifetime of the campaign.
Exponentially backs off on retries, throws on timeout.
"""
attempt = 1
while True:
try:
report_result = report.fetch_report(report_id)
break
except report.ReportPendingException as e:
timeout = (datetime.utcnow().replace(tzinfo=pytz.utc) -
timedelta(seconds=g.az_reporting_timeout))
if queued_date < timeout:
raise report.ReportFailedException(
"campign report timed out (%s/%s)" %
(campaign._fullname, report_id))
else:
sleep_time = math.pow(RETRY_SLEEP_SECONDS, attempt)
attempt = attempt + 1
g.log.warning(
"campaign report still pending, retrying in %d seconds (%s/%s)"
% (RETRY_SLEEP_SECONDS, campaign._fullname, report_id))
time.sleep(sleep_time)
impressions, clicks, spent = _get_total_usage(report_result)
campaign.adserver_spent_pennies = int(spent * 100)
campaign.adserver_impressions = impressions
campaign.adserver_clicks = clicks
campaign.last_lifetime_report = report_id
campaign.last_lifetime_report_run = queued_date
campaign._commit()
def _process_lifetime_campaign_report(campaign, report_id, queued_date):
"""
Processes report for the lifetime of the campaign.
Exponentially backs off on retries, throws on timeout.
"""
attempt = 1
while True:
try:
report_result = report.fetch_report(report_id)
break
except report.ReportPendingException as e:
timeout = (datetime.utcnow().replace(tzinfo=pytz.utc) -
timedelta(seconds=g.az_reporting_timeout))
if queued_date < timeout:
raise report.ReportFailedException("campign report timed out (%s/%s)" %
(campaign._fullname, report_id))
else:
sleep_time = math.pow(RETRY_SLEEP_SECONDS, attempt)
attempt = attempt + 1
g.log.warning("campaign report still pending, retrying in %d seconds (%s/%s)" %
(sleep_time, campaign._fullname, report_id))
time.sleep(sleep_time)
impressions, clicks, spent = _get_total_usage(report_result)
campaign.adserver_spent_pennies = int(spent * 100)
campaign.adserver_impressions = impressions
campaign.adserver_clicks = clicks
campaign.last_lifetime_report = report_id
campaign.last_lifetime_report_run = queued_date
campaign._commit()
def _process_daily_link_report(link, report_id, queued_date):
"""
Processes report grouped by day and flight.
Exponentially backs off on retries, throws on timeout.
"""
attempt = 1
while True:
try:
report_result = report.fetch_report(report_id)
break
except report.ReportPendingException as e:
timeout = (datetime.utcnow().replace(tzinfo=pytz.utc) -
timedelta(seconds=g.az_reporting_timeout))
if queued_date < timeout:
raise report.ReportFailedException("link report timed out (%s/%s)" %
(link._fullname, report_id))
else:
sleep_time = math.pow(RETRY_SLEEP_SECONDS, attempt)
attempt = attempt + 1
g.log.warning("link report still pending, retrying in %d seconds (%s/%s)" %
(sleep_time, link._fullname, report_id))
time.sleep(sleep_time)
g.log.debug(report_result)
campaigns_by_fullname = {campaign._fullname: campaign for campaign
in PromoCampaign._by_link(link._id)}
# report is by date, by flight. each record is a day
# and each detail is a flight for that day.
for record in report_result.get("Records", []):
impressions, clicks, spent = _get_usage(record)
date = _get_date(record)
_insert_daily_link_reporting(
codename=link._fullname,
date=date,
impressions=impressions,
clicks=clicks,
spent_pennies=spent * 100.,
)
campaign_details = defaultdict(_reporting_factory)
for detail in record.get("Details", []):
campaign_fullname = _get_fullname(PromoCampaign, detail)
if not campaign_fullname:
g.log.error("invalid fullname for campaign (%s/%s)" %
(campaign_fullname, flight_id))
continue
campaign = campaigns_by_fullname.get(campaign_fullname)
if not campaign:
flight_id = _get_flight_id(detail)
g.log.warning("no campaign for flight (%s/%s)" %
(campaign_fullname, flight_id))
continue
impressions, clicks, spent = _get_usage(detail)
# if the price changes then there may be multiple records for each campaign/date.
values = campaign_details[(campaign, date)]
values["impressions"] = values["impressions"] + impressions
values["clicks"] = values["clicks"] + clicks
values["spent_pennies"] = values["spent_pennies"] + (spent * 100.)
for (campaign, date), values in campaign_details.iteritems():
# hack around `target_name`s for multi subreddit collections
# being overly long.
if (campaign.target.is_collection and
"/r/" in campaign.target.pretty_name):
subreddit = "multi_%s" % PromoCampaign.SUBREDDIT_TARGET
else:
subreddit = campaign.target_name
_insert_daily_campaign_reporting(
codename=campaign._fullname,
date=date,
subreddit=subreddit,
**values
)
link.last_daily_report = report_id
link.last_daily_report_run = queued_date
link._commit()
def _process_lifetime_campaign_reports(campaigns, report_id, queued_date):
"""
Processes report for the lifetime of the campaigns.
Exponentially backs off on retries, throws on timeout.
"""
attempt = 1
campaign_fullnames = ",".join(c._fullname for c in campaigns)
while True:
try:
report_result = report.fetch_report(report_id)
break
except report.ReportPendingException as e:
timeout = (datetime.utcnow().replace(tzinfo=pytz.utc) -
timedelta(seconds=g.live_config.get("adzerk_reporting_timeout", 500)))
if queued_date < timeout:
raise report.ReportFailedException("campign reports timed out (%s/%s)" %
(campaign_fullnames, report_id))
else:
sleep_time = math.pow(RETRY_SLEEP_SECONDS, attempt)
attempt = attempt + 1
g.log.warning("campaign reports still pending, retrying in %d seconds (%s/%s)" %
(sleep_time, campaign_fullnames, report_id))
time.sleep(sleep_time)
report_records = report_result.get("Records", None)
campaigns_by_fullname = {c._fullname: c for c in campaigns}
campaign_results = defaultdict(_reporting_factory)
if report_records:
for detail in report_records[0].get("Details", []):
campaign_fullname = _get_fullname(PromoCampaign, detail)
if not campaign_fullname:
flight_id = _get_flight_id(detail)
g.log.error("invalid fullname for campaign (%s/%s)" %
(campaign_fullname, flight_id))
continue
campaign = campaigns_by_fullname.get(campaign_fullname)
if not campaign:
flight_id = _get_flight_id(detail)
g.log.warning("no campaign for flight (%s/%s)" %
(campaign_fullname, flight_id))
continue
impressions, clicks, spent = _get_usage(detail)
# if the price changes there may be more than 1 record for a single campaign.
campaign_values = campaign_results[campaign]
campaign_values["impressions"] = campaign_values["impressions"] + impressions
campaign_values["clicks"] = campaign_values["clicks"] + clicks
campaign_values["spent_pennies"] = campaign_values["spent_pennies"] + (spent * 100.)
for campaign, values in campaign_results.items():
campaign.adserver_spent_pennies = values["spent_pennies"]
campaign.adserver_impressions = values["impressions"]
campaign.adserver_clicks = values["clicks"]
campaign.last_lifetime_report = report_id
campaign.last_lifetime_report_run = queued_date
campaign._commit()
def _process_daily_link_report(link, report_id, queued_date):
"""
Processes report grouped by day and flight.
Exponentially backs off on retries, throws on timeout.
"""
attempt = 1
while True:
try:
report_result = report.fetch_report(report_id)
break
except report.ReportPendingException as e:
timeout = (datetime.utcnow().replace(tzinfo=pytz.utc) -
timedelta(seconds=g.az_reporting_timeout))
if queued_date < timeout:
raise report.ReportFailedException(
"link report timed out (%s/%s)" %
(link._fullname, report_id))
else:
sleep_time = math.pow(RETRY_SLEEP_SECONDS, attempt)
attempt = attempt + 1
g.log.warning(
"link report still pending, retrying in %d seconds (%s/%s)"
% (RETRY_SLEEP_SECONDS, link._fullname, report_id))
time.sleep(sleep_time)
g.log.debug(report_result)
campaigns_by_fullname = {
campaign._fullname: campaign
for campaign in PromoCampaign._by_link(link._id)
}
# report is by date, by flight. each record is a day
# and each detail is a flight for that day.
for record in report_result.get("Records", []):
impressions, clicks, spent = _get_usage(record)
date = _get_date(record)
_insert_daily_link_reporting(
codename=link._fullname,
date=date,
impressions=impressions,
clicks=clicks,
spent_pennies=spent * 100.,
)
for detail in record.get("Details", []):
campaign_fullname = _get_fullname(PromoCampaign, detail)
if not campaign_fullname:
g.log.error("invalid fullname for campaign (%s/%s)" %
(campaign_fullname, flight_id))
continue
campaign = campaigns_by_fullname.get(campaign_fullname)
if not campaign:
flight_id = _get_flight_id(detail)
g.log.warning("no campaign for flight (%s/%s)" %
(campaign_fullname, flight_id))
continue
impressions, clicks, spent = _get_usage(detail)
_insert_daily_campaign_reporting(
codename=campaign._fullname,
date=date,
impressions=impressions,
clicks=clicks,
spent_pennies=spent * 100.,
subreddit=campaign.target_name,
)
link.last_daily_report = report_id
link.last_daily_report_run = queued_date
link._commit()
def _process_daily_link_report(link, report_id, queued_date):
"""
Processes report grouped by day and flight.
Exponentially backs off on retries, throws on timeout.
"""
attempt = 1
while True:
try:
report_result = report.fetch_report(report_id)
break
except report.ReportPendingException as e:
timeout = (datetime.utcnow().replace(tzinfo=pytz.utc) -
timedelta(seconds=g.az_reporting_timeout))
if queued_date < timeout:
raise report.ReportFailedException("link report timed out (%s/%s)" %
(link._fullname, report_id))
else:
sleep_time = math.pow(RETRY_SLEEP_SECONDS, attempt)
attempt = attempt + 1
g.log.warning("link report still pending, retrying in %d seconds (%s/%s)" %
(RETRY_SLEEP_SECONDS, link._fullname, report_id))
time.sleep(sleep_time)
g.log.debug(report_result)
campaigns_by_fullname = {campaign._fullname: campaign for campaign
in PromoCampaign._by_link(link._id)}
# report is by date, by flight. each record is a day
# and each detail is a flight for that day.
for record in report_result.get("Records", []):
impressions, clicks, spent = _get_usage(record)
date = _get_date(record)
_insert_daily_link_reporting(
codename=link._fullname,
date=date,
impressions=impressions,
clicks=clicks,
spent_pennies=spent * 100.,
)
for detail in record.get("Details", []):
campaign_fullname = _get_fullname(PromoCampaign, detail)
if not campaign_fullname:
g.log.error("invalid fullname for campaign (%s/%s)" %
(campaign_fullname, flight_id))
continue
campaign = campaigns_by_fullname.get(campaign_fullname)
if not campaign:
flight_id = _get_flight_id(detail)
g.log.warning("no campaign for flight (%s/%s)" %
(campaign_fullname, flight_id))
continue
impressions, clicks, spent = _get_usage(detail)
_insert_daily_campaign_reporting(
codename=campaign._fullname,
date=date,
impressions=impressions,
clicks=clicks,
spent_pennies=spent * 100.,
subreddit=campaign.target_name,
)
link.last_daily_report = report_id
link.last_daily_report_run = queued_date
link._commit()