def get_house_link_names(cls):
now = promote.promo_datetime_now()
pws = PromotionWeights.get_campaigns(now)
campaign_ids = {pw.promo_idx for pw in pws}
q = PromoCampaign._query(PromoCampaign.c._id.in_(campaign_ids),
PromoCampaign.c.priority_name == 'house',
data=True)
return [Link._fullname_from_id36(to36(camp.link_id)) for camp in q]
def get_house_link_names(cls):
now = promote.promo_datetime_now()
campaign_ids = PromotionWeights.get_campaign_ids(now)
q = PromoCampaign._query(PromoCampaign.c._id.in_(campaign_ids),
PromoCampaign.c.priority_name == 'house',
data=True)
link_names = {Link._fullname_from_id36(to36(camp.link_id))
for camp in q}
return sorted(link_names, reverse=True)
def finalize_completed_campaigns(daysago=1):
# PromoCampaign.end_date is utc datetime with year, month, day only
now = datetime.datetime.now(g.tz)
date = now - datetime.timedelta(days=daysago)
date = date.replace(hour=0, minute=0, second=0, microsecond=0)
q = PromoCampaign._query(PromoCampaign.c.end_date == date,
# exclude no transaction
PromoCampaign.c.trans_id != NO_TRANSACTION,
data=True)
# filter out freebies
campaigns = filter(lambda camp: camp.trans_id > NO_TRANSACTION, q)
if not campaigns:
return
# check that traffic is up to date
earliest_campaign = min(campaigns, key=lambda camp: camp.start_date)
start, end = get_total_run(earliest_campaign)
missing_traffic = traffic.get_missing_traffic(start.replace(tzinfo=None),
date.replace(tzinfo=None))
if missing_traffic:
raise ValueError("Can't finalize campaigns finished on %s."
"Missing traffic from %s" % (date, missing_traffic))
links = Link._byID([camp.link_id for camp in campaigns], data=True)
underdelivered_campaigns = []
for camp in campaigns:
if hasattr(camp, 'refund_amount'):
continue
link = links[camp.link_id]
billable_impressions = get_billable_impressions(camp)
billable_amount = get_billable_amount(camp, billable_impressions)
if billable_amount >= camp.total_budget_pennies:
if hasattr(camp, 'cpm'):
text = '%s completed with $%s billable (%s impressions @ $%s).'
text %= (camp, billable_amount, billable_impressions,
camp.bid_dollars)
else:
text = '%s completed with $%s billable (pre-CPM).'
text %= (camp, billable_amount)
PromotionLog.add(link, text)
camp.refund_amount = 0.
camp._commit()
elif charged_or_not_needed(camp):
underdelivered_campaigns.append(camp)
if underdelivered_campaigns:
queries.set_underdelivered_campaigns(underdelivered_campaigns)
def finalize_completed_campaigns(daysago=1):
# PromoCampaign.end_date is utc datetime with year, month, day only
now = datetime.datetime.now(g.tz)
date = now - datetime.timedelta(days=daysago)
date = date.replace(hour=0, minute=0, second=0, microsecond=0)
q = PromoCampaign._query(PromoCampaign.c.end_date == date,
# exclude no transaction
PromoCampaign.c.trans_id != NO_TRANSACTION,
data=True)
# filter out freebies
campaigns = filter(lambda camp: camp.trans_id > NO_TRANSACTION, q)
if not campaigns:
return
# check that traffic is up to date
earliest_campaign = min(campaigns, key=lambda camp: camp.start_date)
start, end = get_total_run(earliest_campaign)
missing_traffic = traffic.get_missing_traffic(start.replace(tzinfo=None),
date.replace(tzinfo=None))
if missing_traffic:
raise ValueError("Can't finalize campaigns finished on %s."
"Missing traffic from %s" % (date, missing_traffic))
links = Link._byID([camp.link_id for camp in campaigns], data=True)
underdelivered_campaigns = []
for camp in campaigns:
if hasattr(camp, 'refund_amount'):
continue
link = links[camp.link_id]
billable_impressions = get_billable_impressions(camp)
billable_amount = get_billable_amount(camp, billable_impressions)
if billable_amount >= camp.bid:
if hasattr(camp, 'cpm'):
text = '%s completed with $%s billable (%s impressions @ $%s).'
text %= (camp, billable_amount, billable_impressions, camp.cpm)
else:
text = '%s completed with $%s billable (pre-CPM).'
text %= (camp, billable_amount)
PromotionLog.add(link, text)
camp.refund_amount = 0.
camp._commit()
elif charged_or_not_needed(camp):
underdelivered_campaigns.append(camp)
if underdelivered_campaigns:
queries.set_underdelivered_campaigns(underdelivered_campaigns)
def finalize_completed_campaigns(daysago=1):
# PromoCampaign.end_date is utc datetime with year, month, day only
now = datetime.now(g.tz)
date = now - timedelta(days=daysago)
date = date.replace(hour=0, minute=0, second=0, microsecond=0)
q = PromoCampaign._query(
PromoCampaign.c.end_date == date,
# exclude no transaction and freebies
PromoCampaign.c.trans_id > 0,
data=True,
)
campaigns = list(q)
if not campaigns:
return
# check that traffic is up to date
earliest_campaign = min(campaigns, key=lambda camp: camp.start_date)
start, end = get_total_run(earliest_campaign)
missing_traffic = traffic.get_missing_traffic(start.replace(tzinfo=None), date.replace(tzinfo=None))
if missing_traffic:
raise ValueError("Can't finalize campaigns finished on %s." "Missing traffic from %s" % (date, missing_traffic))
links = Link._byID([camp.link_id for camp in campaigns], data=True)
underdelivered_campaigns = []
for camp in campaigns:
if hasattr(camp, "refund_amount"):
continue
link = links[camp.link_id]
billable_impressions = get_billable_impressions(camp)
billable_amount = get_billable_amount(camp, billable_impressions)
if billable_amount >= camp.bid:
if hasattr(camp, "cpm"):
text = "%s completed with $%s billable (%s impressions @ $%s)."
text %= (camp, billable_amount, billable_impressions, camp.cpm)
else:
text = "%s completed with $%s billable (pre-CPM)."
text %= (camp, billable_amount)
PromotionLog.add(link, text)
camp.refund_amount = 0.0
camp._commit()
else:
underdelivered_campaigns.append(camp)
if underdelivered_campaigns:
set_underdelivered_campaigns(underdelivered_campaigns)
def get(cls, flight_id):
fullname = g.gencache.get(cls._cache_key(flight_id), stale=True)
if not fullname:
q = PromoCampaign._query(
PromoCampaign.c.external_flight_id == flight_id,
)
q._limit = 1
campaigns = list(q)
if campaigns:
campaign = campaigns[0]
cls.add(campaign)
return campaign._fullname
else:
return None
else:
return fullname
def get(cls, flight_id):
fullname = g.cache.get(cls._cachekey(flight_id))
if not fullname:
q = PromoCampaign._query(
PromoCampaign.c.external_flight_id == flight_id,
data=True,
)
q._limit = 1
campaigns = list(q)
if campaigns:
campaign = campaigns[0]
cls.add(campaign)
return campaign._fullname
else:
return None
else:
return fullname
def accepted_campaigns(offset=0):
now = promo_datetime_now(offset=offset)
promo_weights = PromotionWeights.get_campaigns(now)
all_links = Link._by_fullname(set(x.thing_name for x in promo_weights), data=True, return_dict=True)
accepted_links = {}
for link_fullname, link in all_links.iteritems():
if is_accepted(link):
accepted_links[link._id] = link
accepted_link_ids = accepted_links.keys()
campaign_query = PromoCampaign._query(PromoCampaign.c.link_id == accepted_link_ids, data=True)
campaigns = dict((camp._id, camp) for camp in campaign_query)
for pw in promo_weights:
campaign = campaigns.get(pw.promo_idx)
if not campaign or (not campaign.trans_id and campaign.priority.cpm):
continue
link = accepted_links.get(campaign.link_id)
if not link:
continue
yield (link, campaign, pw.weight)
def accepted_campaigns(offset=0):
now = promo_datetime_now(offset=offset)
promo_weights = PromotionWeights.get_campaigns(now)
all_links = Link._by_fullname(set(x.thing_name for x in promo_weights),
data=True, return_dict=True)
accepted_links = {}
for link_fullname, link in all_links.iteritems():
if is_accepted(link):
accepted_links[link._id] = link
accepted_link_ids = accepted_links.keys()
campaign_query = PromoCampaign._query(PromoCampaign.c.link_id == accepted_link_ids,
data=True)
campaigns = dict((camp._id, camp) for camp in campaign_query)
for pw in promo_weights:
campaign = campaigns.get(pw.promo_idx)
if not campaign or not campaign.trans_id:
continue
link = accepted_links.get(campaign.link_id)
if not link:
continue
yield (link, campaign, pw.weight)
def _process_daily_link_reports(links, report_id, queued_date):
"""
Processes report grouped by day and flight.
Exponentially backs off on retries, throws on timeout.
"""
link_fullnames = ",".join([l._fullname for l in links])
attempt = 1
while True:
try:
report_result = report.fetch_report(report_id)
break
except report.ReportPendingException as e:
timeout = (datetime.utcnow().replace(tzinfo=pytz.utc) -
timedelta(seconds=g.live_config.get("adzerk_reporting_timeout", 500)))
if queued_date < timeout:
raise report.ReportFailedException("link reports timed out (%s/%s)" %
(link_fullnames, report_id))
else:
sleep_time = math.pow(RETRY_SLEEP_SECONDS, attempt)
attempt = attempt + 1
g.log.warning("link reports still pending, retrying in %d seconds (%s/%s)" %
(sleep_time, link_fullnames, report_id))
time.sleep(sleep_time)
g.log.debug(report_result)
link_ids = [l._id for l in links]
campaigns = list(PromoCampaign._query(PromoCampaign.c.link_id.in_(link_ids)))
campaigns_by_fullname = {c._fullname: c for c in campaigns}
links_by_id = { l._id: l for l in links}
# report is by date, by flight. each record is a day (not grouped by campaign)
# and each detail is a flight for that day.
for record in report_result.get("Records", []):
date = _get_date(record)
link_details = defaultdict(_reporting_factory)
campaign_details = defaultdict(_reporting_factory)
for detail in record.get("Details", []):
campaign_fullname = _get_fullname(PromoCampaign, detail)
if not campaign_fullname:
flight_id = _get_flight_id(detail)
g.log.error("invalid fullname for campaign (%s/%s)" %
(campaign_fullname, flight_id))
continue
campaign = campaigns_by_fullname.get(campaign_fullname)
if not campaign:
flight_id = _get_flight_id(detail)
g.log.warning("no campaign for flight (%s/%s)" %
(campaign_fullname, flight_id))
continue
link = links_by_id[campaign.link_id]
impressions, clicks, spent = _get_usage(detail)
# if the price changes then there may be multiple records for each campaign/date.
campaign_values = campaign_details[(campaign, date)]
campaign_values["impressions"] = campaign_values["impressions"] + impressions
campaign_values["clicks"] = campaign_values["clicks"] + clicks
campaign_values["spent_pennies"] = campaign_values["spent_pennies"] + (spent * 100.)
link_values = link_details[(link, date)]
link_values["impressions"] = link_values["impressions"] + impressions
link_values["clicks"] = link_values["clicks"] + clicks
link_values["spent_pennies"] = link_values["spent_pennies"] + (spent * 100.)
for (campaign, date), values in campaign_details.iteritems():
# hack around `target_name`s for multi subreddit collections
# being overly long.
if (campaign.target.is_collection and
"/r/" in campaign.target.pretty_name):
subreddit = "multi_%s" % PromoCampaign.SUBREDDIT_TARGET
else:
subreddit = campaign.target_name
_insert_daily_campaign_reporting(
codename=campaign._fullname,
date=date,
subreddit=subreddit,
**values
)
for (link, date), values in link_details.iteritems():
_insert_daily_link_reporting(
codename=link._fullname,
date=date,
**values
)
link.last_daily_report = report_id
link.last_daily_report_run = queued_date
link._commit()
