def get_house_link_names(cls): now = promote.promo_datetime_now() pws = PromotionWeights.get_campaigns(now) campaign_ids = {pw.promo_idx for pw in pws} q = PromoCampaign._query(PromoCampaign.c._id.in_(campaign_ids), PromoCampaign.c.priority_name == 'house', data=True) return [Link._fullname_from_id36(to36(camp.link_id)) for camp in q]
def get_house_link_names(cls): now = promote.promo_datetime_now() campaign_ids = PromotionWeights.get_campaign_ids(now) q = PromoCampaign._query(PromoCampaign.c._id.in_(campaign_ids), PromoCampaign.c.priority_name == 'house', data=True) link_names = {Link._fullname_from_id36(to36(camp.link_id)) for camp in q} return sorted(link_names, reverse=True)
def finalize_completed_campaigns(daysago=1): # PromoCampaign.end_date is utc datetime with year, month, day only now = datetime.datetime.now(g.tz) date = now - datetime.timedelta(days=daysago) date = date.replace(hour=0, minute=0, second=0, microsecond=0) q = PromoCampaign._query(PromoCampaign.c.end_date == date, # exclude no transaction PromoCampaign.c.trans_id != NO_TRANSACTION, data=True) # filter out freebies campaigns = filter(lambda camp: camp.trans_id > NO_TRANSACTION, q) if not campaigns: return # check that traffic is up to date earliest_campaign = min(campaigns, key=lambda camp: camp.start_date) start, end = get_total_run(earliest_campaign) missing_traffic = traffic.get_missing_traffic(start.replace(tzinfo=None), date.replace(tzinfo=None)) if missing_traffic: raise ValueError("Can't finalize campaigns finished on %s." "Missing traffic from %s" % (date, missing_traffic)) links = Link._byID([camp.link_id for camp in campaigns], data=True) underdelivered_campaigns = [] for camp in campaigns: if hasattr(camp, 'refund_amount'): continue link = links[camp.link_id] billable_impressions = get_billable_impressions(camp) billable_amount = get_billable_amount(camp, billable_impressions) if billable_amount >= camp.total_budget_pennies: if hasattr(camp, 'cpm'): text = '%s completed with $%s billable (%s impressions @ $%s).' text %= (camp, billable_amount, billable_impressions, camp.bid_dollars) else: text = '%s completed with $%s billable (pre-CPM).' text %= (camp, billable_amount) PromotionLog.add(link, text) camp.refund_amount = 0. camp._commit() elif charged_or_not_needed(camp): underdelivered_campaigns.append(camp) if underdelivered_campaigns: queries.set_underdelivered_campaigns(underdelivered_campaigns)
def finalize_completed_campaigns(daysago=1): # PromoCampaign.end_date is utc datetime with year, month, day only now = datetime.datetime.now(g.tz) date = now - datetime.timedelta(days=daysago) date = date.replace(hour=0, minute=0, second=0, microsecond=0) q = PromoCampaign._query(PromoCampaign.c.end_date == date, # exclude no transaction PromoCampaign.c.trans_id != NO_TRANSACTION, data=True) # filter out freebies campaigns = filter(lambda camp: camp.trans_id > NO_TRANSACTION, q) if not campaigns: return # check that traffic is up to date earliest_campaign = min(campaigns, key=lambda camp: camp.start_date) start, end = get_total_run(earliest_campaign) missing_traffic = traffic.get_missing_traffic(start.replace(tzinfo=None), date.replace(tzinfo=None)) if missing_traffic: raise ValueError("Can't finalize campaigns finished on %s." "Missing traffic from %s" % (date, missing_traffic)) links = Link._byID([camp.link_id for camp in campaigns], data=True) underdelivered_campaigns = [] for camp in campaigns: if hasattr(camp, 'refund_amount'): continue link = links[camp.link_id] billable_impressions = get_billable_impressions(camp) billable_amount = get_billable_amount(camp, billable_impressions) if billable_amount >= camp.bid: if hasattr(camp, 'cpm'): text = '%s completed with $%s billable (%s impressions @ $%s).' text %= (camp, billable_amount, billable_impressions, camp.cpm) else: text = '%s completed with $%s billable (pre-CPM).' text %= (camp, billable_amount) PromotionLog.add(link, text) camp.refund_amount = 0. camp._commit() elif charged_or_not_needed(camp): underdelivered_campaigns.append(camp) if underdelivered_campaigns: queries.set_underdelivered_campaigns(underdelivered_campaigns)
def finalize_completed_campaigns(daysago=1): # PromoCampaign.end_date is utc datetime with year, month, day only now = datetime.now(g.tz) date = now - timedelta(days=daysago) date = date.replace(hour=0, minute=0, second=0, microsecond=0) q = PromoCampaign._query( PromoCampaign.c.end_date == date, # exclude no transaction and freebies PromoCampaign.c.trans_id > 0, data=True, ) campaigns = list(q) if not campaigns: return # check that traffic is up to date earliest_campaign = min(campaigns, key=lambda camp: camp.start_date) start, end = get_total_run(earliest_campaign) missing_traffic = traffic.get_missing_traffic(start.replace(tzinfo=None), date.replace(tzinfo=None)) if missing_traffic: raise ValueError("Can't finalize campaigns finished on %s." "Missing traffic from %s" % (date, missing_traffic)) links = Link._byID([camp.link_id for camp in campaigns], data=True) underdelivered_campaigns = [] for camp in campaigns: if hasattr(camp, "refund_amount"): continue link = links[camp.link_id] billable_impressions = get_billable_impressions(camp) billable_amount = get_billable_amount(camp, billable_impressions) if billable_amount >= camp.bid: if hasattr(camp, "cpm"): text = "%s completed with $%s billable (%s impressions @ $%s)." text %= (camp, billable_amount, billable_impressions, camp.cpm) else: text = "%s completed with $%s billable (pre-CPM)." text %= (camp, billable_amount) PromotionLog.add(link, text) camp.refund_amount = 0.0 camp._commit() else: underdelivered_campaigns.append(camp) if underdelivered_campaigns: set_underdelivered_campaigns(underdelivered_campaigns)
def get(cls, flight_id): fullname = g.gencache.get(cls._cache_key(flight_id), stale=True) if not fullname: q = PromoCampaign._query( PromoCampaign.c.external_flight_id == flight_id, ) q._limit = 1 campaigns = list(q) if campaigns: campaign = campaigns[0] cls.add(campaign) return campaign._fullname else: return None else: return fullname
def get(cls, flight_id): fullname = g.cache.get(cls._cachekey(flight_id)) if not fullname: q = PromoCampaign._query( PromoCampaign.c.external_flight_id == flight_id, data=True, ) q._limit = 1 campaigns = list(q) if campaigns: campaign = campaigns[0] cls.add(campaign) return campaign._fullname else: return None else: return fullname
def accepted_campaigns(offset=0): now = promo_datetime_now(offset=offset) promo_weights = PromotionWeights.get_campaigns(now) all_links = Link._by_fullname(set(x.thing_name for x in promo_weights), data=True, return_dict=True) accepted_links = {} for link_fullname, link in all_links.iteritems(): if is_accepted(link): accepted_links[link._id] = link accepted_link_ids = accepted_links.keys() campaign_query = PromoCampaign._query(PromoCampaign.c.link_id == accepted_link_ids, data=True) campaigns = dict((camp._id, camp) for camp in campaign_query) for pw in promo_weights: campaign = campaigns.get(pw.promo_idx) if not campaign or (not campaign.trans_id and campaign.priority.cpm): continue link = accepted_links.get(campaign.link_id) if not link: continue yield (link, campaign, pw.weight)
def accepted_campaigns(offset=0): now = promo_datetime_now(offset=offset) promo_weights = PromotionWeights.get_campaigns(now) all_links = Link._by_fullname(set(x.thing_name for x in promo_weights), data=True, return_dict=True) accepted_links = {} for link_fullname, link in all_links.iteritems(): if is_accepted(link): accepted_links[link._id] = link accepted_link_ids = accepted_links.keys() campaign_query = PromoCampaign._query(PromoCampaign.c.link_id == accepted_link_ids, data=True) campaigns = dict((camp._id, camp) for camp in campaign_query) for pw in promo_weights: campaign = campaigns.get(pw.promo_idx) if not campaign or not campaign.trans_id: continue link = accepted_links.get(campaign.link_id) if not link: continue yield (link, campaign, pw.weight)
def _process_daily_link_reports(links, report_id, queued_date): """ Processes report grouped by day and flight. Exponentially backs off on retries, throws on timeout. """ link_fullnames = ",".join([l._fullname for l in links]) attempt = 1 while True: try: report_result = report.fetch_report(report_id) break except report.ReportPendingException as e: timeout = (datetime.utcnow().replace(tzinfo=pytz.utc) - timedelta(seconds=g.live_config.get("adzerk_reporting_timeout", 500))) if queued_date < timeout: raise report.ReportFailedException("link reports timed out (%s/%s)" % (link_fullnames, report_id)) else: sleep_time = math.pow(RETRY_SLEEP_SECONDS, attempt) attempt = attempt + 1 g.log.warning("link reports still pending, retrying in %d seconds (%s/%s)" % (sleep_time, link_fullnames, report_id)) time.sleep(sleep_time) g.log.debug(report_result) link_ids = [l._id for l in links] campaigns = list(PromoCampaign._query(PromoCampaign.c.link_id.in_(link_ids))) campaigns_by_fullname = {c._fullname: c for c in campaigns} links_by_id = { l._id: l for l in links} # report is by date, by flight. each record is a day (not grouped by campaign) # and each detail is a flight for that day. for record in report_result.get("Records", []): date = _get_date(record) link_details = defaultdict(_reporting_factory) campaign_details = defaultdict(_reporting_factory) for detail in record.get("Details", []): campaign_fullname = _get_fullname(PromoCampaign, detail) if not campaign_fullname: flight_id = _get_flight_id(detail) g.log.error("invalid fullname for campaign (%s/%s)" % (campaign_fullname, flight_id)) continue campaign = campaigns_by_fullname.get(campaign_fullname) if not campaign: flight_id = _get_flight_id(detail) g.log.warning("no campaign for flight (%s/%s)" % (campaign_fullname, flight_id)) continue link = links_by_id[campaign.link_id] impressions, clicks, spent = _get_usage(detail) # if the price changes then there may be multiple records for each campaign/date. campaign_values = campaign_details[(campaign, date)] campaign_values["impressions"] = campaign_values["impressions"] + impressions campaign_values["clicks"] = campaign_values["clicks"] + clicks campaign_values["spent_pennies"] = campaign_values["spent_pennies"] + (spent * 100.) link_values = link_details[(link, date)] link_values["impressions"] = link_values["impressions"] + impressions link_values["clicks"] = link_values["clicks"] + clicks link_values["spent_pennies"] = link_values["spent_pennies"] + (spent * 100.) for (campaign, date), values in campaign_details.iteritems(): # hack around `target_name`s for multi subreddit collections # being overly long. if (campaign.target.is_collection and "/r/" in campaign.target.pretty_name): subreddit = "multi_%s" % PromoCampaign.SUBREDDIT_TARGET else: subreddit = campaign.target_name _insert_daily_campaign_reporting( codename=campaign._fullname, date=date, subreddit=subreddit, **values ) for (link, date), values in link_details.iteritems(): _insert_daily_link_reporting( codename=link._fullname, date=date, **values ) link.last_daily_report = report_id link.last_daily_report_run = queued_date link._commit()