def get_sold_pageviews(srs, start, end, ignore=None): srs, is_single = tup(srs, ret_is_single=True) sr_names = ['' if isinstance(sr, DefaultSR) else sr.name for sr in srs] dates = set(get_date_range(start, end)) ignore = [] if ignore is None else ignore q = (PromotionWeights.query() .filter(PromotionWeights.sr_name.in_(sr_names)) .filter(PromotionWeights.date.in_(dates))) campaign_ids = {pw.promo_idx for pw in q} campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) ret = {sr.name: dict.fromkeys(dates, 0) for sr in srs} for camp in campaigns: if camp.trans_id == NO_TRANSACTION: continue if ignore and camp._id in ignore: continue if camp.impressions <= 0: # pre-CPM campaign continue sr_name = camp.sr_name or DefaultSR.name daily_impressions = camp.impressions / camp.ndays camp_dates = set(get_date_range(camp.start_date, camp.end_date)) for date in camp_dates.intersection(dates): ret[sr_name][date] += daily_impressions if is_single: return ret[srs[0].name] else: return ret
def scheduled_campaigns_by_link(l, date=None): # A promotion/campaign is scheduled/live if it's in # PromotionWeights.get_campaigns(now) and # charged_or_not_needed date = date or promo_datetime_now() if not is_accepted(l): return [] scheduled = PromotionWeights.get_campaigns(date) campaigns = [c.promo_idx for c in scheduled if c.thing_name == l._fullname] # Check authorize accepted = [] for campaign_id in campaigns: try: campaign = PromoCampaign._byID(campaign_id, data=True) if charged_or_not_needed(campaign): accepted.append(campaign_id) except NotFound: g.log.error("PromoCampaign %d scheduled to run on %s not found." % (campaign_id, date.strftime("%Y-%m-%d"))) return accepted
def _handle_generate_lifetime_campaign_report(campaign_id): now = datetime.utcnow() campaign = PromoCampaign._byID(campaign_id, data=True) start = campaign.start_date.replace(tzinfo=pytz.utc) end = campaign.end_date.replace(tzinfo=pytz.utc) now = now.replace(tzinfo=pytz.utc) end = min([now, end]) g.log.info("generating report for campaign %s" % campaign._fullname) report_id = report.queue_report( start=start, end=end, parameters=[{ "flightId": campaign.external_flight_id, }], ) try: _process_lifetime_campaign_report( campaign=campaign, report_id=report_id, queued_date=now, ) g.log.info("successfully processed report for campaign (%s/%s)" % (campaign._fullname, report_id)) except report.ReportFailedException as e: g.log.error(e) # retry if report failed _generate_promo_report(campaign)
def get_sold_pageviews(srs, start, end, ignore=None): srs, is_single = tup(srs, ret_is_single=True) sr_names = ['' if isinstance(sr, DefaultSR) else sr.name for sr in srs] dates = set(get_date_range(start, end)) ignore = [] if ignore is None else ignore q = (PromotionWeights.query().filter( PromotionWeights.sr_name.in_(sr_names)).filter( PromotionWeights.date.in_(dates))) campaign_ids = {pw.promo_idx for pw in q} campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) ret = {sr.name: dict.fromkeys(dates, 0) for sr in srs} for camp in campaigns: if camp.trans_id == NO_TRANSACTION: continue if ignore and camp._id in ignore: continue if camp.impressions <= 0: # pre-CPM campaign continue sr_name = camp.sr_name or DefaultSR.name daily_impressions = camp.impressions / camp.ndays camp_dates = set(get_date_range(camp.start_date, camp.end_date)) for date in camp_dates.intersection(dates): ret[sr_name][date] += daily_impressions if is_single: return ret[srs[0].name] else: return ret
def GET_report(self, start, end, link_text=None, owner=None): now = datetime.now(g.tz).replace(hour=0, minute=0, second=0, microsecond=0) end = end or now - timedelta(days=1) start = start or end - timedelta(days=7) links = [] bad_links = [] owner_name = owner.name if owner else '' if owner: promo_weights = PromotionWeights.get_campaigns(start, end, author_id=owner._id) campaign_ids = [pw.promo_idx for pw in promo_weights] campaigns = PromoCampaign._byID(campaign_ids, data=True) link_ids = {camp.link_id for camp in campaigns.itervalues()} links.extend(Link._byID(link_ids, data=True, return_dict=False)) if link_text is not None: id36s = link_text.replace(',', ' ').split() try: links_from_text = Link._byID36(id36s, data=True) except NotFound: links_from_text = {} bad_links = [id36 for id36 in id36s if id36 not in links_from_text] links.extend(links_from_text.values()) content = PromoteReport(links, link_text, owner_name, bad_links, start, end) if c.render_style == 'csv': return content.as_csv() else: return PromotePage(title=_("sponsored link report"), content=content).render()
def get_promos(date, sr_names=None, link=None): campaign_ids = PromotionWeights.get_campaign_ids(date, sr_names=sr_names, link=link) campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) link_ids = {camp.link_id for camp in campaigns} links = Link._byID(link_ids, data=True) for camp in campaigns: yield camp, links[camp.link_id]
def accept_promotion(link): """ Accepting is campaign agnostic. Accepting the ad just means that it is allowed to run if payment has been processed. If a campagn is able to run, this also requeues it. """ # update the query queue set_promote_status(link, PROMOTE_STATUS.accepted) # campaigns that should be live now must be updated now = promo_datetime_now(0) promotion_weights = PromotionWeights.get_campaigns(now) live_campaigns = {pw.promo_idx for pw in promotion_weights if pw.thing_name == link._fullname} if live_campaigns: campaigns = PromoCampaign._byID(live_campaigns, data=True, return_dict=False) PromotionLog.add(link, "has live campaigns, forcing live") charge_pending(0) # campaign must be charged before it will go live for campaign in campaigns: hooks.get_hook("campaign.edit").call(link=link, campaign=campaign) queue_changed_promo(link, "accepted") # campaigns that were charged and will go live in the future must be updated future_campaigns = [camp for camp in PromoCampaign._by_link(link._id) if camp.start_date > now] transactions = get_transactions(link, future_campaigns) charged_campaigns = [ camp for camp in future_campaigns if (transactions.get(camp._id) and transactions.get(camp._id).is_charged()) ] for campaign in charged_campaigns: hooks.get_hook("campaign.edit").call(link=link, campaign=campaign) if link._spam: link._spam = False link._commit() emailer.accept_promo(link)
def scheduled_campaigns_by_link(l, date=None): # A promotion/campaign is scheduled/live if it's in # PromotionWeights.get_campaigns(now) and # authorize.is_charged_transaction() date = date or promo_datetime_now() if not is_accepted(l): return [] scheduled = PromotionWeights.get_campaigns(date) campaigns = [c.promo_idx for c in scheduled if c.thing_name == l._fullname] # Check authorize accepted = [] for campaign_id in campaigns: try: campaign = PromoCampaign._byID(campaign_id, data=True) if authorize.is_charged_transaction(campaign.trans_id, campaign_id): accepted.append(campaign_id) except NotFound: g.log.error("PromoCampaign %d scheduled to run on %s not found." % (campaign_id, date.strftime("%Y-%m-%d"))) return accepted
def _handle_generate_daily_link_reports(link_ids, campaign_ids): now = datetime.utcnow() links = Link._byID(link_ids, data=True, return_dict=False) campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) if not campaigns: return links_start, links_end = _get_campaigns_date_range(campaigns) now = now.replace(tzinfo=pytz.utc) links_start = links_start.replace(tzinfo=pytz.utc) links_end = links_end.replace(tzinfo=pytz.utc) # if data has already been processed then there's no need # to redo it. use the last time the report was run as a # starting point, but subtract 24hrs since initial numbers # are preliminary. last_run = min(getattr(l, "last_daily_report_run", links_start) for l in links) start = max( last_run - timedelta(hours=24), links_start, ) # in cases where we may be running a report well after a link # has completed ensure we always use the actual start. if start > links_end: start = links_start end = min([now, links_end]) link_fullnames = ",".join([l._fullname for l in links]) g.log.info("generating report for link %s (%s-%s)" % ( link_fullnames, start.strftime('%Y-%m-%d'), end.strftime('%Y-%m-%d'))) report_id = report.queue_report( start=start, end=end, groups=["optionId", "day"], parameters=[{ "campaignId": l.external_campaign_id, } for l in links], ) g.log.info("processing report for link (%s/%s)" % (link_fullnames, report_id)) try: _process_daily_link_reports( links=links, report_id=report_id, queued_date=now, ) g.log.info("successfully processed report for link (%s/%s)" % (link_fullnames, report_id)) except report.ReportFailedException as e: g.log.error(e) # retry if report failed _generate_link_reports(links)
def get_promos(date, sr_names=None, link=None): pws = PromotionWeights.get_campaigns(date, sr_names=sr_names, link=link) campaign_ids = {pw.promo_idx for pw in pws} campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) link_ids = {camp.link_id for camp in campaigns} links = Link._byID(link_ids, data=True) for camp in campaigns: yield camp, links[camp.link_id]
def get_house_campaigns(cls): now = promote.promo_datetime_now() pws = PromotionWeights.get_campaigns(now) campaign_ids = {pw.promo_idx for pw in pws} campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) campaigns = [camp for camp in campaigns if not camp.priority.cpm] return campaigns
def is_live_on_sr(link, srname): if not is_promoted(link): return False live = scheduled_campaigns_by_link(link) srname = srname.lower() srname = srname if srname != DefaultSR.name.lower() else '' campaigns = PromoCampaign._byID(live, return_dict=True) for campaign_id in live: campaign = campaigns.get(campaign_id) if campaign and campaign.sr_name.lower() == srname: return True return False
def get_campaigns_by_date(srs, start, end, ignore=None): srs, is_single = tup(srs, ret_is_single=True) sr_names = ['' if isinstance(sr, DefaultSR) else sr.name for sr in srs] dates = set(get_date_range(start, end)) q = (PromotionWeights.query().filter( PromotionWeights.sr_name.in_(sr_names)).filter( PromotionWeights.date.in_(dates))) if ignore: q = q.filter(PromotionWeights.promo_idx != ignore._id) campaign_ids = {pw.promo_idx for pw in q} campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) transaction_ids = { camp.trans_id for camp in campaigns if camp.trans_id != NO_TRANSACTION } if transaction_ids: transactions = Bid.query().filter(Bid.transaction.in_(transaction_ids)) transaction_by_id = {bid.transaction: bid for bid in transactions} else: transaction_by_id = {} ret = {sr.name: dict.fromkeys(dates) for sr in srs} for srname, date_dict in ret.iteritems(): for date in date_dict: ret[srname][date] = [] for camp in campaigns: if camp.trans_id == NO_TRANSACTION: continue if camp.impressions <= 0: # pre-CPM campaign continue transaction = transaction_by_id[camp.trans_id] if not (transaction.is_auth() or transaction.is_charged()): continue sr_name = camp.sr_name or DefaultSR.name camp_dates = set(get_date_range(camp.start_date, camp.end_date)) for date in camp_dates.intersection(dates): ret[sr_name][date].append(camp) if is_single: return ret[srs[0].name] else: return ret
def get_campaigns_by_date(srs, start, end, ignore=None): srs, is_single = tup(srs, ret_is_single=True) sr_names = ['' if isinstance(sr, DefaultSR) else sr.name for sr in srs] dates = set(get_date_range(start, end)) q = (PromotionWeights.query() .filter(PromotionWeights.sr_name.in_(sr_names)) .filter(PromotionWeights.date.in_(dates))) if ignore: q = q.filter(PromotionWeights.promo_idx != ignore._id) campaign_ids = {pw.promo_idx for pw in q} campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) transaction_ids = {camp.trans_id for camp in campaigns if camp.trans_id != NO_TRANSACTION} if transaction_ids: transactions = Bid.query().filter(Bid.transaction.in_(transaction_ids)) transaction_by_id = {bid.transaction: bid for bid in transactions} else: transaction_by_id = {} ret = {sr.name: dict.fromkeys(dates) for sr in srs} for srname, date_dict in ret.iteritems(): for date in date_dict: ret[srname][date] = [] for camp in campaigns: if camp.trans_id == NO_TRANSACTION: continue if camp.impressions <= 0: # pre-CPM campaign continue transaction = transaction_by_id[camp.trans_id] if not (transaction.is_auth() or transaction.is_charged()): continue sr_names = camp.target.subreddit_names camp_dates = set(get_date_range(camp.start_date, camp.end_date)) for date in camp_dates.intersection(dates): for sr_name in sr_names: ret[sr_name][date].append(camp) if is_single: return ret[srs[0].name] else: return ret
def get_campaigns_by_date(srs, start, end, ignore=None): srs = tup(srs) sr_names = [sr.name for sr in srs] campaign_ids = PromotionWeights.get_campaign_ids(start, end=end, sr_names=sr_names) if ignore: campaign_ids.discard(ignore._id) campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) # filter out deleted campaigns that didn't have their PromotionWeights # deleted campaigns = filter(lambda camp: not camp._deleted, campaigns) transaction_ids = { camp.trans_id for camp in campaigns if camp.trans_id != NO_TRANSACTION } if transaction_ids: transactions = Bid.query().filter(Bid.transaction.in_(transaction_ids)) # index transactions by transaction and campaign id because freebies # reuse the same transaction id (they always use -link id) transaction_by_id = {(bid.transaction, bid.campaign): bid for bid in transactions} else: transaction_by_id = {} dates = set(get_date_range(start, end)) ret = {date: set() for date in dates} for camp in campaigns: if camp.trans_id == NO_TRANSACTION: continue if camp.impressions <= 0: # pre-CPM campaign continue transaction = transaction_by_id[(camp.trans_id, camp._id)] if not (transaction.is_auth() or transaction.is_charged()): continue camp_dates = set(get_date_range(camp.start_date, camp.end_date)) for date in camp_dates.intersection(dates): ret[date].add(camp) return ret
def get_scheduled(date, sr_name=""): campaign_ids = PromotionWeights.get_campaign_ids(date, sr_names=[sr_name]) campaigns = PromoCampaign._byID(campaign_ids, return_dict=False, data=True) links = Link._by_fullname({camp.link_id for camp in campaigns}, return_dict=False, data=True) links = {l._id: l for l in links} kept = [] for camp in campaigns: if camp.trans_id == 0: continue link = links[camp.link_id] if link._spam or not promote.is_accepted(link): continue kept.append(camp._id) return [(camp._fullname, camp.link_id, camp.bid) for camp in kept]
def accept_promotion(link): """ Accepting is campaign agnostic. Accepting the ad just means that it is allowed to run if payment has been processed. If a campagn is able to run, this also requeues it. """ # update the query queue set_promote_status(link, PROMOTE_STATUS.accepted) # campaigns that should be live now must be updated now = promo_datetime_now(0) promotion_weights = PromotionWeights.get_campaigns(now) live_campaigns = { pw.promo_idx for pw in promotion_weights if pw.thing_name == link._fullname } if live_campaigns: campaigns = PromoCampaign._byID(live_campaigns, data=True, return_dict=False) PromotionLog.add(link, 'has live campaigns, forcing live') charge_pending(0) # campaign must be charged before it will go live for campaign in campaigns: hooks.get_hook('campaign.edit').call(link=link, campaign=campaign) queue_changed_promo(link, "accepted") # campaigns that were charged and will go live in the future must be updated future_campaigns = [ camp for camp in PromoCampaign._by_link(link._id) if camp.start_date > now ] transactions = get_transactions(link, future_campaigns) charged_campaigns = [ camp for camp in future_campaigns if (transactions.get(camp._id) and transactions.get(camp._id).is_charged()) ] for campaign in charged_campaigns: hooks.get_hook('campaign.edit').call(link=link, campaign=campaign) if link._spam: link._spam = False link._commit() emailer.accept_promo(link)
def get_scheduled(date, sr_name=''): campaign_ids = PromotionWeights.get_campaign_ids(date, sr_names=[sr_name]) campaigns = PromoCampaign._byID(campaign_ids, return_dict=False, data=True) links = Link._by_fullname({camp.link_id for camp in campaigns}, return_dict=False, data=True) links = {l._id: l for l in links} kept = [] for camp in campaigns: if camp.trans_id == 0: continue link = links[camp.link_id] if link._spam or not promote.is_accepted(link): continue kept.append(camp._id) return [(camp._fullname, camp.link_id, camp.bid) for camp in kept]
def get_campaigns_by_date(srs, start, end, ignore=None): srs = tup(srs) sr_names = [sr.name for sr in srs] campaign_ids = PromotionWeights.get_campaign_ids( start, end=end, sr_names=sr_names) if ignore: campaign_ids.discard(ignore._id) campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) # filter out deleted campaigns that didn't have their PromotionWeights # deleted campaigns = filter(lambda camp: not camp._deleted, campaigns) transaction_ids = {camp.trans_id for camp in campaigns if camp.trans_id != NO_TRANSACTION} if transaction_ids: transactions = Bid.query().filter(Bid.transaction.in_(transaction_ids)) # index transactions by transaction and campaign id because freebies # reuse the same transaction id (they always use -link id) transaction_by_id = { (bid.transaction, bid.campaign): bid for bid in transactions} else: transaction_by_id = {} dates = set(get_date_range(start, end)) ret = {date: set() for date in dates} for camp in campaigns: if camp.trans_id == NO_TRANSACTION: continue if camp.impressions <= 0: # pre-CPM campaign continue transaction = transaction_by_id[(camp.trans_id, camp._id)] if not (transaction.is_auth() or transaction.is_charged()): continue camp_dates = set(get_date_range(camp.start_date, camp.end_date)) for date in camp_dates.intersection(dates): ret[date].add(camp) return ret
def get_scheduled(date, sr_name=""): all_promotions = PromotionWeights.get_campaigns(date) fp_promotions = [p for p in all_promotions if p.sr_name == sr_name] campaigns = PromoCampaign._byID([i.promo_idx for i in fp_promotions], return_dict=False, data=True) links = Link._by_fullname([i.thing_name for i in fp_promotions], return_dict=False, data=True) links = {l._id: l for l in links} kept = [] for camp in campaigns: if camp.trans_id == 0: continue link = links[camp.link_id] if link._spam or not promote.is_accepted(link): continue kept.append(camp._id) return [ ("%s_%s" % (PC_PREFIX, to36(p.promo_idx)), p.thing_name, p.bid) for p in fp_promotions if p.promo_idx in kept ]
def bid_history(cls, start_date, end_date = None, account_id = None): from r2.lib import promote from r2.models import PromoCampaign if not end_date: end_date = datetime.datetime.now(g.tz) start_date = to_date(start_date) end_date = to_date(end_date) q = cls.query() q = q.filter(and_(cls.date >= start_date, cls.date < end_date)) q = list(q) links = Link._by_fullname([x.thing_name for x in q], data=True) d = start_date res = [] while d < end_date: bid = 0 refund = 0 for i in q: if d == i.date: l = links[i.thing_name] if (not promote.is_rejected(l) and not promote.is_unpaid(l) and not l._deleted): try: camp = PromoCampaign._byID(i.promo_idx, data=True) bid += i.bid refund += i.bid if camp.is_freebie() else 0 except NotFound: g.log.error("Skipping missing PromoCampaign in " "bidding.bid_history, campaign id: %d" % i.promo_idx) res.append([d, bid, refund]) d += datetime.timedelta(1) return res
def get_scheduled(date, sr_name=''): all_promotions = PromotionWeights.get_campaigns(date) fp_promotions = [p for p in all_promotions if p.sr_name == sr_name] campaigns = PromoCampaign._byID([i.promo_idx for i in fp_promotions], return_dict=False, data=True) links = Link._by_fullname([i.thing_name for i in fp_promotions], return_dict=False, data=True) links = {l._id: l for l in links} kept = [] for camp in campaigns: if camp.trans_id == 0: continue link = links[camp.link_id] if link._spam or not promote.is_accepted(link): continue kept.append(camp._id) return [('%s_%s' % (PC_PREFIX, to36(p.promo_idx)), p.thing_name, p.bid) for p in fp_promotions if p.promo_idx in kept]
def _handle_generate_lifetime_campaign_reports(campaign_ids): now = datetime.utcnow() campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) start = min(c.start_date for c in campaigns).replace(tzinfo=pytz.utc) end = max(c.end_date for c in campaigns).replace(tzinfo=pytz.utc) now = now.replace(tzinfo=pytz.utc) end = min([now, end]) campaign_fullnames = ",".join(c._fullname for c in campaigns) g.log.info("generating report for campaigns %s (%s-%s)" % ( campaign_fullnames, start.strftime('%Y-%m-%d'), end.strftime('%Y-%m-%d'))) report_id = report.queue_report( start=start, end=end, groups=["optionId"], parameters=[{ "flightId": c.external_flight_id, } for c in campaigns], ) try: _process_lifetime_campaign_reports( campaigns=campaigns, report_id=report_id, queued_date=now, ) g.log.info("successfully processed report for campaigns (%s/%s)" % (campaign_fullnames, report_id)) except report.ReportFailedException as e: g.log.error(e) # retry if report failed _generate_promo_reports(campaigns)