def _handle_check_edits(payload):
existing = Link._by_fullname(payload["link"], data=True)
creative = creatives_service.get_creative(existing)
link = utils.dfp_creative_to_link(
creative, link=Link._by_fullname(payload["link"], data=True))
link.dfp_checking_edits = False
link._commit()
def _handle_adzerk(msg):
data = json.loads(msg.body)
g.log.debug('data: %s' % data)
action = data.get('action')
if action == 'deactivate_link':
link = Link._by_fullname(data['link'], data=True)
_deactivate_link(link)
elif action == 'deactivate_campaign':
link = Link._by_fullname(data['link'], data=True)
campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
_deactivate_campaign(link, campaign)
elif action == 'update_adzerk':
link = Link._by_fullname(data['link'], data=True)
campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
_update_adzerk(link, campaign)
def _handle_adzerk(msg):
data = json.loads(msg.body)
g.log.debug('data: %s' % data)
action = data.get('action')
if action == 'deactivate_orphaned_flight':
_deactivate_orphaned_flight(data['flight'])
return
link = Link._by_fullname(data['link'], data=True)
if data['campaign']:
campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
else:
campaign = None
if action == 'update_adzerk':
if 'triggered_by' in data and data['triggered_by'] is not None:
triggered_by = Account._by_fullname(data['triggered_by'], data=True)
else:
triggered_by = None
_update_adzerk(link, campaign, triggered_by)
elif action == 'deactivate_overdelivered':
_deactivate_overdelivered(link, campaign)
def bid_history(cls, start_date, end_date=None, account_id=None):
from r2.models import Link
from r2.lib import promote
start_date = to_date(start_date)
end_date = to_date(end_date)
q = cls.query()
q = q.filter(and_(cls.date >= start_date, cls.date < end_date))
q = list(q)
links = Link._by_fullname([x.thing_name for x in q], data=True)
d = start_date
res = []
while d < end_date:
bid = 0
refund = 0
for i in q:
if d == i.date:
l = links[i.thing_name]
if not promote.is_rejected(l) and not promote.is_unpaid(
l) and not l._deleted:
camp = l.campaigns[i.promo_idx]
bid += i.bid
refund += i.bid if camp[-1] <= 0 else 0
res.append([d, bid, refund])
d += datetime.timedelta(1)
return res
def set_recent_clicks():
c.recent_clicks = []
if not c.user_is_loggedin:
return
click_cookie = read_user_cookie('recentclicks2')
if click_cookie:
if valid_click_cookie(click_cookie):
names = [x for x in UniqueIterator(click_cookie.split(',')) if x]
if len(names) > 5:
names = names[:5]
set_user_cookie('recentclicks2', ','.join(names))
#eventually this will look at the user preference
names = names[:5]
try:
c.recent_clicks = Link._by_fullname(names,
data=True,
return_dict=False)
except NotFound:
# clear their cookie because it's got bad links in it
set_user_cookie('recentclicks2', '')
else:
#if the cookie wasn't valid, clear it
set_user_cookie('recentclicks2', '')
def thing_lookup(self, tuples):
links = Link._by_fullname([t.link for t in tuples], data=True, return_dict=True, stale=self.stale)
return [
Storage({"thing": links[t.link], "_id": links[t.link]._id, "weight": t.weight, "campaign": t.campaign})
for t in tuples
]
def process_message(msgs, chan):
"""Update get_links(), the Links by Subreddit precomputed query.
get_links() is a CachedResult which is stored in permacache. To
update these objects we need to do a read-modify-write which requires
obtaining a lock. Sharding these updates by subreddit allows us to run
multiple consumers (but ideally just one per shard) to avoid lock
contention.
"""
from r2.lib.db.queries import add_queries, get_links
link_names = {msg.body for msg in msgs}
links = Link._by_fullname(link_names, return_dict=False)
print 'Processing %r' % (links, )
links_by_sr_id = defaultdict(list)
for link in links:
links_by_sr_id[link.sr_id].append(link)
srs_by_id = Subreddit._byID(links_by_sr_id.keys(), stale=True)
for sr_id, links in links_by_sr_id.iteritems():
with g.stats.get_timer("link_vote_processor.subreddit_queries"):
sr = srs_by_id[sr_id]
add_queries(
queries=[get_links(sr, sort, "all") for sort in SORTS],
insert_items=links,
)
def process_message(msgs, chan):
"""Update get_domain_links(), the Links by domain precomputed query.
get_domain_links() is a CachedResult which is stored in permacache. To
update these objects we need to do a read-modify-write which requires
obtaining a lock. Sharding these updates by domain allows us to run
multiple consumers (but ideally just one per shard) to avoid lock
contention.
"""
from r2.lib.db.queries import add_queries, get_domain_links
link_names = {msg.body for msg in msgs}
links = Link._by_fullname(link_names, return_dict=False)
print 'Processing %r' % (links, )
links_by_domain = defaultdict(list)
for link in links:
parsed = UrlParser(link.url)
# update the listings for all permutations of the link's domain
for domain in parsed.domain_permutations():
links_by_domain[domain].append(link)
for d, links in links_by_domain.iteritems():
with g.stats.get_timer("link_vote_processor.domain_queries"):
add_queries(
queries=[
get_domain_links(d, sort, "all") for sort in SORTS
],
insert_items=links,
)
def bid_history(cls, start_date, end_date = None, account_id = None):
from r2.models import Link
from r2.lib import promote
start_date = to_date(start_date)
end_date = to_date(end_date)
q = cls.query()
q = q.filter(and_(cls.date >= start_date, cls.date < end_date))
q = list(q)
links = Link._by_fullname([x.thing_name for x in q], data=True)
d = start_date
res = []
while d < end_date:
bid = 0
refund = 0
for i in q:
if d == i.date:
l = links[i.thing_name]
if (not promote.is_rejected(l) and
not promote.is_unpaid(l) and
not l._deleted and
i.promo_idx in getattr(l, 'campaigns', {})):
camp = l.campaigns[i.promo_idx]
bid += i.bid
refund += i.bid if camp[-1] <= 0 else 0
res.append([d, bid, refund])
d += datetime.timedelta(1)
return res
def set_recent_clicks():
c.recent_clicks = []
if not c.user_is_loggedin:
return
click_cookie = read_user_cookie('recentclicks2')
if click_cookie:
if valid_click_cookie(click_cookie):
names = [ x for x in UniqueIterator(click_cookie.split(',')) if x ]
if len(names) > 5:
names = names[:5]
set_user_cookie('recentclicks2', ','.join(names))
#eventually this will look at the user preference
names = names[:5]
try:
c.recent_clicks = Link._by_fullname(names, data=True,
return_dict=False)
except NotFound:
# clear their cookie because it's got bad links in it
set_user_cookie('recentclicks2', '')
else:
#if the cookie wasn't valid, clear it
set_user_cookie('recentclicks2', '')
def process_message(msgs, chan):
"""Update get_domain_links(), the Links by domain precomputed query.
get_domain_links() is a CachedResult which is stored in permacache. To
update these objects we need to do a read-modify-write which requires
obtaining a lock. Sharding these updates by domain allows us to run
multiple consumers (but ideally just one per shard) to avoid lock
contention.
"""
from r2.lib.db.queries import add_queries, get_domain_links
link_names = {msg.body for msg in msgs}
links = Link._by_fullname(link_names, return_dict=False)
print 'Processing %r' % (links,)
links_by_domain = defaultdict(list)
for link in links:
parsed = UrlParser(link.url)
# update the listings for all permutations of the link's domain
for domain in parsed.domain_permutations():
links_by_domain[domain].append(link)
for d, links in links_by_domain.iteritems():
with g.stats.get_timer("link_vote_processor.domain_queries"):
add_queries(
queries=[
get_domain_links(d, sort, "all") for sort in SORTS],
insert_items=links,
)
def process_message(msgs, chan):
"""Update get_links(), the Links by Subreddit precomputed query.
get_links() is a CachedResult which is stored in permacache. To
update these objects we need to do a read-modify-write which requires
obtaining a lock. Sharding these updates by subreddit allows us to run
multiple consumers (but ideally just one per shard) to avoid lock
contention.
"""
from r2.lib.db.queries import add_queries, get_links
link_names = {msg.body for msg in msgs}
links = Link._by_fullname(link_names, return_dict=False)
print 'Processing %r' % (links,)
links_by_sr_id = defaultdict(list)
for link in links:
links_by_sr_id[link.sr_id].append(link)
srs_by_id = Subreddit._byID(links_by_sr_id.keys(), stale=True)
for sr_id, links in links_by_sr_id.iteritems():
with g.stats.get_timer("link_vote_processor.subreddit_queries"):
sr = srs_by_id[sr_id]
add_queries(
queries=[get_links(sr, sort, "all") for sort in SORTS],
insert_items=links,
)
def GET_report(self, start, end, link_text=None):
now = datetime.now(g.tz).replace(hour=0,
minute=0,
second=0,
microsecond=0)
end = end or now - timedelta(days=1)
start = start or end - timedelta(days=7)
if link_text is not None:
names = link_text.replace(',', ' ').split()
try:
links = Link._by_fullname(names, data=True)
except NotFound:
links = {}
bad_links = [name for name in names if name not in links]
links = links.values()
else:
links = []
bad_links = []
content = PromoteReport(links, link_text, bad_links, start, end)
if c.render_style == 'csv':
return content.as_csv()
else:
return PromotePage('report', content=content).render()
def thing_lookup(self, tuples):
links = Link._by_fullname([t.link for t in tuples], data=True,
return_dict=True, stale=self.stale)
return [Storage({'thing': links[t.link],
'_id': links[t.link]._id,
'weight': t.weight,
'campaign': t.campaign}) for t in tuples]
def thing_lookup(self, tuples):
links = Link._by_fullname([t.link for t in tuples], data=True,
return_dict=True, stale=self.stale)
return [Storage({'thing': links[t.link],
'_id': links[t.link]._id,
'weight': t.weight,
'campaign': t.campaign}) for t in tuples]
def send_account_summary_email(account_thing_id, verbose=False, send_email=send_email):
account = Account._byID(account_thing_id, data=True)
if not should_send_activity_summary_email(account):
return
# if we've never sent an email, only tell about the last 24 hours
a_day_ago = datetime.datetime.now(pytz.utc) - datetime.timedelta(hours=24)
if getattr(account, 'last_email_sent_at', None) is None:
account.last_email_sent_at = a_day_ago
c.content_langs = 'en-US'
# Find all the "active" links for this user. Frontpage uses the c.user global
# to find the right subreddits for the current user
c.user = account
c.user_is_loggedin = True
thing_ids = []
for link in Frontpage.get_links('active', 'all'):
thing_ids.append(link)
active_links_hash = Link._by_fullname(thing_ids, data=True)
active_links = [active_links_hash[t_id] for t_id in thing_ids if active_links_hash[t_id]._active > account.last_email_sent_at]
idx = 0
for ll in active_links:
idx += 1
ll.num = idx
# Find all new spaces created since we last sent the user an email
new_spaces = list(fetch_things2(Subreddit._query(
Subreddit.c._date > account.last_email_sent_at,
sort=asc('_date'))))
# don't bother sending email if there's noting to report.
if len(new_spaces) == 0 and len(active_links) == 0:
return
# Get the date and time
now = datetime.datetime.now(pytz.timezone('US/Eastern'))
date_string = now.strftime("%A %B %d, %Y")
time_string = now.strftime("%I:%M %p")
# Render the template
html_email_template = g.mako_lookup.get_template('summary_email.html')
html_body = html_email_template.render(
last_email_sent_at=account.last_email_sent_at,
new_spaces=new_spaces,
active_links=active_links,
date_string=date_string,
time_string=time_string)
# with open('out.html', 'w') as ff:
# ff.write(html_body)
if verbose:
print "sending email to %s" % (account.email,)
send_email(account.email, html_body, date_string)
account.last_email_sent_at = datetime.datetime.now(pytz.utc)
account._commit()
def get_hot_items(srs, item_type, src):
"""Get hot links from specified srs."""
hot_srs = {sr._id: sr for sr in srs} # for looking up sr by id
hot_link_fullnames = normalized_hot(sr._id for sr in srs)
hot_links = Link._by_fullname(hot_link_fullnames, return_dict=False)
hot_items = []
for l in hot_links:
hot_items.append(ExploreItem(item_type, src, hot_srs[l.sr_id], l))
return hot_items
def get_hot_items(srs, item_type, src):
"""Get hot links from specified srs."""
hot_srs = {sr._id: sr for sr in srs} # for looking up sr by id
hot_link_fullnames = normalized_hot([sr._id for sr in srs])
hot_links = Link._by_fullname(hot_link_fullnames, return_dict=False)
hot_items = []
for l in hot_links:
hot_items.append(ExploreItem(item_type, src, hot_srs[l.sr_id], l))
return hot_items
def get_hot(sr):
"""Get the hottest links for a subreddit. If g.use_query_cache is
True, it'll use the query cache, otherwise it'll use cached_query()
from above."""
q = sr.get_links('hot', 'all')
if isinstance(q, Query):
return cached_query(q, sr)
else:
return Link._by_fullname(list(q)[:150], return_dict = False)
def _get_selfserve_links(self, count):
links = Subreddit._by_name(g.advertising_links_sr).get_links(
'new', 'all')
items = Link._by_fullname(links, data=True, return_dict=False)
id36s = map(
lambda x: self.advertising_link_id36_re.match(x.url).group(1),
items)
ad_links = Link._byID36(id36s, return_dict=False, data=True)
return wrap_links(ad_links, num=count)
def get_rising_items(omit_sr_ids, count=4):
"""Get links that are rising right now."""
all_rising = rising.get_all_rising()
candidate_sr_ids = {sr_id for link, score, sr_id in all_rising}.difference(omit_sr_ids)
link_fullnames = [link for link, score, sr_id in all_rising if sr_id in candidate_sr_ids]
link_fullnames_to_show = random_sample(link_fullnames, count)
rising_links = Link._by_fullname(link_fullnames_to_show, return_dict=False, data=True)
rising_items = [ExploreItem(TYPE_RISING, "ris", Subreddit._byID(l.sr_id), l) for l in rising_links]
return rising_items
def _handle_upsert_campaign(payload):
link = Link._by_fullname(payload["link"], data=True)
campaign = PromoCampaign._by_fullname(payload["campaign"], data=True)
owner = Account._byID(campaign.owner_id)
lineitem = lineitems_service.upsert_lineitem(owner, campaign)
creative = creatives_service.get_creative(link)
lineitems_service.associate_with_creative(
lineitem=lineitem, creative=creative)
def process_message(msg):
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
timer = g.stats.get_timer("link_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
link = Link._by_fullname(vote_data.pop("thing_fullname"))
# create the vote and update the voter's liked/disliked under lock so
# that the vote state and cached query are consistent
lock_key = "vote-%s-%s" % (user._id36, link._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, link, vote_data)
try:
vote = Vote(
user,
link,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
# CUSTOM: voting model
vote_direction=vote_data["vote_direction"],
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
update_user_liked(vote)
timer.intermediate("voter_likes")
vote_valid = vote.is_automatic_initial_vote or vote.effects.affects_score
link_valid = not (link._spam or link._deleted)
if vote_valid and link_valid:
add_to_author_query_q(link)
add_to_subreddit_query_q(link)
add_to_domain_query_q(link)
timer.stop()
timer.flush()
def get_rising_items(omit_sr_ids, count=4):
"""Get links that are rising right now."""
all_rising = rising.get_all_rising()
candidate_sr_ids = {sr_id for link, score, sr_id in all_rising}.difference(omit_sr_ids)
link_fullnames = [link for link, score, sr_id in all_rising if sr_id in candidate_sr_ids]
link_fullnames_to_show = random_sample(link_fullnames, count)
rising_links = Link._by_fullname(link_fullnames_to_show,
return_dict=False,
data=True)
rising_items = [ExploreItem(TYPE_RISING, 'ris', Subreddit._byID(l.sr_id), l)
for l in rising_links]
return rising_items
def process_message(msg):
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
timer = g.stats.get_timer("link_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
link = Link._by_fullname(vote_data.pop("thing_fullname"))
# create the vote and update the voter's liked/disliked under lock so
# that the vote state and cached query are consistent
lock_key = "vote-%s-%s" % (user._id36, link._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, link, vote_data)
try:
vote = Vote(
user,
link,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
update_user_liked(vote)
timer.intermediate("voter_likes")
vote_valid = vote.is_automatic_initial_vote or vote.effects.affects_score
link_valid = not (link._spam or link._deleted)
if vote_valid and link_valid:
add_to_author_query_q(link)
add_to_subreddit_query_q(link)
add_to_domain_query_q(link)
timer.stop()
timer.flush()
def _run(msgs, chan):
items = [json.loads(msg.body) for msg in msgs]
if QUEUE_ALL in items:
# QUEUE_ALL is just an indicator to run make_daily_promotions.
# There's no promotion log to update in this case.
print "Received %s QUEUE_ALL message(s)" % items.count(QUEUE_ALL)
items = [i for i in items if i != QUEUE_ALL]
make_daily_promotions()
links = Link._by_fullname([i["link"] for i in items])
for item in items:
PromotionLog.add(links[item['link']],
"Finished remaking current promotions (this link "
"was: %(message)s" % item)
def _run(msgs, chan):
items = [json.loads(msg.body) for msg in msgs]
if QUEUE_ALL in items:
# QUEUE_ALL is just an indicator to run make_daily_promotions.
# There's no promotion log to update in this case.
print "Received %s QUEUE_ALL message(s)" % items.count(QUEUE_ALL)
items = [i for i in items if i != QUEUE_ALL]
make_daily_promotions()
links = Link._by_fullname([i["link"] for i in items])
for item in items:
PromotionLog.add(links[item['link']],
"Finished remaking current promotions (this link "
"was: %(message)s" % item)
def get_comment_items(srs, src, count=4):
"""Get hot links from srs, plus top comment from each link."""
link_fullnames = normalized_hot([sr._id for sr in srs])
hot_links = Link._by_fullname(link_fullnames[:count], return_dict=False)
top_comments = []
for link in hot_links:
builder = CommentBuilder(
link, operators.desc("_confidence"), comment=None, context=None, num=1, load_more=False
)
listing = NestedListing(builder, parent_name=link._fullname).listing()
top_comments.extend(listing.things)
srs = Subreddit._byID([com.sr_id for com in top_comments])
links = Link._byID([com.link_id for com in top_comments])
comment_items = [ExploreItem(TYPE_COMMENT, src, srs[com.sr_id], links[com.link_id], com) for com in top_comments]
return comment_items
def _handle_adzerk(msg):
data = json.loads(msg.body)
g.log.debug('data: %s' % data)
action = data.get('action')
link = Link._by_fullname(data['link'], data=True)
if data['campaign']:
campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
else:
campaign = None
if action == 'update_adzerk':
_update_adzerk(link, campaign)
elif action == 'deactivate_overdelivered':
_deactivate_overdelivered(link, campaign)
def _handle_adzerk(msg):
data = json.loads(msg.body)
g.log.debug('data: %s' % data)
action = data.get('action')
link = Link._by_fullname(data['link'], data=True)
if data['campaign']:
campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
else:
campaign = None
if action == 'update_adzerk':
_update_adzerk(link, campaign)
elif action == 'deactivate_overdelivered':
_deactivate_overdelivered(link, campaign)
def _handle_upsert_campaign(payload):
link = Link._by_fullname(payload["link"], data=True)
campaign = PromoCampaign._by_fullname(payload["campaign"], data=True)
owner = Account._byID(campaign.owner_id)
author = Account._byID(link.author_id)
try:
lineitem = lineitems_service.upsert_lineitem(owner, campaign)
except ValueError as e:
g.log.error("unable to upsert lineitem: %s" % e)
return
creative = creatives_service.upsert_creative(author, link)
lineitems_service.associate_with_creative(
lineitem=lineitem, creative=creative)
def get_scheduled(date, sr_name=""):
campaign_ids = PromotionWeights.get_campaign_ids(date, sr_names=[sr_name])
campaigns = PromoCampaign._byID(campaign_ids, return_dict=False, data=True)
links = Link._by_fullname({camp.link_id for camp in campaigns}, return_dict=False, data=True)
links = {l._id: l for l in links}
kept = []
for camp in campaigns:
if camp.trans_id == 0:
continue
link = links[camp.link_id]
if link._spam or not promote.is_accepted(link):
continue
kept.append(camp._id)
return [(camp._fullname, camp.link_id, camp.bid) for camp in kept]
def get_scheduled(date, sr_name=''):
campaign_ids = PromotionWeights.get_campaign_ids(date, sr_names=[sr_name])
campaigns = PromoCampaign._byID(campaign_ids, return_dict=False, data=True)
links = Link._by_fullname({camp.link_id for camp in campaigns},
return_dict=False, data=True)
links = {l._id: l for l in links}
kept = []
for camp in campaigns:
if camp.trans_id == 0:
continue
link = links[camp.link_id]
if link._spam or not promote.is_accepted(link):
continue
kept.append(camp._id)
return [(camp._fullname, camp.link_id, camp.bid) for camp in kept]
def promote_v2():
# alter table bids add column campaign integer;
# update bids set campaign = 0;
from r2.models import Link, NotFound, PromoteDates, Bid
from datetime import datetime
from pylons import g
for p in PromoteDates.query():
try:
l = Link._by_fullname(p.thing_name,
data = True, return_dict = False)
if not l:
raise NotFound, p.thing_name
# update the promote status
l.promoted = True
l.promote_status = getattr(l, "promote_status", STATUS.unseen)
l._date = datetime(*(list(p.start_date.timetuple()[:7]) + [g.tz]))
set_status(l, l.promote_status)
# add new campaign
print (l, (p.start_date, p.end_date), p.bid, None)
if not p.bid:
print "no bid? ", l
p.bid = 20
new_campaign(l, (p.start_date, p.end_date), p.bid, None)
print "updated: %s (%s)" % (l, l._date)
except NotFound:
print "NotFound: %s" % p.thing_name
print "updating campaigns"
for b in Bid.query():
l = Link._byID(int(b.thing_id))
print "updating: ", l
campaigns = getattr(l, "campaigns", {}).copy()
indx = b.campaign
if indx in campaigns:
sd, ed, bid, sr, trans_id = campaigns[indx]
campaigns[indx] = sd, ed, bid, sr, b.transaction
l.campaigns = campaigns
l._commit()
else:
print "no campaign information: ", l
def promote_v2():
# alter table bids add column campaign integer;
# update bids set campaign = 0;
from r2.models import Link, NotFound, PromoteDates, Bid
from datetime import datetime
from pylons import g
for p in PromoteDates.query():
try:
l = Link._by_fullname(p.thing_name,
data = True, return_dict = False)
if not l:
raise NotFound, p.thing_name
# update the promote status
l.promoted = True
l.promote_status = getattr(l, "promote_status", STATUS.unseen)
l._date = datetime(*(list(p.start_date.timetuple()[:7]) + [g.tz]))
set_status(l, l.promote_status)
# add new campaign
print (l, (p.start_date, p.end_date), p.bid, None)
if not p.bid:
print "no bid? ", l
p.bid = 20
new_campaign(l, (p.start_date, p.end_date), p.bid, None)
print "updated: %s (%s)" % (l, l._date)
except NotFound:
print "NotFound: %s" % p.thing_name
print "updating campaigns"
for b in Bid.query():
l = Link._byID(int(b.thing_id))
print "updating: ", l
campaigns = getattr(l, "campaigns", {}).copy()
indx = b.campaign
if indx in campaigns:
sd, ed, bid, sr, trans_id = campaigns[indx]
campaigns[indx] = sd, ed, bid, sr, b.transaction
l.campaigns = campaigns
l._commit()
else:
print "no campaign information: ", l
def get_scheduled(date, sr_name=""):
all_promotions = PromotionWeights.get_campaigns(date)
fp_promotions = [p for p in all_promotions if p.sr_name == sr_name]
campaigns = PromoCampaign._byID([i.promo_idx for i in fp_promotions], return_dict=False, data=True)
links = Link._by_fullname([i.thing_name for i in fp_promotions], return_dict=False, data=True)
links = {l._id: l for l in links}
kept = []
for camp in campaigns:
if camp.trans_id == 0:
continue
link = links[camp.link_id]
if link._spam or not promote.is_accepted(link):
continue
kept.append(camp._id)
return [
("%s_%s" % (PC_PREFIX, to36(p.promo_idx)), p.thing_name, p.bid) for p in fp_promotions if p.promo_idx in kept
]
def get_comment_items(srs, src, count=4):
"""Get hot links from srs, plus top comment from each link."""
link_fullnames = normalized_hot([sr._id for sr in srs])
hot_links = Link._by_fullname(link_fullnames[:count], return_dict=False)
top_comments = []
for link in hot_links:
builder = CommentBuilder(link,
operators.desc('_confidence'),
comment=None,
context=None,
load_more=False)
listing = NestedListing(builder, num=1,
parent_name=link._fullname).listing()
top_comments.extend(listing.things)
srs = Subreddit._byID([com.sr_id for com in top_comments])
links = Link._byID([com.link_id for com in top_comments])
comment_items = [
ExploreItem(TYPE_COMMENT, src, srs[com.sr_id], links[com.link_id], com)
for com in top_comments
]
return comment_items
def accepted_campaigns(offset=0):
now = promo_datetime_now(offset=offset)
promo_weights = PromotionWeights.get_campaigns(now)
all_links = Link._by_fullname(set(x.thing_name for x in promo_weights), data=True, return_dict=True)
accepted_links = {}
for link_fullname, link in all_links.iteritems():
if is_accepted(link):
accepted_links[link._id] = link
accepted_link_ids = accepted_links.keys()
campaign_query = PromoCampaign._query(PromoCampaign.c.link_id == accepted_link_ids, data=True)
campaigns = dict((camp._id, camp) for camp in campaign_query)
for pw in promo_weights:
campaign = campaigns.get(pw.promo_idx)
if not campaign or (not campaign.trans_id and campaign.priority.cpm):
continue
link = accepted_links.get(campaign.link_id)
if not link:
continue
yield (link, campaign, pw.weight)
def bid_history(cls, start_date, end_date = None, account_id = None):
from r2.lib import promote
from r2.models import PromoCampaign
if not end_date:
end_date = datetime.datetime.now(g.tz)
start_date = to_date(start_date)
end_date = to_date(end_date)
q = cls.query()
q = q.filter(and_(cls.date >= start_date, cls.date < end_date))
q = list(q)
links = Link._by_fullname([x.thing_name for x in q], data=True)
d = start_date
res = []
while d < end_date:
bid = 0
refund = 0
for i in q:
if d == i.date:
l = links[i.thing_name]
if (not promote.is_rejected(l) and
not promote.is_unpaid(l) and
not l._deleted):
try:
camp = PromoCampaign._byID(i.promo_idx, data=True)
bid += i.bid
refund += i.bid if camp.is_freebie() else 0
except NotFound:
g.log.error("Skipping missing PromoCampaign in "
"bidding.bid_history, campaign id: %d"
% i.promo_idx)
res.append([d, bid, refund])
d += datetime.timedelta(1)
return res
def get_scheduled(date, sr_name=''):
all_promotions = PromotionWeights.get_campaigns(date)
fp_promotions = [p for p in all_promotions if p.sr_name == sr_name]
campaigns = PromoCampaign._byID([i.promo_idx for i in fp_promotions],
return_dict=False,
data=True)
links = Link._by_fullname([i.thing_name for i in fp_promotions],
return_dict=False,
data=True)
links = {l._id: l for l in links}
kept = []
for camp in campaigns:
if camp.trans_id == 0:
continue
link = links[camp.link_id]
if link._spam or not promote.is_accepted(link):
continue
kept.append(camp._id)
return [('%s_%s' % (PC_PREFIX, to36(p.promo_idx)), p.thing_name, p.bid)
for p in fp_promotions if p.promo_idx in kept]
def accepted_campaigns(offset=0):
now = promo_datetime_now(offset=offset)
promo_weights = PromotionWeights.get_campaigns(now)
all_links = Link._by_fullname(set(x.thing_name for x in promo_weights),
data=True, return_dict=True)
accepted_links = {}
for link_fullname, link in all_links.iteritems():
if is_accepted(link):
accepted_links[link._id] = link
accepted_link_ids = accepted_links.keys()
campaign_query = PromoCampaign._query(PromoCampaign.c.link_id == accepted_link_ids,
data=True)
campaigns = dict((camp._id, camp) for camp in campaign_query)
for pw in promo_weights:
campaign = campaigns.get(pw.promo_idx)
if not campaign or not campaign.trans_id:
continue
link = accepted_links.get(campaign.link_id)
if not link:
continue
yield (link, campaign, pw.weight)
def GET_report(self, start, end, link_text=None):
now = datetime.now(g.tz).replace(hour=0, minute=0, second=0,
microsecond=0)
end = end or now - timedelta(days=1)
start = start or end - timedelta(days=7)
if link_text is not None:
names = link_text.replace(',', ' ').split()
try:
links = Link._by_fullname(names, data=True)
except NotFound:
links = {}
bad_links = [name for name in names if name not in links]
links = links.values()
else:
links = []
bad_links = []
content = PromoteReport(links, link_text, bad_links, start, end)
if c.render_style == 'csv':
return content.as_csv()
else:
return PromotePage('report', content=content).render()
def _handle_upsert_promotion(payload):
link = Link._by_fullname(payload["link"], data=True)
author = Account._byID(link.author_id)
creatives_service.upsert_creative(author, link)
def adzerk_request(
keywords, properties, user_id,
num_placements=1,
timeout=1.5,
platform="desktop",
is_refresh=False,
referrer=None,
):
placements = []
divs = ["div%s" % i for i in xrange(num_placements)]
subreddit = None
if isinstance(c.site, Subreddit) and not c.default_sr:
subreddit = c.site.name
for div in divs:
placement = {
"divName": div,
"networkId": g.az_selfserve_network_id,
"siteId": g.az_selfserve_site_ids[platform],
"adTypes": [LEADERBOARD_AD_TYPE],
"eventIds": [EVENT_TYPE_UPVOTE, EVENT_TYPE_DOWNVOTE],
"properties": properties,
}
if subreddit is not None:
placement["properties"] = {
"subreddit": subreddit,
}
placements.append(placement)
keywords = [word.lower() for word in keywords]
data = {
"placements": placements,
"keywords": keywords,
"ip": request.ip,
"enableBotFiltering": True,
"includePricingData": True,
}
page_url = request.headers.get("referer", None)
if page_url is not None:
data["url"] = page_url
if referrer is not None:
data["referrer"] = referrer
if user_id:
data["user"] = {"key": user_id}
url = 'https://%s/api/v2' % g.adzerk_engine_domain
headers = {
'content-type': 'application/json',
'user-agent': request.headers.get('User-Agent'),
}
do_not_track = request.headers.get("DNT", None)
if do_not_track and feature.is_enabled("adzerk_do_not_track"):
headers["DNT"] = do_not_track
timer = g.stats.get_timer("providers.adzerk")
timer.start()
for placement in placements:
g.ad_events.ad_request(
keywords=keywords,
platform=platform,
placement_name=placement["divName"],
placement_types=placement["adTypes"],
is_refresh=is_refresh,
subreddit=c.site,
request=request,
context=c,
)
try:
r = requests.post(url, data=json.dumps(data), headers=headers,
timeout=timeout)
except (requests.exceptions.Timeout, requests.exceptions.SSLError):
g.stats.simple_event('adzerk.request.timeout')
return None
except requests.exceptions.ConnectionError:
g.stats.simple_event('adzerk.request.refused')
return None
except select.error:
return None
finally:
timer.stop()
errored = False
try:
response = adzerk_api.handle_response(r)
except adzerk_api.AdzerkError:
g.stats.simple_event('adzerk.request.badresponse')
g.log.error('adzerk_request: bad response (%s) %r', r.status_code,
r.content)
errored = True
finally:
# Temporarily log request data and response body,
# sample at 1%
if random.random() < g.live_config.get('ad_log_sample_rate', 0):
g.log.info("ad_request [DNT=%s]: %s, ad_response: [%s] %s",
do_not_track, json.dumps(data), r.status_code, r.text)
if errored:
return None
decisions = response['decisions']
if not decisions:
return None
placements_by_div = {placement["divName"]: placement
for placement in placements}
res = []
for div in divs:
decision = decisions[div]
if not decision:
continue
placement = placements_by_div[div]
ad_id = decision['adId']
pricing = decision.get("pricing", {})
revenue = pricing.get("revenue")
rate_type_id = pricing.get("rateType")
rate_type = RATE_TYPE_NAMES.get(rate_type_id, None)
impression_url = decision.get("impressionUrl")
impression_b64_data = UrlParser(impression_url).query_dict.get("e", "")
impression_id, matched_keywords = None, []
try:
# fix padding and string encode
impression_b64_data = str(
impression_b64_data +
("=" * (len(impression_b64_data) % 4))
)
impression_data = json.loads(
base64.urlsafe_b64decode(impression_b64_data),
strict=False,
)
except UnicodeDecodeError:
g.log.info("unable to decode impression data: %s", impression_b64_data)
impression_data = None
except TypeError, ValueError:
impression_data = None
if impression_data is not None:
impression_id = impression_data.get("di")
matched_keywords = impression_data.get("mk")
if matched_keywords:
matched_keywords = matched_keywords.split(",")
# adserver ads are not reddit links, we return the body
if decision['campaignId'] in g.adserver_campaign_ids:
g.ad_events.ad_response(
keywords=keywords,
platform=platform,
placement_name=div,
placement_types=placement["adTypes"],
ad_id=ad_id,
impression_id=impression_id,
matched_keywords=matched_keywords,
rate_type=rate_type,
clearing_price=revenue,
subreddit=c.site,
request=request,
context=c,
)
return AdserverResponse(decision['contents'][0]['body'])
adzerk_flight_id = decision['flightId']
imp_pixel = decision['impressionUrl']
click_url = decision['clickUrl']
events_by_id = {event["id"]: event["url"] for event in decision["events"]}
upvote_pixel = events_by_id[EVENT_TYPE_UPVOTE]
downvote_pixel = events_by_id[EVENT_TYPE_DOWNVOTE]
campaign_fullname = PromoCampaignByFlightIdCache.get(adzerk_flight_id)
contents = decision['contents'][0]
body = json.loads(contents['body'])
link_fullname = body['link']
target = body['target']
priority = None
priority_id = body.get('priorityId', None)
ecpm = body.get('ecpm', None)
moat_query = body.get('moatQuery', None)
if priority_id is not None:
try:
priority_id = int(priority_id)
except ValueError:
pass
for k, v in g.az_selfserve_priorities.iteritems():
if priority_id == v:
priority = k
g.ad_events.ad_response(
keywords=keywords,
platform=platform,
placement_name=div,
placement_types=placement["adTypes"],
ad_id=ad_id,
impression_id=impression_id,
matched_keywords=matched_keywords,
rate_type=rate_type,
clearing_price=revenue,
subreddit=c.site,
link_fullname=link_fullname,
campaign_fullname=campaign_fullname,
priority=priority,
ecpm=ecpm,
request=request,
context=c,
)
if not campaign_fullname:
link = Link._by_fullname(link_fullname, data=True, stale=True)
if promote.is_external(link):
campaign_fullname = promote.EXTERNAL_CAMPAIGN
else:
adzerk_campaign_id = decision['campaignId']
g.stats.simple_event('adzerk.request.orphaned_flight')
g.log.error('adzerk_request: couldn\'t find campaign for flight (az campaign: %s, flight: %s)',
adzerk_campaign_id, adzerk_flight_id)
# deactivate the flight, it will be reactivated if a
# valid campaign actually exists
deactivate_orphaned_flight(adzerk_flight_id)
continue
res.append(AdzerkResponse(
link=link_fullname,
campaign=campaign_fullname,
target=target,
ecpm=ecpm,
priority=priority,
moat_query=moat_query,
imp_pixel=imp_pixel,
click_url=click_url,
upvote_pixel=upvote_pixel,
downvote_pixel=downvote_pixel,
))
def make_daily_promotions(offset=0, test=False):
"""
Arguments:
offset - number of days after today to get the schedule for
test - if True, new schedule will be generated but not launched
Raises Exception with list of campaigns that had errors if there were any
"""
by_srname, links, error_campaigns = get_scheduled(offset)
all_links = set([l._fullname for l in links])
srs = Subreddit._by_name(by_srname.keys())
# over18 check
for srname, adweights in by_srname.iteritems():
if srname:
sr = srs[srname]
if sr.over_18:
sr_links = Link._by_fullname([a.link for a in adweights],
return_dict=False)
for l in sr_links:
l.over_18 = True
if not test:
l._commit()
old_ads = get_live_promotions([LiveAdWeights.ALL_ADS])
old_links = set(x.link for x in old_ads[LiveAdWeights.ALL_ADS])
# links that need to be promoted
new_links = all_links - old_links
# links that have already been promoted
old_links = old_links - all_links
links = Link._by_fullname(new_links.union(old_links),
data=True,
return_dict=True)
for l in old_links:
if is_promoted(links[l]):
if test:
print "unpromote", l
else:
# update the query queue
set_promote_status(links[l], PROMOTE_STATUS.finished)
emailer.finished_promo(links[l])
for l in new_links:
if is_accepted(links[l]):
if test:
print "promote2", l
else:
# update the query queue
set_promote_status(links[l], PROMOTE_STATUS.promoted)
emailer.live_promo(links[l])
# convert the weighted dict to use sr_ids which are more useful
by_srid = {
srs[srname]._id: adweights
for srname, adweights in by_srname.iteritems() if srname != ''
}
if '' in by_srname:
by_srid[''] = by_srname['']
if not test:
set_live_promotions(by_srid)
_mark_promos_updated()
else:
print by_srid
# after launching as many campaigns as possible, raise an exception to
# report any error campaigns. (useful for triggering alerts in irc)
if error_campaigns:
raise Exception("Some scheduled campaigns could not be added to daily "
"promotions: %r" % error_campaigns)
def _handle_upsert_promotion(payload):
link = Link._by_fullname(payload["link"], data=True)
author = Account._byID(link.author_id)
creatives_service.upsert_creative(author, link)
def make_daily_promotions(offset=0, test=False):
"""
Arguments:
offset - number of days after today to get the schedule for
test - if True, new schedule will be generated but not launched
Raises Exception with list of campaigns that had errors if there were any
"""
scheduled_adweights, error_campaigns = get_scheduled(offset)
current_adweights_byid = get_live_promotions([LiveAdWeights.ALL_ADS])
current_adweights = current_adweights_byid[LiveAdWeights.ALL_ADS]
link_names = [aw.link for aw in itertools.chain(scheduled_adweights,
current_adweights)]
links = Link._by_fullname(link_names, data=True)
camp_names = [aw.campaign for aw in itertools.chain(scheduled_adweights,
current_adweights)]
campaigns = PromoCampaign._by_fullname(camp_names, data=True)
srs = Subreddit._by_name([camp.sr_name for camp in campaigns.itervalues()
if camp.sr_name])
expired_links = ({aw.link for aw in current_adweights} -
{aw.link for aw in scheduled_adweights})
for link_name in expired_links:
link = links[link_name]
if is_promoted(link):
if test:
print "unpromote", link_name
else:
# update the query queue
set_promote_status(link, PROMOTE_STATUS.finished)
emailer.finished_promo(link)
by_srid = defaultdict(list)
for adweight in scheduled_adweights:
link = links[adweight.link]
campaign = campaigns[adweight.campaign]
if campaign.sr_name:
sr = srs[campaign.sr_name]
sr_id = sr._id
sr_over_18 = sr.over_18
else:
sr_id = ''
sr_over_18 = False
if sr_over_18:
if test:
print "over18", link._fullname
else:
link.over_18 = True
link._commit()
if is_accepted(link) and not is_promoted(link):
if test:
print "promote2", link._fullname
else:
# update the query queue
set_promote_status(link, PROMOTE_STATUS.promoted)
emailer.live_promo(link)
by_srid[sr_id].append(adweight)
if not test:
set_live_promotions(by_srid)
_mark_promos_updated()
else:
print by_srid
finalize_completed_campaigns(daysago=offset+1)
hooks.get_hook('promote.make_daily_promotions').call(offset=offset)
# after launching as many campaigns as possible, raise an exception to
# report any error campaigns. (useful for triggering alerts in irc)
if error_campaigns:
raise Exception("Some scheduled campaigns could not be added to daily "
"promotions: %r" % error_campaigns)
def _get_selfserve_links(self, count):
links = Subreddit._by_name(g.advertising_links_sr).get_links('new', 'all')
items = Link._by_fullname(links, data=True, return_dict=False)
id36s = map(lambda x: self.advertising_link_id36_re.match(x.url).group(1), items)
ad_links = Link._byID36(id36s, return_dict=False, data=True)
return wrap_links(ad_links, num=count)
def make_daily_promotions(offset=0, test=False):
"""
Arguments:
offset - number of days after today to get the schedule for
test - if True, new schedule will be generated but not launched
Raises Exception with list of campaigns that had errors if there were any
"""
by_srname, links, error_campaigns = get_scheduled(offset)
all_links = set([l._fullname for l in links])
srs = Subreddit._by_name(by_srname.keys())
# over18 check
for srname, adweights in by_srname.iteritems():
if srname:
sr = srs[srname]
if sr.over_18:
sr_links = Link._by_fullname([a.link for a in adweights],
return_dict=False)
for l in sr_links:
l.over_18 = True
if not test:
l._commit()
old_ads = get_live_promotions([LiveAdWeights.ALL_ADS])
old_links = set(x.link for x in old_ads[LiveAdWeights.ALL_ADS])
# links that need to be promoted
new_links = all_links - old_links
# links that have already been promoted
old_links = old_links - all_links
links = Link._by_fullname(new_links.union(old_links), data=True,
return_dict=True)
for l in old_links:
if is_promoted(links[l]):
if test:
print "unpromote", l
else:
# update the query queue
set_promote_status(links[l], PROMOTE_STATUS.finished)
emailer.finished_promo(links[l])
for l in new_links:
if is_accepted(links[l]):
if test:
print "promote2", l
else:
# update the query queue
set_promote_status(links[l], PROMOTE_STATUS.promoted)
emailer.live_promo(links[l])
# convert the weighted dict to use sr_ids which are more useful
by_srid = {srs[srname]._id: adweights for srname, adweights
in by_srname.iteritems()
if srname != ''}
if '' in by_srname:
by_srid[''] = by_srname['']
if not test:
set_live_promotions(by_srid)
_mark_promos_updated()
else:
print by_srid
# after launching as many campaigns as possible, raise an exception to
# report any error campaigns. (useful for triggering alerts in irc)
if error_campaigns:
raise Exception("Some scheduled campaigns could not be added to daily "
"promotions: %r" % error_campaigns)
def make_daily_promotions(offset=0, test=False):
"""
Arguments:
offset - number of days after today to get the schedule for
test - if True, new schedule will be generated but not launched
Raises Exception with list of campaigns that had errors if there were any
"""
scheduled_adweights, error_campaigns = get_scheduled(offset)
current_adweights_byid = get_live_promotions([LiveAdWeights.ALL_ADS])
current_adweights = current_adweights_byid[LiveAdWeights.ALL_ADS]
link_names = [aw.link for aw in itertools.chain(scheduled_adweights,
current_adweights)]
links = Link._by_fullname(link_names, data=True)
camp_names = [aw.campaign for aw in itertools.chain(scheduled_adweights,
current_adweights)]
campaigns = PromoCampaign._by_fullname(camp_names, data=True)
srs = Subreddit._by_name([camp.sr_name for camp in campaigns.itervalues()
if camp.sr_name])
expired_links = ({aw.link for aw in current_adweights} -
{aw.link for aw in scheduled_adweights})
for link_name in expired_links:
link = links[link_name]
if is_promoted(link):
if test:
print "unpromote", link_name
else:
# update the query queue
set_promote_status(link, PROMOTE_STATUS.finished)
emailer.finished_promo(link)
by_srid = defaultdict(list)
for adweight in scheduled_adweights:
link = links[adweight.link]
campaign = campaigns[adweight.campaign]
if campaign.sr_name:
sr = srs[campaign.sr_name]
sr_id = sr._id
sr_over_18 = sr.over_18
else:
sr_id = ''
sr_over_18 = False
if sr_over_18:
if test:
print "over18", link._fullname
else:
link.over_18 = True
link._commit()
if is_accepted(link) and not is_promoted(link):
if test:
print "promote2", link._fullname
else:
# update the query queue
set_promote_status(link, PROMOTE_STATUS.promoted)
emailer.live_promo(link)
by_srid[sr_id].append(adweight)
if not test:
set_live_promotions(by_srid)
_mark_promos_updated()
else:
print by_srid
finalize_completed_campaigns(daysago=offset+1)
hooks.get_hook('promote.make_daily_promotions').call(offset=offset)
# after launching as many campaigns as possible, raise an exception to
# report any error campaigns. (useful for triggering alerts in irc)
if error_campaigns:
raise Exception("Some scheduled campaigns could not be added to daily "
"promotions: %r" % error_campaigns)
