def test_run_link(start_link, count=1000):
'''Inject `count` number of links, starting with `start_link`'''
if isinstance(start_link, basestring):
start_link = int(start_link, 36)
links = Link._byID(range(start_link - count, start_link),
data=True,
return_dict=False)
uploader = LinkUploader(g.CLOUDSEARCH_DOC_API, things=links)
return uploader.inject()
def get_promos(date, sr_names=None, link=None):
campaign_ids = PromotionWeights.get_campaign_ids(date,
sr_names=sr_names,
link=link)
campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False)
link_ids = {camp.link_id for camp in campaigns}
links = Link._byID(link_ids, data=True)
for camp in campaigns:
yield camp, links[camp.link_id]
def finalize_completed_campaigns(daysago=1):
# PromoCampaign.end_date is utc datetime with year, month, day only
now = datetime.datetime.now(g.tz)
date = now - datetime.timedelta(days=daysago)
date = date.replace(hour=0, minute=0, second=0, microsecond=0)
q = PromoCampaign._query(
PromoCampaign.c.end_date == date,
# exclude no transaction
PromoCampaign.c.trans_id != NO_TRANSACTION,
data=True)
# filter out freebies
campaigns = filter(lambda camp: camp.trans_id > NO_TRANSACTION, q)
if not campaigns:
return
# check that traffic is up to date
earliest_campaign = min(campaigns, key=lambda camp: camp.start_date)
start, end = get_total_run(earliest_campaign)
missing_traffic = traffic.get_missing_traffic(start.replace(tzinfo=None),
date.replace(tzinfo=None))
if missing_traffic:
raise ValueError("Can't finalize campaigns finished on %s."
"Missing traffic from %s" % (date, missing_traffic))
links = Link._byID([camp.link_id for camp in campaigns], data=True)
underdelivered_campaigns = []
for camp in campaigns:
if hasattr(camp, 'refund_amount'):
continue
link = links[camp.link_id]
billable_impressions = get_billable_impressions(camp)
billable_amount = get_billable_amount(camp, billable_impressions)
if billable_amount >= camp.total_budget_pennies:
if hasattr(camp, 'cpm'):
text = '%s completed with $%s billable (%s impressions @ $%s).'
text %= (camp, billable_amount, billable_impressions,
camp.bid_dollars)
else:
text = '%s completed with $%s billable (pre-CPM).'
text %= (camp, billable_amount)
PromotionLog.add(link, text)
camp.refund_amount = 0.
camp._commit()
elif charged_or_not_needed(camp):
underdelivered_campaigns.append(camp)
if underdelivered_campaigns:
queries.set_underdelivered_campaigns(underdelivered_campaigns)
def comment_event(self, new_comment, request=None, context=None):
"""Create a 'comment' event for event-collector.
new_comment: An v1.models.Comment object
request, context: Should be pylons.request & pylons.c respectively
"""
from v1.models import Comment, Link
event = Event(
topic="comment_events",
event_type="ss.comment",
time=new_comment._date,
request=request,
context=context,
truncatable_field="comment_body",
)
event.add("comment_id", new_comment._id)
event.add("comment_fullname", new_comment._fullname)
event.add_text("comment_body", new_comment.body)
post = Link._byID(new_comment.link_id)
event.add("post_id", post._id)
event.add("post_fullname", post._fullname)
event.add("post_created_ts", to_epoch_milliseconds(post._date))
if post.promoted:
event.add("post_is_promoted", bool(post.promoted))
if new_comment.parent_id:
parent = Comment._byID(new_comment.parent_id)
else:
# If this is a top-level comment, parent is the same as the post
parent = post
event.add("parent_id", parent._id)
event.add("parent_fullname", parent._fullname)
event.add("parent_created_ts", to_epoch_milliseconds(parent._date))
event.add("user_neutered", new_comment.author_slow._spam)
event.add_subverbify_fields(new_comment.subverbify_slow)
self.save_event(event)
def port_cassaurls(after_id=None, estimate=15231317):
from v1.models import Link, LinksByUrlAndSubverbify
from v1.lib.db import tdb_cassandra
from v1.lib.db.operators import desc
from v1.lib.db.tdb_cassandra import CL
from v1.lib.utils import fetch_things2, in_chunks, progress
q = Link._query(Link.c._spam == (True, False),
sort=desc('_date'),
data=True)
if after_id:
q._after(Link._byID(after_id, data=True))
q = fetch_things2(q, chunk_size=500)
q = progress(q, estimate=estimate)
q = (l for l in q if getattr(l, 'url', 'self') != 'self'
and not getattr(l, 'is_self', False))
chunks = in_chunks(q, 500)
for chunk in chunks:
for l in chunk:
LinksByUrlAndSubverbify.add_link(l)
def get_comment_items(srs, src, count=4):
"""Get hot links from srs, plus top comment from each link."""
link_fullnames = normalized_hot([sr._id for sr in srs])
hot_links = Link._by_fullname(link_fullnames[:count], return_dict=False)
top_comments = []
for link in hot_links:
builder = CommentBuilder(link,
operators.desc('_confidence'),
comment=None,
context=None,
num=1,
load_more=False)
listing = NestedListing(builder, parent_name=link._fullname).listing()
top_comments.extend(listing.things)
srs = Subverbify._byID([com.sr_id for com in top_comments])
links = Link._byID([com.link_id for com in top_comments])
comment_items = [ExploreItem(TYPE_COMMENT,
src,
srs[com.sr_id],
links[com.link_id],
com) for com in top_comments]
return comment_items
def add_props(cls, user, wrapped):
from v1.lib.db.thing import Thing
from v1.lib.menus import QueryButton
from v1.lib.pages import WrappedUser
from v1.models import (
Account,
Link,
ModSR,
MultiVerbify,
Subverbify,
)
target_names = {item.target_fullname for item in wrapped
if hasattr(item, "target_fullname")}
targets = Thing._by_fullname(target_names, data=True)
# get moderators
moderators = Account._byID36({item.mod_id36 for item in wrapped},
data=True)
# get authors for targets that are Links or Comments
target_author_names = {target.author_id for target in targets.values()
if hasattr(target, "author_id")}
target_authors = Account._byID(target_author_names, data=True)
# get parent links for targets that are Comments
parent_link_names = {target.link_id for target in targets.values()
if hasattr(target, "link_id")}
parent_links = Link._byID(parent_link_names, data=True)
# get subverbifys
srs = Subverbify._byID36({item.sr_id36 for item in wrapped}, data=True)
for item in wrapped:
item.moderator = moderators[item.mod_id36]
item.subverbify = srs[item.sr_id36]
item.text = cls._text.get(item.action, '')
item.target = None
item.target_author = None
if hasattr(item, "target_fullname") and item.target_fullname:
item.target = targets[item.target_fullname]
if hasattr(item.target, "author_id"):
author_name = item.target.author_id
item.target_author = target_authors[author_name]
if hasattr(item.target, "link_id"):
parent_link_name = item.target.link_id
item.parent_link = parent_links[parent_link_name]
if isinstance(item.target, Account):
item.target_author = item.target
if c.render_style == "html":
request_path = request.path
# make wrapped users for targets that are accounts
user_targets = filter(lambda target: isinstance(target, Account),
targets.values())
wrapped_user_targets = {user._fullname: WrappedUser(user)
for user in user_targets}
for item in wrapped:
if isinstance(item.target, Account):
user_name = item.target._fullname
item.wrapped_user_target = wrapped_user_targets[user_name]
css_class = 'modactions %s' % item.action
action_button = QueryButton(
'', item.action, query_param='type', css_class=css_class)
action_button.build(base_path=request_path)
item.action_button = action_button
mod_button = QueryButton(
item.moderator.name, item.moderator.name, query_param='mod')
mod_button.build(base_path=request_path)
item.mod_button = mod_button
if isinstance(c.site, ModSR) or isinstance(c.site, MultiVerbify):
rgb = item.subverbify.get_rgb()
item.bgcolor = 'rgb(%s,%s,%s)' % rgb
item.is_multi = True
else:
item.bgcolor = "rgb(255,255,255)"
item.is_multi = False
def process_message(msg):
from v1.lib.comment_tree import write_comment_scores
from v1.lib.db.queries import (
add_queries,
add_to_commentstree_q,
get_comments,
)
from v1.models.builder import get_active_sort_orders_for_link
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
timer = g.stats.get_timer("comment_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
comment = Comment._by_fullname(vote_data.pop("thing_fullname"))
print "Processing vote by %s on %s %s" % (user, comment, vote_data)
try:
vote = Vote(
user,
comment,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
vote_invalid = (not vote.effects.affects_score and
not vote.is_automatic_initial_vote)
comment_invalid = comment._spam or comment._deleted
if vote_invalid or comment_invalid:
timer.stop()
timer.flush()
return
author = Account._byID(comment.author_id)
add_queries(
queries=[get_comments(author, sort, 'all') for sort in SORTS],
insert_items=comment,
)
timer.intermediate("author_queries")
update_threshold = g.live_config['comment_vote_update_threshold']
update_period = g.live_config['comment_vote_update_period']
skip_score_update = (comment.num_votes > update_threshold and
comment.num_votes % update_period != 0)
# skip updating scores if this was the automatic initial vote. those
# updates will be handled by new_comment. Also only update scores
# periodically once a comment has many votes.
if not vote.is_automatic_initial_vote and not skip_score_update:
# check whether this link is using precomputed sorts, if it is
# we'll need to push an update to commentstree_q
link = Link._byID(comment.link_id)
if get_active_sort_orders_for_link(link):
# send this comment to commentstree_q where we will update
# CommentScoresByLink, CommentTree (noop), and CommentOrderer
add_to_commentstree_q(comment)
else:
# the link isn't using precomputed sorts, so just update the
# scores
write_comment_scores(link, [comment])
timer.intermediate("update_scores")
timer.stop()
timer.flush()