def process_message(msgs, chan):
"""Update get_domain_links(), the Links by domain precomputed query.
get_domain_links() is a CachedResult which is stored in permacache. To
update these objects we need to do a read-modify-write which requires
obtaining a lock. Sharding these updates by domain allows us to run
multiple consumers (but ideally just one per shard) to avoid lock
contention.
"""
from v1.lib.db.queries import add_queries, get_domain_links
link_names = {msg.body for msg in msgs}
links = Link._by_fullname(link_names, return_dict=False)
print 'Processing %r' % (links,)
links_by_domain = defaultdict(list)
for link in links:
parsed = UrlParser(link.url)
# update the listings for all permutations of the link's domain
for domain in parsed.domain_permutations():
links_by_domain[domain].append(link)
for d, links in links_by_domain.iteritems():
with g.stats.get_timer("link_vote_processor.domain_queries"):
add_queries(
queries=[
get_domain_links(d, sort, "all") for sort in SORTS],
insert_items=links,
)
def make_daily_promotions():
# charge campaigns so they can go live
charge_pending(offset=0)
charge_pending(offset=1)
# promote links and record ids of promoted links
link_ids = set()
for campaign, link in get_scheduled_promos(offset=0):
link_ids.add(link._id)
promote_link(link, campaign)
# expire finished links
q = Link._query(Link.c.promote_status == PROMOTE_STATUS.promoted,
data=True)
q = q._filter(not_(Link.c._id.in_(link_ids)))
for link in q:
update_promote_status(link, PROMOTE_STATUS.finished)
emailer.finished_promo(link)
# update subverbifys with promos
all_live_promo_srnames(_update=True)
_mark_promos_updated()
finalize_completed_campaigns(daysago=1)
hooks.get_hook('promote.make_daily_promotions').call(offset=0)
def new_promotion(is_self, title, content, author, ip):
"""
Creates a new promotion with the provided title, etc, and sets it
status to be 'unpaid'.
"""
sr = Subverbify._byID(Subverbify.get_promote_srid())
l = Link._submit(
is_self=is_self,
title=title,
content=content,
author=author,
sr=sr,
ip=ip,
)
l.promoted = True
l.disable_comments = False
l.sendreplies = True
PromotionLog.add(l, 'promotion created')
update_promote_status(l, PROMOTE_STATUS.unpaid)
# the user has posted a promotion, so enable the promote menu unless
# they have already opted out
if author.pref_show_promote is not False:
author.pref_show_promote = True
author._commit()
# notify of new promo
emailer.new_promo(l)
return l
def process_message(msgs, chan):
"""Update get_links(), the Links by Subverbify precomputed query.
get_links() is a CachedResult which is stored in permacache. To
update these objects we need to do a read-modify-write which requires
obtaining a lock. Sharding these updates by subverbify allows us to run
multiple consumers (but ideally just one per shard) to avoid lock
contention.
"""
from v1.lib.db.queries import add_queries, get_links
link_names = {msg.body for msg in msgs}
links = Link._by_fullname(link_names, return_dict=False)
print 'Processing %r' % (links,)
links_by_sr_id = defaultdict(list)
for link in links:
links_by_sr_id[link.sr_id].append(link)
srs_by_id = Subverbify._byID(links_by_sr_id.keys(), stale=True)
for sr_id, links in links_by_sr_id.iteritems():
with g.stats.get_timer("link_vote_processor.subverbify_queries"):
sr = srs_by_id[sr_id]
add_queries(
queries=[get_links(sr, sort, "all") for sort in SORTS],
insert_items=links,
)
def test_imgur_link(self):
post = Link(is_self=True,
selftext='''
Some text here.
https://example.com
https://imgur.com''')
url = _get_scrape_url(post)
self.assertEqual(url, 'https://imgur.com')
def test_simple_self_post(self):
post = Link(is_self=True,
selftext='''
Some text here.
https://example.com
https://verbify.com''')
url = _get_scrape_url(post)
self.assertEqual(url, 'https://example.com')
def get_hot_items(srs, item_type, src):
"""Get hot links from specified srs."""
hot_srs = {sr._id: sr for sr in srs} # for looking up sr by id
hot_link_fullnames = normalized_hot([sr._id for sr in srs])
hot_links = Link._by_fullname(hot_link_fullnames, return_dict=False)
hot_items = []
for l in hot_links:
hot_items.append(ExploreItem(item_type, src, hot_srs[l.sr_id], l))
return hot_items
def test_run_link(start_link, count=1000):
'''Inject `count` number of links, starting with `start_link`'''
if isinstance(start_link, basestring):
start_link = int(start_link, 36)
links = Link._byID(range(start_link - count, start_link),
data=True,
return_dict=False)
uploader = LinkUploader(g.CLOUDSEARCH_DOC_API, things=links)
return uploader.inject()
def hot_links_by_url_listing(url, sr=None, num=None, **kw):
try:
links_for_url = Link._by_url(url, sr)
except NotFound:
links_for_url = []
links_for_url.sort(key=lambda link: link._hot, reverse=True)
listing = wrap_links(links_for_url, num=num, **kw)
return listing
def get_promos(date, sr_names=None, link=None):
campaign_ids = PromotionWeights.get_campaign_ids(date,
sr_names=sr_names,
link=link)
campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False)
link_ids = {camp.link_id for camp in campaigns}
links = Link._byID(link_ids, data=True)
for camp in campaigns:
yield camp, links[camp.link_id]
def port_cassaurls(after_id=None, estimate=15231317):
from v1.models import Link, LinksByUrlAndSubverbify
from v1.lib.db import tdb_cassandra
from v1.lib.db.operators import desc
from v1.lib.db.tdb_cassandra import CL
from v1.lib.utils import fetch_things2, in_chunks, progress
q = Link._query(Link.c._spam == (True, False),
sort=desc('_date'),
data=True)
if after_id:
q._after(Link._byID(after_id, data=True))
q = fetch_things2(q, chunk_size=500)
q = progress(q, estimate=estimate)
q = (l for l in q if getattr(l, 'url', 'self') != 'self'
and not getattr(l, 'is_self', False))
chunks = in_chunks(q, 500)
for chunk in chunks:
for l in chunk:
LinksByUrlAndSubverbify.add_link(l)
def finalize_completed_campaigns(daysago=1):
# PromoCampaign.end_date is utc datetime with year, month, day only
now = datetime.datetime.now(g.tz)
date = now - datetime.timedelta(days=daysago)
date = date.replace(hour=0, minute=0, second=0, microsecond=0)
q = PromoCampaign._query(
PromoCampaign.c.end_date == date,
# exclude no transaction
PromoCampaign.c.trans_id != NO_TRANSACTION,
data=True)
# filter out freebies
campaigns = filter(lambda camp: camp.trans_id > NO_TRANSACTION, q)
if not campaigns:
return
# check that traffic is up to date
earliest_campaign = min(campaigns, key=lambda camp: camp.start_date)
start, end = get_total_run(earliest_campaign)
missing_traffic = traffic.get_missing_traffic(start.replace(tzinfo=None),
date.replace(tzinfo=None))
if missing_traffic:
raise ValueError("Can't finalize campaigns finished on %s."
"Missing traffic from %s" % (date, missing_traffic))
links = Link._byID([camp.link_id for camp in campaigns], data=True)
underdelivered_campaigns = []
for camp in campaigns:
if hasattr(camp, 'refund_amount'):
continue
link = links[camp.link_id]
billable_impressions = get_billable_impressions(camp)
billable_amount = get_billable_amount(camp, billable_impressions)
if billable_amount >= camp.total_budget_pennies:
if hasattr(camp, 'cpm'):
text = '%s completed with $%s billable (%s impressions @ $%s).'
text %= (camp, billable_amount, billable_impressions,
camp.bid_dollars)
else:
text = '%s completed with $%s billable (pre-CPM).'
text %= (camp, billable_amount)
PromotionLog.add(link, text)
camp.refund_amount = 0.
camp._commit()
elif charged_or_not_needed(camp):
underdelivered_campaigns.append(camp)
if underdelivered_campaigns:
queries.set_underdelivered_campaigns(underdelivered_campaigns)
def get_rising_items(omit_sr_ids, count=4):
"""Get links that are rising right now."""
all_rising = rising.get_all_rising()
candidate_sr_ids = {sr_id for link, score, sr_id in all_rising}.difference(omit_sr_ids)
link_fullnames = [link for link, score, sr_id in all_rising if sr_id in candidate_sr_ids]
link_fullnames_to_show = random_sample(link_fullnames, count)
rising_links = Link._by_fullname(link_fullnames_to_show,
return_dict=False,
data=True)
rising_items = [ExploreItem(TYPE_RISING, 'ris', Subverbify._byID(l.sr_id), l)
for l in rising_links]
return rising_items
def process_message(msg):
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
timer = g.stats.get_timer("link_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
link = Link._by_fullname(vote_data.pop("thing_fullname"))
# create the vote and update the voter's liked/disliked under lock so
# that the vote state and cached query are consistent
lock_key = "vote-%s-%s" % (user._id36, link._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, link, vote_data)
try:
vote = Vote(
user,
link,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
update_user_liked(vote)
timer.intermediate("voter_likes")
vote_valid = vote.is_automatic_initial_vote or vote.effects.affects_score
link_valid = not (link._spam or link._deleted)
if vote_valid and link_valid:
add_to_author_query_q(link)
add_to_subverbify_query_q(link)
add_to_domain_query_q(link)
timer.stop()
timer.flush()
def get_comment_items(srs, src, count=4):
"""Get hot links from srs, plus top comment from each link."""
link_fullnames = normalized_hot([sr._id for sr in srs])
hot_links = Link._by_fullname(link_fullnames[:count], return_dict=False)
top_comments = []
for link in hot_links:
builder = CommentBuilder(link,
operators.desc('_confidence'),
comment=None,
context=None,
num=1,
load_more=False)
listing = NestedListing(builder, parent_name=link._fullname).listing()
top_comments.extend(listing.things)
srs = Subverbify._byID([com.sr_id for com in top_comments])
links = Link._byID([com.link_id for com in top_comments])
comment_items = [ExploreItem(TYPE_COMMENT,
src,
srs[com.sr_id],
links[com.link_id],
com) for com in top_comments]
return comment_items
def add_target_fields(self, target):
if not target:
return
from v1.models import Comment, Link, Message
self.add("target_id", target._id)
self.add("target_fullname", target._fullname)
self.add("target_age_seconds", target._age.total_seconds())
target_type = target.__class__.__name__.lower()
if target_type == "link" and target.is_self:
target_type = "self"
self.add("target_type", target_type)
# If the target is an Account or Subverbify (or has a "name" attr),
# add the target_name
if hasattr(target, "name"):
self.add("target_name", target.name)
# Add info about the target's author for comments, links, & messages
if isinstance(target, (Comment, Link, Message)):
author = target.author_slow
if target._deleted or author._deleted:
self.add("target_author_id", 0)
self.add("target_author_name", "[deleted]")
else:
self.add("target_author_id", author._id)
self.add("target_author_name", author.name)
# Add info about the url being linked to for link posts
if isinstance(target, Link):
self.add_text("target_title", target.title)
if not target.is_self:
self.add("target_url", target.url)
self.add("target_url_domain", target.link_domain())
# Add info about the link being commented on for comments
if isinstance(target, Comment):
link_fullname = Link._fullname_from_id36(to36(target.link_id))
self.add("link_id", target.link_id)
self.add("link_fullname", link_fullname)
# Add info about when target was originally posted for links/comments
if isinstance(target, (Comment, Link)):
self.add("target_created_ts", to_epoch_milliseconds(target._date))
hooks.get_hook("eventcollector.add_target_fields").call(
event=self,
target=target,
)
def test_image_link(self):
post = Link(is_self=True,
selftext='''
Some text here.
https://example.com
https://verbify.com/a.jpg''')
url = _get_scrape_url(post)
self.assertEqual(url, 'https://verbify.com/a.jpg')
post = Link(is_self=True,
selftext='''
Some text here.
https://example.com
https://verbify.com/a.PNG''')
url = _get_scrape_url(post)
self.assertEqual(url, 'https://verbify.com/a.PNG')
post = Link(is_self=True,
selftext='''
Some text here.
https://example.com
https://verbify.com/a.jpg/b''')
url = _get_scrape_url(post)
self.assertEqual(url, 'https://example.com')
def _mock_link(self, id=1, author_id=1, sr_id=1, can_comment=True,
can_view_promo=True, **kwargs):
kwargs['id'] = id
kwargs['author_id'] = author_id
kwargs['sr_id'] = sr_id
link = Link(**kwargs)
self.autopatch(VByName, "run", return_value=link)
sr = Subverbify(id=sr_id)
self.autopatch(Subverbify, "_byID", return_value=sr)
self.autopatch(Subverbify, "can_comment", return_value=can_comment)
self.autopatch(Link, "can_view_promo", return_value=can_view_promo)
return link
def port_deleted_links(after_id=None):
from v1.models import Link
from v1.lib.db.operators import desc
from v1.models.query_cache import CachedQueryMutator
from v1.lib.db.queries import get_deleted_links
from v1.lib.utils import fetch_things2, in_chunks, progress
q = Link._query(Link.c._deleted == True,
Link.c._spam == (True, False),
sort=desc('_date'),
data=True)
q = fetch_things2(q, chunk_size=500)
q = progress(q, verbosity=1000)
for chunk in in_chunks(q):
with CachedQueryMutator() as m:
for link in chunk:
query = get_deleted_links(link.author_id)
m.insert(query, [link])
def comment_event(self, new_comment, request=None, context=None):
"""Create a 'comment' event for event-collector.
new_comment: An v1.models.Comment object
request, context: Should be pylons.request & pylons.c respectively
"""
from v1.models import Comment, Link
event = Event(
topic="comment_events",
event_type="ss.comment",
time=new_comment._date,
request=request,
context=context,
truncatable_field="comment_body",
)
event.add("comment_id", new_comment._id)
event.add("comment_fullname", new_comment._fullname)
event.add_text("comment_body", new_comment.body)
post = Link._byID(new_comment.link_id)
event.add("post_id", post._id)
event.add("post_fullname", post._fullname)
event.add("post_created_ts", to_epoch_milliseconds(post._date))
if post.promoted:
event.add("post_is_promoted", bool(post.promoted))
if new_comment.parent_id:
parent = Comment._byID(new_comment.parent_id)
else:
# If this is a top-level comment, parent is the same as the post
parent = post
event.add("parent_id", parent._id)
event.add("parent_fullname", parent._fullname)
event.add("parent_created_ts", to_epoch_milliseconds(parent._date))
event.add("user_neutered", new_comment.author_slow._spam)
event.add_subverbify_fields(new_comment.subverbify_slow)
self.save_event(event)
def _mock_comment(self,
id=1, author_id=1, link_id=1, sr_id=1, can_comment=True,
can_view_promo=True, is_moderator=False, **kwargs):
kwargs['id'] = id
kwargs['author_id'] = author_id
kwargs['link_id'] = link_id
kwargs['sr_id'] = sr_id
comment = Comment(**kwargs)
self.autopatch(VByName, "run", return_value=comment)
link = Link(id=link_id, sr_id=sr_id)
self.autopatch(Link, "_byID", return_value=link)
sr = Subverbify(id=sr_id)
self.autopatch(Subverbify, "_byID", return_value=sr)
self.autopatch(Subverbify, "can_comment", return_value=can_comment)
self.autopatch(Link, "can_view_promo", return_value=can_view_promo)
self.autopatch(Subverbify, "is_moderator", return_value=is_moderator)
return comment
def get_scheduled(date, sr_name=''):
campaign_ids = PromotionWeights.get_campaign_ids(date, sr_names=[sr_name])
campaigns = PromoCampaign._byID(campaign_ids, return_dict=False, data=True)
links = Link._by_fullname({camp.link_id
for camp in campaigns},
return_dict=False,
data=True)
links = {l._id: l for l in links}
kept = []
for camp in campaigns:
if camp.trans_id == 0:
continue
link = links[camp.link_id]
if link._spam or not promote.is_accepted(link):
continue
kept.append(camp._id)
return [(camp._fullname, camp.link_id, camp.total_budget_dollars)
for camp in kept]
def gen_keys():
yield promoted_memo_key
# just let this one do its own writing
load_all_verbifys()
yield queries.get_all_comments().iden
l_q = Link._query(
Link.c._spam == (True, False),
Link.c._deleted == (True, False),
sort=desc('_date'),
data=True,
)
for link in fetch_things2(l_q, verbosity):
yield comments_key(link._id)
yield last_modified_key(link, 'comments')
a_q = Account._query(
Account.c._spam == (True, False),
sort=desc('_date'),
)
for account in fetch_things2(a_q, verbosity):
yield messages_key(account._id)
yield last_modified_key(account, 'overview')
yield last_modified_key(account, 'commented')
yield last_modified_key(account, 'submitted')
yield last_modified_key(account, 'liked')
yield last_modified_key(account, 'disliked')
yield queries.get_comments(account, 'new', 'all').iden
yield queries.get_submitted(account, 'new', 'all').iden
yield queries.get_liked(account).iden
yield queries.get_disliked(account).iden
yield queries.get_hidden(account).iden
yield queries.get_saved(account).iden
yield queries.get_inbox_messages(account).iden
yield queries.get_unread_messages(account).iden
yield queries.get_inbox_comments(account).iden
yield queries.get_unread_comments(account).iden
yield queries.get_inbox_selfreply(account).iden
yield queries.get_unread_selfreply(account).iden
yield queries.get_sent(account).iden
sr_q = Subverbify._query(
Subverbify.c._spam == (True, False),
sort=desc('_date'),
)
for sr in fetch_things2(sr_q, verbosity):
yield last_modified_key(sr, 'stylesheet_contents')
yield queries.get_links(sr, 'hot', 'all').iden
yield queries.get_links(sr, 'new', 'all').iden
for sort in 'top', 'controversial':
for time in 'hour', 'day', 'week', 'month', 'year', 'all':
yield queries.get_links(sr,
sort,
time,
merge_batched=False).iden
yield queries.get_spam_links(sr).iden
yield queries.get_spam_comments(sr).iden
yield queries.get_reported_links(sr).iden
yield queries.get_reported_comments(sr).iden
yield queries.get_subverbify_messages(sr).iden
yield queries.get_unread_subverbify_messages(sr).iden
def add_props(cls, user, wrapped):
from v1.lib.db.thing import Thing
from v1.lib.menus import QueryButton
from v1.lib.pages import WrappedUser
from v1.models import (
Account,
Link,
ModSR,
MultiVerbify,
Subverbify,
)
target_names = {item.target_fullname for item in wrapped
if hasattr(item, "target_fullname")}
targets = Thing._by_fullname(target_names, data=True)
# get moderators
moderators = Account._byID36({item.mod_id36 for item in wrapped},
data=True)
# get authors for targets that are Links or Comments
target_author_names = {target.author_id for target in targets.values()
if hasattr(target, "author_id")}
target_authors = Account._byID(target_author_names, data=True)
# get parent links for targets that are Comments
parent_link_names = {target.link_id for target in targets.values()
if hasattr(target, "link_id")}
parent_links = Link._byID(parent_link_names, data=True)
# get subverbifys
srs = Subverbify._byID36({item.sr_id36 for item in wrapped}, data=True)
for item in wrapped:
item.moderator = moderators[item.mod_id36]
item.subverbify = srs[item.sr_id36]
item.text = cls._text.get(item.action, '')
item.target = None
item.target_author = None
if hasattr(item, "target_fullname") and item.target_fullname:
item.target = targets[item.target_fullname]
if hasattr(item.target, "author_id"):
author_name = item.target.author_id
item.target_author = target_authors[author_name]
if hasattr(item.target, "link_id"):
parent_link_name = item.target.link_id
item.parent_link = parent_links[parent_link_name]
if isinstance(item.target, Account):
item.target_author = item.target
if c.render_style == "html":
request_path = request.path
# make wrapped users for targets that are accounts
user_targets = filter(lambda target: isinstance(target, Account),
targets.values())
wrapped_user_targets = {user._fullname: WrappedUser(user)
for user in user_targets}
for item in wrapped:
if isinstance(item.target, Account):
user_name = item.target._fullname
item.wrapped_user_target = wrapped_user_targets[user_name]
css_class = 'modactions %s' % item.action
action_button = QueryButton(
'', item.action, query_param='type', css_class=css_class)
action_button.build(base_path=request_path)
item.action_button = action_button
mod_button = QueryButton(
item.moderator.name, item.moderator.name, query_param='mod')
mod_button.build(base_path=request_path)
item.mod_button = mod_button
if isinstance(c.site, ModSR) or isinstance(c.site, MultiVerbify):
rgb = item.subverbify.get_rgb()
item.bgcolor = 'rgb(%s,%s,%s)' % rgb
item.is_multi = True
else:
item.bgcolor = "rgb(255,255,255)"
item.is_multi = False
def get_link_counts(period = count_period):
links = Link._query(Link.c._date >= utils.timeago(period),
limit=50, data = True)
return dict((l._fullname, (0, l.sr_id)) for l in links)
def inject_test_data(num_links=25, num_comments=25, num_votes=5):
"""Flood your verbify install with test data based on verbify.com."""
print ">>>> Ensuring configured objects exist"
system_user = ensure_account(g.system_user)
ensure_account(g.automoderator_account)
ensure_subverbify(g.default_sr, system_user)
ensure_subverbify(g.takedown_sr, system_user)
ensure_subverbify(g.beta_sr, system_user)
ensure_subverbify(g.promo_sr_name, system_user)
print
print
print ">>>> Fetching real data from verbify.com"
modeler = Modeler()
subverbifys = [
modeler.model_subverbify("pics"),
modeler.model_subverbify("videos"),
modeler.model_subverbify("askhistorians"),
]
extra_settings = {
"pics": {
"show_media": True,
},
"videos": {
"show_media": True,
},
}
print
print
print ">>>> Generating test data"
print ">>> Accounts"
account_query = Account._query(sort="_date", limit=500, data=True)
accounts = [a for a in account_query if a.name != g.system_user]
accounts.extend(
ensure_account(modeler.generate_username())
for i in xrange(50 - len(accounts)))
print ">>> Content"
things = []
for sr_model in subverbifys:
sr_author = random.choice(accounts)
sr = ensure_subverbify(sr_model.name, sr_author)
# make the system user subscribed for easier testing
if sr.add_subscriber(system_user):
sr._incr("_ups", 1)
# apply any custom config we need for this sr
for setting, value in extra_settings.get(sr.name, {}).iteritems():
setattr(sr, setting, value)
sr._commit()
for i in xrange(num_links):
link_author = random.choice(accounts)
url = sr_model.generate_link_url()
is_self = (url == "self")
content = sr_model.generate_selfpost_body() if is_self else url
link = Link._submit(
is_self=is_self,
title=sr_model.generate_link_title(),
content=content,
author=link_author,
sr=sr,
ip="127.0.0.1",
)
queries.new_link(link)
things.append(link)
comments = [None]
for i in xrange(fuzz_number(num_comments)):
comment_author = random.choice(accounts)
comment, inbox_rel = Comment._new(
comment_author,
link,
parent=random.choice(comments),
body=sr_model.generate_comment_body(),
ip="127.0.0.1",
)
queries.new_comment(comment, inbox_rel)
comments.append(comment)
things.append(comment)
for thing in things:
for i in xrange(fuzz_number(num_votes)):
direction = random.choice([
Vote.DIRECTIONS.up,
Vote.DIRECTIONS.unvote,
Vote.DIRECTIONS.down,
])
voter = random.choice(accounts)
cast_vote(voter, thing, direction)
amqp.worker.join()
srs = [Subverbify._by_name(n) for n in ("pics", "videos", "askhistorians")]
LocalizedDefaultSubverbifys.set_global_srs(srs)
LocalizedFeaturedSubverbifys.set_global_srs([Subverbify._by_name('pics')])
from v1.lib.db.operators import desc
from v1.lib.utils import fetch_things2
from v1.models import (
calculate_server_seconds,
Comment,
Link,
Subverbify,
)
LINK_SILDING_START = datetime(2014, 2, 1, 0, 0, tzinfo=g.tz)
COMMENT_SILDING_START = datetime(2012, 10, 1, 0, 0, tzinfo=g.tz)
queries = [
Link._query(
Link.c.sildings != 0,
Link.c._date > LINK_SILDING_START,
data=True,
sort=desc('_date'),
),
Comment._query(
Comment.c.sildings != 0,
Comment.c._date > COMMENT_SILDING_START,
data=True,
sort=desc('_date'),
),
]
seconds_by_srid = defaultdict(int)
silding_price = g.sodium_month_price.pennies
for q in queries:
for things in fetch_things2(q, chunks=True, chunk_size=100):
def test_link_post(self, Link):
post = Link()
post.url = 'https://example.com'
post.is_self = False
url = _get_scrape_url(post)
self.assertEqual(url, 'https://example.com')
def process_message(msg):
from v1.lib.comment_tree import write_comment_scores
from v1.lib.db.queries import (
add_queries,
add_to_commentstree_q,
get_comments,
)
from v1.models.builder import get_active_sort_orders_for_link
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
timer = g.stats.get_timer("comment_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
comment = Comment._by_fullname(vote_data.pop("thing_fullname"))
print "Processing vote by %s on %s %s" % (user, comment, vote_data)
try:
vote = Vote(
user,
comment,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
vote_invalid = (not vote.effects.affects_score and
not vote.is_automatic_initial_vote)
comment_invalid = comment._spam or comment._deleted
if vote_invalid or comment_invalid:
timer.stop()
timer.flush()
return
author = Account._byID(comment.author_id)
add_queries(
queries=[get_comments(author, sort, 'all') for sort in SORTS],
insert_items=comment,
)
timer.intermediate("author_queries")
update_threshold = g.live_config['comment_vote_update_threshold']
update_period = g.live_config['comment_vote_update_period']
skip_score_update = (comment.num_votes > update_threshold and
comment.num_votes % update_period != 0)
# skip updating scores if this was the automatic initial vote. those
# updates will be handled by new_comment. Also only update scores
# periodically once a comment has many votes.
if not vote.is_automatic_initial_vote and not skip_score_update:
# check whether this link is using precomputed sorts, if it is
# we'll need to push an update to commentstree_q
link = Link._byID(comment.link_id)
if get_active_sort_orders_for_link(link):
# send this comment to commentstree_q where we will update
# CommentScoresByLink, CommentTree (noop), and CommentOrderer
add_to_commentstree_q(comment)
else:
# the link isn't using precomputed sorts, so just update the
# scores
write_comment_scores(link, [comment])
timer.intermediate("update_scores")
timer.stop()
timer.flush()