def test_run_link(start_link, count=1000): '''Inject `count` number of links, starting with `start_link`''' if isinstance(start_link, basestring): start_link = int(start_link, 36) links = Link._byID(range(start_link - count, start_link), data=True, return_dict=False) uploader = LinkUploader(g.CLOUDSEARCH_DOC_API, things=links) return uploader.inject()
def get_promos(date, sr_names=None, link=None): campaign_ids = PromotionWeights.get_campaign_ids(date, sr_names=sr_names, link=link) campaigns = PromoCampaign._byID(campaign_ids, data=True, return_dict=False) link_ids = {camp.link_id for camp in campaigns} links = Link._byID(link_ids, data=True) for camp in campaigns: yield camp, links[camp.link_id]
def finalize_completed_campaigns(daysago=1): # PromoCampaign.end_date is utc datetime with year, month, day only now = datetime.datetime.now(g.tz) date = now - datetime.timedelta(days=daysago) date = date.replace(hour=0, minute=0, second=0, microsecond=0) q = PromoCampaign._query( PromoCampaign.c.end_date == date, # exclude no transaction PromoCampaign.c.trans_id != NO_TRANSACTION, data=True) # filter out freebies campaigns = filter(lambda camp: camp.trans_id > NO_TRANSACTION, q) if not campaigns: return # check that traffic is up to date earliest_campaign = min(campaigns, key=lambda camp: camp.start_date) start, end = get_total_run(earliest_campaign) missing_traffic = traffic.get_missing_traffic(start.replace(tzinfo=None), date.replace(tzinfo=None)) if missing_traffic: raise ValueError("Can't finalize campaigns finished on %s." "Missing traffic from %s" % (date, missing_traffic)) links = Link._byID([camp.link_id for camp in campaigns], data=True) underdelivered_campaigns = [] for camp in campaigns: if hasattr(camp, 'refund_amount'): continue link = links[camp.link_id] billable_impressions = get_billable_impressions(camp) billable_amount = get_billable_amount(camp, billable_impressions) if billable_amount >= camp.total_budget_pennies: if hasattr(camp, 'cpm'): text = '%s completed with $%s billable (%s impressions @ $%s).' text %= (camp, billable_amount, billable_impressions, camp.bid_dollars) else: text = '%s completed with $%s billable (pre-CPM).' text %= (camp, billable_amount) PromotionLog.add(link, text) camp.refund_amount = 0. camp._commit() elif charged_or_not_needed(camp): underdelivered_campaigns.append(camp) if underdelivered_campaigns: queries.set_underdelivered_campaigns(underdelivered_campaigns)
def comment_event(self, new_comment, request=None, context=None): """Create a 'comment' event for event-collector. new_comment: An v1.models.Comment object request, context: Should be pylons.request & pylons.c respectively """ from v1.models import Comment, Link event = Event( topic="comment_events", event_type="ss.comment", time=new_comment._date, request=request, context=context, truncatable_field="comment_body", ) event.add("comment_id", new_comment._id) event.add("comment_fullname", new_comment._fullname) event.add_text("comment_body", new_comment.body) post = Link._byID(new_comment.link_id) event.add("post_id", post._id) event.add("post_fullname", post._fullname) event.add("post_created_ts", to_epoch_milliseconds(post._date)) if post.promoted: event.add("post_is_promoted", bool(post.promoted)) if new_comment.parent_id: parent = Comment._byID(new_comment.parent_id) else: # If this is a top-level comment, parent is the same as the post parent = post event.add("parent_id", parent._id) event.add("parent_fullname", parent._fullname) event.add("parent_created_ts", to_epoch_milliseconds(parent._date)) event.add("user_neutered", new_comment.author_slow._spam) event.add_subverbify_fields(new_comment.subverbify_slow) self.save_event(event)
def port_cassaurls(after_id=None, estimate=15231317): from v1.models import Link, LinksByUrlAndSubverbify from v1.lib.db import tdb_cassandra from v1.lib.db.operators import desc from v1.lib.db.tdb_cassandra import CL from v1.lib.utils import fetch_things2, in_chunks, progress q = Link._query(Link.c._spam == (True, False), sort=desc('_date'), data=True) if after_id: q._after(Link._byID(after_id, data=True)) q = fetch_things2(q, chunk_size=500) q = progress(q, estimate=estimate) q = (l for l in q if getattr(l, 'url', 'self') != 'self' and not getattr(l, 'is_self', False)) chunks = in_chunks(q, 500) for chunk in chunks: for l in chunk: LinksByUrlAndSubverbify.add_link(l)
def get_comment_items(srs, src, count=4): """Get hot links from srs, plus top comment from each link.""" link_fullnames = normalized_hot([sr._id for sr in srs]) hot_links = Link._by_fullname(link_fullnames[:count], return_dict=False) top_comments = [] for link in hot_links: builder = CommentBuilder(link, operators.desc('_confidence'), comment=None, context=None, num=1, load_more=False) listing = NestedListing(builder, parent_name=link._fullname).listing() top_comments.extend(listing.things) srs = Subverbify._byID([com.sr_id for com in top_comments]) links = Link._byID([com.link_id for com in top_comments]) comment_items = [ExploreItem(TYPE_COMMENT, src, srs[com.sr_id], links[com.link_id], com) for com in top_comments] return comment_items
def add_props(cls, user, wrapped): from v1.lib.db.thing import Thing from v1.lib.menus import QueryButton from v1.lib.pages import WrappedUser from v1.models import ( Account, Link, ModSR, MultiVerbify, Subverbify, ) target_names = {item.target_fullname for item in wrapped if hasattr(item, "target_fullname")} targets = Thing._by_fullname(target_names, data=True) # get moderators moderators = Account._byID36({item.mod_id36 for item in wrapped}, data=True) # get authors for targets that are Links or Comments target_author_names = {target.author_id for target in targets.values() if hasattr(target, "author_id")} target_authors = Account._byID(target_author_names, data=True) # get parent links for targets that are Comments parent_link_names = {target.link_id for target in targets.values() if hasattr(target, "link_id")} parent_links = Link._byID(parent_link_names, data=True) # get subverbifys srs = Subverbify._byID36({item.sr_id36 for item in wrapped}, data=True) for item in wrapped: item.moderator = moderators[item.mod_id36] item.subverbify = srs[item.sr_id36] item.text = cls._text.get(item.action, '') item.target = None item.target_author = None if hasattr(item, "target_fullname") and item.target_fullname: item.target = targets[item.target_fullname] if hasattr(item.target, "author_id"): author_name = item.target.author_id item.target_author = target_authors[author_name] if hasattr(item.target, "link_id"): parent_link_name = item.target.link_id item.parent_link = parent_links[parent_link_name] if isinstance(item.target, Account): item.target_author = item.target if c.render_style == "html": request_path = request.path # make wrapped users for targets that are accounts user_targets = filter(lambda target: isinstance(target, Account), targets.values()) wrapped_user_targets = {user._fullname: WrappedUser(user) for user in user_targets} for item in wrapped: if isinstance(item.target, Account): user_name = item.target._fullname item.wrapped_user_target = wrapped_user_targets[user_name] css_class = 'modactions %s' % item.action action_button = QueryButton( '', item.action, query_param='type', css_class=css_class) action_button.build(base_path=request_path) item.action_button = action_button mod_button = QueryButton( item.moderator.name, item.moderator.name, query_param='mod') mod_button.build(base_path=request_path) item.mod_button = mod_button if isinstance(c.site, ModSR) or isinstance(c.site, MultiVerbify): rgb = item.subverbify.get_rgb() item.bgcolor = 'rgb(%s,%s,%s)' % rgb item.is_multi = True else: item.bgcolor = "rgb(255,255,255)" item.is_multi = False
def process_message(msg): from v1.lib.comment_tree import write_comment_scores from v1.lib.db.queries import ( add_queries, add_to_commentstree_q, get_comments, ) from v1.models.builder import get_active_sort_orders_for_link vote_data = json.loads(msg.body) hook = hooks.get_hook('vote.validate_vote_data') if hook.call_until_return(msg=msg, vote_data=vote_data) is False: # Corrupt records in the queue. Ignore them. print "Ignoring invalid vote by %s on %s %s" % ( vote_data.get('user_id', '<unknown>'), vote_data.get('thing_fullname', '<unknown>'), vote_data) return timer = g.stats.get_timer("comment_vote_processor") timer.start() user = Account._byID(vote_data.pop("user_id")) comment = Comment._by_fullname(vote_data.pop("thing_fullname")) print "Processing vote by %s on %s %s" % (user, comment, vote_data) try: vote = Vote( user, comment, direction=vote_data["direction"], date=datetime.utcfromtimestamp(vote_data["date"]), data=vote_data["data"], event_data=vote_data.get("event_data"), ) except TypeError as e: # a vote on an invalid type got in the queue, just skip it g.log.exception("Invalid type: %r", e.message) return vote.commit() timer.intermediate("create_vote_object") vote_invalid = (not vote.effects.affects_score and not vote.is_automatic_initial_vote) comment_invalid = comment._spam or comment._deleted if vote_invalid or comment_invalid: timer.stop() timer.flush() return author = Account._byID(comment.author_id) add_queries( queries=[get_comments(author, sort, 'all') for sort in SORTS], insert_items=comment, ) timer.intermediate("author_queries") update_threshold = g.live_config['comment_vote_update_threshold'] update_period = g.live_config['comment_vote_update_period'] skip_score_update = (comment.num_votes > update_threshold and comment.num_votes % update_period != 0) # skip updating scores if this was the automatic initial vote. those # updates will be handled by new_comment. Also only update scores # periodically once a comment has many votes. if not vote.is_automatic_initial_vote and not skip_score_update: # check whether this link is using precomputed sorts, if it is # we'll need to push an update to commentstree_q link = Link._byID(comment.link_id) if get_active_sort_orders_for_link(link): # send this comment to commentstree_q where we will update # CommentScoresByLink, CommentTree (noop), and CommentOrderer add_to_commentstree_q(comment) else: # the link isn't using precomputed sorts, so just update the # scores write_comment_scores(link, [comment]) timer.intermediate("update_scores") timer.stop() timer.flush()