def _restrict_sr(sr): '''Return a cloudsearch appropriate query string that restricts results to only contain results from self.sr ''' if isinstance(sr, MultiReddit): if not sr.sr_ids: raise InvalidQuery srs = ["sr_id:%s" % sr_id for sr_id in sr.sr_ids] return "(or %s)" % ' '.join(srs) elif isinstance(sr, DomainSR): return "site:'%s'" % sr.domain elif isinstance(sr, FriendsSR): if not c.user_is_loggedin or not c.user.friends: raise InvalidQuery # The query limit is roughly 8k bytes. Limit to 200 friends to # avoid getting too close to that limit friend_ids = c.user.friends[:200] friends = ["author_fullname:'%s'" % Account._fullname_from_id36(r2utils.to36(id_)) for id_ in friend_ids] return "(or %s)" % ' '.join(friends) elif not isinstance(sr, FakeSubreddit): return "sr_id:%s" % sr._id return None
def _get_sr_restriction(sr): '''Return a solr-appropriate query string that restricts results to only contain results from sr ''' bq = [] if (not sr) or sr == All or isinstance(sr, DefaultSR): return None elif isinstance(sr, MultiReddit): for sr_id in sr.sr_ids: bq.append("sr_id:%s" % sr_id) elif isinstance(sr, DomainSR): bq = ["site:'%s'" % sr.domain] elif sr == Friends: if not c.user_is_loggedin or not c.user.friends: return None friend_ids = c.user.friends friends = [ "author_fullname:'%s'" % Account._fullname_from_id36(r2utils.to36(id_)) for id_ in friend_ids ] bq.extend(friends) elif isinstance(sr, AllMinus): for sr_id in sr.exclude_sr_ids: bq.append("-sr_id:%s" % sr_id) elif not isinstance(sr, FakeSubreddit): bq = ["sr_id:%s" % sr._id] return ' OR '.join(bq)
def _get_sr_restriction(sr): """Return a cloudsearch appropriate query string that restricts results to only contain results from self.sr """ bq = [] if (not sr) or sr == All or isinstance(sr, DefaultSR): return None elif isinstance(sr, MultiReddit): bq = ["(or"] for sr_id in sr.sr_ids: bq.append("sr_id:%s" % sr_id) bq.append(")") elif isinstance(sr, DomainSR): bq = ["site:'%s'" % sr.domain] elif sr == Friends: if not c.user_is_loggedin or not c.user.friends: return None bq = ["(or"] # The query limit is roughly 8k bytes. Limit to 200 friends to # avoid getting too close to that limit friend_ids = c.user.friends[:200] friends = ["author_fullname:'%s'" % Account._fullname_from_id36(r2utils.to36(id_)) for id_ in friend_ids] bq.extend(friends) bq.append(")") elif isinstance(sr, ModContribSR): bq = ["(or"] for sr_id in sr.sr_ids: bq.append("sr_id:%s" % sr_id) bq.append(")") elif not isinstance(sr, FakeSubreddit): bq = ["sr_id:%s" % sr._id] return " ".join(bq)
def _get_sr_restriction(sr): '''Return a solr-appropriate query string that restricts results to only contain results from self.sr ''' bq = [] if (not sr) or sr == All or isinstance(sr, DefaultSR): return None elif isinstance(sr, MultiReddit): for sr_id in sr.sr_ids: bq.append("sr_id:%s" % sr_id) elif isinstance(sr, DomainSR): bq = ["site:'%s'" % sr.domain] elif sr == Friends: if not c.user_is_loggedin or not c.user.friends: return None friend_ids = c.user.friends friends = ["author_fullname:'%s'" % Account._fullname_from_id36(r2utils.to36(id_)) for id_ in friend_ids] bq.extend(friends) elif isinstance(sr, ModContribSR): for sr_id in sr.sr_ids: bq.append("sr_id:%s" % sr_id) elif not isinstance(sr, FakeSubreddit): bq = ["sr_id:%s" % sr._id] return ' OR '.join(bq)
def _restrict_sr(sr): '''Return a cloudsearch appropriate query string that restricts results to only contain results from sr ''' if isinstance(sr, MultiReddit): if not sr.sr_ids: raise InvalidQuery srs = ["sr_id:%s" % sr_id for sr_id in sr.sr_ids] return "(or %s)" % ' '.join(srs) elif isinstance(sr, DomainSR): return "site:'\"%s\"'" % sr.domain elif isinstance(sr, FriendsSR): if not c.user_is_loggedin or not c.user.friends: raise InvalidQuery # The query limit is roughly 8k bytes. Limit to 200 friends to # avoid getting too close to that limit friend_ids = c.user.friends[:200] friends = [ "author_fullname:'%s'" % Account._fullname_from_id36(r2utils.to36(id_)) for id_ in friend_ids ] return "(or %s)" % ' '.join(friends) elif isinstance(sr, AllMinus): if not sr.exclude_sr_ids: raise InvalidQuery exclude_srs = ["sr_id:%s" % sr_id for sr_id in sr.exclude_sr_ids] return "(not (or %s))" % ' '.join(exclude_srs) elif not isinstance(sr, FakeSubreddit): return "sr_id:%s" % sr._id return None
def _get_sr_restriction(sr): '''Return a cloudsearch appropriate query string that restricts results to only contain results from self.sr ''' bq = [] if (not sr) or sr == All or isinstance(sr, DefaultSR): return None elif isinstance(sr, MultiReddit): bq = ["(or"] for sr_id in sr.sr_ids: bq.append("sr_id:%s" % sr_id) bq.append(")") elif isinstance(sr, DomainSR): bq = ["site:'%s'" % sr.domain] elif sr == Friends: if not c.user_is_loggedin or not c.user.friends: return None bq = ["(or"] # The query limit is roughly 8k bytes. Limit to 200 friends to # avoid getting too close to that limit friend_ids = c.user.friends[:200] friends = [ "author_fullname:'%s'" % Account._fullname_from_id36(r2utils.to36(id_)) for id_ in friend_ids ] bq.extend(friends) bq.append(")") elif isinstance(sr, ModContribSR): bq = ["(or"] for sr_id in sr.sr_ids: bq.append("sr_id:%s" % sr_id) bq.append(")") elif not isinstance(sr, FakeSubreddit): bq = ["sr_id:%s" % sr._id] return ' '.join(bq)
def bulk_upsert(links): updates = filter(lambda user: getattr(user, "dfp_creative_id", False), links) inserts = filter(lambda user: not getattr(user, "dfp_creative_id", False), links) dfp_creative_service = DfpService("CreativeService") creatives = [] if updates: existing_creatives = {} statement = dfp.FilterStatement( "WHERE id IN (%s)" % ", ".join([str(link.dfp_creative_id) for link in updates])) while True: response = dfp_creative_service.execute( "getCreativesByStatement", statement.ToStatement(), ) if "results" in response: for creative in response["results"]: existing_creatives[creative.id] = creative statement.offset += dfp.SUGGESTED_PAGE_LIMIT else: break updated = dfp_creative_service.execute("updateCreatives", [ _link_to_creative( link=link, existing=existing_creatives[link.dfp_creative_id], ) for link in updates]) creatives += updated if inserts: authors = Account._byID([link.author_id for link in inserts], return_dict=False) advertisers = advertisers_service.bulk_upsert(authors) advertisers_by_author = { advertiser.externalId: advertiser for advertiser in advertisers} inserted = dfp_creative_service.execute("createCreatives", [ _link_to_creative( link=link, advertiser=advertisers_by_author[ Account._fullname_from_id36(to36(link.author_id)) ], ) for link in inserts]) creatives += inserted creatives_by_fullname = { utils.get_template_variable(creative, "link_id"): creative for creative in creatives} for link in links: creative = creatives_by_fullname[link._fullname] link.dfp_creative_id = creative.id link._commit() return creatives
def bulk_upsert(links): updates = filter(lambda user: getattr(user, "dfp_creative_id", False), links) inserts = filter(lambda user: not getattr(user, "dfp_creative_id", False), links) dfp_creative_service = DfpService("CreativeService") creatives = [] if updates: existing_creatives = {} statement = dfp.FilterStatement( "WHERE id IN (%s)" % ", ".join([str(link.dfp_creative_id) for link in updates])) while True: response = dfp_creative_service.execute( "getCreativesByStatement", statement.ToStatement(), ) if "results" in response: for creative in response["results"]: existing_creatives[creative.id] = creative statement.offset += dfp.SUGGESTED_PAGE_LIMIT else: break updated = dfp_creative_service.execute("updateCreatives", [ _link_to_creative( link=link, existing=existing_creatives[link.dfp_creative_id], ) for link in updates ]) creatives += updated if inserts: authors = Account._byID([link.author_id for link in inserts], return_dict=False) advertisers = advertisers_service.bulk_upsert(authors) advertisers_by_author = { advertiser.externalId: advertiser for advertiser in advertisers } inserted = dfp_creative_service.execute("createCreatives", [ _link_to_creative( link=link, advertiser=advertisers_by_author[Account._fullname_from_id36( to36(link.author_id))], ) for link in inserts ]) creatives += inserted creatives_by_fullname = { utils.get_template_variable(creative, "link_id"): creative for creative in creatives } for link in links: creative = creatives_by_fullname[link._fullname] link.dfp_creative_id = creative.id link._commit() return creatives