def store_keys(cls, key, listing):
"""Look up query based on key, and update with provided listing.
:param str key: key generated by :py:method:`make_key`
:param list listing: sorted listing generated by
`mr_reduce_max_per_key`, generally by :py:method:`write_permacache`
"""
category, thing_cls, sort, time, uid = cls.split_key(key)
query = None
if category == "user":
if thing_cls == "link":
query = queries._get_submitted(int(uid), sort, time)
elif thing_cls == "comment":
query = queries._get_comments(int(uid), sort, time)
elif category == "sr":
if thing_cls == "link":
query = queries._get_links(int(uid), sort, time)
elif category == "domain":
if thing_cls == "link":
query = queries.get_domain_links(uid, sort, time)
assert query, 'unknown query type for {}'.format(key)
item_tuples = [(thing_fullname, float(value), float(timestamp))
for value, timestamp, thing_fullname in listing]
# we only need locking updates for non-time-based listings, since for
# time- based ones we're the only ones that ever update it
lock = time == 'all'
query._replace(item_tuples, lock=lock)
def store_keys(cls, key, listing):
"""Look up query based on key, and update with provided listing.
:param str key: key generated by :py:method:`make_key`
:param list listing: sorted listing generated by
`mr_reduce_max_per_key`, generally by :py:method:`write_permacache`
"""
category, thing_cls, sort, time, uid = cls.split_key(key)
query = None
if category == "user":
if thing_cls == "link":
query = queries._get_submitted(int(uid), sort, time)
elif thing_cls == "comment":
query = queries._get_comments(int(uid), sort, time)
elif category == "sr":
if thing_cls == "link":
query = queries._get_links(int(uid), sort, time)
elif category == "domain":
if thing_cls == "link":
query = queries.get_domain_links(uid, sort, time)
assert query, 'unknown query type for {}'.format(key)
item_tuples = [
(thing_fullname, float(value), float(timestamp))
for value, timestamp, thing_fullname in listing
]
# we only need locking updates for non-time-based listings, since for
# time- based ones we're the only ones that ever update it
lock = time == 'all'
query._replace(item_tuples, lock=lock)
def store_keys(key, maxes):
category, thing_cls, sort, time, id = key.split("/")
query = None
if category == "user":
if thing_cls == "link":
query = queries._get_submitted(int(id), sort, time)
elif thing_cls == "comment":
query = queries._get_comments(int(id), sort, time)
elif category == "sr":
if thing_cls == "link":
query = queries._get_links(int(id), sort, time)
elif category == "domain":
if thing_cls == "link":
query = queries.get_domain_links(id, sort, time)
assert query, 'unknown query type for %s' % (key,)
item_tuples = [tuple([item[-1]] + [float(x) for x in item[:-1]])
for item in maxes]
# we only need locking updates for non-time-based listings, since for time-
# based ones we're the only ones that ever update it
lock = time == 'all'
query._replace(item_tuples, lock=lock)
def store_keys(key, maxes):
# we're building queries using queries.py, but we could make the
# queries ourselves if we wanted to avoid the individual lookups
# for accounts and subreddits.
# Note that we're only generating the 'sr-' type queries here, but
# we're also able to process the other listings generated by the
# old migrate.mr_permacache for convenience
userrel_fns = dict(liked = queries.get_liked,
disliked = queries.get_disliked,
saved = queries.get_saved,
hidden = queries.get_hidden)
if key.startswith('user-'):
acc_str, keytype, account_id = key.split('-')
account_id = int(account_id)
fn = queries._get_submitted if keytype == 'submitted' else queries._get_comments
q = fn(account_id, 'new', 'all')
q._replace([(fname, float(timestamp))
for (timestamp, fname)
in maxes])
elif key.startswith('sr-'):
sr_str, sort, time, sr_id = key.split('-')
sr_id = int(sr_id)
if sort == 'controversy':
# I screwed this up in the mapper and it's too late to fix
# it
sort = 'controversial'
q = queries._get_links(sr_id, sort, time)
q._replace([tuple([item[-1]] + map(float, item[:-1]))
for item in maxes])
elif key.startswith('domain/'):
d_str, sort, time, domain = key.split('/')
q = queries.get_domain_links(domain, sort, time)
q._replace([tuple([item[-1]] + map(float, item[:-1]))
for item in maxes])
elif key.split('-')[0] in userrel_fns:
key_type, account_id = key.split('-')
account_id = int(account_id)
fn = userrel_fns[key_type]
q = fn(Account._byID(account_id))
q._replace([tuple([item[-1]] + map(float, item[:-1]))
for item in maxes])
def store_keys(key, maxes):
# we're building queries using queries.py, but we could make the
# queries ourselves if we wanted to avoid the individual lookups
# for accounts and subreddits.
# Note that we're only generating the 'sr-' type queries here, but
# we're also able to process the other listings generated by the
# old migrate.mr_permacache for convenience
userrel_fns = dict(liked = queries.get_liked,
disliked = queries.get_disliked,
saved = queries.get_saved,
hidden = queries.get_hidden)
if key.startswith('user-'):
acc_str, keytype, account_id = key.split('-')
account_id = int(account_id)
fn = queries._get_submitted if keytype == 'submitted' else queries._get_comments
q = fn(account_id, 'new', 'all')
q._replace([(fname, float(timestamp))
for (timestamp, fname)
in maxes])
elif key.startswith('sr-'):
sr_str, sort, time, sr_id = key.split('-')
sr_id = int(sr_id)
if sort == 'controversy':
# I screwed this up in the mapper and it's too late to fix
# it
sort = 'controversial'
q = queries._get_links(sr_id, sort, time)
q._replace([tuple([item[-1]] + map(float, item[:-1]))
for item in maxes])
elif key.startswith('domain/'):
d_str, sort, time, domain = key.split('/')
q = queries.get_domain_links(domain, sort, time)
q._replace([tuple([item[-1]] + map(float, item[:-1]))
for item in maxes])
elif key.split('-')[0] in userrel_fns:
key_type, account_id = key.split('-')
account_id = int(account_id)
fn = userrel_fns[key_type]
q = fn(Account._byID(account_id))
q._replace([tuple([item[-1]] + map(float, item[:-1]))
for item in maxes])
def store_keys(key, maxes):
category, thing_cls, sort, time, id = key.split("/")
query = None
if category == "user":
if thing_cls == "link":
query = queries._get_submitted(int(id), sort, time)
elif thing_cls == "comment":
query = queries._get_comments(int(id), sort, time)
elif category == "sr":
if thing_cls == "link":
query = queries._get_links(int(id), sort, time)
elif category == "domain":
if thing_cls == "link":
query = queries.get_domain_links(id, sort, time)
assert query
item_tuples = [tuple([item[-1]] + [float(x) for x in item[:-1]])
for item in maxes]
query._replace(item_tuples)
def get_hot_tuples(sr_ids):
queries_by_sr_id = {sr_id: _get_links(sr_id, sort='hot', time='all')
for sr_id in sr_ids}
CachedResults.fetch_multi(queries_by_sr_id.values())
tuples_by_srid = {sr_id: [] for sr_id in sr_ids}
for sr_id, q in queries_by_sr_id.iteritems():
if not q.data:
continue
link_name, hot, timestamp = q.data[0]
thot = max(hot, 1.)
tuples_by_srid[sr_id].append((-1., -hot, link_name, timestamp))
for link_name, hot, timestamp in q.data[1:MAX_PER_SUBREDDIT]:
ehot = hot / thot
# heapq.merge sorts from smallest to largest so we need to flip
# ehot and hot to get the hottest links first
tuples_by_srid[sr_id].append((-ehot, -hot, link_name, timestamp))
return tuples_by_srid
def get_hot_tuples(sr_ids, ageweight=None):
queries_by_sr_id = {sr_id: _get_links(sr_id, sort='hot', time='all')
for sr_id in sr_ids}
CachedResults.fetch_multi(queries_by_sr_id.values(), stale=True)
tuples_by_srid = {sr_id: [] for sr_id in sr_ids}
now_seconds = epoch_seconds(datetime.now(g.tz))
for sr_id, q in queries_by_sr_id.iteritems():
if not q.data:
continue
hot_factor = get_hot_factor(q.data[0], now_seconds, ageweight)
for link_name, hot, timestamp in q.data[:MAX_PER_SUBREDDIT]:
effective_hot = hot / hot_factor
# heapq.merge sorts from smallest to largest so we need to flip
# ehot and hot to get the hottest links first
tuples_by_srid[sr_id].append(
(-effective_hot, -hot, link_name, timestamp)
)
return tuples_by_srid
def get_hot_tuples(sr_ids):
queries_by_sr_id = {
sr_id: _get_links(sr_id, sort='hot', time='all')
for sr_id in sr_ids
}
CachedResults.fetch_multi(queries_by_sr_id.values())
tuples_by_srid = {sr_id: [] for sr_id in sr_ids}
for sr_id, q in queries_by_sr_id.iteritems():
if not q.data:
continue
link_name, hot, timestamp = q.data[0]
thot = max(hot, 1.)
tuples_by_srid[sr_id].append((-1., -hot, link_name, timestamp))
for link_name, hot, timestamp in q.data[1:MAX_PER_SUBREDDIT]:
ehot = hot / thot
# heapq.merge sorts from smallest to largest so we need to flip
# ehot and hot to get the hottest links first
tuples_by_srid[sr_id].append((-ehot, -hot, link_name, timestamp))
return tuples_by_srid
def get_hot_tuples(sr_ids, ageweight=None):
queries_by_sr_id = {sr_id: _get_links(sr_id, sort='hot', time='all')
for sr_id in sr_ids}
CachedResults.fetch_multi(queries_by_sr_id.values(), stale=True)
tuples_by_srid = {sr_id: [] for sr_id in sr_ids}
now_seconds = epoch_seconds(datetime.now(g.tz))
for sr_id, q in queries_by_sr_id.iteritems():
if not q.data:
continue
hot_factor = get_hot_factor(q.data[0], now_seconds, ageweight)
for link_name, hot, timestamp in q.data[:MAX_PER_SUBREDDIT]:
effective_hot = hot / hot_factor
# heapq.merge sorts from smallest to largest so we need to flip
# ehot and hot to get the hottest links first
tuples_by_srid[sr_id].append(
(-effective_hot, -hot, link_name, timestamp)
)
return tuples_by_srid
