def process(link):
timestamp = link.timestamp
if link.thing_type == 'link':
fname = make_fullname(Link, link.thing_id)
else:
fname = make_fullname(Comment, link.thing_id)
if not link.spam and not link.deleted:
author_id = link.author_id
ups, downs = link.ups, link.downs
sc = score(ups, downs)
contr = controversy(ups, downs)
h = _hot(ups, downs, timestamp)
for tkey, oldest in oldests.iteritems():
if timestamp > oldest:
yield ('%s-top-%s-%d' % (link.thing_type, tkey, author_id),
sc, timestamp, fname)
yield ('%s-controversial-%s-%d' % (link.thing_type, tkey, author_id),
contr, timestamp, fname)
if tkey == 'all':
#yield ('%s-new-%s-%d' % (link.thing_type, tkey, author_id),
# timestamp, timestamp, fname)
yield ('%s-hot-%s-%d' % (link.thing_type, tkey, author_id),
h, timestamp, fname)
def process(link):
timestamp = link.timestamp
if link.thing_type == 'link':
fname = make_fullname(Link, link.thing_id)
else:
fname = make_fullname(Comment, link.thing_id)
if not link.spam and not link.deleted:
author_id = link.author_id
ups, downs = link.ups, link.downs
sc = score(ups, downs)
contr = controversy(ups, downs)
h = _hot(ups, downs, timestamp)
for tkey, oldest in oldests.iteritems():
if timestamp > oldest:
yield ('%s-top-%s-%d' % (link.thing_type, tkey, author_id),
sc, timestamp, fname)
yield ('%s-controversial-%s-%d' % (link.thing_type, tkey, author_id),
contr, timestamp, fname)
if tkey == 'all':
#yield ('%s-new-%s-%d' % (link.thing_type, tkey, author_id),
# timestamp, timestamp, fname)
yield ('%s-hot-%s-%d' % (link.thing_type, tkey, author_id),
h, timestamp, fname)
def process(thing):
if thing.deleted:
return
thing_cls = thingcls_by_name[thing.thing_type]
fname = make_fullname(thing_cls, thing.thing_id)
thing_score = score(thing.ups, thing.downs)
thing_controversy = controversy(thing.ups, thing.downs)
for interval, cutoff in cutoff_by_interval.iteritems():
if thing.timestamp < cutoff:
continue
yield ("user/%s/top/%s/%d" % (thing.thing_type, interval, thing.author_id),
thing_score, thing.timestamp, fname)
yield ("user/%s/controversial/%s/%d" % (thing.thing_type, interval, thing.author_id),
thing_controversy, thing.timestamp, fname)
if thing.spam:
continue
if thing.thing_type == "link":
yield ("sr/link/top/%s/%d" % (interval, thing.sr_id),
thing_score, thing.timestamp, fname)
yield ("sr/link/controversial/%s/%d" % (interval, thing.sr_id),
thing_controversy, thing.timestamp, fname)
if thing.url:
for domain in UrlParser(thing.url).domain_permutations():
yield ("domain/link/top/%s/%s" % (interval, domain),
thing_score, thing.timestamp, fname)
yield ("domain/link/controversial/%s/%s" % (interval, domain),
thing_controversy, thing.timestamp, fname)
def process(link):
assert link.thing_type == 'link'
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
if not link.spam and not link.deleted:
if link.url:
domains = UrlParser(link.url).domain_permutations()
else:
domains = []
ups, downs = link.ups, link.downs
for tkey, oldest in oldests.iteritems():
if timestamp > oldest:
sc = score(ups, downs)
contr = controversy(ups, downs)
h = _hot(ups, downs, timestamp)
for domain in domains:
yield ('domain/top/%s/%s' % (tkey, domain),
sc, timestamp, fname)
yield ('domain/controversial/%s/%s' % (tkey, domain),
contr, timestamp, fname)
if tkey == "all":
yield ('domain/hot/%s/%s' % (tkey, domain),
h, timestamp, fname)
yield ('domain/new/%s/%s' % (tkey, domain),
timestamp, timestamp, fname)
def process(link):
assert link.thing_type == 'link'
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
if not link.spam and not link.deleted:
if link.url:
domains = UrlParser(link.url).domain_permutations()
else:
domains = []
ups, downs = link.ups, link.downs
for tkey, oldest in oldests.iteritems():
if timestamp > oldest:
sc = score(ups, downs)
contr = controversy(ups, downs)
h = _hot(ups, downs, timestamp)
for domain in domains:
yield ('domain/top/%s/%s' % (tkey, domain), sc,
timestamp, fname)
yield ('domain/controversial/%s/%s' % (tkey, domain),
contr, timestamp, fname)
if tkey == "all":
yield ('domain/hot/%s/%s' % (tkey, domain), h,
timestamp, fname)
yield ('domain/new/%s/%s' % (tkey, domain),
timestamp, timestamp, fname)
def process(link):
assert link.thing_type == 'link'
author_id = link.author_id
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
yield 'user-submitted-%d' % author_id, timestamp, fname
if not link.spam:
sr_id = link.sr_id
ups, downs = link.ups, link.downs
yield ('sr-hot-all-%d' % sr_id, _hot(ups, downs, timestamp),
timestamp, fname)
yield 'sr-new-all-%d' % sr_id, timestamp, fname
yield 'sr-top-all-%d' % sr_id, score(ups, downs), timestamp, fname
yield ('sr-controversial-all-%d' % sr_id,
controversy(ups, downs), timestamp, fname)
for time in '1 year', '1 month', '1 week', '1 day', '1 hour':
if timestamp > epoch_seconds(timeago(time)):
tkey = time.split(' ')[1]
yield ('sr-top-%s-%d' % (tkey, sr_id),
score(ups, downs), timestamp, fname)
yield ('sr-controversial-%s-%d' % (tkey, sr_id),
controversy(ups, downs),
timestamp, fname)
def process(link):
assert link.thing_type == 'link'
author_id = link.author_id
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
yield 'user-submitted-%d' % author_id, timestamp, fname
if not link.spam:
sr_id = link.sr_id
ups, downs = link.ups, link.downs
yield ('sr-hot-all-%d' % sr_id, _hot(ups, downs,
timestamp), timestamp, fname)
yield 'sr-new-all-%d' % sr_id, timestamp, fname
yield 'sr-top-all-%d' % sr_id, score(ups, downs), timestamp, fname
yield ('sr-controversial-all-%d' % sr_id, controversy(ups, downs),
timestamp, fname)
for time in '1 year', '1 month', '1 week', '1 day', '1 hour':
if timestamp > epoch_seconds(timeago(time)):
tkey = time.split(' ')[1]
yield ('sr-top-%s-%d' % (tkey, sr_id), score(ups, downs),
timestamp, fname)
yield ('sr-controversial-%s-%d' % (tkey, sr_id),
controversy(ups, downs), timestamp, fname)
def process(link):
if not link.deleted:
author_id = link.author_id
ups = link.ups
downs = link.downs
sc = score(ups, downs)
contr = controversy(ups, downs)
if link.thing_type == 'link':
fname = make_fullname(Link, link.thing_id)
else:
fname = make_fullname(Comment, link.thing_id)
timestamp = link.timestamp
date = datetime.datetime.utcfromtimestamp(timestamp)
yield ('user-top-%s-%d' % (date.year, author_id),
sc, timestamp, fname)
yield ('user-controversial-%s-%d' % (date.year, author_id),
contr, timestamp, fname)
def process(link):
if not link.deleted:
author_id = link.author_id
ups = link.ups
downs = link.downs
sc = score(ups, downs)
contr = controversy(ups, downs)
if link.thing_type == 'link':
fname = make_fullname(Link, link.thing_id)
else:
fname = make_fullname(Comment, link.thing_id)
timestamp = link.timestamp
date = datetime.datetime.utcfromtimestamp(timestamp)
yield ('user-top-%s-%d' % (date.year, author_id),
sc, timestamp, fname)
yield ('user-controversial-%s-%d' % (date.year, author_id),
contr, timestamp, fname)
def process(link):
assert link.thing_type == 'link'
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
if not link.spam:
sr_id = link.sr_id
ups, downs = link.ups, link.downs
for tkey, oldest in oldests.iteritems():
if timestamp > oldest:
yield ('sr-top-%s-%d' % (tkey, sr_id),
score(ups, downs), timestamp, fname)
yield ('sr-controversial-%s-%d' % (tkey, sr_id),
controversy(ups, downs),
timestamp, fname)
def process(thing):
if thing.deleted:
return
thing_cls = thingcls_by_name[thing.thing_type]
fname = make_fullname(thing_cls, thing.thing_id)
thing_score = score(thing.ups, thing.downs)
thing_upvotes = upvotes(thing.ups)
thing_controversy = controversy(thing.ups, thing.downs)
for interval, cutoff in cutoff_by_interval.iteritems():
if thing.timestamp < cutoff:
continue
yield ("user/%s/top/%s/%d" % (thing.thing_type, interval, thing.author_id),
thing_score, thing.timestamp, fname)
yield ("user/%s/%s/%s/%d" % (thing.thing_type, g.voting_upvote_path, interval, thing.author_id),
thing_upvotes, thing.timestamp, fname)
yield ("user/%s/%s/%s/%d" % (thing.thing_type, g.voting_controversial_path, interval, thing.author_id),
thing_controversy, thing.timestamp, fname)
if thing.spam:
continue
if thing.thing_type == "link":
yield ("sr/link/top/%s/%d" % (interval, thing.sr_id),
thing_score, thing.timestamp, fname)
yield ("sr/link/%s/%s/%d" % (g.voting_upvote_path, interval, thing.sr_id),
thing_upvotes, thing.timestamp, fname)
yield ("sr/link/%s/%s/%d" % (g.voting_controversial_path, interval, thing.sr_id),
thing_controversy, thing.timestamp, fname)
if thing.url:
try:
parsed = UrlParser(thing.url)
except ValueError:
continue
for domain in parsed.domain_permutations():
yield ("domain/link/top/%s/%s" % (interval, domain),
thing_score, thing.timestamp, fname)
yield ("domain/link/%s/%s/%s" % (g.voting_upvote_path, interval, domain),
thing_upvotes, thing.timestamp, fname)
yield ("domain/link/%s/%s/%s" % (g.voting_controversial_path, interval, domain),
thing_controversy, thing.timestamp, fname)
def process(link):
assert link.thing_type == "link"
author_id = link.author_id
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
yield "user-submitted-%d" % author_id, timestamp, fname
if not link.spam:
sr_id = link.sr_id
ups, downs = link.ups, link.downs
yield ("sr-hot-all-%d" % sr_id, _hot(ups, downs, timestamp), timestamp, fname)
yield "sr-new-all-%d" % sr_id, timestamp, fname
yield "sr-top-all-%d" % sr_id, score(ups, downs), timestamp, fname
yield ("sr-controversial-all-%d" % sr_id, controversy(ups, downs), timestamp, fname)
for time in "1 year", "1 month", "1 week", "1 day", "1 hour":
if timestamp > epoch_seconds(timeago(time)):
tkey = time.split(" ")[1]
yield ("sr-top-%s-%d" % (tkey, sr_id), score(ups, downs), timestamp, fname)
yield ("sr-controversial-%s-%d" % (tkey, sr_id), controversy(ups, downs), timestamp, fname)
def time_listing_iter(self, thing, cutoff_by_interval):
if thing.deleted:
return
thing_cls = self.thing_cls
fname = make_fullname(thing_cls, thing.thing_id)
scores = {k: func(thing) for k, func in self.LISTING_SORTS.iteritems()}
for interval, cutoff in cutoff_by_interval.iteritems():
if thing.timestamp < cutoff:
continue
for sort, value in scores.iteritems():
aid = thing.author_id
key = self.make_key("user", sort, interval, aid)
yield (key, value, thing.timestamp, fname)
if thing.spam:
continue
if thing.thing_type == "link":
for sort, value in scores.iteritems():
sr_id = thing.sr_id
key = self.make_key("sr", sort, interval, sr_id)
yield (key, value, thing.timestamp, fname)
if not thing.url:
continue
try:
parsed = UrlParser(thing.url)
except ValueError:
continue
for d in parsed.domain_permutations():
for sort, value in scores.iteritems():
key = self.make_key("domain", sort, interval, d)
yield (key, value, thing.timestamp, fname)
def time_listing_iter(self, thing, cutoff_by_interval):
if thing.deleted:
return
thing_cls = self.thing_cls
fname = make_fullname(thing_cls, thing.thing_id)
scores = {k: func(thing) for k, func in self.LISTING_SORTS.iteritems()}
for interval, cutoff in cutoff_by_interval.iteritems():
if thing.timestamp < cutoff:
continue
for sort, value in scores.iteritems():
aid = thing.author_id
key = self.make_key("user", sort, interval, aid)
yield (key, value, thing.timestamp, fname)
if thing.spam:
continue
if thing.thing_type == "link":
for sort, value in scores.iteritems():
sr_id = thing.sr_id
key = self.make_key("sr", sort, interval, sr_id)
yield (key, value, thing.timestamp, fname)
if not thing.url:
continue
try:
parsed = UrlParser(thing.url)
except ValueError:
continue
for d in parsed.domain_permutations():
for sort, value in scores.iteritems():
key = self.make_key("domain", sort, interval, d)
yield (key, value, thing.timestamp, fname)
def process(rel):
if rel.name in names:
yield ("%s-%s" % (names[rel.name], rel.thing1_id), rel.timestamp, make_fullname(thing2_cls, rel.thing2_id))
def process(comment):
assert comment.thing_type == "comment"
yield ("user-commented-%d" % comment.author_id, comment.timestamp, make_fullname(Comment, comment.thing_id))
def process(rel):
if rel.name in names:
yield ('%s-%s' % (names[rel.name], rel.thing1_id), rel.timestamp,
make_fullname(thing2_cls, rel.thing2_id))
def process(comment):
assert comment.thing_type == 'comment'
yield ('user-commented-%d' % comment.author_id, comment.timestamp,
make_fullname(Comment, comment.thing_id))