def process(link):
assert link.thing_type == 'link'
author_id = link.author_id
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
yield 'user-submitted-%d' % author_id, timestamp, fname
if not link.spam:
sr_id = link.sr_id
ups, downs = link.ups, link.downs
yield ('sr-hot-all-%d' % sr_id, _hot(ups, downs, timestamp),
timestamp, fname)
yield 'sr-new-all-%d' % sr_id, timestamp, fname
yield 'sr-top-all-%d' % sr_id, score(ups, downs), timestamp, fname
yield ('sr-controversial-all-%d' % sr_id,
controversy(ups, downs), timestamp, fname)
for time in '1 year', '1 month', '1 week', '1 day', '1 hour':
if timestamp > epoch_seconds(timeago(time)):
tkey = time.split(' ')[1]
yield ('sr-top-%s-%d' % (tkey, sr_id),
score(ups, downs), timestamp, fname)
yield ('sr-controversial-%s-%d' % (tkey, sr_id),
controversy(ups, downs),
timestamp, fname)
def process(link):
assert link.thing_type == 'link'
author_id = link.author_id
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
yield 'user-submitted-%d' % author_id, timestamp, fname
if not link.spam:
sr_id = link.sr_id
ups, downs = link.ups, link.downs
yield ('sr-hot-all-%d' % sr_id, _hot(ups, downs,
timestamp), timestamp, fname)
yield 'sr-new-all-%d' % sr_id, timestamp, fname
yield 'sr-top-all-%d' % sr_id, score(ups, downs), timestamp, fname
yield ('sr-controversial-all-%d' % sr_id, controversy(ups, downs),
timestamp, fname)
for time in '1 year', '1 month', '1 week', '1 day', '1 hour':
if timestamp > epoch_seconds(timeago(time)):
tkey = time.split(' ')[1]
yield ('sr-top-%s-%d' % (tkey, sr_id), score(ups, downs),
timestamp, fname)
yield ('sr-controversial-%s-%d' % (tkey, sr_id),
controversy(ups, downs), timestamp, fname)
def process(link):
assert link.thing_type == 'link'
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
if not link.spam and not link.deleted:
if link.url:
domains = UrlParser(link.url).domain_permutations()
else:
domains = []
ups, downs = link.ups, link.downs
for tkey, oldest in oldests.iteritems():
if timestamp > oldest:
sc = score(ups, downs)
contr = controversy(ups, downs)
h = _hot(ups, downs, timestamp)
for domain in domains:
yield ('domain/top/%s/%s' % (tkey, domain),
sc, timestamp, fname)
yield ('domain/controversial/%s/%s' % (tkey, domain),
contr, timestamp, fname)
if tkey == "all":
yield ('domain/hot/%s/%s' % (tkey, domain),
h, timestamp, fname)
yield ('domain/new/%s/%s' % (tkey, domain),
timestamp, timestamp, fname)
def process(link):
timestamp = link.timestamp
if link.thing_type == 'link':
fname = make_fullname(Link, link.thing_id)
else:
fname = make_fullname(Comment, link.thing_id)
if not link.spam and not link.deleted:
author_id = link.author_id
ups, downs = link.ups, link.downs
sc = score(ups, downs)
contr = controversy(ups, downs)
h = _hot(ups, downs, timestamp)
for tkey, oldest in oldests.iteritems():
if timestamp > oldest:
yield ('%s-top-%s-%d' % (link.thing_type, tkey, author_id),
sc, timestamp, fname)
yield ('%s-controversial-%s-%d' % (link.thing_type, tkey, author_id),
contr, timestamp, fname)
if tkey == 'all':
#yield ('%s-new-%s-%d' % (link.thing_type, tkey, author_id),
# timestamp, timestamp, fname)
yield ('%s-hot-%s-%d' % (link.thing_type, tkey, author_id),
h, timestamp, fname)
def process(link):
assert link.thing_type == 'link'
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
if not link.spam and not link.deleted:
if link.url:
domains = UrlParser(link.url).domain_permutations()
else:
domains = []
ups, downs = link.ups, link.downs
for tkey, oldest in oldests.iteritems():
if timestamp > oldest:
sc = score(ups, downs)
contr = controversy(ups, downs)
h = _hot(ups, downs, timestamp)
for domain in domains:
yield ('domain/top/%s/%s' % (tkey, domain), sc,
timestamp, fname)
yield ('domain/controversial/%s/%s' % (tkey, domain),
contr, timestamp, fname)
if tkey == "all":
yield ('domain/hot/%s/%s' % (tkey, domain), h,
timestamp, fname)
yield ('domain/new/%s/%s' % (tkey, domain),
timestamp, timestamp, fname)
def process(thing):
if thing.deleted:
return
thing_cls = thingcls_by_name[thing.thing_type]
fname = make_fullname(thing_cls, thing.thing_id)
thing_score = score(thing.ups, thing.downs)
thing_controversy = controversy(thing.ups, thing.downs)
for interval, cutoff in cutoff_by_interval.iteritems():
if thing.timestamp < cutoff:
continue
yield ("user/%s/top/%s/%d" % (thing.thing_type, interval, thing.author_id),
thing_score, thing.timestamp, fname)
yield ("user/%s/controversial/%s/%d" % (thing.thing_type, interval, thing.author_id),
thing_controversy, thing.timestamp, fname)
if thing.spam:
continue
if thing.thing_type == "link":
yield ("sr/link/top/%s/%d" % (interval, thing.sr_id),
thing_score, thing.timestamp, fname)
yield ("sr/link/controversial/%s/%d" % (interval, thing.sr_id),
thing_controversy, thing.timestamp, fname)
if thing.url:
for domain in UrlParser(thing.url).domain_permutations():
yield ("domain/link/top/%s/%s" % (interval, domain),
thing_score, thing.timestamp, fname)
yield ("domain/link/controversial/%s/%s" % (interval, domain),
thing_controversy, thing.timestamp, fname)
def process(link):
assert link.thing_type == 'link'
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
if not link.spam and not link.deleted:
author_id = link.author_id
ups, downs = link.ups, link.downs
sc = score(ups, downs)
contr = controversy(ups, downs)
h = _hot(ups, downs, timestamp)
for tkey, oldest in oldests.iteritems():
if timestamp > oldest:
yield ('user-top-%s-%d' % (tkey, author_id), sc, timestamp,
fname)
yield ('user-controversial-%s-%d' % (tkey, author_id),
contr, timestamp, fname)
if tkey == 'all':
yield ('user-new-%s-%d' % (tkey, author_id), timestamp,
timestamp, fname)
yield ('user-hot-%s-%d' % (tkey, author_id), h,
timestamp, fname)
def process(link):
assert link.thing_type == "link"
author_id = link.author_id
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
yield "user-submitted-%d" % author_id, timestamp, fname
if not link.spam:
sr_id = link.sr_id
ups, downs = link.ups, link.downs
yield ("sr-hot-all-%d" % sr_id, _hot(ups, downs, timestamp), timestamp, fname)
yield "sr-new-all-%d" % sr_id, timestamp, fname
yield "sr-top-all-%d" % sr_id, score(ups, downs), timestamp, fname
yield ("sr-controversial-all-%d" % sr_id, controversy(ups, downs), timestamp, fname)
for time in "1 year", "1 month", "1 week", "1 day", "1 hour":
if timestamp > epoch_seconds(timeago(time)):
tkey = time.split(" ")[1]
yield ("sr-top-%s-%d" % (tkey, sr_id), score(ups, downs), timestamp, fname)
yield ("sr-controversial-%s-%d" % (tkey, sr_id), controversy(ups, downs), timestamp, fname)
def _process(t):
thing_id = t.thing_id
id36 = to36(thing_id)
link_id = t.link_id
link_id36 = to36(link_id)
ups, downs, timestamp = t.ups, t.downs, t.timestamp
yield link_id36+'_controversy', id36, sorts.controversy(ups, downs)
yield link_id36+'_hot', id36, sorts._hot(ups, downs, timestamp)
yield link_id36+'_confidence', id36, sorts.confidence(ups, downs)
yield link_id36+'_score', id36, sorts.score(ups, downs)
yield link_id36+'_date', id36, timestamp
def _process(t):
thing_id = t.thing_id
id36 = to36(thing_id)
link_id = t.link_id
link_id36 = to36(link_id)
ups, downs, timestamp = t.ups, t.downs, t.timestamp
yield link_id36+'_controversy', id36, sorts.controversy(ups, downs)
yield link_id36+'_hot', id36, sorts._hot(ups, downs, timestamp)
yield link_id36+'_confidence', id36, sorts.confidence(ups, downs)
yield link_id36+'_score', id36, sorts.score(ups, downs)
yield link_id36+'_date', id36, timestamp
def process(link):
if not link.deleted:
author_id = link.author_id
ups = link.ups
downs = link.downs
sc = score(ups, downs)
contr = controversy(ups, downs)
if link.thing_type == 'link':
fname = make_fullname(Link, link.thing_id)
else:
fname = make_fullname(Comment, link.thing_id)
timestamp = link.timestamp
date = datetime.datetime.utcfromtimestamp(timestamp)
yield ('user-top-%s-%d' % (date.year, author_id),
sc, timestamp, fname)
yield ('user-controversial-%s-%d' % (date.year, author_id),
contr, timestamp, fname)
def process(link):
if not link.deleted:
author_id = link.author_id
ups = link.ups
downs = link.downs
sc = score(ups, downs)
contr = controversy(ups, downs)
if link.thing_type == 'link':
fname = make_fullname(Link, link.thing_id)
else:
fname = make_fullname(Comment, link.thing_id)
timestamp = link.timestamp
date = datetime.datetime.utcfromtimestamp(timestamp)
yield ('user-top-%s-%d' % (date.year, author_id),
sc, timestamp, fname)
yield ('user-controversial-%s-%d' % (date.year, author_id),
contr, timestamp, fname)
def process(link):
assert link.thing_type == 'link'
timestamp = link.timestamp
fname = make_fullname(Link, link.thing_id)
if not link.spam:
sr_id = link.sr_id
ups, downs = link.ups, link.downs
for tkey, oldest in oldests.iteritems():
if timestamp > oldest:
yield ('sr-top-%s-%d' % (tkey, sr_id),
score(ups, downs), timestamp, fname)
yield ('sr-controversial-%s-%d' % (tkey, sr_id),
controversy(ups, downs),
timestamp, fname)
def process(thing):
if thing.deleted:
return
thing_cls = thingcls_by_name[thing.thing_type]
fname = make_fullname(thing_cls, thing.thing_id)
thing_score = score(thing.ups, thing.downs)
thing_upvotes = upvotes(thing.ups)
thing_controversy = controversy(thing.ups, thing.downs)
for interval, cutoff in cutoff_by_interval.iteritems():
if thing.timestamp < cutoff:
continue
yield ("user/%s/top/%s/%d" % (thing.thing_type, interval, thing.author_id),
thing_score, thing.timestamp, fname)
yield ("user/%s/%s/%s/%d" % (thing.thing_type, g.voting_upvote_path, interval, thing.author_id),
thing_upvotes, thing.timestamp, fname)
yield ("user/%s/%s/%s/%d" % (thing.thing_type, g.voting_controversial_path, interval, thing.author_id),
thing_controversy, thing.timestamp, fname)
if thing.spam:
continue
if thing.thing_type == "link":
yield ("sr/link/top/%s/%d" % (interval, thing.sr_id),
thing_score, thing.timestamp, fname)
yield ("sr/link/%s/%s/%d" % (g.voting_upvote_path, interval, thing.sr_id),
thing_upvotes, thing.timestamp, fname)
yield ("sr/link/%s/%s/%d" % (g.voting_controversial_path, interval, thing.sr_id),
thing_controversy, thing.timestamp, fname)
if thing.url:
try:
parsed = UrlParser(thing.url)
except ValueError:
continue
for domain in parsed.domain_permutations():
yield ("domain/link/top/%s/%s" % (interval, domain),
thing_score, thing.timestamp, fname)
yield ("domain/link/%s/%s/%s" % (g.voting_upvote_path, interval, domain),
thing_upvotes, thing.timestamp, fname)
yield ("domain/link/%s/%s/%s" % (g.voting_controversial_path, interval, domain),
thing_controversy, thing.timestamp, fname)
def _controversy(self):
return sorts.controversy(self._ups, self._downs)
def _controversy(self):
return sorts.controversy(self._ups, self._downs)
def controversy_score(thing):
return controversy(thing.ups, thing.downs)
def controversy_score(thing):
return controversy(thing.ups, thing.downs)
