Пример #1
0
    def process(link):

        timestamp = link.timestamp
        if link.thing_type == 'link':
            fname = make_fullname(Link, link.thing_id)
        else:
            fname = make_fullname(Comment, link.thing_id)

        if not link.spam and not link.deleted:
            author_id = link.author_id
            ups, downs = link.ups, link.downs

            sc = score(ups, downs)
            contr = controversy(ups, downs)
            h = _hot(ups, downs, timestamp)

            for tkey, oldest in oldests.iteritems():
                if timestamp > oldest:
                    yield ('%s-top-%s-%d' % (link.thing_type, tkey, author_id),
                           sc, timestamp, fname)
                    yield ('%s-controversial-%s-%d' % (link.thing_type, tkey, author_id),
                           contr, timestamp, fname)
                    if tkey == 'all':
                        #yield ('%s-new-%s-%d' % (link.thing_type, tkey, author_id),
                        #       timestamp, timestamp, fname)
                        yield ('%s-hot-%s-%d' % (link.thing_type, tkey, author_id),
                               h, timestamp, fname)
Пример #2
0
    def process(link):

        timestamp = link.timestamp
        if link.thing_type == 'link':
            fname = make_fullname(Link, link.thing_id)
        else:
            fname = make_fullname(Comment, link.thing_id)

        if not link.spam and not link.deleted:
            author_id = link.author_id
            ups, downs = link.ups, link.downs

            sc = score(ups, downs)
            contr = controversy(ups, downs)
            h = _hot(ups, downs, timestamp)

            for tkey, oldest in oldests.iteritems():
                if timestamp > oldest:
                    yield ('%s-top-%s-%d' % (link.thing_type, tkey, author_id),
                           sc, timestamp, fname)
                    yield ('%s-controversial-%s-%d' % (link.thing_type, tkey, author_id),
                           contr, timestamp, fname)
                    if tkey == 'all':
                        #yield ('%s-new-%s-%d' % (link.thing_type, tkey, author_id),
                        #       timestamp, timestamp, fname)
                        yield ('%s-hot-%s-%d' % (link.thing_type, tkey, author_id),
                               h, timestamp, fname)
Пример #3
0
    def process(thing):
        if thing.deleted:
            return

        thing_cls = thingcls_by_name[thing.thing_type]
        fname = make_fullname(thing_cls, thing.thing_id)
        thing_score = score(thing.ups, thing.downs)
        thing_controversy = controversy(thing.ups, thing.downs)

        for interval, cutoff in cutoff_by_interval.iteritems():
            if thing.timestamp < cutoff:
                continue

            yield ("user/%s/top/%s/%d" % (thing.thing_type, interval, thing.author_id),
                   thing_score, thing.timestamp, fname)
            yield ("user/%s/controversial/%s/%d" % (thing.thing_type, interval, thing.author_id),
                   thing_controversy, thing.timestamp, fname)

            if thing.spam:
                continue

            if thing.thing_type == "link":
                yield ("sr/link/top/%s/%d" % (interval, thing.sr_id),
                       thing_score, thing.timestamp, fname)
                yield ("sr/link/controversial/%s/%d" % (interval, thing.sr_id),
                       thing_controversy, thing.timestamp, fname)

                if thing.url:
                    for domain in UrlParser(thing.url).domain_permutations():
                        yield ("domain/link/top/%s/%s" % (interval, domain),
                               thing_score, thing.timestamp, fname)
                        yield ("domain/link/controversial/%s/%s" % (interval, domain),
                               thing_controversy, thing.timestamp, fname)
Пример #4
0
    def process(link):
        assert link.thing_type == 'link'

        timestamp = link.timestamp
        fname = make_fullname(Link, link.thing_id)

        if not link.spam and not link.deleted:
            if link.url:
                domains = UrlParser(link.url).domain_permutations()
            else:
                domains = []
            ups, downs = link.ups, link.downs

            for tkey, oldest in oldests.iteritems():
                if timestamp > oldest:
                    sc = score(ups, downs)
                    contr = controversy(ups, downs)
                    h = _hot(ups, downs, timestamp)
                    for domain in domains:
                        yield ('domain/top/%s/%s' % (tkey, domain),
                               sc, timestamp, fname)
                        yield ('domain/controversial/%s/%s' % (tkey, domain),
                               contr, timestamp, fname)
                        if tkey == "all":
                            yield ('domain/hot/%s/%s' % (tkey, domain),
                                   h, timestamp, fname)
                            yield ('domain/new/%s/%s' % (tkey, domain),
                                   timestamp, timestamp, fname)
Пример #5
0
    def process(link):
        assert link.thing_type == 'link'

        timestamp = link.timestamp
        fname = make_fullname(Link, link.thing_id)

        if not link.spam and not link.deleted:
            if link.url:
                domains = UrlParser(link.url).domain_permutations()
            else:
                domains = []
            ups, downs = link.ups, link.downs

            for tkey, oldest in oldests.iteritems():
                if timestamp > oldest:
                    sc = score(ups, downs)
                    contr = controversy(ups, downs)
                    h = _hot(ups, downs, timestamp)
                    for domain in domains:
                        yield ('domain/top/%s/%s' % (tkey, domain), sc,
                               timestamp, fname)
                        yield ('domain/controversial/%s/%s' % (tkey, domain),
                               contr, timestamp, fname)
                        if tkey == "all":
                            yield ('domain/hot/%s/%s' % (tkey, domain), h,
                                   timestamp, fname)
                            yield ('domain/new/%s/%s' % (tkey, domain),
                                   timestamp, timestamp, fname)
Пример #6
0
    def process(link):
        assert link.thing_type == 'link'

        author_id = link.author_id
        timestamp = link.timestamp
        fname = make_fullname(Link, link.thing_id)

        yield 'user-submitted-%d' % author_id, timestamp, fname
        if not link.spam:
            sr_id = link.sr_id
            ups, downs = link.ups, link.downs

            yield ('sr-hot-all-%d' % sr_id, _hot(ups, downs, timestamp),
                   timestamp, fname)
            yield 'sr-new-all-%d' % sr_id, timestamp, fname
            yield 'sr-top-all-%d' % sr_id, score(ups, downs), timestamp, fname
            yield ('sr-controversial-all-%d' % sr_id,
                   controversy(ups, downs), timestamp, fname)
            for time in '1 year', '1 month', '1 week', '1 day', '1 hour':
                if timestamp > epoch_seconds(timeago(time)):
                    tkey = time.split(' ')[1]
                    yield ('sr-top-%s-%d' % (tkey, sr_id),
                           score(ups, downs), timestamp, fname)
                    yield ('sr-controversial-%s-%d' % (tkey, sr_id),
                           controversy(ups, downs),
                           timestamp, fname)
Пример #7
0
    def process(link):
        assert link.thing_type == 'link'

        author_id = link.author_id
        timestamp = link.timestamp
        fname = make_fullname(Link, link.thing_id)

        yield 'user-submitted-%d' % author_id, timestamp, fname
        if not link.spam:
            sr_id = link.sr_id
            ups, downs = link.ups, link.downs

            yield ('sr-hot-all-%d' % sr_id, _hot(ups, downs,
                                                 timestamp), timestamp, fname)
            yield 'sr-new-all-%d' % sr_id, timestamp, fname
            yield 'sr-top-all-%d' % sr_id, score(ups, downs), timestamp, fname
            yield ('sr-controversial-all-%d' % sr_id, controversy(ups, downs),
                   timestamp, fname)
            for time in '1 year', '1 month', '1 week', '1 day', '1 hour':
                if timestamp > epoch_seconds(timeago(time)):
                    tkey = time.split(' ')[1]
                    yield ('sr-top-%s-%d' % (tkey, sr_id), score(ups, downs),
                           timestamp, fname)
                    yield ('sr-controversial-%s-%d' % (tkey, sr_id),
                           controversy(ups, downs), timestamp, fname)
Пример #8
0
 def process(link):
     if not link.deleted:
         author_id = link.author_id
         ups = link.ups
         downs = link.downs
         sc = score(ups, downs)
         contr = controversy(ups, downs)
         if link.thing_type == 'link':
             fname = make_fullname(Link, link.thing_id)
         else:
             fname = make_fullname(Comment, link.thing_id)
         timestamp = link.timestamp
         date = datetime.datetime.utcfromtimestamp(timestamp)
         yield ('user-top-%s-%d' % (date.year, author_id),
                sc, timestamp, fname)
         yield ('user-controversial-%s-%d' % (date.year, author_id),
                contr, timestamp, fname)
Пример #9
0
 def process(link):
     if not link.deleted:
         author_id = link.author_id
         ups = link.ups
         downs = link.downs
         sc = score(ups, downs)
         contr = controversy(ups, downs)
         if link.thing_type == 'link':
             fname = make_fullname(Link, link.thing_id)
         else:
             fname = make_fullname(Comment, link.thing_id)
         timestamp = link.timestamp
         date = datetime.datetime.utcfromtimestamp(timestamp)
         yield ('user-top-%s-%d' % (date.year, author_id),
                sc, timestamp, fname)
         yield ('user-controversial-%s-%d' % (date.year, author_id),
                contr, timestamp, fname)
Пример #10
0
    def process(link):
        assert link.thing_type == 'link'

        timestamp = link.timestamp
        fname = make_fullname(Link, link.thing_id)

        if not link.spam:
            sr_id = link.sr_id
            ups, downs = link.ups, link.downs

            for tkey, oldest in oldests.iteritems():
                if timestamp > oldest:
                    yield ('sr-top-%s-%d' % (tkey, sr_id),
                           score(ups, downs), timestamp, fname)
                    yield ('sr-controversial-%s-%d' % (tkey, sr_id),
                           controversy(ups, downs),
                           timestamp, fname)
Пример #11
0
    def process(thing):
        if thing.deleted:
            return

        thing_cls = thingcls_by_name[thing.thing_type]
        fname = make_fullname(thing_cls, thing.thing_id)
        thing_score = score(thing.ups, thing.downs)
        thing_upvotes = upvotes(thing.ups)
        thing_controversy = controversy(thing.ups, thing.downs)

        for interval, cutoff in cutoff_by_interval.iteritems():
            if thing.timestamp < cutoff:
                continue

            yield ("user/%s/top/%s/%d" % (thing.thing_type, interval, thing.author_id),
                   thing_score, thing.timestamp, fname)
            yield ("user/%s/%s/%s/%d" % (thing.thing_type, g.voting_upvote_path, interval, thing.author_id),
                   thing_upvotes, thing.timestamp, fname)
            yield ("user/%s/%s/%s/%d" % (thing.thing_type, g.voting_controversial_path, interval, thing.author_id),
                   thing_controversy, thing.timestamp, fname)

            if thing.spam:
                continue

            if thing.thing_type == "link":
                yield ("sr/link/top/%s/%d" % (interval, thing.sr_id),
                       thing_score, thing.timestamp, fname)
                yield ("sr/link/%s/%s/%d" % (g.voting_upvote_path, interval, thing.sr_id),
                       thing_upvotes, thing.timestamp, fname)
                yield ("sr/link/%s/%s/%d" % (g.voting_controversial_path, interval, thing.sr_id),
                       thing_controversy, thing.timestamp, fname)

                if thing.url:
                    try:
                        parsed = UrlParser(thing.url)
                    except ValueError:
                        continue

                    for domain in parsed.domain_permutations():
                        yield ("domain/link/top/%s/%s" % (interval, domain),
                               thing_score, thing.timestamp, fname)
                        yield ("domain/link/%s/%s/%s" % (g.voting_upvote_path, interval, domain),
                               thing_upvotes, thing.timestamp, fname)
                        yield ("domain/link/%s/%s/%s" % (g.voting_controversial_path, interval, domain),
                               thing_controversy, thing.timestamp, fname)
Пример #12
0
    def process(link):
        assert link.thing_type == "link"

        author_id = link.author_id
        timestamp = link.timestamp
        fname = make_fullname(Link, link.thing_id)

        yield "user-submitted-%d" % author_id, timestamp, fname
        if not link.spam:
            sr_id = link.sr_id
            ups, downs = link.ups, link.downs

            yield ("sr-hot-all-%d" % sr_id, _hot(ups, downs, timestamp), timestamp, fname)
            yield "sr-new-all-%d" % sr_id, timestamp, fname
            yield "sr-top-all-%d" % sr_id, score(ups, downs), timestamp, fname
            yield ("sr-controversial-all-%d" % sr_id, controversy(ups, downs), timestamp, fname)
            for time in "1 year", "1 month", "1 week", "1 day", "1 hour":
                if timestamp > epoch_seconds(timeago(time)):
                    tkey = time.split(" ")[1]
                    yield ("sr-top-%s-%d" % (tkey, sr_id), score(ups, downs), timestamp, fname)
                    yield ("sr-controversial-%s-%d" % (tkey, sr_id), controversy(ups, downs), timestamp, fname)
Пример #13
0
    def time_listing_iter(self, thing, cutoff_by_interval):
        if thing.deleted:
            return

        thing_cls = self.thing_cls
        fname = make_fullname(thing_cls, thing.thing_id)
        scores = {k: func(thing) for k, func in self.LISTING_SORTS.iteritems()}

        for interval, cutoff in cutoff_by_interval.iteritems():
            if thing.timestamp < cutoff:
                continue

            for sort, value in scores.iteritems():
                aid = thing.author_id
                key = self.make_key("user", sort, interval, aid)
                yield (key, value, thing.timestamp, fname)

            if thing.spam:
                continue

            if thing.thing_type == "link":
                for sort, value in scores.iteritems():
                    sr_id = thing.sr_id
                    key = self.make_key("sr", sort, interval, sr_id)
                    yield (key, value, thing.timestamp, fname)

                if not thing.url:
                    continue
                try:
                    parsed = UrlParser(thing.url)
                except ValueError:
                    continue

                for d in parsed.domain_permutations():
                    for sort, value in scores.iteritems():
                        key = self.make_key("domain", sort, interval, d)
                        yield (key, value, thing.timestamp, fname)
Пример #14
0
    def time_listing_iter(self, thing, cutoff_by_interval):
        if thing.deleted:
            return

        thing_cls = self.thing_cls
        fname = make_fullname(thing_cls, thing.thing_id)
        scores = {k: func(thing) for k, func in self.LISTING_SORTS.iteritems()}

        for interval, cutoff in cutoff_by_interval.iteritems():
            if thing.timestamp < cutoff:
                continue

            for sort, value in scores.iteritems():
                aid = thing.author_id
                key = self.make_key("user", sort, interval, aid)
                yield (key, value, thing.timestamp, fname)

            if thing.spam:
                continue

            if thing.thing_type == "link":
                for sort, value in scores.iteritems():
                    sr_id = thing.sr_id
                    key = self.make_key("sr", sort, interval, sr_id)
                    yield (key, value, thing.timestamp, fname)

                if not thing.url:
                    continue
                try:
                    parsed = UrlParser(thing.url)
                except ValueError:
                    continue

                for d in parsed.domain_permutations():
                    for sort, value in scores.iteritems():
                        key = self.make_key("domain", sort, interval, d)
                        yield (key, value, thing.timestamp, fname)
Пример #15
0
 def process(rel):
     if rel.name in names:
         yield ("%s-%s" % (names[rel.name], rel.thing1_id), rel.timestamp, make_fullname(thing2_cls, rel.thing2_id))
Пример #16
0
    def process(comment):
        assert comment.thing_type == "comment"

        yield ("user-commented-%d" % comment.author_id, comment.timestamp, make_fullname(Comment, comment.thing_id))
Пример #17
0
 def process(rel):
     if rel.name in names:
         yield ('%s-%s' % (names[rel.name], rel.thing1_id), rel.timestamp,
                make_fullname(thing2_cls, rel.thing2_id))
Пример #18
0
    def process(comment):
        assert comment.thing_type == 'comment'

        yield ('user-commented-%d' % comment.author_id, comment.timestamp,
               make_fullname(Comment, comment.thing_id))