Esempio n. 1
0
    def _run_changed(msgs, chan):
        start = datetime.now(g.tz)

        changed = map(lambda x: strordict_fullname(x.body), msgs)

        boost = set()
        add = set()

        # an item can request that only its boost fields be updated,
        # so we need to separate those out

        for item in changed:
            fname = item['fullname']
            boost_only = item.get('boost_only', False)

            if fname in add:
                # we're already going to do all of the work
                continue

            if boost_only:
                boost.add(fname)
            else:
                if fname in boost:
                    # we've previously seen an instance of this fname
                    # that requested that only its boosts be updated,
                    # but now we have to update the whole thing
                    boost.remove(fname)

                add.add(fname)

        things = Thing._by_fullname(boost | add, data=True, return_dict=True)

        boost_time = add_time = 0.0
        if boost:
            boost_time = inject([things[fname] for fname in boost],
                                boost_only=True)
        if add:
            add_time = inject([things[fname] for fname in add])

        totaltime = epoch_seconds(datetime.now(g.tz)) - epoch_seconds(start)

        print(
            "%s: %d messages: %d docs (%.2fs), %d boosts (%.2fs) in %.2fs (%d duplicates, %s remaining)"
            % (
                start,
                len(changed),
                len(add),
                add_time,
                len(boost),
                boost_time,
                totaltime,
                len(changed) - len(things),
                msgs[-1].delivery_info.get('message_count', 'unknown'),
            ))
Esempio n. 2
0
    def _run_changed(msgs, chan):
        start = datetime.now(g.tz)

        changed = map(lambda x: strordict_fullname(x.body), msgs)

        boost = set()
        add = set()

        # an item can request that only its boost fields be updated,
        # so we need to separate those out

        for item in changed:
            fname = item["fullname"]
            boost_only = item.get("boost_only", False)

            if fname in add:
                # we're already going to do all of the work
                continue

            if boost_only:
                boost.add(fname)
            else:
                if fname in boost:
                    # we've previously seen an instance of this fname
                    # that requested that only its boosts be updated,
                    # but now we have to update the whole thing
                    boost.remove(fname)

                add.add(fname)

        things = Thing._by_fullname(boost | add, data=True, return_dict=True)

        boost_time = add_time = 0.0
        if boost:
            boost_time = inject([things[fname] for fname in boost], boost_only=True)
        if add:
            add_time = inject([things[fname] for fname in add])

        totaltime = epoch_seconds(datetime.now(g.tz)) - epoch_seconds(start)

        print(
            "%s: %d messages: %d docs (%.2fs), %d boosts (%.2fs) in %.2fs (%d duplicates, %s remaining)"
            % (
                start,
                len(changed),
                len(add),
                add_time,
                len(boost),
                boost_time,
                totaltime,
                len(changed) - len(things),
                msgs[-1].delivery_info.get("message_count", "unknown"),
            )
        )
Esempio n. 3
0
    def _run_changed(msgs, chan):
        print "changed: Processing %d items" % len(msgs)
        msgs = [strordict_fullname(msg.body) for msg in msgs]
        fullnames = set(msg['fullname'] for msg in msgs)

        things = Thing._by_fullname(fullnames, data=True, return_dict=False)
        things = [x for x in things if isinstance(x, indexed_types)]

        update_things = [x for x in things if not x._spam and not x._deleted]
        delete_things = [x for x in things if x._spam or x._deleted]

        with SolrConnection() as s:
            if update_things:
                tokenized = tokenize_things(update_things)
                s.add(tokenized)
            if delete_things:
                for i in delete_things:
                    s.delete(id=i._fullname)
Esempio n. 4
0
    def _run_changed(msgs, chan):
        print "changed: Processing %d items" % len(msgs)
        msgs = [strordict_fullname(msg.body)
                for msg in msgs]
        fullnames = set(msg['fullname'] for msg in msgs if not msg.get('boost_only'))

        things = Thing._by_fullname(fullnames, data=True, return_dict=False)
        things = [x for x in things if isinstance(x, indexed_types)]

        update_things = [x for x in things if not x._spam and not x._deleted]
        delete_things = [x for x in things if x._spam or x._deleted]

        with SolrConnection() as s:
            if update_things:
                tokenized = tokenize_things(update_things)
                s.add(tokenized)
            if delete_things:
                for i in delete_things:
                    s.delete(id=i._fullname)
Esempio n. 5
0
    def _run_changed(msgs, chan):
        changed = map(lambda x: strordict_fullname(x.body), msgs)

        boost = set()
        add = set()

        # an item can request that only its boost fields be updated,
        # so we need to separate those out

        for item in changed:
            fname = item['fullname']
            boost_only = item.get('boost_only', False)

            if fname in add:
                # we're already going to do all of the work
                continue

            if boost_only:
                boost.add(fname)
            else:
                if fname in boost:
                    # we've previously seen an instance of this fname
                    # that requested that only its boosts be updated,
                    # but now we have to update the whole thing
                    boost.remove(fname)

                add.add(fname)

        things = Thing._by_fullname(boost | add, data=True, return_dict=True)

        print ("%d messages: %d docs, %d boosts (%d duplicates, %s remaining)"
               % (len(changed),
                  len(add),
                  len(boost),
                  len(changed) - len(things),
                  msgs[-1].delivery_info.get('message_count', 'unknown'),
               ))

        if boost:
            inject([things[fname] for fname in boost], boost_only=True)
        if add:
            inject([things[fname] for fname in add])
Esempio n. 6
0
    def _run_changed(msgs, chan):
        print "changed: Processing %d items" % len(msgs)
        msgs = [strordict_fullname(msg.body)
                for msg in msgs]
        fullnames = set(msg['fullname'] for msg in msgs)

        things = Thing._by_fullname(fullnames, data=True, return_dict=False)
        things = [x for x in things if isinstance(x, indexed_types)]

        update_things = [x for x in things if not x._spam and not x._deleted]
        delete_things = [x for x in things if x._spam or x._deleted]
	#How often should we commit?
	commit=random()<.1
	if commit:
	    print "COMMITING!"

        with SolrConnection() as s:
            if update_things:
                tokenized = tokenize_things(update_things)
                s.add(tokenized,commit=commit)
            if delete_things:
                for i in delete_things:
                    s.delete(id=i._fullname)