Example #1
0
    def produce_template(cls, constituents, subject, **config):
        """ Helper function used by `merge`.
        Produces the beginnings of a merged conglomerate message that needs to
        be later filled out by a subclass.
        """
        def _extract_timestamp(msg):
            value = msg['timestamp']
            if hasattr(value, 'timetuple'):
                value = time.mktime(value.timetuple())
            return value
        N = len(constituents)
        timestamps = [_extract_timestamp(msg) for msg in constituents]
        average_timestamp = sum(timestamps) / N

        # Avoid circular import
        import fedmsg.meta as fm
        # Optional, so, avoid importing it at the topmost level
        import arrow

        usernames = set(sum([
            list(fm.msg2usernames(msg, **config))
            for msg in constituents], []))
        packages = set(sum([
            list(fm.msg2packages(msg, **config))
            for msg in constituents], []))
        topics = set([msg['topic'] for msg in constituents])
        categories = set([t.split('.')[3] for t in topics])

        # Include metadata about constituent messages in the aggregate
        # http://da.gd/12Eso
        msg_ids = dict([
            (msg['msg_id'], {
                'title': fm.msg2title(msg, **config),
                'subtitle': fm.msg2subtitle(msg, **config),
                'subjective': fm.msg2subjective(msg, subject=subject, **config),
                'link': fm.msg2link(msg, **config),
                'icon': fm.msg2icon(msg, **config),
                'secondary_icon': fm.msg2secondary_icon(msg, **config),
                'usernames': fm.msg2usernames(msg, **config),
                'packages': fm.msg2packages(msg, **config),
                'objects': fm.msg2objects(msg, **config),
            }) for msg in constituents])

        return {
            'start_time': min(timestamps),
            'end_time': max(timestamps),
            'timestamp': average_timestamp,
            'human_time': arrow.get(average_timestamp).humanize(),
            'msg_ids': msg_ids,
            'usernames': usernames,
            'packages': packages,
            'topics': topics,
            'categories': categories,
            'icon': fm.msg2icon(constituents[0], **config),
        }
Example #2
0
    def produce_template(cls, constituents, subject, **config):
        """ Helper function used by `merge`.
        Produces the beginnings of a merged conglomerate message that needs to
        be later filled out by a subclass.
        """
        def _extract_timestamp(msg):
            value = msg['timestamp']
            if hasattr(value, 'timetuple'):
                value = time.mktime(value.timetuple())
            return value
        N = len(constituents)
        timestamps = [_extract_timestamp(msg) for msg in constituents]
        average_timestamp = sum(timestamps) / N

        # Avoid circular import
        import fedmsg.meta as fm
        # Optional, so, avoid importing it at the topmost level
        import arrow

        usernames = set(sum([
            list(fm.msg2usernames(msg, **config))
            for msg in constituents], []))
        packages = set(sum([
            list(fm.msg2packages(msg, **config))
            for msg in constituents], []))
        topics = set([msg['topic'] for msg in constituents])
        categories = set([t.split('.')[3] for t in topics])

        # Include metadata about constituent messages in the aggregate
        # http://da.gd/12Eso
        msg_ids = dict([
            (msg['msg_id'], {
                'title': fm.msg2title(msg, **config),
                'subtitle': fm.msg2subtitle(msg, **config),
                'subjective': fm.msg2subjective(msg, subject=subject, **config),
                'link': fm.msg2link(msg, **config),
                'icon': fm.msg2icon(msg, **config),
                'secondary_icon': fm.msg2secondary_icon(msg, **config),
                'usernames': fm.msg2usernames(msg, **config),
                'packages': fm.msg2packages(msg, **config),
                'objects': fm.msg2objects(msg, **config),
            }) for msg in constituents])

        return {
            'start_time': min(timestamps),
            'end_time': max(timestamps),
            'timestamp': average_timestamp,
            'human_time': arrow.get(average_timestamp).humanize(),
            'msg_ids': msg_ids,
            'usernames': usernames,
            'packages': packages,
            'topics': topics,
            'categories': categories,
            'icon': fm.msg2icon(constituents[0], **config),
        }
def upgrade():
    """ This takes a *really* long time.  Like, hours. """

    config_paths = context.config.get_main_option('fedmsg_config_dir')
    filenames = fedmsg.config._gather_configs_in(config_paths)

    config = fedmsg.config.load_config(filenames=filenames)

    make_processors(**config)

    engine = op.get_bind().engine
    m.init(engine=engine)
    for msg in _page(m.Message.query.order_by(m.Message.timestamp)):
        print "processing", msg.timestamp, msg.topic

        if msg.users and msg.packages:
            continue

        changed = False

        if not msg.users:
            new_usernames = msg2usernames(msg.__json__(), **config)
            print "Updating users to %r" % new_usernames
            changed = changed or new_usernames
            for new_username in new_usernames:
                new_user = m.User.get_or_create(new_username)
                msg.users.append(new_user)

        if not msg.packages:
            new_packagenames = msg2packages(msg.__json__(), **config)
            print "Updating packages to %r" % new_packagenames
            changed = changed or new_usernames
            for new_packagename in new_packagenames:
                new_package = m.Package.get_or_create(new_packagename)
                msg.packages.append(new_package)

        if changed and random.random() < 0.01:
            # Only save if something changed.. and only do it every so often.
            # We do this so that if we crash, we can kind of pick up where
            # we left off.  But if we do it on every change: too slow.
            print " * Saving!"
            m.session.commit()

    m.session.commit()