def produce_template(cls, constituents, subject, **config):
""" Helper function used by `merge`.
Produces the beginnings of a merged conglomerate message that needs to
be later filled out by a subclass.
"""
def _extract_timestamp(msg):
value = msg['timestamp']
if hasattr(value, 'timetuple'):
value = time.mktime(value.timetuple())
return value
N = len(constituents)
timestamps = [_extract_timestamp(msg) for msg in constituents]
average_timestamp = sum(timestamps) / N
# Avoid circular import
import fedmsg.meta as fm
# Optional, so, avoid importing it at the topmost level
import arrow
usernames = set(sum([
list(fm.msg2usernames(msg, **config))
for msg in constituents], []))
packages = set(sum([
list(fm.msg2packages(msg, **config))
for msg in constituents], []))
topics = set([msg['topic'] for msg in constituents])
categories = set([t.split('.')[3] for t in topics])
# Include metadata about constituent messages in the aggregate
# http://da.gd/12Eso
msg_ids = dict([
(msg['msg_id'], {
'title': fm.msg2title(msg, **config),
'subtitle': fm.msg2subtitle(msg, **config),
'subjective': fm.msg2subjective(msg, subject=subject, **config),
'link': fm.msg2link(msg, **config),
'icon': fm.msg2icon(msg, **config),
'secondary_icon': fm.msg2secondary_icon(msg, **config),
'usernames': fm.msg2usernames(msg, **config),
'packages': fm.msg2packages(msg, **config),
'objects': fm.msg2objects(msg, **config),
}) for msg in constituents])
return {
'start_time': min(timestamps),
'end_time': max(timestamps),
'timestamp': average_timestamp,
'human_time': arrow.get(average_timestamp).humanize(),
'msg_ids': msg_ids,
'usernames': usernames,
'packages': packages,
'topics': topics,
'categories': categories,
'icon': fm.msg2icon(constituents[0], **config),
}
def produce_template(cls, constituents, subject, **config):
""" Helper function used by `merge`.
Produces the beginnings of a merged conglomerate message that needs to
be later filled out by a subclass.
"""
def _extract_timestamp(msg):
value = msg['timestamp']
if hasattr(value, 'timetuple'):
value = time.mktime(value.timetuple())
return value
N = len(constituents)
timestamps = [_extract_timestamp(msg) for msg in constituents]
average_timestamp = sum(timestamps) / N
# Avoid circular import
import fedmsg.meta as fm
# Optional, so, avoid importing it at the topmost level
import arrow
usernames = set(sum([
list(fm.msg2usernames(msg, **config))
for msg in constituents], []))
packages = set(sum([
list(fm.msg2packages(msg, **config))
for msg in constituents], []))
topics = set([msg['topic'] for msg in constituents])
categories = set([t.split('.')[3] for t in topics])
# Include metadata about constituent messages in the aggregate
# http://da.gd/12Eso
msg_ids = dict([
(msg['msg_id'], {
'title': fm.msg2title(msg, **config),
'subtitle': fm.msg2subtitle(msg, **config),
'subjective': fm.msg2subjective(msg, subject=subject, **config),
'link': fm.msg2link(msg, **config),
'icon': fm.msg2icon(msg, **config),
'secondary_icon': fm.msg2secondary_icon(msg, **config),
'usernames': fm.msg2usernames(msg, **config),
'packages': fm.msg2packages(msg, **config),
'objects': fm.msg2objects(msg, **config),
}) for msg in constituents])
return {
'start_time': min(timestamps),
'end_time': max(timestamps),
'timestamp': average_timestamp,
'human_time': arrow.get(average_timestamp).humanize(),
'msg_ids': msg_ids,
'usernames': usernames,
'packages': packages,
'topics': topics,
'categories': categories,
'icon': fm.msg2icon(constituents[0], **config),
}
def upgrade():
""" This takes a *really* long time. Like, hours. """
config_paths = context.config.get_main_option('fedmsg_config_dir')
filenames = fedmsg.config._gather_configs_in(config_paths)
config = fedmsg.config.load_config(filenames=filenames)
make_processors(**config)
engine = op.get_bind().engine
m.init(engine=engine)
for msg in _page(m.Message.query.order_by(m.Message.timestamp)):
print "processing", msg.timestamp, msg.topic
if msg.users and msg.packages:
continue
changed = False
if not msg.users:
new_usernames = msg2usernames(msg.__json__(), **config)
print "Updating users to %r" % new_usernames
changed = changed or new_usernames
for new_username in new_usernames:
new_user = m.User.get_or_create(new_username)
msg.users.append(new_user)
if not msg.packages:
new_packagenames = msg2packages(msg.__json__(), **config)
print "Updating packages to %r" % new_packagenames
changed = changed or new_usernames
for new_packagename in new_packagenames:
new_package = m.Package.get_or_create(new_packagename)
msg.packages.append(new_package)
if changed and random.random() < 0.01:
# Only save if something changed.. and only do it every so often.
# We do this so that if we crash, we can kind of pick up where
# we left off. But if we do it on every change: too slow.
print " * Saving!"
m.session.commit()
m.session.commit()
