def run_initial(self, events):
"""Runs the initial batch upload
:param events: an iterable containing events
"""
self_name = type(self).__name__
for i, batch in enumerate(grouper(events, self.INITIAL_BATCH_SIZE, skip_missing=True), 1):
self.logger.debug('%s processing initial batch %d', self_name, i)
for j, processed_batch in enumerate(grouper(
batch, self.BATCH_SIZE, skip_missing=True), 1):
self.logger.info('%s uploading initial chunk #%d (batch %d)', self_name, j, i)
self.upload_records(processed_batch, from_queue=False)
def run_initial_export(self, events):
uploader = DebugUploader(self)
uploader.run_initial(events)
for i, batch in enumerate(grouper(events, 10, skip_missing=True), 1):
print
print cformat('%{white!}Batch {}:%{reset}').format(i)
print MARCXMLGenerator.objects_to_xml(event for event in batch if event is not None)
print
def run(self, records):
"""Runs the batch upload
:param records: an iterable containing queue entries
"""
self_name = type(self).__name__
for i, batch in enumerate(grouper(records, self.BATCH_SIZE, skip_missing=True), 1):
self.logger.info('%s processing batch %d', self_name, i)
try:
for j, proc_batch in enumerate(grouper(
process_records(batch).iteritems(), self.BATCH_SIZE, skip_missing=True), 1):
self.logger.info('%s uploading chunk #%d (batch %d)', self_name, j, i)
self.upload_records({k: v for k, v in proc_batch}, from_queue=True)
except Exception:
self.logger.exception('%s could not upload batch', self_name)
return
self.logger.info('%s finished batch %d', self_name, i)
self.processed_records(batch)
self.logger.info('%s finished', self_name)
def migrate_links(self):
print cformat('%{white!}migrating links')
for avatars in grouper(self._iter_avatars(), 2500, skip_missing=True):
avatars = {int(a.id): a for a in avatars}
users = ((u, avatars[u.id]) for u in User.find(User.id.in_(avatars)))
for user, avatar in committing_iterator(self.flushing_iterator(users, 250), 2500):
registrants = set()
user_shown = False
for type_, entries in avatar.linkedTo.iteritems():
# store registrant roles, in order to avoid duplication below
for role, objects in entries.iteritems():
if (type_ == 'category' and role == 'favorite') or type_ == 'group':
continue
if not objects:
continue
if type_ == 'registration' and role == 'registrant':
registrants |= set(objects)
if not user_shown:
print cformat('%{green}+++%{reset} '
'%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(user.id, user.full_name)
user_shown = True
print cformat('%{blue!}<->%{reset} '
'%{yellow!}{:4d}%{reset}x %{green!}{:12} %{cyan!}{}%{reset}').format(
len(objects), type_, role)
for obj in objects:
try:
UserLink.create_link(user, obj, role, type_)
except Exception as e:
print cformat('%{red!}!!!%{reset} '
'%{red!}linking failed%{reset} (%{yellow!}{}%{reset}): '
'{}').format(unicode(e), obj)
# add old "registrant" entries to registration/registrant
for reg in getattr(avatar, 'registrants', {}).itervalues():
if reg.getConference().getOwner() and reg not in registrants:
UserLink.create_link(user, reg, 'registrant', 'registration')
print cformat('%{cyan!}<->%{reset} '
'%{yellow!} 1%{reset}x %{green!}{:12} %{cyan!}{}%{reset}').format(
'registration', 'registrant')
def migrate_favorite_users(self):
print cformat('%{white!}migrating favorite users')
for avatars in grouper(self._iter_avatars_with_favorite_users(), 2500, skip_missing=True):
avatars = list(avatars)
users = {u.id: u for u in User.find(User.id.in_(int(a.id) for a, _ in avatars))}
for avatar, user_ids in committing_iterator(avatars, 1000):
user = users.get(int(avatar.id))
if user is None:
print cformat('%{red!}!!!%{reset} '
'%{yellow!}User {} does not exist').format(avatar.id)
continue
print cformat('%{green}+++%{reset} '
'%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(user.id, user.full_name)
valid_users = {u.id: u for u in User.find(User.id.in_(user_ids))}
for user_id in user_ids:
target = valid_users.get(user_id)
if target is None:
print cformat('%{yellow!}!!!%{reset} '
'%{yellow!}User {} does not exist').format(user_id)
continue
user.favorite_users.add(target)
print cformat('%{blue!}<->%{reset} '
'%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(target.id, target.full_name)
def test_grouper():
assert list(grouper([], 3)) == []
assert list(grouper(xrange(6), 3)) == [(0, 1, 2), (3, 4, 5)]
assert list(grouper(xrange(7), 3)) == [(0, 1, 2), (3, 4, 5), (6, None, None)]
assert list(grouper(xrange(7), 3, fillvalue='x')) == [(0, 1, 2), (3, 4, 5), (6, 'x', 'x')]
assert list(grouper(xrange(7), 3, skip_missing=True)) == [(0, 1, 2), (3, 4, 5), (6,)]
def test_grouper():
assert list(grouper([], 3)) == []
assert list(grouper(xrange(6), 3)) == [(0, 1, 2), (3, 4, 5)]
assert list(grouper(xrange(7), 3)) == [(0, 1, 2), (3, 4, 5), (6, None, None)]
assert list(grouper(xrange(7), 3, fillvalue='x')) == [(0, 1, 2), (3, 4, 5), (6, 'x', 'x')]
assert list(grouper(xrange(7), 3, skip_missing=True)) == [(0, 1, 2), (3, 4, 5), (6,)]
