def run_initial(self, events): """Runs the initial batch upload :param events: an iterable containing events """ self_name = type(self).__name__ for i, batch in enumerate(grouper(events, self.INITIAL_BATCH_SIZE, skip_missing=True), 1): self.logger.debug('%s processing initial batch %d', self_name, i) for j, processed_batch in enumerate(grouper( batch, self.BATCH_SIZE, skip_missing=True), 1): self.logger.info('%s uploading initial chunk #%d (batch %d)', self_name, j, i) self.upload_records(processed_batch, from_queue=False)
def run_initial_export(self, events): uploader = DebugUploader(self) uploader.run_initial(events) for i, batch in enumerate(grouper(events, 10, skip_missing=True), 1): print print cformat('%{white!}Batch {}:%{reset}').format(i) print MARCXMLGenerator.objects_to_xml(event for event in batch if event is not None) print
def run(self, records): """Runs the batch upload :param records: an iterable containing queue entries """ self_name = type(self).__name__ for i, batch in enumerate(grouper(records, self.BATCH_SIZE, skip_missing=True), 1): self.logger.info('%s processing batch %d', self_name, i) try: for j, proc_batch in enumerate(grouper( process_records(batch).iteritems(), self.BATCH_SIZE, skip_missing=True), 1): self.logger.info('%s uploading chunk #%d (batch %d)', self_name, j, i) self.upload_records({k: v for k, v in proc_batch}, from_queue=True) except Exception: self.logger.exception('%s could not upload batch', self_name) return self.logger.info('%s finished batch %d', self_name, i) self.processed_records(batch) self.logger.info('%s finished', self_name)
def migrate_links(self): print cformat('%{white!}migrating links') for avatars in grouper(self._iter_avatars(), 2500, skip_missing=True): avatars = {int(a.id): a for a in avatars} users = ((u, avatars[u.id]) for u in User.find(User.id.in_(avatars))) for user, avatar in committing_iterator(self.flushing_iterator(users, 250), 2500): registrants = set() user_shown = False for type_, entries in avatar.linkedTo.iteritems(): # store registrant roles, in order to avoid duplication below for role, objects in entries.iteritems(): if (type_ == 'category' and role == 'favorite') or type_ == 'group': continue if not objects: continue if type_ == 'registration' and role == 'registrant': registrants |= set(objects) if not user_shown: print cformat('%{green}+++%{reset} ' '%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(user.id, user.full_name) user_shown = True print cformat('%{blue!}<->%{reset} ' '%{yellow!}{:4d}%{reset}x %{green!}{:12} %{cyan!}{}%{reset}').format( len(objects), type_, role) for obj in objects: try: UserLink.create_link(user, obj, role, type_) except Exception as e: print cformat('%{red!}!!!%{reset} ' '%{red!}linking failed%{reset} (%{yellow!}{}%{reset}): ' '{}').format(unicode(e), obj) # add old "registrant" entries to registration/registrant for reg in getattr(avatar, 'registrants', {}).itervalues(): if reg.getConference().getOwner() and reg not in registrants: UserLink.create_link(user, reg, 'registrant', 'registration') print cformat('%{cyan!}<->%{reset} ' '%{yellow!} 1%{reset}x %{green!}{:12} %{cyan!}{}%{reset}').format( 'registration', 'registrant')
def migrate_favorite_users(self): print cformat('%{white!}migrating favorite users') for avatars in grouper(self._iter_avatars_with_favorite_users(), 2500, skip_missing=True): avatars = list(avatars) users = {u.id: u for u in User.find(User.id.in_(int(a.id) for a, _ in avatars))} for avatar, user_ids in committing_iterator(avatars, 1000): user = users.get(int(avatar.id)) if user is None: print cformat('%{red!}!!!%{reset} ' '%{yellow!}User {} does not exist').format(avatar.id) continue print cformat('%{green}+++%{reset} ' '%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(user.id, user.full_name) valid_users = {u.id: u for u in User.find(User.id.in_(user_ids))} for user_id in user_ids: target = valid_users.get(user_id) if target is None: print cformat('%{yellow!}!!!%{reset} ' '%{yellow!}User {} does not exist').format(user_id) continue user.favorite_users.add(target) print cformat('%{blue!}<->%{reset} ' '%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(target.id, target.full_name)
def test_grouper(): assert list(grouper([], 3)) == [] assert list(grouper(xrange(6), 3)) == [(0, 1, 2), (3, 4, 5)] assert list(grouper(xrange(7), 3)) == [(0, 1, 2), (3, 4, 5), (6, None, None)] assert list(grouper(xrange(7), 3, fillvalue='x')) == [(0, 1, 2), (3, 4, 5), (6, 'x', 'x')] assert list(grouper(xrange(7), 3, skip_missing=True)) == [(0, 1, 2), (3, 4, 5), (6,)]