def _iter_events(self): old_events = self.zodb_root['conferences'] wfr = self.zodb_root['webfactoryregistry'] for conf in verbose_iterator(old_events.itervalues(), len(old_events), attrgetter('id'), lambda x: ''): if wfr.get(conf.id) is None: yield conf
def _iter_events(self): it = self.zodb_root['conferences'].itervalues() total = len(self.zodb_root['conferences']) if self.quiet: it = verbose_iterator(it, total, attrgetter('id'), lambda x: x.__dict__.get('title', '')) for conf in self.flushing_iterator(it): yield conf
def _iter_category_materials(self): it = self.zodb_root['categories'].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root['categories']), attrgetter('id'), attrgetter('name')) for category in self.flushing_iterator(it): for material, resources in self._iter_attachments(category): yield category, material, resources
def _iter_abstracts(self): old_events = self.zodb_root['conferences'] for old_event in verbose_iterator(old_events.itervalues(), len(old_events), attrgetter('id'), lambda x: ''): for abstract in old_event.abstractMgr._abstracts.itervalues(): yield old_event, abstract
def _iter_events(self): it = self.zodb_root["conferences"].itervalues() total = len(self.zodb_root["conferences"]) if self.quiet: it = verbose_iterator(it, total, attrgetter("id"), lambda x: x.__dict__.get("title", "")) for old_event in self.flushing_iterator(it): yield old_event
def _iter_events(self): it = self.zodb_root['conferences'].itervalues() total = len(self.zodb_root['conferences']) if self.quiet: it = verbose_iterator(it, total, attrgetter('id'), lambda x: '') for old_event in self.flushing_iterator(it): yield old_event
def _iter_wfs(self): it = self.zodb_root['webfactoryregistry'].iteritems() total = len(self.zodb_root['webfactoryregistry']) it = verbose_iterator(it, total, itemgetter(0), lambda x: '') for conf_id, wf in it: if conf_id.isdigit(): yield int(conf_id), wf
def migrate_event_series(self): self.print_step("Migrating event series") all_series = self.get_event_series() all_series_ids = set(chain.from_iterable(all_series)) events = { e.id: e for e in Event.find(Event.id.in_(all_series_ids)).options( load_only('id', 'series_id')) } for series in committing_iterator( verbose_iterator(all_series, len(all_series), lambda x: 0, lambda x: '')): series &= events.viewkeys() if len(series) < 2: self.print_warning('Skipping single-event series: {}'.format( sorted(series))) continue es = EventSeries(show_sequence_in_title=False) for id_ in series: events[id_].series = es if not self.quiet: self.print_success(repr(series)) AttachmentFolder.find( AttachmentFolder.title.op('~')('^part\d+$')).update( {AttachmentFolder.is_deleted: True}, synchronize_session=False) db.session.commit()
def _iter_evaluations(self): it = self.zodb_root['conferences'].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title')) for event in self.flushing_iterator(it): for evaluation in getattr(event, '_evaluations', []): if evaluation._questions: yield event, evaluation
def enable_features(self): self.print_step("Enabling registration features") event_ids = [x[0] for x in set(db.session.query(RegistrationForm.event_id) .filter(RegistrationForm.title == PARTICIPATION_FORM_TITLE))] it = verbose_iterator(event_ids, len(event_ids), lambda x: x, lambda x: self.zodb_root['conferences'][str(x)].title) for event_id in committing_iterator(it): set_feature_enabled(self.zodb_root['conferences'][str(event_id)], 'registration', True)
def _iter_evaluations(self): it = self.zodb_root["conferences"].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root["conferences"]), attrgetter("id"), attrgetter("title")) for event in self.flushing_iterator(it): for evaluation in getattr(event, "_evaluations", []): if evaluation._questions: yield event, evaluation
def find_merged_avatars(self): print cformat('%{white!}checking for merged avatars') for avatar in verbose_iterator( self.zodb_root['avatars'].itervalues(), len(self.zodb_root['avatars']), attrgetter('id'), lambda av: '{}, {}'.format(safe_upper(av.surName), av.name)): for merged_avatar in getattr(avatar, '_mergeFrom', []): self.merged_avatars[merged_avatar.getId()] = avatar.getId()
def find_merged_avatars(main_root): print cformat('%{white!}checking for merged avatars') avatar_id_map = {} for avatar in verbose_iterator(main_root['avatars'].itervalues(), len(main_root['avatars']), attrgetter('id'), lambda av: '{}, {}'.format(safe_upper(av.surName), av.name)): for merged_avatar in getattr(avatar, '_mergeFrom', []): avatar_id_map[merged_avatar.getId()] = avatar.getId() return avatar_id_map
def _make_verbose(self, iterator, total): return verbose_iterator( iterator, total, lambda entry: entry[0].id, lambda entry: re.sub( r'\s+', ' ', strip_control_chars(getattr( entry[0], 'title', ''))), print_total_time=True)
def _get_events(self): # we modify the conferences dict so we can't iterate over it while doing so events = self.flushing_iterator( self.zodb_root['conferences'].itervalues()) events = verbose_iterator(events, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title')) return [ event for event in events if is_legacy_id(event.id) or hasattr(event, '_old_id') ]
def _get_categories(self): # we modify the categories dict so we can't iterate over it while doing so cats = self.flushing_iterator( self.zodb_root['categories'].itervalues()) cats = verbose_iterator(cats, len(self.zodb_root['categories']), attrgetter('id'), attrgetter('name')) return [ cat for cat in cats if is_legacy_id(cat.id) or hasattr(cat, '_old_id') ]
def _iter_events(self): it = self.zodb_root['conferences'].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title')) event_ids = {id_ for id_, in db.session.query(Event.id)} for event in self.flushing_iterator(it): if int(event.id) not in event_ids: self.print_error(cformat('%{red!}Event not found in DB'), event_id=event.id) continue yield event
def find_merged_avatars(self): print cformat("%{white!}checking for merged avatars") for avatar in verbose_iterator( self.zodb_root["avatars"].itervalues(), len(self.zodb_root["avatars"]), attrgetter("id"), lambda av: "{}, {}".format(safe_upper(av.surName), av.name), ): for merged_avatar in getattr(avatar, "_mergeFrom", []): self.merged_avatars[merged_avatar.getId()] = avatar.getId()
def _iter_events(self): from sqlalchemy.orm import subqueryload query = (Event.query .filter(~Event.is_deleted) .filter(Event.old_abstracts.any() | (Event.type_ == EventType.conference)) .options(subqueryload('old_abstracts'))) it = iter(query) if self.quiet: it = verbose_iterator(query, query.count(), attrgetter('id'), lambda x: '') confs = self.zodb_root['conferences'] for event in it: yield confs[str(event.id)], event
def progress_iterator(self, description, iterable, total, get_id, get_title, print_every=10): return verbose_iterator(iterable, total, get_id, get_title, print_every=print_every)
def enable_features(self): self.print_step("Enabling registration features") event_ids = [ x[0] for x in set( db.session.query(RegistrationForm.event_id).filter( RegistrationForm.title == PARTICIPATION_FORM_TITLE)) ] it = verbose_iterator( event_ids, len(event_ids), lambda x: x, lambda x: self.zodb_root['conferences'][str(x)].title) for event_id in committing_iterator(it): set_feature_enabled(self.zodb_root['conferences'][str(event_id)], 'registration', True)
def _iter_events(self): from sqlalchemy.orm import subqueryload query = (Event.query.filter(~Event.is_deleted).filter( Event.old_abstracts.any() | (Event.type_ == EventType.conference)).options( subqueryload('old_abstracts'))) it = iter(query) if self.quiet: it = verbose_iterator(query, query.count(), attrgetter('id'), lambda x: '') confs = self.zodb_root['conferences'] for event in it: yield confs[str(event.id)], event
def _iter_participations(self): it = self.zodb_root['conferences'].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title')) for event in self.flushing_iterator(it): try: participation = event._participation except AttributeError: self.print_warning('Event has no participation', event_id=event.id) continue if participation._participantList or participation._pendingParticipantList: yield event, participation
def _iter_events(self): it = self.zodb_root["conferences"].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root["conferences"]), attrgetter("id"), attrgetter("title")) wfr = self.zodb_root["webfactoryregistry"] dmr = self.zodb_root["displayRegistery"] for event in self.flushing_iterator(it): if wfr.get(event.id) is None: # only conferences try: yield event, dmr[event.id] except KeyError: # only happens on a dev db for events created after the removal of displaymgr self.print_error("Skipping event with no displaymgr", event_id=event.id)
def _iter_events(self): query = (Event.query.filter(~Event.is_deleted).filter( db.or_( Event.contributions.any( Contribution.legacy_paper_files.any() | Contribution.legacy_paper_reviewing_roles.any()), (Event.type_ == EventType.conference)))) it = iter(query) if self.quiet: it = verbose_iterator(query, query.count(), attrgetter('id'), lambda x: '') confs = self.zodb_root['conferences'] for event in it: yield confs[str(event.id)], event
def migrate_event_shorturls(self): self.print_step("Migrating event shorturls") todo = {} done = {} for shorturl, conf, event in self._iter_shorturls(): shorturl = convert_to_unicode(shorturl) event_shorturl = convert_to_unicode(conf._sortUrlTag) if event_shorturl.lower() != shorturl.lower(): # warn about mismatch but keep the one from the mapping. # this is a bit weird but like this we never risk breaking urls self.print_warning(cformat( '%{yellow}Shorturl %{yellow!}{}%{reset}%{yellow} from mapping not matching ' 'event shorturl %{yellow!}{}%{reset}%{yellow}').format( shorturl, event_shorturl), event_id=event.id) error = self._validate_shorturl(shorturl) if error == 'url': # show obvious garbage in a less prominent way self.print_warning(cformat( '%{yellow}Shorturl %{yellow!}{}%{reset}%{yellow} is invalid: %{yellow!}{}' ).format(shorturl, error), event_id=event.id) continue elif error: self.print_warning(cformat( '%{red}Shorturl %{yellow!}{}%{reset}%{red} is invalid: %{red!}{}' ).format(shorturl, error), event_id=event.id) continue conflict = done.get(shorturl.lower()) if conflict and conflict[1] != event: # if there's a conflict caused by the previously case-sensitive url shortcuts, # discard them in both events - it's better to get a 404 error than a wrong event self.print_error(cformat( '%{red!}Shorturl %{reset}%{red}{}%{red!} collides with ' '%{reset}%{red}{}%{red!} in event %{reset}%{red}{}%{red!}; discarding both' ).format(shorturl, conflict[0], conflict[1]), event_id=event.id) del done[shorturl.lower()] del todo[conflict[1]] continue done[shorturl.lower()] = (shorturl, event) todo[event] = shorturl it = verbose_iterator(todo.iteritems(), len(todo), lambda x: x[0].id, lambda x: '') for event, shorturl in committing_iterator(it): event.url_shortcut = shorturl self.print_success('{} -> {}'.format(shorturl, event.title), event_id=event.id)
def _iter_shorturls(self): it = self.zodb_root['shorturl'].iteritems() total = len(self.zodb_root['shorturl']) it = verbose_iterator(it, total, lambda x: x[1].id, lambda x: '') for shorturl, conf in it: try: event = Event.get_one(conf.id) except NoResultFound: self.print_warning(cformat( '%{yellow!}Ignoring shorturl %{reset}%{yellow}{}%{yellow!} (event deleted)' ).format(shorturl), always=False, event_id=conf.id) continue yield shorturl, conf, event
def _iter_events(self): event_it = self.zodb_root['conferences'].itervalues() events_query = Event.find(is_deleted=False).order_by(Event.id) event_total = Event.query.count() all_events = {ev.id: ev for ev in events_query} if self.quiet: event_it = verbose_iterator(event_it, event_total, attrgetter('id'), lambda x: '') for conf in self.flushing_iterator(event_it): event = all_events.get(int(conf.id)) if event is None: self.print_error(cformat('%{red!}Event not found in DB'), event_id=conf.id) continue yield conf, event
def _iter_events(self): it = self.zodb_root['conferences'].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title')) wfr = self.zodb_root['webfactoryregistry'] dmr = self.zodb_root['displayRegistery'] for event in self.flushing_iterator(it): if wfr.get(event.id) is None: # only conferences try: yield event, dmr[event.id] except KeyError: # only happens on a dev db for events created after the removal of displaymgr self.print_error('Skipping event with no displaymgr', event_id=event.id)
def _iter_event_materials(self): it = self.zodb_root['conferences'].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title')) for event in self.flushing_iterator(it): for material, resources in self._iter_attachments(event): yield event, event, material, resources for session in event.sessions.itervalues(): for material, resources in self._iter_attachments(session): yield event, session, material, resources for contrib in event.contributions.itervalues(): for material, resources in self._iter_attachments(contrib): yield event, contrib, material, resources for subcontrib in contrib._subConts: for material, resources in self._iter_attachments(subcontrib): yield event, subcontrib, material, resources
def _iter_event_layout_data(self): it = self.zodb_root['conferences'].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title')) wfr = self.zodb_root['webfactoryregistry'] dmr = self.zodb_root['displayRegistery'] for event in self.flushing_iterator(it): wf = wfr.get(event.id) event_type = 'conference' if wf is None else wf.id try: dmgr = dmr[event.id] except KeyError: self.print_error('Skipping event with no displaymgr', event_id=event.id) continue style_mgr = getattr(dmgr, '_styleMngr', None) if event_type == 'conference' else None custom_css = getattr(style_mgr, '_css', None) if event_type == 'conference' else None yield event, event_type, dmgr, event._logo, custom_css
def migrate_event_series(self): self.print_step("Migrating event series") all_series = self.get_event_series() all_series_ids = set(chain.from_iterable(all_series)) events = {e.id: e for e in Event.find(Event.id.in_(all_series_ids)).options(load_only('id', 'series_id'))} for series in committing_iterator(verbose_iterator(all_series, len(all_series), lambda x: 0, lambda x: '')): series &= events.viewkeys() if len(series) < 2: self.print_warning('Skipping single-event series: {}'.format(sorted(series))) continue es = EventSeries(show_sequence_in_title=False) for id_ in series: events[id_].series = es if not self.quiet: self.print_success(repr(series)) AttachmentFolder.find(AttachmentFolder.title.op('~')('^part\d+$')).update({AttachmentFolder.is_deleted: True}, synchronize_session=False) db.session.commit()
def _iter_events(self): event_it = self.zodb_root['conferences'].itervalues() events_query = Event.find(is_deleted=False).order_by(Event.id) event_total = len(self.zodb_root['conferences']) if self.parallel: n, i = self.parallel event_it = (e for e in event_it if int(e.id) % n == i) event_total = int(ceil(event_total / n)) events_query = events_query.filter(Event.id % n == i) all_events = {ev.id: ev for ev in events_query} if self.quiet: event_it = verbose_iterator(event_it, event_total, attrgetter('id'), attrgetter('title')) for conf in self.flushing_iterator(event_it): event = all_events.get(int(conf.id)) if event is None: self.print_error(cformat('%{red!}Event not found in DB'), event_id=conf.id) continue yield conf, event
def _iter_pictures(self): it = iter(db.session.query(Event.id)) if self.quiet: it = verbose_iterator(it, Event.query.count(), attrgetter('id'), lambda x: self.zodb_root['conferences'][str(x.id)].title, 100) for event_id, in self.flushing_iterator(it): try: dmgr = self.zodb_root['displayRegistery'][str(event_id)] except KeyError: self.print_error('Skipping event with no displaymgr', event_id=event_id) continue imgr = getattr(dmgr, '_imagesMngr', None) if imgr: for _, picture in imgr._picList.iteritems(): yield dmgr._conf, picture else: if not self.silence_old_events: self.print_warning('No _imagesMngr attribute!', event_id=event_id)
def run_export_files(self, batch=1000, force=False, max_size=None, verbose=True, initial=False): from indico_citadel.plugin import CitadelPlugin if max_size is None: max_size = CitadelPlugin.settings.get('max_file_size') attachments = (CitadelIdMap.query.join(Attachment).join( AttachmentFile, Attachment.file_id == AttachmentFile.id ).filter(Attachment.type == AttachmentType.file).filter( AttachmentFile.size > 0, AttachmentFile.size <= max_size * 1024 * 1024).filter( db.func.lower(AttachmentFile.extension).in_([ s.lower() for s in CitadelPlugin.settings.get('file_extensions') ])).options( contains_eager(CitadelIdMap.attachment).contains_eager( Attachment.file))) if not force: attachments = attachments.filter( db.or_(CitadelIdMap.attachment_file_id.is_(None), CitadelIdMap.attachment_file_id != Attachment.file_id)) uploader = self.uploader(self) attachments = attachments.yield_per(batch) total = attachments.count() if verbose: attachments = verbose_iterator( attachments, total, attrgetter('id'), lambda obj: re.sub(r'\s+', ' ', strip_control_chars(obj.attachment.title)), print_total_time=True) else: self.plugin.logger.info(f'{total} files need to be uploaded') total, errors, aborted = uploader.upload_files(attachments, initial=initial) return total, errors, aborted
def _iter_minutes(self): it = self.zodb_root['conferences'].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title')) for event in self.flushing_iterator(it): minutes, special = self._get_minutes(event) if minutes: yield event, event, minutes, special for session in event.sessions.itervalues(): minutes, special = self._get_minutes(session) if minutes: yield event, session, minutes, special for contrib in event.contributions.itervalues(): minutes, special = self._get_minutes(contrib) if minutes: yield event, contrib, minutes, special for subcontrib in contrib._subConts: minutes, special = self._get_minutes(subcontrib) if minutes: yield event, subcontrib, minutes, special
def _iter_events(self): it = self.zodb_root["conferences"].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root["conferences"]), attrgetter("id"), attrgetter("title"), 25) return self.flushing_iterator(it)
def _get_events(self): # we modify the conferences dict so we can't iterate over it while doing so events = self.flushing_iterator(self.zodb_root['conferences'].itervalues()) events = verbose_iterator(events, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title')) return [event for event in events if is_legacy_id(event.id) or hasattr(event, '_old_id')]
def find_merged_avatars(self): print cformat('%{white!}checking for merged avatars') for avatar in verbose_iterator(self.zodb_root['avatars'].itervalues(), len(self.zodb_root['avatars']), attrgetter('id'), lambda av: '{}, {}'.format(safe_upper(av.surName), av.name)): for merged_avatar in getattr(avatar, '_mergeFrom', []): self.merged_avatars[merged_avatar.getId()] = avatar.getId()
def _iter_event_ids(self): it = iter(self.zodb_root['conferences']) return verbose_iterator(it, len(self.zodb_root['conferences']), lambda x: x, lambda x: 'n/a', 1000)
def _iter_events(self): it = self.zodb_root['conferences'].itervalues() if self.quiet: it = verbose_iterator(it, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title'), 25) return self.flushing_iterator(it)
def _get_categories(self): # we modify the categories dict so we can't iterate over it while doing so cats = self.flushing_iterator(self.zodb_root['categories'].itervalues()) cats = verbose_iterator(cats, len(self.zodb_root['categories']), attrgetter('id'), attrgetter('name')) return [cat for cat in cats if is_legacy_id(cat.id) or hasattr(cat, '_old_id')]
def _iter_event_ids(self): it = iter(self.zodb_root["conferences"]) return verbose_iterator(it, len(self.zodb_root["conferences"]), lambda x: x, lambda x: "n/a", 1000)