def migrate_event_creation_times(self): self.print_step("Migrating event creation times") for old_event in committing_iterator(self._iter_events()): Event.query.filter_by(id=int(old_event.id)).update({Event.created_dt: old_event._creationDS}, synchronize_session=False) if not self.quiet: self.print_success('', event_id=old_event.id)
def migrate_event_images(self): self.print_step('migrating event images') for event, picture in committing_iterator(self._iter_pictures()): local_file = picture._localFile content_type = mimetypes.guess_type(local_file.fileName)[0] or 'application/octet-stream' storage_backend, storage_path, size = self._get_local_file_info(local_file) if storage_path is None: self.print_warning(cformat('%{yellow}[{}]%{reset} -> %{red!}Not found in filesystem').format( local_file.id), event_id=event.id) continue filename = secure_filename(convert_to_unicode(local_file.fileName), 'image') image = ImageFile(event_id=event.id, filename=filename, content_type=content_type, created_dt=now_utc(), size=size, storage_backend=storage_backend, storage_file_id=storage_path) db.session.add(image) db.session.flush() map_entry = LegacyImageMapping(event_id=event.id, legacy_image_id=local_file.id, image_id=image.id) db.session.add(map_entry) if not self.quiet: self.print_success(cformat('%{cyan}[{}]%{reset} -> %{blue!}{}').format(local_file.id, image), event_id=event.id)
def migrate_layout_settings(self): print cformat('%{white!}migrating layout settings, event logos and custom stylesheets') default_styles = self.zodb_root['MaKaCInfo']['main']._styleMgr._defaultEventStylesheet for event, event_type, dmgr, logo, custom_css in committing_iterator(self._iter_event_layout_data()): if event_type != 'conference': theme = dmgr._defaultstyle if not theme or theme == default_styles[event_type]: continue layout_settings.set(event, 'timetable_theme', theme) if not self.quiet: self.print_success(cformat('- %{cyan}Default timetable theme: {}').format(theme), event_id=event.id) continue settings = self._get_event_settings(event, dmgr) layout_settings.set_multi(event, settings) if not self.quiet: self.print_success(cformat('- %{cyan}Layout settings'), event_id=event.id) if logo or custom_css: sa_event = Event.get(event.id) if not sa_event: self.print_warning('Event does not exist (anymore)! Logo and/or CSS file not saved!', event_id=event.id) continue if logo: self._process_logo(logo, sa_event) if custom_css: self._process_css(custom_css, sa_event)
def migrate_event_locations(self): self.print_step("Migrating event locations") for old_event in committing_iterator(self._iter_events()): custom_location = old_event.places[0] if getattr(old_event, 'places', None) else None custom_room = old_event.rooms[0] if getattr(old_event, 'rooms', None) else None location_name = None room_name = None has_room = False updates = {} if custom_location: location_name = convert_to_unicode(fix_broken_string(custom_location.name, True)) if custom_location.address: updates[Event.own_address] = convert_to_unicode(fix_broken_string(custom_location.address, True)) if custom_room: room_name = convert_to_unicode(fix_broken_string(custom_room.name, True)) if location_name and room_name: mapping = self.room_mapping.get((location_name, room_name)) if mapping: has_room = True updates[Event.own_venue_id] = mapping[0] updates[Event.own_room_id] = mapping[1] # if we don't have a RB room set, use whatever location/room name we have if not has_room: venue_id = self.venue_mapping.get(location_name) if venue_id is not None: updates[Event.own_venue_id] = venue_id updates[Event.own_venue_name] = '' else: updates[Event.own_venue_name] = location_name or '' updates[Event.own_room_name] = room_name or '' if updates: Event.query.filter_by(id=int(old_event.id)).update(updates, synchronize_session=False) if not self.quiet: self.print_success(repr(updates), event_id=old_event.id)
def migrate_event_dates_titles(self): self.print_step("Migrating event dates and titles") for old_event in committing_iterator(self._iter_events()): if 'title' not in old_event.__dict__: self.print_error('Event has no title in ZODB', old_event.id) continue tz = old_event.__dict__.get('timezone', 'UTC') updates = { Event.title: convert_to_unicode(old_event.__dict__['title']) or '(no title)', Event.description: convert_to_unicode(old_event.__dict__['description']) or '', Event.timezone: tz, Event.start_dt: self._fix_naive(old_event, old_event.__dict__['startDate'], tz), Event.end_dt: self._fix_naive(old_event, old_event.__dict__['endDate'], tz) } Event.query.filter_by(id=int(old_event.id)).update(updates, synchronize_session=False) if not self.quiet: self.print_success('', event_id=old_event.id) # deleted events are not in zodb but also need data updates = {Event.title: '***deleted***', Event.description: '', Event.timezone: 'UTC', Event.start_dt: datetime(1970, 1, 1, tzinfo=pytz.utc), Event.end_dt: datetime(1970, 1, 1, tzinfo=pytz.utc)} Event.query.filter_by(is_deleted=True).update(updates, synchronize_session=False) db.session.commit()
def migrate(self): print cformat('%{white!}migrating static sites') for item in committing_iterator(chain.from_iterable( self.zodb_root['modules']['offlineEvents']._idxConf.itervalues())): event_id = item.conference.id if is_legacy_id(event_id): print cformat('%{red!}!!!%{reset} ' '%{white!}{0:6s}%{reset} %{yellow!}Event has non-numeric/broken ID').format(event_id) continue if event_id not in self.zodb_root['conferences']: print cformat('%{red!}!!!%{reset} ' '%{white!}{0:6s}%{reset} %{yellow!}Event deleted, skipping static site').format(event_id) continue event_id = int(event_id) user = self._get_user(item.avatar.id) state = STATE_MAPPING[item.status] requested_dt = item.requestTime file_name, file_path = self._get_file_data(item.file) if file_path is None and state == StaticSiteState.success: print cformat('%{yellow!}!!!%{reset} %{white!}{0:6d}%{reset} ' '%{yellow!}file missing, marking static site as expired.').format(event_id) state = StaticSite.expired static_site = StaticSite(creator=user, event_id=event_id, state=state, requested_dt=requested_dt) if static_site.state == StaticSiteState.success: static_site.path = file_path db.session.add(static_site) print cformat('%{green}+++%{reset} %{white!}{0.event_id:6d}%{reset} ' '%{cyan}{0}%{reset}').format(static_site)
def migrate_event_acls(self): self.print_step('migrating event ACLs') protection_mode_map = {-1: ProtectionMode.public, 0: ProtectionMode.inheriting, 1: ProtectionMode.protected} for legacy_event, event in committing_iterator(self._iter_events(), 5000): ac = legacy_event._Conference__ac self.print_success('', event_id=event.id) old_protection_mode = protection_mode_map[ac._accessProtection] if old_protection_mode == ProtectionMode.public and ac.requiredDomains: event.protection_mode = ProtectionMode.protected self._migrate_domains(event, ac.requiredDomains) else: event.protection_mode = old_protection_mode no_access_contact = convert_to_unicode(getattr(ac, 'contactInfo', '')) if no_access_contact != 'no contact info defined': event.own_no_access_contact = no_access_contact event.access_key = convert_to_unicode(getattr(legacy_event, '_accessKey', '')) if not self.quiet: self.print_success('Protection mode set to {}'.format(event.protection_mode.name, event_id=event.id)) for legacy_acl in ac.allowed: event_acl = self.convert_acl(legacy_acl) if event_acl is None: self.print_warning(cformat('%{red}ACL%{reset}%{yellow} does not exist:%{reset} {}') .format(legacy_acl), event_id=event.id) continue event.update_principal(event_acl, read_access=True, quiet=True) if not self.quiet: self.print_msg(cformat('%{green}[{}]%{reset} {}').format('Event ACL', event_acl))
def migrate_agents(self): print cformat('%{white!}migrating agents') for old_agent in committing_iterator(self.livesync_root['agent_manager']._agents.itervalues()): if not old_agent._active: print cformat('%{yellow}skipping inactive agent {} ({})%{reset}').format(old_agent._id, old_agent._name) continue agent = LiveSyncAgent(name=convert_to_unicode(old_agent._name), initial_data_exported=True) old_agent_class = old_agent.__class__.__name__ if old_agent_class == 'InvenioBatchUploaderAgent': agent.backend_name = 'invenio' agent.settings = { 'server_url': old_agent._url } elif old_agent_class == 'CERNSearchUploadAgent': agent.backend_name = 'cernsearch' agent.settings = { 'server_url': old_agent._url, 'username': old_agent._username, 'password': old_agent._password, } else: print cformat('%{red!}skipping unknown agent type: {}%{reset}').format(old_agent_class) continue print cformat('- %{cyan}{} ({})').format(agent.name, agent.backend_name) db.session.add(agent)
def migrate_event_notes(self): self.print_step('migrating event notes') janitor_user = User.get_one(self.janitor_user_id) self.print_msg('Using janitor user {}'.format(janitor_user), always=True) for event, obj, minutes, special_prot in committing_iterator(self._iter_minutes()): if special_prot: self.print_warning( cformat('%{yellow!}{} minutes have special permissions; skipping them').format(obj), event_id=event.id ) continue path = get_archived_file(minutes, self.archive_dirs)[1] if path is None: self.print_error(cformat('%{red!}{} minutes not found on disk; skipping them').format(obj), event_id=event.id) continue with open(path, 'r') as f: data = convert_to_unicode(f.read()).strip() if not data: self.print_warning(cformat('%{yellow}{} minutes are empty; skipping them').format(obj), always=False, event_id=event.id) continue note = EventNote(linked_object=obj) note.create_revision(RenderMode.html, data, janitor_user) db.session.add(note) if not self.quiet: self.print_success(cformat('%{cyan}{}').format(obj), event_id=event.id)
def vidyo_cleanup(dry_run=False): from indico_vc_vidyo.plugin import VidyoPlugin max_room_event_age = VidyoPlugin.settings.get('num_days_old') VidyoPlugin.logger.info('Deleting Vidyo rooms that are not used or linked to events all older than %d days', max_room_event_age) candidate_rooms = find_old_vidyo_rooms(max_room_event_age) VidyoPlugin.logger.info('%d rooms found', len(candidate_rooms)) if dry_run: for vc_room in candidate_rooms: VidyoPlugin.logger.info('Would delete Vidyo room %s from server', vc_room) return for vc_room in committing_iterator(candidate_rooms, n=20): try: VidyoPlugin.instance.delete_room(vc_room, None) VidyoPlugin.logger.info('Room %s deleted from Vidyo server', vc_room) notify_owner(VidyoPlugin.instance, vc_room) vc_room.status = VCRoomStatus.deleted except RoomNotFoundAPIException: VidyoPlugin.logger.warning('Room %s had been already deleted from the Vidyo server', vc_room) vc_room.status = VCRoomStatus.deleted except APIException: VidyoPlugin.logger.exception('Impossible to delete Vidyo room %s', vc_room)
def migrate_groups(self): print cformat('%{white!}migrating groups') for old_group in committing_iterator(self.zodb_root['groups'].itervalues()): if old_group.__class__.__name__ != 'Group': continue group = LocalGroup(id=int(old_group.id), name=convert_to_unicode(old_group.name).strip()) print cformat('%{green}+++%{reset} %{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(group.id, group.name) members = set() for old_member in old_group.members: if old_member.__class__.__name__ != 'Avatar': print cformat('%{yellow!}!!! Unsupported group member type: {}').format( old_member.__class__.__name__) continue user = User.get(int(old_member.id)) if user is None: print cformat('%{yellow!}!!! User not found: {}').format(old_member.id) continue while user.merged_into_user: user = user.merged_into_user if user.is_deleted: print cformat('%{yellow!}!!! User deleted: {}').format(user.id) continue members.add(user) for member in sorted(members, key=attrgetter('full_name')): print cformat('%{blue!}<->%{reset} %{white!}{:6d} %{yellow}{} ({})').format( member.id, member.full_name, member.email) group.members = members db.session.add(group)
def migrate_regforms(self): self.print_step("Migrating participants") for event, participation in committing_iterator(self._iter_participations(), 10): mig = ParticipationMigration(self, event, participation) with db.session.no_autoflush: mig.run() db.session.add(mig.regform) db.session.flush()
def enable_features(self): self.print_step("Enabling registration features") event_ids = [x[0] for x in set(db.session.query(RegistrationForm.event_id) .filter(RegistrationForm.title == PARTICIPATION_FORM_TITLE))] it = verbose_iterator(event_ids, len(event_ids), lambda x: x, lambda x: self.zodb_root['conferences'][str(x)].title) for event_id in committing_iterator(it): set_feature_enabled(self.zodb_root['conferences'][str(event_id)], 'registration', True)
def migrate_event_menus(self): self.print_step("migrating conference menus") for event, display_mgr in committing_iterator(self._iter_events()): if _get_menu_structure(display_mgr) in DEFAULT_MENU_STRUCTURES: continue self.print_success("", event_id=event.id) db.session.add_all(self._migrate_menu(event, display_mgr._menu)) layout_settings.set(event, "use_custom_menu", True)
def migrate_transactions(self): self.messages.append(cformat("%{magenta!} - Payment Transactions:")) print cformat("%{white!}migrating payment transactions") count = 0 errors = 0 warnings = 0 for event, registrant, transaction in committing_iterator(self._iter_transactions()): try: data = self._get_transaction_data(transaction, event) except ValueError as e: print cformat("%{red!}{0} (evt: {1}, reg: {2})").format(e, event.id, registrant._id) errors += 1 continue if data["provider"] == "_manual" and data["amount"] == 0.0: print cformat( "%{yellow!}Skipping {0[provider]} transaction (evt: {1}, reg: {2}) " "with zero amount: {0[amount]} {0[currency]}" ).format(data, event.id, registrant._id) warnings += 1 continue elif data["amount"] < 0.0: print cformat( "%{yellow!}Skipping {0[provider]} transaction (evt: {1}, reg: {2}) " "with negative amount: {0[amount]} {0[currency]}" ).format(data, event.id, registrant._id) warnings += 1 continue pt = PaymentTransaction( event_id=int(event.id), registrant_id=int(registrant._id), status=TransactionStatus.successful, **data ) count += 1 print cformat("%{cyan}{0}").format(pt) db.session.add(pt) if warnings: warning_msg = cformat( "%{yellow!}There were {0} warnings during the migration " "of the payment transactions." ).format(warnings) self.messages.append(" " + warning_msg) print warning_msg if errors: msg = cformat( "%{red!}There were some errors during the migration of the payment transactions.\n" "{0} transactions were not migrated and will be lost.\n" ).format(errors) else: msg = cformat("%{green!}migration of {0} payment transactions successful\n").format(count) self.messages.append(" " + "\n ".join(msg.split("\n"))) print msg
def migrate_event_keywords(self): self.print_step("Migrating event keywords") for old_event in committing_iterator(self._iter_events()): old_keywords = old_event._keywords keywords = filter(None, map(unicode.strip, map(convert_to_unicode, old_keywords.splitlines()))) if not keywords: continue Event.query.filter_by(id=int(old_event.id)).update({Event.keywords: keywords}, synchronize_session=False) if not self.quiet: self.print_success(repr(keywords), event_id=old_event.id)
def migrate_event_bookings(self): self.vc_rooms_by_extension = {} with VidyoPlugin.instance.plugin_context(): for event_id, csbm in committing_iterator(self.booking_root.iteritems(), n=1000): for bid, booking in csbm._bookings.iteritems(): if booking._type == 'Vidyo': vc_room = self.vc_rooms_by_extension.get(int(booking._extension)) if not vc_room: vc_room = self.migrate_vidyo_room(booking) self.migrate_event_booking(vc_room, booking)
def migrate_surveys(self): self.print_step("Migrating old event evaluations") notification_pending = {es.event_id: es.value for es in EventSetting.find_all(module='evaluation', name='send_notification')} for event, evaluation in committing_iterator(self._iter_evaluations(), 10): survey = self.migrate_survey(evaluation, event) if (survey.notifications_enabled and survey.start_notification_emails and not notification_pending.get(survey.event_id)): survey.start_notification_sent = True db.session.add(survey)
def migrate_event_keywords(self): self.print_step("Migrating event keywords, creation times and visibility") for old_event in committing_iterator(self._iter_events()): updates = {Event.created_dt: old_event._creationDS, Event.visibility: self._convert_visibility(old_event._visibility)} keywords = self._convert_keywords(old_event._keywords) if keywords: updates[Event.keywords] = keywords Event.query.filter_by(id=int(old_event.id)).update(updates, synchronize_session=False) if not self.quiet: self.print_success(repr(keywords), event_id=old_event.id)
def migrate_admins(self): print cformat('%{white!}migrating admins') for avatar in committing_iterator(self.zodb_root['adminlist']._AdminList__list): try: user = User.get(int(avatar.id)) except ValueError: continue if user is None or user.is_deleted: continue user.is_admin = True print cformat('%{green}+++%{reset} %{cyan}{}').format(user)
def migrate_networks(self): self.print_step('migrating networks') for domain in committing_iterator(self._iter_domains()): ip_networks = filter(None, map(self._to_network, set(domain.filterList))) if not ip_networks: self.print_warning(cformat('%{yellow}Domain has no valid IPs: {}') .format(convert_to_unicode(domain.name))) network = IPNetworkGroup(name=convert_to_unicode(domain.name), description=convert_to_unicode(domain.description), networks=ip_networks) db.session.add(network) self.print_success(repr(network)) db.session.flush()
def migrate_event_managers(self): self.print_step("migrating event managers/creators") creator_updates = [] for event in committing_iterator(self._iter_events(), 5000): self.print_success("", event_id=event.id) ac = event._Conference__ac entries = {} # add creator as a manager try: creator = event._Conference__creator except AttributeError: # events created after the removal of the `self.__creator` assignment # should happen only on dev machines self.print_error(cformat("%{red!}Event has no creator attribute"), event_id=event.id) else: user = self.process_principal(event, entries, creator, "Creator", "green!", full_access=True) if user: creator_updates.append({"event_id": int(event.id), "creator_id": user.id}) # add managers for manager in ac.managers: self.process_principal(event, entries, manager, "Manager", "blue!", full_access=True) # add email-based managers emails = getattr(ac, "managersEmail", []) self.process_emails(event, entries, emails, "Manager", "green", full_access=True) # add registrars for registrar in getattr(event, "_Conference__registrars", []): self.process_principal(event, entries, registrar, "Registrar", "cyan", roles={"registration"}) # add submitters for submitter in getattr(ac, "submitters", []): self.process_principal(event, entries, submitter, "Submitter", "magenta!", roles={"submit"}) # email-based (pending) submitters pqm = getattr(event, "_pendingQueuesMgr", None) if pqm is not None: emails = set(getattr(pqm, "_pendingConfSubmitters", [])) self.process_emails(event, entries, emails, "Submitter", "magenta", roles={"submit"}) db.session.add_all(entries.itervalues()) # assign creators if creator_updates: self.print_step("saving event creators") stmt = ( Event.__table__.update() .where(Event.id == db.bindparam("event_id")) .values(creator_id=db.bindparam("creator_id")) ) db.session.execute(stmt, creator_updates) updated = Event.find(Event.creator_id == None).update({Event.creator_id: self.janitor.id}) # noqa db.session.commit() self.print_success("Set the janitor user {} for {} events".format(self.janitor, updated), always=True)
def migrate_event_keywords(self): self.print_step( "Migrating event keywords, creation times and visibility") for old_event in committing_iterator(self._iter_events()): updates = { Event.created_dt: old_event._creationDS, Event.visibility: self._convert_visibility(old_event._visibility) } keywords = self._convert_keywords(old_event._keywords) if keywords: updates[Event.keywords] = keywords Event.query.filter_by(id=int(old_event.id)).update( updates, synchronize_session=False) if not self.quiet: self.print_success(repr(keywords), event_id=old_event.id)
def migrate_networks(self): self.print_step('migrating networks') for domain in committing_iterator(self._iter_domains()): ip_networks = filter(None, map(self._to_network, set(domain.filterList))) if not ip_networks: self.print_warning( cformat('%{yellow}Domain has no valid IPs: {}').format( convert_to_unicode(domain.name))) network = IPNetworkGroup(name=convert_to_unicode(domain.name), description=convert_to_unicode( domain.description), networks=ip_networks) db.session.add(network) self.print_success(repr(network)) db.session.flush()
def migrate_event_logs(self): self.print_step("migrating event logs") msg_email = cformat("%{white!}{:6d}%{reset} %{cyan}{}") msg_action = cformat("%{white!}{:6d}%{reset} %{cyan!}{}") for event in committing_iterator(self._iter_events()): for item in event._logHandler._logLists["emailLog"]: entry = self._migrate_email_log(event, item) db.session.add(entry) if not self.quiet: self.print_success(msg_email.format(entry.event_id, entry)) for item in event._logHandler._logLists["actionLog"]: entry = self._migrate_action_log(event, item) db.session.add(entry) if not self.quiet: self.print_success(msg_action.format(entry.event_id, entry))
def migrate_event_managers(self): self.print_step('migrating event managers/creators') creator_updates = [] for event in committing_iterator(self._iter_events(), 5000): self.print_success('', event_id=event.id) ac = event._Conference__ac entries = {} # add creator as a manager try: creator = event._Conference__creator except AttributeError: # events created after the removal of the `self.__creator` assignment # should happen only on dev machines self.print_error(cformat('%{red!}Event has no creator attribute'), event_id=event.id) else: user = self.process_principal(event, entries, creator, 'Creator', 'green!', full_access=True) if user: creator_updates.append({'event_id': int(event.id), 'creator_id': user.id}) # add managers for manager in ac.managers: self.process_principal(event, entries, manager, 'Manager', 'blue!', full_access=True) # add email-based managers emails = getattr(ac, 'managersEmail', []) self.process_emails(event, entries, emails, 'Manager', 'green', full_access=True) # add registrars for registrar in getattr(event, '_Conference__registrars', []): self.process_principal(event, entries, registrar, 'Registrar', 'cyan', roles={'registration'}) # add submitters for submitter in getattr(ac, 'submitters', []): self.process_principal(event, entries, submitter, 'Submitter', 'magenta!', roles={'submit'}) # email-based (pending) submitters pqm = getattr(event, '_pendingQueuesMgr', None) if pqm is not None: emails = set(getattr(pqm, '_pendingConfSubmitters', [])) self.process_emails(event, entries, emails, 'Submitter', 'magenta', roles={'submit'}) db.session.add_all(entries.itervalues()) # assign creators if creator_updates: self.print_step('saving event creators') stmt = (Event.__table__.update() .where(Event.id == db.bindparam('event_id')) .values(creator_id=db.bindparam('creator_id'))) db.session.execute(stmt, creator_updates) updated = Event.find(Event.creator_id == None).update({Event.creator_id: self.janitor.id}) # noqa db.session.commit() self.print_success('Set the janitor user {} for {} events'.format(self.janitor, updated), always=True)
def migrate_event_logs(self): self.print_step('migrating event logs') msg_email = cformat('%{white!}{:6d}%{reset} %{cyan}{}') msg_action = cformat('%{white!}{:6d}%{reset} %{cyan!}{}') for event in committing_iterator(self._iter_events()): for item in event._logHandler._logLists['emailLog']: entry = self._migrate_email_log(event, item) db.session.add(entry) if not self.quiet: self.print_success(msg_email.format(entry.event_id, entry)) for item in event._logHandler._logLists['actionLog']: entry = self._migrate_action_log(event, item) db.session.add(entry) if not self.quiet: self.print_success(msg_action.format( entry.event_id, entry))
def migrate_event_managers(self): self.print_step('migrating event managers/creators') creator_updates = [] for event in committing_iterator(self._iter_events(), 5000): self.print_success('', event_id=event.id) ac = event._Conference__ac entries = {} # add creator as a manager try: creator = event._Conference__creator except AttributeError: # events created after the removal of the `self.__creator` assignment # should happen only on dev machines self.print_error(cformat('%{red!}Event has no creator attribute'), event_id=event.id) else: user = self.process_principal(event, entries, creator, 'Creator', 'green!', full_access=True) if user: creator_updates.append({'event_id': int(event.id), 'creator_id': user.id}) # add managers for manager in ac.managers: self.process_principal(event, entries, manager, 'Manager', 'blue!', full_access=True) # add email-based managers emails = getattr(ac, 'managersEmail', []) self.process_emails(event, entries, emails, 'Manager', 'green', full_access=True) # add registrars for registrar in getattr(event, '_Conference__registrars', []): self.process_principal(event, entries, registrar, 'Registrar', 'cyan', roles={'registration'}) # add submitters for submitter in getattr(ac, 'submitters', []): self.process_principal(event, entries, submitter, 'Submitter', 'magenta!', roles={'submit'}) # email-based (pending) submitters pqm = getattr(event, '_pendingQueuesMgr', None) if pqm is not None: emails = set(getattr(pqm, '_pendingConfSubmitters', [])) self.process_emails(event, entries, emails, 'Submitter', 'magenta', roles={'submit'}) db.session.add_all(entries.itervalues()) # assign creators if creator_updates: self.print_step('saving event creators') stmt = (Event.__table__.update() .where(Event.id == db.bindparam('event_id')) .values(creator_id=db.bindparam('creator_id'))) db.session.execute(stmt, creator_updates) updated = Event.find(Event.creator_id == None).update({Event.creator_id: self.janitor.id}) # noqa db.session.commit() self.print_success('Set the janitor user {} for {} events'.format(self.janitor, updated), always=True)
def migrate_event_settings(self): self.messages.append(cformat("%{magenta!} - Event Payment Settings:")) print cformat("%{white!}migrating event settings") count = 0 EventSetting.delete_all(payment_event_settings.module) for event in committing_iterator(self._iter_events()): old_payment = event._modPay default_conditions = global_settings.get("conditions") default_register_email = global_settings.get("register_email") default_success_email = global_settings.get("success_email") register_email = getattr(old_payment, "receiptMsg", default_register_email) success_email = getattr(old_payment, "successMsg", default_success_email) conditions = ( getattr(old_payment, "paymentConditions", default_conditions) if ( getattr(old_payment, "paymentConditionsEnabled", False) and convert_to_unicode(getattr(old_payment, "specificPaymentConditions", "")).strip() == "" ) else getattr(old_payment, "specificPaymentConditions", "") ) # The new messages are shown in an "additional info" section, so the old defaults can always go away if convert_to_unicode(register_email) == "Please, see the summary of your order:": register_email = "" if convert_to_unicode(success_email) == "Congratulations, your payment was successful.": success_email = "" # Get rid of the most terrible part of the old default conditions conditions = convert_to_unicode(conditions).replace("CANCELLATION :", "CANCELLATION:") settings = { "enabled": getattr(old_payment, "activated", False), "currency": event._registrationForm._currency, "conditions": conditions, "register_email": register_email, "success_email": success_email, } payment_event_settings.set_multi(event, settings) count += 1 print cformat("%{cyan}<EventSettings(id={id:>6}, enabled={enabled}, " "currency={currency})>").format( id=event.id, **settings ) msg = cformat("%{green!}migration of {0} event payment settings successful\n").format(count) self.messages.append(" " + msg) print msg
def migrate_evaluation_alarms(self): print cformat('%{white!}migrating evaluation alarms') today = date.today() for task in committing_iterator(self._iter_tasks('EvaluationAlarm')): start_date = task.conf._evaluations[0].startDate.date() if start_date < today: print cformat('%{yellow!}!!!%{reset} ' '%{white!}{:6d}%{reset} %{yellow}evaluation starts in the past ({})').format( int(task.conf.id), start_date) elif not task.conf._evaluations[0].visible: print cformat('%{yellow!}!!!%{reset} ' '%{white!}{:6d}%{reset} %{yellow}evaluation is disabled').format(int(task.conf.id)) else: print cformat('%{green}+++%{reset} ' '%{white!}{:6d}%{reset} %{cyan}{}').format(int(task.conf.id), start_date) evaluation_settings.set(task.conf, 'send_notification', True)
def migrate(self): print cformat('%{white!}migrating static sites') for item in committing_iterator( chain.from_iterable(self.zodb_root['modules'] ['offlineEvents']._idxConf.itervalues())): event_id = item.conference.id if is_legacy_id(event_id): print cformat( '%{red!}!!!%{reset} ' '%{white!}{0:6s}%{reset} %{yellow!}Event has non-numeric/broken ID' ).format(event_id) continue if event_id not in self.zodb_root['conferences']: print cformat( '%{red!}!!!%{reset} ' '%{white!}{0:6s}%{reset} %{yellow!}Event deleted, skipping static site' ).format(event_id) continue event_id = int(event_id) user = self._get_user(item.avatar.id) state = STATE_MAPPING[item.status] requested_dt = item.requestTime file_name, file_path = self._get_file_data(item.file) if file_path is None and state == StaticSiteState.success: print cformat( '%{yellow!}!!!%{reset} %{white!}{0:6d}%{reset} ' '%{yellow!}file missing, marking static site as expired.' ).format(event_id) state = StaticSite.expired static_site = StaticSite(creator=user, event_id=event_id, state=state, requested_dt=requested_dt) if static_site.state == StaticSiteState.success: static_site.path = file_path db.session.add(static_site) print cformat( '%{green}+++%{reset} %{white!}{0.event_id:6d}%{reset} ' '%{cyan}{0}%{reset}').format(static_site)
def migrate_event_settings(self): print cformat('%{white!}migrating event settings') default_method_name = PaypalPaymentPlugin.settings.get('method_name') EventSetting.delete_all(PaypalPaymentPlugin.event_settings.module) account_id_re = re.compile(r'^[a-zA-Z0-9]{13}$') for event in committing_iterator(self._iter_events(), 25): pp = event._modPay.payMods['PayPal'] business = pp._business.strip() if not business or (not is_valid_mail(business, multi=False) and not account_id_re.match(business)): print cformat(' - %{yellow!}event {} skipped (business: {})').format(event.id, business or '(none)') continue PaypalPaymentPlugin.event_settings.set(event, 'enabled', True) method_name = convert_to_unicode(pp._title) if method_name.lower() == 'paypal': method_name = default_method_name PaypalPaymentPlugin.event_settings.set(event, 'method_name', method_name) PaypalPaymentPlugin.event_settings.set(event, 'business', pp._business) print cformat(' - %{cyan}event {} (business: {})').format(event.id, pp._business)
def migrate_event_settings(self): print cformat('%{white!}migrating event settings') default_method_name = PaypalPaymentPlugin.settings.get('method_name') EventSetting.delete_all(PaypalPaymentPlugin.event_settings.module) account_id_re = re.compile(r'^[a-zA-Z0-9]{13}$') for event in committing_iterator(self._iter_events(), 25): pp = event._modPay.payMods['PayPal'] business = pp._business.strip() if not business or (not is_valid_mail(business, multi=False) and not account_id_re.match(business)): print cformat(' - %{yellow!}event {} skipped (business: {})').format(event.id, business or '(none)') continue PaypalPaymentPlugin.event_settings.set(event, 'enabled', True) method_name = convert_to_unicode(pp._title) if method_name.lower() == 'paypal': method_name = default_method_name PaypalPaymentPlugin.event_settings.set(event, 'method_name', method_name) PaypalPaymentPlugin.event_settings.set(event, 'business', pp._business) print cformat(' - %{cyan}event {} (business: {})').format(event.id, pp._business)
def migrate_event_series(self): self.print_step("Migrating event series") all_series = self.get_event_series() all_series_ids = set(chain.from_iterable(all_series)) events = {e.id: e for e in Event.find(Event.id.in_(all_series_ids)).options(load_only('id', 'series_id'))} for series in committing_iterator(verbose_iterator(all_series, len(all_series), lambda x: 0, lambda x: '')): series &= events.viewkeys() if len(series) < 2: self.print_warning('Skipping single-event series: {}'.format(sorted(series))) continue es = EventSeries(show_sequence_in_title=False) for id_ in series: events[id_].series = es if not self.quiet: self.print_success(repr(series)) AttachmentFolder.find(AttachmentFolder.title.op('~')('^part\d+$')).update({AttachmentFolder.is_deleted: True}, synchronize_session=False) db.session.commit()
def migrate_event_series(self): self.print_step("Migrating event series") all_series = self.get_event_series() all_series_ids = set(chain.from_iterable(all_series)) events = {e.id: e for e in Event.find(Event.id.in_(all_series_ids)).options(load_only('id', 'series_id'))} for series in committing_iterator(verbose_iterator(all_series, len(all_series), lambda x: 0, lambda x: '')): series &= events.viewkeys() if len(series) < 2: self.print_warning('Skipping single-event series: {}'.format(sorted(series))) continue es = EventSeries(show_sequence_in_title=False) for id_ in series: events[id_].series = es if not self.quiet: self.print_success(repr(series)) AttachmentFolder.find(AttachmentFolder.title.op('~')('^part\d+$')).update({AttachmentFolder.is_deleted: True}, synchronize_session=False) db.session.commit()
def migrate_event_logs(self): self.print_step('migrating event logs') msg_email = cformat('%{white!}{:6d}%{reset} %{cyan}{}') msg_action = cformat('%{white!}{:6d}%{reset} %{cyan!}{}') for event in committing_iterator(self._iter_events()): if not hasattr(event, '_logHandler'): self.print_error('Event has no log handler!', event_id=event.id) continue for item in event._logHandler._logLists['emailLog']: entry = self._migrate_email_log(event, item) db.session.add(entry) if not self.quiet: self.print_success(msg_email.format(entry.event_id, entry)) for item in event._logHandler._logLists['actionLog']: entry = self._migrate_action_log(event, item) db.session.add(entry) if not self.quiet: self.print_success(msg_action.format(entry.event_id, entry))
def migrate_event_abstracts(self): self.print_step("Migrating event abstracts") for conf, event in committing_iterator(self._iter_events()): amgr = conf.abstractMgr duration = amgr._submissionEndDate - amgr._submissionStartDate if (not amgr._activated and not amgr._abstracts and not amgr._notifTpls and duration < timedelta(minutes=1) and not conf.program): continue mig = AbstractMigration(self, conf, event) try: with db.session.begin_nested(): with db.session.no_autoflush: mig.run() db.session.flush() except Exception: self.print_error(cformat('%{red!}MIGRATION FAILED!'), event_id=event.id) traceback.print_exc() raw_input('Press ENTER to continue') db.session.flush()
def migrate_groups(self): it = committing_iterator(self.zodb_root['groups'].itervalues()) used_names = set() for old_group in self.logger.progress_iterator( 'Migrating groups', it, len(self.zodb_root['groups']), attrgetter('id'), lambda x: ''): if old_group.__class__.__name__ != 'Group': continue group_name = orig_group_name = convert_to_unicode( old_group.name).strip() n = 0 while group_name.lower() in used_names: group_name = '{}-{}'.format(orig_group_name, n) n += 1 self.print_warning( 'Duplicate group name: {}, using {} instead'.format( orig_group_name, group_name)) used_names.add(group_name.lower()) group = LocalGroup(id=int(old_group.id), name=group_name) self.print_success( '%[white!]{:6d}%[reset] %[cyan]{}%[reset]'.format( group.id, group.name)) members = set() for old_member in old_group.members: if old_member.__class__.__name__ != 'Avatar': self.print_warning( 'Unsupported group member type: {}'.format( old_member.__class__.__name__)) continue user = self.global_ns.avatar_merged_user.get(old_member.id) if user is None: self.print_warning('User not found: {}'.format( old_member.id)) continue members.add(user) for member in sorted(members, key=attrgetter('full_name')): self.print_info( '%[blue!]<->%[reset] %[white!]{:6d} %[yellow]{} ({})' .format(member.id, member.full_name, member.email)) group.members = members self.global_ns.all_groups[group.id] = group db.session.add(group) db.session.flush()
def update_gravatars(user=None): if user is not None: # explicitly scheduled update (after an email change) if user.picture_source not in (ProfilePictureSource.gravatar, ProfilePictureSource.identicon): return users = [user] else: users = User.query.filter(~User.is_deleted, User.picture_source == ProfilePictureSource.gravatar).all() for user in committing_iterator(users): gravatar, lastmod = get_gravatar_for_user(user, identicon=False, lastmod=user.picture_metadata['lastmod']) if gravatar is None: logger.debug('Gravatar for %r did not change (not modified)', user) continue if crc32(gravatar) == user.picture_metadata['hash']: logger.debug('Gravatar for %r did not change (same hash)', user) user.picture_metadata['lastmod'] = lastmod flag_modified(user, 'picture_metadata') continue set_user_avatar(user, gravatar, 'gravatar', lastmod) logger.info('Gravatar of user %s updated', user)
def migrate_event_acls(self): self.print_step('migrating event ACLs') protection_mode_map = { -1: ProtectionMode.public, 0: ProtectionMode.inheriting, 1: ProtectionMode.protected } for legacy_event, event in committing_iterator(self._iter_events(), 5000): ac = legacy_event._Conference__ac self.print_success('', event_id=event.id) old_protection_mode = protection_mode_map[ac._accessProtection] if old_protection_mode == ProtectionMode.public and ac.requiredDomains: event.protection_mode = ProtectionMode.protected self._migrate_domains(event, ac.requiredDomains) else: event.protection_mode = old_protection_mode no_access_contact = convert_to_unicode( getattr(ac, 'contactInfo', '')) if no_access_contact != 'no contact info defined': event.own_no_access_contact = no_access_contact event.access_key = convert_to_unicode( getattr(legacy_event, '_accessKey', '')) if not self.quiet: self.print_success('Protection mode set to {}'.format( event.protection_mode.name, event_id=event.id)) for legacy_acl in ac.allowed: event_acl = self.convert_acl(legacy_acl) if event_acl is None: self.print_warning(cformat( '%{red}ACL%{reset}%{yellow} does not exist:%{reset} {}' ).format(legacy_acl), event_id=event.id) continue event.update_principal(event_acl, read_access=True, quiet=True) if not self.quiet: self.print_msg( cformat('%{green}[{}]%{reset} {}').format( 'Event ACL', event_acl))
def migrate_event_settings(self): self.messages.append(cformat("%{magenta!} - Event Payment Settings:")) print cformat("%{white!}migrating event settings") count = 0 EventSetting.delete_all(payment_event_settings.module) for event in committing_iterator(self._iter_events()): old_payment = event._modPay default_conditions = payment_settings.get('conditions') default_register_email = payment_settings.get('register_email') default_success_email = payment_settings.get('success_email') register_email = getattr(old_payment, 'receiptMsg', default_register_email) success_email = getattr(old_payment, 'successMsg', default_success_email) conditions = (getattr(old_payment, 'paymentConditions', default_conditions) if (getattr(old_payment, 'paymentConditionsEnabled', False) and convert_to_unicode(getattr(old_payment, 'specificPaymentConditions', '')).strip() == '') else getattr(old_payment, 'specificPaymentConditions', '')) # The new messages are shown in an "additional info" section, so the old defaults can always go away if convert_to_unicode(register_email) == 'Please, see the summary of your order:': register_email = '' if convert_to_unicode(success_email) == 'Congratulations, your payment was successful.': success_email = '' # Get rid of the most terrible part of the old default conditions conditions = convert_to_unicode(conditions).replace('CANCELLATION :', 'CANCELLATION:') settings = { 'enabled': getattr(old_payment, 'activated', False), 'currency': event._registrationForm._currency, 'conditions': conditions, 'register_email': register_email, 'success_email': success_email, } payment_event_settings.set_multi(event, settings) count += 1 print cformat("%{cyan}<EventSettings(id={id:>6}, enabled={enabled}, " "currency={currency})>").format(id=event.id, **settings) msg = cformat("%{green!}migration of {0} event payment settings successful\n").format(count) self.messages.append(' ' + msg) print msg
def migrate_event_abstracts(self): self.print_step("Migrating event abstracts") for conf, event in committing_iterator(self._iter_events()): amgr = conf.abstractMgr duration = amgr._submissionEndDate - amgr._submissionStartDate if (not amgr._activated and not amgr._abstracts and not amgr._notifTpls and duration < timedelta(minutes=1) and not conf.program): continue mig = AbstractMigration(self, conf, event) try: with db.session.begin_nested(): with db.session.no_autoflush: mig.run() db.session.flush() except Exception: self.print_error(cformat('%{red!}MIGRATION FAILED!'), event_id=event.id) traceback.print_exc() raw_input('Press ENTER to continue') db.session.flush()
def migrate_event_categories(self): self.print_step("Migrating event categories") delete_events = set() for conf in committing_iterator(self._iter_events()): try: category_chain = self.category_mapping[int(conf._Conference__owners[0].id)] except (IndexError, KeyError): self.print_error(cformat('%{red!}Event has no category!'), event_id=conf.id) delete_events.add(int(conf.id)) continue Event.query.filter_by(id=int(conf.id)).update({Event.category_id: category_chain[-1]}, synchronize_session=False) if not self.quiet: self.print_success(repr(category_chain), event_id=conf.id) for event_id in delete_events: self.print_warning(cformat('%{yellow!}Deleting broken event {}').format(event_id)) Event.query.filter_by(id=event_id).update({Event.is_deleted: True}, synchronize_session=False) if self.zodb_root['conferences'].has_key(str(event_id)): del self.zodb_root['conferences'][str(event_id)] db.session.commit() transaction.commit()
def migrate_event_images(self): self.print_step('migrating event images') for event, picture in committing_iterator(self._iter_pictures()): local_file = picture._localFile content_type = mimetypes.guess_type( local_file.fileName)[0] or 'application/octet-stream' storage_backend, storage_path, size = self._get_local_file_info( local_file) if storage_path is None: self.print_warning(cformat( '%{yellow}[{}]%{reset} -> %{red!}Not found in filesystem'). format(local_file.id), event_id=event.id) continue filename = secure_filename(convert_to_unicode(local_file.fileName), 'image') image = ImageFile(event_id=event.id, filename=filename, content_type=content_type, created_dt=now_utc(), size=size, storage_backend=storage_backend, storage_file_id=storage_path) db.session.add(image) db.session.flush() map_entry = LegacyImageMapping(event_id=event.id, legacy_image_id=local_file.id, image_id=image.id) db.session.add(map_entry) if not self.quiet: self.print_success( cformat('%{cyan}[{}]%{reset} -> %{blue!}{}').format( local_file.id, image), event_id=event.id)
def migrate_links(self): print cformat('%{white!}migrating links') for avatars in grouper(self._iter_avatars(), 2500, skip_missing=True): avatars = {int(a.id): a for a in avatars} users = ((u, avatars[u.id]) for u in User.find(User.id.in_(avatars))) for user, avatar in committing_iterator(self.flushing_iterator(users, 250), 2500): registrants = set() user_shown = False for type_, entries in avatar.linkedTo.iteritems(): # store registrant roles, in order to avoid duplication below for role, objects in entries.iteritems(): if (type_ == 'category' and role == 'favorite') or type_ == 'group': continue if not objects: continue if type_ == 'registration' and role == 'registrant': registrants |= set(objects) if not user_shown: print cformat('%{green}+++%{reset} ' '%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(user.id, user.full_name) user_shown = True print cformat('%{blue!}<->%{reset} ' '%{yellow!}{:4d}%{reset}x %{green!}{:12} %{cyan!}{}%{reset}').format( len(objects), type_, role) for obj in objects: try: UserLink.create_link(user, obj, role, type_) except Exception as e: print cformat('%{red!}!!!%{reset} ' '%{red!}linking failed%{reset} (%{yellow!}{}%{reset}): ' '{}').format(unicode(e), obj) # add old "registrant" entries to registration/registrant for reg in getattr(avatar, 'registrants', {}).itervalues(): if reg.getConference().getOwner() and reg not in registrants: UserLink.create_link(user, reg, 'registrant', 'registration') print cformat('%{cyan!}<->%{reset} ' '%{yellow!} 1%{reset}x %{green!}{:12} %{cyan!}{}%{reset}').format( 'registration', 'registrant')
def migrate_event_locations(self): self.print_step("Migrating event locations") for old_event in committing_iterator(self._iter_events()): custom_location = old_event.places[0] if getattr( old_event, 'places', None) else None custom_room = old_event.rooms[0] if getattr( old_event, 'rooms', None) else None location_name = None room_name = None has_room = False updates = {} if custom_location: location_name = convert_to_unicode( fix_broken_string(custom_location.name, True)) if custom_location.address: updates[Event.own_address] = convert_to_unicode( fix_broken_string(custom_location.address, True)) if custom_room: room_name = convert_to_unicode( fix_broken_string(custom_room.name, True)) if location_name and room_name: mapping = self.room_mapping.get((location_name, room_name)) if mapping: has_room = True updates[Event.own_venue_id] = mapping[0] updates[Event.own_room_id] = mapping[1] # if we don't have a RB room set, use whatever location/room name we have if not has_room: venue_id = self.venue_mapping.get(location_name) if venue_id is not None: updates[Event.own_venue_id] = venue_id updates[Event.own_venue_name] = '' else: updates[Event.own_venue_name] = location_name or '' updates[Event.own_room_name] = room_name or '' if updates: Event.query.filter_by(id=int(old_event.id)).update( updates, synchronize_session=False) if not self.quiet: self.print_success(repr(updates), event_id=old_event.id)
def migrate_favorite_users(self): print cformat('%{white!}migrating favorite users') for avatars in grouper(self._iter_avatars_with_favorite_users(), 2500, skip_missing=True): avatars = list(avatars) users = {u.id: u for u in User.find(User.id.in_(int(a.id) for a, _ in avatars))} for avatar, user_ids in committing_iterator(avatars, 1000): user = users.get(int(avatar.id)) if user is None: print cformat('%{red!}!!!%{reset} ' '%{yellow!}User {} does not exist').format(avatar.id) continue print cformat('%{green}+++%{reset} ' '%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(user.id, user.full_name) valid_users = {u.id: u for u in User.find(User.id.in_(user_ids))} for user_id in user_ids: target = valid_users.get(user_id) if target is None: print cformat('%{yellow!}!!!%{reset} ' '%{yellow!}User {} does not exist').format(user_id) continue user.favorite_users.add(target) print cformat('%{blue!}<->%{reset} ' '%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(target.id, target.full_name)
def migrate_event_dates_titles(self): self.print_step("Migrating event dates and titles") for old_event in committing_iterator(self._iter_events()): if 'title' not in old_event.__dict__: self.print_error('Event has no title in ZODB', old_event.id) continue tz = old_event.__dict__.get('timezone', 'UTC') updates = { Event.title: convert_to_unicode(old_event.__dict__['title']) or '(no title)', Event.description: convert_to_unicode(old_event.__dict__['description']) or '', Event.timezone: tz, Event.start_dt: self._fix_naive(old_event, old_event.__dict__['startDate'], tz), Event.end_dt: self._fix_naive(old_event, old_event.__dict__['endDate'], tz) } Event.query.filter_by(id=int(old_event.id)).update( updates, synchronize_session=False) if not self.quiet: self.print_success('', event_id=old_event.id) # deleted events are not in zodb but also need data updates = { Event.title: '***deleted***', Event.description: '', Event.timezone: 'UTC', Event.start_dt: datetime(1970, 1, 1, tzinfo=pytz.utc), Event.end_dt: datetime(1970, 1, 1, tzinfo=pytz.utc) } Event.query.filter_by(is_deleted=True).update( updates, synchronize_session=False) db.session.commit()
def migrate_event_notes(self): self.print_step('migrating event notes') janitor_user = User.get_one(self.janitor_user_id) self.print_msg('Using janitor user {}'.format(janitor_user), always=True) for event, obj, minutes, special_prot in committing_iterator( self._iter_minutes()): if special_prot: self.print_warning(cformat( '%{yellow!}{} minutes have special permissions; skipping them' ).format(obj), event_id=event.id) continue path = get_archived_file(minutes, self.archive_dirs)[1] if path is None: self.print_error(cformat( '%{red!}{} minutes not found on disk; skipping them'). format(obj), event_id=event.id) continue with open(path, 'r') as f: data = convert_to_unicode(f.read()).strip() if not data: self.print_warning( cformat('%{yellow}{} minutes are empty; skipping them' ).format(obj), always=False, event_id=event.id) continue note = EventNote(linked_object=obj) note.create_revision(RenderMode.html, data, janitor_user) db.session.add(note) if not self.quiet: self.print_success(cformat('%{cyan}{}').format(obj), event_id=event.id)
def migrate_layout_settings(self): print cformat( '%{white!}migrating layout settings, event logos and custom stylesheets' ) default_styles = self.zodb_root['MaKaCInfo'][ 'main']._styleMgr._defaultEventStylesheet for event, event_type, dmgr, logo, custom_css in committing_iterator( self._iter_event_layout_data()): if event_type != 'conference': theme = dmgr._defaultstyle if not theme or theme == default_styles[event_type]: continue layout_settings.set(event, 'timetable_theme', theme) if not self.quiet: self.print_success(cformat( '- %{cyan}Default timetable theme: {}').format(theme), event_id=event.id) continue settings = self._get_event_settings(event, dmgr) layout_settings.set_multi(event, settings) if not self.quiet: self.print_success(cformat('- %{cyan}Layout settings'), event_id=event.id) if logo or custom_css: sa_event = Event.get(event.id) if not sa_event: self.print_warning( 'Event does not exist (anymore)! Logo and/or CSS file not saved!', event_id=event.id) continue if logo: self._process_logo(logo, sa_event) if custom_css: self._process_css(custom_css, sa_event)
def migrate_event_stubs(self): self.print_step('migrating event stubs') for event_id in committing_iterator(self._iter_event_ids(), 5000): db.session.add(Event(id=int(event_id)))
def migrate_keys(self): print cformat('%{white!}migrating api keys') for idx_key, ak in committing_iterator( self.zodb_root['apikeys'].iteritems()): if idx_key != ak._key: print cformat( '%{red!}!!!%{reset} ' '%{yellow!}Skipping {} - index key {} does not match' ).format(ak._key, idx_key) continue elif str(ak._user.id) not in self.zodb_root['avatars']: print cformat( '%{red!}!!!%{reset} ' '%{yellow!}Skipping {} - user {} does not exist').format( ak._key, ak._user.id) continue elif ak._user.apiKey != ak: print cformat( '%{red!}!!!%{reset} ' '%{yellow!}Skipping {} - user {} has a different api key set' ).format(ak._key, ak._user.id) continue last_used_uri = None if ak._lastPath and ak._lastQuery: last_used_uri = '{}?{}'.format( convert_to_unicode(ak._lastPath), convert_to_unicode(ak._lastQuery)) elif ak._lastPath: last_used_uri = convert_to_unicode(ak._lastPath) api_key = APIKey(token=ak._key, secret=ak._signKey, user_id=ak._user.id, is_blocked=ak._isBlocked, is_persistent_allowed=getattr( ak, '_persistentAllowed', False), created_dt=self._to_utc(ak._createdDT), last_used_dt=self._to_utc(ak._lastUsedDT), last_used_ip=ak._lastUsedIP, last_used_uri=last_used_uri, last_used_auth=ak._lastUseAuthenticated, use_count=ak._useCount) db.session.add(api_key) print cformat( '%{green}+++%{reset} %{cyan}{}%{reset} [%{blue!}{}%{reset}]' ).format(ak._key, ak._user.email) for old_key in ak._oldKeys: # We have no creation time so we use *something* older.. fake_created_dt = self._to_utc( ak._createdDT) - timedelta(hours=1) # We don't have anything besides the api key for old keys, so we use a random secret old_api_key = APIKey(token=old_key, secret=unicode(uuid4()), user_id=ak._user.id, created_dt=fake_created_dt, is_active=False) db.session.add(old_api_key) print cformat( '%{blue!}***%{reset} %{cyan}{}%{reset} [%{yellow}old%{reset}]' ).format(old_key) db.session.flush()
def migrate_users(self): seen_identities = set() for avatar in committing_iterator(self._iter_avatars(), 5000): if getattr(avatar, '_mergeTo', None): self.print_warning('Skipping {} - merged into {}'.format(avatar.id, avatar._mergeTo.id)) merged_user = self.global_ns.avatar_merged_user.get(avatar._mergeTo.id) if merged_user: self.global_ns.avatar_merged_user[avatar.id] = merged_user else: # if the merge target hasn't yet been migrated, keep track of it self.unresolved_merge_targets[avatar._mergeTo.id].add(avatar.id) continue elif avatar.status == 'Not confirmed': self.print_warning('Skipping {} - not activated'.format(avatar.id)) continue elif not avatar.name.strip() and not avatar.surName.strip(): links = {(obj, role): list(objs) for obj, x in avatar.linkedTo.iteritems() for role, objs in x.iteritems() if objs} if not avatar.identities and not links: self.print_warning('Skipping {} - no names and no identities/links'.format(avatar.id)) continue user = self._user_from_avatar(avatar) self._fix_collisions(user, avatar) db.session.add(user) settings = self._settings_from_avatar(avatar) user_settings.set_multi(user, settings) # favorite users cannot be migrated here since the target user might not have been migrated yet for old_categ in avatar.linkedTo['category']['favorite']: if old_categ: self.global_ns.user_favorite_categories[old_categ.id].add(user) db.session.flush() self.print_success('%[white!]{:6d}%[reset] %[cyan]{}%[reset] [%[blue!]{}%[reset]] ' '{{%[cyan!]{}%[reset]}}'.format(user.id, user.full_name, user.email, ', '.join(user.secondary_emails))) # migrate API keys self._migrate_api_keys(avatar, user) # migrate identities of avatars for old_identity in avatar.identities: identity = None username = convert_to_unicode(old_identity.login).strip().lower() if not username: self.print_warning("Empty username: {}. Skipping identity.".format(old_identity)) continue provider = { 'LocalIdentity': 'indico', 'LDAPIdentity': self.ldap_provider_name }.get(old_identity.__class__.__name__) if provider is None: self.print_error("Unsupported provider: {}. Skipping identity.".format( old_identity.__class__.__name__)) continue if (provider, username) in seen_identities: self.print_error("Duplicate identity: {}, {}. Skipping.".format(provider, username)) continue if provider == 'indico' and not self.ignore_local_accounts: identity = Identity(provider=provider, identifier=username) if not hasattr(old_identity, 'algorithm'): # plaintext password if not old_identity.password: # password is empty, skip identity self.print_error("Identity '{}' has empty password. Skipping identity.".format( old_identity.login)) continue identity.password = old_identity.password else: assert old_identity.algorithm == 'bcrypt' identity.password_hash = old_identity.password elif provider == self.ldap_provider_name: identity = Identity(provider=provider, identifier=username) if identity: self.print_info('%[blue!]<->%[reset] %[yellow]{}%[reset]'.format(identity)) user.identities.add(identity) seen_identities.add((provider, username)) if hasattr(avatar, 'personalInfo') and avatar.personalInfo._basket._users: self.favorite_avatars[user.id] = avatar.personalInfo._basket._users # Map old merged identities (no longer in AvatarHolder) # to newly created user for merged_avatar in getattr(avatar, '_mergeFrom', ()): if merged_avatar.id != avatar.id: self.global_ns.avatar_merged_user[merged_avatar.id] = user self.global_ns.avatar_merged_user[avatar.id] = user if avatar.id in self.unresolved_merge_targets: del self.unresolved_merge_targets[avatar.id] self._resolve_merge_targets(avatar.id, user) db.session.flush()
def migrate_transactions(self): self.messages.append(cformat("%{magenta!} - Payment Transactions:")) print cformat("%{white!}migrating payment transactions") count = 0 errors = 0 warnings = 0 for event, registrant, transaction in committing_iterator( self._iter_transactions()): try: data = self._get_transaction_data(transaction, event) except ValueError as e: print cformat("%{red!}{0} (evt: {1}, reg: {2})").format( e, event.id, registrant._id) errors += 1 continue if data['provider'] == '_manual' and data['amount'] == 0.0: print cformat( "%{yellow!}Skipping {0[provider]} transaction (evt: {1}, reg: {2}) " "with zero amount: {0[amount]} {0[currency]}").format( data, event.id, registrant._id) warnings += 1 continue elif data['amount'] < 0.0: print cformat( "%{yellow!}Skipping {0[provider]} transaction (evt: {1}, reg: {2}) " "with negative amount: {0[amount]} {0[currency]}").format( data, event.id, registrant._id) warnings += 1 continue pt = PaymentTransaction(event_id=int(event.id), registrant_id=int(registrant._id), status=TransactionStatus.successful, **data) count += 1 print cformat("%{cyan}{0}").format(pt) db.session.add(pt) if warnings: warning_msg = cformat( "%{yellow!}There were {0} warnings during the migration " "of the payment transactions.").format(warnings) self.messages.append(' ' + warning_msg) print warning_msg if errors: msg = cformat( "%{red!}There were some errors during the migration of the payment transactions.\n" "{0} transactions were not migrated and will be lost.\n" ).format(errors) else: msg = cformat( "%{green!}migration of {0} payment transactions successful\n" ).format(count) self.messages.append(' ' + '\n '.join(msg.split('\n'))) print msg
def migrate_users(self): print cformat('%{white!}migrating users') seen_identities = set() for avatar in committing_iterator(self._iter_avatars(), 5000): if getattr(avatar, '_mergeTo', None): print cformat('%{red!}!!!%{reset} ' '%{yellow!}Skipping {} - merged into {}').format( avatar.id, avatar._mergeTo.id) continue elif avatar.status == 'Not confirmed': print cformat('%{yellow!}!!!%{reset} ' '%{yellow!}Skipping {} - not activated').format( avatar.id) continue elif not avatar.name.strip() and not avatar.surName.strip(): links = {(obj, role): list(objs) for obj, x in avatar.linkedTo.iteritems() for role, objs in x.iteritems() if objs} if not avatar.identities and not links: print cformat( '%{yellow!}!!!%{reset} ' '%{yellow!}Skipping {} - no names and no identities/links' ).format(avatar.id) continue user = self._user_from_avatar(avatar) self._fix_collisions(user, avatar) db.session.add(user) settings = self._settings_from_avatar(avatar) user_settings.set_multi(user, settings) # favorite users cannot be migrated here since the target user might not have been migrated yet # XXX: adapt to new categories for 2.0 user.favorite_categories = set( filter(None, avatar.linkedTo['category']['favorite'])) db.session.flush() print cformat( '%{green}+++%{reset} ' '%{white!}{:6d}%{reset} %{cyan}{}%{reset} [%{blue!}{}%{reset}] ' '{{%{cyan!}{}%{reset}}}').format( user.id, user.full_name, user.email, ', '.join(user.secondary_emails)) # migrate API keys self._migrate_api_keys(avatar, user) # migrate identities of non-deleted avatars if not user.is_deleted: for old_identity in avatar.identities: identity = None username = convert_to_unicode( old_identity.login).strip().lower() if not username: print cformat( "%{red!}!!!%{reset} " "%{yellow!}Empty username: {}. Skipping identity." ).format(old_identity) continue provider = { 'LocalIdentity': 'indico', 'LDAPIdentity': self.ldap_provider_name }.get(old_identity.__class__.__name__) if provider is None: print cformat( "%{red!}!!!%{reset} " "%{yellow!}Unsupported provider: {}. Skipping identity." ).format(old_identity.__class__.__name__) continue if (provider, username) in seen_identities: print cformat( "%{red!}!!!%{reset} " "%{yellow!}Duplicate identity: {}, {}. Skipping." ).format(provider, username) continue if provider == 'indico' and not self.ignore_local_accounts: identity = Identity(provider=provider, identifier=username) if not hasattr(old_identity, 'algorithm'): # plaintext password if not old_identity.password: # password is empty, skip identity print cformat( "%{red!}!!!%{reset} " "%{yellow!}Identity '{}' has empty password. Skipping identity." ).format(old_identity.login) continue identity.password = old_identity.password else: assert old_identity.algorithm == 'bcrypt' identity.password_hash = old_identity.password elif provider == self.ldap_provider_name: identity = Identity(provider=provider, identifier=username) if identity: print cformat( '%{blue!}<->%{reset} %{yellow}{}%{reset}').format( identity) user.identities.add(identity) seen_identities.add((provider, username)) for merged_avatar in getattr(avatar, '_mergeFrom', ()): if merged_avatar.id == avatar.id: continue merged = self._user_from_avatar(merged_avatar, is_deleted=True, merged_into_id=user.id) print cformat( '%{blue!}***%{reset} ' '%{white!}{:6d}%{reset} %{cyan}{}%{reset} [%{blue!}{}%{reset}] ' '{{%{cyan!}{}%{reset}}}').format( merged.id, merged.full_name, merged.email, ', '.join(merged.secondary_emails)) self._fix_collisions(merged, merged_avatar) db.session.add(merged) db.session.flush()