def migrate_event_creation_times(self):
self.print_step("Migrating event creation times")
for old_event in committing_iterator(self._iter_events()):
Event.query.filter_by(id=int(old_event.id)).update({Event.created_dt: old_event._creationDS},
synchronize_session=False)
if not self.quiet:
self.print_success('', event_id=old_event.id)
def migrate_event_images(self):
self.print_step('migrating event images')
for event, picture in committing_iterator(self._iter_pictures()):
local_file = picture._localFile
content_type = mimetypes.guess_type(local_file.fileName)[0] or 'application/octet-stream'
storage_backend, storage_path, size = self._get_local_file_info(local_file)
if storage_path is None:
self.print_warning(cformat('%{yellow}[{}]%{reset} -> %{red!}Not found in filesystem').format(
local_file.id), event_id=event.id)
continue
filename = secure_filename(convert_to_unicode(local_file.fileName), 'image')
image = ImageFile(event_id=event.id,
filename=filename,
content_type=content_type,
created_dt=now_utc(),
size=size,
storage_backend=storage_backend,
storage_file_id=storage_path)
db.session.add(image)
db.session.flush()
map_entry = LegacyImageMapping(event_id=event.id, legacy_image_id=local_file.id, image_id=image.id)
db.session.add(map_entry)
if not self.quiet:
self.print_success(cformat('%{cyan}[{}]%{reset} -> %{blue!}{}').format(local_file.id, image),
event_id=event.id)
def migrate_layout_settings(self):
print cformat('%{white!}migrating layout settings, event logos and custom stylesheets')
default_styles = self.zodb_root['MaKaCInfo']['main']._styleMgr._defaultEventStylesheet
for event, event_type, dmgr, logo, custom_css in committing_iterator(self._iter_event_layout_data()):
if event_type != 'conference':
theme = dmgr._defaultstyle
if not theme or theme == default_styles[event_type]:
continue
layout_settings.set(event, 'timetable_theme', theme)
if not self.quiet:
self.print_success(cformat('- %{cyan}Default timetable theme: {}').format(theme), event_id=event.id)
continue
settings = self._get_event_settings(event, dmgr)
layout_settings.set_multi(event, settings)
if not self.quiet:
self.print_success(cformat('- %{cyan}Layout settings'), event_id=event.id)
if logo or custom_css:
sa_event = Event.get(event.id)
if not sa_event:
self.print_warning('Event does not exist (anymore)! Logo and/or CSS file not saved!',
event_id=event.id)
continue
if logo:
self._process_logo(logo, sa_event)
if custom_css:
self._process_css(custom_css, sa_event)
def migrate_event_locations(self):
self.print_step("Migrating event locations")
for old_event in committing_iterator(self._iter_events()):
custom_location = old_event.places[0] if getattr(old_event, 'places', None) else None
custom_room = old_event.rooms[0] if getattr(old_event, 'rooms', None) else None
location_name = None
room_name = None
has_room = False
updates = {}
if custom_location:
location_name = convert_to_unicode(fix_broken_string(custom_location.name, True))
if custom_location.address:
updates[Event.own_address] = convert_to_unicode(fix_broken_string(custom_location.address, True))
if custom_room:
room_name = convert_to_unicode(fix_broken_string(custom_room.name, True))
if location_name and room_name:
mapping = self.room_mapping.get((location_name, room_name))
if mapping:
has_room = True
updates[Event.own_venue_id] = mapping[0]
updates[Event.own_room_id] = mapping[1]
# if we don't have a RB room set, use whatever location/room name we have
if not has_room:
venue_id = self.venue_mapping.get(location_name)
if venue_id is not None:
updates[Event.own_venue_id] = venue_id
updates[Event.own_venue_name] = ''
else:
updates[Event.own_venue_name] = location_name or ''
updates[Event.own_room_name] = room_name or ''
if updates:
Event.query.filter_by(id=int(old_event.id)).update(updates, synchronize_session=False)
if not self.quiet:
self.print_success(repr(updates), event_id=old_event.id)
def migrate_event_dates_titles(self):
self.print_step("Migrating event dates and titles")
for old_event in committing_iterator(self._iter_events()):
if 'title' not in old_event.__dict__:
self.print_error('Event has no title in ZODB', old_event.id)
continue
tz = old_event.__dict__.get('timezone', 'UTC')
updates = {
Event.title: convert_to_unicode(old_event.__dict__['title']) or '(no title)',
Event.description: convert_to_unicode(old_event.__dict__['description']) or '',
Event.timezone: tz,
Event.start_dt: self._fix_naive(old_event, old_event.__dict__['startDate'], tz),
Event.end_dt: self._fix_naive(old_event, old_event.__dict__['endDate'], tz)
}
Event.query.filter_by(id=int(old_event.id)).update(updates, synchronize_session=False)
if not self.quiet:
self.print_success('', event_id=old_event.id)
# deleted events are not in zodb but also need data
updates = {Event.title: '***deleted***',
Event.description: '',
Event.timezone: 'UTC',
Event.start_dt: datetime(1970, 1, 1, tzinfo=pytz.utc),
Event.end_dt: datetime(1970, 1, 1, tzinfo=pytz.utc)}
Event.query.filter_by(is_deleted=True).update(updates, synchronize_session=False)
db.session.commit()
def migrate(self):
print cformat('%{white!}migrating static sites')
for item in committing_iterator(chain.from_iterable(
self.zodb_root['modules']['offlineEvents']._idxConf.itervalues())):
event_id = item.conference.id
if is_legacy_id(event_id):
print cformat('%{red!}!!!%{reset} '
'%{white!}{0:6s}%{reset} %{yellow!}Event has non-numeric/broken ID').format(event_id)
continue
if event_id not in self.zodb_root['conferences']:
print cformat('%{red!}!!!%{reset} '
'%{white!}{0:6s}%{reset} %{yellow!}Event deleted, skipping static site').format(event_id)
continue
event_id = int(event_id)
user = self._get_user(item.avatar.id)
state = STATE_MAPPING[item.status]
requested_dt = item.requestTime
file_name, file_path = self._get_file_data(item.file)
if file_path is None and state == StaticSiteState.success:
print cformat('%{yellow!}!!!%{reset} %{white!}{0:6d}%{reset} '
'%{yellow!}file missing, marking static site as expired.').format(event_id)
state = StaticSite.expired
static_site = StaticSite(creator=user, event_id=event_id, state=state, requested_dt=requested_dt)
if static_site.state == StaticSiteState.success:
static_site.path = file_path
db.session.add(static_site)
print cformat('%{green}+++%{reset} %{white!}{0.event_id:6d}%{reset} '
'%{cyan}{0}%{reset}').format(static_site)
def migrate_event_acls(self):
self.print_step('migrating event ACLs')
protection_mode_map = {-1: ProtectionMode.public, 0: ProtectionMode.inheriting, 1: ProtectionMode.protected}
for legacy_event, event in committing_iterator(self._iter_events(), 5000):
ac = legacy_event._Conference__ac
self.print_success('', event_id=event.id)
old_protection_mode = protection_mode_map[ac._accessProtection]
if old_protection_mode == ProtectionMode.public and ac.requiredDomains:
event.protection_mode = ProtectionMode.protected
self._migrate_domains(event, ac.requiredDomains)
else:
event.protection_mode = old_protection_mode
no_access_contact = convert_to_unicode(getattr(ac, 'contactInfo', ''))
if no_access_contact != 'no contact info defined':
event.own_no_access_contact = no_access_contact
event.access_key = convert_to_unicode(getattr(legacy_event, '_accessKey', ''))
if not self.quiet:
self.print_success('Protection mode set to {}'.format(event.protection_mode.name, event_id=event.id))
for legacy_acl in ac.allowed:
event_acl = self.convert_acl(legacy_acl)
if event_acl is None:
self.print_warning(cformat('%{red}ACL%{reset}%{yellow} does not exist:%{reset} {}')
.format(legacy_acl), event_id=event.id)
continue
event.update_principal(event_acl, read_access=True, quiet=True)
if not self.quiet:
self.print_msg(cformat('%{green}[{}]%{reset} {}').format('Event ACL', event_acl))
def migrate_agents(self):
print cformat('%{white!}migrating agents')
for old_agent in committing_iterator(self.livesync_root['agent_manager']._agents.itervalues()):
if not old_agent._active:
print cformat('%{yellow}skipping inactive agent {} ({})%{reset}').format(old_agent._id, old_agent._name)
continue
agent = LiveSyncAgent(name=convert_to_unicode(old_agent._name), initial_data_exported=True)
old_agent_class = old_agent.__class__.__name__
if old_agent_class == 'InvenioBatchUploaderAgent':
agent.backend_name = 'invenio'
agent.settings = {
'server_url': old_agent._url
}
elif old_agent_class == 'CERNSearchUploadAgent':
agent.backend_name = 'cernsearch'
agent.settings = {
'server_url': old_agent._url,
'username': old_agent._username,
'password': old_agent._password,
}
else:
print cformat('%{red!}skipping unknown agent type: {}%{reset}').format(old_agent_class)
continue
print cformat('- %{cyan}{} ({})').format(agent.name, agent.backend_name)
db.session.add(agent)
def migrate_event_notes(self):
self.print_step('migrating event notes')
janitor_user = User.get_one(self.janitor_user_id)
self.print_msg('Using janitor user {}'.format(janitor_user), always=True)
for event, obj, minutes, special_prot in committing_iterator(self._iter_minutes()):
if special_prot:
self.print_warning(
cformat('%{yellow!}{} minutes have special permissions; skipping them').format(obj),
event_id=event.id
)
continue
path = get_archived_file(minutes, self.archive_dirs)[1]
if path is None:
self.print_error(cformat('%{red!}{} minutes not found on disk; skipping them').format(obj),
event_id=event.id)
continue
with open(path, 'r') as f:
data = convert_to_unicode(f.read()).strip()
if not data:
self.print_warning(cformat('%{yellow}{} minutes are empty; skipping them').format(obj),
always=False, event_id=event.id)
continue
note = EventNote(linked_object=obj)
note.create_revision(RenderMode.html, data, janitor_user)
db.session.add(note)
if not self.quiet:
self.print_success(cformat('%{cyan}{}').format(obj), event_id=event.id)
def vidyo_cleanup(dry_run=False):
from indico_vc_vidyo.plugin import VidyoPlugin
max_room_event_age = VidyoPlugin.settings.get('num_days_old')
VidyoPlugin.logger.info('Deleting Vidyo rooms that are not used or linked to events all older than %d days',
max_room_event_age)
candidate_rooms = find_old_vidyo_rooms(max_room_event_age)
VidyoPlugin.logger.info('%d rooms found', len(candidate_rooms))
if dry_run:
for vc_room in candidate_rooms:
VidyoPlugin.logger.info('Would delete Vidyo room %s from server', vc_room)
return
for vc_room in committing_iterator(candidate_rooms, n=20):
try:
VidyoPlugin.instance.delete_room(vc_room, None)
VidyoPlugin.logger.info('Room %s deleted from Vidyo server', vc_room)
notify_owner(VidyoPlugin.instance, vc_room)
vc_room.status = VCRoomStatus.deleted
except RoomNotFoundAPIException:
VidyoPlugin.logger.warning('Room %s had been already deleted from the Vidyo server', vc_room)
vc_room.status = VCRoomStatus.deleted
except APIException:
VidyoPlugin.logger.exception('Impossible to delete Vidyo room %s', vc_room)
def migrate_groups(self):
print cformat('%{white!}migrating groups')
for old_group in committing_iterator(self.zodb_root['groups'].itervalues()):
if old_group.__class__.__name__ != 'Group':
continue
group = LocalGroup(id=int(old_group.id), name=convert_to_unicode(old_group.name).strip())
print cformat('%{green}+++%{reset} %{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(group.id, group.name)
members = set()
for old_member in old_group.members:
if old_member.__class__.__name__ != 'Avatar':
print cformat('%{yellow!}!!! Unsupported group member type: {}').format(
old_member.__class__.__name__)
continue
user = User.get(int(old_member.id))
if user is None:
print cformat('%{yellow!}!!! User not found: {}').format(old_member.id)
continue
while user.merged_into_user:
user = user.merged_into_user
if user.is_deleted:
print cformat('%{yellow!}!!! User deleted: {}').format(user.id)
continue
members.add(user)
for member in sorted(members, key=attrgetter('full_name')):
print cformat('%{blue!}<->%{reset} %{white!}{:6d} %{yellow}{} ({})').format(
member.id, member.full_name, member.email)
group.members = members
db.session.add(group)
def migrate_regforms(self):
self.print_step("Migrating participants")
for event, participation in committing_iterator(self._iter_participations(), 10):
mig = ParticipationMigration(self, event, participation)
with db.session.no_autoflush:
mig.run()
db.session.add(mig.regform)
db.session.flush()
def enable_features(self):
self.print_step("Enabling registration features")
event_ids = [x[0] for x in set(db.session.query(RegistrationForm.event_id)
.filter(RegistrationForm.title == PARTICIPATION_FORM_TITLE))]
it = verbose_iterator(event_ids, len(event_ids), lambda x: x,
lambda x: self.zodb_root['conferences'][str(x)].title)
for event_id in committing_iterator(it):
set_feature_enabled(self.zodb_root['conferences'][str(event_id)], 'registration', True)
def migrate_event_menus(self):
self.print_step("migrating conference menus")
for event, display_mgr in committing_iterator(self._iter_events()):
if _get_menu_structure(display_mgr) in DEFAULT_MENU_STRUCTURES:
continue
self.print_success("", event_id=event.id)
db.session.add_all(self._migrate_menu(event, display_mgr._menu))
layout_settings.set(event, "use_custom_menu", True)
def migrate_transactions(self):
self.messages.append(cformat("%{magenta!} - Payment Transactions:"))
print cformat("%{white!}migrating payment transactions")
count = 0
errors = 0
warnings = 0
for event, registrant, transaction in committing_iterator(self._iter_transactions()):
try:
data = self._get_transaction_data(transaction, event)
except ValueError as e:
print cformat("%{red!}{0} (evt: {1}, reg: {2})").format(e, event.id, registrant._id)
errors += 1
continue
if data["provider"] == "_manual" and data["amount"] == 0.0:
print cformat(
"%{yellow!}Skipping {0[provider]} transaction (evt: {1}, reg: {2}) "
"with zero amount: {0[amount]} {0[currency]}"
).format(data, event.id, registrant._id)
warnings += 1
continue
elif data["amount"] < 0.0:
print cformat(
"%{yellow!}Skipping {0[provider]} transaction (evt: {1}, reg: {2}) "
"with negative amount: {0[amount]} {0[currency]}"
).format(data, event.id, registrant._id)
warnings += 1
continue
pt = PaymentTransaction(
event_id=int(event.id), registrant_id=int(registrant._id), status=TransactionStatus.successful, **data
)
count += 1
print cformat("%{cyan}{0}").format(pt)
db.session.add(pt)
if warnings:
warning_msg = cformat(
"%{yellow!}There were {0} warnings during the migration " "of the payment transactions."
).format(warnings)
self.messages.append(" " + warning_msg)
print warning_msg
if errors:
msg = cformat(
"%{red!}There were some errors during the migration of the payment transactions.\n"
"{0} transactions were not migrated and will be lost.\n"
).format(errors)
else:
msg = cformat("%{green!}migration of {0} payment transactions successful\n").format(count)
self.messages.append(" " + "\n ".join(msg.split("\n")))
print msg
def migrate_event_keywords(self):
self.print_step("Migrating event keywords")
for old_event in committing_iterator(self._iter_events()):
old_keywords = old_event._keywords
keywords = filter(None, map(unicode.strip, map(convert_to_unicode, old_keywords.splitlines())))
if not keywords:
continue
Event.query.filter_by(id=int(old_event.id)).update({Event.keywords: keywords}, synchronize_session=False)
if not self.quiet:
self.print_success(repr(keywords), event_id=old_event.id)
def migrate_event_bookings(self):
self.vc_rooms_by_extension = {}
with VidyoPlugin.instance.plugin_context():
for event_id, csbm in committing_iterator(self.booking_root.iteritems(), n=1000):
for bid, booking in csbm._bookings.iteritems():
if booking._type == 'Vidyo':
vc_room = self.vc_rooms_by_extension.get(int(booking._extension))
if not vc_room:
vc_room = self.migrate_vidyo_room(booking)
self.migrate_event_booking(vc_room, booking)
def migrate_surveys(self):
self.print_step("Migrating old event evaluations")
notification_pending = {es.event_id: es.value for es in EventSetting.find_all(module='evaluation',
name='send_notification')}
for event, evaluation in committing_iterator(self._iter_evaluations(), 10):
survey = self.migrate_survey(evaluation, event)
if (survey.notifications_enabled and survey.start_notification_emails and
not notification_pending.get(survey.event_id)):
survey.start_notification_sent = True
db.session.add(survey)
def migrate_event_keywords(self):
self.print_step("Migrating event keywords, creation times and visibility")
for old_event in committing_iterator(self._iter_events()):
updates = {Event.created_dt: old_event._creationDS,
Event.visibility: self._convert_visibility(old_event._visibility)}
keywords = self._convert_keywords(old_event._keywords)
if keywords:
updates[Event.keywords] = keywords
Event.query.filter_by(id=int(old_event.id)).update(updates, synchronize_session=False)
if not self.quiet:
self.print_success(repr(keywords), event_id=old_event.id)
def migrate_admins(self):
print cformat('%{white!}migrating admins')
for avatar in committing_iterator(self.zodb_root['adminlist']._AdminList__list):
try:
user = User.get(int(avatar.id))
except ValueError:
continue
if user is None or user.is_deleted:
continue
user.is_admin = True
print cformat('%{green}+++%{reset} %{cyan}{}').format(user)
def migrate_networks(self):
self.print_step('migrating networks')
for domain in committing_iterator(self._iter_domains()):
ip_networks = filter(None, map(self._to_network, set(domain.filterList)))
if not ip_networks:
self.print_warning(cformat('%{yellow}Domain has no valid IPs: {}')
.format(convert_to_unicode(domain.name)))
network = IPNetworkGroup(name=convert_to_unicode(domain.name),
description=convert_to_unicode(domain.description), networks=ip_networks)
db.session.add(network)
self.print_success(repr(network))
db.session.flush()
def migrate_event_managers(self):
self.print_step("migrating event managers/creators")
creator_updates = []
for event in committing_iterator(self._iter_events(), 5000):
self.print_success("", event_id=event.id)
ac = event._Conference__ac
entries = {}
# add creator as a manager
try:
creator = event._Conference__creator
except AttributeError:
# events created after the removal of the `self.__creator` assignment
# should happen only on dev machines
self.print_error(cformat("%{red!}Event has no creator attribute"), event_id=event.id)
else:
user = self.process_principal(event, entries, creator, "Creator", "green!", full_access=True)
if user:
creator_updates.append({"event_id": int(event.id), "creator_id": user.id})
# add managers
for manager in ac.managers:
self.process_principal(event, entries, manager, "Manager", "blue!", full_access=True)
# add email-based managers
emails = getattr(ac, "managersEmail", [])
self.process_emails(event, entries, emails, "Manager", "green", full_access=True)
# add registrars
for registrar in getattr(event, "_Conference__registrars", []):
self.process_principal(event, entries, registrar, "Registrar", "cyan", roles={"registration"})
# add submitters
for submitter in getattr(ac, "submitters", []):
self.process_principal(event, entries, submitter, "Submitter", "magenta!", roles={"submit"})
# email-based (pending) submitters
pqm = getattr(event, "_pendingQueuesMgr", None)
if pqm is not None:
emails = set(getattr(pqm, "_pendingConfSubmitters", []))
self.process_emails(event, entries, emails, "Submitter", "magenta", roles={"submit"})
db.session.add_all(entries.itervalues())
# assign creators
if creator_updates:
self.print_step("saving event creators")
stmt = (
Event.__table__.update()
.where(Event.id == db.bindparam("event_id"))
.values(creator_id=db.bindparam("creator_id"))
)
db.session.execute(stmt, creator_updates)
updated = Event.find(Event.creator_id == None).update({Event.creator_id: self.janitor.id}) # noqa
db.session.commit()
self.print_success("Set the janitor user {} for {} events".format(self.janitor, updated), always=True)
def migrate_event_keywords(self):
self.print_step(
"Migrating event keywords, creation times and visibility")
for old_event in committing_iterator(self._iter_events()):
updates = {
Event.created_dt: old_event._creationDS,
Event.visibility:
self._convert_visibility(old_event._visibility)
}
keywords = self._convert_keywords(old_event._keywords)
if keywords:
updates[Event.keywords] = keywords
Event.query.filter_by(id=int(old_event.id)).update(
updates, synchronize_session=False)
if not self.quiet:
self.print_success(repr(keywords), event_id=old_event.id)
def migrate_networks(self):
self.print_step('migrating networks')
for domain in committing_iterator(self._iter_domains()):
ip_networks = filter(None,
map(self._to_network, set(domain.filterList)))
if not ip_networks:
self.print_warning(
cformat('%{yellow}Domain has no valid IPs: {}').format(
convert_to_unicode(domain.name)))
network = IPNetworkGroup(name=convert_to_unicode(domain.name),
description=convert_to_unicode(
domain.description),
networks=ip_networks)
db.session.add(network)
self.print_success(repr(network))
db.session.flush()
def migrate_event_logs(self):
self.print_step("migrating event logs")
msg_email = cformat("%{white!}{:6d}%{reset} %{cyan}{}")
msg_action = cformat("%{white!}{:6d}%{reset} %{cyan!}{}")
for event in committing_iterator(self._iter_events()):
for item in event._logHandler._logLists["emailLog"]:
entry = self._migrate_email_log(event, item)
db.session.add(entry)
if not self.quiet:
self.print_success(msg_email.format(entry.event_id, entry))
for item in event._logHandler._logLists["actionLog"]:
entry = self._migrate_action_log(event, item)
db.session.add(entry)
if not self.quiet:
self.print_success(msg_action.format(entry.event_id, entry))
def migrate_event_managers(self):
self.print_step('migrating event managers/creators')
creator_updates = []
for event in committing_iterator(self._iter_events(), 5000):
self.print_success('', event_id=event.id)
ac = event._Conference__ac
entries = {}
# add creator as a manager
try:
creator = event._Conference__creator
except AttributeError:
# events created after the removal of the `self.__creator` assignment
# should happen only on dev machines
self.print_error(cformat('%{red!}Event has no creator attribute'), event_id=event.id)
else:
user = self.process_principal(event, entries, creator, 'Creator', 'green!', full_access=True)
if user:
creator_updates.append({'event_id': int(event.id), 'creator_id': user.id})
# add managers
for manager in ac.managers:
self.process_principal(event, entries, manager, 'Manager', 'blue!', full_access=True)
# add email-based managers
emails = getattr(ac, 'managersEmail', [])
self.process_emails(event, entries, emails, 'Manager', 'green', full_access=True)
# add registrars
for registrar in getattr(event, '_Conference__registrars', []):
self.process_principal(event, entries, registrar, 'Registrar', 'cyan', roles={'registration'})
# add submitters
for submitter in getattr(ac, 'submitters', []):
self.process_principal(event, entries, submitter, 'Submitter', 'magenta!', roles={'submit'})
# email-based (pending) submitters
pqm = getattr(event, '_pendingQueuesMgr', None)
if pqm is not None:
emails = set(getattr(pqm, '_pendingConfSubmitters', []))
self.process_emails(event, entries, emails, 'Submitter', 'magenta', roles={'submit'})
db.session.add_all(entries.itervalues())
# assign creators
if creator_updates:
self.print_step('saving event creators')
stmt = (Event.__table__.update()
.where(Event.id == db.bindparam('event_id'))
.values(creator_id=db.bindparam('creator_id')))
db.session.execute(stmt, creator_updates)
updated = Event.find(Event.creator_id == None).update({Event.creator_id: self.janitor.id}) # noqa
db.session.commit()
self.print_success('Set the janitor user {} for {} events'.format(self.janitor, updated), always=True)
def migrate_event_logs(self):
self.print_step('migrating event logs')
msg_email = cformat('%{white!}{:6d}%{reset} %{cyan}{}')
msg_action = cformat('%{white!}{:6d}%{reset} %{cyan!}{}')
for event in committing_iterator(self._iter_events()):
for item in event._logHandler._logLists['emailLog']:
entry = self._migrate_email_log(event, item)
db.session.add(entry)
if not self.quiet:
self.print_success(msg_email.format(entry.event_id, entry))
for item in event._logHandler._logLists['actionLog']:
entry = self._migrate_action_log(event, item)
db.session.add(entry)
if not self.quiet:
self.print_success(msg_action.format(
entry.event_id, entry))
def migrate_event_managers(self):
self.print_step('migrating event managers/creators')
creator_updates = []
for event in committing_iterator(self._iter_events(), 5000):
self.print_success('', event_id=event.id)
ac = event._Conference__ac
entries = {}
# add creator as a manager
try:
creator = event._Conference__creator
except AttributeError:
# events created after the removal of the `self.__creator` assignment
# should happen only on dev machines
self.print_error(cformat('%{red!}Event has no creator attribute'), event_id=event.id)
else:
user = self.process_principal(event, entries, creator, 'Creator', 'green!', full_access=True)
if user:
creator_updates.append({'event_id': int(event.id), 'creator_id': user.id})
# add managers
for manager in ac.managers:
self.process_principal(event, entries, manager, 'Manager', 'blue!', full_access=True)
# add email-based managers
emails = getattr(ac, 'managersEmail', [])
self.process_emails(event, entries, emails, 'Manager', 'green', full_access=True)
# add registrars
for registrar in getattr(event, '_Conference__registrars', []):
self.process_principal(event, entries, registrar, 'Registrar', 'cyan', roles={'registration'})
# add submitters
for submitter in getattr(ac, 'submitters', []):
self.process_principal(event, entries, submitter, 'Submitter', 'magenta!', roles={'submit'})
# email-based (pending) submitters
pqm = getattr(event, '_pendingQueuesMgr', None)
if pqm is not None:
emails = set(getattr(pqm, '_pendingConfSubmitters', []))
self.process_emails(event, entries, emails, 'Submitter', 'magenta', roles={'submit'})
db.session.add_all(entries.itervalues())
# assign creators
if creator_updates:
self.print_step('saving event creators')
stmt = (Event.__table__.update()
.where(Event.id == db.bindparam('event_id'))
.values(creator_id=db.bindparam('creator_id')))
db.session.execute(stmt, creator_updates)
updated = Event.find(Event.creator_id == None).update({Event.creator_id: self.janitor.id}) # noqa
db.session.commit()
self.print_success('Set the janitor user {} for {} events'.format(self.janitor, updated), always=True)
def migrate_event_settings(self):
self.messages.append(cformat("%{magenta!} - Event Payment Settings:"))
print cformat("%{white!}migrating event settings")
count = 0
EventSetting.delete_all(payment_event_settings.module)
for event in committing_iterator(self._iter_events()):
old_payment = event._modPay
default_conditions = global_settings.get("conditions")
default_register_email = global_settings.get("register_email")
default_success_email = global_settings.get("success_email")
register_email = getattr(old_payment, "receiptMsg", default_register_email)
success_email = getattr(old_payment, "successMsg", default_success_email)
conditions = (
getattr(old_payment, "paymentConditions", default_conditions)
if (
getattr(old_payment, "paymentConditionsEnabled", False)
and convert_to_unicode(getattr(old_payment, "specificPaymentConditions", "")).strip() == ""
)
else getattr(old_payment, "specificPaymentConditions", "")
)
# The new messages are shown in an "additional info" section, so the old defaults can always go away
if convert_to_unicode(register_email) == "Please, see the summary of your order:":
register_email = ""
if convert_to_unicode(success_email) == "Congratulations, your payment was successful.":
success_email = ""
# Get rid of the most terrible part of the old default conditions
conditions = convert_to_unicode(conditions).replace("CANCELLATION :", "CANCELLATION:")
settings = {
"enabled": getattr(old_payment, "activated", False),
"currency": event._registrationForm._currency,
"conditions": conditions,
"register_email": register_email,
"success_email": success_email,
}
payment_event_settings.set_multi(event, settings)
count += 1
print cformat("%{cyan}<EventSettings(id={id:>6}, enabled={enabled}, " "currency={currency})>").format(
id=event.id, **settings
)
msg = cformat("%{green!}migration of {0} event payment settings successful\n").format(count)
self.messages.append(" " + msg)
print msg
def migrate_evaluation_alarms(self):
print cformat('%{white!}migrating evaluation alarms')
today = date.today()
for task in committing_iterator(self._iter_tasks('EvaluationAlarm')):
start_date = task.conf._evaluations[0].startDate.date()
if start_date < today:
print cformat('%{yellow!}!!!%{reset} '
'%{white!}{:6d}%{reset} %{yellow}evaluation starts in the past ({})').format(
int(task.conf.id), start_date)
elif not task.conf._evaluations[0].visible:
print cformat('%{yellow!}!!!%{reset} '
'%{white!}{:6d}%{reset} %{yellow}evaluation is disabled').format(int(task.conf.id))
else:
print cformat('%{green}+++%{reset} '
'%{white!}{:6d}%{reset} %{cyan}{}').format(int(task.conf.id), start_date)
evaluation_settings.set(task.conf, 'send_notification', True)
def migrate(self):
print cformat('%{white!}migrating static sites')
for item in committing_iterator(
chain.from_iterable(self.zodb_root['modules']
['offlineEvents']._idxConf.itervalues())):
event_id = item.conference.id
if is_legacy_id(event_id):
print cformat(
'%{red!}!!!%{reset} '
'%{white!}{0:6s}%{reset} %{yellow!}Event has non-numeric/broken ID'
).format(event_id)
continue
if event_id not in self.zodb_root['conferences']:
print cformat(
'%{red!}!!!%{reset} '
'%{white!}{0:6s}%{reset} %{yellow!}Event deleted, skipping static site'
).format(event_id)
continue
event_id = int(event_id)
user = self._get_user(item.avatar.id)
state = STATE_MAPPING[item.status]
requested_dt = item.requestTime
file_name, file_path = self._get_file_data(item.file)
if file_path is None and state == StaticSiteState.success:
print cformat(
'%{yellow!}!!!%{reset} %{white!}{0:6d}%{reset} '
'%{yellow!}file missing, marking static site as expired.'
).format(event_id)
state = StaticSite.expired
static_site = StaticSite(creator=user,
event_id=event_id,
state=state,
requested_dt=requested_dt)
if static_site.state == StaticSiteState.success:
static_site.path = file_path
db.session.add(static_site)
print cformat(
'%{green}+++%{reset} %{white!}{0.event_id:6d}%{reset} '
'%{cyan}{0}%{reset}').format(static_site)
def migrate_event_settings(self):
print cformat('%{white!}migrating event settings')
default_method_name = PaypalPaymentPlugin.settings.get('method_name')
EventSetting.delete_all(PaypalPaymentPlugin.event_settings.module)
account_id_re = re.compile(r'^[a-zA-Z0-9]{13}$')
for event in committing_iterator(self._iter_events(), 25):
pp = event._modPay.payMods['PayPal']
business = pp._business.strip()
if not business or (not is_valid_mail(business, multi=False) and not account_id_re.match(business)):
print cformat(' - %{yellow!}event {} skipped (business: {})').format(event.id, business or '(none)')
continue
PaypalPaymentPlugin.event_settings.set(event, 'enabled', True)
method_name = convert_to_unicode(pp._title)
if method_name.lower() == 'paypal':
method_name = default_method_name
PaypalPaymentPlugin.event_settings.set(event, 'method_name', method_name)
PaypalPaymentPlugin.event_settings.set(event, 'business', pp._business)
print cformat(' - %{cyan}event {} (business: {})').format(event.id, pp._business)
def migrate_event_settings(self):
print cformat('%{white!}migrating event settings')
default_method_name = PaypalPaymentPlugin.settings.get('method_name')
EventSetting.delete_all(PaypalPaymentPlugin.event_settings.module)
account_id_re = re.compile(r'^[a-zA-Z0-9]{13}$')
for event in committing_iterator(self._iter_events(), 25):
pp = event._modPay.payMods['PayPal']
business = pp._business.strip()
if not business or (not is_valid_mail(business, multi=False) and not account_id_re.match(business)):
print cformat(' - %{yellow!}event {} skipped (business: {})').format(event.id, business or '(none)')
continue
PaypalPaymentPlugin.event_settings.set(event, 'enabled', True)
method_name = convert_to_unicode(pp._title)
if method_name.lower() == 'paypal':
method_name = default_method_name
PaypalPaymentPlugin.event_settings.set(event, 'method_name', method_name)
PaypalPaymentPlugin.event_settings.set(event, 'business', pp._business)
print cformat(' - %{cyan}event {} (business: {})').format(event.id, pp._business)
def migrate_event_series(self):
self.print_step("Migrating event series")
all_series = self.get_event_series()
all_series_ids = set(chain.from_iterable(all_series))
events = {e.id: e for e in Event.find(Event.id.in_(all_series_ids)).options(load_only('id', 'series_id'))}
for series in committing_iterator(verbose_iterator(all_series, len(all_series), lambda x: 0, lambda x: '')):
series &= events.viewkeys()
if len(series) < 2:
self.print_warning('Skipping single-event series: {}'.format(sorted(series)))
continue
es = EventSeries(show_sequence_in_title=False)
for id_ in series:
events[id_].series = es
if not self.quiet:
self.print_success(repr(series))
AttachmentFolder.find(AttachmentFolder.title.op('~')('^part\d+$')).update({AttachmentFolder.is_deleted: True},
synchronize_session=False)
db.session.commit()
def migrate_event_series(self):
self.print_step("Migrating event series")
all_series = self.get_event_series()
all_series_ids = set(chain.from_iterable(all_series))
events = {e.id: e for e in Event.find(Event.id.in_(all_series_ids)).options(load_only('id', 'series_id'))}
for series in committing_iterator(verbose_iterator(all_series, len(all_series), lambda x: 0, lambda x: '')):
series &= events.viewkeys()
if len(series) < 2:
self.print_warning('Skipping single-event series: {}'.format(sorted(series)))
continue
es = EventSeries(show_sequence_in_title=False)
for id_ in series:
events[id_].series = es
if not self.quiet:
self.print_success(repr(series))
AttachmentFolder.find(AttachmentFolder.title.op('~')('^part\d+$')).update({AttachmentFolder.is_deleted: True},
synchronize_session=False)
db.session.commit()
def migrate_event_logs(self):
self.print_step('migrating event logs')
msg_email = cformat('%{white!}{:6d}%{reset} %{cyan}{}')
msg_action = cformat('%{white!}{:6d}%{reset} %{cyan!}{}')
for event in committing_iterator(self._iter_events()):
if not hasattr(event, '_logHandler'):
self.print_error('Event has no log handler!', event_id=event.id)
continue
for item in event._logHandler._logLists['emailLog']:
entry = self._migrate_email_log(event, item)
db.session.add(entry)
if not self.quiet:
self.print_success(msg_email.format(entry.event_id, entry))
for item in event._logHandler._logLists['actionLog']:
entry = self._migrate_action_log(event, item)
db.session.add(entry)
if not self.quiet:
self.print_success(msg_action.format(entry.event_id, entry))
def migrate_event_abstracts(self):
self.print_step("Migrating event abstracts")
for conf, event in committing_iterator(self._iter_events()):
amgr = conf.abstractMgr
duration = amgr._submissionEndDate - amgr._submissionStartDate
if (not amgr._activated and not amgr._abstracts and not amgr._notifTpls and
duration < timedelta(minutes=1) and not conf.program):
continue
mig = AbstractMigration(self, conf, event)
try:
with db.session.begin_nested():
with db.session.no_autoflush:
mig.run()
db.session.flush()
except Exception:
self.print_error(cformat('%{red!}MIGRATION FAILED!'), event_id=event.id)
traceback.print_exc()
raw_input('Press ENTER to continue')
db.session.flush()
def migrate_groups(self):
it = committing_iterator(self.zodb_root['groups'].itervalues())
used_names = set()
for old_group in self.logger.progress_iterator(
'Migrating groups', it, len(self.zodb_root['groups']),
attrgetter('id'), lambda x: ''):
if old_group.__class__.__name__ != 'Group':
continue
group_name = orig_group_name = convert_to_unicode(
old_group.name).strip()
n = 0
while group_name.lower() in used_names:
group_name = '{}-{}'.format(orig_group_name, n)
n += 1
self.print_warning(
'Duplicate group name: {}, using {} instead'.format(
orig_group_name, group_name))
used_names.add(group_name.lower())
group = LocalGroup(id=int(old_group.id), name=group_name)
self.print_success(
'%[white!]{:6d}%[reset] %[cyan]{}%[reset]'.format(
group.id, group.name))
members = set()
for old_member in old_group.members:
if old_member.__class__.__name__ != 'Avatar':
self.print_warning(
'Unsupported group member type: {}'.format(
old_member.__class__.__name__))
continue
user = self.global_ns.avatar_merged_user.get(old_member.id)
if user is None:
self.print_warning('User not found: {}'.format(
old_member.id))
continue
members.add(user)
for member in sorted(members, key=attrgetter('full_name')):
self.print_info(
'%[blue!]<->%[reset] %[white!]{:6d} %[yellow]{} ({})'
.format(member.id, member.full_name, member.email))
group.members = members
self.global_ns.all_groups[group.id] = group
db.session.add(group)
db.session.flush()
def update_gravatars(user=None):
if user is not None:
# explicitly scheduled update (after an email change)
if user.picture_source not in (ProfilePictureSource.gravatar, ProfilePictureSource.identicon):
return
users = [user]
else:
users = User.query.filter(~User.is_deleted, User.picture_source == ProfilePictureSource.gravatar).all()
for user in committing_iterator(users):
gravatar, lastmod = get_gravatar_for_user(user, identicon=False, lastmod=user.picture_metadata['lastmod'])
if gravatar is None:
logger.debug('Gravatar for %r did not change (not modified)', user)
continue
if crc32(gravatar) == user.picture_metadata['hash']:
logger.debug('Gravatar for %r did not change (same hash)', user)
user.picture_metadata['lastmod'] = lastmod
flag_modified(user, 'picture_metadata')
continue
set_user_avatar(user, gravatar, 'gravatar', lastmod)
logger.info('Gravatar of user %s updated', user)
def migrate_event_acls(self):
self.print_step('migrating event ACLs')
protection_mode_map = {
-1: ProtectionMode.public,
0: ProtectionMode.inheriting,
1: ProtectionMode.protected
}
for legacy_event, event in committing_iterator(self._iter_events(),
5000):
ac = legacy_event._Conference__ac
self.print_success('', event_id=event.id)
old_protection_mode = protection_mode_map[ac._accessProtection]
if old_protection_mode == ProtectionMode.public and ac.requiredDomains:
event.protection_mode = ProtectionMode.protected
self._migrate_domains(event, ac.requiredDomains)
else:
event.protection_mode = old_protection_mode
no_access_contact = convert_to_unicode(
getattr(ac, 'contactInfo', ''))
if no_access_contact != 'no contact info defined':
event.own_no_access_contact = no_access_contact
event.access_key = convert_to_unicode(
getattr(legacy_event, '_accessKey', ''))
if not self.quiet:
self.print_success('Protection mode set to {}'.format(
event.protection_mode.name, event_id=event.id))
for legacy_acl in ac.allowed:
event_acl = self.convert_acl(legacy_acl)
if event_acl is None:
self.print_warning(cformat(
'%{red}ACL%{reset}%{yellow} does not exist:%{reset} {}'
).format(legacy_acl),
event_id=event.id)
continue
event.update_principal(event_acl, read_access=True, quiet=True)
if not self.quiet:
self.print_msg(
cformat('%{green}[{}]%{reset} {}').format(
'Event ACL', event_acl))
def migrate_event_settings(self):
self.messages.append(cformat("%{magenta!} - Event Payment Settings:"))
print cformat("%{white!}migrating event settings")
count = 0
EventSetting.delete_all(payment_event_settings.module)
for event in committing_iterator(self._iter_events()):
old_payment = event._modPay
default_conditions = payment_settings.get('conditions')
default_register_email = payment_settings.get('register_email')
default_success_email = payment_settings.get('success_email')
register_email = getattr(old_payment, 'receiptMsg', default_register_email)
success_email = getattr(old_payment, 'successMsg', default_success_email)
conditions = (getattr(old_payment, 'paymentConditions', default_conditions)
if (getattr(old_payment, 'paymentConditionsEnabled', False) and
convert_to_unicode(getattr(old_payment, 'specificPaymentConditions', '')).strip() == '')
else getattr(old_payment, 'specificPaymentConditions', ''))
# The new messages are shown in an "additional info" section, so the old defaults can always go away
if convert_to_unicode(register_email) == 'Please, see the summary of your order:':
register_email = ''
if convert_to_unicode(success_email) == 'Congratulations, your payment was successful.':
success_email = ''
# Get rid of the most terrible part of the old default conditions
conditions = convert_to_unicode(conditions).replace('CANCELLATION :', 'CANCELLATION:')
settings = {
'enabled': getattr(old_payment, 'activated', False),
'currency': event._registrationForm._currency,
'conditions': conditions,
'register_email': register_email,
'success_email': success_email,
}
payment_event_settings.set_multi(event, settings)
count += 1
print cformat("%{cyan}<EventSettings(id={id:>6}, enabled={enabled}, "
"currency={currency})>").format(id=event.id, **settings)
msg = cformat("%{green!}migration of {0} event payment settings successful\n").format(count)
self.messages.append(' ' + msg)
print msg
def migrate_event_abstracts(self):
self.print_step("Migrating event abstracts")
for conf, event in committing_iterator(self._iter_events()):
amgr = conf.abstractMgr
duration = amgr._submissionEndDate - amgr._submissionStartDate
if (not amgr._activated and not amgr._abstracts
and not amgr._notifTpls and duration < timedelta(minutes=1)
and not conf.program):
continue
mig = AbstractMigration(self, conf, event)
try:
with db.session.begin_nested():
with db.session.no_autoflush:
mig.run()
db.session.flush()
except Exception:
self.print_error(cformat('%{red!}MIGRATION FAILED!'),
event_id=event.id)
traceback.print_exc()
raw_input('Press ENTER to continue')
db.session.flush()
def migrate_event_categories(self):
self.print_step("Migrating event categories")
delete_events = set()
for conf in committing_iterator(self._iter_events()):
try:
category_chain = self.category_mapping[int(conf._Conference__owners[0].id)]
except (IndexError, KeyError):
self.print_error(cformat('%{red!}Event has no category!'), event_id=conf.id)
delete_events.add(int(conf.id))
continue
Event.query.filter_by(id=int(conf.id)).update({Event.category_id: category_chain[-1]},
synchronize_session=False)
if not self.quiet:
self.print_success(repr(category_chain), event_id=conf.id)
for event_id in delete_events:
self.print_warning(cformat('%{yellow!}Deleting broken event {}').format(event_id))
Event.query.filter_by(id=event_id).update({Event.is_deleted: True}, synchronize_session=False)
if self.zodb_root['conferences'].has_key(str(event_id)):
del self.zodb_root['conferences'][str(event_id)]
db.session.commit()
transaction.commit()
def migrate_event_images(self):
self.print_step('migrating event images')
for event, picture in committing_iterator(self._iter_pictures()):
local_file = picture._localFile
content_type = mimetypes.guess_type(
local_file.fileName)[0] or 'application/octet-stream'
storage_backend, storage_path, size = self._get_local_file_info(
local_file)
if storage_path is None:
self.print_warning(cformat(
'%{yellow}[{}]%{reset} -> %{red!}Not found in filesystem').
format(local_file.id),
event_id=event.id)
continue
filename = secure_filename(convert_to_unicode(local_file.fileName),
'image')
image = ImageFile(event_id=event.id,
filename=filename,
content_type=content_type,
created_dt=now_utc(),
size=size,
storage_backend=storage_backend,
storage_file_id=storage_path)
db.session.add(image)
db.session.flush()
map_entry = LegacyImageMapping(event_id=event.id,
legacy_image_id=local_file.id,
image_id=image.id)
db.session.add(map_entry)
if not self.quiet:
self.print_success(
cformat('%{cyan}[{}]%{reset} -> %{blue!}{}').format(
local_file.id, image),
event_id=event.id)
def migrate_links(self):
print cformat('%{white!}migrating links')
for avatars in grouper(self._iter_avatars(), 2500, skip_missing=True):
avatars = {int(a.id): a for a in avatars}
users = ((u, avatars[u.id]) for u in User.find(User.id.in_(avatars)))
for user, avatar in committing_iterator(self.flushing_iterator(users, 250), 2500):
registrants = set()
user_shown = False
for type_, entries in avatar.linkedTo.iteritems():
# store registrant roles, in order to avoid duplication below
for role, objects in entries.iteritems():
if (type_ == 'category' and role == 'favorite') or type_ == 'group':
continue
if not objects:
continue
if type_ == 'registration' and role == 'registrant':
registrants |= set(objects)
if not user_shown:
print cformat('%{green}+++%{reset} '
'%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(user.id, user.full_name)
user_shown = True
print cformat('%{blue!}<->%{reset} '
'%{yellow!}{:4d}%{reset}x %{green!}{:12} %{cyan!}{}%{reset}').format(
len(objects), type_, role)
for obj in objects:
try:
UserLink.create_link(user, obj, role, type_)
except Exception as e:
print cformat('%{red!}!!!%{reset} '
'%{red!}linking failed%{reset} (%{yellow!}{}%{reset}): '
'{}').format(unicode(e), obj)
# add old "registrant" entries to registration/registrant
for reg in getattr(avatar, 'registrants', {}).itervalues():
if reg.getConference().getOwner() and reg not in registrants:
UserLink.create_link(user, reg, 'registrant', 'registration')
print cformat('%{cyan!}<->%{reset} '
'%{yellow!} 1%{reset}x %{green!}{:12} %{cyan!}{}%{reset}').format(
'registration', 'registrant')
def migrate_event_locations(self):
self.print_step("Migrating event locations")
for old_event in committing_iterator(self._iter_events()):
custom_location = old_event.places[0] if getattr(
old_event, 'places', None) else None
custom_room = old_event.rooms[0] if getattr(
old_event, 'rooms', None) else None
location_name = None
room_name = None
has_room = False
updates = {}
if custom_location:
location_name = convert_to_unicode(
fix_broken_string(custom_location.name, True))
if custom_location.address:
updates[Event.own_address] = convert_to_unicode(
fix_broken_string(custom_location.address, True))
if custom_room:
room_name = convert_to_unicode(
fix_broken_string(custom_room.name, True))
if location_name and room_name:
mapping = self.room_mapping.get((location_name, room_name))
if mapping:
has_room = True
updates[Event.own_venue_id] = mapping[0]
updates[Event.own_room_id] = mapping[1]
# if we don't have a RB room set, use whatever location/room name we have
if not has_room:
venue_id = self.venue_mapping.get(location_name)
if venue_id is not None:
updates[Event.own_venue_id] = venue_id
updates[Event.own_venue_name] = ''
else:
updates[Event.own_venue_name] = location_name or ''
updates[Event.own_room_name] = room_name or ''
if updates:
Event.query.filter_by(id=int(old_event.id)).update(
updates, synchronize_session=False)
if not self.quiet:
self.print_success(repr(updates), event_id=old_event.id)
def migrate_favorite_users(self):
print cformat('%{white!}migrating favorite users')
for avatars in grouper(self._iter_avatars_with_favorite_users(), 2500, skip_missing=True):
avatars = list(avatars)
users = {u.id: u for u in User.find(User.id.in_(int(a.id) for a, _ in avatars))}
for avatar, user_ids in committing_iterator(avatars, 1000):
user = users.get(int(avatar.id))
if user is None:
print cformat('%{red!}!!!%{reset} '
'%{yellow!}User {} does not exist').format(avatar.id)
continue
print cformat('%{green}+++%{reset} '
'%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(user.id, user.full_name)
valid_users = {u.id: u for u in User.find(User.id.in_(user_ids))}
for user_id in user_ids:
target = valid_users.get(user_id)
if target is None:
print cformat('%{yellow!}!!!%{reset} '
'%{yellow!}User {} does not exist').format(user_id)
continue
user.favorite_users.add(target)
print cformat('%{blue!}<->%{reset} '
'%{white!}{:6d}%{reset} %{cyan}{}%{reset}').format(target.id, target.full_name)
def migrate_event_dates_titles(self):
self.print_step("Migrating event dates and titles")
for old_event in committing_iterator(self._iter_events()):
if 'title' not in old_event.__dict__:
self.print_error('Event has no title in ZODB', old_event.id)
continue
tz = old_event.__dict__.get('timezone', 'UTC')
updates = {
Event.title:
convert_to_unicode(old_event.__dict__['title'])
or '(no title)',
Event.description:
convert_to_unicode(old_event.__dict__['description']) or '',
Event.timezone:
tz,
Event.start_dt:
self._fix_naive(old_event, old_event.__dict__['startDate'],
tz),
Event.end_dt:
self._fix_naive(old_event, old_event.__dict__['endDate'], tz)
}
Event.query.filter_by(id=int(old_event.id)).update(
updates, synchronize_session=False)
if not self.quiet:
self.print_success('', event_id=old_event.id)
# deleted events are not in zodb but also need data
updates = {
Event.title: '***deleted***',
Event.description: '',
Event.timezone: 'UTC',
Event.start_dt: datetime(1970, 1, 1, tzinfo=pytz.utc),
Event.end_dt: datetime(1970, 1, 1, tzinfo=pytz.utc)
}
Event.query.filter_by(is_deleted=True).update(
updates, synchronize_session=False)
db.session.commit()
def migrate_event_notes(self):
self.print_step('migrating event notes')
janitor_user = User.get_one(self.janitor_user_id)
self.print_msg('Using janitor user {}'.format(janitor_user),
always=True)
for event, obj, minutes, special_prot in committing_iterator(
self._iter_minutes()):
if special_prot:
self.print_warning(cformat(
'%{yellow!}{} minutes have special permissions; skipping them'
).format(obj),
event_id=event.id)
continue
path = get_archived_file(minutes, self.archive_dirs)[1]
if path is None:
self.print_error(cformat(
'%{red!}{} minutes not found on disk; skipping them').
format(obj),
event_id=event.id)
continue
with open(path, 'r') as f:
data = convert_to_unicode(f.read()).strip()
if not data:
self.print_warning(
cformat('%{yellow}{} minutes are empty; skipping them'
).format(obj),
always=False,
event_id=event.id)
continue
note = EventNote(linked_object=obj)
note.create_revision(RenderMode.html, data, janitor_user)
db.session.add(note)
if not self.quiet:
self.print_success(cformat('%{cyan}{}').format(obj),
event_id=event.id)
def migrate_layout_settings(self):
print cformat(
'%{white!}migrating layout settings, event logos and custom stylesheets'
)
default_styles = self.zodb_root['MaKaCInfo'][
'main']._styleMgr._defaultEventStylesheet
for event, event_type, dmgr, logo, custom_css in committing_iterator(
self._iter_event_layout_data()):
if event_type != 'conference':
theme = dmgr._defaultstyle
if not theme or theme == default_styles[event_type]:
continue
layout_settings.set(event, 'timetable_theme', theme)
if not self.quiet:
self.print_success(cformat(
'- %{cyan}Default timetable theme: {}').format(theme),
event_id=event.id)
continue
settings = self._get_event_settings(event, dmgr)
layout_settings.set_multi(event, settings)
if not self.quiet:
self.print_success(cformat('- %{cyan}Layout settings'),
event_id=event.id)
if logo or custom_css:
sa_event = Event.get(event.id)
if not sa_event:
self.print_warning(
'Event does not exist (anymore)! Logo and/or CSS file not saved!',
event_id=event.id)
continue
if logo:
self._process_logo(logo, sa_event)
if custom_css:
self._process_css(custom_css, sa_event)
def migrate_event_stubs(self):
self.print_step('migrating event stubs')
for event_id in committing_iterator(self._iter_event_ids(), 5000):
db.session.add(Event(id=int(event_id)))
def migrate_keys(self):
print cformat('%{white!}migrating api keys')
for idx_key, ak in committing_iterator(
self.zodb_root['apikeys'].iteritems()):
if idx_key != ak._key:
print cformat(
'%{red!}!!!%{reset} '
'%{yellow!}Skipping {} - index key {} does not match'
).format(ak._key, idx_key)
continue
elif str(ak._user.id) not in self.zodb_root['avatars']:
print cformat(
'%{red!}!!!%{reset} '
'%{yellow!}Skipping {} - user {} does not exist').format(
ak._key, ak._user.id)
continue
elif ak._user.apiKey != ak:
print cformat(
'%{red!}!!!%{reset} '
'%{yellow!}Skipping {} - user {} has a different api key set'
).format(ak._key, ak._user.id)
continue
last_used_uri = None
if ak._lastPath and ak._lastQuery:
last_used_uri = '{}?{}'.format(
convert_to_unicode(ak._lastPath),
convert_to_unicode(ak._lastQuery))
elif ak._lastPath:
last_used_uri = convert_to_unicode(ak._lastPath)
api_key = APIKey(token=ak._key,
secret=ak._signKey,
user_id=ak._user.id,
is_blocked=ak._isBlocked,
is_persistent_allowed=getattr(
ak, '_persistentAllowed', False),
created_dt=self._to_utc(ak._createdDT),
last_used_dt=self._to_utc(ak._lastUsedDT),
last_used_ip=ak._lastUsedIP,
last_used_uri=last_used_uri,
last_used_auth=ak._lastUseAuthenticated,
use_count=ak._useCount)
db.session.add(api_key)
print cformat(
'%{green}+++%{reset} %{cyan}{}%{reset} [%{blue!}{}%{reset}]'
).format(ak._key, ak._user.email)
for old_key in ak._oldKeys:
# We have no creation time so we use *something* older..
fake_created_dt = self._to_utc(
ak._createdDT) - timedelta(hours=1)
# We don't have anything besides the api key for old keys, so we use a random secret
old_api_key = APIKey(token=old_key,
secret=unicode(uuid4()),
user_id=ak._user.id,
created_dt=fake_created_dt,
is_active=False)
db.session.add(old_api_key)
print cformat(
'%{blue!}***%{reset} %{cyan}{}%{reset} [%{yellow}old%{reset}]'
).format(old_key)
db.session.flush()
def migrate_users(self):
seen_identities = set()
for avatar in committing_iterator(self._iter_avatars(), 5000):
if getattr(avatar, '_mergeTo', None):
self.print_warning('Skipping {} - merged into {}'.format(avatar.id, avatar._mergeTo.id))
merged_user = self.global_ns.avatar_merged_user.get(avatar._mergeTo.id)
if merged_user:
self.global_ns.avatar_merged_user[avatar.id] = merged_user
else:
# if the merge target hasn't yet been migrated, keep track of it
self.unresolved_merge_targets[avatar._mergeTo.id].add(avatar.id)
continue
elif avatar.status == 'Not confirmed':
self.print_warning('Skipping {} - not activated'.format(avatar.id))
continue
elif not avatar.name.strip() and not avatar.surName.strip():
links = {(obj, role): list(objs)
for obj, x in avatar.linkedTo.iteritems()
for role, objs in x.iteritems()
if objs}
if not avatar.identities and not links:
self.print_warning('Skipping {} - no names and no identities/links'.format(avatar.id))
continue
user = self._user_from_avatar(avatar)
self._fix_collisions(user, avatar)
db.session.add(user)
settings = self._settings_from_avatar(avatar)
user_settings.set_multi(user, settings)
# favorite users cannot be migrated here since the target user might not have been migrated yet
for old_categ in avatar.linkedTo['category']['favorite']:
if old_categ:
self.global_ns.user_favorite_categories[old_categ.id].add(user)
db.session.flush()
self.print_success('%[white!]{:6d}%[reset] %[cyan]{}%[reset] [%[blue!]{}%[reset]] '
'{{%[cyan!]{}%[reset]}}'.format(user.id, user.full_name, user.email,
', '.join(user.secondary_emails)))
# migrate API keys
self._migrate_api_keys(avatar, user)
# migrate identities of avatars
for old_identity in avatar.identities:
identity = None
username = convert_to_unicode(old_identity.login).strip().lower()
if not username:
self.print_warning("Empty username: {}. Skipping identity.".format(old_identity))
continue
provider = {
'LocalIdentity': 'indico',
'LDAPIdentity': self.ldap_provider_name
}.get(old_identity.__class__.__name__)
if provider is None:
self.print_error("Unsupported provider: {}. Skipping identity.".format(
old_identity.__class__.__name__))
continue
if (provider, username) in seen_identities:
self.print_error("Duplicate identity: {}, {}. Skipping.".format(provider, username))
continue
if provider == 'indico' and not self.ignore_local_accounts:
identity = Identity(provider=provider, identifier=username)
if not hasattr(old_identity, 'algorithm'): # plaintext password
if not old_identity.password:
# password is empty, skip identity
self.print_error("Identity '{}' has empty password. Skipping identity.".format(
old_identity.login))
continue
identity.password = old_identity.password
else:
assert old_identity.algorithm == 'bcrypt'
identity.password_hash = old_identity.password
elif provider == self.ldap_provider_name:
identity = Identity(provider=provider, identifier=username)
if identity:
self.print_info('%[blue!]<->%[reset] %[yellow]{}%[reset]'.format(identity))
user.identities.add(identity)
seen_identities.add((provider, username))
if hasattr(avatar, 'personalInfo') and avatar.personalInfo._basket._users:
self.favorite_avatars[user.id] = avatar.personalInfo._basket._users
# Map old merged identities (no longer in AvatarHolder)
# to newly created user
for merged_avatar in getattr(avatar, '_mergeFrom', ()):
if merged_avatar.id != avatar.id:
self.global_ns.avatar_merged_user[merged_avatar.id] = user
self.global_ns.avatar_merged_user[avatar.id] = user
if avatar.id in self.unresolved_merge_targets:
del self.unresolved_merge_targets[avatar.id]
self._resolve_merge_targets(avatar.id, user)
db.session.flush()
def migrate_transactions(self):
self.messages.append(cformat("%{magenta!} - Payment Transactions:"))
print cformat("%{white!}migrating payment transactions")
count = 0
errors = 0
warnings = 0
for event, registrant, transaction in committing_iterator(
self._iter_transactions()):
try:
data = self._get_transaction_data(transaction, event)
except ValueError as e:
print cformat("%{red!}{0} (evt: {1}, reg: {2})").format(
e, event.id, registrant._id)
errors += 1
continue
if data['provider'] == '_manual' and data['amount'] == 0.0:
print cformat(
"%{yellow!}Skipping {0[provider]} transaction (evt: {1}, reg: {2}) "
"with zero amount: {0[amount]} {0[currency]}").format(
data, event.id, registrant._id)
warnings += 1
continue
elif data['amount'] < 0.0:
print cformat(
"%{yellow!}Skipping {0[provider]} transaction (evt: {1}, reg: {2}) "
"with negative amount: {0[amount]} {0[currency]}").format(
data, event.id, registrant._id)
warnings += 1
continue
pt = PaymentTransaction(event_id=int(event.id),
registrant_id=int(registrant._id),
status=TransactionStatus.successful,
**data)
count += 1
print cformat("%{cyan}{0}").format(pt)
db.session.add(pt)
if warnings:
warning_msg = cformat(
"%{yellow!}There were {0} warnings during the migration "
"of the payment transactions.").format(warnings)
self.messages.append(' ' + warning_msg)
print warning_msg
if errors:
msg = cformat(
"%{red!}There were some errors during the migration of the payment transactions.\n"
"{0} transactions were not migrated and will be lost.\n"
).format(errors)
else:
msg = cformat(
"%{green!}migration of {0} payment transactions successful\n"
).format(count)
self.messages.append(' ' + '\n '.join(msg.split('\n')))
print msg
def migrate_users(self):
print cformat('%{white!}migrating users')
seen_identities = set()
for avatar in committing_iterator(self._iter_avatars(), 5000):
if getattr(avatar, '_mergeTo', None):
print cformat('%{red!}!!!%{reset} '
'%{yellow!}Skipping {} - merged into {}').format(
avatar.id, avatar._mergeTo.id)
continue
elif avatar.status == 'Not confirmed':
print cformat('%{yellow!}!!!%{reset} '
'%{yellow!}Skipping {} - not activated').format(
avatar.id)
continue
elif not avatar.name.strip() and not avatar.surName.strip():
links = {(obj, role): list(objs)
for obj, x in avatar.linkedTo.iteritems()
for role, objs in x.iteritems() if objs}
if not avatar.identities and not links:
print cformat(
'%{yellow!}!!!%{reset} '
'%{yellow!}Skipping {} - no names and no identities/links'
).format(avatar.id)
continue
user = self._user_from_avatar(avatar)
self._fix_collisions(user, avatar)
db.session.add(user)
settings = self._settings_from_avatar(avatar)
user_settings.set_multi(user, settings)
# favorite users cannot be migrated here since the target user might not have been migrated yet
# XXX: adapt to new categories for 2.0
user.favorite_categories = set(
filter(None, avatar.linkedTo['category']['favorite']))
db.session.flush()
print cformat(
'%{green}+++%{reset} '
'%{white!}{:6d}%{reset} %{cyan}{}%{reset} [%{blue!}{}%{reset}] '
'{{%{cyan!}{}%{reset}}}').format(
user.id, user.full_name, user.email,
', '.join(user.secondary_emails))
# migrate API keys
self._migrate_api_keys(avatar, user)
# migrate identities of non-deleted avatars
if not user.is_deleted:
for old_identity in avatar.identities:
identity = None
username = convert_to_unicode(
old_identity.login).strip().lower()
if not username:
print cformat(
"%{red!}!!!%{reset} "
"%{yellow!}Empty username: {}. Skipping identity."
).format(old_identity)
continue
provider = {
'LocalIdentity': 'indico',
'LDAPIdentity': self.ldap_provider_name
}.get(old_identity.__class__.__name__)
if provider is None:
print cformat(
"%{red!}!!!%{reset} "
"%{yellow!}Unsupported provider: {}. Skipping identity."
).format(old_identity.__class__.__name__)
continue
if (provider, username) in seen_identities:
print cformat(
"%{red!}!!!%{reset} "
"%{yellow!}Duplicate identity: {}, {}. Skipping."
).format(provider, username)
continue
if provider == 'indico' and not self.ignore_local_accounts:
identity = Identity(provider=provider,
identifier=username)
if not hasattr(old_identity,
'algorithm'): # plaintext password
if not old_identity.password:
# password is empty, skip identity
print cformat(
"%{red!}!!!%{reset} "
"%{yellow!}Identity '{}' has empty password. Skipping identity."
).format(old_identity.login)
continue
identity.password = old_identity.password
else:
assert old_identity.algorithm == 'bcrypt'
identity.password_hash = old_identity.password
elif provider == self.ldap_provider_name:
identity = Identity(provider=provider,
identifier=username)
if identity:
print cformat(
'%{blue!}<->%{reset} %{yellow}{}%{reset}').format(
identity)
user.identities.add(identity)
seen_identities.add((provider, username))
for merged_avatar in getattr(avatar, '_mergeFrom', ()):
if merged_avatar.id == avatar.id:
continue
merged = self._user_from_avatar(merged_avatar,
is_deleted=True,
merged_into_id=user.id)
print cformat(
'%{blue!}***%{reset} '
'%{white!}{:6d}%{reset} %{cyan}{}%{reset} [%{blue!}{}%{reset}] '
'{{%{cyan!}{}%{reset}}}').format(
merged.id, merged.full_name, merged.email,
', '.join(merged.secondary_emails))
self._fix_collisions(merged, merged_avatar)
db.session.add(merged)
db.session.flush()
