def _migrate(self, label, query_count, query, model, get_entity):
self.start(label)
total_count = self.connection_source.execute(
text(query_count)).fetchone()[0]
print('Total: {0} rows'.format(total_count))
query = text(query)
batch = SimpleBatch(
self.session_target, self.batch_size, model)
with transaction.manager, batch:
count = 0
for entity_in in self.connection_source.execute(query):
count += 1
# ignore associations for these documents because the
# documents are broken (no latest version)
if entity_in.main_id not in [224228, 436917] and \
entity_in.linked_id not in [224228, 436917]:
batch.add(get_entity(entity_in))
self.progress(count, total_count)
# the transaction will not be committed automatically when doing
# a bulk insertion. `mark_changed` forces a commit.
zope.sqlalchemy.mark_changed(self.session_target)
self.stop()
def migrate(self):
self.start('Mailing lists subscriptions')
total_count = self.connection_source.execute(
text(SQL_ML_QUERY_COUNT)).fetchone()[0]
print('Total: {0} rows'.format(total_count))
query = text(SQL_ML_QUERY)
batch = SimpleBatch(self.session_target, self.batch_size, Mailinglist)
with transaction.manager, batch:
count = 0
for entity_in in self.connection_source.execute(query):
count += 1
batch.add(self.get_subscription(entity_in))
self.progress(count, total_count)
zope.sqlalchemy.mark_changed(self.session_target)
# run analyze on the table (must be outside a transaction)
engine = self.session_target.bind
conn = engine.connect()
old_lvl = conn.connection.isolation_level
conn.connection.set_isolation_level(0)
conn.execute('analyze ' + sympa_schema + '.subscriber_table;')
conn.connection.set_isolation_level(old_lvl)
conn.close()
self.stop()
def _migrate(self, label, query_count, query, model, get_entity):
self.start(label)
total_count = self.connection_source.execute(
text(query_count)).fetchone()[0]
print('Total: {0} rows'.format(total_count))
query = text(query)
batch = SimpleBatch(self.session_target, self.batch_size, model)
with transaction.manager, batch:
count = 0
for entity_in in self.connection_source.execute(query):
count += 1
# ignore associations for these documents because the
# documents are broken (no latest version)
if entity_in.main_id not in [224228, 436917] and \
entity_in.linked_id not in [224228, 436917]:
batch.add(get_entity(entity_in))
self.progress(count, total_count)
# the transaction will not be committed automatically when doing
# a bulk insertion. `mark_changed` forces a commit.
zope.sqlalchemy.mark_changed(self.session_target)
self.stop()
def migrate(self):
self.start('Mailing lists subscriptions')
total_count = self.connection_source.execute(
text(SQL_ML_QUERY_COUNT)).fetchone()[0]
print('Total: {0} rows'.format(total_count))
query = text(SQL_ML_QUERY)
batch = SimpleBatch(
self.session_target, self.batch_size, Mailinglist)
with transaction.manager, batch:
count = 0
for entity_in in self.connection_source.execute(query):
count += 1
batch.add(self.get_subscription(entity_in))
self.progress(count, total_count)
zope.sqlalchemy.mark_changed(self.session_target)
# run analyze on the table (must be outside a transaction)
engine = self.session_target.bind
conn = engine.connect()
old_lvl = conn.connection.isolation_level
conn.connection.set_isolation_level(0)
conn.execute('analyze ' + sympa_schema + '.subscriber_table;')
conn.connection.set_isolation_level(old_lvl)
conn.close()
self.stop()
def migrate(self):
self.start('users')
query_count = text('select count(*) from app_users_private_data')
total_count = self.connection_source.execute(query_count).fetchone()[0]
print('Total: {0} rows'.format(total_count))
query = text('select id, login_name, email, '
'password, password_tmp '
'from app_users_private_data order by id')
duplicate_user_query = text(
'select count(*) as count, login_name from app_users_private_data '
'group by login_name having count(*) > 1')
def get_group_by_id(gid):
return self.connection_source.execute(text(
'select user_id from app_users_groups where '
'group_id = ' + str(gid)))
batch = SimpleBatch(self.session_target, self.batch_size, User)
with transaction.manager, batch:
count = 0
super_admins = get_group_by_id(1)
admins = get_group_by_id(2)
pending = get_group_by_id(4)
# skipping useless logged/3 (everyone)
# TODO: inactive = group_query_builder(5)
duplicates = set()
for item in self.connection_source.execute(duplicate_user_query):
assert item.count == 2
duplicates.add(item.login_name)
for user_in in self.connection_source.execute(query):
count += 1
id = user_in.id
username = user_in.login_name
if username in duplicates:
# Rename the oldest login_name
duplicates.remove(username)
username = username + '_v6_duplicate'
batch.add(dict(
id=user_in.id,
username=username,
email=user_in.email,
_password=user_in.password,
temp_password=user_in.password_tmp,
moderator=id in super_admins or id in admins,
email_validated=id not in pending
))
self.progress(count, total_count)
# the transaction will not be commited automatically when doing
# a bulk insertion. `mark_changed` forces a commit.
zope.sqlalchemy.mark_changed(self.session_target)
self.stop()
def _create_dummy_routes(self, climbing_sites):
route_batch = DocumentBatch(self.session_target, self.batch_size,
Route, ArchiveRoute, DocumentGeometry,
ArchiveDocumentGeometry)
route_locale_batch = DocumentBatch(self.session_target,
self.batch_size, RouteLocale,
ArchiveRouteLocale, None, None)
history_batch = SimpleBatch(self.session_target, self.batch_size,
HistoryMetaData)
version_batch = SimpleBatch(self.session_target, self.batch_size,
DocumentVersion)
with route_batch:
links_for_climbing_site, route_locales, route_locale_archives, \
history_entries, version_entries = \
self._create_routes(climbing_sites, route_batch)
with route_locale_batch:
for route_locale in route_locales:
route_locale_batch.add_document(route_locale)
for route_locale_archive in route_locale_archives:
route_locale_batch.add_archive_documents(
[route_locale_archive])
with history_batch:
for entry in history_entries:
history_batch.add(entry)
with version_batch:
for entry in version_entries:
version_batch.add(entry)
return links_for_climbing_site
def migrate(self):
self.start('users')
query_count = text('select count(*) from app_users_private_data')
total_count = self.connection_source.execute(query_count).fetchone()[0]
print('Total: {0} rows'.format(total_count))
query = text('select id, login_name, topo_name, email, '
'password, username as forum_username '
'from app_users_private_data order by id')
def get_group_by_id(gid):
results = self.connection_source.execute(
text('select user_id from app_users_groups where '
'group_id = ' + str(gid))).fetchall()
return [r[0] for r in results]
batch = SimpleBatch(self.session_target, self.batch_size, User)
with transaction.manager, batch:
count = 0
super_admins = get_group_by_id(1)
admins = get_group_by_id(2)
pending = get_group_by_id(4)
# skipping useless logged/3 (everyone)
# TODO: inactive = group_query_builder(5)
for user_in in self.connection_source.execute(query):
count += 1
id = user_in.id
batch.add(
dict(id=user_in.id,
username=user_in.login_name,
name=user_in.topo_name,
forum_username=user_in.forum_username,
email=user_in.email,
_password=user_in.password,
moderator=id in super_admins or id in admins,
email_validated=id not in pending))
self.progress(count, total_count)
# the transaction will not be commited automatically when doing
# a bulk insertion. `mark_changed` forces a commit.
zope.sqlalchemy.mark_changed(self.session_target)
self.stop()
def _migrate(self, label, query_count, query, model, get_entity):
self.start(label)
total_count = self.connection_source.execute(text(query_count)).fetchone()[0]
print("Total: {0} rows".format(total_count))
query = text(query)
batch = SimpleBatch(self.session_target, self.batch_size, model)
with transaction.manager, batch:
count = 0
for entity_in in self.connection_source.execute(query):
count += 1
batch.add(get_entity(entity_in))
self.progress(count, total_count)
# the transaction will not be commited automatically when doing
# a bulk insertion. `mark_changed` forces a commit.
zope.sqlalchemy.mark_changed(self.session_target)
self.stop()
def _migrate(self, label, query_count, query, model, get_entity):
self.start(label)
total_count = self.connection_source.execute(
text(query_count)).fetchone()[0]
print('Total: {0} rows'.format(total_count))
query = text(query)
batch = SimpleBatch(self.session_target, self.batch_size, model)
with transaction.manager, batch:
count = 0
for entity_in in self.connection_source.execute(query):
count += 1
batch.add(get_entity(entity_in))
self.progress(count, total_count)
# the transaction will not be commited automatically when doing
# a bulk insertion. `mark_changed` forces a commit.
zope.sqlalchemy.mark_changed(self.session_target)
self.stop()
def _create_dummy_routes(self, climbing_sites):
route_batch = DocumentBatch(
self.session_target, self.batch_size,
Route, ArchiveRoute,
DocumentGeometry, ArchiveDocumentGeometry)
route_locale_batch = DocumentBatch(
self.session_target, self.batch_size,
RouteLocale, ArchiveRouteLocale,
None, None)
history_batch = SimpleBatch(
self.session_target, self.batch_size, HistoryMetaData)
version_batch = SimpleBatch(
self.session_target, self.batch_size, DocumentVersion)
with route_batch:
links_for_climbing_site, route_locales, route_locale_archives, \
history_entries, version_entries = \
self._create_routes(climbing_sites, route_batch)
with route_locale_batch:
for route_locale in route_locales:
route_locale_batch.add_document(route_locale)
for route_locale_archive in route_locale_archives:
route_locale_batch.add_archive_documents(
[route_locale_archive])
with history_batch:
for entry in history_entries:
history_batch.add(entry)
with version_batch:
for entry in version_entries:
version_batch.add(entry)
return links_for_climbing_site
def _create_links_for_routes(self, links_for_climbing_site):
association_batch = SimpleBatch(
self.session_target, self.batch_size, Association)
association_log_batch = SimpleBatch(
self.session_target, self.batch_size, AssociationLog)
with association_batch, association_log_batch:
for climbing_site_id in links_for_climbing_site:
route_id, outings_ids = \
links_for_climbing_site[climbing_site_id]
# create association between fake route and climbing site
link_site_route = dict(
parent_document_id=climbing_site_id,
parent_document_type=WAYPOINT_TYPE,
child_document_id=route_id,
child_document_type=ROUTE_TYPE
)
association_batch.add(link_site_route)
link_site_route_log = dict(link_site_route)
link_site_route_log['user_id'] = 2 # c2c user
association_log_batch.add(link_site_route_log)
# create associations between fake route and outings
for outing_id in outings_ids:
link_outing_route = dict(
parent_document_id=route_id,
parent_document_type=ROUTE_TYPE,
child_document_id=outing_id,
child_document_type=OUTING_TYPE
)
association_batch.add(link_outing_route)
link_outing_route_log = dict(link_outing_route)
link_outing_route_log['user_id'] = 2 # c2c user
association_log_batch.add(link_outing_route_log)
def migrate(self):
self.start('users')
query_count = text('select count(*) from app_users_private_data')
total_count = self.connection_source.execute(query_count).fetchone()[0]
print('Total: {0} rows'.format(total_count))
query = text('select id, login_name, topo_name, email, '
'password, username as forum_username, pref_cookies, '
'is_profile_public '
'from app_users_private_data order by id')
def get_group_by_id(gid):
results = self.connection_source.execute(
text('select user_id from app_users_groups where '
'group_id = ' + str(gid))).fetchall()
return [r[0] for r in results]
filter_areas_for_users = []
batch = SimpleBatch(self.session_target, self.batch_size, User)
with transaction.manager, batch:
count = 0
super_admins = get_group_by_id(1)
admins = get_group_by_id(2)
pending = get_group_by_id(4)
# skipping useless logged/3 (everyone)
# TODO: inactive = group_query_builder(5)
for user_in in self.connection_source.execute(query):
count += 1
id = user_in.id
preferences = self.parse_preferences(user_in.pref_cookies, id)
filter_areas = self.get_areas(
preferences.get('filtered_places'))
filter_areas_for_users.extend([(id, area_id)
for area_id in filter_areas])
filter_activities = self.get_activities(
preferences.get('filtered_activities'))
batch.add(
dict(id=user_in.id,
username=user_in.login_name,
name=user_in.topo_name,
forum_username=user_in.forum_username,
email=user_in.email,
_password=user_in.password,
moderator=id in super_admins or id in admins,
email_validated=id not in pending,
feed_filter_activities=filter_activities,
is_profile_public=user_in.is_profile_public))
self.progress(count, total_count)
# the transaction will not be commited automatically when doing
# a bulk insertion. `mark_changed` forces a commit.
zope.sqlalchemy.mark_changed(self.session_target)
# add the area filter rows after all users have been created
all_area_ids = set(
area_id
for area_id, in self.session_target.query(Area.document_id).all())
batch_filter_area = SimpleBatch(self.session_target, self.batch_size,
FilterArea)
with transaction.manager, batch_filter_area:
for user_id, area_id in filter_areas_for_users:
# some areas used in the preferences do no longer exist,
# ignore them
if area_id in all_area_ids:
batch_filter_area.add(
dict(user_id=user_id, area_id=area_id))
zope.sqlalchemy.mark_changed(self.session_target)
self.stop()
def _create_links_for_routes(self, links_for_climbing_site):
association_batch = SimpleBatch(self.session_target, self.batch_size,
Association)
association_log_batch = SimpleBatch(self.session_target,
self.batch_size, AssociationLog)
with association_batch, association_log_batch:
for climbing_site_id in links_for_climbing_site:
route_id, outings_ids = \
links_for_climbing_site[climbing_site_id]
# create association between fake route and climbing site
link_site_route = dict(parent_document_id=climbing_site_id,
parent_document_type=WAYPOINT_TYPE,
child_document_id=route_id,
child_document_type=ROUTE_TYPE)
association_batch.add(link_site_route)
link_site_route_log = dict(link_site_route)
link_site_route_log['user_id'] = 2 # c2c user
association_log_batch.add(link_site_route_log)
# create associations between fake route and outings
for outing_id in outings_ids:
link_outing_route = dict(parent_document_id=route_id,
parent_document_type=ROUTE_TYPE,
child_document_id=outing_id,
child_document_type=OUTING_TYPE)
association_batch.add(link_outing_route)
link_outing_route_log = dict(link_outing_route)
link_outing_route_log['user_id'] = 2 # c2c user
association_log_batch.add(link_outing_route_log)
def migrate(self):
self.start('users')
query_count = text('select count(*) from app_users_private_data')
total_count = self.connection_source.execute(query_count).fetchone()[0]
print('Total: {0} rows'.format(total_count))
query = text('select id, login_name, topo_name, email, '
'password, username as forum_username, pref_cookies, '
'is_profile_public '
'from app_users_private_data order by id')
def get_group_by_id(gid):
results = self.connection_source.execute(text(
'select user_id from app_users_groups where '
'group_id = ' + str(gid))).fetchall()
return [r[0] for r in results]
filter_areas_for_users = []
batch = SimpleBatch(self.session_target, self.batch_size, User)
with transaction.manager, batch:
count = 0
super_admins = get_group_by_id(1)
admins = get_group_by_id(2)
pending = get_group_by_id(4)
# skipping useless logged/3 (everyone)
# TODO: inactive = group_query_builder(5)
for user_in in self.connection_source.execute(query):
count += 1
id = user_in.id
preferences = self.parse_preferences(user_in.pref_cookies, id)
filter_areas = self.get_areas(
preferences.get('filtered_places'))
filter_areas_for_users.extend(
[(id, area_id) for area_id in filter_areas])
filter_activities = self.get_activities(
preferences.get('filtered_activities'))
batch.add(dict(
id=user_in.id,
username=user_in.login_name,
name=user_in.topo_name,
forum_username=user_in.forum_username,
email=user_in.email,
_password=user_in.password,
moderator=id in super_admins or id in admins,
email_validated=id not in pending,
feed_filter_activities=filter_activities,
is_profile_public=user_in.is_profile_public
))
self.progress(count, total_count)
# the transaction will not be commited automatically when doing
# a bulk insertion. `mark_changed` forces a commit.
zope.sqlalchemy.mark_changed(self.session_target)
# add the area filter rows after all users have been created
all_area_ids = set(
area_id for area_id, in
self.session_target.query(Area.document_id).all()
)
batch_filter_area = SimpleBatch(
self.session_target, self.batch_size, FilterArea)
with transaction.manager, batch_filter_area:
for user_id, area_id in filter_areas_for_users:
# some areas used in the preferences do no longer exist,
# ignore them
if area_id in all_area_ids:
batch_filter_area.add(dict(
user_id=user_id,
area_id=area_id))
zope.sqlalchemy.mark_changed(self.session_target)
self.stop()
