def _assert_no_migration_restrictions(self, domain_name):
assert should_use_sql_backend(domain_name)
assert not COUCH_SQL_MIGRATION_BLACKLIST.enabled(
domain_name, NAMESPACE_DOMAIN)
assert not stale_get_export_count(domain_name)
assert not any(
custom_report_domain == domain_name
for custom_report_domain in settings.DOMAIN_MODULE_MAP.keys())
def migrate_domain(domain, dryrun=False):
from couchexport.models import SavedExportSchema
export_count = stale_get_export_count(domain)
metas = []
if export_count:
for old_export in with_progress_bar(stale_get_exports_json(domain),
length=export_count,
prefix=domain):
try:
_, migration_meta = convert_saved_export_to_export_instance(
domain, SavedExportSchema.wrap(old_export), dryrun=dryrun)
except Exception, e:
print 'Failed parsing {}: {}'.format(old_export['_id'], e)
else:
metas.append(migration_meta)
def migrate_domain(domain, dryrun=False, force_convert_columns=False):
from couchexport.models import SavedExportSchema
export_count = stale_get_export_count(domain)
metas = []
if export_count:
for old_export in with_progress_bar(
stale_get_exports_json(domain),
length=export_count,
prefix=domain):
try:
_, migration_meta = convert_saved_export_to_export_instance(
domain,
SavedExportSchema.wrap(old_export),
dryrun=dryrun,
force_convert_columns=force_convert_columns,
)
except Exception, e:
print 'Failed parsing {}: {}'.format(old_export['_id'], e)
raise e
else:
metas.append(migration_meta)
def show_in_navigation(cls, domain=None, project=None, user=None):
return domain and stale_get_export_count(domain) > 0
def test_stale_get_export_count(self):
result = stale_get_export_count('domain2')
self.assertEqual(result, 2)
def test_stale_get_export_count(self):
result = stale_get_export_count('domain2')
self.assertEqual(result, 2)
def migrate_domain(domain, dryrun=False, force_convert_columns=False):
from couchexport.models import SavedExportSchema
export_count = stale_get_export_count(domain)
metas = []
if export_count:
for old_export in with_progress_bar(get_exports_json(domain),
length=export_count,
prefix=domain):
with CriticalSection(['saved-export-{}'.format(old_export['_id'])],
timeout=120):
try:
_, migration_meta = convert_saved_export_to_export_instance(
domain,
SavedExportSchema.get(old_export['_id']),
dryrun=dryrun,
force_convert_columns=force_convert_columns,
)
except Exception as e:
print('Failed parsing {}: {}'.format(old_export['_id'], e))
raise
else:
metas.append(migration_meta)
if not dryrun:
set_toggle(OLD_EXPORTS.slug, domain, False, namespace=NAMESPACE_DOMAIN)
toggle_js_domain_cachebuster.clear(domain)
# Remote app migrations must have access to UserDefined columns and tables
if any(map(lambda meta: meta.is_remote_app_migration, metas)):
set_toggle(ALLOW_USER_DEFINED_EXPORT_COLUMNS.slug,
domain,
True,
namespace=NAMESPACE_DOMAIN)
toggle_js_domain_cachebuster.clear(domain)
for meta in metas:
if not meta.skipped_tables and not meta.skipped_columns:
continue
output = '* Export information for export: {} *'.format(
meta.old_export_url)
schema_id_output = 'Generated schema: {}'.format(
meta.generated_schema_id)
print('')
print('*' * len(output))
print(output)
print('* {}{} *'.format(
schema_id_output, ' ' * (len(output) - len(schema_id_output) - 4)))
print('*' * len(output))
print('')
if meta.skipped_tables:
print('# Skipped tables #')
for table_meta in meta.skipped_tables:
table_meta.pretty_print()
if meta.skipped_columns:
print('# Skipped columns #')
for column_meta in meta.skipped_columns:
column_meta.pretty_print()
return metas
def show_in_navigation(cls, domain=None, project=None, user=None):
return stale_get_export_count(domain) > 0
def handle(self, **options):
dryrun = options.pop('dryrun')
limit = options.pop('limit')
force_convert_columns = options.pop('force_convert_columns')
count = 0
if dryrun:
print(
'*** Running in dryrun mode. Will not save any conversion ***')
print('*** Migrating {} exports ***'.format(limit or 'ALL'))
skipped_domains = []
for doc in Domain.get_all(include_docs=False):
domain = doc['key']
if stale_get_export_count(domain):
print("Inspecting '{}'".format(domain))
if not force_convert_columns:
try:
metas = migrate_domain(
domain,
dryrun=True,
force_convert_columns=force_convert_columns)
except Exception:
print('Migration raised an exception, skipping.')
traceback.print_exc()
skipped_domains.append(domain)
continue
has_skipped_tables = any(meta.skipped_tables
for meta in metas)
has_skipped_columns = any(meta.skipped_columns
for meta in metas)
has_case_history = any(meta.has_case_history
for meta in metas)
is_remote_app_migration = any(meta.is_remote_app_migration
for meta in metas)
if has_skipped_tables or has_skipped_columns or has_case_history:
print(
'Skipping {} because we would have skipped columns'
.format(domain))
skipped_domains.append(domain)
continue
if is_remote_app_migration:
print('Skipping {} because it contains remote apps'.
format(domain))
skipped_domains.append(domain)
continue
if not dryrun:
print('Migrating {}'.format(domain))
try:
migrate_domain(
domain,
dryrun=False,
force_convert_columns=force_convert_columns)
except Exception:
print('Migration raised an exception, skipping.')
skipped_domains.append(domain)
continue
else:
print(
'No skipped tables/columns. Not migrating since dryrun is specified'
)
count += 1
if limit is not None and count >= limit:
break
send_HTML_email(
'Export migration results',
'{}@{}'.format('commcarehq-ops+admins', 'dimagi.com'), '''
Skipped domains: {} <br />
Successfully migrated: {}
'''.format(
', '.join(skipped_domains),
count,
))
