def _assert_no_migration_restrictions(self, domain_name): assert should_use_sql_backend(domain_name) assert not COUCH_SQL_MIGRATION_BLACKLIST.enabled( domain_name, NAMESPACE_DOMAIN) assert not stale_get_export_count(domain_name) assert not any( custom_report_domain == domain_name for custom_report_domain in settings.DOMAIN_MODULE_MAP.keys())
def migrate_domain(domain, dryrun=False): from couchexport.models import SavedExportSchema export_count = stale_get_export_count(domain) metas = [] if export_count: for old_export in with_progress_bar(stale_get_exports_json(domain), length=export_count, prefix=domain): try: _, migration_meta = convert_saved_export_to_export_instance( domain, SavedExportSchema.wrap(old_export), dryrun=dryrun) except Exception, e: print 'Failed parsing {}: {}'.format(old_export['_id'], e) else: metas.append(migration_meta)
def migrate_domain(domain, dryrun=False, force_convert_columns=False): from couchexport.models import SavedExportSchema export_count = stale_get_export_count(domain) metas = [] if export_count: for old_export in with_progress_bar( stale_get_exports_json(domain), length=export_count, prefix=domain): try: _, migration_meta = convert_saved_export_to_export_instance( domain, SavedExportSchema.wrap(old_export), dryrun=dryrun, force_convert_columns=force_convert_columns, ) except Exception, e: print 'Failed parsing {}: {}'.format(old_export['_id'], e) raise e else: metas.append(migration_meta)
def show_in_navigation(cls, domain=None, project=None, user=None): return domain and stale_get_export_count(domain) > 0
def test_stale_get_export_count(self): result = stale_get_export_count('domain2') self.assertEqual(result, 2)
def migrate_domain(domain, dryrun=False, force_convert_columns=False): from couchexport.models import SavedExportSchema export_count = stale_get_export_count(domain) metas = [] if export_count: for old_export in with_progress_bar(get_exports_json(domain), length=export_count, prefix=domain): with CriticalSection(['saved-export-{}'.format(old_export['_id'])], timeout=120): try: _, migration_meta = convert_saved_export_to_export_instance( domain, SavedExportSchema.get(old_export['_id']), dryrun=dryrun, force_convert_columns=force_convert_columns, ) except Exception as e: print('Failed parsing {}: {}'.format(old_export['_id'], e)) raise else: metas.append(migration_meta) if not dryrun: set_toggle(OLD_EXPORTS.slug, domain, False, namespace=NAMESPACE_DOMAIN) toggle_js_domain_cachebuster.clear(domain) # Remote app migrations must have access to UserDefined columns and tables if any(map(lambda meta: meta.is_remote_app_migration, metas)): set_toggle(ALLOW_USER_DEFINED_EXPORT_COLUMNS.slug, domain, True, namespace=NAMESPACE_DOMAIN) toggle_js_domain_cachebuster.clear(domain) for meta in metas: if not meta.skipped_tables and not meta.skipped_columns: continue output = '* Export information for export: {} *'.format( meta.old_export_url) schema_id_output = 'Generated schema: {}'.format( meta.generated_schema_id) print('') print('*' * len(output)) print(output) print('* {}{} *'.format( schema_id_output, ' ' * (len(output) - len(schema_id_output) - 4))) print('*' * len(output)) print('') if meta.skipped_tables: print('# Skipped tables #') for table_meta in meta.skipped_tables: table_meta.pretty_print() if meta.skipped_columns: print('# Skipped columns #') for column_meta in meta.skipped_columns: column_meta.pretty_print() return metas
def show_in_navigation(cls, domain=None, project=None, user=None): return stale_get_export_count(domain) > 0
def handle(self, **options): dryrun = options.pop('dryrun') limit = options.pop('limit') force_convert_columns = options.pop('force_convert_columns') count = 0 if dryrun: print( '*** Running in dryrun mode. Will not save any conversion ***') print('*** Migrating {} exports ***'.format(limit or 'ALL')) skipped_domains = [] for doc in Domain.get_all(include_docs=False): domain = doc['key'] if stale_get_export_count(domain): print("Inspecting '{}'".format(domain)) if not force_convert_columns: try: metas = migrate_domain( domain, dryrun=True, force_convert_columns=force_convert_columns) except Exception: print('Migration raised an exception, skipping.') traceback.print_exc() skipped_domains.append(domain) continue has_skipped_tables = any(meta.skipped_tables for meta in metas) has_skipped_columns = any(meta.skipped_columns for meta in metas) has_case_history = any(meta.has_case_history for meta in metas) is_remote_app_migration = any(meta.is_remote_app_migration for meta in metas) if has_skipped_tables or has_skipped_columns or has_case_history: print( 'Skipping {} because we would have skipped columns' .format(domain)) skipped_domains.append(domain) continue if is_remote_app_migration: print('Skipping {} because it contains remote apps'. format(domain)) skipped_domains.append(domain) continue if not dryrun: print('Migrating {}'.format(domain)) try: migrate_domain( domain, dryrun=False, force_convert_columns=force_convert_columns) except Exception: print('Migration raised an exception, skipping.') skipped_domains.append(domain) continue else: print( 'No skipped tables/columns. Not migrating since dryrun is specified' ) count += 1 if limit is not None and count >= limit: break send_HTML_email( 'Export migration results', '{}@{}'.format('commcarehq-ops+admins', 'dimagi.com'), ''' Skipped domains: {} <br /> Successfully migrated: {} '''.format( ', '.join(skipped_domains), count, ))