def handle(self, domain=None, reset=False, chunk_size=100, all=None, **options): exporters = options.get('exporters') if not domain: raise CommandError(USAGE) if all: exporters = list(EXPORTERS) for exporter_slug in exporters: try: exporter = EXPORTERS[exporter_slug] except KeyError: raise CommandError(USAGE) self.stdout.write("\nRunning exporter: {}\n{}".format(exporter_slug, '-' * 50)) export_filename = get_export_filename(exporter_slug, domain) if os.path.exists(export_filename): reset_export = False self.stderr.write( "WARNING: export file for {} exists. " "Resuming export progress. Delete file to reset progress.".format(exporter_slug) ) else: reset_export = True # always reset if the file doesn't already exist exporter.by_domain(domain) total, skips = exporter.migrate(reset=reset_export, chunk_size=chunk_size) if skips: sys.exit(skips)
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100, limit_to_db=None): from corehq.apps.dump_reload.sql.dump import get_all_model_iterators_builders_for_domain if not self.domain: raise MigrationError("Must specify domain") if os.path.exists(get_export_filename(self.slug, self.domain)): raise MigrationError("{} exporter doesn't support resume. " "To re-run the export use 'reset'".format( self.slug)) migrator = self.migrator_class(self.slug, self.domain) with migrator: builders = get_all_model_iterators_builders_for_domain( self.model_class, self.domain, limit_to_db) for model_class, builder in builders: for iterator in builder.iterators(): for obj in iterator: migrator.process_object(obj) if migrator.total_blobs % chunk_size == 0: print("Processed {} {} objects".format( migrator.total_blobs, self.slug)) return migrator.total_blobs, 0
def handle(self, domain=None, reset=False, chunk_size=100, all=None, limit_to_db=None, **options): exporters = options.get('exporters') if not domain: raise CommandError(USAGE) if all: exporters = list(EXPORTERS) for exporter_slug in exporters: try: exporter_cls = EXPORTERS[exporter_slug] except KeyError: raise CommandError(USAGE) self.stdout.write("\nRunning exporter: {}\n{}".format( exporter_slug, '-' * 50)) export_filename = get_export_filename(exporter_slug, domain) if os.path.exists(export_filename): raise CommandError(f"Export file '{export_filename}' exists. " f"Remove the file and re-run the command.") exporter = exporter_cls(domain) total, skips = exporter.migrate(export_filename, chunk_size=chunk_size, limit_to_db=limit_to_db) if skips: sys.exit(skips)
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100): from corehq.apps.dump_reload.sql.dump import get_all_model_querysets_for_domain from corehq.apps.dump_reload.sql.dump import allow_form_processing_queries if not self.domain: raise MigrationError("Must specify domain") if os.path.exists(get_export_filename(self.slug, self.domain)): raise MigrationError( "{} exporter doesn't support resume. " "To re-run the export use 'reset'".format(self.slug) ) migrator = self.migrator_class(self.slug, self.domain) with migrator: with allow_form_processing_queries(): for model_class, queryset in get_all_model_querysets_for_domain(self.model_class, self.domain): for obj in queryset.iterator(): migrator.process_object(obj) if migrator.total_blobs % chunk_size == 0: print("Processed {} {} objects".format(migrator.total_blobs, self.slug)) return migrator.total_blobs, 0