def handle(self, domain=None, reset=False,
chunk_size=100, all=None, **options):
exporters = options.get('exporters')
if not domain:
raise CommandError(USAGE)
if all:
exporters = list(EXPORTERS)
for exporter_slug in exporters:
try:
exporter = EXPORTERS[exporter_slug]
except KeyError:
raise CommandError(USAGE)
self.stdout.write("\nRunning exporter: {}\n{}".format(exporter_slug, '-' * 50))
export_filename = get_export_filename(exporter_slug, domain)
if os.path.exists(export_filename):
reset_export = False
self.stderr.write(
"WARNING: export file for {} exists. "
"Resuming export progress. Delete file to reset progress.".format(exporter_slug)
)
else:
reset_export = True # always reset if the file doesn't already exist
exporter.by_domain(domain)
total, skips = exporter.migrate(reset=reset_export, chunk_size=chunk_size)
if skips:
sys.exit(skips)
def migrate(self,
filename=None,
reset=False,
max_retry=2,
chunk_size=100,
limit_to_db=None):
from corehq.apps.dump_reload.sql.dump import get_all_model_iterators_builders_for_domain
if not self.domain:
raise MigrationError("Must specify domain")
if os.path.exists(get_export_filename(self.slug, self.domain)):
raise MigrationError("{} exporter doesn't support resume. "
"To re-run the export use 'reset'".format(
self.slug))
migrator = self.migrator_class(self.slug, self.domain)
with migrator:
builders = get_all_model_iterators_builders_for_domain(
self.model_class, self.domain, limit_to_db)
for model_class, builder in builders:
for iterator in builder.iterators():
for obj in iterator:
migrator.process_object(obj)
if migrator.total_blobs % chunk_size == 0:
print("Processed {} {} objects".format(
migrator.total_blobs, self.slug))
return migrator.total_blobs, 0
def handle(self,
domain=None,
reset=False,
chunk_size=100,
all=None,
limit_to_db=None,
**options):
exporters = options.get('exporters')
if not domain:
raise CommandError(USAGE)
if all:
exporters = list(EXPORTERS)
for exporter_slug in exporters:
try:
exporter_cls = EXPORTERS[exporter_slug]
except KeyError:
raise CommandError(USAGE)
self.stdout.write("\nRunning exporter: {}\n{}".format(
exporter_slug, '-' * 50))
export_filename = get_export_filename(exporter_slug, domain)
if os.path.exists(export_filename):
raise CommandError(f"Export file '{export_filename}' exists. "
f"Remove the file and re-run the command.")
exporter = exporter_cls(domain)
total, skips = exporter.migrate(export_filename,
chunk_size=chunk_size,
limit_to_db=limit_to_db)
if skips:
sys.exit(skips)
def migrate(self, filename=None, reset=False, max_retry=2, chunk_size=100):
from corehq.apps.dump_reload.sql.dump import get_all_model_querysets_for_domain
from corehq.apps.dump_reload.sql.dump import allow_form_processing_queries
if not self.domain:
raise MigrationError("Must specify domain")
if os.path.exists(get_export_filename(self.slug, self.domain)):
raise MigrationError(
"{} exporter doesn't support resume. "
"To re-run the export use 'reset'".format(self.slug)
)
migrator = self.migrator_class(self.slug, self.domain)
with migrator:
with allow_form_processing_queries():
for model_class, queryset in get_all_model_querysets_for_domain(self.model_class, self.domain):
for obj in queryset.iterator():
migrator.process_object(obj)
if migrator.total_blobs % chunk_size == 0:
print("Processed {} {} objects".format(migrator.total_blobs, self.slug))
return migrator.total_blobs, 0
