def copy_docs(self, sourcedb, domain, simulate, startkey=None, endkey=None, doc_ids=None,
type=None, since=None, exclude_types=None, postgres_db=None, exclude_attachments=False):
if not doc_ids:
doc_ids = [result["id"] for result in sourcedb.view("domain/docs", startkey=startkey,
endkey=endkey, reduce=False)]
total = len(doc_ids)
count = 0
msg = "Found %s matching documents in domain: %s" % (total, domain)
msg += " of type: %s" % (type) if type else ""
msg += " since: %s" % (since) if since else ""
print msg
err_log = self._get_err_log()
queue = Queue(150)
for i in range(NUM_PROCESSES):
Worker(queue, sourcedb, self.targetdb, exclude_types, total, simulate, err_log, exclude_attachments).start()
for doc in iter_docs(sourcedb, doc_ids, chunksize=100):
count += 1
queue.put((doc, count))
# shutdown workers
for i in range(NUM_PROCESSES):
queue.put(None)
err_log.close()
if os.stat(err_log.name)[6] == 0:
os.remove(err_log.name)
else:
print 'Failed document IDs written to %s' % err_log.name
if postgres_db:
copy_postgres_data_for_docs(postgres_db, doc_ids=doc_ids, simulate=simulate)
def copy_docs(self,
sourcedb,
domain,
simulate,
startkey=None,
endkey=None,
doc_ids=None,
doc_type=None,
since=None,
exclude_types=None,
postgres_db=None,
exclude_attachments=False):
if not doc_ids:
doc_ids = [
result["id"]
for result in sourcedb.view("by_domain_doc_type_date/view",
startkey=startkey,
endkey=endkey,
reduce=False)
]
total = len(doc_ids)
count = 0
msg = "Found %s matching documents in domain: %s" % (total, domain)
msg += " of type: %s" % (doc_type) if doc_type else ""
msg += " since: %s" % (since) if since else ""
print msg
err_log = self._get_err_log()
if self.run_multi_process:
queue = Queue(150)
for i in range(NUM_PROCESSES):
Worker(queue, sourcedb, self.targetdb, exclude_types, total,
simulate, err_log, exclude_attachments).start()
for doc in iter_docs(sourcedb, doc_ids, chunksize=100):
count += 1
queue.put((doc, count))
# shutdown workers
for i in range(NUM_PROCESSES):
queue.put(None)
else:
for doc in iter_docs(sourcedb, doc_ids, chunksize=100):
target = self.targetdb.get_db_for_doc_type(doc['doc_type'])
count += 1
copy_doc(doc, count, sourcedb, target, exclude_types, total,
simulate, exclude_attachments)
err_log.close()
if os.stat(err_log.name)[6] == 0:
os.remove(err_log.name)
else:
print 'Failed document IDs written to %s' % err_log.name
if postgres_db:
copy_postgres_data_for_docs(postgres_db,
doc_ids=doc_ids,
simulate=simulate)
def handle(self, *args, **options):
if len(args) < 2:
raise CommandError('Usage is copy_case, %s' % self.args)
source_couch = CouchConfig(args[0])
case_id = args[1]
doc_ids = [case_id]
domain = args[2] if len(args) > 2 else None
if should_use_sql_backend(domain):
raise CommandError('This command only works for couch-based domains.')
def _migrate_case(case_id):
print 'getting case %s' % case_id
case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id))
original_domain = case.domain
if domain is not None:
case.domain = domain
case.save(force_update=True)
return case, original_domain
case, orig_domain = _migrate_case(case_id)
print 'copying %s parent cases' % len(case.indices)
for index in case.indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
# hack, set the domain back to make sure we get the reverse indices correctly
case.domain = orig_domain
with OverrideDB(CommCareCase, source_couch.get_db_for_class(CommCareCase)):
child_indices = get_reverse_indices(case)
print 'copying %s child cases' % len(child_indices)
for index in child_indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
print 'copying %s xforms' % len(case.xform_ids)
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
doc.pop('external_blobs', None)
return XFormInstance.wrap(doc)
xforms = source_couch.get_db_for_class(XFormInstance).all_docs(
keys=case.xform_ids,
include_docs=True,
wrapper=form_wrapper,
).all()
for form in xforms:
if domain is not None:
form.domain = domain
form.save(force_update=True)
print 'saved %s' % form._id
doc_ids.append(form._id)
if options['postgres_db']:
copy_postgres_data_for_docs(options['postgres_db'], doc_ids)
def handle(self, *args, **options):
if len(args) < 2:
raise CommandError("Usage is copy_case, %s" % self.args)
source_couch = CouchConfig(args[0])
case_id = args[1]
doc_ids = [case_id]
domain = args[2] if len(args) > 2 else None
def _migrate_case(case_id):
print "getting case %s" % case_id
case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id))
original_domain = case.domain
if domain is not None:
case.domain = domain
case.save(force_update=True)
return case, original_domain
case, orig_domain = _migrate_case(case_id)
print "copying %s parent cases" % len(case.indices)
for index in case.indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
# hack, set the domain back to make sure we get the reverse indices correctly
case.domain = orig_domain
with OverrideDB(CommCareCase, source_couch.get_db_for_class(CommCareCase)):
child_indices = get_reverse_indices(case)
print "copying %s child cases" % len(child_indices)
for index in child_indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
print "copying %s xforms" % len(case.xform_ids)
def form_wrapper(row):
doc = row["doc"]
doc.pop("_attachments", None)
return XFormInstance.wrap(doc)
xforms = (
source_couch.get_db_for_class(XFormInstance)
.all_docs(keys=case.xform_ids, include_docs=True, wrapper=form_wrapper)
.all()
)
for form in xforms:
if domain is not None:
form.domain = domain
form.save(force_update=True)
print "saved %s" % form._id
doc_ids.append(form._id)
if options["postgres_db"]:
copy_postgres_data_for_docs(options["postgres_db"], doc_ids)
def handle(self, *args, **options):
if len(args) < 2:
raise CommandError('Usage is copy_case, %s' % self.args)
sourcedb = Database(args[0])
case_id = args[1]
doc_ids = [case_id]
domain = args[2] if len(args) > 2 else None
def _migrate_case(case_id):
print 'getting case %s' % case_id
case = CommCareCase.wrap(sourcedb.get(case_id))
original_domain = case.domain
if domain is not None:
case.domain = domain
case.save(force_update=True)
return case, original_domain
case, orig_domain = _migrate_case(case_id)
print 'copying %s parent cases' % len(case.indices)
for index in case.indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
# hack, set the domain back to make sure we get the reverse indices correctly
case.domain = orig_domain
child_indices = reverse_indices(sourcedb, case)
print 'copying %s child cases' % len(child_indices)
for index in child_indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
print 'copying %s xforms' % len(case.xform_ids)
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
return XFormInstance.wrap(doc)
xforms = sourcedb.all_docs(
keys=case.xform_ids,
include_docs=True,
wrapper=form_wrapper,
).all()
for form in xforms:
if domain is not None:
form.domain = domain
form.save(force_update=True)
print 'saved %s' % form._id
doc_ids.append(form._id)
if options['postgres_db']:
copy_postgres_data_for_docs(options['postgres_db'], doc_ids)
class Command(LabelCommand):
help = "Copy a case and all related forms"
args = '<sourcedb> <case_id> <domain>'
option_list = LabelCommand.option_list + (make_option(
'--postgres-db',
action='store',
dest='postgres_db',
default='',
help=
"Name of postgres database to pull additional data from. This should map to a "
"key in settings.DATABASES. If not specified no additional postgres data will be "
"copied. This is currently used to pull CommCare Supply models."), )
def handle(self, *args, **options):
if len(args) < 2:
raise CommandError('Usage is copy_case, %s' % self.args)
sourcedb = Database(args[0])
case_id = args[1]
doc_ids = [case_id]
domain = args[2] if len(args) > 2 else None
def _migrate_case(case_id):
print 'getting case %s' % case_id
case = CommCareCase.wrap(sourcedb.get(case_id))
original_domain = case.domain
if domain is not None:
case.domain = domain
case.save(force_update=True)
return case, original_domain
case, orig_domain = _migrate_case(case_id)
print 'copying %s parent cases' % len(case.indices)
for index in case.indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
# hack, set the domain back to make sure we get the reverse indices correctly
case.domain = orig_domain
with OverrideDB(CommCareCase, sourcedb):
child_indices = get_reverse_indices(case)
print 'copying %s child cases' % len(child_indices)
for index in child_indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
print 'copying %s xforms' % len(case.xform_ids)
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
return XFormInstance.wrap(doc)
xforms = sourcedb.all_docs(
keys=case.xform_ids,
include_docs=True,
wrapper=form_wrapper,
).all()
for form in xforms:
if domain is not None:
form.domain = domain
form.save(force_update=True)
print 'saved %s' % form._id
doc_ids.append(form._id)
if options['postgres_db']:
copy_postgres_data_for_docs(options['postgres_db'], doc_ids)
class Command(BaseCommand):
help = "Copy a case and all related forms"
def add_arguments(self, parser):
parser.add_argument('sourcedb', )
parser.add_argument('case_id', )
parser.add_argument(
'domain',
nargs='?',
)
parser.add_argument(
'--postgres-db',
action='store',
dest='postgres_db',
default='',
help=
"Name of postgres database to pull additional data from. This should map to a "
"key in settings.DATABASES. If not specified no additional postgres data will be "
"copied. This is currently used to pull CommCare Supply models.",
)
def handle(self, sourcedb, case_id, domain, **options):
# FIXME broken b/c https://github.com/dimagi/commcare-hq/pull/15896
source_couch = CouchConfig(sourcedb)
doc_ids = [case_id]
if should_use_sql_backend(domain):
raise CommandError(
'This command only works for couch-based domains.')
def _migrate_case(case_id):
print('getting case %s' % case_id)
case = CommCareCase.wrap(
source_couch.get_db_for_class(CommCareCase).get(case_id))
original_domain = case.domain
if domain is not None:
case.domain = domain
case.save(force_update=True)
return case, original_domain
case, orig_domain = _migrate_case(case_id)
print('copying %s parent cases' % len(case.indices))
for index in case.indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
# hack, set the domain back to make sure we get the reverse indices correctly
case.domain = orig_domain
with OverrideDB(CommCareCase,
source_couch.get_db_for_class(CommCareCase)):
child_indices = get_reverse_indices(case)
print('copying %s child cases' % len(child_indices))
for index in child_indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
print('copying %s xforms' % len(case.xform_ids))
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
doc.pop('external_blobs', None)
return XFormInstance.wrap(doc)
xforms = source_couch.get_db_for_class(XFormInstance).all_docs(
keys=case.xform_ids,
include_docs=True,
wrapper=form_wrapper,
).all()
for form in xforms:
if domain is not None:
form.domain = domain
form.save(force_update=True)
print('saved %s' % form._id)
doc_ids.append(form._id)
if options['postgres_db']:
copy_postgres_data_for_docs(options['postgres_db'], doc_ids)
