def copy_docs(self, sourcedb, domain, simulate, startkey=None, endkey=None, doc_ids=None, type=None, since=None, exclude_types=None, postgres_db=None, exclude_attachments=False): if not doc_ids: doc_ids = [result["id"] for result in sourcedb.view("domain/docs", startkey=startkey, endkey=endkey, reduce=False)] total = len(doc_ids) count = 0 msg = "Found %s matching documents in domain: %s" % (total, domain) msg += " of type: %s" % (type) if type else "" msg += " since: %s" % (since) if since else "" print msg err_log = self._get_err_log() queue = Queue(150) for i in range(NUM_PROCESSES): Worker(queue, sourcedb, self.targetdb, exclude_types, total, simulate, err_log, exclude_attachments).start() for doc in iter_docs(sourcedb, doc_ids, chunksize=100): count += 1 queue.put((doc, count)) # shutdown workers for i in range(NUM_PROCESSES): queue.put(None) err_log.close() if os.stat(err_log.name)[6] == 0: os.remove(err_log.name) else: print 'Failed document IDs written to %s' % err_log.name if postgres_db: copy_postgres_data_for_docs(postgres_db, doc_ids=doc_ids, simulate=simulate)
def copy_docs(self, sourcedb, domain, simulate, startkey=None, endkey=None, doc_ids=None, doc_type=None, since=None, exclude_types=None, postgres_db=None, exclude_attachments=False): if not doc_ids: doc_ids = [ result["id"] for result in sourcedb.view("by_domain_doc_type_date/view", startkey=startkey, endkey=endkey, reduce=False) ] total = len(doc_ids) count = 0 msg = "Found %s matching documents in domain: %s" % (total, domain) msg += " of type: %s" % (doc_type) if doc_type else "" msg += " since: %s" % (since) if since else "" print msg err_log = self._get_err_log() if self.run_multi_process: queue = Queue(150) for i in range(NUM_PROCESSES): Worker(queue, sourcedb, self.targetdb, exclude_types, total, simulate, err_log, exclude_attachments).start() for doc in iter_docs(sourcedb, doc_ids, chunksize=100): count += 1 queue.put((doc, count)) # shutdown workers for i in range(NUM_PROCESSES): queue.put(None) else: for doc in iter_docs(sourcedb, doc_ids, chunksize=100): target = self.targetdb.get_db_for_doc_type(doc['doc_type']) count += 1 copy_doc(doc, count, sourcedb, target, exclude_types, total, simulate, exclude_attachments) err_log.close() if os.stat(err_log.name)[6] == 0: os.remove(err_log.name) else: print 'Failed document IDs written to %s' % err_log.name if postgres_db: copy_postgres_data_for_docs(postgres_db, doc_ids=doc_ids, simulate=simulate)
def handle(self, *args, **options): if len(args) < 2: raise CommandError('Usage is copy_case, %s' % self.args) source_couch = CouchConfig(args[0]) case_id = args[1] doc_ids = [case_id] domain = args[2] if len(args) > 2 else None if should_use_sql_backend(domain): raise CommandError('This command only works for couch-based domains.') def _migrate_case(case_id): print 'getting case %s' % case_id case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id)) original_domain = case.domain if domain is not None: case.domain = domain case.save(force_update=True) return case, original_domain case, orig_domain = _migrate_case(case_id) print 'copying %s parent cases' % len(case.indices) for index in case.indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) # hack, set the domain back to make sure we get the reverse indices correctly case.domain = orig_domain with OverrideDB(CommCareCase, source_couch.get_db_for_class(CommCareCase)): child_indices = get_reverse_indices(case) print 'copying %s child cases' % len(child_indices) for index in child_indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) print 'copying %s xforms' % len(case.xform_ids) def form_wrapper(row): doc = row['doc'] doc.pop('_attachments', None) doc.pop('external_blobs', None) return XFormInstance.wrap(doc) xforms = source_couch.get_db_for_class(XFormInstance).all_docs( keys=case.xform_ids, include_docs=True, wrapper=form_wrapper, ).all() for form in xforms: if domain is not None: form.domain = domain form.save(force_update=True) print 'saved %s' % form._id doc_ids.append(form._id) if options['postgres_db']: copy_postgres_data_for_docs(options['postgres_db'], doc_ids)
def handle(self, *args, **options): if len(args) < 2: raise CommandError("Usage is copy_case, %s" % self.args) source_couch = CouchConfig(args[0]) case_id = args[1] doc_ids = [case_id] domain = args[2] if len(args) > 2 else None def _migrate_case(case_id): print "getting case %s" % case_id case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id)) original_domain = case.domain if domain is not None: case.domain = domain case.save(force_update=True) return case, original_domain case, orig_domain = _migrate_case(case_id) print "copying %s parent cases" % len(case.indices) for index in case.indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) # hack, set the domain back to make sure we get the reverse indices correctly case.domain = orig_domain with OverrideDB(CommCareCase, source_couch.get_db_for_class(CommCareCase)): child_indices = get_reverse_indices(case) print "copying %s child cases" % len(child_indices) for index in child_indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) print "copying %s xforms" % len(case.xform_ids) def form_wrapper(row): doc = row["doc"] doc.pop("_attachments", None) return XFormInstance.wrap(doc) xforms = ( source_couch.get_db_for_class(XFormInstance) .all_docs(keys=case.xform_ids, include_docs=True, wrapper=form_wrapper) .all() ) for form in xforms: if domain is not None: form.domain = domain form.save(force_update=True) print "saved %s" % form._id doc_ids.append(form._id) if options["postgres_db"]: copy_postgres_data_for_docs(options["postgres_db"], doc_ids)
def handle(self, *args, **options): if len(args) < 2: raise CommandError('Usage is copy_case, %s' % self.args) sourcedb = Database(args[0]) case_id = args[1] doc_ids = [case_id] domain = args[2] if len(args) > 2 else None def _migrate_case(case_id): print 'getting case %s' % case_id case = CommCareCase.wrap(sourcedb.get(case_id)) original_domain = case.domain if domain is not None: case.domain = domain case.save(force_update=True) return case, original_domain case, orig_domain = _migrate_case(case_id) print 'copying %s parent cases' % len(case.indices) for index in case.indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) # hack, set the domain back to make sure we get the reverse indices correctly case.domain = orig_domain child_indices = reverse_indices(sourcedb, case) print 'copying %s child cases' % len(child_indices) for index in child_indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) print 'copying %s xforms' % len(case.xform_ids) def form_wrapper(row): doc = row['doc'] doc.pop('_attachments', None) return XFormInstance.wrap(doc) xforms = sourcedb.all_docs( keys=case.xform_ids, include_docs=True, wrapper=form_wrapper, ).all() for form in xforms: if domain is not None: form.domain = domain form.save(force_update=True) print 'saved %s' % form._id doc_ids.append(form._id) if options['postgres_db']: copy_postgres_data_for_docs(options['postgres_db'], doc_ids)
class Command(LabelCommand): help = "Copy a case and all related forms" args = '<sourcedb> <case_id> <domain>' option_list = LabelCommand.option_list + (make_option( '--postgres-db', action='store', dest='postgres_db', default='', help= "Name of postgres database to pull additional data from. This should map to a " "key in settings.DATABASES. If not specified no additional postgres data will be " "copied. This is currently used to pull CommCare Supply models."), ) def handle(self, *args, **options): if len(args) < 2: raise CommandError('Usage is copy_case, %s' % self.args) sourcedb = Database(args[0]) case_id = args[1] doc_ids = [case_id] domain = args[2] if len(args) > 2 else None def _migrate_case(case_id): print 'getting case %s' % case_id case = CommCareCase.wrap(sourcedb.get(case_id)) original_domain = case.domain if domain is not None: case.domain = domain case.save(force_update=True) return case, original_domain case, orig_domain = _migrate_case(case_id) print 'copying %s parent cases' % len(case.indices) for index in case.indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) # hack, set the domain back to make sure we get the reverse indices correctly case.domain = orig_domain with OverrideDB(CommCareCase, sourcedb): child_indices = get_reverse_indices(case) print 'copying %s child cases' % len(child_indices) for index in child_indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) print 'copying %s xforms' % len(case.xform_ids) def form_wrapper(row): doc = row['doc'] doc.pop('_attachments', None) return XFormInstance.wrap(doc) xforms = sourcedb.all_docs( keys=case.xform_ids, include_docs=True, wrapper=form_wrapper, ).all() for form in xforms: if domain is not None: form.domain = domain form.save(force_update=True) print 'saved %s' % form._id doc_ids.append(form._id) if options['postgres_db']: copy_postgres_data_for_docs(options['postgres_db'], doc_ids)
class Command(BaseCommand): help = "Copy a case and all related forms" def add_arguments(self, parser): parser.add_argument('sourcedb', ) parser.add_argument('case_id', ) parser.add_argument( 'domain', nargs='?', ) parser.add_argument( '--postgres-db', action='store', dest='postgres_db', default='', help= "Name of postgres database to pull additional data from. This should map to a " "key in settings.DATABASES. If not specified no additional postgres data will be " "copied. This is currently used to pull CommCare Supply models.", ) def handle(self, sourcedb, case_id, domain, **options): # FIXME broken b/c https://github.com/dimagi/commcare-hq/pull/15896 source_couch = CouchConfig(sourcedb) doc_ids = [case_id] if should_use_sql_backend(domain): raise CommandError( 'This command only works for couch-based domains.') def _migrate_case(case_id): print('getting case %s' % case_id) case = CommCareCase.wrap( source_couch.get_db_for_class(CommCareCase).get(case_id)) original_domain = case.domain if domain is not None: case.domain = domain case.save(force_update=True) return case, original_domain case, orig_domain = _migrate_case(case_id) print('copying %s parent cases' % len(case.indices)) for index in case.indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) # hack, set the domain back to make sure we get the reverse indices correctly case.domain = orig_domain with OverrideDB(CommCareCase, source_couch.get_db_for_class(CommCareCase)): child_indices = get_reverse_indices(case) print('copying %s child cases' % len(child_indices)) for index in child_indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) print('copying %s xforms' % len(case.xform_ids)) def form_wrapper(row): doc = row['doc'] doc.pop('_attachments', None) doc.pop('external_blobs', None) return XFormInstance.wrap(doc) xforms = source_couch.get_db_for_class(XFormInstance).all_docs( keys=case.xform_ids, include_docs=True, wrapper=form_wrapper, ).all() for form in xforms: if domain is not None: form.domain = domain form.save(force_update=True) print('saved %s' % form._id) doc_ids.append(form._id) if options['postgres_db']: copy_postgres_data_for_docs(options['postgres_db'], doc_ids)