def test_get_db_for_doc_type(self): config = CouchConfig(config=self._config) self.assertEqual( config.get_db_for_doc_type('CommCareCase').uri, self.remote_db_uri) self.assertEqual( config.get_db_for_doc_type('CommCareUser').uri, '{}__users'.format(self.remote_db_uri))
def handle(self, *args, **options): if len(args) < 2: raise CommandError('Usage is copy_case, %s' % self.args) source_couch = CouchConfig(args[0]) case_id = args[1] doc_ids = [case_id] domain = args[2] if len(args) > 2 else None if should_use_sql_backend(domain): raise CommandError('This command only works for couch-based domains.') def _migrate_case(case_id): print 'getting case %s' % case_id case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id)) original_domain = case.domain if domain is not None: case.domain = domain case.save(force_update=True) return case, original_domain case, orig_domain = _migrate_case(case_id) print 'copying %s parent cases' % len(case.indices) for index in case.indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) # hack, set the domain back to make sure we get the reverse indices correctly case.domain = orig_domain with OverrideDB(CommCareCase, source_couch.get_db_for_class(CommCareCase)): child_indices = get_reverse_indices(case) print 'copying %s child cases' % len(child_indices) for index in child_indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) print 'copying %s xforms' % len(case.xform_ids) def form_wrapper(row): doc = row['doc'] doc.pop('_attachments', None) doc.pop('external_blobs', None) return XFormInstance.wrap(doc) xforms = source_couch.get_db_for_class(XFormInstance).all_docs( keys=case.xform_ids, include_docs=True, wrapper=form_wrapper, ).all() for form in xforms: if domain is not None: form.domain = domain form.save(force_update=True) print 'saved %s' % form._id doc_ids.append(form._id) if options['postgres_db']: copy_postgres_data_for_docs(options['postgres_db'], doc_ids)
def _get_designs(self): # Instantiate here to make sure that it's instantiated after the dbs settings # are patched for tests couch_config = CouchConfig() db = Database( couch_config.get_db_uri_for_app_label(self.app_config.label), create=True) return [ DesignInfo(app_label=self.app_config.label, db=db, design_path=self.dir) ]
def handle(self, *args, **options): if len(args) < 2: raise CommandError("Usage is copy_case, %s" % self.args) source_couch = CouchConfig(args[0]) case_id = args[1] doc_ids = [case_id] domain = args[2] if len(args) > 2 else None def _migrate_case(case_id): print "getting case %s" % case_id case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id)) original_domain = case.domain if domain is not None: case.domain = domain case.save(force_update=True) return case, original_domain case, orig_domain = _migrate_case(case_id) print "copying %s parent cases" % len(case.indices) for index in case.indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) # hack, set the domain back to make sure we get the reverse indices correctly case.domain = orig_domain with OverrideDB(CommCareCase, source_couch.get_db_for_class(CommCareCase)): child_indices = get_reverse_indices(case) print "copying %s child cases" % len(child_indices) for index in child_indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) print "copying %s xforms" % len(case.xform_ids) def form_wrapper(row): doc = row["doc"] doc.pop("_attachments", None) return XFormInstance.wrap(doc) xforms = ( source_couch.get_db_for_class(XFormInstance) .all_docs(keys=case.xform_ids, include_docs=True, wrapper=form_wrapper) .all() ) for form in xforms: if domain is not None: form.domain = domain form.save(force_update=True) print "saved %s" % form._id doc_ids.append(form._id) if options["postgres_db"]: copy_postgres_data_for_docs(options["postgres_db"], doc_ids)
def test_all_db_uris_by_slug(self): config = CouchConfig(config=self._config) self.assertDictContainsSubset( { None: self.remote_db_uri, 'users': '{}__users'.format(self.remote_db_uri), 'fixtures': '{}__fixtures'.format(self.remote_db_uri), 'meta': '{}__meta'.format(self.remote_db_uri), }, config.all_db_uris_by_slug)
def test_all_db_uris_by_slug(self): config = CouchConfig(config=self._config) for key, value in { None: self.remote_db_uri, 'users': '{}__users'.format(self.remote_db_uri), 'fixtures': '{}__fixtures'.format(self.remote_db_uri), 'meta': '{}__meta'.format(self.remote_db_uri), }.items(): self.assertEqual(config.all_db_uris_by_slug[key], value, f"key: {key!r}")
def _get_couch_database_configs_from_string(self, db_string): sourcedb_parse_result = urlparse(db_string) return CouchConfig({ 'default': { 'COUCH_HTTPS': sourcedb_parse_result.scheme == 'https', 'COUCH_SERVER_ROOT': sourcedb_parse_result.hostname, 'COUCH_USERNAME': sourcedb_parse_result.username, 'COUCH_PASSWORD': sourcedb_parse_result.password, 'COUCH_DATABASE_NAME': sourcedb_parse_result.path.lstrip('/') } })
def handle(self, sourcedb, case_id, domain, **options): # FIXME broken b/c https://github.com/dimagi/commcare-hq/pull/15896 source_couch = CouchConfig(sourcedb) doc_ids = [case_id] if should_use_sql_backend(domain): raise CommandError('This command only works for couch-based domains.') def _migrate_case(case_id): print('getting case %s' % case_id) case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id)) original_domain = case.domain if domain is not None: case.domain = domain case.save(force_update=True) return case, original_domain
def handle(self, *args, **options): if len(args) < 2: raise CommandError('Usage is copy_case, %s' % self.args) source_couch = CouchConfig(args[0]) case_id = args[1] doc_ids = [case_id] domain = args[2] if len(args) > 2 else None if should_use_sql_backend(domain): raise CommandError('This command only works for couch-based domains.') def _migrate_case(case_id): print 'getting case %s' % case_id case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id)) original_domain = case.domain if domain is not None: case.domain = domain case.save(force_update=True) return case, original_domain
def test_get_db_for_db_name(self): config = CouchConfig(config=self._config) self.assertEqual(self.remote_db_uri, config.get_db_for_db_name('test_cchq').uri) self.assertEqual('{}__users'.format(self.remote_db_uri), config.get_db_for_db_name('test_cchq__users').uri)
def test_get_db_for_db_name_not_found(self): config = CouchConfig(config=self._config) with self.assertRaises(DatabaseNotFound): config.get_db_for_db_name('missing')
def handle(self, *args, **options): if len(args) not in [2, 3]: raise CommandError('Usage is copy_domain %s' % self.args) self.exclude_dbs = ( # these have data we don't want to copy 'receiverwrapper', 'couchlog', 'auditcare', 'fluff-bihar', 'fluff-opm', 'fluff-mc', 'fluff-cvsu', 'mvp-indicators', 'm4change', # todo: missing domain/docs, but probably want to add back 'meta', ) self.source_couch = source_couch = CouchConfig(args[0]) domain = args[1].strip() simulate = options['simulate'] exclude_attachments = options['exclude_attachments'] self.run_multi_process = options['run_multi_process'] since = json_format_date(iso_string_to_date( options['since'])) if options['since'] else None if options['list_types']: for sourcedb_name, sourcedb in self.iter_source_dbs(): self.list_types(sourcedb, domain, since) sys.exit(0) if simulate: print "\nSimulated run, no data will be copied.\n" if options['postgres_db'] and options['postgres_password']: settings.DATABASES[options['postgres_db']]['PASSWORD'] = options[ 'postgres_password'] self.targetdb = CouchConfig( args[2]) if len(args) == 3 else CouchConfig() try: domain_doc = Domain.get_by_name(domain) except ResourceNotFound: domain_doc = None if domain_doc is None: self.copy_domain(source_couch, domain) if options['doc_types']: doc_types = options['doc_types'].split(',') for doc_type in doc_types: sourcedb = source_couch.get_db_for_doc_type(doc_type) startkey = [ x for x in [domain, doc_type, since] if x is not None ] endkey = [x for x in [domain, doc_type, {}] if x is not None] self.copy_docs(sourcedb, domain, simulate, startkey, endkey, doc_type=doc_type, since=since, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments) elif options['id_file']: path = options['id_file'] if not os.path.isfile(path): print "Path '%s' does not exist or is not a file" % path sys.exit(1) with open(path) as input: doc_ids = [line.rstrip('\n') for line in input] if not doc_ids: print "Path '%s' does not contain any document ID's" % path sys.exit(1) for sourcedb_name, sourcedb in self.iter_source_dbs(): self.copy_docs(sourcedb, domain, simulate, doc_ids=doc_ids, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments) else: startkey = [domain] endkey = [domain, {}] exclude_types = DEFAULT_EXCLUDE_TYPES + options[ 'doc_types_exclude'].split(',') for sourcedb_name, sourcedb in self.iter_source_dbs(): self.copy_docs(sourcedb, domain, simulate, startkey, endkey, exclude_types=exclude_types, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments)
class Command(BaseCommand): help = "Copies the contents of a domain to another database. " \ "If tagetdb is not specified, the target is the database " \ "specified by COUCH_DATABASE in your settings." args = '<sourcedb> <domain> [<targetdb>]' option_list = BaseCommand.option_list + ( make_option('--include', action='store', dest='doc_types', default='', help='Comma-separated list of Document Types to copy'), make_option( '--exclude', action='store', dest='doc_types_exclude', default='', help='Comma-separated list of Document Types to NOT copy.'), make_option( '--exclude-attachments', action='store_true', dest='exclude_attachments', default=False, help="Don't copy document attachments, just the docs themselves."), make_option( '--since', action='store', dest='since', default='', help= 'Only copy documents newer than this date. Format: yyyy-MM-dd. Only ' ), make_option( '--list-types', action='store_true', dest='list_types', default=False, help= 'Don\'t copy anything, just list all the available document types.' ), make_option('--simulate', action='store_true', dest='simulate', default=False, help='Don\'t copy anything, print what would be copied.'), make_option( '--id-file', action='store', dest='id_file', default='', help= "File containing one document ID per line. Only docs with these ID's will be copied" ), make_option( '--postgres-db', action='store', dest='postgres_db', default='', help= "Name of postgres database to pull additional data from. This should map to a " "key in settings.DATABASES. If not specified no additional postgres data will be " "copied. This is currently used to pull CommCare Supply models."), make_option( '--postgres-password', action='store', dest='postgres_password', default='', help= "Password for postgres database to pull additional data from. If not specified will " "default to the value in settings.DATABASES"), make_option( '--dont-run-multi-process', action='store_false', dest='run_multi_process', default=True, help= "If set to true this spawn multiple processes which should speed up the time taken to " "copy. This must be false if running in a supervised process")) def iter_source_dbs(self): for sourcedb_name, sourcedb in self.source_couch.all_dbs_by_slug.items( ): if sourcedb_name not in self.exclude_dbs: print "In {} db".format(sourcedb_name or "the main") yield sourcedb_name, sourcedb def handle(self, *args, **options): if len(args) not in [2, 3]: raise CommandError('Usage is copy_domain %s' % self.args) self.exclude_dbs = ( # these have data we don't want to copy 'receiverwrapper', 'couchlog', 'auditcare', 'fluff-bihar', 'fluff-opm', 'fluff-mc', 'fluff-cvsu', 'mvp-indicators', 'm4change', # todo: missing domain/docs, but probably want to add back 'meta', ) self.source_couch = source_couch = CouchConfig(args[0]) domain = args[1].strip() simulate = options['simulate'] exclude_attachments = options['exclude_attachments'] self.run_multi_process = options['run_multi_process'] since = json_format_date(iso_string_to_date( options['since'])) if options['since'] else None if options['list_types']: for sourcedb_name, sourcedb in self.iter_source_dbs(): self.list_types(sourcedb, domain, since) sys.exit(0) if simulate: print "\nSimulated run, no data will be copied.\n" if options['postgres_db'] and options['postgres_password']: settings.DATABASES[options['postgres_db']]['PASSWORD'] = options[ 'postgres_password'] self.targetdb = CouchConfig( args[2]) if len(args) == 3 else CouchConfig() try: domain_doc = Domain.get_by_name(domain) except ResourceNotFound: domain_doc = None if domain_doc is None: self.copy_domain(source_couch, domain) if options['doc_types']: doc_types = options['doc_types'].split(',') for doc_type in doc_types: sourcedb = source_couch.get_db_for_doc_type(doc_type) startkey = [ x for x in [domain, doc_type, since] if x is not None ] endkey = [x for x in [domain, doc_type, {}] if x is not None] self.copy_docs(sourcedb, domain, simulate, startkey, endkey, doc_type=doc_type, since=since, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments) elif options['id_file']: path = options['id_file'] if not os.path.isfile(path): print "Path '%s' does not exist or is not a file" % path sys.exit(1) with open(path) as input: doc_ids = [line.rstrip('\n') for line in input] if not doc_ids: print "Path '%s' does not contain any document ID's" % path sys.exit(1) for sourcedb_name, sourcedb in self.iter_source_dbs(): self.copy_docs(sourcedb, domain, simulate, doc_ids=doc_ids, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments) else: startkey = [domain] endkey = [domain, {}] exclude_types = DEFAULT_EXCLUDE_TYPES + options[ 'doc_types_exclude'].split(',') for sourcedb_name, sourcedb in self.iter_source_dbs(): self.copy_docs(sourcedb, domain, simulate, startkey, endkey, exclude_types=exclude_types, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments) def list_types(self, sourcedb, domain, since): doc_types = sourcedb.view("by_domain_doc_type_date/view", startkey=[domain], endkey=[domain, {}], reduce=True, group=True, group_level=2) doc_count = dict([(row['key'][1], row['value']) for row in doc_types]) if since: for doc_type in sorted(doc_count.iterkeys()): num_since = sourcedb.view("by_domain_doc_type_date/view", startkey=[domain, doc_type, since], endkey=[domain, doc_type, {}], reduce=True).all() num = num_since[0]['value'] if num_since else 0 print "{0:<30}- {1:<6} total {2}".format( doc_type, num, doc_count[doc_type]) else: for doc_type in sorted(doc_count.iterkeys()): print "{0:<30}- {1}".format(doc_type, doc_count[doc_type]) def copy_docs(self, sourcedb, domain, simulate, startkey=None, endkey=None, doc_ids=None, doc_type=None, since=None, exclude_types=None, postgres_db=None, exclude_attachments=False): if not doc_ids: doc_ids = [ result["id"] for result in sourcedb.view("by_domain_doc_type_date/view", startkey=startkey, endkey=endkey, reduce=False) ] total = len(doc_ids) count = 0 msg = "Found %s matching documents in domain: %s" % (total, domain) msg += " of type: %s" % (doc_type) if doc_type else "" msg += " since: %s" % (since) if since else "" print msg err_log = self._get_err_log() if self.run_multi_process: queue = Queue(150) for i in range(NUM_PROCESSES): Worker(queue, sourcedb, self.targetdb, exclude_types, total, simulate, err_log, exclude_attachments).start() for doc in iter_docs(sourcedb, doc_ids, chunksize=100): count += 1 queue.put((doc, count)) # shutdown workers for i in range(NUM_PROCESSES): queue.put(None) else: for doc in iter_docs(sourcedb, doc_ids, chunksize=100): target = self.targetdb.get_db_for_doc_type(doc['doc_type']) count += 1 copy_doc(doc, count, sourcedb, target, exclude_types, total, simulate, exclude_attachments) err_log.close() if os.stat(err_log.name)[6] == 0: os.remove(err_log.name) else: print 'Failed document IDs written to %s' % err_log.name if postgres_db: copy_postgres_data_for_docs(postgres_db, doc_ids=doc_ids, simulate=simulate) def copy_domain(self, source_couch, domain): print "Copying domain doc" sourcedb = source_couch.get_db_for_class(Domain) result = sourcedb.view("domain/domains", key=domain, reduce=False, include_docs=True).first() if result and 'doc' in result: domain_doc = Domain.wrap(result['doc']) dt = DocumentTransform(domain_doc._obj, sourcedb) save(dt, self.targetdb.get_db_for_doc_type(domain_doc['doc_type'])) else: print "Domain doc not found for domain %s." % domain def _get_err_log(self): name = 'copy_domain.err.%s' for i in range(1000): # arbitrarily large number candidate = name % i if not os.path.isfile(candidate): return open(candidate, 'a', buffering=1)
def test_remote_db_uri(self): config = CouchConfig(config=self._config) self.assertEqual(config.db_uri, self.remote_db_uri)
def test_default_db_uri(self): config = CouchConfig() self.assertEqual(config.db_uri, settings.COUCH_DATABASE)
def test_get_db_for_db_name(self): config = CouchConfig(db_uri=self.remote_db_uri) self.assertEqual(self.remote_db_uri, config.get_db_for_db_name('cchq').uri) self.assertEqual('{}__users'.format(self.remote_db_uri), config.get_db_for_db_name('cchq__users').uri)
def test_get_db_for_doc_type(self): config = CouchConfig(db_uri=self.remote_db_uri) self.assertEqual(config.get_db_for_doc_type('CommCareCase').uri, self.remote_db_uri) self.assertEqual(config.get_db_for_doc_type('CommCareUser').uri, '{}__users'.format(self.remote_db_uri))
class Command(BaseCommand): help = "Copies the contents of a domain to another database. " \ "If tagetdb is not specified, the target is the database " \ "specified by COUCH_DATABASE in your settings." args = '<sourcedb> <domain> [<targetdb>]' option_list = BaseCommand.option_list + ( make_option('--include', action='store', dest='doc_types', default='', help='Comma-separated list of Document Types to copy'), make_option('--exclude', action='store', dest='doc_types_exclude', default='', help='Comma-separated list of Document Types to NOT copy.'), make_option('--exclude-attachments', action='store_true', dest='exclude_attachments', default=False, help="Don't copy document attachments, just the docs themselves."), make_option('--since', action='store', dest='since', default='', help='Only copy documents newer than this date. Format: yyyy-MM-dd. Only '), make_option('--list-types', action='store_true', dest='list_types', default=False, help='Don\'t copy anything, just list all the available document types.'), make_option('--simulate', action='store_true', dest='simulate', default=False, help='Don\'t copy anything, print what would be copied.'), make_option('--id-file', action='store', dest='id_file', default='', help="File containing one document ID per line. Only docs with these ID's will be copied"), make_option('--postgres-db', action='store', dest='postgres_db', default='', help="Name of postgres database to pull additional data from. This should map to a " "key in settings.DATABASES. If not specified no additional postgres data will be " "copied. This is currently used to pull CommCare Supply models."), make_option('--postgres-password', action='store', dest='postgres_password', default='', help="Password for postgres database to pull additional data from. If not specified will " "default to the value in settings.DATABASES"), make_option('--dont-run-multi-process', action='store_false', dest='run_multi_process', default=True, help="If set to true this spawn multiple processes which should speed up the time taken to " "copy. This must be false if running in a supervised process") ) def iter_source_dbs(self): for sourcedb_name, sourcedb in self.source_couch.all_dbs_by_slug.items(): if sourcedb_name not in self.exclude_dbs: print "In {} db".format(sourcedb_name or "the main") yield sourcedb_name, sourcedb def handle(self, *args, **options): if len(args) not in [2, 3]: raise CommandError('Usage is copy_domain %s' % self.args) self.exclude_dbs = ( # these have data we don't want to copy 'receiverwrapper', 'couchlog', 'auditcare', 'fluff-bihar', 'fluff-opm', 'fluff-mc', 'fluff-cvsu', 'mvp-indicators', 'm4change', # todo: missing domain/docs, but probably want to add back 'meta', ) self.source_couch = source_couch = CouchConfig(args[0]) domain = args[1].strip() simulate = options['simulate'] exclude_attachments = options['exclude_attachments'] self.run_multi_process = options['run_multi_process'] since = json_format_date(iso_string_to_date(options['since'])) if options['since'] else None if options['list_types']: for sourcedb_name, sourcedb in self.iter_source_dbs(): self.list_types(sourcedb, domain, since) sys.exit(0) if simulate: print "\nSimulated run, no data will be copied.\n" if options['postgres_db'] and options['postgres_password']: settings.DATABASES[options['postgres_db']]['PASSWORD'] = options['postgres_password'] self.targetdb = CouchConfig(args[2]) if len(args) == 3 else CouchConfig() try: domain_doc = Domain.get_by_name(domain) except ResourceNotFound: domain_doc = None if domain_doc is None: self.copy_domain(source_couch, domain) if options['doc_types']: doc_types = options['doc_types'].split(',') for doc_type in doc_types: sourcedb = source_couch.get_db_for_doc_type(doc_type) startkey = [x for x in [domain, doc_type, since] if x is not None] endkey = [x for x in [domain, doc_type, {}] if x is not None] self.copy_docs(sourcedb, domain, simulate, startkey, endkey, doc_type=doc_type, since=since, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments) elif options['id_file']: path = options['id_file'] if not os.path.isfile(path): print "Path '%s' does not exist or is not a file" % path sys.exit(1) with open(path) as input: doc_ids = [line.rstrip('\n') for line in input] if not doc_ids: print "Path '%s' does not contain any document ID's" % path sys.exit(1) for sourcedb_name, sourcedb in self.iter_source_dbs(): self.copy_docs(sourcedb, domain, simulate, doc_ids=doc_ids, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments) else: startkey = [domain] endkey = [domain, {}] exclude_types = DEFAULT_EXCLUDE_TYPES + options['doc_types_exclude'].split(',') for sourcedb_name, sourcedb in self.iter_source_dbs(): self.copy_docs(sourcedb, domain, simulate, startkey, endkey, exclude_types=exclude_types, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments) def list_types(self, sourcedb, domain, since): doc_types = sourcedb.view("by_domain_doc_type_date/view", startkey=[domain], endkey=[domain, {}], reduce=True, group=True, group_level=2) doc_count = dict([(row['key'][1], row['value']) for row in doc_types]) if since: for doc_type in sorted(doc_count.iterkeys()): num_since = sourcedb.view("by_domain_doc_type_date/view", startkey=[domain, doc_type, since], endkey=[domain, doc_type, {}], reduce=True).all() num = num_since[0]['value'] if num_since else 0 print "{0:<30}- {1:<6} total {2}".format(doc_type, num, doc_count[doc_type]) else: for doc_type in sorted(doc_count.iterkeys()): print "{0:<30}- {1}".format(doc_type, doc_count[doc_type]) def copy_docs(self, sourcedb, domain, simulate, startkey=None, endkey=None, doc_ids=None, doc_type=None, since=None, exclude_types=None, postgres_db=None, exclude_attachments=False): if not doc_ids: doc_ids = [result["id"] for result in sourcedb.view("by_domain_doc_type_date/view", startkey=startkey, endkey=endkey, reduce=False)] total = len(doc_ids) count = 0 msg = "Found %s matching documents in domain: %s" % (total, domain) msg += " of type: %s" % (doc_type) if doc_type else "" msg += " since: %s" % (since) if since else "" print msg err_log = self._get_err_log() if self.run_multi_process: queue = Queue(150) for i in range(NUM_PROCESSES): Worker(queue, sourcedb, self.targetdb, exclude_types, total, simulate, err_log, exclude_attachments).start() for doc in iter_docs(sourcedb, doc_ids, chunksize=100): count += 1 queue.put((doc, count)) # shutdown workers for i in range(NUM_PROCESSES): queue.put(None) else: for doc in iter_docs(sourcedb, doc_ids, chunksize=100): target = self.targetdb.get_db_for_doc_type(doc['doc_type']) count += 1 copy_doc(doc, count, sourcedb, target, exclude_types, total, simulate, exclude_attachments) err_log.close() if os.stat(err_log.name)[6] == 0: os.remove(err_log.name) else: print 'Failed document IDs written to %s' % err_log.name if postgres_db: copy_postgres_data_for_docs(postgres_db, doc_ids=doc_ids, simulate=simulate) def copy_domain(self, source_couch, domain): print "Copying domain doc" sourcedb = source_couch.get_db_for_class(Domain) result = sourcedb.view( "domain/domains", key=domain, reduce=False, include_docs=True ).first() if result and 'doc' in result: domain_doc = Domain.wrap(result['doc']) dt = DocumentTransform(domain_doc._obj, sourcedb) save(dt, self.targetdb.get_db_for_doc_type(domain_doc['doc_type'])) else: print "Domain doc not found for domain %s." % domain def _get_err_log(self): name = 'copy_domain.err.%s' for i in range(1000): # arbitrarily large number candidate = name % i if not os.path.isfile(candidate): return open(candidate, 'a', buffering=1)
def test_get_db_for_db_name_not_found(self): config = CouchConfig(db_uri=self.remote_db_uri) with self.assertRaises(DatabaseNotFound): config.get_db_for_db_name('missing')
def handle(self, *args, **options): if len(args) not in [2, 3]: raise CommandError('Usage is copy_domain %s' % self.args) self.exclude_dbs = ( # these have data we don't want to copy 'receiverwrapper', 'couchlog', 'auditcare', 'fluff-bihar', 'fluff-opm', 'fluff-mc', 'fluff-cvsu', 'mvp-indicators', 'm4change', # todo: missing domain/docs, but probably want to add back 'meta', ) self.source_couch = source_couch = CouchConfig(args[0]) domain = args[1].strip() simulate = options['simulate'] exclude_attachments = options['exclude_attachments'] self.run_multi_process = options['run_multi_process'] since = json_format_date(iso_string_to_date(options['since'])) if options['since'] else None if options['list_types']: for sourcedb_name, sourcedb in self.iter_source_dbs(): self.list_types(sourcedb, domain, since) sys.exit(0) if simulate: print "\nSimulated run, no data will be copied.\n" if options['postgres_db'] and options['postgres_password']: settings.DATABASES[options['postgres_db']]['PASSWORD'] = options['postgres_password'] self.targetdb = CouchConfig(args[2]) if len(args) == 3 else CouchConfig() try: domain_doc = Domain.get_by_name(domain) except ResourceNotFound: domain_doc = None if domain_doc is None: self.copy_domain(source_couch, domain) if options['doc_types']: doc_types = options['doc_types'].split(',') for doc_type in doc_types: sourcedb = source_couch.get_db_for_doc_type(doc_type) startkey = [x for x in [domain, doc_type, since] if x is not None] endkey = [x for x in [domain, doc_type, {}] if x is not None] self.copy_docs(sourcedb, domain, simulate, startkey, endkey, doc_type=doc_type, since=since, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments) elif options['id_file']: path = options['id_file'] if not os.path.isfile(path): print "Path '%s' does not exist or is not a file" % path sys.exit(1) with open(path) as input: doc_ids = [line.rstrip('\n') for line in input] if not doc_ids: print "Path '%s' does not contain any document ID's" % path sys.exit(1) for sourcedb_name, sourcedb in self.iter_source_dbs(): self.copy_docs(sourcedb, domain, simulate, doc_ids=doc_ids, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments) else: startkey = [domain] endkey = [domain, {}] exclude_types = DEFAULT_EXCLUDE_TYPES + options['doc_types_exclude'].split(',') for sourcedb_name, sourcedb in self.iter_source_dbs(): self.copy_docs(sourcedb, domain, simulate, startkey, endkey, exclude_types=exclude_types, postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments)