def get_excluded_apps_and_models(excludes):
"""
:param excludes: list of app labels ("app_label.model_name" or "app_label") to exclude
:return: Tuple containing two sets: Set of AppConfigs to exclude, Set of model classes to excluded.
"""
excluded_apps = set()
excluded_models = set()
for exclude in excludes:
if '.' in exclude:
try:
model = apps.get_model(exclude)
except LookupError:
raise DomainDumpError('Unknown model in excludes: %s' % exclude)
excluded_models.add(model)
else:
try:
app_config = apps.get_app_config(exclude)
except LookupError:
from corehq.util.couch import get_document_class_by_doc_type
from corehq.util.exceptions import DocumentClassNotFound
# ignore this if it's a couch doc type
try:
get_document_class_by_doc_type(exclude)
except DocumentClassNotFound:
raise DomainDumpError('Unknown app in excludes: %s' % exclude)
excluded_apps.add(app_config)
return excluded_apps, excluded_models
def get_apps_and_models(app_or_model_label):
"""
:param app_or_model_label: list of app labels ("app_label.model_name" or "app_label")
:return: Tuple containing two sets: Set of AppConfigs, Set of model classes
"""
specified_apps = set()
specified_models = set()
for label in app_or_model_label:
if '.' in label:
try:
model = apps.get_model(label)
except LookupError:
raise DomainDumpError('Unknown model: %s' % label)
specified_models.add(model)
else:
try:
app_config = apps.get_app_config(label)
except LookupError:
from corehq.util.couch import get_document_class_by_doc_type
from corehq.util.exceptions import DocumentClassNotFound
# ignore this if it's a couch doc type
try:
get_document_class_by_doc_type(label)
except DocumentClassNotFound:
raise DomainDumpError('Unknown app in excludes: %s' % label)
specified_apps.add(app_config)
return specified_apps, specified_models
def _get_couchdb_counts(domain):
couch_db_counts = Counter()
for provider in DOC_PROVIDERS:
if isinstance(provider, DocTypeIDProvider):
for doc_type in provider.doc_types:
if doc_type == 'CommCareUser':
continue # want to split deleted
doc_class = get_document_class_by_doc_type(doc_type)
count = get_doc_count_in_domain_by_class(domain, doc_class)
couch_db_counts.update({doc_type: count})
for row in CommCareMultimedia.get_db().view('hqmedia/by_domain',
key=domain,
include_docs=False):
couch_db_counts.update(['CommCareMultimedia'])
mobile_user_count = get_mobile_user_count(domain)
couch_db_counts.update({
'WebUser':
get_web_user_count(domain),
'CommCareUser':
mobile_user_count,
'CommCareUser-Deleted':
get_doc_count_in_domain_by_class(domain, CommCareUser) -
mobile_user_count
})
# this is very slow, excluding for now
# for _, doc_ids in SyncLogIDProvider().get_doc_ids(domain):
# couch_db_counts['SyncLog'] += len(doc_ids)
#
return couch_db_counts
def _get_couchdb_counts(domain):
couch_db_counts = Counter()
for provider in DOC_PROVIDERS:
if isinstance(provider, DocTypeIDProvider):
doc_type = provider.doc_type
if doc_type == 'CommCareUser':
continue # want to split deleted
doc_class = get_document_class_by_doc_type(doc_type)
count = get_doc_count_in_domain_by_class(domain, doc_class)
couch_db_counts.update({doc_type: count})
for _ in CommCareMultimedia.get_db().view('hqmedia/by_domain',
key=domain,
include_docs=False):
couch_db_counts.update(['CommCareMultimedia'])
mobile_user_count = get_mobile_user_count(domain)
couch_db_counts.update({
'WebUser':
get_web_user_count(domain),
'CommCareUser':
mobile_user_count,
'CommCareUser-Deleted':
get_doc_count_in_domain_by_class(domain, CommCareUser) -
mobile_user_count
})
return couch_db_counts
def _get_couchdb_counts(domain):
couch_db_counts = Counter()
for provider in DOC_PROVIDERS:
if isinstance(provider, DocTypeIDProvider):
for doc_type in provider.doc_types:
if doc_type == 'CommCareUser':
continue # want to split deleted
doc_class = get_document_class_by_doc_type(doc_type)
count = get_doc_count_in_domain_by_class(domain, doc_class)
couch_db_counts.update({doc_type: count})
for row in CommCareMultimedia.get_db().view('hqmedia/by_domain', key=domain, include_docs=False):
couch_db_counts.update(['CommCareMultimedia'])
mobile_user_count = get_mobile_user_count(domain)
couch_db_counts.update({
'WebUser': get_web_user_count(domain),
'CommCareUser': mobile_user_count,
'CommCareUser-Deleted': get_doc_count_in_domain_by_class(domain, CommCareUser) - mobile_user_count
})
# this is very slow, excluding for now
# for _, doc_ids in SyncLogIDProvider().get_doc_ids(domain):
# couch_db_counts['SyncLog'] += len(doc_ids)
#
return couch_db_counts
def get_doc_ids(self, domain):
doc_class = get_document_class_by_doc_type(self.doc_type)
key = self.key_generator(self.doc_type, domain) if self.key_generator else domain
doc_ids = [
row['id']
for row in doc_class.get_db().view(self.view_name, key=key, include_docs=False)
]
return [(doc_class, doc_ids)]
def get_doc_ids(self, domain):
doc_class = get_document_class_by_doc_type(self.doc_type)
key_kwargs = self.key_generator(self.doc_type, domain)
doc_ids = [
row['id'] for row in doc_class.get_db().view(
self.view_name, include_docs=False, reduce=False, **key_kwargs)
]
return [(doc_class, doc_ids)]
def get_doc_ids(self, domain):
doc_class = get_document_class_by_doc_type(self.doc_type)
key_kwargs = self.key_generator(self.doc_type, domain)
doc_ids = [
row['id']
for row in doc_class.get_db().view(
self.view_name, include_docs=False, reduce=False,
**key_kwargs)
]
return [(doc_class, doc_ids)]
def _make_doc(doc_type, domain):
doc_class = get_document_class_by_doc_type(doc_type)
properties_by_key = doc_class._properties_by_key
self.assertIn('domain', properties_by_key, doc_type)
doc = doc_class(domain=domain)
for key, prop in properties_by_key.items():
if key != 'domain' and prop.required:
doc[key] = _get_property_value(prop)
doc = doc_class.wrap(doc.to_json()) # dump and wrap to set default props etc.
res = doc_class.get_db().save_doc(doc)
self.assertTrue(res['ok'])
return doc
def _make_doc(doc_type, domain):
doc_class = get_document_class_by_doc_type(doc_type)
properties_by_key = doc_class._properties_by_key
self.assertIn('domain', properties_by_key, doc_type)
doc = doc_class(domain=domain)
for key, prop in properties_by_key.items():
if key != 'domain' and prop.required:
doc[key] = _get_property_value(prop)
doc = doc_class.wrap(
doc.to_json()) # dump and wrap to set default props etc.
res = doc_class.get_db().save_doc(doc)
self.assertTrue(res['ok'])
return doc
def post_commit(self, operation, committed_docs, success_ids, errors):
if errors:
raise Exception("Errors loading data", errors)
success_doc_types = []
for doc in committed_docs:
doc_id = doc['_id']
doc_type = drop_suffix(doc['doc_type'])
doc_class = get_document_class_by_doc_type(doc_type)
doc_label = '(couch) {}.{}'.format(doc_class._meta.app_label, doc_type)
if doc_id in success_ids:
success_doc_types.append(doc_label)
self.success_counter.update(success_doc_types)
if self.stdout:
self.stdout.write('Loaded {} couch docs'.format(sum(self.success_counter.values())))
def post_commit(self, operation, committed_docs, success_ids, errors):
if errors:
raise Exception("Errors loading data", errors)
success_doc_types = []
for doc in committed_docs:
doc_id = doc['_id']
doc_type = doc['doc_type']
doc_class = get_document_class_by_doc_type(doc_type)
doc_label = '(couch) {}.{}'.format(doc_class._meta.app_label, doc_type)
if doc_id in success_ids:
success_doc_types.append(doc_label)
self.success_counter.update(success_doc_types)
if self.stdout:
self.stdout.write('Loaded {} couch docs'.format(sum(self.success_counter.values())))
def _get_couchdb_counts(domain):
couch_db_counts = Counter()
for provider in DOC_PROVIDERS:
if isinstance(provider, DocTypeIDProvider):
for doc_type in provider.doc_types:
if doc_type == 'CommCareUser':
continue # want to split deleted
doc_class = get_document_class_by_doc_type(doc_type)
count = get_doc_count_in_domain_by_class(domain, doc_class)
couch_db_counts.update({doc_type: count})
mobile_user_count = get_mobile_user_count(domain)
couch_db_counts.update({
'WebUser': get_web_user_count(domain),
'CommCareUser': mobile_user_count,
'CommCareUser-Deleted': get_doc_count_in_domain_by_class(domain, CommCareUser) - mobile_user_count
})
return couch_db_counts
def _doc_to_doc_class(doc, doc_to_doc_class):
if doc_to_doc_class:
doc_class = doc_to_doc_class(doc)
if doc_class:
return doc_class
return get_document_class_by_doc_type(doc['doc_type'])
def get_db_uri_for_doc_type(self, doc_type):
return self.get_db_uri_for_class(get_document_class_by_doc_type(doc_type))
def _doc_to_doc_class(doc, doc_to_doc_class):
if doc_to_doc_class:
doc_class = doc_to_doc_class(doc)
if doc_class:
return doc_class
return get_document_class_by_doc_type(doc['doc_type'])
def get_db_uri_for_doc_type(self, doc_type):
return self.get_db_uri_for_class(
get_document_class_by_doc_type(doc_type))
def get_doc_ids(self, domain):
for doc_type in self.doc_types:
doc_class = get_document_class_by_doc_type(doc_type)
doc_ids = get_doc_ids_in_domain_by_type(domain, doc_type)
yield doc_class, doc_ids
def get_doc_ids(self, domain):
for doc_type in self.doc_types:
doc_class = get_document_class_by_doc_type(doc_type)
doc_ids = get_doc_ids_in_domain_by_type(domain, doc_type)
yield doc_class, doc_ids
