def load_objects(self, object_strings, force=False):
# Keep a count of the installed objects
load_stats_by_db = {}
total_object_counts = []
def _process_chunk(chunk, total_object_counts=total_object_counts, load_stats_by_db=load_stats_by_db):
chunk_stats = load_objects(chunk)
total_object_counts.append(len(chunk))
self.stdout.write('Loaded {} SQL objects'.format(sum(total_object_counts)))
_update_stats(load_stats_by_db, chunk_stats)
chunk = []
for line in object_strings:
line = line.strip()
if not line:
continue
chunk.append(json.loads(line))
if len(chunk) >= 1000:
_process_chunk(chunk)
chunk = []
if chunk:
_process_chunk(chunk)
_reset_sequences(list(load_stats_by_db.values()))
loaded_model_counts = Counter()
for db_stats in load_stats_by_db.values():
model_labels = ('(sql) {}'.format(get_model_label(model)) for model in db_stats.model_counter.elements())
loaded_model_counts.update(model_labels)
return sum(total_object_counts), loaded_model_counts
def load_objects(self, object_strings, force=False):
# Keep a count of the installed objects
load_stats_by_db = {}
total_object_counts = []
def _process_chunk(chunk, total_object_counts=total_object_counts, load_stats_by_db=load_stats_by_db):
chunk_stats = load_objects(chunk)
total_object_counts.append(len(chunk))
self.stdout.write('Loaded {} SQL objects'.format(sum(total_object_counts)))
_update_stats(load_stats_by_db, chunk_stats)
chunk = []
for line in object_strings:
line = line.strip()
if not line:
continue
chunk.append(json.loads(line))
if len(chunk) >= 1000:
_process_chunk(chunk)
chunk = []
if chunk:
_process_chunk(chunk)
_reset_sequences(list(load_stats_by_db.values()))
loaded_model_counts = Counter()
for db_stats in load_stats_by_db.values():
model_labels = ('(sql) {}'.format(get_model_label(model)) for model in db_stats.model_counter.elements())
loaded_model_counts.update(model_labels)
return sum(total_object_counts), loaded_model_counts
def _get_sql_counts(domain):
counter = Counter()
for model_class, queryset in get_querysets_to_dump(domain, []):
if model_class in (User, XFormInstanceSQL, CommCareCaseSQL):
continue # User is very slow, others we want to break out
counter[get_model_label(model_class)] += queryset.count()
counter += _get_sql_forms_by_doc_type(domain)
counter += _get_sql_cases_by_doc_type(domain)
return counter
def _get_sql_counts(domain):
counter = Counter()
for model_class, queryset in get_querysets_to_dump(domain, []):
if model_class in (User, XFormInstanceSQL, CommCareCaseSQL):
continue # User is very slow, others we want to break out
counter[get_model_label(model_class)] += queryset.count()
counter += get_primary_db_form_counts(domain)
counter += get_primary_db_case_counts(domain)
return counter
def get_objects_to_dump_from_builders(builders, stats_counter=None, stdout=None):
if stats_counter is None:
stats_counter = Counter()
for model_class, builder in builders:
model_label = get_model_label(model_class)
for iterator in builder.iterators():
for obj in iterator:
stats_counter.update([model_label])
yield obj
if stdout:
stdout.write('Dumped {} {}\n'.format(stats_counter[model_label], model_label))
def _get_sql_counts(domain):
counter = Counter()
for model_class, builder in get_model_iterator_builders_to_dump(domain, []):
if model_class in (User, XFormInstanceSQL, CommCareCaseSQL):
continue # User is very slow, others we want to break out
for queryset in builder.querysets():
counter[get_model_label(model_class)] += queryset.count()
counter += get_primary_db_form_counts(domain)
counter += get_primary_db_case_counts(domain)
return counter
def _ignore_model(model):
if not model._meta.managed:
return True
if model._meta.app_label in IGNORE_APPS:
return True
if get_model_label(model) in IGNORE_MODELS:
return True
if model._meta.proxy:
return model._meta.concrete_model in covered_models
def get_objects_to_dump(domain, excludes, stats_counter=None, stdout=None):
"""
:param domain: domain name to filter with
:param app_list: List of (app_config, model_class) tuples to dump
:param excluded_models: List of model_class classes to exclude
:return: generator yielding models objects
"""
if stats_counter is None:
stats_counter = Counter()
for model_class, query in get_querysets_to_dump(domain, excludes):
model_label = get_model_label(model_class)
for obj in query.iterator():
stats_counter.update([model_label])
yield obj
if stdout:
stdout.write('Dumped {} {}\n'.format(stats_counter[model_label], model_label))
def get_objects_to_dump(domain, excludes, stats_counter=None, stdout=None):
"""
:param domain: domain name to filter with
:param app_list: List of (app_config, model_class) tuples to dump
:param excluded_models: List of model_class classes to exclude
:return: generator yielding models objects
"""
if stats_counter is None:
stats_counter = Counter()
for model_class, query in get_querysets_to_dump(domain, excludes):
model_label = get_model_label(model_class)
for obj in query.iterator():
stats_counter.update([model_label])
yield obj
if stdout:
stdout.write('Dumped {} {}\n'.format(stats_counter[model_label], model_label))
def get_all_model_iterators_builders_for_domain(model_class, domain, limit_to_db=None):
if settings.USE_PARTITIONED_DATABASE and hasattr(model_class, 'partition_attr'):
using = plproxy_config.form_processing_dbs
else:
using = [router.db_for_read(model_class)]
if limit_to_db:
if limit_to_db not in using:
raise DomainDumpError('DB specified is not valide for '
'model class: {} not in {}'.format(limit_to_db, using))
using = [limit_to_db]
for db_alias in using:
if not model_class._meta.proxy and router.allow_migrate_model(db_alias, model_class):
iterator_builders = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)]
for builder in iterator_builders:
yield model_class, builder.build(domain, model_class, db_alias)
def get_all_model_iterators_builders_for_domain(model_class, domain, limit_to_db=None):
using = router.db_for_read(model_class)
if settings.USE_PARTITIONED_DATABASE and using == partition_config.get_proxy_db():
using = partition_config.get_form_processing_dbs()
else:
using = [using]
if limit_to_db:
if limit_to_db not in using:
raise DomainDumpError('DB specified is not valide for '
'model class: {} not in {}'.format(limit_to_db, using))
using = [limit_to_db]
for db_alias in using:
if not model_class._meta.proxy and router.allow_migrate_model(db_alias, model_class):
iterator_builder = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)]
yield model_class, iterator_builder.build(domain, model_class, db_alias)
def get_model_iterator_builders_to_dump(domain, excludes, limit_to_db=None):
"""
:param domain: domain name to filter with
:param app_list: List of (app_config, model_class) tuples to dump
:param excluded_models: List of model_class classes to exclude
:return: generator yielding query sets
"""
excluded_apps, excluded_models = get_excluded_apps_and_models(excludes)
app_config_models = _get_app_list(excluded_apps)
# Collate the objects to be serialized.
for model_class in serializers.sort_dependencies(list(app_config_models.items())):
if model_class in excluded_models:
continue
iterator_builders = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)]
for model_class, builder in get_all_model_iterators_builders_for_domain(
model_class, domain, iterator_builders, limit_to_db=limit_to_db
):
yield model_class, builder
def _get_doc_counts_from_db(domain):
return {
get_model_label(doc_class): len(doc_ids)
for doc_class, doc_ids in get_doc_ids_to_dump(domain) if doc_ids
}
def get_model_domain_filters(model_class, domain):
label = get_model_label(model_class)
filter = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[label]
return filter.get_filters(domain)
def _get_doc_counts_from_db(domain):
return {
get_model_label(doc_class): len(doc_ids)
for doc_class, doc_ids in get_doc_ids_to_dump(domain) if doc_ids
}
def get_model_domain_filters(model_class, domain):
label = get_model_label(model_class)
filter = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[label]
return filter.get_filters(domain)
