def load_objects(self, object_strings, force=False): # Keep a count of the installed objects load_stats_by_db = {} total_object_counts = [] def _process_chunk(chunk, total_object_counts=total_object_counts, load_stats_by_db=load_stats_by_db): chunk_stats = load_objects(chunk) total_object_counts.append(len(chunk)) self.stdout.write('Loaded {} SQL objects'.format(sum(total_object_counts))) _update_stats(load_stats_by_db, chunk_stats) chunk = [] for line in object_strings: line = line.strip() if not line: continue chunk.append(json.loads(line)) if len(chunk) >= 1000: _process_chunk(chunk) chunk = [] if chunk: _process_chunk(chunk) _reset_sequences(list(load_stats_by_db.values())) loaded_model_counts = Counter() for db_stats in load_stats_by_db.values(): model_labels = ('(sql) {}'.format(get_model_label(model)) for model in db_stats.model_counter.elements()) loaded_model_counts.update(model_labels) return sum(total_object_counts), loaded_model_counts
def _get_sql_counts(domain): counter = Counter() for model_class, queryset in get_querysets_to_dump(domain, []): if model_class in (User, XFormInstanceSQL, CommCareCaseSQL): continue # User is very slow, others we want to break out counter[get_model_label(model_class)] += queryset.count() counter += _get_sql_forms_by_doc_type(domain) counter += _get_sql_cases_by_doc_type(domain) return counter
def _get_sql_counts(domain): counter = Counter() for model_class, queryset in get_querysets_to_dump(domain, []): if model_class in (User, XFormInstanceSQL, CommCareCaseSQL): continue # User is very slow, others we want to break out counter[get_model_label(model_class)] += queryset.count() counter += get_primary_db_form_counts(domain) counter += get_primary_db_case_counts(domain) return counter
def get_objects_to_dump_from_builders(builders, stats_counter=None, stdout=None): if stats_counter is None: stats_counter = Counter() for model_class, builder in builders: model_label = get_model_label(model_class) for iterator in builder.iterators(): for obj in iterator: stats_counter.update([model_label]) yield obj if stdout: stdout.write('Dumped {} {}\n'.format(stats_counter[model_label], model_label))
def _get_sql_counts(domain): counter = Counter() for model_class, builder in get_model_iterator_builders_to_dump(domain, []): if model_class in (User, XFormInstanceSQL, CommCareCaseSQL): continue # User is very slow, others we want to break out for queryset in builder.querysets(): counter[get_model_label(model_class)] += queryset.count() counter += get_primary_db_form_counts(domain) counter += get_primary_db_case_counts(domain) return counter
def _ignore_model(model): if not model._meta.managed: return True if model._meta.app_label in IGNORE_APPS: return True if get_model_label(model) in IGNORE_MODELS: return True if model._meta.proxy: return model._meta.concrete_model in covered_models
def get_objects_to_dump(domain, excludes, stats_counter=None, stdout=None): """ :param domain: domain name to filter with :param app_list: List of (app_config, model_class) tuples to dump :param excluded_models: List of model_class classes to exclude :return: generator yielding models objects """ if stats_counter is None: stats_counter = Counter() for model_class, query in get_querysets_to_dump(domain, excludes): model_label = get_model_label(model_class) for obj in query.iterator(): stats_counter.update([model_label]) yield obj if stdout: stdout.write('Dumped {} {}\n'.format(stats_counter[model_label], model_label))
def get_all_model_iterators_builders_for_domain(model_class, domain, limit_to_db=None): if settings.USE_PARTITIONED_DATABASE and hasattr(model_class, 'partition_attr'): using = plproxy_config.form_processing_dbs else: using = [router.db_for_read(model_class)] if limit_to_db: if limit_to_db not in using: raise DomainDumpError('DB specified is not valide for ' 'model class: {} not in {}'.format(limit_to_db, using)) using = [limit_to_db] for db_alias in using: if not model_class._meta.proxy and router.allow_migrate_model(db_alias, model_class): iterator_builders = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)] for builder in iterator_builders: yield model_class, builder.build(domain, model_class, db_alias)
def get_all_model_iterators_builders_for_domain(model_class, domain, limit_to_db=None): using = router.db_for_read(model_class) if settings.USE_PARTITIONED_DATABASE and using == partition_config.get_proxy_db(): using = partition_config.get_form_processing_dbs() else: using = [using] if limit_to_db: if limit_to_db not in using: raise DomainDumpError('DB specified is not valide for ' 'model class: {} not in {}'.format(limit_to_db, using)) using = [limit_to_db] for db_alias in using: if not model_class._meta.proxy and router.allow_migrate_model(db_alias, model_class): iterator_builder = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)] yield model_class, iterator_builder.build(domain, model_class, db_alias)
def get_model_iterator_builders_to_dump(domain, excludes, limit_to_db=None): """ :param domain: domain name to filter with :param app_list: List of (app_config, model_class) tuples to dump :param excluded_models: List of model_class classes to exclude :return: generator yielding query sets """ excluded_apps, excluded_models = get_excluded_apps_and_models(excludes) app_config_models = _get_app_list(excluded_apps) # Collate the objects to be serialized. for model_class in serializers.sort_dependencies(list(app_config_models.items())): if model_class in excluded_models: continue iterator_builders = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)] for model_class, builder in get_all_model_iterators_builders_for_domain( model_class, domain, iterator_builders, limit_to_db=limit_to_db ): yield model_class, builder
def _get_doc_counts_from_db(domain): return { get_model_label(doc_class): len(doc_ids) for doc_class, doc_ids in get_doc_ids_to_dump(domain) if doc_ids }
def get_model_domain_filters(model_class, domain): label = get_model_label(model_class) filter = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[label] return filter.get_filters(domain)