コード例 #1
0
ファイル: load.py プロジェクト: dimagi/commcare-hq
    def load_objects(self, object_strings, force=False):
        # Keep a count of the installed objects
        load_stats_by_db = {}
        total_object_counts = []

        def _process_chunk(chunk, total_object_counts=total_object_counts, load_stats_by_db=load_stats_by_db):
            chunk_stats = load_objects(chunk)
            total_object_counts.append(len(chunk))
            self.stdout.write('Loaded {} SQL objects'.format(sum(total_object_counts)))
            _update_stats(load_stats_by_db, chunk_stats)

        chunk = []
        for line in object_strings:
            line = line.strip()
            if not line:
                continue
            chunk.append(json.loads(line))
            if len(chunk) >= 1000:
                _process_chunk(chunk)
                chunk = []

        if chunk:
            _process_chunk(chunk)

        _reset_sequences(list(load_stats_by_db.values()))

        loaded_model_counts = Counter()
        for db_stats in load_stats_by_db.values():
            model_labels = ('(sql) {}'.format(get_model_label(model)) for model in db_stats.model_counter.elements())
            loaded_model_counts.update(model_labels)

        return sum(total_object_counts), loaded_model_counts
コード例 #2
0
    def load_objects(self, object_strings, force=False):
        # Keep a count of the installed objects
        load_stats_by_db = {}
        total_object_counts = []

        def _process_chunk(chunk, total_object_counts=total_object_counts, load_stats_by_db=load_stats_by_db):
            chunk_stats = load_objects(chunk)
            total_object_counts.append(len(chunk))
            self.stdout.write('Loaded {} SQL objects'.format(sum(total_object_counts)))
            _update_stats(load_stats_by_db, chunk_stats)

        chunk = []
        for line in object_strings:
            line = line.strip()
            if not line:
                continue
            chunk.append(json.loads(line))
            if len(chunk) >= 1000:
                _process_chunk(chunk)
                chunk = []

        if chunk:
            _process_chunk(chunk)

        _reset_sequences(list(load_stats_by_db.values()))

        loaded_model_counts = Counter()
        for db_stats in load_stats_by_db.values():
            model_labels = ('(sql) {}'.format(get_model_label(model)) for model in db_stats.model_counter.elements())
            loaded_model_counts.update(model_labels)

        return sum(total_object_counts), loaded_model_counts
コード例 #3
0
def _get_sql_counts(domain):
    counter = Counter()
    for model_class, queryset in get_querysets_to_dump(domain, []):
        if model_class in (User, XFormInstanceSQL, CommCareCaseSQL):
            continue  # User is very slow, others we want to break out
        counter[get_model_label(model_class)] += queryset.count()

    counter += _get_sql_forms_by_doc_type(domain)
    counter += _get_sql_cases_by_doc_type(domain)
    return counter
コード例 #4
0
def _get_sql_counts(domain):
    counter = Counter()
    for model_class, queryset in get_querysets_to_dump(domain, []):
        if model_class in (User, XFormInstanceSQL, CommCareCaseSQL):
            continue  # User is very slow, others we want to break out
        counter[get_model_label(model_class)] += queryset.count()

    counter += get_primary_db_form_counts(domain)
    counter += get_primary_db_case_counts(domain)
    return counter
コード例 #5
0
ファイル: dump.py プロジェクト: challabeehyv/commcare-hq
def get_objects_to_dump_from_builders(builders, stats_counter=None, stdout=None):
    if stats_counter is None:
        stats_counter = Counter()
    for model_class, builder in builders:
        model_label = get_model_label(model_class)
        for iterator in builder.iterators():
            for obj in iterator:
                stats_counter.update([model_label])
                yield obj
        if stdout:
            stdout.write('Dumped {} {}\n'.format(stats_counter[model_label], model_label))
コード例 #6
0
def _get_sql_counts(domain):
    counter = Counter()
    for model_class, builder in get_model_iterator_builders_to_dump(domain, []):
        if model_class in (User, XFormInstanceSQL, CommCareCaseSQL):
            continue  # User is very slow, others we want to break out
        for queryset in builder.querysets():
            counter[get_model_label(model_class)] += queryset.count()

    counter += get_primary_db_form_counts(domain)
    counter += get_primary_db_case_counts(domain)
    return counter
コード例 #7
0
    def _ignore_model(model):
        if not model._meta.managed:
            return True

        if model._meta.app_label in IGNORE_APPS:
            return True

        if get_model_label(model) in IGNORE_MODELS:
            return True

        if model._meta.proxy:
            return model._meta.concrete_model in covered_models
コード例 #8
0
def get_objects_to_dump(domain, excludes, stats_counter=None, stdout=None):
    """
    :param domain: domain name to filter with
    :param app_list: List of (app_config, model_class) tuples to dump
    :param excluded_models: List of model_class classes to exclude
    :return: generator yielding models objects
    """
    if stats_counter is None:
        stats_counter = Counter()
    for model_class, query in get_querysets_to_dump(domain, excludes):
        model_label = get_model_label(model_class)
        for obj in query.iterator():
            stats_counter.update([model_label])
            yield obj
        if stdout:
            stdout.write('Dumped {} {}\n'.format(stats_counter[model_label], model_label))
コード例 #9
0
def get_objects_to_dump(domain, excludes, stats_counter=None, stdout=None):
    """
    :param domain: domain name to filter with
    :param app_list: List of (app_config, model_class) tuples to dump
    :param excluded_models: List of model_class classes to exclude
    :return: generator yielding models objects
    """
    if stats_counter is None:
        stats_counter = Counter()
    for model_class, query in get_querysets_to_dump(domain, excludes):
        model_label = get_model_label(model_class)
        for obj in query.iterator():
            stats_counter.update([model_label])
            yield obj
        if stdout:
            stdout.write('Dumped {} {}\n'.format(stats_counter[model_label], model_label))
コード例 #10
0
def get_all_model_iterators_builders_for_domain(model_class, domain, limit_to_db=None):
    if settings.USE_PARTITIONED_DATABASE and hasattr(model_class, 'partition_attr'):
        using = plproxy_config.form_processing_dbs
    else:
        using = [router.db_for_read(model_class)]

    if limit_to_db:
        if limit_to_db not in using:
            raise DomainDumpError('DB specified is not valide for '
                                  'model class: {} not in {}'.format(limit_to_db, using))
        using = [limit_to_db]

    for db_alias in using:
        if not model_class._meta.proxy and router.allow_migrate_model(db_alias, model_class):
            iterator_builders = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)]
            for builder in iterator_builders:
                yield model_class, builder.build(domain, model_class, db_alias)
コード例 #11
0
ファイル: dump.py プロジェクト: dimagi/commcare-hq
def get_all_model_iterators_builders_for_domain(model_class, domain, limit_to_db=None):
    using = router.db_for_read(model_class)
    if settings.USE_PARTITIONED_DATABASE and using == partition_config.get_proxy_db():
        using = partition_config.get_form_processing_dbs()
    else:
        using = [using]

    if limit_to_db:
        if limit_to_db not in using:
            raise DomainDumpError('DB specified is not valide for '
                                  'model class: {} not in {}'.format(limit_to_db, using))
        using = [limit_to_db]

    for db_alias in using:
        if not model_class._meta.proxy and router.allow_migrate_model(db_alias, model_class):
            iterator_builder = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)]
            yield model_class, iterator_builder.build(domain, model_class, db_alias)
コード例 #12
0
ファイル: dump.py プロジェクト: challabeehyv/commcare-hq
def get_model_iterator_builders_to_dump(domain, excludes, limit_to_db=None):
    """
    :param domain: domain name to filter with
    :param app_list: List of (app_config, model_class) tuples to dump
    :param excluded_models: List of model_class classes to exclude
    :return: generator yielding query sets
    """
    excluded_apps, excluded_models = get_excluded_apps_and_models(excludes)
    app_config_models = _get_app_list(excluded_apps)

    # Collate the objects to be serialized.
    for model_class in serializers.sort_dependencies(list(app_config_models.items())):
        if model_class in excluded_models:
            continue

        iterator_builders = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)]
        for model_class, builder in get_all_model_iterators_builders_for_domain(
            model_class, domain, iterator_builders, limit_to_db=limit_to_db
        ):
            yield model_class, builder
コード例 #13
0
def _get_doc_counts_from_db(domain):
    return {
        get_model_label(doc_class): len(doc_ids)
        for doc_class, doc_ids in get_doc_ids_to_dump(domain) if doc_ids
    }
コード例 #14
0
def get_model_domain_filters(model_class, domain):
    label = get_model_label(model_class)
    filter = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[label]
    return filter.get_filters(domain)
コード例 #15
0
def _get_doc_counts_from_db(domain):
    return {
        get_model_label(doc_class): len(doc_ids)
        for doc_class, doc_ids in get_doc_ids_to_dump(domain) if doc_ids
    }
コード例 #16
0
def get_model_domain_filters(model_class, domain):
    label = get_model_label(model_class)
    filter = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[label]
    return filter.get_filters(domain)