def get_sql_form_ids(domain, doc_type):
sql_ids = set()
state = doc_type_to_state[doc_type]
for db_alias in get_sql_db_aliases_in_use():
queryset = XFormInstanceSQL.objects.using(db_alias) \
.filter(domain=domain, state=state).values_list('form_id', flat=True)
sql_ids.update(list(queryset))
return sql_ids
def get_sql_case_ids(domain, doc_type):
sql_ids = set()
deleted = doc_type == 'CommCareCase-Deleted'
for db_alias in get_sql_db_aliases_in_use():
queryset = CommCareCaseSQL.objects.using(db_alias) \
.filter(domain=domain, deleted=deleted).values_list('case_id', flat=True)
sql_ids.update(list(queryset))
return sql_ids
def _get_sql_cases_by_doc_type(domain):
counter = Counter()
for db_alias in get_sql_db_aliases_in_use():
queryset = CommCareCaseSQL.objects.using(db_alias).filter(domain=domain)
counter['CommCareCase'] += queryset.filter(deleted=False).count()
counter['CommCareCase-Deleted'] += queryset.filter(deleted=True).count()
return counter
def _get_sql_cases_by_doc_type(domain):
counter = Counter()
for db_alias in get_sql_db_aliases_in_use():
queryset = CommCareCaseSQL.objects.using(db_alias).filter(
domain=domain)
counter['CommCareCase'] += queryset.filter(deleted=False).count()
counter['CommCareCase-Deleted'] += queryset.filter(
deleted=True).count()
return counter
def _compare_cases(domain, doc_type):
sql_ids = set()
for db_alias in get_sql_db_aliases_in_use():
queryset = CommCareCaseSQL.objects.using(db_alias) \
.filter(domain=domain, deleted=False).values_list('case_id', flat=True)
sql_ids.update(list(queryset))
es_ids = set(es.CaseES().remove_default_filters().filter(
es.filters.term('domain', domain)).get_ids())
return sql_ids - es_ids
def _get_sql_forms_by_doc_type(domain):
counter = Counter()
for db_alias in get_sql_db_aliases_in_use():
queryset = XFormInstanceSQL.objects.using(db_alias).filter(domain=domain)
for doc_type, state in doc_type_to_state.items():
counter[doc_type] += queryset.filter(state=state).count()
where_clause = 'state & {0} = {0}'.format(XFormInstanceSQL.DELETED)
counter['XFormInstance-Deleted'] += queryset.extra(where=[where_clause]).count()
return counter
def _get_sql_cases_by_doc_type(domain, startdate=None, enddate=None):
counter = Counter()
for db_alias in get_sql_db_aliases_in_use():
queryset = CommCareCaseSQL.objects.using(db_alias).filter(domain=domain)
if startdate is not None:
queryset = queryset.filter(server_modified_on__gte=startdate)
if enddate is not None:
queryset = queryset.filter(server_modified_on__lt=enddate)
counter['CommCareCase'] += queryset.filter(deleted=False).count()
counter['CommCareCase-Deleted'] += queryset.filter(deleted=True).count()
return counter
def _get_sql_forms_by_doc_type(domain):
counter = Counter()
for db_alias in get_sql_db_aliases_in_use():
queryset = XFormInstanceSQL.objects.using(db_alias).filter(
domain=domain)
for doc_type, state in doc_type_to_state.items():
counter[doc_type] += queryset.filter(state=state).count()
where_clause = 'state & {0} = {0}'.format(XFormInstanceSQL.DELETED)
counter['XFormInstance-Deleted'] += queryset.extra(
where=[where_clause]).count()
return counter
def _compare_xforms(domain, doc_type):
sql_ids = set()
state = doc_type_to_state[doc_type]
for db_alias in get_sql_db_aliases_in_use():
queryset = XFormInstanceSQL.objects.using(db_alias) \
.filter(domain=domain, state=state).values_list('form_id', flat=True)
sql_ids.update(list(queryset))
es_ids = set(es.FormES().remove_default_filters().filter(
es.filters.term('domain', domain)).filter(
es.filters.term('doc_type', doc_type.lower())).get_ids())
return sql_ids - es_ids
def _get_sql_cases_by_doc_type(domain, startdate=None, enddate=None):
counter = Counter()
for db_alias in get_sql_db_aliases_in_use():
queryset = CommCareCaseSQL.objects.using(db_alias).filter(
domain=domain)
if startdate is not None:
queryset = queryset.filter(server_modified_on__gte=startdate)
if enddate is not None:
queryset = queryset.filter(server_modified_on__lt=enddate)
counter['CommCareCase'] += queryset.filter(deleted=False).count()
counter['CommCareCase-Deleted'] += queryset.filter(
deleted=True).count()
return counter
def get_sql_case_ids(domain, doc_type, startdate, enddate):
sql_ids = set()
deleted = doc_type == 'CommCareCase-Deleted'
for db_alias in get_sql_db_aliases_in_use():
queryset = CommCareCaseSQL.objects.using(db_alias) \
.filter(domain=domain, deleted=deleted)
if startdate:
queryset = queryset.filter(server_modified_on__gte=startdate)
if enddate:
queryset = queryset.filter(server_modified_on__lt=enddate)
sql_ids.update(list(queryset.values_list('case_id', flat=True)))
return sql_ids
def _compare_cases(domain, doc_type):
sql_ids = set()
for db_alias in get_sql_db_aliases_in_use():
queryset = CommCareCaseSQL.objects.using(db_alias) \
.filter(domain=domain, deleted=False).values_list('case_id', flat=True)
sql_ids.update(list(queryset))
es_ids = set(
es.CaseES()
.remove_default_filters()
.filter(es.filters.term('domain', domain))
.get_ids()
)
return sql_ids - es_ids
def get_sql_case_ids(domain, doc_type, startdate, enddate):
sql_ids = set()
deleted = doc_type == 'CommCareCase-Deleted'
for db_alias in get_sql_db_aliases_in_use():
queryset = CommCareCaseSQL.objects.using(db_alias) \
.filter(domain=domain, deleted=deleted)
if startdate:
queryset = queryset.filter(server_modified_on__gte=startdate)
if enddate:
queryset = queryset.filter(server_modified_on__lt=enddate)
sql_ids.update(list(queryset.values_list('case_id', flat=True)))
return sql_ids
def get_sql_form_ids(domain, doc_type, startdate, enddate):
sql_ids = set()
state = doc_type_to_state[doc_type]
for db_alias in get_sql_db_aliases_in_use():
queryset = XFormInstanceSQL.objects.using(db_alias) \
.filter(domain=domain, state=state) \
.exclude(xmlns=DEVICE_LOG_XMLNS)
if startdate:
queryset = queryset.filter(received_on__gte=startdate)
if enddate:
queryset = queryset.filter(received_on__lt=enddate)
sql_ids.update(list(queryset.values_list('form_id', flat=True)))
return sql_ids
def get_sql_form_ids(domain, doc_type, startdate, enddate):
sql_ids = set()
state = doc_type_to_state[doc_type]
for db_alias in get_sql_db_aliases_in_use():
queryset = XFormInstanceSQL.objects.using(db_alias) \
.filter(domain=domain, state=state) \
.exclude(xmlns=DEVICE_LOG_XMLNS)
if startdate:
queryset = queryset.filter(received_on__gte=startdate)
if enddate:
queryset = queryset.filter(received_on__lt=enddate)
sql_ids.update(list(queryset.values_list('form_id', flat=True)))
return sql_ids
def delete_all_v2_ledgers(domain=None):
logger.debug("Deleting all V2 ledgers for domain %s", domain)
def _delete_ledgers_for_case(case_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
if not domain:
for db in get_sql_db_aliases_in_use():
for ledger in LedgerReindexAccessor().get_docs(db, None, limit=10000):
_delete_ledgers_for_case(ledger.case_id)
else:
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
_delete_ledgers_for_case(case_id)
def delete_all_v2_ledgers(domain=None):
logger.debug("Deleting all V2 ledgers for domain %s", domain)
def _delete_ledgers_for_case(case_id):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
if not domain:
for db in get_sql_db_aliases_in_use():
for ledger in LedgerReindexAccessor().get_docs(db, None, limit=10000):
_delete_ledgers_for_case(ledger.case_id)
else:
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
_delete_ledgers_for_case(case_id)
def resumable_sql_model_iterator(iteration_key,
reindex_accessor,
chunk_size=100,
event_handler=None):
"""Perform one-time resumable iteration over documents
Iteration can be efficiently stopped and resumed. The iteration may
omit documents that are added after the iteration begins or resumes
and may include deleted documents.
:param iteration_key: A unique key identifying the iteration. This
key will be used in combination with `doc_types` to maintain state
about an iteration that is in progress. The state will be maintained
indefinitely unless it is removed with `discard_state()`.
:param reindex_accessor: A ``ReindexAccessor`` object.
:param chunk_size: Number of documents to yield before updating the
iteration checkpoint. In the worst case about this many documents
that were previously yielded may be yielded again if the iteration
is stopped and later resumed.
"""
def data_function(from_db, filter_value, last_id):
return reindex_accessor.get_docs(from_db,
filter_value,
last_id,
limit=chunk_size)
if reindex_accessor.is_sharded():
db_list = get_sql_db_aliases_in_use()
else:
db_list = ['default']
args_provider = SqlModelArgsProvider(
reindex_accessor.startkey_attribute_name,
reindex_accessor.startkey_min_value,
db_list,
)
class ResumableModelIterator(ResumableFunctionIterator):
def __iter__(self):
for doc in super(ResumableModelIterator, self).__iter__():
yield reindex_accessor.doc_to_json(doc)
item_getter = reindex_accessor.get_doc
return ResumableModelIterator(iteration_key, data_function, args_provider,
item_getter, event_handler)
def _compare_xforms(domain, doc_type):
sql_ids = set()
state = doc_type_to_state[doc_type]
for db_alias in get_sql_db_aliases_in_use():
queryset = XFormInstanceSQL.objects.using(db_alias) \
.filter(domain=domain, state=state).values_list('form_id', flat=True)
sql_ids.update(list(queryset))
es_ids = set(
es.FormES()
.remove_default_filters()
.filter(es.filters.term('domain', domain))
.filter(es.filters.term('doc_type', doc_type.lower()))
.get_ids()
)
return sql_ids - es_ids
def get_total_document_count(self):
return sum(
self.reindex_accessor.get_doc_count(from_db)
for from_db in get_sql_db_aliases_in_use())
def sql_db_aliases(self):
return get_sql_db_aliases_in_use() if self.is_sharded() else ['default']
def __init__(self, model_filter_attribute):
self.model_filter_attribute = model_filter_attribute
self.db_list = get_sql_db_aliases_in_use()
def __init__(self, model_filter_attribute):
self.model_filter_attribute = model_filter_attribute
self.db_list = get_sql_db_aliases_in_use()
def get_total_document_count(self):
return sum(
self.reindex_accessor.get_doc_count(from_db)
for from_db in get_sql_db_aliases_in_use()
)
