def _get_sql_counts(domain):
counter = Counter()
for model_class, queryset in get_querysets_to_dump(domain, []):
if model_class in (User, XFormInstanceSQL, CommCareCaseSQL):
continue # User is very slow, others we want to break out
counter[get_model_label(model_class)] += queryset.count()
counter += get_primary_db_form_counts(domain)
counter += get_primary_db_case_counts(domain)
return counter
def _get_sql_counts(domain):
counter = Counter()
for model_class, builder in get_model_iterator_builders_to_dump(domain, []):
if model_class in (User, XFormInstanceSQL, CommCareCaseSQL):
continue # User is very slow, others we want to break out
for queryset in builder.querysets():
counter[get_model_label(model_class)] += queryset.count()
counter += get_primary_db_form_counts(domain)
counter += get_primary_db_case_counts(domain)
return counter
def _get_primary_db_counts(domain):
db_counts = Counter()
db_counts.update(get_primary_db_form_counts(domain))
db_counts.update(get_primary_db_case_counts(domain))
mobile_user_count = get_mobile_user_count(domain)
db_counts.update({
'WebUser': get_web_user_count(domain),
'CommCareUser': mobile_user_count,
'CommCareUser-Deleted': get_doc_count_in_domain_by_class(domain, CommCareUser) - mobile_user_count
})
return db_counts
def _get_primary_db_counts(domain):
db_counts = Counter()
db_counts.update(get_primary_db_form_counts(domain))
db_counts.update(get_primary_db_case_counts(domain))
mobile_user_count = get_mobile_user_count(domain)
db_counts.update({
'WebUser':
get_web_user_count(domain),
'CommCareUser':
mobile_user_count,
'CommCareUser-Deleted':
get_doc_count_in_domain_by_class(domain, CommCareUser) -
mobile_user_count
})
return db_counts
def push_missing_docs_to_es():
if settings.SERVER_ENVIRONMENT not in settings.ICDS_ENVS:
return
current_date = date.today() - timedelta(weeks=12)
interval = timedelta(days=1)
case_doc_type = 'CommCareCase'
xform_doc_type = 'XFormInstance'
doc_differences = dict()
while current_date <= date.today() + interval:
end_date = current_date + interval
primary_xforms = get_primary_db_form_counts('icds-cas', current_date,
end_date).get(
xform_doc_type, -1)
es_xforms = get_es_counts_by_doc_type(
'icds-cas', (FormES, ),
(submitted(gte=current_date, lt=end_date), )).get(
xform_doc_type.lower(), -2)
if primary_xforms != es_xforms:
doc_differences[(current_date,
xform_doc_type)] = primary_xforms - es_xforms
resave_documents.delay(xform_doc_type, current_date, end_date)
primary_cases = get_primary_db_case_counts('icds-cas', current_date,
end_date).get(
case_doc_type, -1)
es_cases = get_es_counts_by_doc_type(
'icds-cas', (CaseES, ),
(server_modified_range(gte=current_date, lt=end_date), )).get(
case_doc_type, -2)
if primary_cases != es_cases:
doc_differences[(current_date,
case_doc_type)] = primary_xforms - es_xforms
resave_documents.delay(case_doc_type, current_date, end_date)
current_date += interval
if doc_differences:
message = "\n".join([
"{}, {}: {}".format(k[0], k[1], v)
for k, v in doc_differences.items()
])
send_mail_async.delay(
subject="Results from push_missing_docs_to_es",
message=message,
from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=["{}@{}.com".format("jmoney", "dimagi")])
def push_missing_docs_to_es():
if settings.SERVER_ENVIRONMENT not in settings.ICDS_ENVS:
return
current_date = date.today() - timedelta(weeks=12)
interval = timedelta(days=1)
case_doc_type = 'CommCareCase'
xform_doc_type = 'XFormInstance'
doc_differences = dict()
while current_date <= date.today() + interval:
end_date = current_date + interval
primary_xforms = get_primary_db_form_counts(
'icds-cas', current_date, end_date
).get(xform_doc_type, -1)
es_xforms = get_es_counts_by_doc_type(
'icds-cas', (FormES,), (submitted(gte=current_date, lt=end_date),)
).get(xform_doc_type.lower(), -2)
if primary_xforms != es_xforms:
doc_differences[(current_date, xform_doc_type)] = primary_xforms - es_xforms
primary_cases = get_primary_db_case_counts(
'icds-cas', current_date, end_date
).get(case_doc_type, -1)
es_cases = get_es_counts_by_doc_type(
'icds-cas', (CaseES,), (server_modified_range(gte=current_date, lt=end_date),)
).get(case_doc_type, -2)
if primary_cases != es_cases:
doc_differences[(current_date, case_doc_type)] = primary_xforms - es_xforms
current_date += interval
if doc_differences:
message = "\n".join([
"{}, {}: {}".format(k[0], k[1], v)
for k, v in doc_differences.items()
])
send_mail_async.delay(
subject="Results from push_missing_docs_to_es",
message=message,
from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=["{}@{}.com".format("jmoney", "dimagi")]
)
