def handle(self, domain, **options):
csv = options.get('csv')
couch_counts = map_counter_doc_types(_get_couchdb_counts(domain))
sql_counts = map_counter_doc_types(_get_sql_counts(domain))
es_counts = map_counter_doc_types(get_es_counts_by_doc_type(domain))
all_doc_types = set(couch_counts) | set(sql_counts) | set(es_counts)
output_rows = []
for doc_type in sorted(all_doc_types, key=lambda d: d.lower()):
couch = couch_counts.get(doc_type, '')
sql = sql_counts.get(doc_type, '')
es = es_counts.get(doc_type, '')
output_rows.append((doc_type, couch, sql, es))
if csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter(
[50, 20, 20, 20],
_get_row_color
)
SimpleTableWriter(self.stdout, row_formatter).write_table(
['Doc Type', 'Couch', 'SQL', 'ES'], output_rows
)
def handle(self, domain, **options):
csv = options.get('csv')
couch_counts = map_counter_doc_types(_get_couchdb_counts(domain))
sql_counts = map_counter_doc_types(_get_sql_counts(domain))
es_counts = map_counter_doc_types(get_es_counts_by_doc_type(domain))
all_doc_types = set(couch_counts) | set(sql_counts) | set(es_counts)
output_rows = []
for doc_type in sorted(all_doc_types, key=lambda d: d.lower()):
couch = couch_counts.get(doc_type, '')
sql = sql_counts.get(doc_type, '')
es = es_counts.get(doc_type, '')
output_rows.append((doc_type, couch, sql, es))
if csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter(
[50, 20, 20, 20],
_get_row_color
)
SimpleTableWriter(self.stdout, row_formatter).write_table(
['Doc Type', 'Couch', 'SQL', 'ES'], output_rows
)
def push_missing_docs_to_es():
if settings.SERVER_ENVIRONMENT not in settings.ICDS_ENVS:
return
current_date = date.today() - timedelta(weeks=12)
interval = timedelta(days=1)
case_doc_type = 'CommCareCase'
xform_doc_type = 'XFormInstance'
doc_differences = dict()
while current_date <= date.today() + interval:
end_date = current_date + interval
primary_xforms = get_primary_db_form_counts('icds-cas', current_date,
end_date).get(
xform_doc_type, -1)
es_xforms = get_es_counts_by_doc_type(
'icds-cas', (FormES, ),
(submitted(gte=current_date, lt=end_date), )).get(
xform_doc_type.lower(), -2)
if primary_xforms != es_xforms:
doc_differences[(current_date,
xform_doc_type)] = primary_xforms - es_xforms
resave_documents.delay(xform_doc_type, current_date, end_date)
primary_cases = get_primary_db_case_counts('icds-cas', current_date,
end_date).get(
case_doc_type, -1)
es_cases = get_es_counts_by_doc_type(
'icds-cas', (CaseES, ),
(server_modified_range(gte=current_date, lt=end_date), )).get(
case_doc_type, -2)
if primary_cases != es_cases:
doc_differences[(current_date,
case_doc_type)] = primary_xforms - es_xforms
resave_documents.delay(case_doc_type, current_date, end_date)
current_date += interval
if doc_differences:
message = "\n".join([
"{}, {}: {}".format(k[0], k[1], v)
for k, v in doc_differences.items()
])
send_mail_async.delay(
subject="Results from push_missing_docs_to_es",
message=message,
from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=["{}@{}.com".format("jmoney", "dimagi")])
def push_missing_docs_to_es():
if settings.SERVER_ENVIRONMENT not in settings.ICDS_ENVS:
return
current_date = date.today() - timedelta(weeks=12)
interval = timedelta(days=1)
case_doc_type = 'CommCareCase'
xform_doc_type = 'XFormInstance'
doc_differences = dict()
while current_date <= date.today() + interval:
end_date = current_date + interval
primary_xforms = get_primary_db_form_counts(
'icds-cas', current_date, end_date
).get(xform_doc_type, -1)
es_xforms = get_es_counts_by_doc_type(
'icds-cas', (FormES,), (submitted(gte=current_date, lt=end_date),)
).get(xform_doc_type.lower(), -2)
if primary_xforms != es_xforms:
doc_differences[(current_date, xform_doc_type)] = primary_xforms - es_xforms
primary_cases = get_primary_db_case_counts(
'icds-cas', current_date, end_date
).get(case_doc_type, -1)
es_cases = get_es_counts_by_doc_type(
'icds-cas', (CaseES,), (server_modified_range(gte=current_date, lt=end_date),)
).get(case_doc_type, -2)
if primary_cases != es_cases:
doc_differences[(current_date, case_doc_type)] = primary_xforms - es_xforms
current_date += interval
if doc_differences:
message = "\n".join([
"{}, {}: {}".format(k[0], k[1], v)
for k, v in doc_differences.items()
])
send_mail_async.delay(
subject="Results from push_missing_docs_to_es",
message=message,
from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=["{}@{}.com".format("jmoney", "dimagi")]
)
def get_doc_counts_for_domain(domain):
"""
:param domain:
:return: List of tuples: ``(doc_type, primary_db_count, es_count)``
"""
primary_db_counts = map_counter_doc_types(_get_primary_db_counts(domain))
es_counts = map_counter_doc_types(
get_es_counts_by_doc_type(domain, (es.CaseES, es.FormES)))
es_counts.update(get_es_user_counts_by_doc_type(domain))
all_doc_types = set(primary_db_counts) | set(es_counts)
output_rows = []
for doc_type in sorted(all_doc_types, key=lambda d: d.lower()):
output_rows.append(
(doc_type, primary_db_counts[doc_type], es_counts[doc_type]))
return output_rows
def get_doc_counts_for_domain(domain):
"""
:param domain:
:return: List of tuples: ``(doc_type, primary_db_count, es_count)``
"""
primary_db_counts = map_counter_doc_types(_get_primary_db_counts(domain))
es_counts = map_counter_doc_types(
get_es_counts_by_doc_type(domain, (es.CaseES, es.FormES))
)
es_counts.update(get_es_user_counts_by_doc_type(domain))
all_doc_types = set(primary_db_counts) | set(es_counts)
output_rows = []
for doc_type in sorted(all_doc_types, key=lambda d: d.lower()):
output_rows.append((
doc_type,
primary_db_counts[doc_type],
es_counts[doc_type]
))
return output_rows
