def handle(self, domain, **options): csv = options.get('csv') couch_counts = map_counter_doc_types(_get_couchdb_counts(domain)) sql_counts = map_counter_doc_types(_get_sql_counts(domain)) es_counts = map_counter_doc_types(get_es_counts_by_doc_type(domain)) all_doc_types = set(couch_counts) | set(sql_counts) | set(es_counts) output_rows = [] for doc_type in sorted(all_doc_types, key=lambda d: d.lower()): couch = couch_counts.get(doc_type, '') sql = sql_counts.get(doc_type, '') es = es_counts.get(doc_type, '') output_rows.append((doc_type, couch, sql, es)) if csv: row_formatter = CSVRowFormatter() else: row_formatter = TableRowFormatter( [50, 20, 20, 20], _get_row_color ) SimpleTableWriter(self.stdout, row_formatter).write_table( ['Doc Type', 'Couch', 'SQL', 'ES'], output_rows )
def handle(self, domain, **options): csv = options.get('csv') couch_counts = map_counter_doc_types(_get_couchdb_counts(domain)) sql_counts = map_counter_doc_types(_get_sql_counts(domain)) es_counts = map_counter_doc_types(get_es_counts_by_doc_type(domain)) all_doc_types = set(couch_counts) | set(sql_counts) | set(es_counts) output_rows = [] for doc_type in sorted(all_doc_types, key=lambda d: d.lower()): couch = couch_counts.get(doc_type, '') sql = sql_counts.get(doc_type, '') es = es_counts.get(doc_type, '') output_rows.append((doc_type, couch, sql, es)) if csv: row_formatter = CSVRowFormatter() else: row_formatter = TableRowFormatter( [50, 20, 20, 20], _get_row_color ) SimpleTableWriter(self.stdout, row_formatter).write_table( ['Doc Type', 'Couch', 'SQL', 'ES'], output_rows )
def push_missing_docs_to_es(): if settings.SERVER_ENVIRONMENT not in settings.ICDS_ENVS: return current_date = date.today() - timedelta(weeks=12) interval = timedelta(days=1) case_doc_type = 'CommCareCase' xform_doc_type = 'XFormInstance' doc_differences = dict() while current_date <= date.today() + interval: end_date = current_date + interval primary_xforms = get_primary_db_form_counts('icds-cas', current_date, end_date).get( xform_doc_type, -1) es_xforms = get_es_counts_by_doc_type( 'icds-cas', (FormES, ), (submitted(gte=current_date, lt=end_date), )).get( xform_doc_type.lower(), -2) if primary_xforms != es_xforms: doc_differences[(current_date, xform_doc_type)] = primary_xforms - es_xforms resave_documents.delay(xform_doc_type, current_date, end_date) primary_cases = get_primary_db_case_counts('icds-cas', current_date, end_date).get( case_doc_type, -1) es_cases = get_es_counts_by_doc_type( 'icds-cas', (CaseES, ), (server_modified_range(gte=current_date, lt=end_date), )).get( case_doc_type, -2) if primary_cases != es_cases: doc_differences[(current_date, case_doc_type)] = primary_xforms - es_xforms resave_documents.delay(case_doc_type, current_date, end_date) current_date += interval if doc_differences: message = "\n".join([ "{}, {}: {}".format(k[0], k[1], v) for k, v in doc_differences.items() ]) send_mail_async.delay( subject="Results from push_missing_docs_to_es", message=message, from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=["{}@{}.com".format("jmoney", "dimagi")])
def push_missing_docs_to_es(): if settings.SERVER_ENVIRONMENT not in settings.ICDS_ENVS: return current_date = date.today() - timedelta(weeks=12) interval = timedelta(days=1) case_doc_type = 'CommCareCase' xform_doc_type = 'XFormInstance' doc_differences = dict() while current_date <= date.today() + interval: end_date = current_date + interval primary_xforms = get_primary_db_form_counts( 'icds-cas', current_date, end_date ).get(xform_doc_type, -1) es_xforms = get_es_counts_by_doc_type( 'icds-cas', (FormES,), (submitted(gte=current_date, lt=end_date),) ).get(xform_doc_type.lower(), -2) if primary_xforms != es_xforms: doc_differences[(current_date, xform_doc_type)] = primary_xforms - es_xforms primary_cases = get_primary_db_case_counts( 'icds-cas', current_date, end_date ).get(case_doc_type, -1) es_cases = get_es_counts_by_doc_type( 'icds-cas', (CaseES,), (server_modified_range(gte=current_date, lt=end_date),) ).get(case_doc_type, -2) if primary_cases != es_cases: doc_differences[(current_date, case_doc_type)] = primary_xforms - es_xforms current_date += interval if doc_differences: message = "\n".join([ "{}, {}: {}".format(k[0], k[1], v) for k, v in doc_differences.items() ]) send_mail_async.delay( subject="Results from push_missing_docs_to_es", message=message, from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=["{}@{}.com".format("jmoney", "dimagi")] )
def get_doc_counts_for_domain(domain): """ :param domain: :return: List of tuples: ``(doc_type, primary_db_count, es_count)`` """ primary_db_counts = map_counter_doc_types(_get_primary_db_counts(domain)) es_counts = map_counter_doc_types( get_es_counts_by_doc_type(domain, (es.CaseES, es.FormES))) es_counts.update(get_es_user_counts_by_doc_type(domain)) all_doc_types = set(primary_db_counts) | set(es_counts) output_rows = [] for doc_type in sorted(all_doc_types, key=lambda d: d.lower()): output_rows.append( (doc_type, primary_db_counts[doc_type], es_counts[doc_type])) return output_rows
def get_doc_counts_for_domain(domain): """ :param domain: :return: List of tuples: ``(doc_type, primary_db_count, es_count)`` """ primary_db_counts = map_counter_doc_types(_get_primary_db_counts(domain)) es_counts = map_counter_doc_types( get_es_counts_by_doc_type(domain, (es.CaseES, es.FormES)) ) es_counts.update(get_es_user_counts_by_doc_type(domain)) all_doc_types = set(primary_db_counts) | set(es_counts) output_rows = [] for doc_type in sorted(all_doc_types, key=lambda d: d.lower()): output_rows.append(( doc_type, primary_db_counts[doc_type], es_counts[doc_type] )) return output_rows