def set_cleanliness_flags_for_enabled_domains(force_full=False): """ Updates cleanliness for all domains that have the toggle enabled """ for domain in Domain.get_all_names(): if OWNERSHIP_CLEANLINESS.enabled(domain): set_cleanliness_flags_for_domain(domain, force_full=force_full)
def handle(self, *args, **options): for domain in Domain.get_all_names(): fields_definition = CustomDataFieldsDefinition.get_or_create( domain, 'UserFields' ) user_ids = (CommCareUser.ids_by_domain(domain) + CommCareUser.ids_by_domain(domain, is_active=False)) existing_field_slugs = set([field.slug for field in fields_definition.fields]) for user in iter_docs(CommCareUser.get_db(), user_ids): user_data = user.get('user_data', {}) for key in user_data.keys(): if key and key not in existing_field_slugs: existing_field_slugs.add(key) fields_definition.fields.append(CustomDataField( slug=key, label=key, is_required=False )) # Only save a definition for domains which use custom user data if fields_definition.fields: fields_definition.save()
def handle(self, *args, **options): for domain in Domain.get_all_names(): fields_definition = cdm.CustomDataFieldsDefinition.get_or_create( domain, 'UserFields') had_fields = bool(fields_definition.fields) user_ids = (CommCareUser.ids_by_domain(domain) + CommCareUser.ids_by_domain(domain, is_active=False)) existing_field_slugs = set( [field.slug for field in fields_definition.fields]) for user in iter_docs(CommCareUser.get_db(), user_ids): user_data = user.get('user_data', {}) for key in user_data.keys(): if (key and key not in existing_field_slugs and not cdm.is_system_key(key)): existing_field_slugs.add(key) fields_definition.fields.append( cdm.CustomDataField( slug=key, label=key, is_required=False, )) for field in fields_definition.fields: if cdm.is_system_key(field.slug): fields_definition.fields.remove(field) # Only save a definition for domains which use custom user data if fields_definition.fields or had_fields: fields_definition.save() print 'finished domain "{}"'.format(domain)
def handle(self, *args, **options): for domain in Domain.get_all_names(): fields_definition = cdm.CustomDataFieldsDefinition.get_or_create( domain, 'UserFields' ) had_fields = bool(fields_definition.fields) user_ids = (CommCareUser.ids_by_domain(domain) + CommCareUser.ids_by_domain(domain, is_active=False)) existing_field_slugs = set([field.slug for field in fields_definition.fields]) for user in iter_docs(CommCareUser.get_db(), user_ids): user_data = user.get('user_data', {}) for key in user_data.keys(): if (key and key not in existing_field_slugs and not cdm.is_system_key(key)): existing_field_slugs.add(key) fields_definition.fields.append(cdm.CustomDataField( slug=key, label=key, is_required=False, )) for field in fields_definition.fields: if cdm.is_system_key(field.slug): fields_definition.fields.remove(field) # Only save a definition for domains which use custom user data if fields_definition.fields or had_fields: fields_definition.save() print 'finished domain "{}"'.format(domain.name)
def handle(self, *args, **options): for domain in Domain.get_all_names(): fields_definition = cdm.CustomDataFieldsDefinition.get_or_create( domain, 'LocationFields') had_fields = bool(fields_definition.fields) existing_field_slugs = set( [field.slug for field in fields_definition.fields]) for location in Location.by_domain(domain): location_data = location.metadata for key in location_data.keys(): if (key and key not in existing_field_slugs and not cdm.is_system_key(key)): existing_field_slugs.add(key) fields_definition.fields.append( cdm.CustomDataField( slug=key, label=key, is_required=False, )) for field in fields_definition.fields: if cdm.is_system_key(field.slug): fields_definition.fields.remove(field) # Only save a definition for domains which use custom location data if fields_definition.fields or had_fields: fields_definition.save() print 'finished domain "{}"'.format(domain)
def _copy(config): # unfortunately the only couch view we have for this needs to go by domain # will be a bit slow database = Domain.get_db() assert database.uri == config.source_db.uri, 'can only use "copy" with the main HQ DB as the source' domain_names = Domain.get_all_names() for domain in domain_names: for doc_type in config.doc_types: ids_of_this_type = [row['id'] for row in database.view( 'domain/docs', startkey=[domain, doc_type], endkey=[domain, doc_type, {}], reduce=False, include_docs=False, )] if ids_of_this_type: new_revs = dict([ (row['id'], row['value']['rev']) for row in config.dest_db.view('_all_docs', keys=ids_of_this_type, include_docs=False) if 'error' not in row ]) for id_group in chunked(ids_of_this_type, 500): docs = get_docs(database, id_group) for doc in docs: if doc['_id'] in new_revs: doc['_rev'] = new_revs[doc['_id']] config.dest_db.bulk_save(docs) print 'copied {} {}s from {}'.format(len(ids_of_this_type), doc_type, domain) print 'copy docs complete'
def handle(self, *args, **options): for domain in Domain.get_all_names(): fields_definition = cdm.CustomDataFieldsDefinition.get_or_create( domain, 'LocationFields' ) had_fields = bool(fields_definition.fields) existing_field_slugs = set([field.slug for field in fields_definition.fields]) for location in Location.by_domain(domain): location_data = location.metadata for key in location_data.keys(): if (key and key not in existing_field_slugs and not cdm.is_system_key(key)): existing_field_slugs.add(key) fields_definition.fields.append(cdm.CustomDataField( slug=key, label=key, is_required=False, )) for field in fields_definition.fields: if cdm.is_system_key(field.slug): fields_definition.fields.remove(field) # Only save a definition for domains which use custom location data if fields_definition.fields or had_fields: fields_definition.save() print 'finished domain "{}"'.format(domain)
def calculate_users_in_all_domains(): for domain in Domain.get_all_names(): num_users = CommCareUser.total_by_domain(domain) record_date = datetime.date.today() - relativedelta(days=1) user_history = DomainUserHistory.create(domain=domain, num_users=num_users, record_date=record_date) user_history.save()
def handle(self, **options): all_domains = Domain.get_all_names() total = len(all_domains) finished = 0 for domain in all_domains: update_supply_points(domain) finished += 1 if finished % 100 == 0: print("Processed {} of {} domains".format(finished, total))
def set_cleanliness_flags_for_all_domains(force_full=False): """ Updates cleanliness for all domains """ for domain in Domain.get_all_names(): try: set_cleanliness_flags_for_domain(domain, force_full=force_full) except InvalidDomainError as e: notify_exception(None, unicode(e))
def handle(self, *args, **options): all_domains = Domain.get_all_names() total = len(all_domains) finished = 0 for domain in all_domains: update_supply_points(domain) finished += 1 if finished % 100 == 0: print "Processed {} of {} domains".format(finished, total)
def warn_subscriptions_without_domain(): domains_with_active_subscription = Subscription.visible_objects.filter( is_active=True, ).values_list('subscriber__domain', flat=True).distinct() for domain_name in set(domains_with_active_subscription) - set( Domain.get_all_names()): log_accounting_error( 'Domain %s has an active subscription but does not exist.' % domain_name)
def set_cleanliness_flags_for_enabled_domains(force_full=False): """ Updates cleanliness for all domains that have the toggle enabled """ for domain in Domain.get_all_names(): if OWNERSHIP_CLEANLINESS.enabled(domain): try: set_cleanliness_flags_for_domain(domain, force_full=force_full) except InvalidDomainError as e: notify_exception(None, unicode(e))
def calculate_users_in_all_domains(): for domain in Domain.get_all_names(): num_users = CommCareUser.total_by_domain(domain) record_date = datetime.date.today() - relativedelta(days=1) user_history = DomainUserHistory.create( domain=domain, num_users=num_users, record_date=record_date ) user_history.save()
def calculate_users_in_all_domains(today=None): today = today or datetime.date.today() for domain in Domain.get_all_names(): num_users = CommCareUser.total_by_domain(domain) record_date = today - relativedelta(days=1) DomainUserHistory.objects.create( domain=domain, num_users=num_users, record_date=record_date )
def warn_active_subscriptions_per_domain_not_one(): for domain_name in Domain.get_all_names(): active_subscription_count = Subscription.visible_objects.filter( subscriber__domain=domain_name, is_active=True, ).count() if active_subscription_count > 1: log_accounting_error("Multiple active subscriptions found for domain %s" % domain_name) elif active_subscription_count == 0 and Domain.get_by_name(domain_name).is_active: log_accounting_error("There is no active subscription for domain %s" % domain_name)
def warn_active_subscriptions_per_domain_not_one(): for domain_name in Domain.get_all_names(): active_subscription_count = Subscription.objects.filter( subscriber__domain=domain_name, is_active=True, ).count() if active_subscription_count > 1: log_accounting_error("Multiple active subscriptions found for domain %s" % domain_name) elif active_subscription_count == 0: log_accounting_error("There is no active subscription for domain %s" % domain_name)
def handle(self, doc_types, *args, **options): input = raw_input( "\n".join( [ "\n\nReally delete documents of the following types: {}?", "This operation is not reversible. Enter a number N to delete the first " 'N found, or type "delete all" to delete everything.', "", ] ).format(doc_types) ) if input == "delete all": remaining = None else: try: remaining = int(input) except ValueError: print "aborting" sys.exit() doc_types = doc_types.split(",") deleted = 0 # unfortunately the only couch view we have for this needs to go by domain # will be a bit slow domain_names = Domain.get_all_names() for doc_type in doc_types: db = get_db_by_doc_type(doc_type) if not db: print "Cannot find db for {}, skipping".format(doc_type) continue for domain in domain_names: docs = [ row["doc"] for row in db.view( "by_domain_doc_type_date/view", startkey=[domain, doc_type], endkey=[domain, doc_type, {}], reduce=False, include_docs=True, ) ][:remaining] if docs: count = len(docs) print "deleting {} {}s from {}".format(count, doc_type, domain) db.delete_docs(docs) deleted += count if remaining is not None: remaining -= count if remaining <= 0: return print "successfully deleted {} documents".format(deleted)
def build_last_month_MALT(): last_month = last_month_dict() domains = Domain.get_all_names() task_results = [] for chunk in chunked(domains, 1000): task_results.append(update_malt.delay(last_month, chunk)) for result in task_results: result.get(disable_sync_subtasks=False) send_MALT_complete_email(last_month)
def handle(self, *args, **kwargs): from casexml.apps.phone.cleanliness import set_cleanliness_flags_for_domain if len(args) == 1: domain = args[0] set_cleanliness_flags_for_domain(domain) else: assert len(args) == 0 for domain in Domain.get_all_names(): if OWNERSHIP_CLEANLINESS.enabled(domain): print 'updating flags for {}'.format(domain) set_cleanliness_flags_for_domain(domain)
def warn_subscriptions_without_domain(): domains_with_active_subscription = Subscription.visible_objects.filter( is_active=True, ).values_list('subscriber__domain', flat=True).distinct() for domain_name in set(domains_with_active_subscription) - set(Domain.get_all_names()): # we need to put a try/except here so that sentry captures logging try: raise ActiveSubscriptionWithoutDomain() except ActiveSubscriptionWithoutDomain: log_accounting_error( f'Domain {domain_name} has an active subscription but does not exist.' )
def handle(self, doc_types, *args, **options): user_input = input('\n'.join([ '\n\nReally delete documents of the following types: {}?', 'This operation is not reversible. Enter a number N to delete the first ' 'N found, or type "delete all" to delete everything.', '', ]).format(doc_types)) if user_input == 'delete all': remaining = None else: try: remaining = int(user_input) except ValueError: print('aborting') sys.exit() doc_types = doc_types.split(',') deleted = 0 # unfortunately the only couch view we have for this needs to go by domain # will be a bit slow domain_names = Domain.get_all_names() for doc_type in doc_types: db = get_db_by_doc_type(doc_type) if not db: print("Cannot find db for {}, skipping".format(doc_type)) continue for domain in domain_names: docs = [ row['doc'] for row in db.view( 'by_domain_doc_type_date/view', startkey=[domain, doc_type], endkey=[domain, doc_type, {}], reduce=False, include_docs=True, ) ][:remaining] if docs: count = len(docs) print('deleting {} {}s from {}'.format( count, doc_type, domain)) db.delete_docs(docs) deleted += count if remaining is not None: remaining -= count if remaining <= 0: return print('successfully deleted {} documents'.format(deleted))
def check_for_sql_cases_without_existing_domain(): missing_domains_with_cases = set() for domain in set( _get_all_domains_that_have_ever_had_subscriptions()) - set( Domain.get_all_names()): if CaseAccessorSQL.get_case_ids_in_domain(domain): missing_domains_with_cases |= {domain} if missing_domains_with_cases: mail_admins_async.delay( 'There exist SQL cases belonging to a missing domain', six.text_type(missing_domains_with_cases)) elif _is_monday(): mail_admins_async.delay('All SQL cases belong to valid domains', '')
def check_for_sql_cases_without_existing_domain(): missing_domains_with_cases = set() for domain in set(_get_all_domains_that_have_ever_had_subscriptions()) - set(Domain.get_all_names()): if CaseAccessorSQL.get_case_ids_in_domain(domain): missing_domains_with_cases |= {domain} if missing_domains_with_cases: mail_admins_async.delay( 'There exist SQL cases belonging to a missing domain', six.text_type(missing_domains_with_cases) ) elif _is_monday(): mail_admins_async.delay( 'All SQL cases belong to valid domains', '' )
def calculate_users_in_all_domains(today=None): today = today or datetime.date.today() for domain in Domain.get_all_names(): num_users = CommCareUser.total_by_domain(domain) record_date = today - relativedelta(days=1) try: DomainUserHistory.objects.create(domain=domain, num_users=num_users, record_date=record_date) except Exception as e: log_accounting_error( "Something went wrong while creating DomainUserHistory for domain %s: %s" % (domain, e), show_stack_trace=True, )
def check_for_sql_forms_without_existing_domain(): missing_domains_with_forms = set() for domain in set(_get_all_domains_that_have_ever_had_subscriptions()) - set(Domain.get_all_names()): for doc_type in doc_type_to_state: if FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type): missing_domains_with_forms |= {domain} if missing_domains_with_forms: mail_admins_async.delay( 'There exist SQL forms belonging to a missing domain', six.text_type(missing_domains_with_forms) ) elif _is_monday(): mail_admins_async.delay( 'All SQL forms belong to valid domains', '' )
def check_for_sql_forms_without_existing_domain(): missing_domains_with_forms = set() for domain in set( _get_all_domains_that_have_ever_had_subscriptions()) - set( Domain.get_all_names()): for doc_type in doc_type_to_state: if FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type): missing_domains_with_forms |= {domain} if missing_domains_with_forms: mail_admins_async.delay( 'There exist SQL forms belonging to a missing domain', six.text_type(missing_domains_with_forms)) elif _is_monday(): mail_admins_async.delay('All SQL forms belong to valid domains', '')
def check_for_sql_forms_without_existing_domain(): missing_domains_with_forms = set() for domain in set( _get_all_domains_that_have_ever_had_subscriptions()) - set( Domain.get_all_names()): accessor = FormReindexAccessor(domain=domain, include_deleted=True) for db_name in accessor.sql_db_aliases: if _has_docs(accessor, db_name): missing_domains_with_forms |= {domain} break if missing_domains_with_forms: mail_admins_async.delay( 'There exist SQL forms belonging to a missing domain', str(missing_domains_with_forms)) elif _is_monday(): mail_admins_async.delay('All SQL forms belong to valid domains', '')
def check_for_elasticsearch_data_without_existing_domain(): issue_found = False deleted_domain_names = set( _get_all_domains_that_have_ever_had_subscriptions()) - set( Domain.get_all_names()) for domain_name in deleted_domain_names: for hqESQuery in [ AppES, CaseES, CaseSearchES, FormES, GroupES, LedgerES, UserES ]: query = hqESQuery().domain(domain_name) count = query.count() if query.count() != 0: issue_found = True mail_admins_async.delay( 'ES index "%s" contains %s items belonging to missing domain "%s"' % (query.index, count, domain_name), '') if not issue_found and _is_monday(): mail_admins_async.delay('All data in ES belongs to valid domains', '')
def handle(self, doc_types, *args, **options): input = raw_input('\n'.join([ '\n\nReally delete documents of the following types: {}?', 'This operation is not reversible. Enter a number N to delete the first ' 'N found, or type "delete all" to delete everything.', '', ]).format(doc_types)) if input == 'delete all': remaining = None else: try: remaining = int(input) except ValueError: print 'aborting' sys.exit() doc_types = doc_types.split(',') deleted = 0 # unfortunately the only couch view we have for this needs to go by domain # will be a bit slow domain_names = Domain.get_all_names() database = Domain.get_db() for domain in domain_names: for doc_type in doc_types: docs = [row['doc'] for row in database.view( 'domain/docs', startkey=[domain, doc_type], endkey=[domain, doc_type, {}], reduce=False, include_docs=True, )][:remaining] if docs: count = len(docs) print 'deleting {} {}s from {}'.format(count, doc_type, domain) database.delete_docs(docs) deleted += count if remaining is not None: remaining -= count if remaining <= 0: return print 'successfully deleted {} documents'.format(deleted)
def calculate_users_in_all_domains(today=None): today = today or datetime.date.today() for domain in Domain.get_all_names(): num_users = CommCareUser.total_by_domain(domain) record_date = today - relativedelta(days=1) try: DomainUserHistory.objects.create(domain=domain, num_users=num_users, record_date=record_date) except Exception as e: log_accounting_error( "Something went wrong while creating DomainUserHistory for domain %s: %s" % (domain, e), show_stack_trace=True, ) # kick off the auto-deactivation of mobile workers after we calculate the # DomainUserHistory for projects. This ensures this feature is never abused # to get around our billing system. from corehq.apps.enterprise.tasks import auto_deactivate_mobile_workers auto_deactivate_mobile_workers.delay()
def warn_active_subscriptions_per_domain_not_one(): for domain_name in Domain.get_all_names(): active_subscription_count = Subscription.visible_objects.filter( subscriber__domain=domain_name, is_active=True, ).count() # we need to put a try/except here so that sentry captures logging try: if active_subscription_count > 1: raise MultipleActiveSubscriptionsError() elif active_subscription_count == 0 and Domain.get_by_name(domain_name).is_active: raise NoActiveSubscriptionError() except NoActiveSubscriptionError: log_accounting_error( f"There is no active subscription for domain {domain_name}" ) except MultipleActiveSubscriptionsError: log_accounting_error( f"Multiple active subscriptions found for domain {domain_name}" )
def check_for_ucr_tables_without_existing_domain(): all_domain_names = Domain.get_all_names() connection_name = ConnectionManager().get_django_db_alias(UCR_ENGINE_ID) table_names = connections[connection_name].introspection.table_names() ucr_table_names = [name for name in table_names if name.startswith('config_report')] missing_domains_to_tables = defaultdict(list) for ucr_table_name in ucr_table_names: table_domain = ucr_table_name.split('_')[2] if table_domain not in all_domain_names: missing_domains_to_tables[table_domain].append(ucr_table_name) if missing_domains_to_tables: for missing_domain in missing_domains_to_tables: mail_admins_async.delay( 'Missing domain "%s" has remaining UCR tables' % missing_domain, six.text_type(missing_domains_to_tables[missing_domain]) ) elif _is_monday(): mail_admins_async.delay('All UCR tables belong to valid domains', '')
def generate_malt(monthspans, domains=None): """ Populates MALTRow SQL table with app submission data for a given list of months :param monthspans: list of DateSpan objects :param domains: list of domain names """ domain_names = domains or Domain.get_all_names() for domain_name in domain_names: last_submission_date = get_last_form_submission_received(domain_name) last_malt_run_dates_by_month = _get_last_run_date_for_malt_by_month(domain_name, monthspans) for monthspan in monthspans: # if the MALTRow last_run_date is none, use the start date of the month last_malt_run_date = last_malt_run_dates_by_month.get(monthspan.startdate.date(), monthspan.startdate) if last_submission_date and last_submission_date >= last_malt_run_date: # use this date to populate last_run_date for all MALTRows with this domain and month run_date = datetime.datetime.utcnow() logger.info(f"Building MALT for {domain_name} for {monthspan} up to {run_date}") all_users = get_all_user_rows(domain_name, include_inactive=False, include_docs=True) for users in chunked(all_users, 1000): users_by_id = {user['id']: CouchUser.wrap_correctly(user['doc']) for user in users} malt_row_dicts = _get_malt_row_dicts(domain_name, monthspan, users_by_id, run_date) if malt_row_dicts: _save_malt_row_dicts_to_db(malt_row_dicts)
def check_for_elasticsearch_data_without_existing_domain(): issue_found = False deleted_domain_names = set(_get_all_domains_that_have_ever_had_subscriptions()) - set(Domain.get_all_names()) for domain_name in deleted_domain_names: for hqESQuery in [AppES, CaseES, CaseSearchES, FormES, GroupES, LedgerES, UserES]: query = hqESQuery().domain(domain_name) count = query.count() if query.count() != 0: issue_found = True mail_admins_async.delay( 'ES index "%s" contains %s items belonging to missing domain "%s"' % ( query.index, count, domain_name ), '' ) if not issue_found and _is_monday(): mail_admins_async.delay( 'All data in ES belongs to valid domains', '' )
def warn_subscriptions_without_domain(): domains_with_active_subscription = Subscription.visible_objects.filter( is_active=True, ).values_list('subscriber__domain', flat=True).distinct() for domain_name in set(domains_with_active_subscription) - set(Domain.get_all_names()): log_accounting_error('Domain %s has an active subscription but does not exist.' % domain_name)
def update_current_MALT(): today = datetime.date.today() this_month_dict = {'month': today.month, 'year': today.year} domains = Domain.get_all_names() for chunk in chunked(domains, 1000): update_malt.delay(this_month_dict, chunk)
def _get_missing_domains(): return set(_get_all_domains_that_have_ever_had_subscriptions()) - set( Domain.get_all_names())