def set_cleanliness_flags_for_enabled_domains(force_full=False):
"""
Updates cleanliness for all domains that have the toggle enabled
"""
for domain in Domain.get_all_names():
if OWNERSHIP_CLEANLINESS.enabled(domain):
set_cleanliness_flags_for_domain(domain, force_full=force_full)
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = CustomDataFieldsDefinition.get_or_create(
domain,
'UserFields'
)
user_ids = (CommCareUser.ids_by_domain(domain) +
CommCareUser.ids_by_domain(domain, is_active=False))
existing_field_slugs = set([field.slug for field in fields_definition.fields])
for user in iter_docs(CommCareUser.get_db(), user_ids):
user_data = user.get('user_data', {})
for key in user_data.keys():
if key and key not in existing_field_slugs:
existing_field_slugs.add(key)
fields_definition.fields.append(CustomDataField(
slug=key,
label=key,
is_required=False
))
# Only save a definition for domains which use custom user data
if fields_definition.fields:
fields_definition.save()
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = cdm.CustomDataFieldsDefinition.get_or_create(
domain, 'UserFields')
had_fields = bool(fields_definition.fields)
user_ids = (CommCareUser.ids_by_domain(domain) +
CommCareUser.ids_by_domain(domain, is_active=False))
existing_field_slugs = set(
[field.slug for field in fields_definition.fields])
for user in iter_docs(CommCareUser.get_db(), user_ids):
user_data = user.get('user_data', {})
for key in user_data.keys():
if (key and key not in existing_field_slugs
and not cdm.is_system_key(key)):
existing_field_slugs.add(key)
fields_definition.fields.append(
cdm.CustomDataField(
slug=key,
label=key,
is_required=False,
))
for field in fields_definition.fields:
if cdm.is_system_key(field.slug):
fields_definition.fields.remove(field)
# Only save a definition for domains which use custom user data
if fields_definition.fields or had_fields:
fields_definition.save()
print 'finished domain "{}"'.format(domain)
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = cdm.CustomDataFieldsDefinition.get_or_create(
domain,
'UserFields'
)
had_fields = bool(fields_definition.fields)
user_ids = (CommCareUser.ids_by_domain(domain) +
CommCareUser.ids_by_domain(domain, is_active=False))
existing_field_slugs = set([field.slug for field in fields_definition.fields])
for user in iter_docs(CommCareUser.get_db(), user_ids):
user_data = user.get('user_data', {})
for key in user_data.keys():
if (key and key not in existing_field_slugs
and not cdm.is_system_key(key)):
existing_field_slugs.add(key)
fields_definition.fields.append(cdm.CustomDataField(
slug=key,
label=key,
is_required=False,
))
for field in fields_definition.fields:
if cdm.is_system_key(field.slug):
fields_definition.fields.remove(field)
# Only save a definition for domains which use custom user data
if fields_definition.fields or had_fields:
fields_definition.save()
print 'finished domain "{}"'.format(domain.name)
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = cdm.CustomDataFieldsDefinition.get_or_create(
domain, 'LocationFields')
had_fields = bool(fields_definition.fields)
existing_field_slugs = set(
[field.slug for field in fields_definition.fields])
for location in Location.by_domain(domain):
location_data = location.metadata
for key in location_data.keys():
if (key and key not in existing_field_slugs
and not cdm.is_system_key(key)):
existing_field_slugs.add(key)
fields_definition.fields.append(
cdm.CustomDataField(
slug=key,
label=key,
is_required=False,
))
for field in fields_definition.fields:
if cdm.is_system_key(field.slug):
fields_definition.fields.remove(field)
# Only save a definition for domains which use custom location data
if fields_definition.fields or had_fields:
fields_definition.save()
print 'finished domain "{}"'.format(domain)
def _copy(config):
# unfortunately the only couch view we have for this needs to go by domain
# will be a bit slow
database = Domain.get_db()
assert database.uri == config.source_db.uri, 'can only use "copy" with the main HQ DB as the source'
domain_names = Domain.get_all_names()
for domain in domain_names:
for doc_type in config.doc_types:
ids_of_this_type = [row['id'] for row in database.view(
'domain/docs',
startkey=[domain, doc_type],
endkey=[domain, doc_type, {}],
reduce=False,
include_docs=False,
)]
if ids_of_this_type:
new_revs = dict([
(row['id'], row['value']['rev'])
for row in config.dest_db.view('_all_docs', keys=ids_of_this_type, include_docs=False)
if 'error' not in row
])
for id_group in chunked(ids_of_this_type, 500):
docs = get_docs(database, id_group)
for doc in docs:
if doc['_id'] in new_revs:
doc['_rev'] = new_revs[doc['_id']]
config.dest_db.bulk_save(docs)
print 'copied {} {}s from {}'.format(len(ids_of_this_type), doc_type, domain)
print 'copy docs complete'
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = cdm.CustomDataFieldsDefinition.get_or_create(
domain,
'LocationFields'
)
had_fields = bool(fields_definition.fields)
existing_field_slugs = set([field.slug for field in fields_definition.fields])
for location in Location.by_domain(domain):
location_data = location.metadata
for key in location_data.keys():
if (key and key not in existing_field_slugs
and not cdm.is_system_key(key)):
existing_field_slugs.add(key)
fields_definition.fields.append(cdm.CustomDataField(
slug=key,
label=key,
is_required=False,
))
for field in fields_definition.fields:
if cdm.is_system_key(field.slug):
fields_definition.fields.remove(field)
# Only save a definition for domains which use custom location data
if fields_definition.fields or had_fields:
fields_definition.save()
print 'finished domain "{}"'.format(domain)
def calculate_users_in_all_domains():
for domain in Domain.get_all_names():
num_users = CommCareUser.total_by_domain(domain)
record_date = datetime.date.today() - relativedelta(days=1)
user_history = DomainUserHistory.create(domain=domain,
num_users=num_users,
record_date=record_date)
user_history.save()
def handle(self, **options):
all_domains = Domain.get_all_names()
total = len(all_domains)
finished = 0
for domain in all_domains:
update_supply_points(domain)
finished += 1
if finished % 100 == 0:
print("Processed {} of {} domains".format(finished, total))
def set_cleanliness_flags_for_all_domains(force_full=False):
"""
Updates cleanliness for all domains
"""
for domain in Domain.get_all_names():
try:
set_cleanliness_flags_for_domain(domain, force_full=force_full)
except InvalidDomainError as e:
notify_exception(None, unicode(e))
def set_cleanliness_flags_for_all_domains(force_full=False):
"""
Updates cleanliness for all domains
"""
for domain in Domain.get_all_names():
try:
set_cleanliness_flags_for_domain(domain, force_full=force_full)
except InvalidDomainError as e:
notify_exception(None, unicode(e))
def handle(self, *args, **options):
all_domains = Domain.get_all_names()
total = len(all_domains)
finished = 0
for domain in all_domains:
update_supply_points(domain)
finished += 1
if finished % 100 == 0:
print "Processed {} of {} domains".format(finished, total)
def warn_subscriptions_without_domain():
domains_with_active_subscription = Subscription.visible_objects.filter(
is_active=True, ).values_list('subscriber__domain',
flat=True).distinct()
for domain_name in set(domains_with_active_subscription) - set(
Domain.get_all_names()):
log_accounting_error(
'Domain %s has an active subscription but does not exist.' %
domain_name)
def set_cleanliness_flags_for_enabled_domains(force_full=False):
"""
Updates cleanliness for all domains that have the toggle enabled
"""
for domain in Domain.get_all_names():
if OWNERSHIP_CLEANLINESS.enabled(domain):
try:
set_cleanliness_flags_for_domain(domain, force_full=force_full)
except InvalidDomainError as e:
notify_exception(None, unicode(e))
def calculate_users_in_all_domains():
for domain in Domain.get_all_names():
num_users = CommCareUser.total_by_domain(domain)
record_date = datetime.date.today() - relativedelta(days=1)
user_history = DomainUserHistory.create(
domain=domain,
num_users=num_users,
record_date=record_date
)
user_history.save()
def set_cleanliness_flags_for_enabled_domains(force_full=False):
"""
Updates cleanliness for all domains that have the toggle enabled
"""
for domain in Domain.get_all_names():
if OWNERSHIP_CLEANLINESS.enabled(domain):
try:
set_cleanliness_flags_for_domain(domain, force_full=force_full)
except InvalidDomainError as e:
notify_exception(None, unicode(e))
def calculate_users_in_all_domains(today=None):
today = today or datetime.date.today()
for domain in Domain.get_all_names():
num_users = CommCareUser.total_by_domain(domain)
record_date = today - relativedelta(days=1)
DomainUserHistory.objects.create(
domain=domain,
num_users=num_users,
record_date=record_date
)
def warn_active_subscriptions_per_domain_not_one():
for domain_name in Domain.get_all_names():
active_subscription_count = Subscription.visible_objects.filter(
subscriber__domain=domain_name,
is_active=True,
).count()
if active_subscription_count > 1:
log_accounting_error("Multiple active subscriptions found for domain %s" % domain_name)
elif active_subscription_count == 0 and Domain.get_by_name(domain_name).is_active:
log_accounting_error("There is no active subscription for domain %s" % domain_name)
def warn_active_subscriptions_per_domain_not_one():
for domain_name in Domain.get_all_names():
active_subscription_count = Subscription.objects.filter(
subscriber__domain=domain_name,
is_active=True,
).count()
if active_subscription_count > 1:
log_accounting_error("Multiple active subscriptions found for domain %s" % domain_name)
elif active_subscription_count == 0:
log_accounting_error("There is no active subscription for domain %s" % domain_name)
def handle(self, doc_types, *args, **options):
input = raw_input(
"\n".join(
[
"\n\nReally delete documents of the following types: {}?",
"This operation is not reversible. Enter a number N to delete the first "
'N found, or type "delete all" to delete everything.',
"",
]
).format(doc_types)
)
if input == "delete all":
remaining = None
else:
try:
remaining = int(input)
except ValueError:
print "aborting"
sys.exit()
doc_types = doc_types.split(",")
deleted = 0
# unfortunately the only couch view we have for this needs to go by domain
# will be a bit slow
domain_names = Domain.get_all_names()
for doc_type in doc_types:
db = get_db_by_doc_type(doc_type)
if not db:
print "Cannot find db for {}, skipping".format(doc_type)
continue
for domain in domain_names:
docs = [
row["doc"]
for row in db.view(
"by_domain_doc_type_date/view",
startkey=[domain, doc_type],
endkey=[domain, doc_type, {}],
reduce=False,
include_docs=True,
)
][:remaining]
if docs:
count = len(docs)
print "deleting {} {}s from {}".format(count, doc_type, domain)
db.delete_docs(docs)
deleted += count
if remaining is not None:
remaining -= count
if remaining <= 0:
return
print "successfully deleted {} documents".format(deleted)
def build_last_month_MALT():
last_month = last_month_dict()
domains = Domain.get_all_names()
task_results = []
for chunk in chunked(domains, 1000):
task_results.append(update_malt.delay(last_month, chunk))
for result in task_results:
result.get(disable_sync_subtasks=False)
send_MALT_complete_email(last_month)
def handle(self, *args, **kwargs):
from casexml.apps.phone.cleanliness import set_cleanliness_flags_for_domain
if len(args) == 1:
domain = args[0]
set_cleanliness_flags_for_domain(domain)
else:
assert len(args) == 0
for domain in Domain.get_all_names():
if OWNERSHIP_CLEANLINESS.enabled(domain):
print 'updating flags for {}'.format(domain)
set_cleanliness_flags_for_domain(domain)
def warn_subscriptions_without_domain():
domains_with_active_subscription = Subscription.visible_objects.filter(
is_active=True,
).values_list('subscriber__domain', flat=True).distinct()
for domain_name in set(domains_with_active_subscription) - set(Domain.get_all_names()):
# we need to put a try/except here so that sentry captures logging
try:
raise ActiveSubscriptionWithoutDomain()
except ActiveSubscriptionWithoutDomain:
log_accounting_error(
f'Domain {domain_name} has an active subscription but does not exist.'
)
def handle(self, doc_types, *args, **options):
user_input = input('\n'.join([
'\n\nReally delete documents of the following types: {}?',
'This operation is not reversible. Enter a number N to delete the first '
'N found, or type "delete all" to delete everything.',
'',
]).format(doc_types))
if user_input == 'delete all':
remaining = None
else:
try:
remaining = int(user_input)
except ValueError:
print('aborting')
sys.exit()
doc_types = doc_types.split(',')
deleted = 0
# unfortunately the only couch view we have for this needs to go by domain
# will be a bit slow
domain_names = Domain.get_all_names()
for doc_type in doc_types:
db = get_db_by_doc_type(doc_type)
if not db:
print("Cannot find db for {}, skipping".format(doc_type))
continue
for domain in domain_names:
docs = [
row['doc'] for row in db.view(
'by_domain_doc_type_date/view',
startkey=[domain, doc_type],
endkey=[domain, doc_type, {}],
reduce=False,
include_docs=True,
)
][:remaining]
if docs:
count = len(docs)
print('deleting {} {}s from {}'.format(
count, doc_type, domain))
db.delete_docs(docs)
deleted += count
if remaining is not None:
remaining -= count
if remaining <= 0:
return
print('successfully deleted {} documents'.format(deleted))
def check_for_sql_cases_without_existing_domain():
missing_domains_with_cases = set()
for domain in set(
_get_all_domains_that_have_ever_had_subscriptions()) - set(
Domain.get_all_names()):
if CaseAccessorSQL.get_case_ids_in_domain(domain):
missing_domains_with_cases |= {domain}
if missing_domains_with_cases:
mail_admins_async.delay(
'There exist SQL cases belonging to a missing domain',
six.text_type(missing_domains_with_cases))
elif _is_monday():
mail_admins_async.delay('All SQL cases belong to valid domains', '')
def check_for_sql_cases_without_existing_domain():
missing_domains_with_cases = set()
for domain in set(_get_all_domains_that_have_ever_had_subscriptions()) - set(Domain.get_all_names()):
if CaseAccessorSQL.get_case_ids_in_domain(domain):
missing_domains_with_cases |= {domain}
if missing_domains_with_cases:
mail_admins_async.delay(
'There exist SQL cases belonging to a missing domain',
six.text_type(missing_domains_with_cases)
)
elif _is_monday():
mail_admins_async.delay(
'All SQL cases belong to valid domains', ''
)
def calculate_users_in_all_domains(today=None):
today = today or datetime.date.today()
for domain in Domain.get_all_names():
num_users = CommCareUser.total_by_domain(domain)
record_date = today - relativedelta(days=1)
try:
DomainUserHistory.objects.create(domain=domain,
num_users=num_users,
record_date=record_date)
except Exception as e:
log_accounting_error(
"Something went wrong while creating DomainUserHistory for domain %s: %s"
% (domain, e),
show_stack_trace=True,
)
def check_for_sql_forms_without_existing_domain():
missing_domains_with_forms = set()
for domain in set(_get_all_domains_that_have_ever_had_subscriptions()) - set(Domain.get_all_names()):
for doc_type in doc_type_to_state:
if FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type):
missing_domains_with_forms |= {domain}
if missing_domains_with_forms:
mail_admins_async.delay(
'There exist SQL forms belonging to a missing domain',
six.text_type(missing_domains_with_forms)
)
elif _is_monday():
mail_admins_async.delay(
'All SQL forms belong to valid domains', ''
)
def check_for_sql_forms_without_existing_domain():
missing_domains_with_forms = set()
for domain in set(
_get_all_domains_that_have_ever_had_subscriptions()) - set(
Domain.get_all_names()):
for doc_type in doc_type_to_state:
if FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type):
missing_domains_with_forms |= {domain}
if missing_domains_with_forms:
mail_admins_async.delay(
'There exist SQL forms belonging to a missing domain',
six.text_type(missing_domains_with_forms))
elif _is_monday():
mail_admins_async.delay('All SQL forms belong to valid domains', '')
def check_for_sql_forms_without_existing_domain():
missing_domains_with_forms = set()
for domain in set(
_get_all_domains_that_have_ever_had_subscriptions()) - set(
Domain.get_all_names()):
accessor = FormReindexAccessor(domain=domain, include_deleted=True)
for db_name in accessor.sql_db_aliases:
if _has_docs(accessor, db_name):
missing_domains_with_forms |= {domain}
break
if missing_domains_with_forms:
mail_admins_async.delay(
'There exist SQL forms belonging to a missing domain',
str(missing_domains_with_forms))
elif _is_monday():
mail_admins_async.delay('All SQL forms belong to valid domains', '')
def check_for_elasticsearch_data_without_existing_domain():
issue_found = False
deleted_domain_names = set(
_get_all_domains_that_have_ever_had_subscriptions()) - set(
Domain.get_all_names())
for domain_name in deleted_domain_names:
for hqESQuery in [
AppES, CaseES, CaseSearchES, FormES, GroupES, LedgerES, UserES
]:
query = hqESQuery().domain(domain_name)
count = query.count()
if query.count() != 0:
issue_found = True
mail_admins_async.delay(
'ES index "%s" contains %s items belonging to missing domain "%s"'
% (query.index, count, domain_name), '')
if not issue_found and _is_monday():
mail_admins_async.delay('All data in ES belongs to valid domains', '')
def handle(self, doc_types, *args, **options):
input = raw_input('\n'.join([
'\n\nReally delete documents of the following types: {}?',
'This operation is not reversible. Enter a number N to delete the first '
'N found, or type "delete all" to delete everything.',
'',
]).format(doc_types))
if input == 'delete all':
remaining = None
else:
try:
remaining = int(input)
except ValueError:
print 'aborting'
sys.exit()
doc_types = doc_types.split(',')
deleted = 0
# unfortunately the only couch view we have for this needs to go by domain
# will be a bit slow
domain_names = Domain.get_all_names()
database = Domain.get_db()
for domain in domain_names:
for doc_type in doc_types:
docs = [row['doc'] for row in database.view(
'domain/docs',
startkey=[domain, doc_type],
endkey=[domain, doc_type, {}],
reduce=False,
include_docs=True,
)][:remaining]
if docs:
count = len(docs)
print 'deleting {} {}s from {}'.format(count, doc_type, domain)
database.delete_docs(docs)
deleted += count
if remaining is not None:
remaining -= count
if remaining <= 0:
return
print 'successfully deleted {} documents'.format(deleted)
def calculate_users_in_all_domains(today=None):
today = today or datetime.date.today()
for domain in Domain.get_all_names():
num_users = CommCareUser.total_by_domain(domain)
record_date = today - relativedelta(days=1)
try:
DomainUserHistory.objects.create(domain=domain,
num_users=num_users,
record_date=record_date)
except Exception as e:
log_accounting_error(
"Something went wrong while creating DomainUserHistory for domain %s: %s"
% (domain, e),
show_stack_trace=True,
)
# kick off the auto-deactivation of mobile workers after we calculate the
# DomainUserHistory for projects. This ensures this feature is never abused
# to get around our billing system.
from corehq.apps.enterprise.tasks import auto_deactivate_mobile_workers
auto_deactivate_mobile_workers.delay()
def warn_active_subscriptions_per_domain_not_one():
for domain_name in Domain.get_all_names():
active_subscription_count = Subscription.visible_objects.filter(
subscriber__domain=domain_name,
is_active=True,
).count()
# we need to put a try/except here so that sentry captures logging
try:
if active_subscription_count > 1:
raise MultipleActiveSubscriptionsError()
elif active_subscription_count == 0 and Domain.get_by_name(domain_name).is_active:
raise NoActiveSubscriptionError()
except NoActiveSubscriptionError:
log_accounting_error(
f"There is no active subscription for domain {domain_name}"
)
except MultipleActiveSubscriptionsError:
log_accounting_error(
f"Multiple active subscriptions found for domain {domain_name}"
)
def check_for_ucr_tables_without_existing_domain():
all_domain_names = Domain.get_all_names()
connection_name = ConnectionManager().get_django_db_alias(UCR_ENGINE_ID)
table_names = connections[connection_name].introspection.table_names()
ucr_table_names = [name for name in table_names if name.startswith('config_report')]
missing_domains_to_tables = defaultdict(list)
for ucr_table_name in ucr_table_names:
table_domain = ucr_table_name.split('_')[2]
if table_domain not in all_domain_names:
missing_domains_to_tables[table_domain].append(ucr_table_name)
if missing_domains_to_tables:
for missing_domain in missing_domains_to_tables:
mail_admins_async.delay(
'Missing domain "%s" has remaining UCR tables' % missing_domain,
six.text_type(missing_domains_to_tables[missing_domain])
)
elif _is_monday():
mail_admins_async.delay('All UCR tables belong to valid domains', '')
def check_for_ucr_tables_without_existing_domain():
all_domain_names = Domain.get_all_names()
connection_name = ConnectionManager().get_django_db_alias(UCR_ENGINE_ID)
table_names = connections[connection_name].introspection.table_names()
ucr_table_names = [name for name in table_names if name.startswith('config_report')]
missing_domains_to_tables = defaultdict(list)
for ucr_table_name in ucr_table_names:
table_domain = ucr_table_name.split('_')[2]
if table_domain not in all_domain_names:
missing_domains_to_tables[table_domain].append(ucr_table_name)
if missing_domains_to_tables:
for missing_domain in missing_domains_to_tables:
mail_admins_async.delay(
'Missing domain "%s" has remaining UCR tables' % missing_domain,
six.text_type(missing_domains_to_tables[missing_domain])
)
elif _is_monday():
mail_admins_async.delay('All UCR tables belong to valid domains', '')
def generate_malt(monthspans, domains=None):
"""
Populates MALTRow SQL table with app submission data for a given list of months
:param monthspans: list of DateSpan objects
:param domains: list of domain names
"""
domain_names = domains or Domain.get_all_names()
for domain_name in domain_names:
last_submission_date = get_last_form_submission_received(domain_name)
last_malt_run_dates_by_month = _get_last_run_date_for_malt_by_month(domain_name, monthspans)
for monthspan in monthspans:
# if the MALTRow last_run_date is none, use the start date of the month
last_malt_run_date = last_malt_run_dates_by_month.get(monthspan.startdate.date(), monthspan.startdate)
if last_submission_date and last_submission_date >= last_malt_run_date:
# use this date to populate last_run_date for all MALTRows with this domain and month
run_date = datetime.datetime.utcnow()
logger.info(f"Building MALT for {domain_name} for {monthspan} up to {run_date}")
all_users = get_all_user_rows(domain_name, include_inactive=False, include_docs=True)
for users in chunked(all_users, 1000):
users_by_id = {user['id']: CouchUser.wrap_correctly(user['doc']) for user in users}
malt_row_dicts = _get_malt_row_dicts(domain_name, monthspan, users_by_id, run_date)
if malt_row_dicts:
_save_malt_row_dicts_to_db(malt_row_dicts)
def check_for_elasticsearch_data_without_existing_domain():
issue_found = False
deleted_domain_names = set(_get_all_domains_that_have_ever_had_subscriptions()) - set(Domain.get_all_names())
for domain_name in deleted_domain_names:
for hqESQuery in [AppES, CaseES, CaseSearchES, FormES, GroupES, LedgerES, UserES]:
query = hqESQuery().domain(domain_name)
count = query.count()
if query.count() != 0:
issue_found = True
mail_admins_async.delay(
'ES index "%s" contains %s items belonging to missing domain "%s"' % (
query.index, count, domain_name
), ''
)
if not issue_found and _is_monday():
mail_admins_async.delay(
'All data in ES belongs to valid domains', ''
)
def warn_subscriptions_without_domain():
domains_with_active_subscription = Subscription.visible_objects.filter(
is_active=True,
).values_list('subscriber__domain', flat=True).distinct()
for domain_name in set(domains_with_active_subscription) - set(Domain.get_all_names()):
log_accounting_error('Domain %s has an active subscription but does not exist.' % domain_name)
def update_current_MALT():
today = datetime.date.today()
this_month_dict = {'month': today.month, 'year': today.year}
domains = Domain.get_all_names()
for chunk in chunked(domains, 1000):
update_malt.delay(this_month_dict, chunk)
def _get_missing_domains():
return set(_get_all_domains_that_have_ever_had_subscriptions()) - set(
Domain.get_all_names())
