def save(self):
from corehq.apps.users.views.utils import log_commcare_user_locations_changes
selected_users = set(self.cleaned_data['selected_ids'])
previous_users = set([u['id'] for u in self.get_users_at_location()])
to_remove = previous_users - selected_users
to_add = selected_users - previous_users
users_to_be_updated = set.union(to_add, to_remove)
# fetch before updates and fetch from Couch to avoid any ES lag
targeted_users_old_locations = {
user_doc['_id']: {
'location_id': user_doc['location_id'],
'assigned_location_ids': user_doc['assigned_location_ids']
}
for user_doc in iter_docs(CommCareUser.get_db(),
users_to_be_updated)
}
self.unassign_users(to_remove)
self.assign_users(to_add)
self.cache_users_at_location(selected_users)
# re-fetch users to get fresh locations
for updated_user_doc in iter_docs(CommCareUser.get_db(),
users_to_be_updated):
updated_user = CommCareUser.wrap_correctly(updated_user_doc)
user_old_locations = targeted_users_old_locations[
updated_user.get_id]
log_commcare_user_locations_changes(
self.request,
updated_user,
old_location_id=user_old_locations['location_id'],
old_assigned_location_ids=user_old_locations[
'assigned_location_ids'])
def update_analytics_indexes():
"""
Mostly for testing; wait until analytics data sources are up to date
so that calls to analytics functions return up-to-date
(modeled very closely after the same function in couchforms.analytics)
"""
CommCareUser.get_db().view('users/by_domain', limit=1).all()
def update_analytics_indexes():
"""
Mostly for testing; wait until analytics data sources are up to date
so that calls to analytics functions return up-to-date
(modeled very closely after the same function in couchforms.analytics)
"""
CommCareUser.get_db().view('users/by_domain', limit=1).all()
def get_all_commcare_users_by_domain(domain):
"""Returns all CommCareUsers by domain regardless of their active status"""
from corehq.apps.users.models import CommCareUser
key = [domain, CommCareUser.__name__]
ids = [user['id'] for user in CommCareUser.get_db().view(
'domain/docs',
startkey=key,
endkey=key + [{}],
reduce=False,
include_docs=False)]
return [CommCareUser.wrap(user) for user in iter_docs(CommCareUser.get_db(), ids)]
def delete_connections_field_task(domain):
to_save = []
for user in iter_docs(CommCareUser.get_db(), CommCareUser.ids_by_domain(domain)):
if 'connections' in user['user_data']:
del user['user_data']['connections']
to_save.append(user)
if len(to_save) > 500:
CommCareUser.get_db().bulk_save(to_save)
to_save = []
if to_save:
CommCareUser.get_db().bulk_save(to_save)
def bulk_auto_deactivate_commcare_users(user_ids, domain):
"""
Deactivates CommCareUsers in bulk.
Please pre-chunk ids to a reasonable size. Also please reference the
save() method in CommCareUser when making changes.
:param user_ids: list of user IDs
:param domain: name of domain user IDs belong to
"""
from corehq.apps.users.models import UserHistory, CommCareUser
from corehq.apps.users.model_log import UserModelAction
last_modified = json_format_datetime(datetime.datetime.utcnow())
user_docs_to_bulk_save = []
for user_doc in get_docs(CommCareUser.get_db(), keys=user_ids):
if user_doc['is_active']:
user_doc['is_active'] = False
user_doc['last_modified'] = last_modified
user_docs_to_bulk_save.append(user_doc)
# bulk save django Users
user_query = User.objects.filter(
username__in=[u["username"] for u in user_docs_to_bulk_save])
user_query.update(is_active=False)
# bulk save in couch
CommCareUser.get_db().bulk_save(user_docs_to_bulk_save)
# bulk create all the UserHistory logs
UserHistory.objects.bulk_create([
UserHistory(by_domain=domain,
for_domain=domain,
user_type=CommCareUser.doc_type,
user_repr=u['username'].split('@')[0],
changed_by_repr=SYSTEM_USER_ID,
user_id=u['_id'],
changed_by=SYSTEM_USER_ID,
changes={'is_active': False},
changed_via=USER_CHANGE_VIA_AUTO_DEACTIVATE,
change_messages={},
action=UserModelAction.UPDATE.value,
user_upload_record_id=None) for u in user_docs_to_bulk_save
])
# clear caches and fire signals
for user_doc in user_docs_to_bulk_save:
commcare_user = CommCareUser.wrap(user_doc)
commcare_user.clear_quickcache_for_user()
commcare_user.fire_signals()
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = CustomDataFieldsDefinition.get_or_create(
domain,
'UserFields'
)
user_ids = (CommCareUser.ids_by_domain(domain) +
CommCareUser.ids_by_domain(domain, is_active=False))
existing_field_slugs = set([field.slug for field in fields_definition.fields])
for user in iter_docs(CommCareUser.get_db(), user_ids):
user_data = user.get('user_data', {})
for key in user_data.keys():
if key and key not in existing_field_slugs:
existing_field_slugs.add(key)
fields_definition.fields.append(CustomDataField(
slug=key,
label=key,
is_required=False
))
# Only save a definition for domains which use custom user data
if fields_definition.fields:
fields_definition.save()
def message_form(self):
if self.request.method == 'POST':
return MessageForm(self.request.POST, **self.form_kwargs)
broadcast = self.broadcast
schedule = broadcast.schedule
schedule_instances = get_alert_schedule_instances_for_schedule(schedule)
recipients = [
(instance.recipient_type, instance.recipient_id)
for instance in schedule_instances
]
ret = []
if recipients:
for doc_type, doc_id in recipients:
user = CommCareUser.wrap(CommCareUser.get_db().get(doc_id))
ret.append({"id": doc_id, "text": user.raw_username})
initial = {
'schedule_name': broadcast.name,
'send_frequency': 'immediately',
'recipients': ret,
'content': 'sms',
# only works for SMS
'message': schedule.memoized_events[0].content.message,
}
return MessageForm(initial=initial, **self.form_kwargs)
def get_commcare_users_by_filters(domain, user_filters, count_only=False):
"""
Returns CommCareUsers in domain per given filters. If user_filters is empty
returns all users in the domain
args:
user_filters: a dict with below structure.
{'role_id': <Role ID to filter users by>,
'search_string': <string to search users by username>}
kwargs:
count_only: If True, returns count of search results
"""
role_id = user_filters.get('role_id', None)
search_string = user_filters.get('search_string', None)
query = UserES().domain(domain).mobile_users()
if not role_id and not search_string:
if count_only:
query.count()
else:
return get_all_commcare_users_by_domain(domain)
if role_id:
query = query.role_id(role_id)
if search_string:
query = query.search_string_query(search_string, default_fields=['first_name', 'last_name', 'username'])
if count_only:
return query.count()
user_ids = [u['_id'] for u in query.source(['_id']).run().hits]
return map(CommCareUser.wrap, iter_docs(CommCareUser.get_db(), user_ids))
def get_all_user_rows(domain, include_web_users=True, include_mobile_users=True,
include_inactive=True, count_only=False, include_docs=False):
from corehq.apps.users.models import CommCareUser, WebUser
assert include_web_users or include_mobile_users
doc_types = []
if include_mobile_users:
doc_types.append(CommCareUser.__name__)
if include_web_users:
doc_types.append(WebUser.__name__)
states = ['active']
if include_inactive:
states.append('inactive')
for flag in states:
for doc_type in doc_types:
key = [flag, domain, doc_type]
for row in CommCareUser.get_db().view(
'users/by_domain',
startkey=key,
endkey=key + [{}],
reduce=count_only,
include_docs=include_docs
):
yield row
def rows(self):
rows = []
locations = SQLLocation.objects.filter(parent__location_id=self.config['location_id'])
dg = []
for date in list(rrule.rrule(rrule.MONTHLY, dtstart=self.config['startdate'],
until=self.config['enddate'])):
dg.extend(DeliveryGroups().submitting(locations, date.month))
for child in dg:
total_responses = 0
total_possible = 0
submitted, rr_value = randr_value(child.location_id, self.config['startdate'], self.config['enddate'])
if child.is_archived and not rr_value:
continue
group_summaries = GroupSummary.objects.filter(
org_summary__date__lte=self.config['startdate'],
org_summary__location_id=child.location_id,
title=SupplyPointStatusTypes.R_AND_R_FACILITY
)
for group_summary in group_summaries:
if group_summary:
total_responses += group_summary.responded
total_possible += group_summary.total
hist_resp_rate = rr_format_percent(total_responses, total_possible)
url = make_url(FacilityDetailsReport, self.config['domain'],
'?location_id=%s&filter_by_program=%s&'
'datespan_type=%s&datespan_first=%s&datespan_second=%s',
(self.config['location_id'],
self.config['program'], self.config['datespan_type'],
self.config['datespan_first'], self.config['datespan_second']))
contact = CommCareUser.get_db().view(
'locations/users_by_location_id',
startkey=[child.location_id],
endkey=[child.location_id, {}],
include_docs=True
).first()
if contact and contact['doc']:
contact = CommCareUser.wrap(contact['doc'])
role = contact.user_data.get('role') or ""
args = (contact.first_name, contact.last_name, role, contact.default_phone_number)
contact_string = "%s %s (%s) %s" % args
else:
contact_string = ""
rows.append(
[
child.site_code,
link_format(child.name, url),
get_span(submitted) % (rr_value.strftime("%d %b %Y") if rr_value else "Not reported"),
contact_string,
hist_resp_rate
]
)
return rows
def _user_to_change_meta(user):
user_doc = user.to_json()
return change_meta_from_doc(
document=user_doc,
data_source_type=data_sources.SOURCE_COUCH,
data_source_name=CommCareUser.get_db().dbname,
)
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = cdm.CustomDataFieldsDefinition.get_or_create(
domain,
'UserFields'
)
had_fields = bool(fields_definition.fields)
user_ids = (CommCareUser.ids_by_domain(domain) +
CommCareUser.ids_by_domain(domain, is_active=False))
existing_field_slugs = set([field.slug for field in fields_definition.fields])
for user in iter_docs(CommCareUser.get_db(), user_ids):
user_data = user.get('user_data', {})
for key in user_data.keys():
if (key and key not in existing_field_slugs
and not cdm.is_system_key(key)):
existing_field_slugs.add(key)
fields_definition.fields.append(cdm.CustomDataField(
slug=key,
label=key,
is_required=False,
))
for field in fields_definition.fields:
if cdm.is_system_key(field.slug):
fields_definition.fields.remove(field)
# Only save a definition for domains which use custom user data
if fields_definition.fields or had_fields:
fields_definition.save()
print 'finished domain "{}"'.format(domain.name)
def get_all_user_rows(domain,
include_web_users=True,
include_mobile_users=True,
include_inactive=True,
count_only=False,
include_docs=False):
from corehq.apps.users.models import CommCareUser, WebUser
assert include_web_users or include_mobile_users
doc_types = []
if include_mobile_users:
doc_types.append(CommCareUser.__name__)
if include_web_users:
doc_types.append(WebUser.__name__)
states = ['active']
if include_inactive:
states.append('inactive')
for flag in states:
for doc_type in doc_types:
key = [flag, domain, doc_type]
for row in CommCareUser.get_db().view('users/by_domain',
startkey=key,
endkey=key + [{}],
reduce=count_only,
include_docs=include_docs):
yield row
def cache_users_at_location(self, selected_users):
user_cache_list = []
for doc in iter_docs(CommCareUser.get_db(), selected_users):
display_username = user_display_string(
doc['username'], doc.get('first_name', ''), doc.get('last_name', ''))
user_cache_list.append({'text': display_username, 'id': doc['_id']})
self.get_users_at_location.set_cached_value(self).to(user_cache_list)
def location_reporting_owner_ids(self):
"""
Include all users that are assigned to the selected
locations or those locations descendants.
"""
from corehq.apps.locations.models import SQLLocation, LOCATION_REPORTING_PREFIX
from corehq.apps.users.models import CommCareUser
results = []
selected_location_group_ids = EMWF.selected_location_reporting_group_ids(self.request)
for group_id in selected_location_group_ids:
loc = SQLLocation.objects.get(
location_id=group_id.replace(LOCATION_REPORTING_PREFIX, '')
)
for l in [loc] + list(loc.get_descendants()):
users = CommCareUser.get_db().view(
'locations/users_by_location_id',
startkey=[l.location_id],
endkey=[l.location_id, {}],
include_docs=True
).all()
results += [u['id'] for u in users]
return results
def get_users_messages(self):
locations = SQLLocation.active_objects.filter(
domain=self.domain, location_type__administrative=False)
for sql_location in locations:
in_charges = list(
map(
CommCareUser.wrap,
iter_docs(CommCareUser.get_db(), [
in_charge.user_id for in_charge in
sql_location.facilityincharge_set.all()
])))
web_users = [
web_user for web_user in get_web_users_by_location(
self.domain, sql_location.location_id)
if has_notifications_enabled(self.domain, web_user)
]
message, kwargs = self.get_message_for_location(sql_location)
for user in web_users + in_charges:
phone_number = get_preferred_phone_number_for_recipient(user)
if not phone_number:
continue
kwargs['name'] = user.full_name
if message:
yield user, phone_number, message % kwargs
def get_commcare_users_by_filters(domain, user_filters, count_only=False):
"""
Returns CommCareUsers in domain per given filters. If user_filters is empty
returns all users in the domain
args:
user_filters: a dict with below structure.
{'role_id': <Role ID to filter users by>,
'search_string': <string to search users by username>,
'location_id': <Location ID to filter users by>}
kwargs:
count_only: If True, returns count of search results
"""
if not any([
user_filters.get('role_id', None),
user_filters.get('search_string', None),
user_filters.get('location_id', None), count_only
]):
return get_all_commcare_users_by_domain(domain)
query = _get_es_query(domain, user_filters)
if count_only:
return query.count()
user_ids = query.scroll_ids()
return map(CommCareUser.wrap, iter_docs(CommCareUser.get_db(), user_ids))
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = cdm.CustomDataFieldsDefinition.get_or_create(
domain, 'UserFields')
had_fields = bool(fields_definition.fields)
user_ids = (CommCareUser.ids_by_domain(domain) +
CommCareUser.ids_by_domain(domain, is_active=False))
existing_field_slugs = set(
[field.slug for field in fields_definition.fields])
for user in iter_docs(CommCareUser.get_db(), user_ids):
user_data = user.get('user_data', {})
for key in user_data.keys():
if (key and key not in existing_field_slugs
and not cdm.is_system_key(key)):
existing_field_slugs.add(key)
fields_definition.fields.append(
cdm.CustomDataField(
slug=key,
label=key,
is_required=False,
))
for field in fields_definition.fields:
if cdm.is_system_key(field.slug):
fields_definition.fields.remove(field)
# Only save a definition for domains which use custom user data
if fields_definition.fields or had_fields:
fields_definition.save()
print 'finished domain "{}"'.format(domain)
def _user_to_change_meta(user):
user_doc = user.to_json()
return change_meta_from_doc(
document=user_doc,
data_source_type=data_sources.COUCH,
data_source_name=CommCareUser.get_db().dbname,
)
def get_commcare_users_by_filters(domain, user_filters, count_only=False):
"""
Returns CommCareUsers in domain per given filters. If user_filters is empty
returns all users in the domain
args:
user_filters: a dict with below structure.
{'role_id': <Role ID to filter users by>,
'search_string': <string to search users by username>}
kwargs:
count_only: If True, returns count of search results
"""
role_id = user_filters.get('role_id', None)
search_string = user_filters.get('search_string', None)
query = UserES().domain(domain).mobile_users()
if not role_id and not search_string:
if count_only:
query.count()
else:
return get_all_commcare_users_by_domain(domain)
if role_id:
query = query.role_id(role_id)
if search_string:
query = query.search_string_query(
search_string,
default_fields=['first_name', 'last_name', 'username'])
if count_only:
return query.count()
user_ids = [u['_id'] for u in query.source(['_id']).run().hits]
return map(CommCareUser.wrap, iter_docs(CommCareUser.get_db(), user_ids))
def restore_domain_membership(user, check=False):
doc_id = user._id
db = CommCareUser.get_db()
revisions = get_doc_revisions(db, doc_id)
for rev in revisions[1:]:
doc = get_doc_rev(db, doc_id, rev)
if not doc:
continue
prev_user = CommCareUser.wrap(doc)
if user_looks_ok(prev_user):
if user.location_id != prev_user.domain_membership.location_id:
continue
if user.assigned_location_ids != prev_user.domain_membership.assigned_location_ids:
continue
if check:
print('Ready to patch user: {} ({})'.format(
user.domain, doc_id))
old = json.dumps(user.domain_membership.to_json(), indent=2)
print('Old domain membership: \n{}\n'.format(old))
new = json.dumps(prev_user.domain_membership.to_json(),
indent=2)
print('New domain membership: \n{}\n'.format(new))
if not confirm('Proceed with updating user?'):
return
print("Patching user: {} ({})".format(user.domain, doc_id))
prev_domain_membership = prev_user.domain_membership
user.domain_membership = prev_domain_membership
user.save()
return
print('Unable to fix user: {} ({})'.format(user.domain, doc_id))
def _add_user_info_to_cases(bad_cases):
user_info = _get_user_info(
case['form_user_id'] for case in bad_cases if 'form_user_id' in case)
auth_user_ids = [case['auth_user_id'] for case in bad_cases
if 'auth_user_id' in case]
auth_usernames = {user_doc['_id']: user_doc['username'] for user_doc in
iter_docs(CommCareUser.get_db(), auth_user_ids)}
for case in bad_cases:
user_dict = user_info.get(case.get('form_user_id'))
if user_dict:
case['username'] = user_dict['username']
device_id = case['form_device_id']
if device_id == 'Formplayer':
# Hazard a guess as to what device ID was used in the restore
if case['form_user_id'] == case['auth_user_id']:
device_id = "WebAppsLogin"
else:
auth_username = auth_usernames.get(case['auth_user_id'])
device_id = "WebAppsLogin*{}*as*{}".format(
auth_username, user_dict['username']).replace('.', '_')
try:
device_number = user_dict['device_ids'].index(device_id) + 1
except ValueError:
device_number = -1
case['real_device_number'] = six.text_type(device_number)
def cache_users_at_location(self, selected_users):
user_cache_list = []
for doc in iter_docs(CommCareUser.get_db(), selected_users):
display_username = user_display_string(
doc['username'], doc.get('first_name', ''), doc.get('last_name', ''))
user_cache_list.append({'text': display_username, 'id': doc['_id']})
self.get_users_at_location.set_cached_value(self).to(user_cache_list)
def _users_by_location(location_id, include_docs=True, wrap=True):
view = CommCareUser.view if wrap else CommCareUser.get_db().view
return view(
'locations/users_by_location_id',
startkey=[location_id],
endkey=[location_id, {}],
include_docs=include_docs,
)
def setUp(self):
self.database = FakeCouchDb()
self.case_orig_db = CommCareCase.get_db()
self.form_orig_db = XFormInstance.get_db()
self.user_orig_db = CommCareUser.get_db()
CommCareCase.set_db(self.database)
XFormInstance.set_db(self.database)
CommCareUser.set_db(self.database)
def setUp(self):
self.database = FakeCouchDb()
self.case_orig_db = CommCareCase.get_db()
self.form_orig_db = XFormInstance.get_db()
self.user_orig_db = CommCareUser.get_db()
CommCareCase.set_db(self.database)
XFormInstance.set_db(self.database)
CommCareUser.set_db(self.database)
def delete_all_users():
from django.contrib.auth.models import User
def _clear_cache(doc):
user = CouchUser.wrap_correctly(doc, allow_deleted_doc_types=True)
user.clear_quickcache_for_user()
iter_bulk_delete(CommCareUser.get_db(), get_all_user_ids(), doc_callback=_clear_cache)
User.objects.all().delete()
def _get_user_info(user_ids):
return {
user_doc['_id']: {
'username': user_doc['username'].split('@')[0],
'device_ids': [d['device_id'] for d in user_doc['devices']],
}
for user_doc in iter_docs(CommCareUser.get_db(), user_ids)
}
def setUpClass(cls):
super(AppStatusIntegrationTest, cls).setUpClass()
delete_all_docs_by_doc_type(Domain.get_db(),
['Domain', 'Domain-Deleted'])
delete_all_docs_by_doc_type(CommCareUser.get_db(),
['CommCareUser', 'WebUser'])
delete_all_docs_by_doc_type(Application.get_db(),
['Application', 'Application-Deleted'])
cls.domain_records = [
Domain(name=cls.domain,
hr_name='One',
creating_user_id='abc',
is_active=True),
]
for domain in cls.domain_records:
domain.save()
cls.user_records = [
# TODO: Handle WebUsers who have multiple domains
# WebUser.create(
# cls.domain,
# 'web-user',
# '***',
# date_joined=datetime.utcnow(),
# first_name='A',
# last_name='B',
# email='*****@*****.**',
# is_active=True,
# is_staff=False,
# is_superuser=True,
# ),
CommCareUser.create(
cls.domain,
'commcare-user',
'***',
date_joined=datetime.utcnow(),
email='*****@*****.**',
is_active=True,
is_staff=True,
is_superuser=False,
),
]
cls.form_records = [
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
create_form_for_test(cls.domain, user_id=cls.user_records[0]._id),
]
cls.sync_records = []
for user in cls.user_records:
restore_user = OTARestoreCommCareUser(user.domain, user)
device = MockDevice(cls.domain_records[0], restore_user)
cls.sync_records.append(device.sync())
cls.batch = create_batch(cls.slug)
def handle(self, domain, **options):
ids = (CommCareUser.ids_by_domain(domain, is_active=True) +
CommCareUser.ids_by_domain(domain, is_active=False))
for doc in iter_docs(CommCareUser.get_db(), ids):
user = CommCareUser.wrap(doc)
try:
self.process_user(user)
except Exception as e:
print("Error processing user %s: %s" % (user._id, e))
def get_user_reindexer():
return ElasticPillowReindexer(
pillow=get_user_pillow(),
change_provider=CouchViewChangeProvider(
couch_db=CommCareUser.get_db(), view_name="users/by_username", view_kwargs={"include_docs": True}
),
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
)
def delete_all_users():
from corehq.apps.users.models import CouchUser
from django.contrib.auth.models import User
def _clear_cache(doc):
user = CouchUser.wrap_correctly(doc, allow_deleted_doc_types=True)
user.clear_quickcache_for_user()
iter_bulk_delete(CommCareUser.get_db(), get_all_user_ids(), doc_callback=_clear_cache)
User.objects.all().delete()
def handle(self, *args, **options):
self.stdout.write("Population location_id field...\n")
relevant_ids = set([
r['id'] for r in CouchUser.get_db().view(
'users/by_username',
reduce=False,
).all()
])
to_save = []
domain_cache = {}
exclude = (
"drewpsi",
"psi",
"psi-ors",
"psi-test",
"psi-test2",
"psi-test3",
"psi-unicef",
"psi-unicef-wb",
)
def _is_location_domain(domain):
if domain in domain_cache:
return domain_cache[domain]
else:
domain_obj = Domain.get_by_name(domain)
val = domain_obj.uses_locations
domain_cache[domain] = val
return val
for user_doc in iter_docs(CommCareUser.get_db(), relevant_ids):
if user_doc['doc_type'] == 'WebUser':
continue
if user_doc['domain'] in exclude:
continue
if not _is_location_domain(user_doc['domain']):
continue
user = CommCareUser.get(user_doc['_id'])
if user._locations:
user_doc['location_id'] = user._locations[0]._id
to_save.append(user_doc)
if len(to_save) > 500:
self.stdout.write("Saving 500")
CouchUser.get_db().bulk_save(to_save)
to_save = []
if to_save:
CouchUser.get_db().bulk_save(to_save)
def get_ids():
for flag in ['active', 'inactive']:
key = [flag, domain, CommCareUser.__name__]
for user in CommCareUser.get_db().view('users/by_domain',
startkey=key,
endkey=key + [{}],
reduce=False,
include_docs=False):
yield user['id']
def _get_count_of_commcare_users_in_domain(active_flag, domain):
result = CommCareUser.get_db().view(
'users/by_domain',
startkey=[active_flag, domain, 'CommCareUser'],
endkey=[active_flag, domain, 'CommCareUser', {}],
group=True,
group_level=2,
stale=stale_ok(),
).one()
return result['value'] if result else 0
def clean_recipients(self):
data = self.cleaned_data['recipients']
# TODO Will need to add more than user ids
# TODO batch id verification
for user_id in data:
user = CommCareUser.get_db().get(user_id)
assert user[
'domain'] == self.domain, "User must be in the same domain"
return data
def _cc_users_by_location(domain, location_id, include_docs=True, wrap=True):
from corehq.apps.users.models import CommCareUser
view = CommCareUser.view if wrap else CommCareUser.get_db().view
return view(
'locations/users_by_location_id',
startkey=[domain, location_id, CommCareUser.__name__],
endkey=[domain, location_id, CommCareUser.__name__, {}],
include_docs=include_docs,
reduce=False,
)
def _get_count_of_commcare_users_in_domain(active_flag, domain):
result = CommCareUser.get_db().view(
'users/by_domain',
startkey=[active_flag, domain, 'CommCareUser'],
endkey=[active_flag, domain, 'CommCareUser', {}],
group=True,
group_level=2,
stale=stale_ok(),
).one()
return result['value'] if result else 0
def db_comparisons(request):
comparison_config = [{
'description':
'Users (base_doc is "CouchUser")',
'couch_db':
CommCareUser.get_db(),
'view_name':
'users/by_username',
'es_query':
UserES().remove_default_filter('active').remove_default_filter(
'mobile_worker').size(0),
'sql_rows':
User.objects.count(),
}, {
'description': 'Domains (doc_type is "Domain")',
'couch_db': Domain.get_db(),
'view_name': 'domain/by_status',
'es_query': DomainES().size(0),
'sql_rows': None,
}, {
'description':
'Forms (doc_type is "XFormInstance")',
'couch_db':
XFormInstance.get_db(),
'view_name':
'couchforms/by_xmlns',
'es_query':
FormES().remove_default_filter('has_xmlns').remove_default_filter(
'has_user').size(0),
'sql_rows':
FormData.objects.count(),
}, {
'description': 'Cases (doc_type is "CommCareCase")',
'couch_db': CommCareCase.get_db(),
'view_name': 'case/by_owner',
'es_query': CaseES().size(0),
'sql_rows': None,
}]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description':
comp['description'],
'couch_docs':
comp['couch_db'].view(
comp['view_name'],
reduce=True,
).one()['value'],
'es_docs':
comp['es_query'].run().total,
'sql_rows':
comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def handle(self, *args, **options):
self.stdout.write("Population location_id field...\n")
relevant_ids = set([r['id'] for r in CouchUser.get_db().view(
'users/by_username',
reduce=False,
).all()])
to_save = []
domain_cache = {}
exclude = (
"drewpsi",
"psi",
"psi-ors",
"psi-test",
"psi-test2",
"psi-test3",
"psi-unicef",
"psi-unicef-wb",
)
def _is_location_domain(domain):
if domain in domain_cache:
return domain_cache[domain]
else:
domain_obj = Domain.get_by_name(domain)
val = domain_obj.uses_locations
domain_cache[domain] = val
return val
for user_doc in iter_docs(CommCareUser.get_db(), relevant_ids):
if user_doc['doc_type'] == 'WebUser':
continue
if user_doc['domain'] in exclude:
continue
if not _is_location_domain(user_doc['domain']):
continue
user = CommCareUser.get(user_doc['_id'])
if user._locations:
user_doc['location_id'] = user._locations[0]._id
to_save.append(user_doc)
if len(to_save) > 500:
self.stdout.write("Saving 500")
CouchUser.get_db().bulk_save(to_save)
to_save = []
if to_save:
CouchUser.get_db().bulk_save(to_save)
def send_message_to_admins(self, message):
in_charge_users = map(CommCareUser.wrap, iter_docs(
CommCareUser.get_db(),
[in_charge.user_id for in_charge in self.sql_location.facilityincharge_set.all()]
))
for in_charge_user in in_charge_users:
phone_number = get_preferred_phone_number_for_recipient(in_charge_user)
if not phone_number:
continue
send_sms(self.sql_location.domain, in_charge_user, phone_number,
message % {'name': in_charge_user.full_name, 'location': self.sql_location.name})
def get_ids():
for flag in ['active', 'inactive']:
key = [flag, domain, CommCareUser.__name__]
for user in CommCareUser.get_db().view(
'users/by_domain',
startkey=key,
endkey=key + [{}],
reduce=False,
include_docs=False
):
yield user['id']
def handle(self, domain, **options):
ids = (
CommCareUser.ids_by_domain(domain, is_active=True) +
CommCareUser.ids_by_domain(domain, is_active=False)
)
for doc in iter_docs(CommCareUser.get_db(), ids):
user = CommCareUser.wrap(doc)
try:
self.process_user(user)
except Exception as e:
print("Error processing user %s: %s" % (user._id, e))
def build(self):
return ElasticPillowReindexer(pillow=get_user_pillow(),
change_provider=CouchViewChangeProvider(
couch_db=CommCareUser.get_db(),
view_name='users/by_username',
view_kwargs={
'include_docs': True,
}),
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
**self.options)
def _get_users_by_loc_id(location_type):
"""Find any existing users previously assigned to this type"""
loc_ids = SQLLocation.objects.filter(
location_type=location_type).location_ids()
user_ids = list(UserES().domain(location_type.domain).show_inactive().term(
'user_location_id', list(loc_ids)).values_list('_id', flat=True))
return {
user_doc['user_location_id']: CommCareUser.wrap(user_doc)
for user_doc in iter_docs(CommCareUser.get_db(), user_ids)
if 'user_location_id' in user_doc
}
def user_exists(username):
"""
:param username:
:return: namedtuple(exists:bool, is_deleted:bool)
"""
result = CommCareUser.get_db().view(
'users/by_username',
key=username,
include_docs=False,
reduce=False,
)
if result:
return UserExists(True, False)
result = CommCareUser.get_db().view('deleted_users_by_username/view',
key=username,
include_docs=False,
reduce=False).count()
exists = bool(result)
return UserExists(exists, exists)
def _validate_usernames(self):
keys = [["active", self.domain, "CommCareUser", username] for username in self.usernames]
result = CommCareUser.get_db().view(
'users/by_domain',
keys=keys,
reduce=False,
include_docs=False
).all()
if len(result) != len(self.usernames):
usernames_found = set([r['key'][-1] for r in result])
usernames_missing = set(self.usernames) - usernames_found
self.errors.append(f"Could not find user(s): {', '.join(usernames_missing)}")
def _create_or_unarchive_users(location_type):
users_by_loc = _get_users_by_loc_id(location_type)
with IterDB(CommCareUser.get_db()) as iter_db:
for loc in SQLLocation.objects.filter(location_type=location_type):
user = users_by_loc.get(loc.location_id, None) or make_location_user(loc)
user.is_active = True
user.user_location_id = loc.location_id
user.set_location(loc, commit=False)
iter_db.save(user)
loc.user_id = user._id
loc.save()
def _create_or_unarchive_users(location_type):
users_by_loc = _get_users_by_loc_id(location_type)
with IterDB(CommCareUser.get_db()) as iter_db:
for loc in SQLLocation.objects.filter(location_type=location_type):
user = users_by_loc.get(loc.location_id, None) or make_location_user(loc)
user.is_active = True
user.user_location_id = loc.location_id
user.set_location(loc, commit=False)
iter_db.save(user)
loc.user_id = user._id
loc.save()
def _get_users_by_loc_id(location_type):
"""Find any existing users previously assigned to this type"""
loc_ids = SQLLocation.objects.filter(location_type=location_type).location_ids()
user_ids = list(UserES()
.domain(location_type.domain)
.show_inactive()
.term('user_location_id', list(loc_ids))
.values_list('_id', flat=True))
return {
user_doc['user_location_id']: CommCareUser.wrap(user_doc)
for user_doc in iter_docs(CommCareUser.get_db(), user_ids)
if 'user_location_id' in user_doc
}
def _archive_users(location_type):
def archive(user_doc):
if user_doc['is_active']:
user_doc['is_active'] = False
return DocUpdate(user_doc)
user_ids = (SQLLocation.objects.filter(
location_type=location_type).values_list('user_id', flat=True))
iter_update(CommCareUser.get_db(), archive, user_ids)
for loc in SQLLocation.objects.filter(location_type=location_type):
loc.user_id = ''
loc.save()
def user_exists(username):
"""
:param username:
:return: namedtuple(exists:bool, is_deleted:bool)
"""
result = CommCareUser.get_db().view(
'users/by_username',
key=username,
include_docs=False,
reduce=False,
)
if result:
return UserExists(True, False)
result = CommCareUser.get_db().view(
'deleted_users_by_username/view',
key=username,
include_docs=False,
reduce=False
).count()
exists = bool(result)
return UserExists(exists, exists)
def build(self):
return ElasticPillowReindexer(
pillow_or_processor=get_user_pillow_old(),
change_provider=CouchViewChangeProvider(
couch_db=CommCareUser.get_db(),
view_name='users/by_username',
view_kwargs={
'include_docs': True,
}
),
elasticsearch=get_es_new(),
index_info=USER_INDEX_INFO,
**self.options
)
def _archive_users(location_type):
def archive(user_doc):
if user_doc['is_active']:
user_doc['is_active'] = False
return DocUpdate(user_doc)
user_ids = (SQLLocation.objects
.filter(location_type=location_type)
.values_list('user_id', flat=True))
iter_update(CommCareUser.get_db(), archive, user_ids)
for loc in SQLLocation.objects.filter(location_type=location_type):
loc.user_id = ''
loc.save()
def handle(self, *args, **options):
if len(args) == 0:
raise CommandError("Usage: python manage.py resync_location_user_data %s" % self.args)
domain = args[0]
ids = (
CommCareUser.ids_by_domain(domain, is_active=True) +
CommCareUser.ids_by_domain(domain, is_active=False)
)
for doc in iter_docs(CommCareUser.get_db(), ids):
user = CommCareUser.wrap(doc)
try:
self.process_user(user)
except Exception as e:
print "Error processing user %s: %s" % (user._id, e)
def db_comparisons(request):
comparison_config = [
{
'description': 'Users (base_doc is "CouchUser")',
'couch_db': CommCareUser.get_db(),
'view_name': 'users/by_username',
'es_query': UserES().remove_default_filter('active')
.remove_default_filter('mobile_worker')
.size(0),
'sql_rows': User.objects.count(),
},
{
'description': 'Domains (doc_type is "Domain")',
'couch_db': Domain.get_db(),
'view_name': 'domain/by_status',
'es_query': DomainES().size(0),
'sql_rows': None,
},
{
'description': 'Forms (doc_type is "XFormInstance")',
'couch_db': XFormInstance.get_db(),
'view_name': 'couchforms/by_xmlns',
'es_query': FormES().remove_default_filter('has_xmlns')
.remove_default_filter('has_user')
.size(0),
'sql_rows': FormData.objects.count(),
},
{
'description': 'Cases (doc_type is "CommCareCase")',
'couch_db': CommCareCase.get_db(),
'view_name': 'case/by_owner',
'es_query': CaseES().size(0),
'sql_rows': None,
}
]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description': comp['description'],
'couch_docs': comp['couch_db'].view(
comp['view_name'],
reduce=True,
).one()['value'],
'es_docs': comp['es_query'].run().total,
'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def db_comparisons(request):
def _simple_view_couch_query(db, view_name):
return db.view(view_name, reduce=True).one()['value']
def _count_real_forms():
all_forms = _simple_view_couch_query(XFormInstance.get_db(), 'couchforms/by_xmlns')
device_logs = XFormInstance.get_db().view('couchforms/by_xmlns', key=DEVICE_LOG_XMLNS).one()['value']
return all_forms - device_logs
comparison_config = [
{
'description': 'Users (base_doc is "CouchUser")',
'couch_docs': _simple_view_couch_query(CommCareUser.get_db(), 'users/by_username'),
'es_query': UserES().remove_default_filter('active').size(0),
'sql_rows': User.objects.count(),
},
{
'description': 'Domains (doc_type is "Domain")',
'couch_docs': _simple_view_couch_query(Domain.get_db(), 'domain/by_status'),
'es_query': DomainES().size(0),
'sql_rows': None,
},
{
'description': 'Forms (doc_type is "XFormInstance")',
'couch_docs': _count_real_forms(),
'es_query': FormES().remove_default_filter('has_xmlns')
.remove_default_filter('has_user')
.size(0),
'sql_rows': FormData.objects.count(),
},
{
'description': 'Cases (doc_type is "CommCareCase")',
'couch_docs': _simple_view_couch_query(CommCareCase.get_db(), 'case/by_owner'),
'es_query': CaseES().size(0),
'sql_rows': CaseData.objects.count(),
}
]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description': comp['description'],
'couch_docs': comp['couch_docs'],
'es_docs': comp['es_query'].run().total,
'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def db_comparisons(request):
def _simple_view_couch_query(db, view_name):
return db.view(view_name, reduce=True).one()["value"]
def _count_real_forms():
all_forms = _simple_view_couch_query(XFormInstance.get_db(), "couchforms/by_xmlns")
device_logs = XFormInstance.get_db().view("couchforms/by_xmlns", key=DEVICE_LOG_XMLNS).one()["value"]
return all_forms - device_logs
comparison_config = [
{
"description": 'Users (base_doc is "CouchUser")',
"couch_docs": _simple_view_couch_query(CommCareUser.get_db(), "users/by_username"),
"es_query": UserES().remove_default_filter("active").size(0),
"sql_rows": User.objects.count(),
},
{
"description": 'Domains (doc_type is "Domain")',
"couch_docs": _simple_view_couch_query(Domain.get_db(), "domain/by_status"),
"es_query": DomainES().size(0),
"sql_rows": None,
},
{
"description": 'Forms (doc_type is "XFormInstance")',
"couch_docs": _count_real_forms(),
"es_query": FormES().remove_default_filter("has_xmlns").remove_default_filter("has_user").size(0),
"sql_rows": FormData.objects.count(),
},
{
"description": 'Cases (doc_type is "CommCareCase")',
"couch_docs": _simple_view_couch_query(CommCareCase.get_db(), "case/by_owner"),
"es_query": CaseES().size(0),
"sql_rows": CaseData.objects.count(),
},
]
comparisons = []
for comp in comparison_config:
comparisons.append(
{
"description": comp["description"],
"couch_docs": comp["couch_docs"],
"es_docs": comp["es_query"].run().total,
"sql_rows": comp["sql_rows"] if comp["sql_rows"] else "n/a",
}
)
return json_response(comparisons)
