def print_domain_summary(self, domain):
users_not_blocked = {}
user_query = UserES().domain(domain).source(['email', 'username'])
total_users = user_query.count()
chunk_size = 30 # Hubspot recommends fewer than 100 emails per request
num_chunks = int(math.ceil(float(total_users) / float(chunk_size)))
for chunk in range(num_chunks):
blocked_users = (user_query.size(chunk_size).start(
chunk * chunk_size).run().hits)
blocked_emails = []
for user in blocked_users:
username = user.get('username')
user_email = user.get('email')
blocked_emails.append(username)
if user_email and user_email != username:
blocked_emails.append(user_email)
users_not_blocked.update(
get_first_conversion_status_for_emails(
list(set(blocked_emails))))
if users_not_blocked:
self.stdout.write(
self.style.ERROR(
f"\n\nFound {len(users_not_blocked)} users in "
f"HubSpot who are members of the project {domain} "
f"that is blocking HubSpot data:"))
self.stdout.write("\nEmail\tFirst Conversion")
for user, status in users_not_blocked.items():
self.stdout.write(f"{user}\t{status}")
self.stdout.write('\n\n')
else:
self.stdout.write(
self.style.SUCCESS(
f"All users in project {domain} are absent on HubSpot."))
def test_unknown_user_pillow(self):
FormProcessorTestUtils.delete_all_xforms()
user_id = 'test-unknown-user'
metadata = TestFormMetadata(domain=TEST_DOMAIN,
user_id='test-unknown-user')
form = get_form_ready_to_save(metadata)
FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models(
[form])
# send to kafka
topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM
since = self._get_kafka_seq()
producer.send_change(topic, _form_to_change_meta(form))
# send to elasticsearch
pillow = get_xform_pillow()
pillow.process_changes(since=since, forever=False)
self.elasticsearch.indices.refresh(self.index_info.index)
# the default query doesn't include unknown users so should have no results
self.assertEqual(0, UserES().run().total)
# clear the default filters which hide unknown users
user_es = UserES().remove_default_filters()
results = user_es.run()
self.assertEqual(1, results.total)
user_doc = results.hits[0]
self.assertEqual(TEST_DOMAIN, user_doc['domain'])
self.assertEqual(user_id, user_doc['_id'])
self.assertEqual('UnknownUser', user_doc['doc_type'])
def test_unknown_user_pillow(self):
FormProcessorTestUtils.delete_all_xforms()
user_id = 'test-unknown-user'
metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user')
form = get_form_ready_to_save(metadata)
FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models([form])
# send to kafka
topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM
since = self._get_kafka_seq()
producer.send_change(topic, _form_to_change_meta(form))
# send to elasticsearch
pillow = get_unknown_users_pillow()
pillow.process_changes(since=since, forever=False)
self.elasticsearch.indices.refresh(self.index_info.index)
# the default query doesn't include unknown users so should have no results
self.assertEqual(0, UserES().run().total)
# clear the default filters which hide unknown users
user_es = UserES().remove_default_filters()
results = user_es.run()
self.assertEqual(1, results.total)
user_doc = results.hits[0]
self.assertEqual(TEST_DOMAIN, user_doc['domain'])
self.assertEqual(user_id, user_doc['_id'])
self.assertEqual('UnknownUser', user_doc['doc_type'])
def _query(self):
if self.show_archived:
return UserES().mobile_users().domain(
self.domain).show_only_inactive().sort('username.exact')
else:
return UserES().mobile_users().domain(
self.domain).sort('username.exact')
def get_users_by_location(self, user_supported_locations):
if not self.national_user:
user_query = UserES().mobile_users().domain(
self.domain).location(user_supported_locations).fields(
['username', 'assigned_location_ids', 'last_login'])
else:
user_query = UserES().mobile_users().domain(self.domain).fields(
['username', 'assigned_location_ids', 'last_login'])
return [u for u in user_query.run().hits]
def get_users_at_location(self):
user_query = UserES().domain(
self.domain_object.name
).mobile_users().location(
self.location.location_id
).fields(['_id', 'username', 'first_name', 'last_name'])
return [
dict(id=u['_id'], text=user_display_string(
u['username'], u.get('first_name', ''), u.get('last_name', '')
)) for u in user_query.run().hits]
def get_users_at_location(self):
user_query = UserES().domain(
self.domain_object.name
).mobile_users().location(
self.location.location_id
).fields(['_id', 'username', 'first_name', 'last_name'])
return [
dict(id=u['_id'], text=user_display_string(
u['username'], u.get('first_name', ''), u.get('last_name', '')
)) for u in user_query.run().hits]
def user_query(self):
user_query = UserES().domain(self.domain).filter(mobile_users())
locations_id = self.locations_id
if locations_id:
user_query = user_query.location(locations_id)
elif not self.request.couch_user.has_permission(
self.domain, 'access_all_locations'):
# EnikshayLocationFilter.get_value should always return a
# location_id for restricted users
raise PermissionDenied()
return user_query
def supply_points_users(self):
query = UserES().mobile_users().domain(self.config['domain']).term(
"location_id", list(self.config['reporting_supply_points']))
with_reporters = set()
with_in_charge = set()
for hit in query.run().hits:
with_reporters.add(hit['location_id'])
if 'In Charge' in hit['user_data'].get('role', []):
with_in_charge.add(hit['location_id'])
return with_reporters, with_in_charge
def supply_points_users(self, supply_points):
query = UserES().mobile_users().domain(self.config['domain']).term("location_id",
[sp for sp in supply_points])
with_reporters = set()
with_in_charge = set()
for hit in query.run().hits:
with_reporters.add(hit['location_id'])
if hit['user_data'].get('role') == 'In Charge':
with_in_charge.add(hit['location_id'])
return with_reporters, with_in_charge
def test_domain_allow_enterprise(self):
self._send_user_to_es()
self.assertEqual(['superman'],
UserES().domain(self.domain).values_list('username',
flat=True))
self.assertEqual([],
UserES().domain(self.source_domain).values_list(
'username', flat=True))
self.assertEqual(['superman'],
UserES().domain(self.domain,
allow_enterprise=True).values_list(
'username', flat=True))
def supply_points_users(self):
query = UserES().mobile_users().domain(self.config['domain']).term(
"location_id", list(self.config['reporting_supply_points'])
)
with_reporters = set()
with_in_charge = set()
for hit in query.run().hits:
with_reporters.add(hit['location_id'])
if 'In Charge' in hit['user_data'].get('role', []):
with_in_charge.add(hit['location_id'])
return with_reporters, with_in_charge
def test_domain_allow_mirroring(self):
source_domain = self.domain + "-source"
mirror = DomainPermissionsMirror(source=source_domain, mirror=self.domain)
mirror.save()
self._send_user_to_es()
self.assertEqual(['superman'], UserES().domain(self.domain).values_list('username', flat=True))
self.assertEqual([], UserES().domain(source_domain).values_list('username', flat=True))
self.assertEqual(
['superman'],
UserES().domain(self.domain, allow_mirroring=True).values_list('username', flat=True)
)
mirror.delete()
def get_commcare_users_by_filters(domain, user_filters, count_only=False):
"""
Returns CommCareUsers in domain per given filters. If user_filters is empty
returns all users in the domain
args:
user_filters: a dict with below structure.
{'role_id': <Role ID to filter users by>,
'search_string': <string to search users by username>}
kwargs:
count_only: If True, returns count of search results
"""
role_id = user_filters.get('role_id', None)
search_string = user_filters.get('search_string', None)
query = UserES().domain(domain).mobile_users()
if not role_id and not search_string:
if count_only:
query.count()
else:
return get_all_commcare_users_by_domain(domain)
if role_id:
query = query.role_id(role_id)
if search_string:
query = query.search_string_query(
search_string,
default_fields=['first_name', 'last_name', 'username'])
if count_only:
return query.count()
user_ids = [u['_id'] for u in query.source(['_id']).run().hits]
return map(CommCareUser.wrap, iter_docs(CommCareUser.get_db(), user_ids))
def supply_points_users(self):
query = UserES().mobile_users().domain(self.config['domain']).term(
"location_id", self.supply_points_locations_ids)
with_reporters = set()
for hit in query.run().hits:
with_reporters.add(hit['location_id'])
with_in_charge = set(
FacilityInCharge.objects.filter(
location__location_id__in=self.supply_points_locations_ids).
values_list('location__location_id', flat=True).distinct())
return with_reporters, with_in_charge
def supply_points_users(self):
query = UserES().mobile_users().domain(self.config['domain']).term(
"location_id",
self.supply_points_locations_ids
)
with_reporters = set()
for hit in query.run().hits:
with_reporters.add(hit['location_id'])
with_in_charge = set(FacilityInCharge.objects.filter(
location__location_id__in=self.supply_points_locations_ids
).values_list('location__location_id', flat=True).distinct())
return with_reporters, with_in_charge
def _get_es_query(domain, user_filters):
role_id = user_filters.get('role_id', None)
search_string = user_filters.get('search_string', None)
location_id = user_filters.get('location_id', None)
query = UserES().domain(domain).mobile_users().remove_default_filter('active')
if role_id:
query = query.role_id(role_id)
if search_string:
query = query.search_string_query(search_string, default_fields=['first_name', 'last_name', 'username'])
if location_id:
location_ids = SQLLocation.objects.get_locations_and_children_ids([location_id])
query = query.location(location_ids)
return query
def get_user_by_org_unit(domain, org_unit_id, top_org_unit_name):
"""
Look up user ID by a DHIS2 organisation unit ID
"""
result = (
UserES().domain(domain).mobile_users()
# .term('user_data.dhis_org_id', org_unit_id)
.run())
# cf. http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-dynamic-mapping.html
# If/when we upgrade elasticsearch, we can filter on dynamic mappings and
# uncomment the ".term" line above. Until then, check it ourselves ...
for doc in result.hits:
if doc['user_data'].get('dhis_org_id') == org_unit_id:
return CommCareUser.wrap(doc)
# No user is assigned to this organisation unit (i.e. region or facility).
# Try its parent org unit.
Dhis2OrgUnit.objects = FixtureManager(Dhis2OrgUnit, domain,
ORG_UNIT_FIXTURES)
org_units = {ou.id: ou for ou in Dhis2OrgUnit.objects.all()}
if (org_unit_id in org_units
and org_units[org_unit_id].name != top_org_unit_name
and org_units[org_unit_id].parent_id):
return get_user_by_org_unit(domain, org_units[org_unit_id].parent_id,
top_org_unit_name)
# We don't know that org unit ID, or we're at the top for this project, or we're at the top of DHIS2
return None
def login_as_user_query(domain, couch_user, search_string, limit, offset):
'''
Takes in various parameters to determine which users to populate the login as screen.
:param domain: String domain
:param couch_user: The CouchUser that is using the Login As feature
:param search_string: The query that filters the users returned. Filters based on the
`search_fields` as well as any fields defined in `user_data_fields`.
:param limit: The max amount of users returned.
:param offset: From where to start the query.
:returns: An EsQuery instance.
'''
search_fields = [
"base_username", "last_name", "first_name", "phone_numbers"
]
user_es = (UserES().domain(domain).start(offset).size(limit).sort(
'username.exact').search_string_query(search_string, search_fields))
if not couch_user.has_permission(domain, 'access_all_locations'):
loc_ids = SQLLocation.objects.accessible_to_user(
domain, couch_user).location_ids()
user_es = user_es.location(list(loc_ids))
if _limit_login_as(couch_user, domain):
user_filters = [login_as_user_filter(couch_user.username)]
if couch_user.has_permission(domain, 'access_default_login_as_user'):
user_filters.append(login_as_user_filter('default'))
user_es = user_es.filter(
queries.nested('user_data_es', filters.OR(*user_filters)))
return user_es.mobile_users()
def _sync_location_user(self):
if not self.location.location_id:
return
if self.location.location_type.has_user and not self.location.user_id:
# make sure there's a location user
res = list(UserES().domain(self.domain).show_inactive().term(
'user_location_id',
self.location.location_id).values_list('_id', flat=True))
user_id = res[0] if res else None
if user_id:
user = CommCareUser.get(user_id)
else:
user = make_location_user(self.location)
user.is_active = True
user.user_location_id = self.location.location_id
user.set_location(self.location, commit=False)
user.save()
self.location.user_id = user._id
self.location.save()
elif self.location.user_id and not self.location.location_type.has_user:
# archive the location user
user = CommCareUser.get_by_user_id(self.location.user_id,
self.domain)
if user:
user.is_active = False
user.save()
self.location.user_id = ''
self.location.save()
def _get_admins_with_roles():
# domain_memberships isn't a nested mapping in ES, so this only checks that
# they have a domain membership that's an admin, and one with a role_id,
# not that it's both on the same membership
return (UserES().web_users().term(
'domain_memberships.is_admin',
True).non_null('domain_memberships.role_id').get_ids())
def paginate_web_users(request, domain):
limit = int(request.GET.get('limit', 10))
page = int(request.GET.get('page', 1))
skip = limit * (page - 1)
query = request.GET.get('query')
result = (
UserES().domain(domain).web_users().sort('username.exact')
.search_string_query(query, ["username", "last_name", "first_name"])
.start(skip).size(limit).run()
)
web_users = [WebUser.wrap(w) for w in result.hits]
web_users_fmt = [{
'email': u.get_email(),
'domain': domain,
'name': u.full_name,
'role': u.role_label(domain),
'phoneNumbers': u.phone_numbers,
'id': u.get_id,
'editUrl': reverse('user_account', args=[domain, u.get_id]),
'removeUrl': (
reverse('remove_web_user', args=[domain, u.user_id])
if request.user.username != u.username else None
),
} for u in web_users]
return JsonResponse({
'users': web_users_fmt,
'total': result.total,
'page': page,
'query': query,
})
def get_user_ids(self):
res = (UserES()
.OR(filters.AND(user_filters.mobile_users(), filters.non_null('location_id')),
filters.AND(user_filters.web_users(), filters.non_null('domain_memberships.location_id')))
.exclude_source()
.run())
return list(res.doc_ids)
def login_as_user_query(domain,
couch_user,
search_string,
limit,
offset,
user_data_fields=None):
'''
Takes in various parameters to determine which users to populate the login as screen.
:param domain: String domain
:param couch_user: The CouchUser that is using the Login As feature
:param search_string: The query that filters the users returned. Filters based on the
`search_fields` as well as any fields defined in `user_data_fields`.
:param limit: The max amount of users returned.
:param offset: From where to start the query.
:param user_data_fields: A list of custom user data fields that should also be searched
by the `search_string`
:returns: An EsQuery instance.
'''
search_fields = [
"base_username", "last_name", "first_name", "phone_numbers"
]
should_criteria_query = [
queries.search_string_query(search_string, search_fields),
]
if user_data_fields:
or_criteria = []
for field in user_data_fields:
or_criteria.append(
filters.AND(
filters.term('user_data_es.key', field),
filters.term('user_data_es.value', search_string),
), )
should_criteria_query.append(
queries.nested_filter('user_data_es', filters.OR(*or_criteria)))
user_es = (
UserES().domain(domain).start(offset).size(limit).
sort('username.exact').set_query(
queries.BOOL_CLAUSE(
queries.SHOULD_CLAUSE(
should_criteria_query,
# It should either match on the search fields like username or it
# should match on the custom user data fields. If this were 2, then
# it would require the search string to match both on the search fields and
# the custom user data fields.
minimum_should_match=1,
), )))
if not couch_user.has_permission(domain, 'access_all_locations'):
loc_ids = SQLLocation.objects.accessible_to_user(
domain, couch_user).location_ids()
user_es = user_es.location(list(loc_ids))
return user_es.mobile_users()
def _ids_to_users(self, user_ids):
users = (UserES().domain(self.domain).user_ids(user_ids).values(
'_id', 'username', 'first_name', 'last_name'))
return {
u['_id']: user_display_string(u['username'], u['first_name'],
u['last_name'])
for u in users
}
def get_all_users(self):
user_query = (UserES().domain(
self.domain_object.name).mobile_users().fields(
['_id', 'username', 'first_name', 'last_name']))
return [(u['_id'],
user_display_string(u['username'], u.get('first_name', ''),
u.get('last_name', '')))
for u in user_query.run().hits]
def active_user_es_query(self, query):
if not self.request.can_access_all_locations:
return super().active_user_es_query(query)
else:
search_fields = ["first_name", "last_name", "base_username"]
return (UserES().domains(
self.enterprise_domains).search_string_query(
query, default_fields=search_fields))
def get_commcare_users_by_filters(domain, user_filters, count_only=False):
"""
Returns CommCareUsers in domain per given filters. If user_filters is empty
returns all users in the domain
args:
user_filters: a dict with below structure.
{'role_id': <Role ID to filter users by>,
'search_string': <string to search users by username>}
kwargs:
count_only: If True, returns count of search results
"""
role_id = user_filters.get('role_id', None)
search_string = user_filters.get('search_string', None)
query = UserES().domain(domain).mobile_users()
if not role_id and not search_string:
if count_only:
query.count()
else:
return get_all_commcare_users_by_domain(domain)
if role_id:
query = query.role_id(role_id)
if search_string:
query = query.search_string_query(search_string, default_fields=['first_name', 'last_name', 'username'])
if count_only:
return query.count()
user_ids = [u['_id'] for u in query.source(['_id']).run().hits]
return map(CommCareUser.wrap, iter_docs(CommCareUser.get_db(), user_ids))
def user_es_call(domain, q, fields, size, start_at):
query = (UserES()
.domain(domain)
.fields(fields)
.size(size)
.start(start_at))
if q is not None:
query.set_query({"query_string": {"query": q}})
return query.run().hits
def get_bad_user_ids():
res = (UserES().mobile_users().empty("location_id").fields(
["_id", "domain", "username",
"user_data.commcare_location_id"]).run().hits)
return [
u['_id'] for u in res
if u.get('user_data', {}).get('commcare_location_id')
or u.get('user_data', {}).get('commtrack-supply-point')
]
def _get_assigned_location_ids(self):
location_ids = list(self._location_details_by_location_id.keys())
assigned_location_ids_per_username = {}
user_details = UserES().location(location_ids).values_list('base_username', 'assigned_location_ids')
for username, assigned_location_ids in user_details:
if not isinstance(assigned_location_ids, list):
assigned_location_ids = [assigned_location_ids]
assigned_location_ids_per_username[username] = assigned_location_ids
return assigned_location_ids_per_username
def get_users_location_ids(domain, user_ids):
"""Get the ids of the locations the users are assigned to"""
result = (UserES().domain(domain).user_ids(user_ids).non_null(
'assigned_location_ids').fields(['assigned_location_ids']).run())
location_ids = [
r['assigned_location_ids'] for r in result.hits
if 'assigned_location_ids' in r
]
return list(chain(*location_ids))
def get_search_users_in_domain_es_query(domain, search_string, limit, offset):
"""
returns a UserES object
"""
default_search_fields = ["base_username", "last_name", "first_name"]
return (UserES().domain(domain).search_string_query(
search_string,
default_search_fields).start(offset).size(limit).sort('username.exact')
)
def rendered_content(self):
from corehq.apps.users.views.mobile.users import EditCommCareUserView
users = get_users_by_location_id(self.config['domain'],
self.config['location_id'])
in_charges = FacilityInCharge.objects.filter(
location=self.location).values_list('user_id', flat=True)
if self.location.parent.location_type.name == 'district':
children = self.location.parent.get_descendants()
availaible_in_charges = list(
chain.from_iterable([
filter(
lambda u: 'In Charge' in u.user_data.get('role', []),
get_users_by_location_id(self.config['domain'],
child.location_id))
for child in children
]))
else:
availaible_in_charges = filter(
lambda u: 'In Charge' in u.user_data.get('role', []),
get_users_by_location_id(self.domain, self.location_id))
user_to_dict = lambda sms_user: {
'id':
sms_user.get_id,
'full_name':
sms_user.full_name,
'phone_numbers':
sms_user.phone_numbers,
'in_charge':
sms_user.get_id in in_charges,
'location_name':
sms_user.location.sql_location.name,
'url':
reverse(EditCommCareUserView.urlname,
args=[self.config['domain'], sms_user.get_id])
}
web_users = [{
'id': web_user['_id'],
'first_name': web_user['first_name'],
'last_name': web_user['last_name'],
'email': web_user['email']
} for web_user in UserES().web_users().domain(
self.config['domain']).term("domain_memberships.location_id",
self.config['location_id']).run().hits]
return render_to_string(
'ewsghana/partials/users_tables.html', {
'users': [user_to_dict(user) for user in users],
'domain':
self.domain,
'location_id':
self.location_id,
'web_users':
web_users,
'district_in_charges':
[user_to_dict(user) for user in availaible_in_charges]
})
def _assert_user_in_es(self, username, is_webuser=False):
results = UserES().run()
self.assertEqual(1, results.total)
user_doc = results.hits[0]
self.assertEqual(username, user_doc['username'])
if not is_webuser:
self.assertEqual(DOMAIN, user_doc['domain'])
self.assertEqual('CommCareUser', user_doc['doc_type'])
else:
self.assertEqual('WebUser', user_doc['doc_type'])
def stream_user_sources(user_ids):
results = (UserES().user_ids(user_ids).fields(
['_id', '__group_ids', '__group_names']).scroll())
for result in results:
group_ids = result.get('__group_ids', [])
group_ids = set(group_ids) if isinstance(group_ids,
list) else {group_ids}
group_names = result.get('__group_names', [])
group_names = set(group_names) if isinstance(group_names,
list) else {group_names}
yield UserSource(result['_id'], group_ids, group_names)
def test_unknown_user_reindexer(self):
FormProcessorTestUtils.delete_all_xforms()
user_id = 'test-unknown-user'
metadata = TestFormMetadata(domain=self.domain, user_id='test-unknown-user')
form = get_form_ready_to_save(metadata)
FormProcessorInterface(domain=self.domain).save_processed_models([form])
ensure_index_deleted(USER_INDEX)
call_command('ptop_fast_reindex_unknownusers', noinput=True, bulk=True)
# the default query doesn't include unknown users so should have no results
self.assertEqual(0, UserES().run().total)
user_es = UserES()
# hack: clear the default filters which hide unknown users
# todo: find a better way to do this.
user_es._default_filters = ESQuery.default_filters
results = user_es.run()
self.assertEqual(1, results.total)
user_doc = results.hits[0]
self.assertEqual(self.domain, user_doc['domain'])
self.assertEqual(user_id, user_doc['_id'])
self.assertEqual('UnknownUser', user_doc['doc_type'])
form.delete()
delete_es_index(USER_INDEX)