def user_choices(self):
users = UserES().domain(self.domain).fields(['_id',
'username']).run().hits
return [(user['_id'], raw_username(user['username']))
for user in users]
def track_periodic_data():
"""
Sync data that is neither event or page based with hubspot/Kissmetrics
:return:
"""
# Start by getting a list of web users mapped to their domains
three_months_ago = date.today() - timedelta(days=90)
user_query = (UserES().web_users().last_logged_in(
gte=three_months_ago).sort('date_joined', desc=True).source(
['domains', 'email', 'date_joined']).analytics_enabled())
total_users = user_query.count()
chunk_size = 100
num_chunks = int(math.ceil(float(total_users) / float(chunk_size)))
# Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD
hubspot_number_of_users = 0
hubspot_number_of_domains_with_forms_gt_threshold = 0
for chunk in range(num_chunks):
users_to_domains = (user_query.size(chunk_size).start(
chunk * chunk_size).run().hits)
# users_to_domains is a list of dicts
domains_to_forms = (FormES().terms_aggregation(
'domain',
'domain').size(0).run().aggregations.domain.counts_by_bucket())
domains_to_mobile_users = (UserES().mobile_users().terms_aggregation(
'domain',
'domain').size(0).run().aggregations.domain.counts_by_bucket())
# Keep track of india and www data seperately
env = get_instance_string()
for num_forms in domains_to_forms.values():
if num_forms > HUBSPOT_THRESHOLD:
hubspot_number_of_domains_with_forms_gt_threshold += 1
# For each web user, iterate through their domains and select the max number of form submissions and
# max number of mobile workers
submit = []
for user in users_to_domains:
email = user.get('email')
if not _email_is_valid(email):
continue
hubspot_number_of_users += 1
date_created = user.get('date_joined')
max_forms = 0
max_workers = 0
max_export = 0
max_report = 0
for domain in user['domains']:
if domain in domains_to_forms and domains_to_forms[
domain] > max_forms:
max_forms = domains_to_forms[domain]
if domain in domains_to_mobile_users and domains_to_mobile_users[
domain] > max_workers:
max_workers = domains_to_mobile_users[domain]
if _get_export_count(domain) > max_export:
max_export = _get_export_count(domain)
if _get_report_count(domain) > max_report:
max_report = _get_report_count(domain)
project_spaces_created = ", ".join(
get_domains_created_by_user(email))
user_json = {
'email':
email,
'properties': [{
'property':
'{}max_form_submissions_in_a_domain'.format(env),
'value':
max_forms
}, {
'property':
'{}max_mobile_workers_in_a_domain'.format(env),
'value':
max_workers
}, {
'property':
'{}project_spaces_created_by_user'.format(env),
'value':
project_spaces_created,
}, {
'property':
'{}over_300_form_submissions'.format(env),
'value':
max_forms > HUBSPOT_THRESHOLD
}, {
'property': '{}date_created'.format(env),
'value': date_created
}, {
'property':
'{}max_exports_in_a_domain'.format(env),
'value':
max_export
}, {
'property':
'{}max_custom_reports_in_a_domain'.format(env),
'value':
max_report
}]
}
submit.append(user_json)
submit_json = json.dumps(submit)
submit_data_to_hub_and_kiss(submit_json)
update_datadog_metrics({
DATADOG_WEB_USERS_GAUGE:
hubspot_number_of_users,
DATADOG_DOMAINS_EXCEEDING_FORMS_GAUGE:
hubspot_number_of_domains_with_forms_gt_threshold
})
def user_lookup(request, domain):
return JsonResponse({
'results': (UserES().domain(domain).fields([
'_id', 'base_username'
]).search_string_query(request.GET['term']).size(10).run().hits)
})
def get_users_by_location_filter(self, location_ids):
return UserES().domain(self.domain).location(location_ids).values_list(
'_id', flat=True)
def all_users(self):
return map(
_report_user_dict,
sorted(UserES().mobile_users().domain(self.domain).run().hits,
key=lambda user: user['username']))
def get_mobile_users(domains):
return set(
UserES().show_inactive().mobile_users().domain(domains).scroll_ids())
def track_periodic_data():
"""
Sync data that is neither event or page based with hubspot/Kissmetrics
:return:
"""
# Start by getting a list of web users mapped to their domains
six_months_ago = date.today() - timedelta(days=180)
users_to_domains = UserES().web_users().last_logged_in(gte=six_months_ago)\
.fields(['domains', 'email', 'date_joined'])\
.run().hits
# users_to_domains is a list of dicts
domains_to_forms = FormES().terms_aggregation('domain', 'domain').size(0).run()\
.aggregations.domain.counts_by_bucket()
domains_to_mobile_users = UserES().mobile_users().terms_aggregation('domain', 'domain').size(0).run()\
.aggregations.domain.counts_by_bucket()
# Keep track of india and www data seperately
env = get_instance_string()
# Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD
number_of_users = 0
number_of_domains_with_forms_gt_threshold = 0
for num_forms in domains_to_forms.values():
if num_forms > HUBSPOT_THRESHOLD:
number_of_domains_with_forms_gt_threshold += 1
# For each web user, iterate through their domains and select the max number of form submissions and
# max number of mobile workers
submit = []
for user in users_to_domains:
email = user.get('email')
if not _email_is_valid(email):
continue
number_of_users += 1
date_created = user.get('date_joined')
max_forms = 0
max_workers = 0
for domain in user['domains']:
if domain in domains_to_forms and domains_to_forms[
domain] > max_forms:
max_forms = domains_to_forms[domain]
if domain in domains_to_mobile_users and domains_to_mobile_users[
domain] > max_workers:
max_workers = domains_to_mobile_users[domain]
project_spaces_created = ", ".join(get_domains_created_by_user(email))
user_json = {
'email':
email,
'properties': [{
'property':
'{}max_form_submissions_in_a_domain'.format(env),
'value':
max_forms
}, {
'property':
'{}max_mobile_workers_in_a_domain'.format(env),
'value':
max_workers
}, {
'property':
'{}project_spaces_created_by_user'.format(env),
'value':
project_spaces_created,
}, {
'property':
'{}over_300_form_submissions'.format(env),
'value':
max_forms > HUBSPOT_THRESHOLD
}, {
'property': '{}date_created'.format(env),
'value': date_created
}]
}
submit.append(user_json)
submit_json = json.dumps(submit)
submit_data_to_hub_and_kiss(submit_json)
update_datadog_metrics({
DATADOG_WEB_USERS_GAUGE:
number_of_users,
DATADOG_DOMAINS_EXCEEDING_FORMS_GAUGE:
number_of_domains_with_forms_gt_threshold
})
def get_active_users(cls):
six_months_ago = date.today() - timedelta(days=180)
users = UserES().web_users().last_logged_in(
gte=six_months_ago).run().hits
return [WebUser.wrap(u) for u in users]
def members(self):
# FYI use '_id' instead of '__group_ids' in query below in case of
# ES not updating immediately
return get_simplified_users(UserES().mobile_users().domain(
self.domain).term("_id", self.group.get_user_ids()))
def track_periodic_data():
"""
Sync data that is neither event or page based with hubspot/Kissmetrics
:return:
"""
# Start by getting a list of web users mapped to their domains
if not KISSMETRICS_ENABLED and not HUBSPOT_ENABLED:
return
three_months_ago = date.today() - timedelta(days=90)
user_query = (UserES().web_users().last_logged_in(
gte=three_months_ago).sort('date_joined', desc=True).source(
['domains', 'email', 'date_joined',
'username']).analytics_enabled())
total_users = user_query.count()
chunk_size = 100
num_chunks = int(math.ceil(float(total_users) / float(chunk_size)))
# Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD
hubspot_number_of_users = 0
hubspot_number_of_domains_with_forms_gt_threshold = 0
blocked_domains = get_blocked_hubspot_domains()
blocked_email_domains = get_blocked_hubspot_email_domains()
for chunk in range(num_chunks):
users_to_domains = (user_query.size(chunk_size).start(
chunk * chunk_size).run().hits)
# users_to_domains is a list of dicts
domains_to_forms = (FormES().terms_aggregation(
'domain.exact',
'domain').size(0).run().aggregations.domain.counts_by_bucket())
domains_to_mobile_users = (UserES().mobile_users().terms_aggregation(
'domain.exact',
'domain').size(0).run().aggregations.domain.counts_by_bucket())
# Keep track of india and www data seperately
env = get_instance_string()
for num_forms in domains_to_forms.values():
if num_forms > HUBSPOT_THRESHOLD:
hubspot_number_of_domains_with_forms_gt_threshold += 1
# For each web user, iterate through their domains and select the max number of form submissions and
# max number of mobile workers
submit = []
for user in users_to_domains:
email = user.get('email') or user.get('username')
if not _email_is_valid(email):
continue
email_domain = email.split('@')[-1]
if email_domain in blocked_email_domains:
metrics_gauge(
'commcare.hubspot_data.rejected.periodic_task.email_domain',
1,
tags={
'email_domain': email_domain,
})
continue
username_email_domain = user.get('username').split('@')[-1]
if username_email_domain in blocked_email_domains:
metrics_gauge(
'commcare.hubspot_data.rejected.periodic_task.username',
1,
tags={
'username': username_email_domain,
})
continue
hubspot_number_of_users += 1
date_created = user.get('date_joined')
max_forms = 0
max_workers = 0
max_export = 0
max_report = 0
is_member_of_blocked_domain = False
for domain in user['domains']:
if domain in blocked_domains:
metrics_gauge(
'commcare.hubspot_data.rejected.periodic_task.domain',
1,
tags={
'domain': domain,
})
is_member_of_blocked_domain = True
break
if domain in domains_to_forms and domains_to_forms[
domain] > max_forms:
max_forms = domains_to_forms[domain]
if domain in domains_to_mobile_users and domains_to_mobile_users[
domain] > max_workers:
max_workers = domains_to_mobile_users[domain]
if _get_export_count(domain) > max_export:
max_export = _get_export_count(domain)
if _get_report_count(domain) > max_report:
max_report = _get_report_count(domain)
if is_member_of_blocked_domain:
continue
project_spaces_created = ", ".join(
get_domains_created_by_user(email))
user_json = {
'email':
email,
'properties': [{
'property':
'{}max_form_submissions_in_a_domain'.format(env),
'value':
max_forms
}, {
'property':
'{}max_mobile_workers_in_a_domain'.format(env),
'value':
max_workers
}, {
'property':
'{}project_spaces_created_by_user'.format(env),
'value':
project_spaces_created,
}, {
'property':
'{}over_300_form_submissions'.format(env),
'value':
max_forms > HUBSPOT_THRESHOLD
}, {
'property': '{}date_created'.format(env),
'value': date_created
}, {
'property':
'{}max_exports_in_a_domain'.format(env),
'value':
max_export
}, {
'property':
'{}max_custom_reports_in_a_domain'.format(env),
'value':
max_report
}]
}
submit.append(user_json)
submit_json = json.dumps(submit)
submit_data_to_hub_and_kiss(submit_json)
metrics_gauge('commcare.hubspot.web_users_processed',
hubspot_number_of_users,
multiprocess_mode=MPM_LIVESUM)
metrics_gauge('commcare.hubspot.domains_with_forms_gt_threshold',
hubspot_number_of_domains_with_forms_gt_threshold,
multiprocess_mode=MPM_MAX)
