def get_mobile_workers_data(domains, datespan, interval,
datefield='created_on'):
"""
Returns mobile workers that have used SMS.
Returned based on date mobile worker is created
"""
sms_users = (SMSES()
.to_commcare_user()
.domain(domains)
.terms_facet('couch_recipient', 'users', USER_COUNT_UPPER_BOUND)
.size(0))
users = [u['term'] for u in sms_users.run().facet('users', 'terms')]
users_after_date = (UserES()
.domain(domains)
.filter({"ids": {"values": users}})
.mobile_users()
.show_inactive()
.created(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram('date', datefield, interval)
.size(0))
histo_data = users_after_date.run().facet('date', 'entries')
users_before_date = (UserES()
.domain(domains)
.filter({"ids": {"values": users}})
.mobile_users()
.show_inactive()
.created(lt=datespan.startdate)
.size(0)).run().total
return format_return_data(histo_data, users_before_date, datespan)
def get_users_all_stats(domains, datespan, interval,
user_type_mobile=None,
require_submissions=False):
query = UserES().domain(domains).show_inactive().size(0)
if user_type_mobile:
query = query.mobile_users()
elif user_type_mobile is not None:
query = query.web_users()
if require_submissions:
query = query.user_ids(get_submitted_users())
histo_data = (
query
.created(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram('date', 'created_on', interval)
.run().facet('date', 'entries')
)
users_before_date = len(
query
.created(lt=datespan.startdate)
.run().doc_ids
)
return format_return_data(histo_data, users_before_date, datespan)
def get_mobile_workers_data(domains,
datespan,
interval,
datefield='created_on'):
"""
Returns mobile workers that have used SMS.
Returned based on date mobile worker is created
"""
sms_users = (SMSES().to_commcare_user().domain(domains).terms_aggregation(
'couch_recipient', 'users').size(0))
users = sms_users.run().aggregations.users.keys
users_after_date = (UserES().domain(domains).filter({
"ids": {
"values": users
}
}).mobile_users().show_inactive().created(
gte=datespan.startdate,
lte=datespan.enddate).date_histogram('date', datefield,
interval).size(0))
histo_data = users_after_date.run().aggregations.date.as_facet_result()
users_before_date = (UserES().domain(domains).filter({
"ids": {
"values": users
}
}).mobile_users().show_inactive().created(
lt=datespan.startdate).size(0)).run().total
return format_return_data(histo_data, users_before_date, datespan)
def cleanup_blocked_hubspot_contacts():
"""
Remove any data stored about users from blocked domains and email domains
from Hubspot in case it somehow got there.
:return:
"""
if not HUBSPOT_ENABLED:
return
# First delete any user information from users that are members of
# blocked domains
blocked_domains = get_blocked_hubspot_domains()
for domain in blocked_domains:
user_query = UserES().domain(domain).source(['email', 'username'])
total_users = user_query.count()
chunk_size = 30 # Hubspot recommends fewer than 100 emails per request
num_chunks = int(math.ceil(float(total_users) / float(chunk_size)))
for chunk in range(num_chunks):
blocked_users = (user_query
.size(chunk_size)
.start(chunk * chunk_size)
.run()
.hits)
blocked_emails = []
for user in blocked_users:
username = user.get('username')
user_email = user.get('email')
blocked_emails.append(username)
if user_email and user_email != username:
blocked_emails.append(user_email)
ids_to_delete = _get_contact_ids_for_emails(set(blocked_emails))
num_deleted = sum(_delete_hubspot_contact(vid) for vid in ids_to_delete)
metrics_gauge(
'commcare.hubspot_data.deleted_user.blocked_domain',
num_deleted,
tags={
'domain': domain,
'ids_deleted': ids_to_delete,
}
)
# Next delete any user info from users that have emails or usernames ending
# in blocked email-domains
blocked_email_domains = get_blocked_hubspot_email_domains()
for email_domain in blocked_email_domains:
ids_to_delete = _get_contact_ids_for_email_domain(email_domain)
num_deleted = sum(_delete_hubspot_contact(vid) for vid in ids_to_delete)
metrics_gauge(
'commcare.hubspot_data.deleted_user.blocked_email_domain',
num_deleted,
tags={
'email_domain': email_domain,
'ids_deleted': ids_to_delete,
}
)
def get_user_ids(user_type_mobile):
"""
Returns the set of mobile user IDs if user_type_mobile is True,
else returns the set of web user IDs.
"""
query = UserES().show_inactive()
if user_type_mobile:
query = query.mobile_users()
else:
query = query.web_users()
return set(query.run().doc_ids)
def get_user_ids(user_type_mobile):
"""
Returns the set of mobile user IDs if user_type_mobile is True,
else returns the set of web user IDs.
"""
query = UserES().show_inactive()
if user_type_mobile:
query = query.mobile_users()
else:
query = query.web_users()
return set(query.run().doc_ids)
def get_user_ids(user_type_mobile, domains):
"""
Returns the set of mobile user IDs if user_type_mobile is True,
else returns the set of web user IDs.
"""
query = UserES().show_inactive()
if domains:
query = query.domain(domains)
if user_type_mobile:
query = query.mobile_users()
else:
query = query.web_users()
return set(query.get_ids())
def report_context(self):
context = super(CaseReassignmentInterface, self).report_context
if not self.request.can_access_all_locations:
accessible_location_ids = (
SQLLocation.active_objects.accessible_location_ids(
self.request.domain, self.request.couch_user))
user_query = UserES().location(accessible_location_ids)
active_users = get_simplified_users(user_query)
context.update(groups=[
dict(ownerid=group.get_id, name=group.name, type="group")
for group in self.accessible_case_sharing_locations(
self.request.couch_user)
], )
else:
active_users = self.get_all_users_by_domain(user_filter=tuple(
HQUserType.all()),
simplified=True)
context.update(groups=[
dict(ownerid=group.get_id, name=group.name, type="group")
for group in self.all_case_sharing_groups
], )
context.update(users=[
dict(ownerid=user.user_id,
name=user.username_in_report,
type="user") for user in active_users
], )
return context
def previous_six_months(self):
now = datetime.datetime.utcnow()
six_month_summary = []
last_month_summary = None
performance_threshold = get_performance_threshold(self.domain)
filtered_users = self.get_users_by_filter()
active_not_deleted_users = UserES().domain(self.domain).values_list(
"_id", flat=True)
for i in range(-6, 1):
year, month = add_months(now.year, now.month, i)
month_as_date = datetime.date(year, month, 1)
this_month_summary = MonthlyPerformanceSummary(
domain=self.domain,
performance_threshold=performance_threshold,
month=month_as_date,
previous_summary=last_month_summary,
selected_users=filtered_users,
active_not_deleted_users=active_not_deleted_users,
)
six_month_summary.append(this_month_summary)
if last_month_summary is not None:
last_month_summary.set_next_month_summary(this_month_summary)
this_month_summary.set_num_inactive_users(
len(this_month_summary.get_dropouts()))
this_month_summary.set_percent_active()
last_month_summary = this_month_summary
return six_month_summary[1:]
def _user_query(self):
return (UserES().domain(
self.definition.domain).mobile_users().show_inactive().filter(
filters.nested(
'user_data_es',
filters.AND(filters.term('user_data_es.key', PROFILE_SLUG),
filters.term('user_data_es.value', self.id)))))
def previous_months_summary(self, months=6):
now = datetime.datetime.utcnow()
six_month_summary = []
last_month_summary = None
performance_threshold = get_performance_threshold(self.domain)
filtered_users = self.get_users_by_filter()
active_not_deleted_users = UserES().domain(self.domain).values_list(
"_id", flat=True)
for i in range(-months, 1):
year, month = add_months(now.year, now.month, i)
month_as_date = datetime.date(year, month, 1)
this_month_summary = MonthlyPerformanceSummary(
domain=self.domain,
performance_threshold=performance_threshold,
month=month_as_date,
previous_summary=last_month_summary,
selected_users=filtered_users,
active_not_deleted_users=active_not_deleted_users,
)
six_month_summary.append(this_month_summary)
if last_month_summary is not None:
last_month_summary.set_next_month_summary(this_month_summary)
last_month_summary = this_month_summary
# these steps have to be done in a second outer loop so that 'next month summary' is available
# whenever it is needed
for summary in six_month_summary:
summary.finalize()
summary.set_percent_active()
return six_month_summary[1:]
def get_mobile_users(domains):
return set(
UserES()
.show_inactive()
.mobile_users()
.domain(domains)
.scroll_ids()
)
def get_mobile_users(domains):
return set(
UserES()
.show_inactive()
.mobile_users()
.domain(domains)
.run().doc_ids
)
def send_mass_emails(username, real_email, subject, html, text):
if real_email:
recipients = [{
'username': h['username'],
'first_name': h['first_name'] or 'CommCare User',
} for h in UserES().web_users().run().hits]
else:
recipients = [{
'username': username,
'first_name': 'CommCare User',
}]
successes = []
failures = []
for recipient in recipients:
context = recipient
context.update({
'url_prefix':
'' if settings.STATIC_CDN else 'http://' + get_site_domain(),
})
html_template = Template(html)
text_template = Template(text)
text_content = render_to_string(
"hqadmin/email/mass_email_base.txt", {
'email_body': text_template.render(Context(context)),
})
html_content = render_to_string(
"hqadmin/email/mass_email_base.html", {
'email_body': html_template.render(Context(context)),
})
try:
send_HTML_email(subject,
recipient['username'],
html_content,
text_content=text_content)
successes.append((recipient['username'], None))
except Exception as e:
failures.append((recipient['username'], e))
message = (
"Subject: {subject},\n"
"Total successes: {success_count} \n Total errors: {failure_count} \n"
"".format(subject=subject,
success_count=len(successes),
failure_count=len(failures)))
send_html_email_async("Mass email summary",
username,
message,
text_content=message,
file_attachments=[
_mass_email_attachment('successes', successes),
_mass_email_attachment('failures', failures)
])
def user_lookup(request, domain):
return JsonResponse({
'results': (
UserES()
.domain(domain)
.fields(['_id', 'base_username'])
.search_string_query(request.GET['term'])
.size(10)
.run().hits)
})
def db_comparisons(request):
comparison_config = [{
'description':
'Users (base_doc is "CouchUser")',
'couch_db':
CommCareUser.get_db(),
'view_name':
'users/by_username',
'es_query':
UserES().remove_default_filter('active').remove_default_filter(
'mobile_worker').size(0),
'sql_rows':
User.objects.count(),
}, {
'description': 'Domains (doc_type is "Domain")',
'couch_db': Domain.get_db(),
'view_name': 'domain/by_status',
'es_query': DomainES().size(0),
'sql_rows': None,
}, {
'description':
'Forms (doc_type is "XFormInstance")',
'couch_db':
XFormInstance.get_db(),
'view_name':
'couchforms/by_xmlns',
'es_query':
FormES().remove_default_filter('has_xmlns').remove_default_filter(
'has_user').size(0),
'sql_rows':
FormData.objects.count(),
}, {
'description': 'Cases (doc_type is "CommCareCase")',
'couch_db': CommCareCase.get_db(),
'view_name': 'case/by_owner',
'es_query': CaseES().size(0),
'sql_rows': None,
}]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description':
comp['description'],
'couch_docs':
comp['couch_db'].view(
comp['view_name'],
reduce=True,
).one()['value'],
'es_docs':
comp['es_query'].run().total,
'sql_rows':
comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def _get_users_by_loc_id(location_type):
"""Find any existing users previously assigned to this type"""
loc_ids = SQLLocation.objects.filter(
location_type=location_type).location_ids()
user_ids = list(UserES().domain(location_type.domain).show_inactive().term(
'user_location_id', list(loc_ids)).values_list('_id', flat=True))
return {
user_doc['user_location_id']: CommCareUser.wrap(user_doc)
for user_doc in iter_docs(CommCareUser.get_db(), user_ids)
if 'user_location_id' in user_doc
}
def remove_blocked_domain_contacts_from_hubspot(stdout=None):
"""
Removes contacts from Hubspot that are members of blocked domains.
:param stdout: the stdout of a management command (if applicable)
"""
blocked_domains = get_blocked_hubspot_domains()
for domain in blocked_domains:
if stdout:
stdout.write(f"\n\nChecking DOMAIN {domain}...")
user_query = UserES().domain(domain).source(['email', 'username'])
total_users = user_query.count()
chunk_size = 30 # Hubspot recommends fewer than 100 emails per request
num_chunks = int(math.ceil(float(total_users) / float(chunk_size)))
for chunk in range(num_chunks):
blocked_users = (user_query
.size(chunk_size)
.start(chunk * chunk_size)
.run()
.hits)
blocked_emails = []
for user in blocked_users:
username = user.get('username')
user_email = user.get('email')
blocked_emails.append(username)
if user_email and user_email != username:
blocked_emails.append(user_email)
ids_to_delete = _get_contact_ids_to_delete(set(blocked_emails))
if stdout:
stdout.write(f"Found {len(ids_to_delete)} id(s) to delete.")
num_deleted = sum(_delete_hubspot_contact(vid) for vid in ids_to_delete)
metrics_counter(
'commcare.hubspot_data.deleted_user.blocked_domain',
num_deleted,
tags={
'domain': domain,
}
)
def get_number_of_web_users(domain, year, month):
date_start, date_end = get_start_and_end_dates_of_month(year, month)
users_existing_in_domain = set(
doc['base_username']
for doc in UserES().domain(domain).web_users().created(
lte=date_end).run().hits)
users_who_accepted_invites = set(
email for email in Invitation.objects.filter(
domain=domain,
is_accepted=True,
invited_on__gte=date_start,
invited_on__lt=date_end).values_list('email', flat=True))
return len(users_existing_in_domain.union(users_who_accepted_invites))
def get_user_ids(user_type_mobile, domains):
"""
Returns the set of mobile user IDs if user_type_mobile is True,
else returns the set of web user IDs.
"""
query = UserES().show_inactive()
if domains:
query = query.domain(domains)
if user_type_mobile:
query = query.mobile_users()
else:
query = query.web_users()
return set(query.get_ids())
def commtrack_form_submissions(domains,
datespan,
interval,
datefield='received_on'):
mobile_workers = UserES().exclude_source().mobile_users().show_inactive(
).run().doc_ids
forms_after_date = (FormES().domain(domains).submitted(
gte=datespan.startdate, lte=datespan.enddate).date_histogram(
'date', datefield, interval).user_id(mobile_workers).size(0))
histo_data = forms_after_date.run().aggregations.date.as_facet_result()
forms_before_date = (FormES().domain(domains).submitted(
lt=datespan.startdate).user_id(mobile_workers).size(0))
forms_before_date = forms_before_date.run().total
return format_return_data(histo_data, forms_before_date, datespan)
def db_comparisons(request):
def _simple_view_couch_query(db, view_name):
return db.view(view_name, reduce=True).one()['value']
comparison_config = [
{
'description': 'Users (base_doc is "CouchUser")',
'couch_docs': _simple_view_couch_query(CommCareUser.get_db(), 'users/by_username'),
'es_query': UserES().remove_default_filter('active').size(0),
'sql_rows': User.objects.count(),
},
{
'description': 'Domains (doc_type is "Domain")',
'couch_docs': _simple_view_couch_query(Domain.get_db(), 'domain/by_status'),
'es_query': DomainES().size(0),
'sql_rows': None,
},
{
'description': 'Forms (doc_type is "XFormInstance")',
'couch_docs': get_number_of_forms_all_domains_in_couch(),
'es_query': FormES().remove_default_filter('has_xmlns')
.remove_default_filter('has_user')
.size(0),
'sql_rows': FormData.objects.exclude(domain__isnull=True).count(),
},
{
'description': 'Cases (doc_type is "CommCareCase")',
'couch_docs': get_total_case_count(),
'es_query': CaseES().size(0),
'sql_rows': CaseData.objects.exclude(domain__isnull=True).count(),
}
]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description': comp['description'],
'couch_docs': comp['couch_docs'],
'es_docs': comp['es_query'].run().total,
'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def commtrack_form_submissions(domains,
datespan,
interval,
datefield='received_on'):
mobile_workers = [
a['_id'] for a in UserES().fields(
[]).mobile_users().show_inactive().run().raw_hits
]
forms_after_date = (FormES().domain(domains).submitted(
gte=datespan.startdate, lte=datespan.enddate).date_histogram(
'date', datefield, interval).user_id(mobile_workers).size(0))
histo_data = forms_after_date.run().facet('date', 'entries')
forms_before_date = (FormES().domain(domains).submitted(
lt=datespan.startdate).user_id(mobile_workers).size(0))
forms_before_date = forms_before_date.run().total
return format_return_data(histo_data, forms_before_date, datespan)
def get_users_all_stats(domains, datespan, interval, user_type_mobile=None, require_submissions=False):
query = UserES().domain(domains).show_inactive().size(0)
if user_type_mobile:
query = query.mobile_users()
elif user_type_mobile is not None:
query = query.web_users()
if require_submissions:
query = query.user_ids(get_submitted_users(domains))
histo_data = (
query.created(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram("date", "created_on", interval)
.run()
.aggregations.date.as_facet_result()
)
users_before_date = query.created(lt=datespan.startdate).count()
return format_return_data(histo_data, users_before_date, datespan)
def track_periodic_data():
"""
Sync data that is neither event or page based with hubspot/Kissmetrics
:return:
"""
# Start by getting a list of web users mapped to their domains
if not KISSMETRICS_ENABLED and not HUBSPOT_ENABLED:
return
three_months_ago = date.today() - timedelta(days=90)
user_query = (UserES()
.web_users()
.last_logged_in(gte=three_months_ago)
.sort('date_joined', desc=True)
.source(['domains', 'email', 'date_joined', 'username'])
.analytics_enabled())
total_users = user_query.count()
chunk_size = 100
num_chunks = int(math.ceil(float(total_users) / float(chunk_size)))
# Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD
hubspot_number_of_users = 0
hubspot_number_of_domains_with_forms_gt_threshold = 0
blocked_domains = get_blocked_hubspot_domains()
blocked_email_domains = get_blocked_hubspot_email_domains()
for chunk in range(num_chunks):
users_to_domains = (user_query
.size(chunk_size)
.start(chunk * chunk_size)
.run()
.hits)
# users_to_domains is a list of dicts
domains_to_forms = (FormES()
.terms_aggregation('domain.exact', 'domain')
.size(0)
.run()
.aggregations.domain.counts_by_bucket())
domains_to_mobile_users = (UserES()
.mobile_users()
.terms_aggregation('domain.exact', 'domain')
.size(0)
.run()
.aggregations
.domain
.counts_by_bucket())
# Keep track of india and www data seperately
env = get_instance_string()
for num_forms in domains_to_forms.values():
if num_forms > HUBSPOT_THRESHOLD:
hubspot_number_of_domains_with_forms_gt_threshold += 1
# For each web user, iterate through their domains and select the max number of form submissions and
# max number of mobile workers
submit = []
for user in users_to_domains:
email = user.get('email') or user.get('username')
if not _email_is_valid(email):
continue
email_domain = email.split('@')[-1]
if email_domain in blocked_email_domains:
metrics_gauge(
'commcare.hubspot_data.rejected.periodic_task.email_domain',
1,
tags={
'email_domain': email_domain,
}
)
continue
username_email_domain = user.get('username').split('@')[-1]
if username_email_domain in blocked_email_domains:
metrics_gauge(
'commcare.hubspot_data.rejected.periodic_task.username',
1,
tags={
'username': username_email_domain,
}
)
continue
hubspot_number_of_users += 1
date_created = user.get('date_joined')
max_forms = 0
max_workers = 0
max_export = 0
max_report = 0
is_member_of_blocked_domain = False
for domain in user['domains']:
if domain in blocked_domains:
metrics_gauge(
'commcare.hubspot_data.rejected.periodic_task.domain',
1,
tags={
'domain': domain,
}
)
is_member_of_blocked_domain = True
break
if domain in domains_to_forms and domains_to_forms[domain] > max_forms:
max_forms = domains_to_forms[domain]
if domain in domains_to_mobile_users and domains_to_mobile_users[domain] > max_workers:
max_workers = domains_to_mobile_users[domain]
if _get_export_count(domain) > max_export:
max_export = _get_export_count(domain)
if _get_report_count(domain) > max_report:
max_report = _get_report_count(domain)
if is_member_of_blocked_domain:
continue
project_spaces_created = ", ".join(get_domains_created_by_user(email))
user_json = {
'email': email,
'properties': [
{
'property': '{}max_form_submissions_in_a_domain'.format(env),
'value': max_forms
},
{
'property': '{}max_mobile_workers_in_a_domain'.format(env),
'value': max_workers
},
{
'property': '{}project_spaces_created_by_user'.format(env),
'value': project_spaces_created,
},
{
'property': '{}over_300_form_submissions'.format(env),
'value': max_forms > HUBSPOT_THRESHOLD
},
{
'property': '{}date_created'.format(env),
'value': date_created
},
{
'property': '{}max_exports_in_a_domain'.format(env),
'value': max_export
},
{
'property': '{}max_custom_reports_in_a_domain'.format(env),
'value': max_report
}
]
}
submit.append(user_json)
submit_json = json.dumps(submit)
submit_data_to_hub_and_kiss(submit_json)
metrics_gauge('commcare.hubspot.web_users_processed', hubspot_number_of_users,
multiprocess_mode=MPM_LIVESUM)
metrics_gauge(
'commcare.hubspot.domains_with_forms_gt_threshold', hubspot_number_of_domains_with_forms_gt_threshold,
multiprocess_mode=MPM_MAX
)
def user_choices(self):
users = UserES().domain(self.domain).fields(['_id',
'username']).run().hits
return [(user['_id'], raw_username(user['username']))
for user in users]
def members(self):
# FYI use '_id' instead of '__group_ids' in query below in case of
# ES not updating immediately
return get_simplified_users(UserES().mobile_users().domain(
self.domain).term("_id", self.group.get_user_ids()))
def all_users(self):
return get_simplified_users(
UserES().mobile_users().domain(self.domain)
)
def get_active_users(cls):
six_months_ago = date.today() - timedelta(days=180)
users = UserES().web_users().last_logged_in(gte=six_months_ago).run().hits
return [WebUser.wrap(u) for u in users]
def _user_query(self):
return (UserES().domain(
self.definition.domain).mobile_users().show_inactive().metadata(
PROFILE_SLUG, self.id))
def track_periodic_data():
"""
Sync data that is neither event or page based with hubspot/Kissmetrics
:return:
"""
# Start by getting a list of web users mapped to their domains
six_months_ago = date.today() - timedelta(days=180)
users_to_domains = (UserES().web_users().last_logged_in(
gte=six_months_ago).fields(['domains', 'email', 'date_joined'
]).analytics_enabled().run().hits)
# users_to_domains is a list of dicts
domains_to_forms = FormES().terms_aggregation('domain', 'domain').size(0).run()\
.aggregations.domain.counts_by_bucket()
domains_to_mobile_users = UserES().mobile_users().terms_aggregation('domain', 'domain').size(0).run()\
.aggregations.domain.counts_by_bucket()
# Keep track of india and www data seperately
env = get_instance_string()
# Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD
number_of_users = 0
number_of_domains_with_forms_gt_threshold = 0
for num_forms in domains_to_forms.values():
if num_forms > HUBSPOT_THRESHOLD:
number_of_domains_with_forms_gt_threshold += 1
# For each web user, iterate through their domains and select the max number of form submissions and
# max number of mobile workers
submit = []
for user in users_to_domains:
email = user.get('email')
if not _email_is_valid(email):
continue
number_of_users += 1
date_created = user.get('date_joined')
max_forms = 0
max_workers = 0
for domain in user['domains']:
if domain in domains_to_forms and domains_to_forms[
domain] > max_forms:
max_forms = domains_to_forms[domain]
if domain in domains_to_mobile_users and domains_to_mobile_users[
domain] > max_workers:
max_workers = domains_to_mobile_users[domain]
project_spaces_created = ", ".join(get_domains_created_by_user(email))
user_json = {
'email':
email,
'properties': [{
'property':
'{}max_form_submissions_in_a_domain'.format(env),
'value':
max_forms
}, {
'property':
'{}max_mobile_workers_in_a_domain'.format(env),
'value':
max_workers
}, {
'property':
'{}project_spaces_created_by_user'.format(env),
'value':
project_spaces_created,
}, {
'property':
'{}over_300_form_submissions'.format(env),
'value':
max_forms > HUBSPOT_THRESHOLD
}, {
'property': '{}date_created'.format(env),
'value': date_created
}]
}
submit.append(user_json)
submit_json = json.dumps(submit)
submit_data_to_hub_and_kiss(submit_json)
update_datadog_metrics({
DATADOG_WEB_USERS_GAUGE:
number_of_users,
DATADOG_DOMAINS_EXCEEDING_FORMS_GAUGE:
number_of_domains_with_forms_gt_threshold
})
def track_periodic_data():
"""
Sync data that is neither event or page based with hubspot/Kissmetrics
:return:
"""
# Start by getting a list of web users mapped to their domains
six_months_ago = date.today() - timedelta(days=180)
users_to_domains = UserES().web_users().last_logged_in(gte=six_months_ago)\
.fields(['domains', 'email', 'date_joined'])\
.run().hits
# users_to_domains is a list of dicts
domains_to_forms = FormES().terms_aggregation('domain', 'domain').size(0).run()\
.aggregations.domain.counts_by_bucket()
domains_to_mobile_users = UserES().mobile_users().terms_aggregation('domain', 'domain').size(0).run()\
.aggregations.domain.counts_by_bucket()
# Keep track of india and www data seperately
env = get_instance_string()
# For each web user, iterate through their domains and select the max number of form submissions and
# max number of mobile workers
submit = []
for user in users_to_domains:
email = user.get('email')
if not email:
continue
date_created = user.get('date_joined')
max_forms = 0
max_workers = 0
for domain in user['domains']:
if domain in domains_to_forms and domains_to_forms[
domain] > max_forms:
max_forms = domains_to_forms[domain]
if domain in domains_to_mobile_users and domains_to_mobile_users[
domain] > max_workers:
max_workers = domains_to_mobile_users[domain]
project_spaces_created = ", ".join(get_domains_created_by_user(email))
user_json = {
'email':
email,
'properties': [{
'property':
'{}max_form_submissions_in_a_domain'.format(env),
'value':
max_forms
}, {
'property':
'{}max_mobile_workers_in_a_domain'.format(env),
'value':
max_workers
}, {
'property':
'{}project_spaces_created_by_user'.format(env),
'value':
project_spaces_created,
}, {
'property':
'{}over_300_form_submissions'.format(env),
'value':
max_forms > 300
}, {
'property': '{}date_created'.format(env),
'value': date_created
}]
}
submit.append(user_json)
submit_json = json.dumps(submit)
submit_data_to_hub_and_kiss(submit_json)
def get_users_by_location_filter(self, location_ids):
return UserES().domain(self.domain).location(location_ids).values_list(
'_id', flat=True)
