def track_web_user_registration_hubspot(request, web_user, properties): if not settings.ANALYTICS_IDS.get('HUBSPOT_API_ID'): return tracking_info = { 'created_account_in_hq': True, 'is_a_commcare_user': True, 'lifecyclestage': 'lead', } env = get_instance_string() tracking_info['{}date_created'.format(env)] = web_user.date_joined.isoformat() if (hasattr(web_user, 'phone_numbers') and len(web_user.phone_numbers) > 0): tracking_info.update({ 'phone': web_user.phone_numbers[0], }) if web_user.atypical_user: tracking_info.update({ 'atypical_user': True }) tracking_info.update(get_ab_test_properties(web_user)) tracking_info.update(properties) send_hubspot_form( HUBSPOT_SIGNUP_FORM_ID, request, user=web_user, extra_fields=tracking_info )
def get_domain_membership_properties(couch_user): env = get_instance_string() return { "{}number_of_project_spaces".format(env): len(couch_user.domains), "{}project_spaces_list".format(env): '\n'.join(couch_user.domains), }
def track_web_user_registration_hubspot(request, web_user, properties): if not settings.ANALYTICS_IDS.get('HUBSPOT_API_ID'): return tracking_info = { 'created_account_in_hq': 'true', 'is_a_commcare_user': '******', 'lifecyclestage': 'lead', } env = get_instance_string() tracking_info['{}date_created'.format( env)] = web_user.date_joined.isoformat() if (hasattr(web_user, 'phone_numbers') and len(web_user.phone_numbers) > 0): tracking_info.update({ 'phone': web_user.phone_numbers[0], }) if web_user.atypical_user: tracking_info.update({'atypical_user': '******'}) tracking_info.update(get_ab_test_properties(web_user)) tracking_info.update(properties) send_hubspot_form(HUBSPOT_SIGNUP_FORM_ID, request, user=web_user, extra_fields=tracking_info)
def get_subscription_properties_by_user(couch_user): def _is_paying_subscription(subscription, plan_version): NON_PAYING_SERVICE_TYPES = [ SubscriptionType.TRIAL, SubscriptionType.EXTENDED_TRIAL, SubscriptionType.SANDBOX, SubscriptionType.INTERNAL, ] NON_PAYING_PRO_BONO_STATUSES = [ProBonoStatus.YES, ProBonoStatus.DISCOUNTED] return ( plan_version.plan.visibility != SoftwarePlanVisibility.TRIAL and subscription.service_type not in NON_PAYING_SERVICE_TYPES and subscription.pro_bono_status not in NON_PAYING_PRO_BONO_STATUSES and plan_version.plan.edition != SoftwarePlanEdition.COMMUNITY ) # Note: using "yes" and "no" instead of True and False because spec calls # for using these values. (True is just converted to "True" in KISSmetrics) all_subscriptions = [] paying_subscribed_editions = [] subscribed_editions = [] for domain_name in couch_user.domains: plan_version, subscription = Subscription.get_subscribed_plan_by_domain(domain_name) subscribed_editions.append(plan_version.plan.edition) if subscription is not None: all_subscriptions.append(subscription) if subscription is not None and _is_paying_subscription(subscription, plan_version): paying_subscribed_editions.append(plan_version.plan.edition) def _is_one_of_editions(edition): return "yes" if edition in subscribed_editions else "no" def _is_a_pro_bono_status(status): return "yes" if status in [s.pro_bono_status for s in all_subscriptions] else "no" def _is_on_extended_trial(): service_types = [s.service_type for s in all_subscriptions] return "yes" if SubscriptionType.EXTENDED_TRIAL in service_types else "no" def _max_edition(): for edition in paying_subscribed_editions: assert edition in [e[0] for e in SoftwarePlanEdition.CHOICES] return max(paying_subscribed_editions) if paying_subscribed_editions else "" env = get_instance_string() return { "{}is_on_community_plan".format(env): _is_one_of_editions(SoftwarePlanEdition.COMMUNITY), "{}is_on_standard_plan".format(env): _is_one_of_editions(SoftwarePlanEdition.STANDARD), "{}is_on_pro_plan".format(env): _is_one_of_editions(SoftwarePlanEdition.PRO), "{}is_on_pro_bono_plan".format(env): _is_a_pro_bono_status(ProBonoStatus.YES), "{}is_on_discounted_plan".format(env): _is_a_pro_bono_status(ProBonoStatus.DISCOUNTED), "{}is_on_extended_trial_plan".format(env): _is_on_extended_trial(), "{}max_edition_of_paying_plan".format(env): _max_edition(), }
def get_subscription_properties_by_user(couch_user): def _is_paying_subscription(subscription, plan_version): NON_PAYING_SERVICE_TYPES = [ SubscriptionType.TRIAL, SubscriptionType.EXTENDED_TRIAL, SubscriptionType.SANDBOX, SubscriptionType.INTERNAL, ] NON_PAYING_PRO_BONO_STATUSES = [ ProBonoStatus.YES, ProBonoStatus.DISCOUNTED, ] return (plan_version.plan.visibility != SoftwarePlanVisibility.TRIAL and subscription.service_type not in NON_PAYING_SERVICE_TYPES and subscription.pro_bono_status not in NON_PAYING_PRO_BONO_STATUSES and plan_version.plan.edition != SoftwarePlanEdition.COMMUNITY) # Note: using "yes" and "no" instead of True and False because spec calls # for using these values. (True is just converted to "True" in KISSmetrics) all_subscriptions = [] paying_subscribed_editions = [] subscribed_editions = [] for domain_name in couch_user.domains: subscription = Subscription.get_active_subscription_by_domain( domain_name) plan_version = (subscription.plan_version if subscription is not None else DefaultProductPlan.get_default_plan_version()) subscribed_editions.append(plan_version.plan.edition) if subscription is not None: all_subscriptions.append(subscription) if subscription is not None and _is_paying_subscription( subscription, plan_version): paying_subscribed_editions.append(plan_version.plan.edition) def _is_one_of_editions(edition): return 'yes' if edition in subscribed_editions else 'no' def _is_a_pro_bono_status(status): return 'yes' if status in [ s.pro_bono_status for s in all_subscriptions ] else 'no' def _is_on_extended_trial(): service_types = [s.service_type for s in all_subscriptions] return 'yes' if SubscriptionType.EXTENDED_TRIAL in service_types else 'no' def _max_edition(): for edition in paying_subscribed_editions: assert edition in [e[0] for e in SoftwarePlanEdition.CHOICES] return max( paying_subscribed_editions) if paying_subscribed_editions else '' env = get_instance_string() return { '{}is_on_community_plan'.format(env): _is_one_of_editions(SoftwarePlanEdition.COMMUNITY), '{}is_on_standard_plan'.format(env): _is_one_of_editions(SoftwarePlanEdition.STANDARD), '{}is_on_pro_plan'.format(env): _is_one_of_editions(SoftwarePlanEdition.PRO), '{}is_on_pro_bono_plan'.format(env): _is_a_pro_bono_status(ProBonoStatus.YES), '{}is_on_discounted_plan'.format(env): _is_a_pro_bono_status(ProBonoStatus.DISCOUNTED), '{}is_on_extended_trial_plan'.format(env): _is_on_extended_trial(), '{}max_edition_of_paying_plan'.format(env): _max_edition() }
def track_periodic_data(): """ Sync data that is neither event or page based with hubspot/Kissmetrics :return: """ # Start by getting a list of web users mapped to their domains six_months_ago = date.today() - timedelta(days=180) users_to_domains = (UserES().web_users().last_logged_in( gte=six_months_ago).fields(['domains', 'email', 'date_joined' ]).analytics_enabled().run().hits) # users_to_domains is a list of dicts domains_to_forms = FormES().terms_aggregation('domain', 'domain').size(0).run()\ .aggregations.domain.counts_by_bucket() domains_to_mobile_users = UserES().mobile_users().terms_aggregation('domain', 'domain').size(0).run()\ .aggregations.domain.counts_by_bucket() # Keep track of india and www data seperately env = get_instance_string() # Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD number_of_users = 0 number_of_domains_with_forms_gt_threshold = 0 for num_forms in domains_to_forms.values(): if num_forms > HUBSPOT_THRESHOLD: number_of_domains_with_forms_gt_threshold += 1 # For each web user, iterate through their domains and select the max number of form submissions and # max number of mobile workers submit = [] for user in users_to_domains: email = user.get('email') if not _email_is_valid(email): continue number_of_users += 1 date_created = user.get('date_joined') max_forms = 0 max_workers = 0 for domain in user['domains']: if domain in domains_to_forms and domains_to_forms[ domain] > max_forms: max_forms = domains_to_forms[domain] if domain in domains_to_mobile_users and domains_to_mobile_users[ domain] > max_workers: max_workers = domains_to_mobile_users[domain] project_spaces_created = ", ".join(get_domains_created_by_user(email)) user_json = { 'email': email, 'properties': [{ 'property': '{}max_form_submissions_in_a_domain'.format(env), 'value': max_forms }, { 'property': '{}max_mobile_workers_in_a_domain'.format(env), 'value': max_workers }, { 'property': '{}project_spaces_created_by_user'.format(env), 'value': project_spaces_created, }, { 'property': '{}over_300_form_submissions'.format(env), 'value': max_forms > HUBSPOT_THRESHOLD }, { 'property': '{}date_created'.format(env), 'value': date_created }] } submit.append(user_json) submit_json = json.dumps(submit) submit_data_to_hub_and_kiss(submit_json) update_datadog_metrics({ DATADOG_WEB_USERS_GAUGE: number_of_users, DATADOG_DOMAINS_EXCEEDING_FORMS_GAUGE: number_of_domains_with_forms_gt_threshold })
def track_periodic_data(): """ Sync data that is neither event or page based with hubspot/Kissmetrics :return: """ # Start by getting a list of web users mapped to their domains six_months_ago = date.today() - timedelta(days=180) users_to_domains = UserES().web_users().last_logged_in(gte=six_months_ago)\ .fields(['domains', 'email', 'date_joined'])\ .run().hits # users_to_domains is a list of dicts domains_to_forms = FormES().terms_aggregation('domain', 'domain').size(0).run()\ .aggregations.domain.counts_by_bucket() domains_to_mobile_users = UserES().mobile_users().terms_aggregation('domain', 'domain').size(0).run()\ .aggregations.domain.counts_by_bucket() # Keep track of india and www data seperately env = get_instance_string() # For each web user, iterate through their domains and select the max number of form submissions and # max number of mobile workers submit = [] for user in users_to_domains: email = user.get('email') if not email: continue date_created = user.get('date_joined') max_forms = 0 max_workers = 0 for domain in user['domains']: if domain in domains_to_forms and domains_to_forms[ domain] > max_forms: max_forms = domains_to_forms[domain] if domain in domains_to_mobile_users and domains_to_mobile_users[ domain] > max_workers: max_workers = domains_to_mobile_users[domain] project_spaces_created = ", ".join(get_domains_created_by_user(email)) user_json = { 'email': email, 'properties': [{ 'property': '{}max_form_submissions_in_a_domain'.format(env), 'value': max_forms }, { 'property': '{}max_mobile_workers_in_a_domain'.format(env), 'value': max_workers }, { 'property': '{}project_spaces_created_by_user'.format(env), 'value': project_spaces_created, }, { 'property': '{}over_300_form_submissions'.format(env), 'value': max_forms > 300 }, { 'property': '{}date_created'.format(env), 'value': date_created }] } submit.append(user_json) submit_json = json.dumps(submit) submit_data_to_hub_and_kiss(submit_json)
def track_periodic_data(): """ Sync data that is neither event or page based with hubspot/Kissmetrics :return: """ # Start by getting a list of web users mapped to their domains if not KISSMETRICS_ENABLED and not HUBSPOT_ENABLED: return three_months_ago = date.today() - timedelta(days=90) user_query = (UserES() .web_users() .last_logged_in(gte=three_months_ago) .sort('date_joined', desc=True) .source(['domains', 'email', 'date_joined', 'username']) .analytics_enabled()) total_users = user_query.count() chunk_size = 100 num_chunks = int(math.ceil(float(total_users) / float(chunk_size))) # Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD hubspot_number_of_users = 0 hubspot_number_of_domains_with_forms_gt_threshold = 0 blocked_domains = get_blocked_hubspot_domains() blocked_email_domains = get_blocked_hubspot_email_domains() for chunk in range(num_chunks): users_to_domains = (user_query .size(chunk_size) .start(chunk * chunk_size) .run() .hits) # users_to_domains is a list of dicts domains_to_forms = (FormES() .terms_aggregation('domain.exact', 'domain') .size(0) .run() .aggregations.domain.counts_by_bucket()) domains_to_mobile_users = (UserES() .mobile_users() .terms_aggregation('domain.exact', 'domain') .size(0) .run() .aggregations .domain .counts_by_bucket()) # Keep track of india and www data seperately env = get_instance_string() for num_forms in domains_to_forms.values(): if num_forms > HUBSPOT_THRESHOLD: hubspot_number_of_domains_with_forms_gt_threshold += 1 # For each web user, iterate through their domains and select the max number of form submissions and # max number of mobile workers submit = [] for user in users_to_domains: email = user.get('email') or user.get('username') if not _email_is_valid(email): continue email_domain = email.split('@')[-1] if email_domain in blocked_email_domains: metrics_gauge( 'commcare.hubspot_data.rejected.periodic_task.email_domain', 1, tags={ 'email_domain': email_domain, } ) continue username_email_domain = user.get('username').split('@')[-1] if username_email_domain in blocked_email_domains: metrics_gauge( 'commcare.hubspot_data.rejected.periodic_task.username', 1, tags={ 'username': username_email_domain, } ) continue hubspot_number_of_users += 1 date_created = user.get('date_joined') max_forms = 0 max_workers = 0 max_export = 0 max_report = 0 is_member_of_blocked_domain = False for domain in user['domains']: if domain in blocked_domains: metrics_gauge( 'commcare.hubspot_data.rejected.periodic_task.domain', 1, tags={ 'domain': domain, } ) is_member_of_blocked_domain = True break if domain in domains_to_forms and domains_to_forms[domain] > max_forms: max_forms = domains_to_forms[domain] if domain in domains_to_mobile_users and domains_to_mobile_users[domain] > max_workers: max_workers = domains_to_mobile_users[domain] if _get_export_count(domain) > max_export: max_export = _get_export_count(domain) if _get_report_count(domain) > max_report: max_report = _get_report_count(domain) if is_member_of_blocked_domain: continue project_spaces_created = ", ".join(get_domains_created_by_user(email)) user_json = { 'email': email, 'properties': [ { 'property': '{}max_form_submissions_in_a_domain'.format(env), 'value': max_forms }, { 'property': '{}max_mobile_workers_in_a_domain'.format(env), 'value': max_workers }, { 'property': '{}project_spaces_created_by_user'.format(env), 'value': project_spaces_created, }, { 'property': '{}over_300_form_submissions'.format(env), 'value': max_forms > HUBSPOT_THRESHOLD }, { 'property': '{}date_created'.format(env), 'value': date_created }, { 'property': '{}max_exports_in_a_domain'.format(env), 'value': max_export }, { 'property': '{}max_custom_reports_in_a_domain'.format(env), 'value': max_report } ] } submit.append(user_json) submit_json = json.dumps(submit) submit_data_to_hub_and_kiss(submit_json) metrics_gauge('commcare.hubspot.web_users_processed', hubspot_number_of_users, multiprocess_mode=MPM_LIVESUM) metrics_gauge( 'commcare.hubspot.domains_with_forms_gt_threshold', hubspot_number_of_domains_with_forms_gt_threshold, multiprocess_mode=MPM_MAX )
def track_periodic_data(): """ Sync data that is neither event or page based with hubspot/Kissmetrics :return: """ # Start by getting a list of web users mapped to their domains three_months_ago = date.today() - timedelta(days=90) user_query = (UserES() .web_users() .last_logged_in(gte=three_months_ago) .sort('date_joined', desc=True) .source(['domains', 'email', 'date_joined']) .analytics_enabled()) total_users = user_query.count() chunk_size = 100 num_chunks = int(math.ceil(float(total_users) / float(chunk_size))) # Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD hubspot_number_of_users = 0 hubspot_number_of_domains_with_forms_gt_threshold = 0 for chunk in range(num_chunks): users_to_domains = (user_query .size(chunk_size) .start(chunk * chunk_size) .run() .hits) # users_to_domains is a list of dicts domains_to_forms = (FormES() .terms_aggregation('domain', 'domain') .size(0) .run() .aggregations.domain.counts_by_bucket()) domains_to_mobile_users = (UserES() .mobile_users() .terms_aggregation('domain', 'domain') .size(0) .run() .aggregations .domain .counts_by_bucket()) # Keep track of india and www data seperately env = get_instance_string() for num_forms in domains_to_forms.values(): if num_forms > HUBSPOT_THRESHOLD: hubspot_number_of_domains_with_forms_gt_threshold += 1 # For each web user, iterate through their domains and select the max number of form submissions and # max number of mobile workers submit = [] for user in users_to_domains: email = user.get('email') if not _email_is_valid(email): continue hubspot_number_of_users += 1 date_created = user.get('date_joined') max_forms = 0 max_workers = 0 max_export = 0 max_report = 0 for domain in user['domains']: if domain in domains_to_forms and domains_to_forms[domain] > max_forms: max_forms = domains_to_forms[domain] if domain in domains_to_mobile_users and domains_to_mobile_users[domain] > max_workers: max_workers = domains_to_mobile_users[domain] if _get_export_count(domain) > max_export: max_export = _get_export_count(domain) if _get_report_count(domain) > max_report: max_report = _get_report_count(domain) project_spaces_created = ", ".join(get_domains_created_by_user(email)) user_json = { 'email': email, 'properties': [ { 'property': '{}max_form_submissions_in_a_domain'.format(env), 'value': max_forms }, { 'property': '{}max_mobile_workers_in_a_domain'.format(env), 'value': max_workers }, { 'property': '{}project_spaces_created_by_user'.format(env), 'value': project_spaces_created, }, { 'property': '{}over_300_form_submissions'.format(env), 'value': max_forms > HUBSPOT_THRESHOLD }, { 'property': '{}date_created'.format(env), 'value': date_created }, { 'property': '{}max_exports_in_a_domain'.format(env), 'value': max_export }, { 'property': '{}max_custom_reports_in_a_domain'.format(env), 'value': max_report } ] } submit.append(user_json) submit_json = json.dumps(submit) submit_data_to_hub_and_kiss(submit_json) update_datadog_metrics({ DATADOG_WEB_USERS_GAUGE: hubspot_number_of_users, DATADOG_DOMAINS_EXCEEDING_FORMS_GAUGE: hubspot_number_of_domains_with_forms_gt_threshold })
def track_periodic_data(): """ Sync data that is neither event or page based with hubspot/Kissmetrics :return: """ # Start by getting a list of web users mapped to their domains six_months_ago = date.today() - timedelta(days=180) users_to_domains = ( UserES().web_users().last_logged_in(gte=six_months_ago).fields(["domains", "email", "date_joined"]).run().hits ) # users_to_domains is a list of dicts domains_to_forms = ( FormES().terms_aggregation("domain", "domain").size(0).run().aggregations.domain.counts_by_bucket() ) domains_to_mobile_users = ( UserES() .mobile_users() .terms_aggregation("domain", "domain") .size(0) .run() .aggregations.domain.counts_by_bucket() ) # Keep track of india and www data seperately env = get_instance_string() # Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD number_of_users = 0 number_of_domains_with_forms_gt_threshold = 0 for num_forms in domains_to_forms.values(): if num_forms > HUBSPOT_THRESHOLD: number_of_domains_with_forms_gt_threshold += 1 # For each web user, iterate through their domains and select the max number of form submissions and # max number of mobile workers submit = [] for user in users_to_domains: email = user.get("email") if not _email_is_valid(email): continue number_of_users += 1 date_created = user.get("date_joined") max_forms = 0 max_workers = 0 for domain in user["domains"]: if domain in domains_to_forms and domains_to_forms[domain] > max_forms: max_forms = domains_to_forms[domain] if domain in domains_to_mobile_users and domains_to_mobile_users[domain] > max_workers: max_workers = domains_to_mobile_users[domain] project_spaces_created = ", ".join(get_domains_created_by_user(email)) user_json = { "email": email, "properties": [ {"property": "{}max_form_submissions_in_a_domain".format(env), "value": max_forms}, {"property": "{}max_mobile_workers_in_a_domain".format(env), "value": max_workers}, {"property": "{}project_spaces_created_by_user".format(env), "value": project_spaces_created}, {"property": "{}over_300_form_submissions".format(env), "value": max_forms > HUBSPOT_THRESHOLD}, {"property": "{}date_created".format(env), "value": date_created}, ], } submit.append(user_json) submit_json = json.dumps(submit) submit_data_to_hub_and_kiss(submit_json) update_datadog_metrics( { DATADOG_WEB_USERS_GAUGE: number_of_users, DATADOG_DOMAINS_EXCEEDING_FORMS_GAUGE: number_of_domains_with_forms_gt_threshold, } )
def track_periodic_data(): """ Sync data that is neither event or page based with hubspot/Kissmetrics :return: """ # Start by getting a list of web users mapped to their domains if not KISSMETRICS_ENABLED and not HUBSPOT_ENABLED: return three_months_ago = date.today() - timedelta(days=90) user_query = (UserES().web_users().last_logged_in( gte=three_months_ago).sort('date_joined', desc=True).source( ['domains', 'email', 'date_joined']).analytics_enabled()) total_users = user_query.count() chunk_size = 100 num_chunks = int(math.ceil(float(total_users) / float(chunk_size))) # Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD hubspot_number_of_users = 0 hubspot_number_of_domains_with_forms_gt_threshold = 0 for chunk in range(num_chunks): users_to_domains = (user_query.size(chunk_size).start( chunk * chunk_size).run().hits) # users_to_domains is a list of dicts domains_to_forms = (FormES().terms_aggregation( 'domain', 'domain').size(0).run().aggregations.domain.counts_by_bucket()) domains_to_mobile_users = (UserES().mobile_users().terms_aggregation( 'domain', 'domain').size(0).run().aggregations.domain.counts_by_bucket()) # Keep track of india and www data seperately env = get_instance_string() for num_forms in domains_to_forms.values(): if num_forms > HUBSPOT_THRESHOLD: hubspot_number_of_domains_with_forms_gt_threshold += 1 # For each web user, iterate through their domains and select the max number of form submissions and # max number of mobile workers submit = [] for user in users_to_domains: email = user.get('email') if not _email_is_valid(email): continue hubspot_number_of_users += 1 date_created = user.get('date_joined') max_forms = 0 max_workers = 0 max_export = 0 max_report = 0 for domain in user['domains']: if domain in domains_to_forms and domains_to_forms[ domain] > max_forms: max_forms = domains_to_forms[domain] if domain in domains_to_mobile_users and domains_to_mobile_users[ domain] > max_workers: max_workers = domains_to_mobile_users[domain] if _get_export_count(domain) > max_export: max_export = _get_export_count(domain) if _get_report_count(domain) > max_report: max_report = _get_report_count(domain) project_spaces_created = ", ".join( get_domains_created_by_user(email)) user_json = { 'email': email, 'properties': [{ 'property': '{}max_form_submissions_in_a_domain'.format(env), 'value': max_forms }, { 'property': '{}max_mobile_workers_in_a_domain'.format(env), 'value': max_workers }, { 'property': '{}project_spaces_created_by_user'.format(env), 'value': project_spaces_created, }, { 'property': '{}over_300_form_submissions'.format(env), 'value': max_forms > HUBSPOT_THRESHOLD }, { 'property': '{}date_created'.format(env), 'value': date_created }, { 'property': '{}max_exports_in_a_domain'.format(env), 'value': max_export }, { 'property': '{}max_custom_reports_in_a_domain'.format(env), 'value': max_report }] } submit.append(user_json) submit_json = json.dumps(submit) submit_data_to_hub_and_kiss(submit_json) update_datadog_metrics({ DATADOG_WEB_USERS_GAUGE: hubspot_number_of_users, DATADOG_DOMAINS_EXCEEDING_FORMS_GAUGE: hubspot_number_of_domains_with_forms_gt_threshold })