def test_app_submission_breakdown(self, combination_count_list):
"""
The breakdown of this report is (app, device, userid, username): count
"""
domain = 'test-data-analytics'
received = datetime(2016, 3, 24)
month = DateSpan.from_month(3, 2016)
for app, device, userid, username, count in combination_count_list:
for i in range(count):
save_to_es_analytics_db(domain, received, app, device, userid, username)
self.es.indices.refresh(XFORM_INDEX_INFO.index)
data_back = get_app_submission_breakdown_es(domain, month)
normalized_data_back = set(data_back)
self.assertEqual(set(combination_count_list), normalized_data_back)
def test_app_submission_breakdown(self, combination_count_list):
"""
The breakdown of this report is (app, device, userid, username): count
"""
domain = 'test-data-analytics'
received = datetime(2016, 3, 24)
month = DateSpan.from_month(3, 2016)
for app, device, userid, username, count in combination_count_list:
for i in range(count):
save_to_es_analytics_db(domain, received, app, device, userid, username)
self.es.indices.refresh(XFORM_INDEX_INFO.index)
data_back = get_app_submission_breakdown_es(domain, month)
normalized_data_back = set(data_back)
self.assertEqual(set(combination_count_list), normalized_data_back)
def _get_malt_row_dicts(domain_name, monthspan, users_by_id, run_date):
"""
Only processes domains that have had a form submission since the startdate of the month
Includes expensive elasticsearch query
:param domain_name: domain name
:param monthspan: DateSpan of month to process
:param users_by_id: list of dictionaries [{user_id: user_obj}, ...]
"""
malt_row_dicts = []
app_rows = get_app_submission_breakdown_es(domain_name, monthspan, list(users_by_id))
for app_row in app_rows:
user = users_by_id[app_row.user_id]
malt_row_dict = _build_malt_row_dict(app_row, domain_name, user, monthspan, run_date)
malt_row_dicts.append(malt_row_dict)
return malt_row_dicts
def _get_malt_row_dicts(self, domain_name, monthspan, users_by_id):
malt_row_dicts = []
for users in chunked(list(users_by_id), 1000):
apps_submitted_for = get_app_submission_breakdown_es(domain_name, monthspan, users)
for app_row in apps_submitted_for:
app_id = app_row.app_id
num_of_forms = app_row.doc_count
try:
app_data = self._app_data(domain_name, app_id)
user_id, username, user_type, email = self._user_data(
app_row.user_id,
app_row.username,
users_by_id
)
except Exception as ex:
logger.error("Failed to get rows for user {id}, app {app_id}. Exception is {ex}".format
(id=user_id, app_id=app_id, ex=str(ex)), exc_info=True)
continue
malt_dict = {
'month': monthspan.startdate,
'user_id': user_id,
'username': username,
'email': email,
'user_type': user_type,
'domain_name': domain_name,
'num_of_forms': num_of_forms,
'app_id': app_id or MISSING_APP_ID,
'device_id': app_row.device_id,
'wam': AMPLIFY_COUCH_TO_SQL_MAP.get(app_data.wam, NOT_SET),
'pam': AMPLIFY_COUCH_TO_SQL_MAP.get(app_data.pam, NOT_SET),
'use_threshold': app_data.use_threshold,
'experienced_threshold': app_data.experienced_threshold,
'is_app_deleted': app_data.is_app_deleted,
}
malt_row_dicts.append(malt_dict)
return malt_row_dicts
def _get_malt_row_dicts(self, domain_name, monthspan, users_by_id):
malt_row_dicts = []
for users in chunked(list(users_by_id), 1000):
apps_submitted_for = get_app_submission_breakdown_es(domain_name, monthspan, users)
for app_row in apps_submitted_for:
app_id = app_row.app_id
num_of_forms = app_row.doc_count
try:
app_data = self._app_data(domain_name, app_id)
user_id, username, user_type, email = self._user_data(
app_row.user_id,
app_row.username,
users_by_id
)
except Exception as ex:
logger.error("Failed to get rows for user {id}, app {app_id}. Exception is {ex}".format
(id=user_id, app_id=app_id, ex=str(ex)), exc_info=True)
continue
malt_dict = {
'month': monthspan.startdate,
'user_id': user_id,
'username': username,
'email': email,
'user_type': user_type,
'domain_name': domain_name,
'num_of_forms': num_of_forms,
'app_id': app_id or MISSING_APP_ID,
'device_id': app_row.device_id,
'wam': AMPLIFY_COUCH_TO_SQL_MAP.get(app_data.wam, NOT_SET),
'pam': AMPLIFY_COUCH_TO_SQL_MAP.get(app_data.pam, NOT_SET),
'use_threshold': app_data.use_threshold,
'experienced_threshold': app_data.experienced_threshold,
'is_app_deleted': app_data.is_app_deleted,
}
malt_row_dicts.append(malt_dict)
return malt_row_dicts
