def test_successfully_updates(self):
self._save_form_data(self.app_id, datetime.datetime(2020, 1, 15))
self.es.indices.refresh(XFORM_INDEX_INFO.index)
monthspan = DateSpan.from_month(1, 2020)
generate_malt([monthspan], domains=[self.domain.name])
malt_row = MALTRow.objects.get(user_id=self.user._id,
app_id=self.app_id,
device_id=self.DEVICE_ID,
month=monthspan.startdate)
self.assertEqual(malt_row.num_of_forms, 1)
# hacky way to simulate last run date in between form submissions
malt_row.last_run_date = datetime.datetime(2020, 1, 17)
malt_row.save()
self._save_form_data(self.app_id, datetime.datetime(2020, 1, 20))
self.es.indices.refresh(XFORM_INDEX_INFO.index)
# mock bulk_create to avoid raising an actual error in the db transaction because this results in errors
# when trying to make future changes within the same transaction
with patch.object(MALTRow.objects,
'bulk_create',
side_effect=IntegrityError):
generate_malt([monthspan], domains=[self.domain.name])
malt_row = MALTRow.objects.get(user_id=self.user._id,
app_id=self.app_id,
device_id=self.DEVICE_ID,
month=monthspan.startdate)
# ensure it updates
self.assertEqual(malt_row.num_of_forms, 2)
def _last_month_datespan():
today = datetime.date.today()
first_of_this_month = datetime.date(day=1,
month=today.month,
year=today.year)
last_month = first_of_this_month - datetime.timedelta(days=1)
return DateSpan.from_month(last_month.month, last_month.year)
def datespan(self):
if self._datespan is None:
datespan = DateSpan.from_month(self.month, self.year)
self.request.datespan = datespan
self.context.update(dict(datespan=datespan))
self._datespan = datespan
return self._datespan
def datespan(self):
now = datetime.datetime.utcnow()
year, month = add_months(now.year, now.month, -1)
last_month = DateSpan.from_month(month, year)
self.request.datespan = last_month
self.context.update(dict(datespan=last_month))
return last_month
def datespan(self):
now = datetime.datetime.utcnow()
year, month = add_months(now.year, now.month, -1)
last_month = DateSpan.from_month(month, year)
self.request.datespan = last_month
self.context.update(dict(datespan=last_month))
return last_month
def setUp(self) -> None:
super().setUp()
self.domain = 'domain'
self.monthspan = DateSpan.from_month(1, 2022)
self.run_date = self.monthspan.computed_enddate
self.web_user = create_user_for_malt_tests(is_web_user=True,
user_id='user_id_1',
username='******')
self.mobile_user = create_user_for_malt_tests(is_web_user=False,
user_id='user_id_2',
username='******')
self.app_data = create_malt_app_data()
self.users_by_id = {
'user_id_1': self.web_user,
'user_id_2': self.mobile_user
}
app_data_patcher = patch(
'corehq.apps.data_analytics.malt_generator._get_malt_app_data')
self.mock_get_malt_app_data = app_data_patcher.start()
self.mock_get_malt_app_data.return_value = self.app_data
self.addCleanup(app_data_patcher.stop)
breakdown_es_patcher = patch(
'corehq.apps.data_analytics.malt_generator.get_app_submission_breakdown_es'
)
self.mock_app_submission_breakdown = breakdown_es_patcher.start()
self.mock_app_submission_breakdown.return_value = [
create_mock_nested_query_row(user_id='user_id_1'),
create_mock_nested_query_row(user_id='user_id_2'),
]
self.addCleanup(breakdown_es_patcher.stop)
def datespan(self):
if self._datespan is None:
datespan = DateSpan.from_month(self.month, self.year)
self.request.datespan = datespan
self.context.update(dict(datespan=datespan))
self._datespan = datespan
return self._datespan
def test_successfully_creates(self):
self._save_form_data(self.app_id, datetime.datetime(2019, 12, 31))
self._save_form_data(self.app_id, datetime.datetime(2020, 1, 1))
self._save_form_data(self.app_id, datetime.datetime(2020, 1, 31))
self.es.indices.refresh(XFORM_INDEX_INFO.index)
monthspan = DateSpan.from_month(1, 2020)
generate_malt([monthspan], domains=[self.domain.name])
malt_row = MALTRow.objects.get(user_id=self.user._id,
app_id=self.app_id,
device_id=self.DEVICE_ID,
month=monthspan.startdate)
self.assertEqual(malt_row.num_of_forms, 2)
self.assertEqual(malt_row.wam, True)
self.assertFalse(
MALTRow.objects.filter(
month=DateSpan.from_month(12, 2019).startdate).exists())
def handle(self, month_years, **options):
datespan_list = []
for arg in month_years:
month_year = dateutil.parser.parse(arg)
datespan_list.append(
DateSpan.from_month(month_year.month, month_year.year))
print("Building Malt table... for time range {}".format(datespan_list))
generate_malt(datespan_list)
print("Finished!")
def handle(self, month_years, **options):
datespan_list = []
for arg in month_years:
month_year = dateutil.parser.parse(arg)
datespan_list.append(DateSpan.from_month(month_year.month, month_year.year))
generator = MALTTableGenerator(datespan_list)
print("Building Malt table... for time range {}".format(datespan_list))
generator.build_table()
print("Finished!")
def handle(self, *args, **options):
datespan_list = []
for arg in args:
month_year = dateutil.parser.parse(arg)
datespan_list.append(DateSpan.from_month(month_year.month, month_year.year))
generator = MALTTableGenerator(datespan_list)
print "Building Malt table... for time range {}".format(datespan_list)
generator.build_table()
print "Finished!"
def test_returns_expected_value_for_month(self):
self.monthspan = DateSpan.from_month(3, 2020)
actual_malt_row_dict = _build_malt_row_dict(self.app_row, self.domain,
self.user, self.monthspan,
self.run_date)
self.assertEqual(actual_malt_row_dict['month'],
datetime.datetime(2020, 3, 1, 0, 0))
def send_MALT_complete_email(month_dict):
month = DateSpan.from_month(month_dict['month'], month_dict['year'])
message = 'MALT generation for month {} is now ready. To download go to'\
' http://www.commcarehq.org/hq/admin/download_malt/'.format(
month
)
send_HTML_email('MALT is ready',
settings.DATA_EMAIL,
message,
text_content=message)
def test_multiple_months(self):
self._save_form_data(self.app_id, datetime.datetime(2019, 12, 15))
self._save_form_data(self.app_id, datetime.datetime(2020, 1, 15))
self._save_form_data(self.app_id, datetime.datetime(2020, 1, 16))
self.es.indices.refresh(XFORM_INDEX_INFO.index)
monthspans = [
DateSpan.from_month(12, 2019),
DateSpan.from_month(1, 2020)
]
generate_malt(monthspans, domains=[self.domain.name])
december_malt = MALTRow.objects.get(domain_name=self.domain,
month=DateSpan.from_month(
12, 2019).startdate)
january_malt = MALTRow.objects.get(domain_name=self.domain,
month=DateSpan.from_month(
1, 2020).startdate)
self.assertEqual(december_malt.num_of_forms, 1)
self.assertEqual(january_malt.num_of_forms, 2)
def setUp(self):
super().setUp()
self.domain = 'domain'
self.monthspan = DateSpan.from_month(1, 2022)
self.run_date = self.monthspan.computed_enddate
self.app_row = create_mock_nested_query_row()
self.user = create_user_for_malt_tests(is_web_user=True)
self.app_data = create_malt_app_data()
app_data_patcher = patch(
'corehq.apps.data_analytics.malt_generator._get_malt_app_data')
self.mock_get_malt_app_data = app_data_patcher.start()
self.mock_get_malt_app_data.return_value = self.app_data
self.addCleanup(app_data_patcher.stop)
def test_app_submission_breakdown(self, combination_count_list):
"""
The breakdown of this report is (app, device, userid, username): count
"""
domain = 'test-data-analytics'
received = datetime(2016, 3, 24)
month = DateSpan.from_month(3, 2016)
for app, device, userid, username, count in combination_count_list:
for i in range(count):
save_to_es_analytics_db(domain, received, app, device, userid, username)
self.es.indices.refresh(XFORM_INDEX_INFO.index)
data_back = get_app_submission_breakdown_es(domain, month)
normalized_data_back = set(data_back)
self.assertEqual(set(combination_count_list), normalized_data_back)
def test_app_submission_breakdown(self, combination_count_list):
"""
The breakdown of this report is (app, device, userid, username): count
"""
domain = 'test-data-analytics'
received = datetime(2016, 3, 24)
month = DateSpan.from_month(3, 2016)
for app, device, userid, username, count in combination_count_list:
for i in range(count):
save_to_es_analytics_db(domain, received, app, device, userid, username)
self.es.indices.refresh(XFORM_INDEX_INFO.index)
data_back = get_app_submission_breakdown_es(domain, month)
normalized_data_back = set(data_back)
self.assertEqual(set(combination_count_list), normalized_data_back)
def test_does_not_update(self):
self._save_form_data(self.app_id, datetime.datetime(2020, 1, 15))
self.es.indices.refresh(XFORM_INDEX_INFO.index)
monthspan = DateSpan.from_month(1, 2020)
generate_malt([monthspan], domains=[self.domain.name])
malt_row = MALTRow.objects.get(user_id=self.user._id,
app_id=self.app_id,
device_id=self.DEVICE_ID,
month=monthspan.startdate)
previous_run_date = malt_row.last_run_date
generate_malt([monthspan], domains=[self.domain.name])
malt_row = MALTRow.objects.get(user_id=self.user._id,
app_id=self.app_id,
device_id=self.DEVICE_ID,
month=monthspan.startdate)
self.assertEqual(malt_row.last_run_date, previous_run_date)
def update_current_MALT():
today = datetime.date.today()
this_month = DateSpan.from_month(today.month, today.year)
MALTTableGenerator([this_month]).build_table()
def update_malt(month_dict, domains):
month = DateSpan.from_month(month_dict['month'], month_dict['year'])
generate_malt([month], domains=domains)
def datespan(self):
return DateSpan.from_month(self.month, self.year)
def _last_month_datespan():
today = datetime.date.today()
first_of_this_month = datetime.date(day=1, month=today.month, year=today.year)
last_month = first_of_this_month - datetime.timedelta(days=1)
return DateSpan.from_month(last_month.month, last_month.year)
def update_current_MALT():
today = datetime.date.today()
this_month = DateSpan.from_month(today.month, today.year)
MALTTableGenerator([this_month]).build_table()
def datespan(self):
return DateSpan.from_month(self.month, self.year)
def datespan(self):
return DateSpan.from_month(month, year)
class MaltGeneratorTest(TestCase):
DOMAIN_NAME = "test"
USERNAME = "******"
DEVICE_ID = "my_phone"
UNKNOWN_ID = "UNKNOWN_ID"
correct_date = datetime.datetime.now()
out_of_range_date = correct_date - datetime.timedelta(days=32)
malt_month = DateSpan.from_month(correct_date.month, correct_date.year)
@classmethod
def setUpClass(cls):
super(MaltGeneratorTest, cls).setUpClass()
cls.es = get_es_new()
ensure_index_deleted(XFORM_INDEX_INFO.index)
initialize_index_and_mapping(cls.es, XFORM_INDEX_INFO)
cls._setup_domain_user()
cls._setup_apps()
cls._setup_forms()
cls.es.indices.refresh(XFORM_INDEX_INFO.index)
cls.run_malt_generation()
@classmethod
def tearDownClass(cls):
cls.domain.delete()
MALTRow.objects.all().delete()
ensure_index_deleted(XFORM_INDEX_INFO.index)
super(MaltGeneratorTest, cls).tearDownClass()
@classmethod
def _setup_domain_user(cls):
cls.domain = Domain(name=cls.DOMAIN_NAME)
cls.domain.save()
cls.user = CommCareUser.create(cls.DOMAIN_NAME, cls.USERNAME, '*****')
cls.user.save()
cls.user_id = cls.user._id
@classmethod
def _setup_apps(cls):
cls.non_wam_app = Application.new_app(cls.DOMAIN_NAME, "app 1")
cls.wam_app = Application.new_app(cls.DOMAIN_NAME, "app 2")
cls.wam_app.amplifies_workers = AMPLIFIES_YES
cls.non_wam_app.save()
cls.wam_app.save()
cls.non_wam_app_id = cls.non_wam_app._id
cls.wam_app_id = cls.wam_app._id
@classmethod
def _setup_forms(cls):
def _save_form_data(app_id, received_on=cls.correct_date, device_id=cls.DEVICE_ID):
save_to_es_analytics_db(
domain=cls.DOMAIN_NAME,
received_on=received_on,
device_id=device_id,
user_id=cls.user_id,
app_id=app_id,
)
def _save_multiple_forms(app_ids, received_on):
for app_id in app_ids:
_save_form_data(app_id, received_on=received_on)
out_of_range_form_apps = [
cls.non_wam_app_id,
cls.wam_app_id,
]
in_range_form_apps = [
# should be included in MALT
cls.non_wam_app_id,
cls.non_wam_app_id,
cls.non_wam_app_id,
cls.wam_app_id,
cls.wam_app_id,
# should be included in MALT
'',
]
_save_multiple_forms(out_of_range_form_apps, cls.out_of_range_date)
_save_multiple_forms(in_range_form_apps, cls.correct_date)
# should be included in MALT
_save_form_data(cls.non_wam_app_id, device_id=COMMCONNECT_DEVICE_ID)
@classmethod
def run_malt_generation(cls):
generator = MALTTableGenerator([cls.malt_month])
generator.build_table()
def _assert_malt_row_exists(self, query_filters):
rows = MALTRow.objects.filter(username=self.USERNAME, **query_filters)
self.assertEqual(rows.count(), 1)
def test_wam_yes_malt_counts(self):
# 2 forms for WAM.YES app
self._assert_malt_row_exists({
'app_id': self.wam_app_id,
'num_of_forms': 2,
'wam': YES,
})
def test_wam_not_set_malt_counts(self):
# 3 forms from self.DEVICE_ID for WAM not-set app
self._assert_malt_row_exists({
'app_id': self.non_wam_app_id,
'num_of_forms': 3,
'wam': NOT_SET,
'device_id': self.DEVICE_ID,
})
# 1 form from COMMONCONNECT_DEVICE_ID for WAM not-set app
self._assert_malt_row_exists({
'app_id': self.non_wam_app_id,
'num_of_forms': 1,
'wam': NOT_SET,
'device_id': COMMCONNECT_DEVICE_ID,
})
def test_missing_app_id_is_included(self):
# apps with MISSING_APP_ID should be included in MALT
self._assert_malt_row_exists({
'app_id': MISSING_APP_ID,
'num_of_forms': 1,
'wam': NOT_SET,
})
def datespan(self):
return DateSpan.from_month(self.month, self.year, inclusive=True)
def datespan(self):
return DateSpan.from_month(self.month, self.year, inclusive=True)
def last_month_datespan():
last_month = get_last_month()
return DateSpan.from_month(last_month.month, last_month.year)
def datespan(self):
return DateSpan.from_month(month, year)