def test_successfully_updates(self): self._save_form_data(self.app_id, datetime.datetime(2020, 1, 15)) self.es.indices.refresh(XFORM_INDEX_INFO.index) monthspan = DateSpan.from_month(1, 2020) generate_malt([monthspan], domains=[self.domain.name]) malt_row = MALTRow.objects.get(user_id=self.user._id, app_id=self.app_id, device_id=self.DEVICE_ID, month=monthspan.startdate) self.assertEqual(malt_row.num_of_forms, 1) # hacky way to simulate last run date in between form submissions malt_row.last_run_date = datetime.datetime(2020, 1, 17) malt_row.save() self._save_form_data(self.app_id, datetime.datetime(2020, 1, 20)) self.es.indices.refresh(XFORM_INDEX_INFO.index) # mock bulk_create to avoid raising an actual error in the db transaction because this results in errors # when trying to make future changes within the same transaction with patch.object(MALTRow.objects, 'bulk_create', side_effect=IntegrityError): generate_malt([monthspan], domains=[self.domain.name]) malt_row = MALTRow.objects.get(user_id=self.user._id, app_id=self.app_id, device_id=self.DEVICE_ID, month=monthspan.startdate) # ensure it updates self.assertEqual(malt_row.num_of_forms, 2)
def _last_month_datespan(): today = datetime.date.today() first_of_this_month = datetime.date(day=1, month=today.month, year=today.year) last_month = first_of_this_month - datetime.timedelta(days=1) return DateSpan.from_month(last_month.month, last_month.year)
def datespan(self): if self._datespan is None: datespan = DateSpan.from_month(self.month, self.year) self.request.datespan = datespan self.context.update(dict(datespan=datespan)) self._datespan = datespan return self._datespan
def datespan(self): now = datetime.datetime.utcnow() year, month = add_months(now.year, now.month, -1) last_month = DateSpan.from_month(month, year) self.request.datespan = last_month self.context.update(dict(datespan=last_month)) return last_month
def setUp(self) -> None: super().setUp() self.domain = 'domain' self.monthspan = DateSpan.from_month(1, 2022) self.run_date = self.monthspan.computed_enddate self.web_user = create_user_for_malt_tests(is_web_user=True, user_id='user_id_1', username='******') self.mobile_user = create_user_for_malt_tests(is_web_user=False, user_id='user_id_2', username='******') self.app_data = create_malt_app_data() self.users_by_id = { 'user_id_1': self.web_user, 'user_id_2': self.mobile_user } app_data_patcher = patch( 'corehq.apps.data_analytics.malt_generator._get_malt_app_data') self.mock_get_malt_app_data = app_data_patcher.start() self.mock_get_malt_app_data.return_value = self.app_data self.addCleanup(app_data_patcher.stop) breakdown_es_patcher = patch( 'corehq.apps.data_analytics.malt_generator.get_app_submission_breakdown_es' ) self.mock_app_submission_breakdown = breakdown_es_patcher.start() self.mock_app_submission_breakdown.return_value = [ create_mock_nested_query_row(user_id='user_id_1'), create_mock_nested_query_row(user_id='user_id_2'), ] self.addCleanup(breakdown_es_patcher.stop)
def test_successfully_creates(self): self._save_form_data(self.app_id, datetime.datetime(2019, 12, 31)) self._save_form_data(self.app_id, datetime.datetime(2020, 1, 1)) self._save_form_data(self.app_id, datetime.datetime(2020, 1, 31)) self.es.indices.refresh(XFORM_INDEX_INFO.index) monthspan = DateSpan.from_month(1, 2020) generate_malt([monthspan], domains=[self.domain.name]) malt_row = MALTRow.objects.get(user_id=self.user._id, app_id=self.app_id, device_id=self.DEVICE_ID, month=monthspan.startdate) self.assertEqual(malt_row.num_of_forms, 2) self.assertEqual(malt_row.wam, True) self.assertFalse( MALTRow.objects.filter( month=DateSpan.from_month(12, 2019).startdate).exists())
def handle(self, month_years, **options): datespan_list = [] for arg in month_years: month_year = dateutil.parser.parse(arg) datespan_list.append( DateSpan.from_month(month_year.month, month_year.year)) print("Building Malt table... for time range {}".format(datespan_list)) generate_malt(datespan_list) print("Finished!")
def handle(self, month_years, **options): datespan_list = [] for arg in month_years: month_year = dateutil.parser.parse(arg) datespan_list.append(DateSpan.from_month(month_year.month, month_year.year)) generator = MALTTableGenerator(datespan_list) print("Building Malt table... for time range {}".format(datespan_list)) generator.build_table() print("Finished!")
def handle(self, *args, **options): datespan_list = [] for arg in args: month_year = dateutil.parser.parse(arg) datespan_list.append(DateSpan.from_month(month_year.month, month_year.year)) generator = MALTTableGenerator(datespan_list) print "Building Malt table... for time range {}".format(datespan_list) generator.build_table() print "Finished!"
def test_returns_expected_value_for_month(self): self.monthspan = DateSpan.from_month(3, 2020) actual_malt_row_dict = _build_malt_row_dict(self.app_row, self.domain, self.user, self.monthspan, self.run_date) self.assertEqual(actual_malt_row_dict['month'], datetime.datetime(2020, 3, 1, 0, 0))
def send_MALT_complete_email(month_dict): month = DateSpan.from_month(month_dict['month'], month_dict['year']) message = 'MALT generation for month {} is now ready. To download go to'\ ' http://www.commcarehq.org/hq/admin/download_malt/'.format( month ) send_HTML_email('MALT is ready', settings.DATA_EMAIL, message, text_content=message)
def test_multiple_months(self): self._save_form_data(self.app_id, datetime.datetime(2019, 12, 15)) self._save_form_data(self.app_id, datetime.datetime(2020, 1, 15)) self._save_form_data(self.app_id, datetime.datetime(2020, 1, 16)) self.es.indices.refresh(XFORM_INDEX_INFO.index) monthspans = [ DateSpan.from_month(12, 2019), DateSpan.from_month(1, 2020) ] generate_malt(monthspans, domains=[self.domain.name]) december_malt = MALTRow.objects.get(domain_name=self.domain, month=DateSpan.from_month( 12, 2019).startdate) january_malt = MALTRow.objects.get(domain_name=self.domain, month=DateSpan.from_month( 1, 2020).startdate) self.assertEqual(december_malt.num_of_forms, 1) self.assertEqual(january_malt.num_of_forms, 2)
def setUp(self): super().setUp() self.domain = 'domain' self.monthspan = DateSpan.from_month(1, 2022) self.run_date = self.monthspan.computed_enddate self.app_row = create_mock_nested_query_row() self.user = create_user_for_malt_tests(is_web_user=True) self.app_data = create_malt_app_data() app_data_patcher = patch( 'corehq.apps.data_analytics.malt_generator._get_malt_app_data') self.mock_get_malt_app_data = app_data_patcher.start() self.mock_get_malt_app_data.return_value = self.app_data self.addCleanup(app_data_patcher.stop)
def test_app_submission_breakdown(self, combination_count_list): """ The breakdown of this report is (app, device, userid, username): count """ domain = 'test-data-analytics' received = datetime(2016, 3, 24) month = DateSpan.from_month(3, 2016) for app, device, userid, username, count in combination_count_list: for i in range(count): save_to_es_analytics_db(domain, received, app, device, userid, username) self.es.indices.refresh(XFORM_INDEX_INFO.index) data_back = get_app_submission_breakdown_es(domain, month) normalized_data_back = set(data_back) self.assertEqual(set(combination_count_list), normalized_data_back)
def test_does_not_update(self): self._save_form_data(self.app_id, datetime.datetime(2020, 1, 15)) self.es.indices.refresh(XFORM_INDEX_INFO.index) monthspan = DateSpan.from_month(1, 2020) generate_malt([monthspan], domains=[self.domain.name]) malt_row = MALTRow.objects.get(user_id=self.user._id, app_id=self.app_id, device_id=self.DEVICE_ID, month=monthspan.startdate) previous_run_date = malt_row.last_run_date generate_malt([monthspan], domains=[self.domain.name]) malt_row = MALTRow.objects.get(user_id=self.user._id, app_id=self.app_id, device_id=self.DEVICE_ID, month=monthspan.startdate) self.assertEqual(malt_row.last_run_date, previous_run_date)
def update_current_MALT(): today = datetime.date.today() this_month = DateSpan.from_month(today.month, today.year) MALTTableGenerator([this_month]).build_table()
def update_malt(month_dict, domains): month = DateSpan.from_month(month_dict['month'], month_dict['year']) generate_malt([month], domains=domains)
def datespan(self): return DateSpan.from_month(self.month, self.year)
def datespan(self): return DateSpan.from_month(month, year)
class MaltGeneratorTest(TestCase): DOMAIN_NAME = "test" USERNAME = "******" DEVICE_ID = "my_phone" UNKNOWN_ID = "UNKNOWN_ID" correct_date = datetime.datetime.now() out_of_range_date = correct_date - datetime.timedelta(days=32) malt_month = DateSpan.from_month(correct_date.month, correct_date.year) @classmethod def setUpClass(cls): super(MaltGeneratorTest, cls).setUpClass() cls.es = get_es_new() ensure_index_deleted(XFORM_INDEX_INFO.index) initialize_index_and_mapping(cls.es, XFORM_INDEX_INFO) cls._setup_domain_user() cls._setup_apps() cls._setup_forms() cls.es.indices.refresh(XFORM_INDEX_INFO.index) cls.run_malt_generation() @classmethod def tearDownClass(cls): cls.domain.delete() MALTRow.objects.all().delete() ensure_index_deleted(XFORM_INDEX_INFO.index) super(MaltGeneratorTest, cls).tearDownClass() @classmethod def _setup_domain_user(cls): cls.domain = Domain(name=cls.DOMAIN_NAME) cls.domain.save() cls.user = CommCareUser.create(cls.DOMAIN_NAME, cls.USERNAME, '*****') cls.user.save() cls.user_id = cls.user._id @classmethod def _setup_apps(cls): cls.non_wam_app = Application.new_app(cls.DOMAIN_NAME, "app 1") cls.wam_app = Application.new_app(cls.DOMAIN_NAME, "app 2") cls.wam_app.amplifies_workers = AMPLIFIES_YES cls.non_wam_app.save() cls.wam_app.save() cls.non_wam_app_id = cls.non_wam_app._id cls.wam_app_id = cls.wam_app._id @classmethod def _setup_forms(cls): def _save_form_data(app_id, received_on=cls.correct_date, device_id=cls.DEVICE_ID): save_to_es_analytics_db( domain=cls.DOMAIN_NAME, received_on=received_on, device_id=device_id, user_id=cls.user_id, app_id=app_id, ) def _save_multiple_forms(app_ids, received_on): for app_id in app_ids: _save_form_data(app_id, received_on=received_on) out_of_range_form_apps = [ cls.non_wam_app_id, cls.wam_app_id, ] in_range_form_apps = [ # should be included in MALT cls.non_wam_app_id, cls.non_wam_app_id, cls.non_wam_app_id, cls.wam_app_id, cls.wam_app_id, # should be included in MALT '', ] _save_multiple_forms(out_of_range_form_apps, cls.out_of_range_date) _save_multiple_forms(in_range_form_apps, cls.correct_date) # should be included in MALT _save_form_data(cls.non_wam_app_id, device_id=COMMCONNECT_DEVICE_ID) @classmethod def run_malt_generation(cls): generator = MALTTableGenerator([cls.malt_month]) generator.build_table() def _assert_malt_row_exists(self, query_filters): rows = MALTRow.objects.filter(username=self.USERNAME, **query_filters) self.assertEqual(rows.count(), 1) def test_wam_yes_malt_counts(self): # 2 forms for WAM.YES app self._assert_malt_row_exists({ 'app_id': self.wam_app_id, 'num_of_forms': 2, 'wam': YES, }) def test_wam_not_set_malt_counts(self): # 3 forms from self.DEVICE_ID for WAM not-set app self._assert_malt_row_exists({ 'app_id': self.non_wam_app_id, 'num_of_forms': 3, 'wam': NOT_SET, 'device_id': self.DEVICE_ID, }) # 1 form from COMMONCONNECT_DEVICE_ID for WAM not-set app self._assert_malt_row_exists({ 'app_id': self.non_wam_app_id, 'num_of_forms': 1, 'wam': NOT_SET, 'device_id': COMMCONNECT_DEVICE_ID, }) def test_missing_app_id_is_included(self): # apps with MISSING_APP_ID should be included in MALT self._assert_malt_row_exists({ 'app_id': MISSING_APP_ID, 'num_of_forms': 1, 'wam': NOT_SET, })
def datespan(self): return DateSpan.from_month(self.month, self.year, inclusive=True)
def last_month_datespan(): last_month = get_last_month() return DateSpan.from_month(last_month.month, last_month.year)