def _dateprop(self, prop): date = self.parse_date(self.case[prop]) if isinstance(date, datetime.datetime): user_time = PhoneTime(date, self.timezone).user_time(self.timezone) return user_time.ui_string(self.date_format) else: return ''
def get_num_missed_windows(case): """ Get the number of reminder events that were missed on registration day. """ domain_obj = Domain.get_by_name(case.domain, strict=True) # unlike in most other projects, case.opened_on is actually in UTC # because it was submitted from cloudcare opened_timestamp = PhoneTime(case.opened_on, pytz.UTC).user_time(domain_obj.get_default_timezone()).done() day_of_week = opened_timestamp.weekday() time_of_day = opened_timestamp.time() # In order to use timedelta, we need a datetime current_time = datetime.combine(date(2000, 1, 1), time_of_day) window_time = datetime.combine(date(2000, 1, 1), WINDOWS[day_of_week][0]) if current_time < window_time: return 0 window_interval = (WINDOWS[day_of_week][1] - 60) / 5 for i in range(1, 5): window_time += timedelta(minutes=window_interval) if current_time < window_time: return i return 5
def report_date_to_json(request, domain, date, is_phonetime=True): timezone = get_timezone(request, domain) if date: if is_phonetime: user_time = PhoneTime(date, timezone).user_time(timezone) else: user_time = ServerTime(date).user_time(timezone) user_time.ui_string(SERVER_DATETIME_FORMAT) else: return ''
def test_phone_to_server(self): cases = [ ('2015-03-20T08:00:00', pytz.FixedOffset(-4 * 60), '2015-03-20T12:00:00'), ('2015-03-20T20:30:00', pytz.FixedOffset(-4 * 60), '2015-03-21T00:30:00'), ] for in_, tz, out in cases: phone_dt = dateutil.parser.parse(in_) server_dt = PhoneTime(phone_dt, tz).server_time().done() self.assertEqual(server_dt.isoformat(), in_)
def test_phone_to_server(self): cases = [ ('2015-03-20T08:00:00', pytz.FixedOffset(-4 * 60), '2015-03-20T12:00:00'), ('2015-03-20T20:30:00', pytz.FixedOffset(-4 * 60), '2015-03-21T00:30:00'), ] for in_, tz, out in cases: phone_dt = dateutil.parser.parse(in_) if get_timezone_data_migration_complete(): server_dt = phone_dt else: server_dt = PhoneTime(phone_dt, tz).server_time().done() self.assertEqual(server_dt.isoformat(), out)
def test_phone_to_server(self): cases = [ ('2015-03-20T08:00:00', pytz.FixedOffset(-4 * 60), '2015-03-20T12:00:00'), ('2015-03-20T20:30:00', pytz.FixedOffset(-4 * 60), '2015-03-21T00:30:00'), ] for in_, tz, out in cases: phone_dt = dateutil.parser.parse(in_) server_dt = PhoneTime(phone_dt, tz).server_time().done() if phone_timezones_have_been_processed(): # no change self.assertEqual(server_dt.isoformat(), in_) else: self.assertEqual(server_dt.isoformat(), out)
def get_display_data(data, prop_def, processors=None, timezone=pytz.utc, info_url=None): # when prop_def came from a couchdbkit document, it will be a LazyDict with # a broken pop method. This conversion also has the effect of a shallow # copy, which we want. prop_def = dict(prop_def) default_processors = { 'yesno': yesno, 'doc_info': lambda value: pretty_doc_info(get_doc_info_by_id( data['domain'], value)) } processors = processors or {} processors.update(default_processors) expr_name = _get_expr_name(prop_def) expr = prop_def.pop('expr') name = prop_def.pop('name', None) or _format_slug_string_for_display(expr) format = prop_def.pop('format', None) process = prop_def.pop('process', None) timeago = prop_def.get('timeago', False) val = eval_expr(expr, data) if prop_def.pop('parse_date', None): val = _parse_date_or_datetime(val) # is_utc is deprecated in favor of is_phone_time # but preserving here for backwards compatibility # is_utc = False is just reinterpreted as is_phone_time = True is_phone_time = prop_def.pop('is_phone_time', not prop_def.pop('is_utc', True)) if isinstance(val, datetime.datetime): if not is_phone_time: val = ServerTime(val).user_time(timezone).done() else: val = PhoneTime(val, timezone).user_time(timezone).done() try: val = conditional_escape(processors[process](val)) except KeyError: val = mark_safe(_to_html(val, timeago=timeago)) if format: val = mark_safe(format.format(val)) return { "expr": expr_name, "name": name, "value": val, "info_url": info_url.replace("__placeholder__", expr) if info_url is not None else None, }
def submission_or_completion_time(self): time = iso_string_to_datetime(safe_index(self.form, self.report.time_field.split('.'))) if self.report.by_submission_time: user_time = ServerTime(time).user_time(self.report.timezone) else: user_time = PhoneTime(time, self.report.timezone).user_time(self.report.timezone) return user_time.ui_string(USER_DATETIME_FORMAT_WITH_SEC)
def get_display_data(data: dict, prop_def: DisplayConfig, timezone=pytz.utc): expr_name = _get_expr_name(prop_def) name = prop_def.name or _format_slug_string_for_display(expr_name) val = eval_expr(prop_def.expr, data) processor = VALUE_DISPLAY_PROCESSORS.get(prop_def.process, None) if processor: try: val = processor(val, data) except Exception: # ignore exceptions from date parsing pass if isinstance(val, datetime.datetime): if not prop_def.is_phone_time: val = ServerTime(val).user_time(timezone).done() else: val = PhoneTime(val, timezone).user_time(timezone).done() if not processor or not processor.returns_html: val = _to_html(val, timeago=prop_def.timeago) if prop_def.format: val = format_html(prop_def.format, val) return { "expr": expr_name, "name": name, "value": val, "has_history": prop_def.has_history, }
def get_formatted_response(self): timezone = get_timezone_for_request() if self.type == 'DateTime' and timezone \ and isinstance(self.response, datetime.datetime): return (PhoneTime(self.response, timezone).user_time(timezone) .ui_string()) else: return self.response
def report_date_to_json(request, domain, date): timezone = get_timezone(request, domain) if date: return (PhoneTime( date, timezone).user_time(timezone).ui_string(SERVER_DATETIME_FORMAT)) else: return ''
def date_to_json(self, date): if date: try: date = force_to_datetime(date) return (PhoneTime(date, self.timezone).user_time(self.timezone) .ui_string('%d-%m-%Y')) except ValueError: return '' else: return ''
def rows(self): def form_data_link(instance_id): return "<a class='ajax_dialog' target='_new' href='%(url)s'>%(text)s</a>" % { "url": absolute_reverse('render_form_data', args=[self.domain, instance_id]), "text": _("View Form") } submissions = [ res['_source'] for res in self.es_results.get('hits', {}).get('hits', []) ] for form in submissions: uid = form["form"]["meta"]["userID"] username = form["form"]["meta"].get("username") try: if username not in ['demo_user', 'admin']: full_name = get_cached_property(CouchUser, uid, 'full_name', expiry=7 * 24 * 60 * 60) name = '"%s"' % full_name if full_name else "" else: name = "" except (ResourceNotFound, IncompatibleDocument): name = "<b>[unregistered]</b>" time = iso_string_to_datetime( safe_index(form, self.time_field.split('.'))) if self.by_submission_time: user_time = ServerTime(time).user_time(self.timezone) else: user_time = PhoneTime(time, self.timezone).user_time(self.timezone) init_cells = [ form_data_link(form["_id"]), (username or _('No data for username')) + (" %s" % name if name else ""), user_time.ui_string(USER_DATETIME_FORMAT_WITH_SEC), xmlns_to_name(self.domain, form.get("xmlns"), app_id=form.get("app_id")), ] def cell(field): return form["form"].get(field) init_cells.extend([cell(field) for field in self.other_fields]) yield init_cells
def get_display_data(data, prop_def, processors=None, timezone=pytz.utc): # when prop_def came from a couchdbkit document, it will be a LazyDict with # a broken pop method. This conversion also has the effect of a shallow # copy, which we want. prop_def = dict(prop_def) default_processors = { 'yesno': yesno, 'doc_info': lambda value: pretty_doc_info(get_doc_info_by_id( data['domain'], value)) } processors = processors or {} processors.update(default_processors) expr_name = _get_expr_name(prop_def) expr = prop_def.pop('expr') name = prop_def.pop('name', None) or _format_slug_string_for_display(expr) format = prop_def.pop('format', None) process = prop_def.pop('process', None) timeago = prop_def.get('timeago', False) has_history = prop_def.pop('has_history', False) val = eval_expr(expr, data) if prop_def.pop('parse_date', None): try: val = _parse_date_or_datetime(val) except Exception: # ignore exceptions from date parsing pass is_phone_time = prop_def.pop('is_phone_time', False) if isinstance(val, datetime.datetime): if not is_phone_time: val = ServerTime(val).user_time(timezone).done() else: val = PhoneTime(val, timezone).user_time(timezone).done() try: val = conditional_escape(processors[process](val)) except KeyError: val = mark_safe(_to_html(val, timeago=timeago)) if format: val = mark_safe(format.format(val)) return { "expr": expr_name, "name": name, "value": val, "has_history": has_history, }
class ImporterTest(TestCase): def setUp(self): super(ImporterTest, self).setUp() self.domain_obj = create_domain("importer-test") self.domain = self.domain_obj.name self.default_case_type = 'importer-test-casetype' self.couch_user = WebUser.create(None, "test", "foobar", None, None) self.couch_user.add_domain_membership(self.domain, is_admin=True) self.couch_user.save() self.accessor = CaseAccessors(self.domain) self.factory = CaseFactory(domain=self.domain, case_defaults={ 'case_type': self.default_case_type, }) delete_all_cases() def tearDown(self): self.couch_user.delete() self.domain_obj.delete() super(ImporterTest, self).tearDown() def _config(self, col_names, search_column=None, case_type=None, search_field='case_id', create_new_cases=True): return ImporterConfig( couch_user_id=self.couch_user._id, case_type=case_type or self.default_case_type, excel_fields=col_names, case_fields=[''] * len(col_names), custom_fields=col_names, search_column=search_column or col_names[0], search_field=search_field, create_new_cases=create_new_cases, ) @run_with_all_backends @patch('corehq.apps.case_importer.tasks.bulk_import_async.update_state') def testImportFileMissing(self, update_state): # by using a made up upload_id, we ensure it's not referencing any real file case_upload = CaseUploadRecord(upload_id=str(uuid.uuid4()), task_id=str(uuid.uuid4())) case_upload.save() res = bulk_import_async.delay(self._config(['anything']), self.domain, case_upload.upload_id) self.assertIsInstance(res.result, Ignore) update_state.assert_called_with( state=states.FAILURE, meta=get_interned_exception( 'Sorry, your session has expired. Please start over and try again.' )) self.assertEqual(0, len(get_case_ids_in_domain(self.domain))) @run_with_all_backends def testImportBasic(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) self.assertEqual(5, res['created_count']) self.assertEqual(0, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(1, res['num_chunks']) case_ids = self.accessor.get_case_ids_in_domain() cases = list(self.accessor.get_cases(case_ids)) self.assertEqual(5, len(cases)) properties_seen = set() for case in cases: self.assertEqual(self.couch_user._id, case.user_id) self.assertEqual(self.couch_user._id, case.owner_id) self.assertEqual(self.default_case_type, case.type) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) self.assertFalse( case.get_case_property(prop) in properties_seen) properties_seen.add(case.get_case_property(prop)) @run_with_all_backends def testImportNamedColumns(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ) res = do_import(file, config, self.domain) self.assertEqual(4, res['created_count']) self.assertEqual(4, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def testImportTrailingWhitespace(self): cols = ['case_id', 'age', 'sex\xa0', 'location'] config = self._config(cols) file = make_worksheet_wrapper( ['case_id', 'age', 'sex\xa0', 'location'], ['case_id-0', 'age-0', 'sex\xa0-0', 'location-0'], ) res = do_import(file, config, self.domain) self.assertEqual(1, res['created_count']) case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) case = self.accessor.get_case(case_ids[0]) self.assertTrue(bool(case.get_case_property( 'sex'))) # make sure the value also got properly set @run_with_all_backends def testCaseIdMatching(self): # bootstrap a stub case [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'update': { 'importer_test_prop': 'foo' }, })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) [case] = self.accessor.get_cases(case_ids) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) # shouldn't touch existing properties self.assertEqual('foo', case.get_case_property('importer_test_prop')) @run_with_all_backends def testCaseLookupTypeCheck(self): [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'case_type': 'nonmatch-type', })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) # because the type is wrong these shouldn't match self.assertEqual(3, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(4, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def testCaseLookupDomainCheck(self): self.factory.domain = 'wrong-domain' [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, })) self.assertEqual(0, len(self.accessor.get_case_ids_in_domain())) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) # because the domain is wrong these shouldn't match self.assertEqual(3, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(3, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def testExternalIdMatching(self): # bootstrap a stub case external_id = 'importer-test-external-id' [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'external_id': external_id, })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = make_worksheet_wrapper( ['external_id', 'age', 'sex', 'location'], ['importer-test-external-id', 'age-0', 'sex-0', 'location-0'], ['importer-test-external-id', 'age-1', 'sex-1', 'location-1'], ['importer-test-external-id', 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def test_external_id_matching_on_create_with_custom_column_name(self): headers = ['id_column', 'age', 'sex', 'location'] external_id = 'external-id-test' config = self._config(headers[1:], search_column='id_column', search_field='external_id') file = make_worksheet_wrapper( ['id_column', 'age', 'sex', 'location'], ['external-id-test', 'age-0', 'sex-0', 'location-0'], ['external-id-test', 'age-1', 'sex-1', 'location-1'], ) res = do_import(file, config, self.domain) self.assertFalse(res['errors']) self.assertEqual(1, res['created_count']) self.assertEqual(1, res['match_count']) case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) case = self.accessor.get_case(case_ids[0]) self.assertEqual(external_id, case.external_id) def testNoCreateNew(self): config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=False) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) # no matching and no create new set - should do nothing self.assertEqual(0, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(0, len(get_case_ids_in_domain(self.domain))) def testBlankRows(self): # don't create new cases for rows left blank config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=True) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [None, None, None, None], ['', '', '', ''], ) res = do_import(file, config, self.domain) # no matching and no create new set - should do nothing self.assertEqual(0, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(0, len(get_case_ids_in_domain(self.domain))) @patch('corehq.apps.case_importer.do_import.CASEBLOCK_CHUNKSIZE', 2) def testBasicChunking(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) # 5 cases in chunks of 2 = 3 chunks self.assertEqual(3, res['num_chunks']) self.assertEqual(5, res['created_count']) self.assertEqual(5, len(get_case_ids_in_domain(self.domain))) @run_with_all_backends def testExternalIdChunking(self): # bootstrap a stub case external_id = 'importer-test-external-id' headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = make_worksheet_wrapper( ['external_id', 'age', 'sex', 'location'], ['importer-test-external-id', 'age-0', 'sex-0', 'location-0'], ['importer-test-external-id', 'age-1', 'sex-1', 'location-1'], ['importer-test-external-id', 'age-2', 'sex-2', 'location-2'], ) # the first one should create the case, and the remaining two should update it res = do_import(file, config, self.domain) self.assertEqual(1, res['created_count']) self.assertEqual(2, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(2, res['num_chunks']) # the lookup causes an extra chunk # should just create the one case case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) [case] = self.accessor.get_cases(case_ids) self.assertEqual(external_id, case.external_id) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) @run_with_all_backends def testParentCase(self): headers = ['parent_id', 'name', 'case_id'] config = self._config(headers, create_new_cases=True, search_column='case_id') rows = 3 [parent_case] = self.factory.create_or_update_case( CaseStructure(attrs={'create': True})) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) file = make_worksheet_wrapper( ['parent_id', 'name', 'case_id'], [parent_case.case_id, 'name-0', 'case_id-0'], [parent_case.case_id, 'name-1', 'case_id-1'], [parent_case.case_id, 'name-2', 'case_id-2'], ) file_missing = make_worksheet_wrapper( ['parent_id', 'name', 'case_id'], ['parent_id-0', 'name-0', 'case_id-0'], ['parent_id-1', 'name-1', 'case_id-1'], ['parent_id-2', 'name-2', 'case_id-2'], ) # Should successfully match on `rows` cases res = do_import(file, config, self.domain) self.assertEqual(rows, res['created_count']) # Should be unable to find parent case on `rows` cases res = do_import(file_missing, config, self.domain) error_column_name = 'parent_id' self.assertEqual( rows, len(res['errors'][exceptions.InvalidParentId.title] [error_column_name]['rows']), "All cases should have missing parent") def import_mock_file(self, rows): config = self._config(rows[0]) xls_file = make_worksheet_wrapper(*rows) return do_import(xls_file, config, self.domain) @run_with_all_backends def testLocationOwner(self): # This is actually testing several different things, but I figure it's # worth it, as each of these tests takes a non-trivial amount of time. non_case_sharing = LocationType.objects.create(domain=self.domain, name='lt1', shares_cases=False) case_sharing = LocationType.objects.create(domain=self.domain, name='lt2', shares_cases=True) location = make_loc('loc-1', 'Loc 1', self.domain, case_sharing.code) make_loc('loc-2', 'Loc 2', self.domain, case_sharing.code) duplicate_loc = make_loc('loc-3', 'Loc 2', self.domain, case_sharing.code) improper_loc = make_loc('loc-4', 'Loc 4', self.domain, non_case_sharing.code) res = self.import_mock_file([ ['case_id', 'name', 'owner_id', 'owner_name'], ['', 'location-owner-id', location.group_id, ''], ['', 'location-owner-code', '', location.site_code], ['', 'location-owner-name', '', location.name], ['', 'duplicate-location-name', '', duplicate_loc.name], ['', 'non-case-owning-name', '', improper_loc.name], ]) case_ids = self.accessor.get_case_ids_in_domain() cases = {c.name: c for c in list(self.accessor.get_cases(case_ids))} self.assertEqual(cases['location-owner-id'].owner_id, location.group_id) self.assertEqual(cases['location-owner-code'].owner_id, location.group_id) self.assertEqual(cases['location-owner-name'].owner_id, location.group_id) error_message = exceptions.DuplicateLocationName.title error_column_name = None self.assertIn(error_message, res['errors']) self.assertEqual( res['errors'][error_message][error_column_name]['rows'], [5]) error_message = exceptions.InvalidOwner.title self.assertIn(error_message, res['errors']) error_column_name = 'owner_name' self.assertEqual( res['errors'][error_message][error_column_name]['rows'], [6]) @run_with_all_backends def test_opened_on(self): case = self.factory.create_case() new_date = '2015-04-30T14:41:53.000000Z' with flag_enabled('BULK_UPLOAD_DATE_OPENED'): self.import_mock_file([['case_id', 'date_opened'], [case.case_id, new_date]]) case = CaseAccessors(self.domain).get_case(case.case_id) self.assertEqual(case.opened_on, PhoneTime(parse_datetime(new_date)).done())
def date_to_json(self, date): if date: return (PhoneTime(date, self.timezone).user_time( self.timezone).ui_string(SERVER_DATETIME_FORMAT)) else: return ''
def test_utc_phonetime(self): dt = datetime.datetime.utcnow() self.assertEqual( PhoneTime(dt, pytz.UTC).user_time(pytz.FixedOffset(9 * 60 + 30)).done(), ServerTime(dt).user_time(pytz.FixedOffset(9 * 60 + 30)).done())
def date_to_json(self, date): if date: return (PhoneTime(date, self.timezone).user_time( self.timezone).ui_string('%d/%m/%Y')) else: return ''
class ImporterTest(TestCase): def setUp(self): super(ImporterTest, self).setUp() self.domain_obj = create_domain("importer-test") self.domain = self.domain_obj.name self.default_case_type = 'importer-test-casetype' self.couch_user = WebUser.create(None, "test", "foobar", None, None) self.couch_user.add_domain_membership(self.domain, is_admin=True) self.couch_user.save() self.subdomain1 = create_domain('subdomain1') self.subdomain2 = create_domain('subdomain2') self.ignored_domain = create_domain('ignored-domain') create_enterprise_permissions( self.couch_user.username, self.domain, [self.subdomain1.name, self.subdomain2.name], [self.ignored_domain.name]) self.factory = CaseFactory(domain=self.domain, case_defaults={ 'case_type': self.default_case_type, }) delete_all_cases() def tearDown(self): self.couch_user.delete(self.domain, deleted_by=None) self.domain_obj.delete() self.subdomain1.delete() self.subdomain2.delete() self.ignored_domain.delete() super(ImporterTest, self).tearDown() def _config(self, col_names, search_column=None, case_type=None, search_field='case_id', create_new_cases=True): return ImporterConfig( couch_user_id=self.couch_user._id, case_type=case_type or self.default_case_type, excel_fields=col_names, case_fields=[''] * len(col_names), custom_fields=col_names, search_column=search_column or col_names[0], search_field=search_field, create_new_cases=create_new_cases, ) @patch('corehq.apps.case_importer.tasks.bulk_import_async.update_state') def testImportFileMissing(self, update_state): # by using a made up upload_id, we ensure it's not referencing any real file case_upload = CaseUploadRecord(upload_id=str(uuid.uuid4()), task_id=str(uuid.uuid4())) case_upload.save() res = bulk_import_async.delay( self._config(['anything']).to_json(), self.domain, case_upload.upload_id) self.assertIsInstance(res.result, Ignore) update_state.assert_called_with( state=states.FAILURE, meta=get_interned_exception( 'There was an unexpected error retrieving the file you uploaded. ' 'Please try again and contact support if the problem persists.' )) self.assertEqual( 0, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) def testImportBasic(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) self.assertEqual(5, res['created_count']) self.assertEqual(0, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(1, res['num_chunks']) case_ids = CommCareCase.objects.get_case_ids_in_domain(self.domain) cases = CommCareCase.objects.get_cases(case_ids, self.domain) self.assertEqual(5, len(cases)) properties_seen = set() for case in cases: self.assertEqual(self.couch_user._id, case.user_id) self.assertEqual(self.couch_user._id, case.owner_id) self.assertEqual(self.default_case_type, case.type) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) self.assertFalse( case.get_case_property(prop) in properties_seen) properties_seen.add(case.get_case_property(prop)) def testCreateCasesWithDuplicateExternalIds(self): config = self._config( ['case_id', 'age', 'sex', 'location', 'external_id']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location', 'external_id'], ['case_id-0', 'age-0', 'sex-0', 'location-0', 'external_id-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1', 'external_id-0'], ['case_id-2', 'age-2', 'sex-2', 'location-2', 'external_id-1'], ) res = do_import(file, config, self.domain) self.assertEqual(3, res['created_count']) self.assertEqual(0, res['match_count']) self.assertFalse(res['errors']) case_ids = CommCareCase.objects.get_case_ids_in_domain(self.domain) self.assertItemsEqual([ case.external_id for case in CommCareCase.objects.get_cases(case_ids, self.domain) ], ['external_id-0', 'external_id-0', 'external_id-1']) def testImportNamedColumns(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ) res = do_import(file, config, self.domain) self.assertEqual(4, res['created_count']) self.assertEqual( 4, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) def testImportTrailingWhitespace(self): cols = ['case_id', 'age', 'sex\xa0', 'location'] config = self._config(cols) file = make_worksheet_wrapper( ['case_id', 'age', 'sex\xa0', 'location'], ['case_id-0', 'age-0', 'sex\xa0-0', 'location-0'], ) res = do_import(file, config, self.domain) self.assertEqual(1, res['created_count']) case_ids = CommCareCase.objects.get_case_ids_in_domain(self.domain) self.assertEqual(1, len(case_ids)) case = CommCareCase.objects.get_case(case_ids[0], self.domain) self.assertTrue(bool(case.get_case_property( 'sex'))) # make sure the value also got properly set def testCaseIdMatching(self): # bootstrap a stub case [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'update': { 'importer_test_prop': 'foo' }, })) self.assertEqual( 1, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one case_ids = CommCareCase.objects.get_case_ids_in_domain(self.domain) self.assertEqual(1, len(case_ids)) [case] = CommCareCase.objects.get_cases(case_ids, self.domain) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) # shouldn't touch existing properties self.assertEqual('foo', case.get_case_property('importer_test_prop')) def testCaseLookupTypeCheck(self): [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'case_type': 'nonmatch-type', })) self.assertEqual( 1, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) # because the type is wrong these shouldn't match self.assertEqual(3, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual( 4, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) def testCaseLookupDomainCheck(self): self.factory.domain = 'wrong-domain' [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, })) self.assertEqual( 0, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) # because the domain is wrong these shouldn't match self.assertEqual(3, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual( 3, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) def testExternalIdMatching(self): # bootstrap a stub case external_id = 'importer-test-external-id' [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'external_id': external_id, })) self.assertEqual( 1, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = make_worksheet_wrapper( ['external_id', 'age', 'sex', 'location'], ['importer-test-external-id', 'age-0', 'sex-0', 'location-0'], ['importer-test-external-id', 'age-1', 'sex-1', 'location-1'], ['importer-test-external-id', 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one self.assertEqual( 1, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) def test_external_id_matching_on_create_with_custom_column_name(self): headers = ['id_column', 'age', 'sex', 'location'] external_id = 'external-id-test' config = self._config(headers[1:], search_column='id_column', search_field='external_id') file = make_worksheet_wrapper( ['id_column', 'age', 'sex', 'location'], ['external-id-test', 'age-0', 'sex-0', 'location-0'], ['external-id-test', 'age-1', 'sex-1', 'location-1'], ) res = do_import(file, config, self.domain) self.assertFalse(res['errors']) self.assertEqual(1, res['created_count']) self.assertEqual(1, res['match_count']) case_ids = CommCareCase.objects.get_case_ids_in_domain(self.domain) self.assertEqual(1, len(case_ids)) case = CommCareCase.objects.get_case(case_ids[0], self.domain) self.assertEqual(external_id, case.external_id) def testNoCreateNew(self): config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=False) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) # no matching and no create new set - should do nothing self.assertEqual(0, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual( 0, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) def testBlankRows(self): # don't create new cases for rows left blank config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=True) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [None, None, None, None], ['', '', '', ''], ) res = do_import(file, config, self.domain) # no matching and no create new set - should do nothing self.assertEqual(0, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual( 0, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) @patch('corehq.apps.case_importer.do_import.CASEBLOCK_CHUNKSIZE', 2) def testBasicChunking(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) # 5 cases in chunks of 2 = 3 chunks self.assertEqual(3, res['num_chunks']) self.assertEqual(5, res['created_count']) self.assertEqual( 5, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) def testExternalIdChunking(self): # bootstrap a stub case external_id = 'importer-test-external-id' headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = make_worksheet_wrapper( ['external_id', 'age', 'sex', 'location'], ['importer-test-external-id', 'age-0', 'sex-0', 'location-0'], ['importer-test-external-id', 'age-1', 'sex-1', 'location-1'], ['importer-test-external-id', 'age-2', 'sex-2', 'location-2'], ) # the first one should create the case, and the remaining two should update it res = do_import(file, config, self.domain) self.assertEqual(1, res['created_count']) self.assertEqual(2, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(2, res['num_chunks']) # the lookup causes an extra chunk # should just create the one case case_ids = CommCareCase.objects.get_case_ids_in_domain(self.domain) self.assertEqual(1, len(case_ids)) [case] = CommCareCase.objects.get_cases(case_ids, self.domain) self.assertEqual(external_id, case.external_id) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) def testParentCase(self): headers = ['parent_id', 'name', 'case_id'] config = self._config(headers, create_new_cases=True, search_column='case_id') rows = 3 [parent_case] = self.factory.create_or_update_case( CaseStructure(attrs={'create': True})) self.assertEqual( 1, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) file = make_worksheet_wrapper( ['parent_id', 'name', 'case_id'], [parent_case.case_id, 'name-0', 'case_id-0'], [parent_case.case_id, 'name-1', 'case_id-1'], [parent_case.case_id, 'name-2', 'case_id-2'], ) # Should successfully match on `rows` cases res = do_import(file, config, self.domain) self.assertEqual(rows, res['created_count']) # Should create child cases cases = CommCareCase.objects.get_reverse_indexed_cases( self.domain, [parent_case.case_id]) self.assertEqual(len(cases), 3) self.assertEqual( CommCareCaseIndex.objects.get_extension_case_ids( self.domain, [parent_case.case_id]), [], ) file_missing = make_worksheet_wrapper( ['parent_id', 'name', 'case_id'], ['parent_id-0', 'name-0', 'case_id-0'], ['parent_id-1', 'name-1', 'case_id-1'], ['parent_id-2', 'name-2', 'case_id-2'], ) # Should be unable to find parent case on `rows` cases res = do_import(file_missing, config, self.domain) error_column_name = 'parent_id' self.assertEqual( rows, len(res['errors'][exceptions.InvalidParentId.title] [error_column_name]['rows']), "All cases should have missing parent") def testExtensionCase(self): headers = [ 'parent_id', 'name', 'case_id', 'parent_relationship_type', 'parent_identifier' ] config = self._config(headers, create_new_cases=True, search_column='case_id') [parent_case] = self.factory.create_or_update_case( CaseStructure(attrs={'create': True})) self.assertEqual( 1, len(CommCareCase.objects.get_case_ids_in_domain(self.domain))) file = make_worksheet_wrapper( headers, [parent_case.case_id, 'name-0', 'case_id-0', 'extension', 'host'], [ parent_case.case_id, 'name-1', 'case_id-1', 'extension', 'mother' ], [parent_case.case_id, 'name-2', 'case_id-2', 'child', 'parent'], ) # Should successfully match on `rows` cases res = do_import(file, config, self.domain) self.assertEqual(res['created_count'], 3) # Of the 3, 2 should be extension cases extension_case_ids = CommCareCaseIndex.objects.get_extension_case_ids( self.domain, [parent_case.case_id]) self.assertEqual(len(extension_case_ids), 2) extension_cases = CommCareCase.objects.get_cases( extension_case_ids, self.domain) # Check that identifier is set correctly self.assertEqual({'host', 'mother'}, {c.indices[0].identifier for c in extension_cases}) @flag_enabled('DOMAIN_PERMISSIONS_MIRROR') def test_multiple_domain_case_import(self): headers_with_domain = ['case_id', 'name', 'artist', 'domain'] config_1 = self._config(headers_with_domain, create_new_cases=True, search_column='case_id') case_with_domain_file = make_worksheet_wrapper( ['case_id', 'name', 'artist', 'domain'], ['', 'name-0', 'artist-0', self.domain], ['', 'name-1', 'artist-1', self.subdomain1.name], ['', 'name-2', 'artist-2', self.subdomain2.name], ['', 'name-3', 'artist-3', self.domain], ['', 'name-4', 'artist-4', self.domain], ['', 'name-5', 'artist-5', 'not-existing-domain'], ['', 'name-6', 'artist-6', self.ignored_domain.name], ) res = do_import(case_with_domain_file, config_1, self.domain) self.assertEqual(5, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(2, res['failed_count']) # Asserting current domain cur_case_ids = CommCareCase.objects.get_case_ids_in_domain(self.domain) cur_cases = CommCareCase.objects.get_cases(cur_case_ids, self.domain) self.assertEqual(3, len(cur_cases)) #Asserting current domain case property cases = {c.name: c for c in cur_cases} self.assertEqual(cases['name-0'].get_case_property('artist'), 'artist-0') # Asserting subdomain 1 s1_case_ids = CommCareCase.objects.get_case_ids_in_domain( self.subdomain1.name) s1_cases = CommCareCase.objects.get_cases(s1_case_ids, self.domain) self.assertEqual(1, len(s1_cases)) # Asserting subdomain 1 case property s1_cases_pro = {c.name: c for c in s1_cases} self.assertEqual(s1_cases_pro['name-1'].get_case_property('artist'), 'artist-1') # Asserting subdomain 2 s2_case_ids = CommCareCase.objects.get_case_ids_in_domain( self.subdomain2.name) s2_cases = CommCareCase.objects.get_cases(s2_case_ids, self.domain) self.assertEqual(1, len(s2_cases)) # Asserting subdomain 2 case property s2_cases_pro = {c.name: c for c in s2_cases} self.assertEqual(s2_cases_pro['name-2'].get_case_property('artist'), 'artist-2') @flag_disabled('DOMAIN_PERMISSIONS_MIRROR') def test_multiple_domain_case_import_mirror_domain_disabled(self): headers_with_domain = ['case_id', 'name', 'artist', 'domain'] config_1 = self._config(headers_with_domain, create_new_cases=True, search_column='case_id') case_with_domain_file = make_worksheet_wrapper( ['case_id', 'name', 'artist', 'domain'], ['', 'name-0', 'artist-0', self.domain], ['', 'name-1', 'artist-1', 'domain-1'], ['', 'name-2', 'artist-2', 'domain-2'], ['', 'name-3', 'artist-3', self.domain], ['', 'name-4', 'artist-4', self.domain], ['', 'name-5', 'artist-5', 'not-existing-domain']) res = do_import(case_with_domain_file, config_1, self.domain) self.assertEqual(6, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(0, res['failed_count']) case_ids = CommCareCase.objects.get_case_ids_in_domain(self.domain) # Asserting current domain cur_cases = CommCareCase.objects.get_cases(case_ids, self.domain) self.assertEqual(6, len(cur_cases)) #Asserting domain case property cases = {c.name: c for c in cur_cases} self.assertEqual(cases['name-0'].get_case_property('domain'), self.domain) def import_mock_file(self, rows): config = self._config(rows[0]) xls_file = make_worksheet_wrapper(*rows) return do_import(xls_file, config, self.domain) def testLocationOwner(self): # This is actually testing several different things, but I figure it's # worth it, as each of these tests takes a non-trivial amount of time. non_case_sharing = LocationType.objects.create(domain=self.domain, name='lt1', shares_cases=False) case_sharing = LocationType.objects.create(domain=self.domain, name='lt2', shares_cases=True) location = make_loc('loc-1', 'Loc 1', self.domain, case_sharing.code) make_loc('loc-2', 'Loc 2', self.domain, case_sharing.code) duplicate_loc = make_loc('loc-3', 'Loc 2', self.domain, case_sharing.code) improper_loc = make_loc('loc-4', 'Loc 4', self.domain, non_case_sharing.code) res = self.import_mock_file([ ['case_id', 'name', 'owner_id', 'owner_name'], ['', 'location-owner-id', location.location_id, ''], ['', 'location-owner-code', '', location.site_code], ['', 'location-owner-name', '', location.name], ['', 'duplicate-location-name', '', duplicate_loc.name], ['', 'non-case-owning-name', '', improper_loc.name], ]) case_ids = CommCareCase.objects.get_case_ids_in_domain(self.domain) cases = { c.name: c for c in CommCareCase.objects.get_cases(case_ids, self.domain) } self.assertEqual(cases['location-owner-id'].owner_id, location.location_id) self.assertEqual(cases['location-owner-code'].owner_id, location.location_id) self.assertEqual(cases['location-owner-name'].owner_id, location.location_id) error_message = exceptions.DuplicateLocationName.title error_column_name = None self.assertIn(error_message, res['errors']) self.assertEqual( res['errors'][error_message][error_column_name]['rows'], [5]) error_message = exceptions.InvalidOwner.title self.assertIn(error_message, res['errors']) error_column_name = 'owner_name' self.assertEqual( res['errors'][error_message][error_column_name]['rows'], [6]) def test_opened_on(self): case = self.factory.create_case() new_date = '2015-04-30T14:41:53.000000Z' with flag_enabled('BULK_UPLOAD_DATE_OPENED'): self.import_mock_file([['case_id', 'date_opened'], [case.case_id, new_date]]) case = CommCareCase.objects.get_case(case.case_id, self.domain) self.assertEqual(case.opened_on, PhoneTime(parse_datetime(new_date)).done())