def test_user_data_profile(self): definition = CustomDataFieldsDefinition( domain='cloudcare-tests', field_type=UserFieldsView.field_type) definition.save() definition.set_fields([ Field(slug='word', label='A Word'), ]) definition.save() profile = CustomDataFieldsProfile(name='prof', fields={'word': 'supernova'}, definition=definition) profile.save() user = CommCareUser.create( 'cloudcare-tests', '*****@*****.**', 'do you want to know a secret', None, None, uuid=uuid.uuid4().hex, metadata={PROFILE_SLUG: profile.id}, ) user_data = get_user_contributions_to_touchforms_session( user)['user_data'] self.assertEqual(profile.id, user_data[PROFILE_SLUG]) self.assertEqual('supernova', user_data['word']) definition.delete()
def test_sync_custom_user_data(self): definition = CustomDataFieldsDefinition( domain=TEST_DOMAIN, field_type=UserFieldsView.field_type) definition.save() definition.set_fields([ Field(slug='from_profile', label='From Profile'), ]) definition.save() profile = CustomDataFieldsProfile( name='callcenter_profile', fields={'from_profile': 'yes'}, definition=definition, ) profile.save() self.user.update_metadata({ '': 'blank_key', 'blank_val': '', 'ok': 'good', 'name with spaces': 'ok', '8starts_with_a_number': '0', 'xml_starts_with_xml': '0', '._starts_with_punctuation': '0', PROFILE_SLUG: profile.id, }) sync_call_center_user_case(self.user) case = self._get_user_case() self.assertIsNotNone(case) self.assertEqual(case.get_case_property('blank_val'), '') self.assertEqual(case.get_case_property('ok'), 'good') self.assertEqual(case.get_case_property(PROFILE_SLUG), str(profile.id)) self.assertEqual(case.get_case_property('from_profile'), 'yes') self.user.pop_metadata(PROFILE_SLUG) definition.delete()
def test_migration(self): doc = CustomDataFieldsDefinition( domain='botswana', field_type='UserFields', fields=[ CustomDataField( slug='color', is_required=True, label='Color', choices=['red', 'orange', 'yellow'], ), ], ) doc.save(sync_to_sql=False) call_command('populate_custom_data_fields') self.assertIsNone( Command.diff_couch_and_sql( doc.to_json(), SQLCustomDataFieldsDefinition.objects.get( couch_id=doc.get_id))) # Additional call should apply any updates doc = CustomDataFieldsDefinition.get(doc.get_id) doc.field_type = 'LocationFields' doc.save(sync_to_sql=False) call_command('populate_custom_data_fields') self.assertIsNone( Command.diff_couch_and_sql( doc.to_json(), SQLCustomDataFieldsDefinition.objects.get( couch_id=doc.get_id)))
def test_custom_data_fields(self): for domain_name in [self.domain.name, self.domain2.name]: CustomDataFieldsDefinition.get_or_create(domain_name, 'UserFields') self.domain.delete() self._assert_custom_data_fields_counts(self.domain.name, 0) self._assert_custom_data_fields_counts(self.domain2.name, 1)
def get_custom_data_models(domain, limit_types=None): fields = defaultdict(dict) for field_view in [LocationFieldsView, ProductFieldsView, UserFieldsView]: if limit_types and field_view.field_type not in limit_types: continue model = CustomDataFieldsDefinition.get(domain, field_view.field_type) if model: fields[field_view.field_type]['fields'] = [{ 'slug': field.slug, 'is_required': field.is_required, 'label': field.label, 'choices': field.choices, 'regex': field.regex, 'regex_msg': field.regex_msg, } for field in model.get_fields()] if field_view.show_profiles: fields[field_view.field_type]['profiles'] = [ profile.to_json() for profile in model.get_profiles() ] return fields
def get_location_data_fields(domain): from corehq.apps.locations.views import LocationFieldsView fields_definition = CustomDataFieldsDefinition.get(domain, LocationFieldsView.field_type) if fields_definition: return fields_definition.get_fields() else: return []
def handle(self, *args, **options): for domain in Domain.get_all_names(): fields_definition = CustomDataFieldsDefinition.get_or_create( domain, 'UserFields' ) user_ids = (CommCareUser.ids_by_domain(domain) + CommCareUser.ids_by_domain(domain, is_active=False)) existing_field_slugs = set([field.slug for field in fields_definition.fields]) for user in iter_docs(CommCareUser.get_db(), user_ids): user_data = user.get('user_data', {}) for key in user_data.keys(): if key and key not in existing_field_slugs: existing_field_slugs.add(key) fields_definition.fields.append(CustomDataField( slug=key, label=key, is_required=False )) # Only save a definition for domains which use custom user data if fields_definition.fields: fields_definition.save()
def save_custom_fields(domain, definition_name, custom_fields): if custom_fields: fields_definitions = CustomDataFieldsDefinition.get_or_create(domain, definition_name) need_save = False for custom_field in custom_fields: name = custom_field.get('name') label = custom_field.get('label') choices = custom_field.get('choices') or [] existing_fields = [field for field in fields_definitions.fields if field.slug == name] if not existing_fields: need_save = True fields_definitions.fields.append( CustomDataField( slug=name, label=label or name, is_required=False, choices=choices, is_multiple_choice=custom_field.get('is_multiple_choice', False) ) ) else: existing_field = existing_fields[0] if set(existing_field.choices) != set(choices): existing_field.choices = choices need_save = True if need_save: fields_definitions.save()
def update_custom_data_models(domain_link, limit_types=None): if domain_link.is_remote: master_results = remote_custom_data_models(domain_link, limit_types) else: master_results = local_custom_data_models(domain_link.master_domain, limit_types) for field_type, data in master_results.items(): field_definitions = data.get('fields', []) model = CustomDataFieldsDefinition.get_or_create(domain_link.linked_domain, field_type) model.set_fields([ Field( slug=field_def['slug'], is_required=field_def['is_required'], label=field_def['label'], choices=field_def['choices'], regex=field_def['regex'], regex_msg=field_def['regex_msg'], ) for field_def in field_definitions ]) model.save() old_profiles = {profile.name: profile for profile in model.get_profiles()} for profile in data.get('profiles'): old_profile = old_profiles.get(profile['name'], None) if old_profile: old_profile.fields = profile['fields'] old_profile.save() else: CustomDataFieldsProfile( name=profile['name'], definition=model, fields=profile['fields'], ).save()
def handle(self, *args, **options): for domain in Domain.get_all(): if domain['commtrack_enabled']: fields_definition = CustomDataFieldsDefinition.get_or_create( domain['name'], 'ProductFields' ) product_ids = Product.ids_by_domain(domain['name']) existing_field_slugs = set( [field.slug for field in fields_definition.fields] ) for product in iter_docs(Product.get_db(), product_ids): product_data = product.get('product_data', {}) for key in product_data.keys(): if key and key not in existing_field_slugs: existing_field_slugs.add(key) fields_definition.fields.append(CustomDataField( slug=key, label=key, is_required=False )) # Only save a definition for domains which use custom product data if fields_definition.fields: fields_definition.save()
def setUpClass(cls): super().setUpClass() cls.domain = 'user-esaccessors-test' cls.definition = CustomDataFieldsDefinition(domain=cls.domain, field_type=UserFieldsView.field_type) cls.definition.save() cls.definition.set_fields([ Field(slug='job', label='Job'), ]) cls.definition.save() cls.profile = CustomDataFieldsProfile( name='daily_planet_staff', fields={'job': 'reporter'}, definition=cls.definition, ) cls.profile.save() cls.user = CommCareUser.create( cls.domain, 'superman', 'secret agent man', None, None, first_name='clark', last_name='kent', is_active=True, metadata={PROFILE_SLUG: cls.profile.id, 'office': 'phone_booth'}, ) cls.user.save()
def get_usercase_default_properties(domain): from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition from corehq.apps.users.views.mobile.custom_data_fields import CUSTOM_USER_DATA_FIELD_TYPE fields_def = CustomDataFieldsDefinition.get_or_create( domain, CUSTOM_USER_DATA_FIELD_TYPE) return [f.slug for f in fields_def.fields]
def setUpClass(cls): super(TestLocationsExport, cls).setUpClass() cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type) cls.loc_fields.set_fields([ Field(slug=slug) for slug in cls.custom_fields ]) cls.loc_fields.save() cls.boston = cls.locations['Boston'] cls.boston.metadata = { field: '{}-试验'.format(field) for field in cls.custom_fields + ['不知道'] } cls.boston.external_id = 'external_id' cls.boston.latitude = Decimal('42.36') cls.boston.longitude = Decimal('71.06') cls.boston.save() exporter = LocationExporter(cls.domain) writer = MockExportWriter() exporter.write_data(writer) cls.headers = dict(exporter.get_headers()) cls.city_headers = cls.headers['city'][0] cls.boston_data = [row for row in writer.data['city'] if row[0] == cls.boston.location_id][0]
def get_location_data_model(domain): from .views import LocationFieldsView from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition return CustomDataFieldsDefinition.get_or_create( domain, LocationFieldsView.field_type, )
def parse_mobile_users(domain, user_filters, task=None, total_count=None): from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView fields_definition = CustomDataFieldsDefinition.get_or_create( domain, UserFieldsView.field_type ) unrecognized_user_data_keys = set() user_groups_length = 0 max_location_length = 0 user_dicts = [] domains_list = [domain] is_multi_domain_download = False if 'domains' in user_filters: domains_list = user_filters['domains'] if domains_list != [domain]: is_multi_domain_download = True current_user_downloaded_count = 0 for current_domain in domains_list: location_cache = LocationIdToSiteCodeCache(current_domain) for n, user in enumerate(get_commcare_users_by_filters(current_domain, user_filters)): group_memoizer = load_memoizer(current_domain) group_names = sorted([ group_memoizer.get(id).name for id in Group.by_user_id(user.user_id, wrap=False) ], key=alphanumeric_sort_key) user_dict = make_mobile_user_dict(user, group_names, location_cache, current_domain, fields_definition) user_dicts.append(user_dict) unrecognized_user_data_keys.update(user_dict['uncategorized_data']) user_groups_length = max(user_groups_length, len(group_names)) max_location_length = max(max_location_length, len(user_dict["location_code"])) if task: DownloadBase.set_progress(task, n + current_user_downloaded_count, total_count) current_user_downloaded_count += n + 1 user_headers = [ 'username', 'password', 'name', 'phone-number', 'email', 'language', 'role', 'user_id', 'is_active', 'User IMEIs (read only)', 'registered_on (read only)', 'last_submission (read only)', 'last_sync (read only)' ] if domain_has_privilege(domain, privileges.APP_USER_PROFILES): user_headers += ['user_profile'] user_data_fields = [f.slug for f in fields_definition.get_fields(include_system=False)] user_headers.extend(build_data_headers(user_data_fields)) user_headers.extend(build_data_headers( unrecognized_user_data_keys, header_prefix='uncategorized_data' )) user_headers.extend(json_to_headers( {'group': list(range(1, user_groups_length + 1))} )) if domain_has_privilege(domain, privileges.LOCATIONS): user_headers.extend(json_to_headers( {'location_code': list(range(1, max_location_length + 1))} )) if is_multi_domain_download: user_headers += ['domain'] return user_headers, get_user_rows(user_dicts, user_headers)
def test_sync_to_sql(self): doc = CustomDataFieldsDefinition( domain='botswana', field_type='UserFields', fields=[ CustomDataField( slug='color', is_required=True, label='Color', choices=['red', 'orange', 'yellow'], ), CustomDataField( slug='size', is_required=False, label='Size', regex='^[0-9]+$', regex_msg='Εισαγάγετε', ), ], ) doc.save() self.assertIsNone( Command.diff_couch_and_sql( doc.to_json(), SQLCustomDataFieldsDefinition.objects.get( couch_id=doc.get_id))) doc.fields[0].label = 'Hue' doc.save() self.assertEquals( 'Hue', SQLCustomDataFieldsDefinition.objects.get( couch_id=doc.get_id).get_fields()[0].label)
def dump_users_and_groups(domain, download_id, user_filters): from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView def _load_memoizer(domain): group_memoizer = GroupMemoizer(domain=domain) # load groups manually instead of calling group_memoizer.load_all() # so that we can detect blank groups blank_groups = set() for group in Group.by_domain(domain): if group.name: group_memoizer.add_group(group) else: blank_groups.add(group) if blank_groups: raise GroupNameError(blank_groups=blank_groups) return group_memoizer writer = Excel2007ExportWriter(format_as_text=True) group_memoizer = _load_memoizer(domain) location_cache = LocationIdToSiteCodeCache(domain) user_data_model = CustomDataFieldsDefinition.get_or_create( domain, UserFieldsView.field_type ) user_headers, user_rows = parse_users( group_memoizer, domain, user_data_model, location_cache, user_filters ) group_headers, group_rows = parse_groups(group_memoizer.groups) headers = [ ('users', [user_headers]), ('groups', [group_headers]), ] rows = [ ('users', user_rows), ('groups', group_rows), ] use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled filename = "{}_users_{}.xlsx".format(domain, uuid.uuid4().hex) file_path = get_download_file_path(use_transfer, filename) writer.open( header_table=headers, file=file_path, ) writer.write(rows) writer.close() expose_download(use_transfer, file_path, filename, download_id, 'xlsx')
def setUpClass(cls): super(LocationFixturesDataTest, cls).setUpClass() cls.user = create_restore_user(cls.domain, 'user', '123') cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type) cls.loc_fields.fields = [ CustomDataField(slug='baseball_team'), CustomDataField(slug='favorite_passtime'), ] cls.loc_fields.save() cls.field_slugs = [f.slug for f in cls.loc_fields.fields]
def update_custom_data_models(domain_link, limit_types=None): if domain_link.is_remote: master_results = remote_custom_data_models(domain_link, limit_types) else: master_results = local_custom_data_models(domain_link.master_domain, limit_types) for field_type, field_definitions in master_results.items(): model = CustomDataFieldsDefinition.get_or_create(domain_link.linked_domain, field_type) model.fields = [CustomDataField.wrap(field_def) for field_def in field_definitions] model.save()
def dump_users_and_groups(response, domain): def _load_memoizer(domain): group_memoizer = GroupMemoizer(domain=domain) # load groups manually instead of calling group_memoizer.load_all() # so that we can detect blank groups blank_groups = set() for group in Group.by_domain(domain): if group.name: group_memoizer.add_group(group) else: blank_groups.add(group) if blank_groups: raise GroupNameError(blank_groups=blank_groups) return group_memoizer export_file = StringIO() writer = Excel2007ExportWriter() group_memoizer = _load_memoizer(domain) user_data_model = CustomDataFieldsDefinition.get_or_create( domain, UserFieldsView.field_type ) user_data_fields = [f.slug for f in user_data_model.fields] user_headers, user_rows = parse_users(group_memoizer, CommCareUser.by_domain(domain), user_data_fields) group_headers, group_rows = parse_groups(group_memoizer.groups) headers = [ ('users', [user_headers]), ('groups', [group_headers]), ] rows = [ ('users', user_rows), ('groups', group_rows), ] commtrack_enabled = Domain.get_by_name(domain).commtrack_enabled if commtrack_enabled: headers.append( ('locations', [['username', 'location-sms-code', 'location name (optional)']]) ) rows.append( ('locations', get_location_rows(domain)) ) writer.open( header_table=headers, file=export_file, ) writer.write(rows) writer.close() response.write(export_file.getvalue())
def setUpClass(cls): super().setUpClass() cls.domain_obj = create_domain(cls.domain) # APP_USER_PROFILES is on ENTERPRISE and above cls.setup_subscription(cls.domain, SoftwarePlanEdition.ENTERPRISE) cls.group_memoizer = GroupMemoizer(domain=cls.domain_obj.name) cls.group_memoizer.load_all() cls.definition = CustomDataFieldsDefinition( domain=cls.domain_obj.name, field_type=UserFieldsView.field_type) cls.definition.save() cls.definition.set_fields([ Field( slug='born', label='Year of Birth', ), Field( slug='_type', label='Type', choices=['fiction', 'non-fiction'], ), ]) cls.definition.save() cls.profile = CustomDataFieldsProfile( name='Novelist', fields={'_type': 'fiction'}, definition=cls.definition, ) cls.profile.save() cls.user1 = CommCareUser.create(cls.domain_obj.name, 'edith', 'badpassword', None, None, first_name='Edith', last_name='Wharton', metadata={'born': 1862}) cls.user2 = CommCareUser.create( cls.domain_obj.name, 'george', 'anotherbadpassword', None, None, first_name='George', last_name='Eliot', metadata={ 'born': 1849, PROFILE_SLUG: cls.profile.id }, )
def handle(self, domain, **options): self.user_data = CustomDataFieldsDefinition.get_or_create( domain, UserFieldsView.field_type) self.location_data = CustomDataFieldsDefinition.get_or_create( domain, LocationFieldsView.field_type) print("\nOLD:") self.show(self.user_data) self.update_definition(self.user_data, AGENCY_USER_FIELDS) print("\nNEW:") self.show(self.user_data) if self.confirm(): self.user_data.save() print("\nOLD:") self.show(self.location_data) self.update_definition(self.location_data, AGENCY_LOCATION_FIELDS) print("\nNEW:") self.show(self.location_data) if self.confirm(): self.location_data.save()
def get_by_domain_and_type(domain, field_type): from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition return CustomDataFieldsDefinition.view( 'custom_data_fields/by_field_type', key=[domain, field_type], include_docs=True, reduce=False, # if there's more than one, # it's probably because a few were created at the same time # due to a race condition limit=1, ).one()
def case_properties(self): props = set([]) if CALLCENTER.enabled(self.domain): from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition user_fields = CustomDataFieldsDefinition.get_or_create(self.domain, 'UserFields') props |= {field.slug for field in user_fields.fields} for app in self.applications: builder = ParentCasePropertyBuilder(app, ("name",)) props |= set(builder.get_properties(self.case_type)) return props
def dump_users_and_groups(domain, download_id, user_filters, task): from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView def _load_memoizer(domain): group_memoizer = GroupMemoizer(domain=domain) # load groups manually instead of calling group_memoizer.load_all() # so that we can detect blank groups blank_groups = set() for group in Group.by_domain(domain): if group.name: group_memoizer.add_group(group) else: blank_groups.add(group) if blank_groups: raise GroupNameError(blank_groups=blank_groups) return group_memoizer group_memoizer = _load_memoizer(domain) location_cache = LocationIdToSiteCodeCache(domain) users_groups_count = count_users_and_groups(domain, user_filters, group_memoizer) DownloadBase.set_progress(task, 0, users_groups_count) user_data_model = CustomDataFieldsDefinition.get_or_create( domain, UserFieldsView.field_type) user_headers, user_rows = parse_users( group_memoizer, domain, user_data_model, location_cache, user_filters, task, users_groups_count, ) group_headers, group_rows = parse_groups(group_memoizer.groups) headers = [ ('users', [user_headers]), ('groups', [group_headers]), ] rows = [ ('users', user_rows), ('groups', group_rows), ] filename = "{}_users_{}.xlsx".format(domain, uuid.uuid4().hex) _dump_xlsx_and_expose_download(filename, headers, rows, download_id, task, users_groups_count)
def prepare_domain(domain_name): from corehq.apps.commtrack.tests.util import bootstrap_domain domain = bootstrap_domain(domain_name) previous = None for name, administrative in [ ("MOHSW", True), ("MSDZONE", True), ("REGION", True), ("DISTRICT", True), ("FACILITY", False) ]: previous, _ = LocationType.objects.get_or_create( domain=domain_name, name=name, parent_type=previous, administrative=administrative, ) account = BillingAccount.get_or_create_account_by_domain( domain.name, created_by="automated-test", )[0] plan = DefaultProductPlan.get_default_plan_version( edition=SoftwarePlanEdition.ADVANCED ) commtrack = domain.commtrack_settings commtrack.actions.append( CommtrackActionConfig(action='receipts', keyword='delivered', caption='Delivered') ) commtrack.save() subscription = Subscription.new_domain_subscription( account, domain.name, plan ) subscription.is_active = True subscription.save() ils_config = ILSGatewayConfig(enabled=True, domain=domain.name, all_stock_data=True) ils_config.save() fields_definition = CustomDataFieldsDefinition.get_or_create(domain.name, 'LocationFields') fields_definition.fields.append(CustomDataField( slug='group', label='Group', is_required=False, choices=['A', 'B', 'C'], is_multiple_choice=False )) fields_definition.save() return domain
def _parse_custom_properties(product): product_data_model = CustomDataFieldsDefinition.get_or_create( domain, ProductFieldsView.field_type) product_data_fields = [f.slug for f in product_data_model.fields] model_data = {} uncategorized_data = {} for prop, val in product.product_data.items(): if prop in product_data_fields: model_data['data: ' + prop] = encode_if_needed(val) else: uncategorized_data['uncategorized_data: ' + prop] = encode_if_needed(val) return model_data, uncategorized_data
def setUpClass(cls): super(TestIndexedLocationsFixture, cls).setUpClass() cls.user = create_restore_user(cls.domain, 'user', '123') cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type) cls.loc_fields.fields = [ CustomDataField(slug='is_test', index_in_fixture=True), CustomDataField(slug='favorite_color'), ] cls.loc_fields.save() cls.field_slugs = [f.slug for f in cls.loc_fields.fields] for location in cls.locations.values(): location.metadata = { 'is_test': 'no', 'favorite_color': 'blue', } location.save()
def test_download_reupload_no_changes(self): # Make sure there's a bunch of data loc_fields = CustomDataFieldsDefinition.get_or_create( self.domain, 'LocationFields') loc_fields.fields = [ CustomDataField(slug='favorite_color'), CustomDataField(slug='language') ] loc_fields.save() self.locations['City111'].latitude = Decimal('42.36') self.locations['City111'].longitude = Decimal('71.06') self.locations['City111'].external_id = '123' self.locations['County11'].metadata = { 'favorite_color': 'purple', 'language': 'en' } self.locations['City111'].save() self.locations['County11'].external_id = '321' self.locations['County11'].metadata = {'favorite_color': 'blue'} self.locations['County11'].save() # Export locations exporter = LocationExporter(self.domain) writer = MockExportWriter() exporter.write_data(writer) # Re-upload that export worksheets = [] for sheet_title, headers in exporter.get_headers(): rows = [[val if val is not None else '' for val in row] for row in writer.data[sheet_title]] sheet = IteratorJSONReader(headers + rows) sheet.title = sheet_title worksheets.append(sheet) mock_importer = Mock() mock_importer.worksheets = worksheets with patch('corehq.apps.locations.models.SQLLocation.save') as save_location, \ patch('corehq.apps.locations.models.LocationType.save') as save_type: result = new_locations_import(self.domain, mock_importer, self.user) # The upload should succeed and not perform any updates assert_errors(result, []) self.assertFalse(save_location.called) self.assertFalse(save_type.called)
def setUpClass(cls): super().setUpClass() cls.domain = 'a-domain' cls.definition = CustomDataFieldsDefinition( domain=cls.domain, field_type=UserFieldsView.field_type) cls.definition.save() cls.definition.set_fields([ Field( slug='corners', is_required=True, label='Number of corners', regex='^[0-9]+$', regex_msg='This should be a number', ), Field( slug='prefix', is_required=False, label='Prefix', choices=['tri', 'tetra', 'penta'], ), Field( slug='color', is_required=False, label='Color', ), ]) cls.definition.save() cls.profile3 = CustomDataFieldsProfile( name='three', fields={ 'corners': 3, 'prefix': 'tri' }, definition=cls.definition, ) cls.profile3.save() cls.profile5 = CustomDataFieldsProfile( name='five', fields={ 'corners': 5, 'prefix': 'penta' }, definition=cls.definition, ) cls.profile5.save()
def setUpClass(cls): super().setUpClass() delete_all_users() cls.domain_name = 'mydomain' cls.domain = Domain.get_or_create_with_name(name=cls.domain_name) cls.other_domain = Domain.get_or_create_with_name(name='other-domain') cls.uploading_user = WebUser.create(cls.domain_name, "*****@*****.**", 'password', None, None, is_superuser=True) permissions = Permissions(edit_apps=True, view_apps=True, view_reports=True) cls.role = UserRole.get_or_create_with_permissions( cls.domain.name, permissions, 'edit-apps') cls.other_role = UserRole.get_or_create_with_permissions( cls.domain.name, permissions, 'admin') cls.patcher = patch('corehq.apps.user_importer.tasks.UserUploadRecord') cls.patcher.start() cls.definition = CustomDataFieldsDefinition( domain=cls.domain_name, field_type=UserFieldsView.field_type) cls.definition.save() cls.definition.set_fields([ Field( slug='key', is_required=False, label='Key', regex='^[A-G]', regex_msg='Starts with A-G', ), Field(slug='mode', is_required=False, label='Mode', choices=['major', 'minor']), ]) cls.definition.save() cls.profile = CustomDataFieldsProfile( name='melancholy', fields={'mode': 'minor'}, definition=cls.definition, ) cls.profile.save()
def _parse_custom_properties(product): product_data_model = CustomDataFieldsDefinition.get_or_create( domain, ProductFieldsView.field_type ) product_data_fields = [f.slug for f in product_data_model.fields] model_data = {} uncategorized_data = {} for prop, val in six.iteritems(product.product_data): if prop in product_data_fields: model_data['data: ' + prop] = encode_if_needed(val) else: uncategorized_data['uncategorized_data: ' + prop] = encode_if_needed(val) return model_data, uncategorized_data
def remove_unused_custom_fields_from_users(domain): """ Removes all unused custom data fields from all users in the domain """ from corehq.apps.users.views.mobile.custom_data_fields import CUSTOM_USER_DATA_FIELD_TYPE fields_definition = CustomDataFieldsDefinition.get( domain, CUSTOM_USER_DATA_FIELD_TYPE) assert fields_definition, 'remove_unused_custom_fields_from_users called without a valid definition' configured_field_keys = set( [f.slug for f in fields_definition.get_fields()]) for user in get_all_commcare_users_by_domain(domain): keys_to_delete = _get_invalid_user_data_fields(user, configured_field_keys) if keys_to_delete: for key in keys_to_delete: user.pop_metadata(key) user.save()
def _get_domain_info(domain): domain_info = domain_info_by_domain.get(domain) if domain_info: return domain_info if domain == upload_domain: domain_group_memoizer = group_memoizer or GroupMemoizer(domain) else: domain_group_memoizer = GroupMemoizer(domain) domain_group_memoizer.load_all() can_assign_locations = domain_has_privilege(domain, privileges.LOCATIONS) location_cache = None if can_assign_locations: location_cache = SiteCodeToLocationCache(domain) domain_obj = Domain.get_by_name(domain) allowed_group_names = [ group.name for group in domain_group_memoizer.groups ] roles_by_name = { role.name: role for role in UserRole.by_domain(domain) } profiles_by_name = {} definition = CustomDataFieldsDefinition.get(domain, UserFieldsView.field_type) if definition: profiles_by_name = { profile.name: profile for profile in definition.get_profiles() } domain_user_specs = [ spec for spec in user_specs if spec.get('domain', upload_domain) == domain ] validators = get_user_import_validators(domain_obj, domain_user_specs, allowed_group_names, list(roles_by_name), list(profiles_by_name), upload_domain) domain_info = DomainInfo(validators, can_assign_locations, location_cache, roles_by_name, profiles_by_name, domain_group_memoizer) domain_info_by_domain[domain] = domain_info return domain_info
def setUpClass(cls): super(TestIndexedLocationsFixture, cls).setUpClass() cls.user = create_restore_user(cls.domain, 'user', '123') cls.loc_fields = CustomDataFieldsDefinition.get_or_create( cls.domain, LocationFieldsView.field_type) cls.loc_fields.fields = [ CustomDataField(slug='is_test', index_in_fixture=True), CustomDataField(slug='favorite_color'), ] cls.loc_fields.save() cls.field_slugs = [f.slug for f in cls.loc_fields.fields] for location in cls.locations.values(): location.metadata = { 'is_test': 'no', 'favorite_color': 'blue', } location.save()
def _setup_profile(self): definition = CustomDataFieldsDefinition( domain=self.domain, field_type=UserFieldsView.field_type) definition.save() definition.set_fields([ Field( slug='conflicting_field', label='Conflicting Field', choices=['yes', 'no'], ), ]) definition.save() profile = CustomDataFieldsProfile( name='character', fields={'conflicting_field': 'yes'}, definition=definition, ) profile.save() return profile.id
def test_download_reupload_no_changes(self): # Make sure there's a bunch of data loc_fields = CustomDataFieldsDefinition.get_or_create( self.domain, 'LocationFields') loc_fields.fields = [CustomDataField(slug='favorite_color'), CustomDataField(slug='language')] loc_fields.save() self.locations['City111'].latitude = Decimal('42.36') self.locations['City111'].longitude = Decimal('71.06') self.locations['City111'].external_id = '123' self.locations['County11'].metadata = {'favorite_color': 'purple', 'language': 'en'} self.locations['City111'].save() self.locations['County11'].external_id = '321' self.locations['County11'].metadata = {'favorite_color': 'blue'} self.locations['County11'].save() # Export locations exporter = LocationExporter(self.domain) writer = MockExportWriter() exporter.write_data(writer) # Re-upload that export worksheets = [] for sheet_title, headers in exporter.get_headers(): rows = [[val if val is not None else '' for val in row] for row in writer.data[sheet_title]] sheet = IteratorJSONReader(headers + rows) sheet.title = sheet_title worksheets.append(sheet) mock_importer = Mock() mock_importer.worksheets = worksheets with patch('corehq.apps.locations.models.SQLLocation.save') as save_location, \ patch('corehq.apps.locations.models.LocationType.save') as save_type: result = new_locations_import(self.domain, mock_importer, self.user) # The upload should succeed and not perform any updates assert_errors(result, []) self.assertFalse(save_location.called) self.assertFalse(save_type.called)
def dump_users_and_groups(domain, download_id, user_filters, task): from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView def _load_memoizer(domain): group_memoizer = GroupMemoizer(domain=domain) # load groups manually instead of calling group_memoizer.load_all() # so that we can detect blank groups blank_groups = set() for group in Group.by_domain(domain): if group.name: group_memoizer.add_group(group) else: blank_groups.add(group) if blank_groups: raise GroupNameError(blank_groups=blank_groups) return group_memoizer writer = Excel2007ExportWriter(format_as_text=True) group_memoizer = _load_memoizer(domain) location_cache = LocationIdToSiteCodeCache(domain) users_groups_count = count_users_and_groups(domain, user_filters, group_memoizer) DownloadBase.set_progress(task, 0, users_groups_count) user_data_model = CustomDataFieldsDefinition.get_or_create( domain, UserFieldsView.field_type ) user_headers, user_rows = parse_users( group_memoizer, domain, user_data_model, location_cache, user_filters, task, users_groups_count, ) group_headers, group_rows = parse_groups(group_memoizer.groups) headers = [ ('users', [user_headers]), ('groups', [group_headers]), ] rows = [ ('users', user_rows), ('groups', group_rows), ] use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled filename = "{}_users_{}.xlsx".format(domain, uuid.uuid4().hex) file_path = get_download_file_path(use_transfer, filename) writer.open( header_table=headers, file=file_path, ) writer.write(rows) writer.close() expose_download(use_transfer, file_path, filename, download_id, 'xlsx') DownloadBase.set_progress(task, users_groups_count, users_groups_count)
def get_usercase_default_properties(domain): from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition from corehq.apps.users.views.mobile.custom_data_fields import CUSTOM_USER_DATA_FIELD_TYPE fields_def = CustomDataFieldsDefinition.get_or_create(domain, CUSTOM_USER_DATA_FIELD_TYPE) return [f.slug for f in fields_def.fields]