def test_user_data_profile(self):
definition = CustomDataFieldsDefinition(
domain='cloudcare-tests', field_type=UserFieldsView.field_type)
definition.save()
definition.set_fields([
Field(slug='word', label='A Word'),
])
definition.save()
profile = CustomDataFieldsProfile(name='prof',
fields={'word': 'supernova'},
definition=definition)
profile.save()
user = CommCareUser.create(
'cloudcare-tests',
'*****@*****.**',
'do you want to know a secret',
None,
None,
uuid=uuid.uuid4().hex,
metadata={PROFILE_SLUG: profile.id},
)
user_data = get_user_contributions_to_touchforms_session(
user)['user_data']
self.assertEqual(profile.id, user_data[PROFILE_SLUG])
self.assertEqual('supernova', user_data['word'])
definition.delete()
def test_sync_custom_user_data(self):
definition = CustomDataFieldsDefinition(
domain=TEST_DOMAIN, field_type=UserFieldsView.field_type)
definition.save()
definition.set_fields([
Field(slug='from_profile', label='From Profile'),
])
definition.save()
profile = CustomDataFieldsProfile(
name='callcenter_profile',
fields={'from_profile': 'yes'},
definition=definition,
)
profile.save()
self.user.update_metadata({
'': 'blank_key',
'blank_val': '',
'ok': 'good',
'name with spaces': 'ok',
'8starts_with_a_number': '0',
'xml_starts_with_xml': '0',
'._starts_with_punctuation': '0',
PROFILE_SLUG: profile.id,
})
sync_call_center_user_case(self.user)
case = self._get_user_case()
self.assertIsNotNone(case)
self.assertEqual(case.get_case_property('blank_val'), '')
self.assertEqual(case.get_case_property('ok'), 'good')
self.assertEqual(case.get_case_property(PROFILE_SLUG), str(profile.id))
self.assertEqual(case.get_case_property('from_profile'), 'yes')
self.user.pop_metadata(PROFILE_SLUG)
definition.delete()
def test_migration(self):
doc = CustomDataFieldsDefinition(
domain='botswana',
field_type='UserFields',
fields=[
CustomDataField(
slug='color',
is_required=True,
label='Color',
choices=['red', 'orange', 'yellow'],
),
],
)
doc.save(sync_to_sql=False)
call_command('populate_custom_data_fields')
self.assertIsNone(
Command.diff_couch_and_sql(
doc.to_json(),
SQLCustomDataFieldsDefinition.objects.get(
couch_id=doc.get_id)))
# Additional call should apply any updates
doc = CustomDataFieldsDefinition.get(doc.get_id)
doc.field_type = 'LocationFields'
doc.save(sync_to_sql=False)
call_command('populate_custom_data_fields')
self.assertIsNone(
Command.diff_couch_and_sql(
doc.to_json(),
SQLCustomDataFieldsDefinition.objects.get(
couch_id=doc.get_id)))
def test_custom_data_fields(self):
for domain_name in [self.domain.name, self.domain2.name]:
CustomDataFieldsDefinition.get_or_create(domain_name, 'UserFields')
self.domain.delete()
self._assert_custom_data_fields_counts(self.domain.name, 0)
self._assert_custom_data_fields_counts(self.domain2.name, 1)
def get_custom_data_models(domain, limit_types=None):
fields = defaultdict(dict)
for field_view in [LocationFieldsView, ProductFieldsView, UserFieldsView]:
if limit_types and field_view.field_type not in limit_types:
continue
model = CustomDataFieldsDefinition.get(domain, field_view.field_type)
if model:
fields[field_view.field_type]['fields'] = [{
'slug':
field.slug,
'is_required':
field.is_required,
'label':
field.label,
'choices':
field.choices,
'regex':
field.regex,
'regex_msg':
field.regex_msg,
} for field in model.get_fields()]
if field_view.show_profiles:
fields[field_view.field_type]['profiles'] = [
profile.to_json() for profile in model.get_profiles()
]
return fields
def get_location_data_fields(domain):
from corehq.apps.locations.views import LocationFieldsView
fields_definition = CustomDataFieldsDefinition.get(domain, LocationFieldsView.field_type)
if fields_definition:
return fields_definition.get_fields()
else:
return []
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = CustomDataFieldsDefinition.get_or_create(
domain,
'UserFields'
)
user_ids = (CommCareUser.ids_by_domain(domain) +
CommCareUser.ids_by_domain(domain, is_active=False))
existing_field_slugs = set([field.slug for field in fields_definition.fields])
for user in iter_docs(CommCareUser.get_db(), user_ids):
user_data = user.get('user_data', {})
for key in user_data.keys():
if key and key not in existing_field_slugs:
existing_field_slugs.add(key)
fields_definition.fields.append(CustomDataField(
slug=key,
label=key,
is_required=False
))
# Only save a definition for domains which use custom user data
if fields_definition.fields:
fields_definition.save()
def save_custom_fields(domain, definition_name, custom_fields):
if custom_fields:
fields_definitions = CustomDataFieldsDefinition.get_or_create(domain, definition_name)
need_save = False
for custom_field in custom_fields:
name = custom_field.get('name')
label = custom_field.get('label')
choices = custom_field.get('choices') or []
existing_fields = [field for field in fields_definitions.fields if field.slug == name]
if not existing_fields:
need_save = True
fields_definitions.fields.append(
CustomDataField(
slug=name,
label=label or name,
is_required=False,
choices=choices,
is_multiple_choice=custom_field.get('is_multiple_choice', False)
)
)
else:
existing_field = existing_fields[0]
if set(existing_field.choices) != set(choices):
existing_field.choices = choices
need_save = True
if need_save:
fields_definitions.save()
def update_custom_data_models(domain_link, limit_types=None):
if domain_link.is_remote:
master_results = remote_custom_data_models(domain_link, limit_types)
else:
master_results = local_custom_data_models(domain_link.master_domain, limit_types)
for field_type, data in master_results.items():
field_definitions = data.get('fields', [])
model = CustomDataFieldsDefinition.get_or_create(domain_link.linked_domain, field_type)
model.set_fields([
Field(
slug=field_def['slug'],
is_required=field_def['is_required'],
label=field_def['label'],
choices=field_def['choices'],
regex=field_def['regex'],
regex_msg=field_def['regex_msg'],
) for field_def in field_definitions
])
model.save()
old_profiles = {profile.name: profile for profile in model.get_profiles()}
for profile in data.get('profiles'):
old_profile = old_profiles.get(profile['name'], None)
if old_profile:
old_profile.fields = profile['fields']
old_profile.save()
else:
CustomDataFieldsProfile(
name=profile['name'],
definition=model,
fields=profile['fields'],
).save()
def handle(self, *args, **options):
for domain in Domain.get_all():
if domain['commtrack_enabled']:
fields_definition = CustomDataFieldsDefinition.get_or_create(
domain['name'],
'ProductFields'
)
product_ids = Product.ids_by_domain(domain['name'])
existing_field_slugs = set(
[field.slug for field in fields_definition.fields]
)
for product in iter_docs(Product.get_db(), product_ids):
product_data = product.get('product_data', {})
for key in product_data.keys():
if key and key not in existing_field_slugs:
existing_field_slugs.add(key)
fields_definition.fields.append(CustomDataField(
slug=key,
label=key,
is_required=False
))
# Only save a definition for domains which use custom product data
if fields_definition.fields:
fields_definition.save()
def setUpClass(cls):
super().setUpClass()
cls.domain = 'user-esaccessors-test'
cls.definition = CustomDataFieldsDefinition(domain=cls.domain,
field_type=UserFieldsView.field_type)
cls.definition.save()
cls.definition.set_fields([
Field(slug='job', label='Job'),
])
cls.definition.save()
cls.profile = CustomDataFieldsProfile(
name='daily_planet_staff',
fields={'job': 'reporter'},
definition=cls.definition,
)
cls.profile.save()
cls.user = CommCareUser.create(
cls.domain,
'superman',
'secret agent man',
None,
None,
first_name='clark',
last_name='kent',
is_active=True,
metadata={PROFILE_SLUG: cls.profile.id, 'office': 'phone_booth'},
)
cls.user.save()
def get_usercase_default_properties(domain):
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
from corehq.apps.users.views.mobile.custom_data_fields import CUSTOM_USER_DATA_FIELD_TYPE
fields_def = CustomDataFieldsDefinition.get_or_create(
domain, CUSTOM_USER_DATA_FIELD_TYPE)
return [f.slug for f in fields_def.fields]
def setUpClass(cls):
super(TestLocationsExport, cls).setUpClass()
cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type)
cls.loc_fields.set_fields([
Field(slug=slug) for slug in cls.custom_fields
])
cls.loc_fields.save()
cls.boston = cls.locations['Boston']
cls.boston.metadata = {
field: '{}-试验'.format(field) for field in cls.custom_fields + ['不知道']
}
cls.boston.external_id = 'external_id'
cls.boston.latitude = Decimal('42.36')
cls.boston.longitude = Decimal('71.06')
cls.boston.save()
exporter = LocationExporter(cls.domain)
writer = MockExportWriter()
exporter.write_data(writer)
cls.headers = dict(exporter.get_headers())
cls.city_headers = cls.headers['city'][0]
cls.boston_data = [row for row in writer.data['city'] if row[0] == cls.boston.location_id][0]
def get_location_data_model(domain):
from .views import LocationFieldsView
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
return CustomDataFieldsDefinition.get_or_create(
domain,
LocationFieldsView.field_type,
)
def get_location_data_model(domain):
from .views import LocationFieldsView
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
return CustomDataFieldsDefinition.get_or_create(
domain,
LocationFieldsView.field_type,
)
def parse_mobile_users(domain, user_filters, task=None, total_count=None):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
fields_definition = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
unrecognized_user_data_keys = set()
user_groups_length = 0
max_location_length = 0
user_dicts = []
domains_list = [domain]
is_multi_domain_download = False
if 'domains' in user_filters:
domains_list = user_filters['domains']
if domains_list != [domain]:
is_multi_domain_download = True
current_user_downloaded_count = 0
for current_domain in domains_list:
location_cache = LocationIdToSiteCodeCache(current_domain)
for n, user in enumerate(get_commcare_users_by_filters(current_domain, user_filters)):
group_memoizer = load_memoizer(current_domain)
group_names = sorted([
group_memoizer.get(id).name for id in Group.by_user_id(user.user_id, wrap=False)
], key=alphanumeric_sort_key)
user_dict = make_mobile_user_dict(user, group_names, location_cache, current_domain, fields_definition)
user_dicts.append(user_dict)
unrecognized_user_data_keys.update(user_dict['uncategorized_data'])
user_groups_length = max(user_groups_length, len(group_names))
max_location_length = max(max_location_length, len(user_dict["location_code"]))
if task:
DownloadBase.set_progress(task, n + current_user_downloaded_count, total_count)
current_user_downloaded_count += n + 1
user_headers = [
'username', 'password', 'name', 'phone-number', 'email',
'language', 'role', 'user_id', 'is_active', 'User IMEIs (read only)',
'registered_on (read only)', 'last_submission (read only)', 'last_sync (read only)'
]
if domain_has_privilege(domain, privileges.APP_USER_PROFILES):
user_headers += ['user_profile']
user_data_fields = [f.slug for f in fields_definition.get_fields(include_system=False)]
user_headers.extend(build_data_headers(user_data_fields))
user_headers.extend(build_data_headers(
unrecognized_user_data_keys,
header_prefix='uncategorized_data'
))
user_headers.extend(json_to_headers(
{'group': list(range(1, user_groups_length + 1))}
))
if domain_has_privilege(domain, privileges.LOCATIONS):
user_headers.extend(json_to_headers(
{'location_code': list(range(1, max_location_length + 1))}
))
if is_multi_domain_download:
user_headers += ['domain']
return user_headers, get_user_rows(user_dicts, user_headers)
def test_sync_to_sql(self):
doc = CustomDataFieldsDefinition(
domain='botswana',
field_type='UserFields',
fields=[
CustomDataField(
slug='color',
is_required=True,
label='Color',
choices=['red', 'orange', 'yellow'],
),
CustomDataField(
slug='size',
is_required=False,
label='Size',
regex='^[0-9]+$',
regex_msg='Εισαγάγετε',
),
],
)
doc.save()
self.assertIsNone(
Command.diff_couch_and_sql(
doc.to_json(),
SQLCustomDataFieldsDefinition.objects.get(
couch_id=doc.get_id)))
doc.fields[0].label = 'Hue'
doc.save()
self.assertEquals(
'Hue',
SQLCustomDataFieldsDefinition.objects.get(
couch_id=doc.get_id).get_fields()[0].label)
def dump_users_and_groups(domain, download_id, user_filters):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
writer = Excel2007ExportWriter(format_as_text=True)
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
user_headers, user_rows = parse_users(
group_memoizer,
domain,
user_data_model,
location_cache,
user_filters
)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
filename = "{}_users_{}.xlsx".format(domain, uuid.uuid4().hex)
file_path = get_download_file_path(use_transfer, filename)
writer.open(
header_table=headers,
file=file_path,
)
writer.write(rows)
writer.close()
expose_download(use_transfer, file_path, filename, download_id, 'xlsx')
def setUpClass(cls):
super(LocationFixturesDataTest, cls).setUpClass()
cls.user = create_restore_user(cls.domain, 'user', '123')
cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type)
cls.loc_fields.fields = [
CustomDataField(slug='baseball_team'),
CustomDataField(slug='favorite_passtime'),
]
cls.loc_fields.save()
cls.field_slugs = [f.slug for f in cls.loc_fields.fields]
def setUpClass(cls):
super(LocationFixturesDataTest, cls).setUpClass()
cls.user = create_restore_user(cls.domain, 'user', '123')
cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type)
cls.loc_fields.fields = [
CustomDataField(slug='baseball_team'),
CustomDataField(slug='favorite_passtime'),
]
cls.loc_fields.save()
cls.field_slugs = [f.slug for f in cls.loc_fields.fields]
def update_custom_data_models(domain_link, limit_types=None):
if domain_link.is_remote:
master_results = remote_custom_data_models(domain_link, limit_types)
else:
master_results = local_custom_data_models(domain_link.master_domain, limit_types)
for field_type, field_definitions in master_results.items():
model = CustomDataFieldsDefinition.get_or_create(domain_link.linked_domain, field_type)
model.fields = [CustomDataField.wrap(field_def) for field_def in field_definitions]
model.save()
def update_custom_data_models(domain_link, limit_types=None):
if domain_link.is_remote:
master_results = remote_custom_data_models(domain_link, limit_types)
else:
master_results = local_custom_data_models(domain_link.master_domain, limit_types)
for field_type, field_definitions in master_results.items():
model = CustomDataFieldsDefinition.get_or_create(domain_link.linked_domain, field_type)
model.fields = [CustomDataField.wrap(field_def) for field_def in field_definitions]
model.save()
def dump_users_and_groups(response, domain):
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
export_file = StringIO()
writer = Excel2007ExportWriter()
group_memoizer = _load_memoizer(domain)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
user_data_fields = [f.slug for f in user_data_model.fields]
user_headers, user_rows = parse_users(group_memoizer,
CommCareUser.by_domain(domain),
user_data_fields)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
commtrack_enabled = Domain.get_by_name(domain).commtrack_enabled
if commtrack_enabled:
headers.append(
('locations', [['username', 'location-sms-code', 'location name (optional)']])
)
rows.append(
('locations', get_location_rows(domain))
)
writer.open(
header_table=headers,
file=export_file,
)
writer.write(rows)
writer.close()
response.write(export_file.getvalue())
def setUpClass(cls):
super().setUpClass()
cls.domain_obj = create_domain(cls.domain)
# APP_USER_PROFILES is on ENTERPRISE and above
cls.setup_subscription(cls.domain, SoftwarePlanEdition.ENTERPRISE)
cls.group_memoizer = GroupMemoizer(domain=cls.domain_obj.name)
cls.group_memoizer.load_all()
cls.definition = CustomDataFieldsDefinition(
domain=cls.domain_obj.name, field_type=UserFieldsView.field_type)
cls.definition.save()
cls.definition.set_fields([
Field(
slug='born',
label='Year of Birth',
),
Field(
slug='_type',
label='Type',
choices=['fiction', 'non-fiction'],
),
])
cls.definition.save()
cls.profile = CustomDataFieldsProfile(
name='Novelist',
fields={'_type': 'fiction'},
definition=cls.definition,
)
cls.profile.save()
cls.user1 = CommCareUser.create(cls.domain_obj.name,
'edith',
'badpassword',
None,
None,
first_name='Edith',
last_name='Wharton',
metadata={'born': 1862})
cls.user2 = CommCareUser.create(
cls.domain_obj.name,
'george',
'anotherbadpassword',
None,
None,
first_name='George',
last_name='Eliot',
metadata={
'born': 1849,
PROFILE_SLUG: cls.profile.id
},
)
def handle(self, domain, **options):
self.user_data = CustomDataFieldsDefinition.get_or_create(
domain, UserFieldsView.field_type)
self.location_data = CustomDataFieldsDefinition.get_or_create(
domain, LocationFieldsView.field_type)
print("\nOLD:")
self.show(self.user_data)
self.update_definition(self.user_data, AGENCY_USER_FIELDS)
print("\nNEW:")
self.show(self.user_data)
if self.confirm():
self.user_data.save()
print("\nOLD:")
self.show(self.location_data)
self.update_definition(self.location_data, AGENCY_LOCATION_FIELDS)
print("\nNEW:")
self.show(self.location_data)
if self.confirm():
self.location_data.save()
def get_by_domain_and_type(domain, field_type):
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
return CustomDataFieldsDefinition.view(
'custom_data_fields/by_field_type',
key=[domain, field_type],
include_docs=True,
reduce=False,
# if there's more than one,
# it's probably because a few were created at the same time
# due to a race condition
limit=1,
).one()
def get_by_domain_and_type(domain, field_type):
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
return CustomDataFieldsDefinition.view(
'custom_data_fields/by_field_type',
key=[domain, field_type],
include_docs=True,
reduce=False,
# if there's more than one,
# it's probably because a few were created at the same time
# due to a race condition
limit=1,
).one()
def case_properties(self):
props = set([])
if CALLCENTER.enabled(self.domain):
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
user_fields = CustomDataFieldsDefinition.get_or_create(self.domain, 'UserFields')
props |= {field.slug for field in user_fields.fields}
for app in self.applications:
builder = ParentCasePropertyBuilder(app, ("name",))
props |= set(builder.get_properties(self.case_type))
return props
def dump_users_and_groups(domain, download_id, user_filters, task):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
users_groups_count = count_users_and_groups(domain, user_filters,
group_memoizer)
DownloadBase.set_progress(task, 0, users_groups_count)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain, UserFieldsView.field_type)
user_headers, user_rows = parse_users(
group_memoizer,
domain,
user_data_model,
location_cache,
user_filters,
task,
users_groups_count,
)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
filename = "{}_users_{}.xlsx".format(domain, uuid.uuid4().hex)
_dump_xlsx_and_expose_download(filename, headers, rows, download_id, task,
users_groups_count)
def prepare_domain(domain_name):
from corehq.apps.commtrack.tests.util import bootstrap_domain
domain = bootstrap_domain(domain_name)
previous = None
for name, administrative in [
("MOHSW", True),
("MSDZONE", True),
("REGION", True),
("DISTRICT", True),
("FACILITY", False)
]:
previous, _ = LocationType.objects.get_or_create(
domain=domain_name,
name=name,
parent_type=previous,
administrative=administrative,
)
account = BillingAccount.get_or_create_account_by_domain(
domain.name,
created_by="automated-test",
)[0]
plan = DefaultProductPlan.get_default_plan_version(
edition=SoftwarePlanEdition.ADVANCED
)
commtrack = domain.commtrack_settings
commtrack.actions.append(
CommtrackActionConfig(action='receipts',
keyword='delivered',
caption='Delivered')
)
commtrack.save()
subscription = Subscription.new_domain_subscription(
account,
domain.name,
plan
)
subscription.is_active = True
subscription.save()
ils_config = ILSGatewayConfig(enabled=True, domain=domain.name, all_stock_data=True)
ils_config.save()
fields_definition = CustomDataFieldsDefinition.get_or_create(domain.name, 'LocationFields')
fields_definition.fields.append(CustomDataField(
slug='group',
label='Group',
is_required=False,
choices=['A', 'B', 'C'],
is_multiple_choice=False
))
fields_definition.save()
return domain
def prepare_domain(domain_name):
from corehq.apps.commtrack.tests.util import bootstrap_domain
domain = bootstrap_domain(domain_name)
previous = None
for name, administrative in [
("MOHSW", True),
("MSDZONE", True),
("REGION", True),
("DISTRICT", True),
("FACILITY", False)
]:
previous, _ = LocationType.objects.get_or_create(
domain=domain_name,
name=name,
parent_type=previous,
administrative=administrative,
)
account = BillingAccount.get_or_create_account_by_domain(
domain.name,
created_by="automated-test",
)[0]
plan = DefaultProductPlan.get_default_plan_version(
edition=SoftwarePlanEdition.ADVANCED
)
commtrack = domain.commtrack_settings
commtrack.actions.append(
CommtrackActionConfig(action='receipts',
keyword='delivered',
caption='Delivered')
)
commtrack.save()
subscription = Subscription.new_domain_subscription(
account,
domain.name,
plan
)
subscription.is_active = True
subscription.save()
ils_config = ILSGatewayConfig(enabled=True, domain=domain.name, all_stock_data=True)
ils_config.save()
fields_definition = CustomDataFieldsDefinition.get_or_create(domain.name, 'LocationFields')
fields_definition.fields.append(CustomDataField(
slug='group',
label='Group',
is_required=False,
choices=['A', 'B', 'C'],
is_multiple_choice=False
))
fields_definition.save()
return domain
def _parse_custom_properties(product):
product_data_model = CustomDataFieldsDefinition.get_or_create(
domain, ProductFieldsView.field_type)
product_data_fields = [f.slug for f in product_data_model.fields]
model_data = {}
uncategorized_data = {}
for prop, val in product.product_data.items():
if prop in product_data_fields:
model_data['data: ' + prop] = encode_if_needed(val)
else:
uncategorized_data['uncategorized_data: ' +
prop] = encode_if_needed(val)
return model_data, uncategorized_data
def setUpClass(cls):
super(TestIndexedLocationsFixture, cls).setUpClass()
cls.user = create_restore_user(cls.domain, 'user', '123')
cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type)
cls.loc_fields.fields = [
CustomDataField(slug='is_test', index_in_fixture=True),
CustomDataField(slug='favorite_color'),
]
cls.loc_fields.save()
cls.field_slugs = [f.slug for f in cls.loc_fields.fields]
for location in cls.locations.values():
location.metadata = {
'is_test': 'no',
'favorite_color': 'blue',
}
location.save()
def test_download_reupload_no_changes(self):
# Make sure there's a bunch of data
loc_fields = CustomDataFieldsDefinition.get_or_create(
self.domain, 'LocationFields')
loc_fields.fields = [
CustomDataField(slug='favorite_color'),
CustomDataField(slug='language')
]
loc_fields.save()
self.locations['City111'].latitude = Decimal('42.36')
self.locations['City111'].longitude = Decimal('71.06')
self.locations['City111'].external_id = '123'
self.locations['County11'].metadata = {
'favorite_color': 'purple',
'language': 'en'
}
self.locations['City111'].save()
self.locations['County11'].external_id = '321'
self.locations['County11'].metadata = {'favorite_color': 'blue'}
self.locations['County11'].save()
# Export locations
exporter = LocationExporter(self.domain)
writer = MockExportWriter()
exporter.write_data(writer)
# Re-upload that export
worksheets = []
for sheet_title, headers in exporter.get_headers():
rows = [[val if val is not None else '' for val in row]
for row in writer.data[sheet_title]]
sheet = IteratorJSONReader(headers + rows)
sheet.title = sheet_title
worksheets.append(sheet)
mock_importer = Mock()
mock_importer.worksheets = worksheets
with patch('corehq.apps.locations.models.SQLLocation.save') as save_location, \
patch('corehq.apps.locations.models.LocationType.save') as save_type:
result = new_locations_import(self.domain, mock_importer,
self.user)
# The upload should succeed and not perform any updates
assert_errors(result, [])
self.assertFalse(save_location.called)
self.assertFalse(save_type.called)
def setUpClass(cls):
super().setUpClass()
cls.domain = 'a-domain'
cls.definition = CustomDataFieldsDefinition(
domain=cls.domain, field_type=UserFieldsView.field_type)
cls.definition.save()
cls.definition.set_fields([
Field(
slug='corners',
is_required=True,
label='Number of corners',
regex='^[0-9]+$',
regex_msg='This should be a number',
),
Field(
slug='prefix',
is_required=False,
label='Prefix',
choices=['tri', 'tetra', 'penta'],
),
Field(
slug='color',
is_required=False,
label='Color',
),
])
cls.definition.save()
cls.profile3 = CustomDataFieldsProfile(
name='three',
fields={
'corners': 3,
'prefix': 'tri'
},
definition=cls.definition,
)
cls.profile3.save()
cls.profile5 = CustomDataFieldsProfile(
name='five',
fields={
'corners': 5,
'prefix': 'penta'
},
definition=cls.definition,
)
cls.profile5.save()
def setUpClass(cls):
super().setUpClass()
delete_all_users()
cls.domain_name = 'mydomain'
cls.domain = Domain.get_or_create_with_name(name=cls.domain_name)
cls.other_domain = Domain.get_or_create_with_name(name='other-domain')
cls.uploading_user = WebUser.create(cls.domain_name,
"*****@*****.**",
'password',
None,
None,
is_superuser=True)
permissions = Permissions(edit_apps=True,
view_apps=True,
view_reports=True)
cls.role = UserRole.get_or_create_with_permissions(
cls.domain.name, permissions, 'edit-apps')
cls.other_role = UserRole.get_or_create_with_permissions(
cls.domain.name, permissions, 'admin')
cls.patcher = patch('corehq.apps.user_importer.tasks.UserUploadRecord')
cls.patcher.start()
cls.definition = CustomDataFieldsDefinition(
domain=cls.domain_name, field_type=UserFieldsView.field_type)
cls.definition.save()
cls.definition.set_fields([
Field(
slug='key',
is_required=False,
label='Key',
regex='^[A-G]',
regex_msg='Starts with A-G',
),
Field(slug='mode',
is_required=False,
label='Mode',
choices=['major', 'minor']),
])
cls.definition.save()
cls.profile = CustomDataFieldsProfile(
name='melancholy',
fields={'mode': 'minor'},
definition=cls.definition,
)
cls.profile.save()
def _parse_custom_properties(product):
product_data_model = CustomDataFieldsDefinition.get_or_create(
domain,
ProductFieldsView.field_type
)
product_data_fields = [f.slug for f in product_data_model.fields]
model_data = {}
uncategorized_data = {}
for prop, val in six.iteritems(product.product_data):
if prop in product_data_fields:
model_data['data: ' + prop] = encode_if_needed(val)
else:
uncategorized_data['uncategorized_data: ' + prop] = encode_if_needed(val)
return model_data, uncategorized_data
def remove_unused_custom_fields_from_users(domain):
"""
Removes all unused custom data fields from all users in the domain
"""
from corehq.apps.users.views.mobile.custom_data_fields import CUSTOM_USER_DATA_FIELD_TYPE
fields_definition = CustomDataFieldsDefinition.get(
domain, CUSTOM_USER_DATA_FIELD_TYPE)
assert fields_definition, 'remove_unused_custom_fields_from_users called without a valid definition'
configured_field_keys = set(
[f.slug for f in fields_definition.get_fields()])
for user in get_all_commcare_users_by_domain(domain):
keys_to_delete = _get_invalid_user_data_fields(user,
configured_field_keys)
if keys_to_delete:
for key in keys_to_delete:
user.pop_metadata(key)
user.save()
def _get_domain_info(domain):
domain_info = domain_info_by_domain.get(domain)
if domain_info:
return domain_info
if domain == upload_domain:
domain_group_memoizer = group_memoizer or GroupMemoizer(domain)
else:
domain_group_memoizer = GroupMemoizer(domain)
domain_group_memoizer.load_all()
can_assign_locations = domain_has_privilege(domain,
privileges.LOCATIONS)
location_cache = None
if can_assign_locations:
location_cache = SiteCodeToLocationCache(domain)
domain_obj = Domain.get_by_name(domain)
allowed_group_names = [
group.name for group in domain_group_memoizer.groups
]
roles_by_name = {
role.name: role
for role in UserRole.by_domain(domain)
}
profiles_by_name = {}
definition = CustomDataFieldsDefinition.get(domain,
UserFieldsView.field_type)
if definition:
profiles_by_name = {
profile.name: profile
for profile in definition.get_profiles()
}
domain_user_specs = [
spec for spec in user_specs
if spec.get('domain', upload_domain) == domain
]
validators = get_user_import_validators(domain_obj, domain_user_specs,
allowed_group_names,
list(roles_by_name),
list(profiles_by_name),
upload_domain)
domain_info = DomainInfo(validators, can_assign_locations,
location_cache, roles_by_name,
profiles_by_name, domain_group_memoizer)
domain_info_by_domain[domain] = domain_info
return domain_info
def setUpClass(cls):
super(TestIndexedLocationsFixture, cls).setUpClass()
cls.user = create_restore_user(cls.domain, 'user', '123')
cls.loc_fields = CustomDataFieldsDefinition.get_or_create(
cls.domain, LocationFieldsView.field_type)
cls.loc_fields.fields = [
CustomDataField(slug='is_test', index_in_fixture=True),
CustomDataField(slug='favorite_color'),
]
cls.loc_fields.save()
cls.field_slugs = [f.slug for f in cls.loc_fields.fields]
for location in cls.locations.values():
location.metadata = {
'is_test': 'no',
'favorite_color': 'blue',
}
location.save()
def _setup_profile(self):
definition = CustomDataFieldsDefinition(
domain=self.domain, field_type=UserFieldsView.field_type)
definition.save()
definition.set_fields([
Field(
slug='conflicting_field',
label='Conflicting Field',
choices=['yes', 'no'],
),
])
definition.save()
profile = CustomDataFieldsProfile(
name='character',
fields={'conflicting_field': 'yes'},
definition=definition,
)
profile.save()
return profile.id
def test_download_reupload_no_changes(self):
# Make sure there's a bunch of data
loc_fields = CustomDataFieldsDefinition.get_or_create(
self.domain, 'LocationFields')
loc_fields.fields = [CustomDataField(slug='favorite_color'),
CustomDataField(slug='language')]
loc_fields.save()
self.locations['City111'].latitude = Decimal('42.36')
self.locations['City111'].longitude = Decimal('71.06')
self.locations['City111'].external_id = '123'
self.locations['County11'].metadata = {'favorite_color': 'purple',
'language': 'en'}
self.locations['City111'].save()
self.locations['County11'].external_id = '321'
self.locations['County11'].metadata = {'favorite_color': 'blue'}
self.locations['County11'].save()
# Export locations
exporter = LocationExporter(self.domain)
writer = MockExportWriter()
exporter.write_data(writer)
# Re-upload that export
worksheets = []
for sheet_title, headers in exporter.get_headers():
rows = [[val if val is not None else '' for val in row]
for row in writer.data[sheet_title]]
sheet = IteratorJSONReader(headers + rows)
sheet.title = sheet_title
worksheets.append(sheet)
mock_importer = Mock()
mock_importer.worksheets = worksheets
with patch('corehq.apps.locations.models.SQLLocation.save') as save_location, \
patch('corehq.apps.locations.models.LocationType.save') as save_type:
result = new_locations_import(self.domain, mock_importer, self.user)
# The upload should succeed and not perform any updates
assert_errors(result, [])
self.assertFalse(save_location.called)
self.assertFalse(save_type.called)
def dump_users_and_groups(domain, download_id, user_filters, task):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
writer = Excel2007ExportWriter(format_as_text=True)
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
users_groups_count = count_users_and_groups(domain, user_filters, group_memoizer)
DownloadBase.set_progress(task, 0, users_groups_count)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
user_headers, user_rows = parse_users(
group_memoizer,
domain,
user_data_model,
location_cache,
user_filters,
task,
users_groups_count,
)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
filename = "{}_users_{}.xlsx".format(domain, uuid.uuid4().hex)
file_path = get_download_file_path(use_transfer, filename)
writer.open(
header_table=headers,
file=file_path,
)
writer.write(rows)
writer.close()
expose_download(use_transfer, file_path, filename, download_id, 'xlsx')
DownloadBase.set_progress(task, users_groups_count, users_groups_count)
def get_usercase_default_properties(domain):
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
from corehq.apps.users.views.mobile.custom_data_fields import CUSTOM_USER_DATA_FIELD_TYPE
fields_def = CustomDataFieldsDefinition.get_or_create(domain, CUSTOM_USER_DATA_FIELD_TYPE)
return [f.slug for f in fields_def.fields]
