def test_custom_data_fields(self):
for domain_name in [self.domain.name, self.domain2.name]:
CustomDataFieldsDefinition.get_or_create(domain_name, 'UserFields')
self.domain.delete()
self._assert_custom_data_fields_counts(self.domain.name, 0)
self._assert_custom_data_fields_counts(self.domain2.name, 1)
def handle(self, *args, **options):
for domain in Domain.get_all_names():
fields_definition = CustomDataFieldsDefinition.get_or_create(
domain,
'UserFields'
)
user_ids = (CommCareUser.ids_by_domain(domain) +
CommCareUser.ids_by_domain(domain, is_active=False))
existing_field_slugs = set([field.slug for field in fields_definition.fields])
for user in iter_docs(CommCareUser.get_db(), user_ids):
user_data = user.get('user_data', {})
for key in user_data.keys():
if key and key not in existing_field_slugs:
existing_field_slugs.add(key)
fields_definition.fields.append(CustomDataField(
slug=key,
label=key,
is_required=False
))
# Only save a definition for domains which use custom user data
if fields_definition.fields:
fields_definition.save()
def handle(self, *args, **options):
for domain in Domain.get_all():
if domain['commtrack_enabled']:
fields_definition = CustomDataFieldsDefinition.get_or_create(
domain['name'],
'ProductFields'
)
product_ids = Product.ids_by_domain(domain['name'])
existing_field_slugs = set(
[field.slug for field in fields_definition.fields]
)
for product in iter_docs(Product.get_db(), product_ids):
product_data = product.get('product_data', {})
for key in product_data.keys():
if key and key not in existing_field_slugs:
existing_field_slugs.add(key)
fields_definition.fields.append(CustomDataField(
slug=key,
label=key,
is_required=False
))
# Only save a definition for domains which use custom product data
if fields_definition.fields:
fields_definition.save()
def get_location_data_model(domain):
from .views import LocationFieldsView
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
return CustomDataFieldsDefinition.get_or_create(
domain,
LocationFieldsView.field_type,
)
def update_custom_data_models(domain_link, limit_types=None):
if domain_link.is_remote:
master_results = remote_custom_data_models(domain_link, limit_types)
else:
master_results = local_custom_data_models(domain_link.master_domain, limit_types)
for field_type, data in master_results.items():
field_definitions = data.get('fields', [])
model = CustomDataFieldsDefinition.get_or_create(domain_link.linked_domain, field_type)
model.set_fields([
Field(
slug=field_def['slug'],
is_required=field_def['is_required'],
label=field_def['label'],
choices=field_def['choices'],
regex=field_def['regex'],
regex_msg=field_def['regex_msg'],
) for field_def in field_definitions
])
model.save()
old_profiles = {profile.name: profile for profile in model.get_profiles()}
for profile in data.get('profiles'):
old_profile = old_profiles.get(profile['name'], None)
if old_profile:
old_profile.fields = profile['fields']
old_profile.save()
else:
CustomDataFieldsProfile(
name=profile['name'],
definition=model,
fields=profile['fields'],
).save()
def save_custom_fields(domain, definition_name, custom_fields):
if custom_fields:
fields_definitions = CustomDataFieldsDefinition.get_or_create(domain, definition_name)
need_save = False
for custom_field in custom_fields:
name = custom_field.get('name')
label = custom_field.get('label')
choices = custom_field.get('choices') or []
existing_fields = [field for field in fields_definitions.fields if field.slug == name]
if not existing_fields:
need_save = True
fields_definitions.fields.append(
CustomDataField(
slug=name,
label=label or name,
is_required=False,
choices=choices,
is_multiple_choice=custom_field.get('is_multiple_choice', False)
)
)
else:
existing_field = existing_fields[0]
if set(existing_field.choices) != set(choices):
existing_field.choices = choices
need_save = True
if need_save:
fields_definitions.save()
def setUpClass(cls):
super(TestLocationsExport, cls).setUpClass()
cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type)
cls.loc_fields.set_fields([
Field(slug=slug) for slug in cls.custom_fields
])
cls.loc_fields.save()
cls.boston = cls.locations['Boston']
cls.boston.metadata = {
field: '{}-试验'.format(field) for field in cls.custom_fields + ['不知道']
}
cls.boston.external_id = 'external_id'
cls.boston.latitude = Decimal('42.36')
cls.boston.longitude = Decimal('71.06')
cls.boston.save()
exporter = LocationExporter(cls.domain)
writer = MockExportWriter()
exporter.write_data(writer)
cls.headers = dict(exporter.get_headers())
cls.city_headers = cls.headers['city'][0]
cls.boston_data = [row for row in writer.data['city'] if row[0] == cls.boston.location_id][0]
def get_usercase_default_properties(domain):
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
from corehq.apps.users.views.mobile.custom_data_fields import CUSTOM_USER_DATA_FIELD_TYPE
fields_def = CustomDataFieldsDefinition.get_or_create(
domain, CUSTOM_USER_DATA_FIELD_TYPE)
return [f.slug for f in fields_def.fields]
def get_location_data_model(domain):
from .views import LocationFieldsView
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
return CustomDataFieldsDefinition.get_or_create(
domain,
LocationFieldsView.field_type,
)
def parse_mobile_users(domain, user_filters, task=None, total_count=None):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
fields_definition = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
unrecognized_user_data_keys = set()
user_groups_length = 0
max_location_length = 0
user_dicts = []
domains_list = [domain]
is_multi_domain_download = False
if 'domains' in user_filters:
domains_list = user_filters['domains']
if domains_list != [domain]:
is_multi_domain_download = True
current_user_downloaded_count = 0
for current_domain in domains_list:
location_cache = LocationIdToSiteCodeCache(current_domain)
for n, user in enumerate(get_commcare_users_by_filters(current_domain, user_filters)):
group_memoizer = load_memoizer(current_domain)
group_names = sorted([
group_memoizer.get(id).name for id in Group.by_user_id(user.user_id, wrap=False)
], key=alphanumeric_sort_key)
user_dict = make_mobile_user_dict(user, group_names, location_cache, current_domain, fields_definition)
user_dicts.append(user_dict)
unrecognized_user_data_keys.update(user_dict['uncategorized_data'])
user_groups_length = max(user_groups_length, len(group_names))
max_location_length = max(max_location_length, len(user_dict["location_code"]))
if task:
DownloadBase.set_progress(task, n + current_user_downloaded_count, total_count)
current_user_downloaded_count += n + 1
user_headers = [
'username', 'password', 'name', 'phone-number', 'email',
'language', 'role', 'user_id', 'is_active', 'User IMEIs (read only)',
'registered_on (read only)', 'last_submission (read only)', 'last_sync (read only)'
]
if domain_has_privilege(domain, privileges.APP_USER_PROFILES):
user_headers += ['user_profile']
user_data_fields = [f.slug for f in fields_definition.get_fields(include_system=False)]
user_headers.extend(build_data_headers(user_data_fields))
user_headers.extend(build_data_headers(
unrecognized_user_data_keys,
header_prefix='uncategorized_data'
))
user_headers.extend(json_to_headers(
{'group': list(range(1, user_groups_length + 1))}
))
if domain_has_privilege(domain, privileges.LOCATIONS):
user_headers.extend(json_to_headers(
{'location_code': list(range(1, max_location_length + 1))}
))
if is_multi_domain_download:
user_headers += ['domain']
return user_headers, get_user_rows(user_dicts, user_headers)
def dump_users_and_groups(domain, download_id, user_filters):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
writer = Excel2007ExportWriter(format_as_text=True)
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
user_headers, user_rows = parse_users(
group_memoizer,
domain,
user_data_model,
location_cache,
user_filters
)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
filename = "{}_users_{}.xlsx".format(domain, uuid.uuid4().hex)
file_path = get_download_file_path(use_transfer, filename)
writer.open(
header_table=headers,
file=file_path,
)
writer.write(rows)
writer.close()
expose_download(use_transfer, file_path, filename, download_id, 'xlsx')
def setUpClass(cls):
super(LocationFixturesDataTest, cls).setUpClass()
cls.user = create_restore_user(cls.domain, 'user', '123')
cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type)
cls.loc_fields.fields = [
CustomDataField(slug='baseball_team'),
CustomDataField(slug='favorite_passtime'),
]
cls.loc_fields.save()
cls.field_slugs = [f.slug for f in cls.loc_fields.fields]
def update_custom_data_models(domain_link, limit_types=None):
if domain_link.is_remote:
master_results = remote_custom_data_models(domain_link, limit_types)
else:
master_results = local_custom_data_models(domain_link.master_domain, limit_types)
for field_type, field_definitions in master_results.items():
model = CustomDataFieldsDefinition.get_or_create(domain_link.linked_domain, field_type)
model.fields = [CustomDataField.wrap(field_def) for field_def in field_definitions]
model.save()
def setUpClass(cls):
super(LocationFixturesDataTest, cls).setUpClass()
cls.user = create_restore_user(cls.domain, 'user', '123')
cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type)
cls.loc_fields.fields = [
CustomDataField(slug='baseball_team'),
CustomDataField(slug='favorite_passtime'),
]
cls.loc_fields.save()
cls.field_slugs = [f.slug for f in cls.loc_fields.fields]
def update_custom_data_models(domain_link, limit_types=None):
if domain_link.is_remote:
master_results = remote_custom_data_models(domain_link, limit_types)
else:
master_results = local_custom_data_models(domain_link.master_domain, limit_types)
for field_type, field_definitions in master_results.items():
model = CustomDataFieldsDefinition.get_or_create(domain_link.linked_domain, field_type)
model.fields = [CustomDataField.wrap(field_def) for field_def in field_definitions]
model.save()
def dump_users_and_groups(response, domain):
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
export_file = StringIO()
writer = Excel2007ExportWriter()
group_memoizer = _load_memoizer(domain)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
user_data_fields = [f.slug for f in user_data_model.fields]
user_headers, user_rows = parse_users(group_memoizer,
CommCareUser.by_domain(domain),
user_data_fields)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
commtrack_enabled = Domain.get_by_name(domain).commtrack_enabled
if commtrack_enabled:
headers.append(
('locations', [['username', 'location-sms-code', 'location name (optional)']])
)
rows.append(
('locations', get_location_rows(domain))
)
writer.open(
header_table=headers,
file=export_file,
)
writer.write(rows)
writer.close()
response.write(export_file.getvalue())
def handle(self, domain, **options):
self.user_data = CustomDataFieldsDefinition.get_or_create(
domain, UserFieldsView.field_type)
self.location_data = CustomDataFieldsDefinition.get_or_create(
domain, LocationFieldsView.field_type)
print("\nOLD:")
self.show(self.user_data)
self.update_definition(self.user_data, AGENCY_USER_FIELDS)
print("\nNEW:")
self.show(self.user_data)
if self.confirm():
self.user_data.save()
print("\nOLD:")
self.show(self.location_data)
self.update_definition(self.location_data, AGENCY_LOCATION_FIELDS)
print("\nNEW:")
self.show(self.location_data)
if self.confirm():
self.location_data.save()
def case_properties(self):
props = set([])
if CALLCENTER.enabled(self.domain):
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
user_fields = CustomDataFieldsDefinition.get_or_create(self.domain, 'UserFields')
props |= {field.slug for field in user_fields.fields}
for app in self.applications:
builder = ParentCasePropertyBuilder(app, ("name",))
props |= set(builder.get_properties(self.case_type))
return props
def prepare_domain(domain_name):
from corehq.apps.commtrack.tests.util import bootstrap_domain
domain = bootstrap_domain(domain_name)
previous = None
for name, administrative in [
("MOHSW", True),
("MSDZONE", True),
("REGION", True),
("DISTRICT", True),
("FACILITY", False)
]:
previous, _ = LocationType.objects.get_or_create(
domain=domain_name,
name=name,
parent_type=previous,
administrative=administrative,
)
account = BillingAccount.get_or_create_account_by_domain(
domain.name,
created_by="automated-test",
)[0]
plan = DefaultProductPlan.get_default_plan_version(
edition=SoftwarePlanEdition.ADVANCED
)
commtrack = domain.commtrack_settings
commtrack.actions.append(
CommtrackActionConfig(action='receipts',
keyword='delivered',
caption='Delivered')
)
commtrack.save()
subscription = Subscription.new_domain_subscription(
account,
domain.name,
plan
)
subscription.is_active = True
subscription.save()
ils_config = ILSGatewayConfig(enabled=True, domain=domain.name, all_stock_data=True)
ils_config.save()
fields_definition = CustomDataFieldsDefinition.get_or_create(domain.name, 'LocationFields')
fields_definition.fields.append(CustomDataField(
slug='group',
label='Group',
is_required=False,
choices=['A', 'B', 'C'],
is_multiple_choice=False
))
fields_definition.save()
return domain
def dump_users_and_groups(domain, download_id, user_filters, task):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
users_groups_count = count_users_and_groups(domain, user_filters,
group_memoizer)
DownloadBase.set_progress(task, 0, users_groups_count)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain, UserFieldsView.field_type)
user_headers, user_rows = parse_users(
group_memoizer,
domain,
user_data_model,
location_cache,
user_filters,
task,
users_groups_count,
)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
filename = "{}_users_{}.xlsx".format(domain, uuid.uuid4().hex)
_dump_xlsx_and_expose_download(filename, headers, rows, download_id, task,
users_groups_count)
def prepare_domain(domain_name):
from corehq.apps.commtrack.tests.util import bootstrap_domain
domain = bootstrap_domain(domain_name)
previous = None
for name, administrative in [
("MOHSW", True),
("MSDZONE", True),
("REGION", True),
("DISTRICT", True),
("FACILITY", False)
]:
previous, _ = LocationType.objects.get_or_create(
domain=domain_name,
name=name,
parent_type=previous,
administrative=administrative,
)
account = BillingAccount.get_or_create_account_by_domain(
domain.name,
created_by="automated-test",
)[0]
plan = DefaultProductPlan.get_default_plan_version(
edition=SoftwarePlanEdition.ADVANCED
)
commtrack = domain.commtrack_settings
commtrack.actions.append(
CommtrackActionConfig(action='receipts',
keyword='delivered',
caption='Delivered')
)
commtrack.save()
subscription = Subscription.new_domain_subscription(
account,
domain.name,
plan
)
subscription.is_active = True
subscription.save()
ils_config = ILSGatewayConfig(enabled=True, domain=domain.name, all_stock_data=True)
ils_config.save()
fields_definition = CustomDataFieldsDefinition.get_or_create(domain.name, 'LocationFields')
fields_definition.fields.append(CustomDataField(
slug='group',
label='Group',
is_required=False,
choices=['A', 'B', 'C'],
is_multiple_choice=False
))
fields_definition.save()
return domain
def setUpClass(cls):
super(TestIndexedLocationsFixture, cls).setUpClass()
cls.user = create_restore_user(cls.domain, 'user', '123')
cls.loc_fields = CustomDataFieldsDefinition.get_or_create(cls.domain, LocationFieldsView.field_type)
cls.loc_fields.fields = [
CustomDataField(slug='is_test', index_in_fixture=True),
CustomDataField(slug='favorite_color'),
]
cls.loc_fields.save()
cls.field_slugs = [f.slug for f in cls.loc_fields.fields]
for location in cls.locations.values():
location.metadata = {
'is_test': 'no',
'favorite_color': 'blue',
}
location.save()
def _parse_custom_properties(product):
product_data_model = CustomDataFieldsDefinition.get_or_create(
domain, ProductFieldsView.field_type)
product_data_fields = [f.slug for f in product_data_model.fields]
model_data = {}
uncategorized_data = {}
for prop, val in product.product_data.items():
if prop in product_data_fields:
model_data['data: ' + prop] = encode_if_needed(val)
else:
uncategorized_data['uncategorized_data: ' +
prop] = encode_if_needed(val)
return model_data, uncategorized_data
def test_download_reupload_no_changes(self):
# Make sure there's a bunch of data
loc_fields = CustomDataFieldsDefinition.get_or_create(
self.domain, 'LocationFields')
loc_fields.fields = [
CustomDataField(slug='favorite_color'),
CustomDataField(slug='language')
]
loc_fields.save()
self.locations['City111'].latitude = Decimal('42.36')
self.locations['City111'].longitude = Decimal('71.06')
self.locations['City111'].external_id = '123'
self.locations['County11'].metadata = {
'favorite_color': 'purple',
'language': 'en'
}
self.locations['City111'].save()
self.locations['County11'].external_id = '321'
self.locations['County11'].metadata = {'favorite_color': 'blue'}
self.locations['County11'].save()
# Export locations
exporter = LocationExporter(self.domain)
writer = MockExportWriter()
exporter.write_data(writer)
# Re-upload that export
worksheets = []
for sheet_title, headers in exporter.get_headers():
rows = [[val if val is not None else '' for val in row]
for row in writer.data[sheet_title]]
sheet = IteratorJSONReader(headers + rows)
sheet.title = sheet_title
worksheets.append(sheet)
mock_importer = Mock()
mock_importer.worksheets = worksheets
with patch('corehq.apps.locations.models.SQLLocation.save') as save_location, \
patch('corehq.apps.locations.models.LocationType.save') as save_type:
result = new_locations_import(self.domain, mock_importer,
self.user)
# The upload should succeed and not perform any updates
assert_errors(result, [])
self.assertFalse(save_location.called)
self.assertFalse(save_type.called)
def setUpClass(cls):
super(TestIndexedLocationsFixture, cls).setUpClass()
cls.user = create_restore_user(cls.domain, 'user', '123')
cls.loc_fields = CustomDataFieldsDefinition.get_or_create(
cls.domain, LocationFieldsView.field_type)
cls.loc_fields.fields = [
CustomDataField(slug='is_test', index_in_fixture=True),
CustomDataField(slug='favorite_color'),
]
cls.loc_fields.save()
cls.field_slugs = [f.slug for f in cls.loc_fields.fields]
for location in cls.locations.values():
location.metadata = {
'is_test': 'no',
'favorite_color': 'blue',
}
location.save()
def _parse_custom_properties(product):
product_data_model = CustomDataFieldsDefinition.get_or_create(
domain,
ProductFieldsView.field_type
)
product_data_fields = [f.slug for f in product_data_model.fields]
model_data = {}
uncategorized_data = {}
for prop, val in six.iteritems(product.product_data):
if prop in product_data_fields:
model_data['data: ' + prop] = encode_if_needed(val)
else:
uncategorized_data['uncategorized_data: ' + prop] = encode_if_needed(val)
return model_data, uncategorized_data
def test_download_reupload_no_changes(self):
# Make sure there's a bunch of data
loc_fields = CustomDataFieldsDefinition.get_or_create(
self.domain, 'LocationFields')
loc_fields.fields = [CustomDataField(slug='favorite_color'),
CustomDataField(slug='language')]
loc_fields.save()
self.locations['City111'].latitude = Decimal('42.36')
self.locations['City111'].longitude = Decimal('71.06')
self.locations['City111'].external_id = '123'
self.locations['County11'].metadata = {'favorite_color': 'purple',
'language': 'en'}
self.locations['City111'].save()
self.locations['County11'].external_id = '321'
self.locations['County11'].metadata = {'favorite_color': 'blue'}
self.locations['County11'].save()
# Export locations
exporter = LocationExporter(self.domain)
writer = MockExportWriter()
exporter.write_data(writer)
# Re-upload that export
worksheets = []
for sheet_title, headers in exporter.get_headers():
rows = [[val if val is not None else '' for val in row]
for row in writer.data[sheet_title]]
sheet = IteratorJSONReader(headers + rows)
sheet.title = sheet_title
worksheets.append(sheet)
mock_importer = Mock()
mock_importer.worksheets = worksheets
with patch('corehq.apps.locations.models.SQLLocation.save') as save_location, \
patch('corehq.apps.locations.models.LocationType.save') as save_type:
result = new_locations_import(self.domain, mock_importer, self.user)
# The upload should succeed and not perform any updates
assert_errors(result, [])
self.assertFalse(save_location.called)
self.assertFalse(save_type.called)
def update_custom_data_models(domain_link, limit_types=None):
if domain_link.is_remote:
master_results = remote_custom_data_models(domain_link, limit_types)
else:
master_results = local_custom_data_models(domain_link.master_domain,
limit_types)
for field_type, field_definitions in master_results.items():
model = CustomDataFieldsDefinition.get_or_create(
domain_link.linked_domain, field_type)
model.set_fields([
Field(
slug=field_def['slug'],
is_required=field_def['is_required'],
label=field_def['label'],
choices=field_def['choices'],
regex=field_def['regex'],
regex_msg=field_def['regex_msg'],
) for field_def in field_definitions
])
model.save()
def dump_users_and_groups(domain, download_id, user_filters, task):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
writer = Excel2007ExportWriter(format_as_text=True)
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
users_groups_count = count_users_and_groups(domain, user_filters, group_memoizer)
DownloadBase.set_progress(task, 0, users_groups_count)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
user_headers, user_rows = parse_users(
group_memoizer,
domain,
user_data_model,
location_cache,
user_filters,
task,
users_groups_count,
)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
filename = "{}_users_{}.xlsx".format(domain, uuid.uuid4().hex)
file_path = get_download_file_path(use_transfer, filename)
writer.open(
header_table=headers,
file=file_path,
)
writer.write(rows)
writer.close()
expose_download(use_transfer, file_path, filename, download_id, 'xlsx')
DownloadBase.set_progress(task, users_groups_count, users_groups_count)
def get_data_fields(self):
data_model = CustomDataFieldsDefinition.get_or_create(
self.domain,
UserFieldsView.field_type,
)
return sorted(data_model.fields)
def parse_users(group_memoizer, domain, user_filters, task=None, total_count=None):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
fields_definition = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
location_cache = LocationIdToSiteCodeCache(domain)
def _get_group_names(user):
return sorted([
group_memoizer.get(id).name for id in Group.by_user_id(user.user_id, wrap=False)
], key=alphanumeric_sort_key)
def _get_devices(user):
"""
Returns a comma-separated list of IMEI numbers of the user's devices, sorted with most-recently-used first
"""
return ', '.join([device.device_id for device in sorted(
user.devices, key=lambda d: d.last_used, reverse=True
)])
def _make_user_dict(user, group_names, location_cache):
model_data, uncategorized_data = (
fields_definition.get_model_and_uncategorized(user.metadata)
)
role = user.get_role(domain)
profile = None
if PROFILE_SLUG in user.metadata and domain_has_privilege(domain, privileges.APP_USER_PROFILES):
try:
profile = CustomDataFieldsProfile.objects.get(id=user.metadata[PROFILE_SLUG])
except CustomDataFieldsProfile.DoesNotExist:
profile = None
activity = user.reporting_metadata
location_codes = []
try:
location_codes.append(location_cache.get(user.location_id))
except SQLLocation.DoesNotExist:
pass
for location_id in user.assigned_location_ids:
# skip if primary location_id, as it is already added to the start of list above
if location_id != user.location_id:
try:
location_codes.append(location_cache.get(location_id))
except SQLLocation.DoesNotExist:
pass
def _format_date(date):
return date.strftime('%Y-%m-%d %H:%M:%S') if date else ''
return {
'data': model_data,
'uncategorized_data': uncategorized_data,
'group': group_names,
'name': user.full_name,
'password': "******", # dummy display string for passwords
'phone-number': user.phone_number,
'email': user.email,
'username': user.raw_username,
'language': user.language,
'user_id': user._id,
'is_active': str(user.is_active),
'User IMEIs (read only)': _get_devices(user),
'location_code': location_codes,
'role': role.name if role else '',
'user_profile': profile.name if profile else '',
'registered_on (read only)': _format_date(user.created_on),
'last_submission (read only)': _format_date(activity.last_submission_for_user.submission_date),
'last_sync (read only)': activity.last_sync_for_user.sync_date,
}
unrecognized_user_data_keys = set()
user_groups_length = 0
max_location_length = 0
user_dicts = []
for n, user in enumerate(get_commcare_users_by_filters(domain, user_filters)):
group_names = _get_group_names(user)
user_dict = _make_user_dict(user, group_names, location_cache)
user_dicts.append(user_dict)
unrecognized_user_data_keys.update(user_dict['uncategorized_data'])
user_groups_length = max(user_groups_length, len(group_names))
max_location_length = max(max_location_length, len(user_dict["location_code"]))
if task:
DownloadBase.set_progress(task, n, total_count)
user_headers = [
'username', 'password', 'name', 'phone-number', 'email',
'language', 'role', 'user_id', 'is_active', 'User IMEIs (read only)',
'registered_on (read only)', 'last_submission (read only)', 'last_sync (read only)'
]
if domain_has_privilege(domain, privileges.APP_USER_PROFILES):
user_headers += ['user_profile']
user_data_fields = [f.slug for f in fields_definition.get_fields(include_system=False)]
user_headers.extend(build_data_headers(user_data_fields))
user_headers.extend(build_data_headers(
unrecognized_user_data_keys,
header_prefix='uncategorized_data'
))
user_headers.extend(json_to_headers(
{'group': list(range(1, user_groups_length + 1))}
))
if domain_has_privilege(domain, privileges.LOCATIONS):
user_headers.extend(json_to_headers(
{'location_code': list(range(1, max_location_length + 1))}
))
def _user_rows():
for user_dict in user_dicts:
row = dict(flatten_json(user_dict))
yield [row.get(header) or '' for header in user_headers]
return user_headers, _user_rows()
def get_usercase_default_properties(domain):
from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
from corehq.apps.users.views.mobile.custom_data_fields import CUSTOM_USER_DATA_FIELD_TYPE
fields_def = CustomDataFieldsDefinition.get_or_create(domain, CUSTOM_USER_DATA_FIELD_TYPE)
return [f.slug for f in fields_def.fields]
