def dump_locations(domain, download_id, include_consumption, headers_only, task=None):
exporter = LocationExporter(domain, include_consumption=include_consumption,
headers_only=headers_only, async_task=task)
fd, path = tempfile.mkstemp()
writer = Excel2007ExportWriter()
writer.open(header_table=exporter.get_headers(), file=path)
with writer:
exporter.write_data(writer)
with open(path, 'rb') as file_:
db = get_blob_db()
expiry_mins = 60
db.put(
file_,
domain=domain,
parent_id=domain,
type_code=CODES.tempfile,
key=download_id,
timeout=expiry_mins,
)
file_format = Format.from_format(Excel2007ExportWriter.format)
expose_blob_download(
download_id,
expiry=expiry_mins * 60,
mimetype=file_format.mimetype,
content_disposition=safe_filename_header('{}_locations'.format(domain), file_format.extension),
download_id=download_id,
)
def _export_data_dictionary(domain):
queryset = CaseType.objects.filter(domain=domain).prefetch_related(
Prefetch('properties', queryset=CaseProperty.objects.order_by('name')))
export_data = {}
for case_type in queryset:
export_data[case_type.name or _("No Name")] = [{
_('Case Property'):
prop.name,
_('Group'):
prop.group,
_('Data Type'):
prop.data_type,
_('Description'):
prop.description,
_('Deprecated'):
prop.deprecated
} for prop in case_type.properties.all()]
headers = (_('Case Property'), _('Group'), _('Data Type'),
_('Description'), _('Deprecated'))
outfile = io.BytesIO()
writer = Excel2007ExportWriter()
header_table = [(tab_name, [headers]) for tab_name in export_data]
writer.open(header_table=header_table, file=outfile)
for tab_name, tab in export_data.items():
tab_rows = []
for row in tab:
tab_rows.append([row.get(header, '') for header in headers])
writer.write([(tab_name, tab_rows)])
writer.close()
return outfile
def write_to_file(locations):
"""
locations = [
('loc_type1', {
'headers': ['header1', 'header2', ...]
'rows': [
{
'header1': val1
'header2': val2
},
{...},
]
})
]
"""
outfile = StringIO()
writer = Excel2007ExportWriter()
header_table = [(tab_name, [tab['headers']])
for tab_name, tab in locations]
writer.open(header_table=header_table, file=outfile)
for tab_name, tab in locations:
headers = tab['headers']
tab_rows = [[row.get(header, '') for header in headers]
for row in tab['rows']]
writer.write([(tab_name, tab_rows)])
writer.close()
return outfile.getvalue()
def _export_data_dictionary(domain):
export_fhir_data = toggles.FHIR_INTEGRATION.enabled(domain)
case_type_headers = [
_('Case Type'),
_('FHIR Resource Type'),
_('Remove Resource Type(Y)')
]
case_prop_headers = [
_('Case Property'),
_('Group'),
_('Data Type'),
_('Description'),
_('Deprecated')
]
case_type_data, case_prop_data = _generate_data_for_export(
domain, export_fhir_data)
outfile = io.BytesIO()
writer = Excel2007ExportWriter()
header_table = _get_headers_for_export(export_fhir_data, case_type_headers,
case_prop_headers, case_prop_data)
writer.open(header_table=header_table, file=outfile)
if export_fhir_data:
_export_fhir_data(writer, case_type_headers, case_type_data)
_export_case_prop_data(writer, case_prop_headers, case_prop_data)
writer.close()
return outfile
def dump_users_and_groups(response, domain):
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
export_file = StringIO()
writer = Excel2007ExportWriter()
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
user_headers, user_rows = parse_users(
group_memoizer,
domain,
user_data_model,
location_cache
)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
domain_obj = Domain.get_by_name(domain)
# This is only for domains using the multiple locations feature flag
if domain_obj.commtrack_enabled and domain_obj.supports_multiple_locations_per_user:
headers.append(
('locations', [['username', 'location-sms-code', 'location name (optional)']])
)
rows.append(
('locations', get_location_rows(domain))
)
writer.open(
header_table=headers,
file=export_file,
)
writer.write(rows)
writer.close()
response.write(export_file.getvalue())
def dump_locations(response, domain, include_consumption=False):
file = StringIO()
writer = Excel2007ExportWriter()
location_types = defined_location_types(domain)
if include_consumption:
defaults = get_default_column_data(domain, location_types)
else:
defaults = {
'headers': {},
'values': {}
}
common_types = ['site_code', 'name', 'parent_site_code', 'latitude', 'longitude']
writer.open(
header_table=[
(loc_type, [
common_types +
get_custom_property_names(domain, loc_type, common_types) +
defaults['headers'].get(loc_type, [])
])
for loc_type in location_types
],
file=file,
)
for loc_type in location_types:
tab_rows = []
locations = Location.filter_by_type(domain, loc_type)
for loc in locations:
parent_site_code = loc.parent.site_code if loc.parent else ''
custom_prop_values = []
for prop in location_custom_properties(domain, loc.location_type):
if prop.name not in common_types:
custom_prop_values.append(
loc[prop.name] or ''
)
if loc._id in defaults['values']:
default_column_values = defaults['values'][loc._id]
else:
default_column_values = []
tab_rows.append(
[
loc.site_code,
loc.name,
parent_site_code,
loc.latitude or '',
loc.longitude or ''
] + custom_prop_values + default_column_values
)
writer.write([(loc_type, tab_rows)])
writer.close()
response.write(file.getvalue())
def download_products(request, domain):
def _get_products(domain):
for p_doc in iter_docs(Product.get_db(), Product.ids_by_domain(domain)):
# filter out archived products from export
if not ('is_archived' in p_doc and p_doc['is_archived']):
yield Product.wrap(p_doc)
def _build_row(keys, product):
row = []
for key in keys:
row.append(product.get(key, '') or '')
return row
file = StringIO()
writer = Excel2007ExportWriter()
product_keys = [
'id',
'name',
'unit',
'product_id',
'description',
'category',
'program_id',
'cost',
]
data_keys = set()
products = []
for product in _get_products(domain):
product_dict = product.to_dict()
custom_properties = product.custom_property_dict()
data_keys.update(custom_properties.keys())
product_dict.update(custom_properties)
products.append(product_dict)
keys = product_keys + list(data_keys)
writer.open(
header_table=[
('products', [keys])
],
file=file,
)
for product in products:
writer.write([('products', [_build_row(keys, product)])])
writer.close()
response = HttpResponse(mimetype=Format.from_format('xlsx').mimetype)
response['Content-Disposition'] = 'attachment; filename="products.xlsx"'
response.write(file.getvalue())
return response
def dump_users_and_groups(domain, download_id, user_filters):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
writer = Excel2007ExportWriter()
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain,
UserFieldsView.field_type
)
user_headers, user_rows = parse_users(
group_memoizer,
domain,
user_data_model,
location_cache,
user_filters
)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
filename = "user_export_{}_{}.xlsx".format(domain, uuid.uuid4().hex)
file_path = get_download_file_path(use_transfer, filename)
writer.open(
header_table=headers,
file=file_path,
)
writer.write(rows)
writer.close()
expose_download(use_transfer, file_path, filename, download_id, 'xlsx')
def formdefs(request, domain, app_id):
# TODO: Looks like this function is never used
langs = [json.loads(request.GET.get('lang', '"en"'))]
format = request.GET.get('format', 'json')
app = get_app(domain, app_id)
def get_questions(form):
xform = XForm(form.source)
prefix = '/%s/' % xform.data_node.tag_name
def remove_prefix(string):
if string.startswith(prefix):
return string[len(prefix):]
else:
raise Exception()
def transform_question(q):
return {
'id': remove_prefix(q['value']),
'type': q['tag'],
'text': q['label'] if q['tag'] != 'hidden' else ''
}
return [transform_question(q) for q in xform.get_questions(langs)]
formdefs = [{
'name':
"%s, %s" % (f['form'].get_module().name['en'], f['form'].name['en'])
if f['type'] == 'module_form' else 'User Registration',
'columns': ['id', 'type', 'text'],
'rows':
get_questions(f['form'])
} for f in app.get_forms(bare=False)]
if format == 'xlsx':
f = StringIO()
writer = Excel2007ExportWriter()
writer.open([(sheet['name'], [FormattedRow(sheet['columns'])])
for sheet in formdefs], f)
writer.write([(sheet['name'], [
FormattedRow([
cell for (_, cell) in sorted(
row.items(),
key=lambda item: sheet['columns'].index(item[0]))
]) for row in sheet['rows']
]) for sheet in formdefs])
writer.close()
response = HttpResponse(
f.getvalue(), content_type=Format.from_format('xlsx').mimetype)
set_file_download(response, 'formdefs.xlsx')
return response
else:
return json_response(formdefs)
def _dump_xlsx_and_expose_download(filename, headers, rows, download_id, task, total_count, owner_id):
writer = Excel2007ExportWriter(format_as_text=True)
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
file_path = get_download_file_path(use_transfer, filename)
writer.open(
header_table=headers,
file=file_path,
)
writer.write(rows)
writer.close()
expose_download(use_transfer, file_path, filename, download_id, 'xlsx', owner_ids=[owner_id])
DownloadBase.set_progress(task, total_count, total_count)
def dump_locations(domain,
download_id,
include_consumption,
headers_only,
owner_id,
root_location_ids=None,
task=None,
**kwargs):
exporter = LocationExporter(domain,
include_consumption=include_consumption,
root_location_ids=root_location_ids,
headers_only=headers_only,
async_task=task,
**kwargs)
fd, path = tempfile.mkstemp()
writer = Excel2007ExportWriter()
writer.open(header_table=exporter.get_headers(), file=path)
with writer:
exporter.write_data(writer)
with open(path, 'rb') as file_:
db = get_blob_db()
expiry_mins = 60
db.put(
file_,
domain=domain,
parent_id=domain,
type_code=CODES.tempfile,
key=download_id,
timeout=expiry_mins,
)
file_format = Format.from_format(Excel2007ExportWriter.format)
filename = '{}_locations'.format(domain)
if len(root_location_ids) == 1:
root_location = SQLLocation.objects.get(
location_id=root_location_ids[0])
filename += '_{}'.format(root_location.name)
expose_blob_download(
download_id,
expiry=expiry_mins * 60,
mimetype=file_format.mimetype,
content_disposition=safe_filename_header(filename,
file_format.extension),
download_id=download_id,
owner_ids=[owner_id],
)
def dump_locations(response, domain):
file = StringIO()
writer = Excel2007ExportWriter()
location_types = defined_location_types(domain)
defaults = get_default_column_data(domain, location_types)
common_types = ['id', 'name', 'parent_id', 'latitude', 'longitude']
writer.open(
header_table=[
(loc_type, [
common_types +
get_custom_property_names(domain, loc_type) +
defaults['headers'].get(loc_type, [])
])
for loc_type in location_types
],
file=file,
)
for loc_type in location_types:
tab_rows = []
locations = Location.filter_by_type(domain, loc_type)
for loc in locations:
parent_id = loc.parent._id if loc.parent else ''
custom_prop_values = [loc[prop.name] or '' for prop in location_custom_properties(domain, loc.location_type)]
if loc._id in defaults['values']:
default_column_values = defaults['values'][loc._id]
else:
default_column_values = []
tab_rows.append(
[
loc._id,
loc.name,
parent_id,
loc.latitude or '',
loc.longitude or ''
] + custom_prop_values + default_column_values
)
writer.write([(loc_type, tab_rows)])
writer.close()
response.write(file.getvalue())
def dump_users_and_groups(response, domain):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
export_file = StringIO()
writer = Excel2007ExportWriter()
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain, UserFieldsView.field_type)
user_headers, user_rows = parse_users(group_memoizer, domain,
user_data_model, location_cache)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
writer.open(
header_table=headers,
file=export_file,
)
writer.write(rows)
writer.close()
response.write(export_file.getvalue())
def download_products(request, domain):
def _parse_custom_properties(product):
product_data_model = CustomDataFieldsDefinition.get_or_create(
domain, ProductFieldsView.field_type)
product_data_fields = [f.slug for f in product_data_model.fields]
model_data = {}
uncategorized_data = {}
for prop, val in six.iteritems(product.product_data):
if prop in product_data_fields:
model_data['data: ' + prop] = encode_if_needed(val)
else:
uncategorized_data['uncategorized_data: ' +
prop] = encode_if_needed(val)
return model_data, uncategorized_data
def _get_products(domain):
for p_doc in iter_docs(Product.get_db(),
Product.ids_by_domain(domain)):
# filter out archived products from export
if not ('is_archived' in p_doc and p_doc['is_archived']):
yield Product.wrap(p_doc)
def _build_row(keys, product):
row = []
for key in keys:
row.append(product.get(key, '') or '')
return row
file = BytesIO()
writer = Excel2007ExportWriter()
product_keys = [
'id',
'name',
'unit',
'product_id',
'description',
'category',
'program_id',
'cost',
]
model_data = set()
uncategorized_data = set()
products = []
for product in _get_products(domain):
product_dict = product.to_dict()
product_model, product_uncategorized = _parse_custom_properties(
product)
model_data.update(product_model)
uncategorized_data.update(product_uncategorized)
product_dict.update(product_model)
product_dict.update(product_uncategorized)
products.append(product_dict)
keys = product_keys + list(model_data) + list(uncategorized_data)
writer.open(
header_table=[('products', [keys])],
file=file,
)
for product in products:
writer.write([('products', [_build_row(keys, product)])])
writer.close()
response = HttpResponse(content_type=Format.from_format('xlsx').mimetype)
response['Content-Disposition'] = 'attachment; filename="products.xlsx"'
response.write(file.getvalue())
return response
def dump_users_and_groups(response, domain):
file = StringIO()
writer = Excel2007ExportWriter()
users = CommCareUser.by_domain(domain)
user_data_keys = set()
user_groups_length = 0
user_dicts = []
group_data_keys = set()
group_dicts = []
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
for user in users:
data = user.user_data
group_names = sorted(map(lambda id: group_memoizer.get(id).name,
Group.by_user(user, wrap=False)),
key=alphanumeric_sort_key)
# exclude password and user_id
user_dicts.append({
'data': data,
'group': group_names,
'name': user.full_name,
# dummy display string for passwords
'password': "******",
'phone-number': user.phone_number,
'email': user.email,
'username': user.raw_username,
'language': user.language,
'user_id': user._id,
})
user_data_keys.update(user.user_data.keys() if user.user_data else {})
user_groups_length = max(user_groups_length, len(group_names))
sorted_groups = sorted(group_memoizer.groups,
key=lambda group: alphanumeric_sort_key(group.name))
for group in sorted_groups:
group_dicts.append({
'id': group.get_id,
'name': group.name,
'case-sharing': group.case_sharing,
'reporting': group.reporting,
'data': group.metadata,
})
group_data_keys.update(group.metadata.keys() if group.metadata else {})
# include obscured password column for adding new users
user_headers = [
'username', 'password', 'name', 'phone-number', 'email', 'language',
'user_id'
]
user_headers.extend(
json_to_headers(
{'data': dict([(key, None) for key in user_data_keys])}))
user_headers.extend(
json_to_headers({'group': range(1, user_groups_length + 1)}))
group_headers = ['id', 'name', 'case-sharing?', 'reporting?']
group_headers.extend(
json_to_headers(
{'data': dict([(key, None) for key in group_data_keys])}))
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
commtrack_enabled = Domain.get_by_name(domain).commtrack_enabled
if commtrack_enabled:
headers.append(
('locations',
[['username', 'location-sms-code', 'location name (optional)']]))
writer.open(
header_table=headers,
file=file,
)
def get_user_rows():
for user_dict in user_dicts:
row = dict(flatten_json(user_dict))
yield [row.get(header) or '' for header in user_headers]
def get_group_rows():
for group_dict in group_dicts:
row = dict(flatten_json(group_dict))
yield [row.get(header) or '' for header in group_headers]
rows = [
('users', get_user_rows()),
('groups', get_group_rows()),
]
if commtrack_enabled:
rows.append(('locations', get_location_rows(domain)))
writer.write(rows)
writer.close()
response.write(file.getvalue())
def dump_users_and_groups(domain, download_id):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
writer = Excel2007ExportWriter()
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain, UserFieldsView.field_type)
user_headers, user_rows = parse_users(group_memoizer, domain,
user_data_model, location_cache)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
file_path = _get_download_file_path(domain, use_transfer)
writer.open(
header_table=headers,
file=file_path,
)
writer.write(rows)
writer.close()
common_kwargs = dict(
mimetype=Format.from_format('xlsx').mimetype,
content_disposition='attachment; filename="{fname}"'.format(
fname='{}_users.xlsx'.format(domain)),
download_id=download_id,
)
if use_transfer:
expose_file_download(file_path,
use_transfer=use_transfer,
**common_kwargs)
else:
expose_cached_download(FileWrapper(open(file_path, 'r')),
expiry=(1 * 60 * 60),
file_extension='xlsx',
**common_kwargs)