def _get_all_case_properties(domain):
# moved here to avoid circular import
from corehq.apps.export.models.new import CaseExportDataSchema
case_type_to_properties = {}
case_properties_from_apps = all_case_properties_by_domain(
domain, include_parent_properties=False
)
for case_type in get_case_types_from_apps(domain):
properties = set()
schema = CaseExportDataSchema.generate_schema_from_builds(domain, None, case_type)
# only the first schema contains case properties. The others contain meta info
group_schema = schema.group_schemas[0]
for item in group_schema.items:
if len(item.path) > 1:
continue
if item.tag:
name = item.tag
else:
name = item.path[-1].name
if '/' not in name:
# Filter out index and parent properties as some are stored as parent/prop in item.path
properties.add(name)
case_type_props_from_app = case_properties_from_apps.get(case_type, {})
properties |= set(case_type_props_from_app)
case_type_to_properties[case_type] = properties
return case_type_to_properties
def get_all_case_properties(domain, case_types=None):
# moved here to avoid circular import
from corehq.apps.export.models.new import CaseExportDataSchema
case_type_to_properties = {}
case_properties_from_apps = all_case_properties_by_domain(
domain, case_types=case_types, include_parent_properties=False)
if case_types is None:
case_types = get_case_types_from_apps(domain)
for case_type in case_types:
properties = set()
schema = CaseExportDataSchema.generate_schema_from_builds(
domain, None, case_type)
for group_schema in schema.group_schemas:
for item in group_schema.items:
if item.tag:
name = item.tag
else:
name = '/'.join([p.name for p in item.path])
properties.add(name)
case_type_props_from_app = case_properties_from_apps.get(case_type, {})
properties |= set(case_type_props_from_app)
case_type_to_properties[case_type] = properties
return case_type_to_properties
def _get_all_case_properties(domain):
# moved here to avoid circular import
from corehq.apps.export.models.new import CaseExportDataSchema
case_type_to_properties = {}
case_properties_from_apps = all_case_properties_by_domain(
domain, include_parent_properties=False)
for case_type in get_case_types_from_apps(domain):
properties = set()
schema = CaseExportDataSchema.generate_schema_from_builds(
domain, None, case_type)
# only the first schema contains case properties. The others contain meta info
group_schema = schema.group_schemas[0]
for item in group_schema.items:
if len(item.path) > 1:
continue
if item.tag:
name = item.tag
else:
name = item.path[-1].name
if '/' not in name:
# Filter out index and parent properties as some are stored as parent/prop in item.path
properties.add(name)
case_type_props_from_app = case_properties_from_apps.get(case_type, {})
properties |= set(case_type_props_from_app)
case_type_to_properties[case_type] = properties
return case_type_to_properties
def _get_all_case_properties(domain):
case_type_to_properties = {}
case_properties_from_apps = all_case_properties_by_domain(
domain, include_parent_properties=False)
for case_type in get_case_types_from_apps(domain):
properties = set()
schema = CaseExportDataSchema.generate_schema_from_builds(
domain, None, case_type)
for group_schema in schema.group_schemas:
for item in group_schema.items:
if item.tag:
name = item.tag
else:
name = '/'.join([p.name for p in item.path])
properties.add(name)
case_type_props_from_app = case_properties_from_apps.get(case_type, {})
properties |= set(case_type_props_from_app)
case_type_to_properties[case_type] = properties
return case_type_to_properties
def _get_all_case_properties(domain):
case_type_to_properties = {}
case_properties_from_apps = all_case_properties_by_domain(
domain, include_parent_properties=False
)
for case_type in get_case_types_from_apps(domain):
properties = set()
schema = CaseExportDataSchema.generate_schema_from_builds(domain, None, case_type)
for group_schema in schema.group_schemas:
for item in group_schema.items:
if item.tag:
name = item.tag
else:
name = '/'.join([p.name for p in item.path])
properties.add(name)
case_type_props_from_app = case_properties_from_apps.get(case_type, {})
properties |= set(case_type_props_from_app)
case_type_to_properties[case_type] = properties
return case_type_to_properties
def _process_file_and_get_upload(uploaded_file_handle,
request,
domain,
max_columns=None):
extension = os.path.splitext(
uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in valid_extensions:
raise SpreadsheetFileExtError(
'The file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel file.')
# stash content in the default storage for subsequent views
case_upload = CaseUpload.create(uploaded_file_handle,
filename=uploaded_file_handle.name,
domain=domain)
request.session[EXCEL_SESSION_ID] = case_upload.upload_id
case_upload.check_file()
invalid_column_names = set()
with case_upload.get_spreadsheet() as spreadsheet:
columns = spreadsheet.get_header_columns()
validate_column_names(columns, invalid_column_names)
row_count = spreadsheet.max_row
if invalid_column_names:
error_message = format_html(
_("Column names must be <a target='_blank' href='https://www.w3schools.com/xml/xml_elements.asp'>"
"valid XML elements</a> and cannot start with a number or contain spaces or most special characters."
" Please update the following: {}.").format(
', '.join(invalid_column_names)))
raise ImporterRawError(error_message)
if row_count == 0:
raise ImporterError(
'Your spreadsheet is empty. Please try again with a different spreadsheet.'
)
if max_columns is not None and len(columns) > max_columns:
raise ImporterError('Your spreadsheet has too many columns. '
'A maximum of %(max_columns)s is supported.' %
{'max_columns': MAX_CASE_IMPORTER_COLUMNS})
case_types_from_apps = sorted(get_case_types_from_apps(domain))
unrecognized_case_types = sorted([
t for t in get_case_types_for_domain_es(domain)
if t not in case_types_from_apps
])
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
raise ImporterError(
'No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.')
context = {
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'slug': base.ImportCases.slug,
}
return case_upload, context
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the excel file
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain, 'Please choose an Excel file to import.')
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ALLOWED_EXTENSIONS:
return render_error(request, domain,
'The file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel file.')
# stash content in the default storage for subsequent views
case_upload = CaseUpload.create(uploaded_file_handle,
filename=uploaded_file_handle.name)
request.session[EXCEL_SESSION_ID] = case_upload.upload_id
try:
case_upload.check_file()
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
with case_upload.get_spreadsheet() as spreadsheet:
columns = spreadsheet.get_header_columns()
row_count = spreadsheet.max_row
if row_count == 0:
return render_error(request, domain,
'Your spreadsheet is empty. '
'Please try again with a different spreadsheet.')
case_types_from_apps = get_case_types_from_apps(domain)
unrecognized_case_types = [t for t in CaseAccessors(domain).get_case_types()
if t not in case_types_from_apps]
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
return render_error(
request,
domain,
'No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'
)
return render(
request,
"case_importer/excel_config.html", {
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'report': {
'name': 'Import: Configuration'
},
'slug': base.ImportCases.slug
}
)
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the Excel file
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain,
'Please choose an Excel file to import.')
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(
uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ALLOWED_EXTENSIONS:
return render_error(
request, domain,
_('The file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel file.'))
# stash content in the default storage for subsequent views
case_upload = CaseUpload.create(uploaded_file_handle,
filename=uploaded_file_handle.name,
domain=domain)
request.session[EXCEL_SESSION_ID] = case_upload.upload_id
try:
case_upload.check_file()
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
except SpreadsheetFileExtError:
return render_error(
request, domain,
_("Please upload file with extension .xls or .xlsx"))
invalid_column_names = set()
with case_upload.get_spreadsheet() as spreadsheet:
columns = spreadsheet.get_header_columns()
validate_column_names(columns, invalid_column_names)
row_count = spreadsheet.max_row
if invalid_column_names:
error_message = format_html(
_("Column names must be <a target='_blank' href='https://www.w3schools.com/xml/xml_elements.asp'>"
"valid XML elements</a> and cannot start with a number or contain spaces or most special characters."
" Please update the following: {}.").format(
', '.join(invalid_column_names)))
return render_error(request, domain, error_message)
if row_count == 0:
return render_error(
request, domain,
_('Your spreadsheet is empty. Please try again with a different spreadsheet.'
))
if len(columns) > MAX_CASE_IMPORTER_COLUMNS:
return render_error(
request, domain,
_('Your spreadsheet has too many columns. '
'A maximum of %(max_columns)s is supported.') %
{'max_columns': MAX_CASE_IMPORTER_COLUMNS})
case_types_from_apps = sorted(get_case_types_from_apps(domain))
unrecognized_case_types = sorted([
t for t in get_case_types_for_domain_es(domain)
if t not in case_types_from_apps
])
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
return render_error(
request, domain,
_('No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'))
context = {
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'slug': base.ImportCases.slug,
}
context.update(_case_importer_breadcrumb_context(_('Case Options'),
domain))
request.use_select2_v4 = True
return render(request, "case_importer/excel_config.html", context)
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the excel file
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain,
'Please choose an Excel file to import.')
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(
uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ALLOWED_EXTENSIONS:
return render_error(
request, domain,
_('The file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel file.'))
# stash content in the default storage for subsequent views
case_upload = CaseUpload.create(uploaded_file_handle,
filename=uploaded_file_handle.name)
request.session[EXCEL_SESSION_ID] = case_upload.upload_id
try:
case_upload.check_file()
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
except SpreadsheetFileExtError:
return render_error(
request, domain,
_("Please upload file with extension .xls or .xlsx"))
with case_upload.get_spreadsheet() as spreadsheet:
columns = spreadsheet.get_header_columns()
row_count = spreadsheet.max_row
if row_count == 0:
return render_error(
request, domain,
_('Your spreadsheet is empty. Please try again with a different spreadsheet.'
))
if len(columns) > MAX_CASE_IMPORTER_COLUMNS:
return render_error(
request, domain,
_('Your spreadsheet has too many columns. '
'A maximum of %(max_columns)s is supported.') %
{'max_columns': MAX_CASE_IMPORTER_COLUMNS})
case_types_from_apps = get_case_types_from_apps(domain)
unrecognized_case_types = [
t for t in get_case_types_for_domain_es(domain)
if t not in case_types_from_apps
]
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
return render_error(
request, domain,
_('No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'))
return render(
request, "case_importer/excel_config.html", {
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'report': {
'name': 'Import: Configuration'
},
'slug': base.ImportCases.slug
})
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the excel file
named_columns:
Whether or not the first row of the excel sheet contains
header strings for the columns. This defaults to True and
should potentially not be an option as it is always used
due to how important it is to see column headers
in the rest of the importer.
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain, 'Please choose an Excel file to import.')
named_columns = request.POST.get('named_columns') == "on"
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ExcelFile.ALLOWED_EXTENSIONS:
return render_error(request, domain,
'The Excel file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel 97/2000 .xls file.')
# stash content in the default storage for subsequent views
file_ref = expose_cached_download(
uploaded_file_handle.read(),
expiry=1*60*60,
file_extension=file_extention_from_filename(uploaded_file_handle.name),
)
request.session[EXCEL_SESSION_ID] = file_ref.download_id
try:
spreadsheet = importer_util.get_spreadsheet(file_ref, named_columns)
except ImporterError as e:
return render_error(request, domain, _get_importer_error_message(e))
columns = spreadsheet.get_header_columns()
row_count = spreadsheet.get_num_rows()
if row_count == 0:
return render_error(request, domain,
'Your spreadsheet is empty. '
'Please try again with a different spreadsheet.')
case_types_from_apps = get_case_types_from_apps(domain)
unrecognized_case_types = [t for t in get_case_types_for_domain(domain)
if t not in case_types_from_apps]
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
return render_error(
request,
domain,
'No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'
)
return render(
request,
"importer/excel_config.html", {
'named_columns': named_columns,
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'report': {
'name': 'Import: Configuration'
},
'slug': base.ImportCases.slug
}
)
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the excel file
named_columns:
Whether or not the first row of the excel sheet contains
header strings for the columns. This defaults to True and
should potentially not be an option as it is always used
due to how important it is to see column headers
in the rest of the importer.
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain, 'Please choose an Excel file to import.')
named_columns = request.POST.get('named_columns') == "on"
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ExcelFile.ALLOWED_EXTENSIONS:
return render_error(request, domain,
'The Excel file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel 97/2000 .xls file.')
# stash content in the default storage for subsequent views
file_ref = expose_cached_download(
uploaded_file_handle.read(),
expiry=1*60*60,
file_extension=file_extention_from_filename(uploaded_file_handle.name),
)
request.session[EXCEL_SESSION_ID] = file_ref.download_id
try:
spreadsheet = importer_util.get_spreadsheet(file_ref, named_columns)
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
columns = spreadsheet.get_header_columns()
row_count = spreadsheet.get_num_rows()
if row_count == 0:
return render_error(request, domain,
'Your spreadsheet is empty. '
'Please try again with a different spreadsheet.')
case_types_from_apps = get_case_types_from_apps(domain)
unrecognized_case_types = [t for t in CaseAccessors(domain).get_case_types()
if t not in case_types_from_apps]
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
return render_error(
request,
domain,
'No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'
)
return render(
request,
"importer/excel_config.html", {
'named_columns': named_columns,
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'report': {
'name': 'Import: Configuration'
},
'slug': base.ImportCases.slug
}
)
def excel_config(request, domain):
"""
Step one of three.
This is the initial post when the user uploads the Excel file
"""
if request.method != 'POST':
return HttpResponseRedirect(base.ImportCases.get_url(domain=domain))
if not request.FILES:
return render_error(request, domain, 'Please choose an Excel file to import.')
uploaded_file_handle = request.FILES['file']
extension = os.path.splitext(uploaded_file_handle.name)[1][1:].strip().lower()
# NOTE: We may not always be able to reference files from subsequent
# views if your worker changes, so we have to store it elsewhere
# using the soil framework.
if extension not in importer_util.ALLOWED_EXTENSIONS:
return render_error(request, domain, _(
'The file you chose could not be processed. '
'Please check that it is saved as a Microsoft '
'Excel file.'
))
# stash content in the default storage for subsequent views
case_upload = CaseUpload.create(uploaded_file_handle,
filename=uploaded_file_handle.name,
domain=domain)
request.session[EXCEL_SESSION_ID] = case_upload.upload_id
try:
case_upload.check_file()
except ImporterError as e:
return render_error(request, domain, get_importer_error_message(e))
except SpreadsheetFileExtError:
return render_error(request, domain, _("Please upload file with extension .xls or .xlsx"))
invalid_column_names = set()
with case_upload.get_spreadsheet() as spreadsheet:
columns = spreadsheet.get_header_columns()
validate_column_names(columns, invalid_column_names)
row_count = spreadsheet.max_row
if invalid_column_names:
error_message = format_html(
_("Column names must be <a target='_blank' href='https://www.w3schools.com/xml/xml_elements.asp'>"
"valid XML elements</a> and cannot start with a number or contain spaces or most special characters."
" Please update the following: {}.").format(
', '.join(invalid_column_names)))
return render_error(request, domain, error_message)
if row_count == 0:
return render_error(request, domain, _(
'Your spreadsheet is empty. Please try again with a different spreadsheet.'
))
if len(columns) > MAX_CASE_IMPORTER_COLUMNS:
return render_error(request, domain, _(
'Your spreadsheet has too many columns. '
'A maximum of %(max_columns)s is supported.'
) % {'max_columns': MAX_CASE_IMPORTER_COLUMNS})
case_types_from_apps = sorted(get_case_types_from_apps(domain))
unrecognized_case_types = sorted([t for t in get_case_types_for_domain_es(domain)
if t not in case_types_from_apps])
if len(case_types_from_apps) == 0 and len(unrecognized_case_types) == 0:
return render_error(request, domain, _(
'No cases have been submitted to this domain and there are no '
'applications yet. You cannot import case details from an Excel '
'file until you have existing cases or applications.'
))
context = {
'columns': columns,
'unrecognized_case_types': unrecognized_case_types,
'case_types_from_apps': case_types_from_apps,
'domain': domain,
'slug': base.ImportCases.slug,
}
context.update(_case_importer_breadcrumb_context(_('Case Options'), domain))
return render(request, "case_importer/excel_config.html", context)
