def test_file_ext(self):
with self.assertRaises(SpreadsheetFileExtError) as cxt:
with open_any_workbook(get_file('badext', 'ext')):
pass
self.assertRegexpMatches(
cxt.exception.message,
r'File .*/ext/badext.ext does not end in .xls or .xlsx')
def test_file_ext(self):
with self.assertRaises(SpreadsheetFileExtError) as cxt:
with open_any_workbook(get_file('badext', 'ext')):
pass
expected_error = (
r'File .*/ext/badext.ext does not have a valid extension. Valid '
'extensions are: csv, xls, xlsx')
self.assertRegex(str(cxt.exception), expected_error)
def get_spreadsheet(filename):
try:
with open_any_workbook(filename) as workbook:
yield WorksheetWrapper.from_workbook(workbook)
except SpreadsheetFileEncrypted as e:
raise ImporterExcelFileEncrypted(e.message)
except SpreadsheetFileNotFound as e:
raise ImporterFileNotFound(e.message)
except SpreadsheetFileInvalidError as e:
raise ImporterExcelError(e.message)
def get_spreadsheet(filename):
try:
with open_any_workbook(filename) as workbook:
yield WorksheetWrapper.from_workbook(workbook)
except SpreadsheetFileEncrypted as e:
raise ImporterExcelFileEncrypted(six.text_type(e))
except SpreadsheetFileNotFound as e:
raise ImporterFileNotFound(six.text_type(e))
except SpreadsheetFileInvalidError as e:
raise ImporterExcelError(six.text_type(e))
def _process_bulk_upload(bulk_file, domain):
filename = make_temp_file(bulk_file.read(), file_extention_from_filename(bulk_file.name))
errors = []
with open_any_workbook(filename) as workbook:
for worksheet in workbook.worksheets:
case_type = worksheet.title
for row in itertools.islice(worksheet.iter_rows(), 1, None):
name, group, data_type, description, deprecated = [cell.value for cell in row[:5]]
if name:
error = save_case_property(name, case_type, domain, data_type, description, group, deprecated)
if error:
errors.append(error)
return errors
def _process_bulk_upload(bulk_file, domain):
filename = make_temp_file(bulk_file.read(),
file_extention_from_filename(bulk_file.name))
errors = []
import_fhir_data = toggles.FHIR_INTEGRATION.enabled(domain)
fhir_resource_type_by_case_type = {}
expected_columns_in_prop_sheet = 5
if import_fhir_data:
expected_columns_in_prop_sheet = 7
with open_any_workbook(filename) as workbook:
for worksheet in workbook.worksheets:
if worksheet.title == FHIR_RESOURCE_TYPE_MAPPING_SHEET:
if import_fhir_data:
_errors, fhir_resource_type_by_case_type = _process_fhir_resource_type_mapping_sheet(
domain, worksheet)
errors.extend(_errors)
continue
case_type = worksheet.title
for (i, row) in enumerate(
itertools.islice(worksheet.iter_rows(), 1, None)):
if len(row) < expected_columns_in_prop_sheet:
error = _('Not enough columns')
else:
error, fhir_resource_prop_path, fhir_resource_type, remove_path = None, None, None, None
name, group, data_type, description, deprecated = [
cell.value for cell in row[:5]
]
if import_fhir_data:
fhir_resource_prop_path, remove_path = row[5:]
remove_path = remove_path == 'Y' if remove_path else False
fhir_resource_type = fhir_resource_type_by_case_type.get(
case_type)
if fhir_resource_prop_path and not fhir_resource_type:
error = _('Could not find resource type for {}'
).format(case_type)
if not error:
error = save_case_property(name, case_type, domain,
data_type, description,
group, deprecated,
fhir_resource_prop_path,
fhir_resource_type,
remove_path)
if error:
errors.append(
_('Error in case type {}, row {}: {}').format(
case_type, i, error))
return errors
def _process_bulk_upload(bulk_file, domain):
filename = make_temp_file(bulk_file.read(), file_extention_from_filename(bulk_file.name))
errors = []
with open_any_workbook(filename) as workbook:
for worksheet in workbook.worksheets:
case_type = worksheet.title
for (i, row) in enumerate(itertools.islice(worksheet.iter_rows(), 1, None)):
if len(row) < 5:
error = _('Not enough columns')
else:
name, group, data_type, description, deprecated = [cell.value for cell in row[:5]]
error = save_case_property(name, case_type, domain, data_type, description, group, deprecated)
if error:
errors.append(_('Error in case type {}, row {}: {}').format(case_type, i, error))
return errors
def _process_bulk_upload(bulk_file, domain):
filename = make_temp_file(bulk_file.read(), file_extention_from_filename(bulk_file.name))
errors = []
import_fhir_data = toggles.FHIR_INTEGRATION.enabled(domain)
fhir_resource_type_by_case_type = {}
expected_columns_in_prop_sheet = 5
data_type_map = {t.label: t.value for t in CaseProperty.DataType}
if import_fhir_data:
expected_columns_in_prop_sheet = 7
worksheets = []
allowed_value_info = {}
prop_row_info = {}
seen_props = defaultdict(set)
missing_valid_values = set()
with open_any_workbook(filename) as workbook:
for worksheet in workbook.worksheets:
if worksheet.title.endswith(ALLOWED_VALUES_SHEET_SUFFIX):
case_type = worksheet.title[:-len(ALLOWED_VALUES_SHEET_SUFFIX)]
allowed_value_info[case_type] = defaultdict(dict)
prop_row_info[case_type] = defaultdict(list)
for (i, row) in enumerate(itertools.islice(worksheet.iter_rows(), 1, None), start=2):
row_len = len(row)
if row_len < 1:
# simply ignore any fully blank rows
continue
if row_len < 3:
# if missing value or description, fill in "blank"
row += [Cell(value='') for _ in range(3 - row_len)]
row = [cell.value if cell.value is not None else '' for cell in row]
prop_name, allowed_value, description = [str(val) for val in row[0:3]]
if allowed_value and not prop_name:
msg_format = _('Error in valid values for case type {}, row {}: missing case property')
msg_val = msg_format.format(case_type, i)
errors.append(msg_val)
else:
allowed_value_info[case_type][prop_name][allowed_value] = description
prop_row_info[case_type][prop_name].append(i)
else:
worksheets.append(worksheet)
for worksheet in worksheets:
if worksheet.title == FHIR_RESOURCE_TYPE_MAPPING_SHEET:
if import_fhir_data:
_errors, fhir_resource_type_by_case_type = _process_fhir_resource_type_mapping_sheet(
domain, worksheet)
errors.extend(_errors)
continue
case_type = worksheet.title
for (i, row) in enumerate(itertools.islice(worksheet.iter_rows(), 1, None), start=2):
if len(row) < expected_columns_in_prop_sheet:
error = _('Not enough columns')
else:
error, fhir_resource_prop_path, fhir_resource_type, remove_path = None, None, None, None
name, group, data_type_display, description, deprecated = [cell.value for cell in row[:5]]
# Fall back to value from file if data_type_display is not found in the map.
# This allows existing error path to report accurately the value that isn't found,
# and also has a side-effect of allowing older files (pre change to export
# display values) to import successfully.
data_type = data_type_map.get(data_type_display, data_type_display)
seen_props[case_type].add(name)
if import_fhir_data:
fhir_resource_prop_path, remove_path = row[5:]
remove_path = remove_path == 'Y' if remove_path else False
fhir_resource_type = fhir_resource_type_by_case_type.get(case_type)
if fhir_resource_prop_path and not fhir_resource_type:
error = _('Could not find resource type for {}').format(case_type)
if not error:
if case_type in allowed_value_info:
allowed_values = allowed_value_info[case_type][name]
else:
allowed_values = None
missing_valid_values.add(case_type)
error = save_case_property(name, case_type, domain, data_type, description, group,
deprecated, fhir_resource_prop_path, fhir_resource_type,
remove_path, allowed_values)
if error:
errors.append(_('Error in case type {}, row {}: {}').format(case_type, i, error))
for case_type in missing_valid_values:
errors.append(_('Missing valid values sheet for case type {}').format(case_type))
for case_type in allowed_value_info:
for prop_name in allowed_value_info[case_type]:
if prop_name not in seen_props[case_type]:
msg_format = _(
'Error in valid values for case type {}, nonexistent property listed ({}), row(s): {}')
msg_val = msg_format.format(
case_type, prop_name, ', '.join(str(v) for v in prop_row_info[case_type][prop_name]))
errors.append(msg_val)
return errors
def test_file_ext(self):
with self.assertRaises(SpreadsheetFileExtError) as cxt:
with open_any_workbook(get_file('badext', 'ext')):
pass
self.assertRegexpMatches(cxt.exception.message,
r'File .*/ext/badext.ext does not end in .xls or .xlsx')
