def generate_xlsform(instance, created=False):
""" Generate XLSForm file """
output_file = "/tmp/{}-{}.xml".format(
random.getrandbits(32), instance.pk)
xls_filepath = "{}{}".format(settings.PROJECT_PATH, instance.xls_file.url)
xls2xform_convert(xls_filepath, output_file)
xlsform_file = File(open(output_file))
filename = "{}.xml".format(instance.xls_file.name)
instance.xlsform_file.save(filename, xlsform_file)
def test_xls2xform_convert__e2e_with_extra_columns__does_not_use_excessive_memory(
self, ):
"""Degenerate form with many blank columns"""
process = psutil.Process(os.getpid())
pre_mem = process.memory_info().rss
xls2xform_convert(
xlsform_path=os.path.join(example_xls.PATH, "extra_columns.xlsx"),
xform_path=os.path.join(test_output.PATH, "extra_columns.xml"),
)
post_mem = process.memory_info().rss
# in v1.8.0, memory usage grew by over 16x
self.assertLess(post_mem, pre_mem * 2)
def wrapper(xlsform_path):
"""
Return XLS2XForm result, including any generated warnings.
Calls xls2xform_convert using the supplied xlsform_path.
- XLSForm path is always required.
- If xform_path is blank, use the XLSForm filename and path.
If any of the paths end up not being resolved, error message boxes are
opened to indicate this clearly to the user.
Parameters.
:param xlsform_path: str. Path to XLSForm to convert.
:return: tuple (output header message, message content)
"""
xform_path_used = None
xlsform_path_used = None
try:
header = "Generate XForm task was run. Output below."
valid_xlsform_path = utils.validate_path(
"XLSForm path", xlsform_path, ".xlsx")
valid_xform_path = valid_xlsform_path.replace(".xlsx", ".xml")
content = xls2xform_convert(
xlsform_path=valid_xlsform_path, xform_path=valid_xform_path,
validate=False)
content.append(xform_patch.xform_empty_question_label_patch(
valid_xform_path))
except Exception as e:
header = "Generate XForm task not run. Error(s) below."
content = str(e)
else:
xform_path_used = valid_xform_path
xlsform_path_used = valid_xlsform_path
result = utils.format_output(header=header, content=content)
return result, xform_path_used, xlsform_path_used
def post():
with TemporaryDirectory() as temp_dir_name:
try:
with open(os.path.join(temp_dir_name, "tmp.xml"),
"w+") as xform, open(
os.path.join(temp_dir_name, "tmp.xlsx"),
"wb") as xlsform:
xlsform.write(request.get_data())
convert_status = xls2xform.xls2xform_convert(
xlsform_path=str(xlsform.name),
xform_path=str(xform.name),
validate=True,
pretty_print=False,
)
if convert_status:
logger.warning(convert_status)
if os.path.isfile(xform.name):
return response(status=200,
result=xform.read(),
warnings=convert_status)
else:
return response(error=convert_status)
except Exception as e:
logger.error(e)
return response(error=str(e))
def wrapper(xlsform_path):
"""
Return XLS2XForm result, including any generated warnings.
Calls xls2xform_convert using the supplied xlsform_path.
- XLSForm path is always required.
- If xform_path is blank, use the XLSForm filename and path.
If any of the paths end up not being resolved, error message boxes are
opened to indicate this clearly to the user.
Parameters.
:param xlsform_path: str. Path to XLSForm to convert.
:return: tuple (output header message, message content)
"""
xform_path_used = None
xlsform_path_used = None
try:
header = "Generate XForm task was run. Output below."
valid_xlsform_path = utils.validate_path("XLSForm path", xlsform_path,
".xlsx")
valid_xform_path = valid_xlsform_path.replace(".xlsx", ".xml")
content = xls2xform_convert(xlsform_path=valid_xlsform_path,
xform_path=valid_xform_path,
validate=False)
content.append(
xform_patch.xform_empty_question_label_patch(valid_xform_path))
except Exception as e:
header = "Generate XForm task not run. Error(s) below."
content = str(e)
else:
xform_path_used = valid_xform_path
xlsform_path_used = valid_xlsform_path
result = utils.format_output(header=header, content=content)
return result, xform_path_used, xlsform_path_used
def post():
xlsform_formid_fallback_header = request.headers.get(
"X-XlsForm-FormId-Fallback")
if xlsform_formid_fallback_header is not None:
xlsform_formid_fallback = sanitize(xlsform_formid_fallback_header)
else:
xlsform_formid_fallback = str(uuid())
request_data = request.get_data()
file_ext = ".xlsx" if has_zip_magic_number(request_data) else ".xls"
with TemporaryDirectory() as temp_dir_name:
try:
with open(
os.path.join(temp_dir_name,
xlsform_formid_fallback + ".xml"),
"w+") as xform, open(
os.path.join(temp_dir_name,
xlsform_formid_fallback + file_ext),
"wb",
) as xlsform:
xlsform.write(request_data)
convert_status = xls2xform.xls2xform_convert(
xlsform_path=str(xlsform.name),
xform_path=str(xform.name),
validate=True,
pretty_print=False,
)
if convert_status:
logger.warning(convert_status)
if os.path.isfile(xform.name):
itemsets_path = os.path.join(temp_dir_name,
"itemsets.csv")
if os.path.isfile(itemsets_path):
try:
with open(itemsets_path, "r") as itemsets:
return response(
status=200,
result=xform.read(),
itemsets=itemsets.read(),
warnings=convert_status,
)
except Exception as e:
logger.error(e)
return response(error=str(e))
else:
return response(status=200,
result=xform.read(),
warnings=convert_status)
else:
return response(error=convert_status)
except Exception as e:
logger.error(e)
return response(error=str(e))
def test_xls2xform_convert__e2e_row_with_no_column_value(self):
"""Programmatically-created XLSX files may have rows without column values"""
md = """
| survey | | | | |
| | type | name | label | hint |
| | text | state | State | |
| | text | city | City | A hint |
"""
wb = md_table_to_workbook(md)
with get_temp_dir() as tmp:
wb_path = os.path.join(tmp, "empty_cell.xlsx")
wb.save(wb_path)
wb.close()
xls2xform_convert(
xlsform_path=wb_path,
xform_path=get_xml_path(wb_path),
)
xform_path = os.path.join(tmp, "empty_cell.xml")
self.assertTrue(os.path.exists(xform_path))
def test_xls2xform_convert__e2e_with_settings_misspelling(self):
"""Should warn about settings misspelling when running full pipeline."""
file_name = "extra_sheet_names"
warnings = xls2xform_convert(
xlsform_path=os.path.join(example_xls.PATH, file_name + ".xlsx"),
xform_path=os.path.join(test_output.PATH, file_name + ".xml"),
validate=False,
pretty_print=False,
enketo=False,
)
expected = (
"When looking for a sheet named 'settings', the following sheets "
"with similar names were found: 'stettings'")
self.assertIn(expected, "\n".join(warnings))
def xlsform_convert(self, validate=True):
msg = xls2xform_convert(self.path, self.outpath, validate=validate, pretty_print = False )
self.assert_itemsets_moved()
return msg