def runTest(self):
for filename in ["select_one_external.xlsx"]:
path_to_excel_file = os.path.join(DIR, "example_xls", filename)
#Get the xform output path:
root_filename, ext = os.path.splitext(filename)
output_path = os.path.join(
DIR, "test_output", root_filename + ".xml")
expected_output_path = os.path.join(
DIR, "test_expected_output", root_filename + ".xml")
output_csv = os.path.join(
DIR, "test_output", root_filename + ".csv")
#Do the conversion:
json_survey = pyxform.xls2json.parse_file_to_json(
path_to_excel_file)
self.assertTrue(sheet_to_csv(path_to_excel_file, output_csv, "external_choices"))
self.assertFalse(sheet_to_csv(path_to_excel_file, output_csv, "non-existant sheet"))
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(output_path)
#Compare with the expected output:
with codecs.open(expected_output_path, 'rb', encoding="utf-8") as\
expected_file:
with codecs.open(output_path, 'rb', encoding="utf-8") as \
actual_file:
self.assertMultiLineEqual(
expected_file.read(), actual_file.read())
def runTest(self):
files_to_test = ["instance_xmlns_test.xls"]
for file_to_test in files_to_test:
path_to_excel_file = utils.path_to_text_fixture(file_to_test)
# Get the xform output path:
directory, filename = os.path.split(path_to_excel_file)
root_filename, ext = os.path.splitext(filename)
path_to_output_xform = os.path.join(
directory, root_filename + "_output.xml")
path_to_expected_xform = os.path.join(
directory, root_filename + ".xml")
# Do the conversion:
json_survey = xls2json.parse_file_to_json(path_to_excel_file)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(path_to_output_xform)
# Compare with the expected output:
with codecs.open(path_to_expected_xform, 'rb', encoding="utf-8"
) as expected_file:
expected = ETree.fromstring(expected_file.read())
result = ETree.fromstring(survey.to_xml())
def write_line(x): sys.stdout.write(x + "\n")
reporter = write_line
self.assertTrue(xml_compare(
expected, result, reporter=reporter))
os.remove(path_to_output_xform)
def json_workbook(request):
error = None
warningsList = []
if not (os.access(SERVER_TMP_DIR, os.F_OK)):
os.mkdir(SERVER_TMP_DIR)
# Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR)
form_name = request.POST.get("name", "form")
output_filename = form_name + ".xml"
out_path = os.path.join(temp_dir, output_filename)
if os.access(out_path, os.F_OK):
os.remove(out_path)
try:
json_survey = xls2json.workbook_to_json(
json.loads(request.POST["workbookJson"]), form_name=form_name, warnings=warningsList
)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(out_path, warnings=warningsList)
except Exception as e:
error = str(e)
return HttpResponse(
json.dumps(
{"dir": os.path.split(temp_dir)[-1], "name": output_filename, "error": error, "warnings": warningsList},
indent=4,
),
mimetype="application/json",
)
def runTest(self):
filename = "search_and_select.xlsx"
path_to_excel_file = os.path.join(DIR, "example_xls", filename)
# Get the xform output path:
root_filename, ext = os.path.splitext(filename)
output_path = os.path.join(DIR, "test_output", root_filename + ".xml")
expected_output_path = os.path.join(DIR, "test_expected_output",
root_filename + ".xml")
# Do the conversion:
warnings = []
json_survey = pyxform.xls2json.parse_file_to_json(
path_to_excel_file, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(output_path, warnings=warnings)
# Compare with the expected output:
with codecs.open(expected_output_path, 'rb', encoding="utf-8") \
as expected_file:
with codecs.open(output_path, 'rb', encoding="utf-8") \
as actual_file:
self.assertMultiLineEqual(
expected_file.read(), actual_file.read())
# cleanup
os.remove(output_path)
def runTest(self):
for filename in ["select_one_external.xlsx"]:
self.get_file_path(filename)
expected_output_path = os.path.join(
DIR, "test_expected_output", self.root_filename + ".xml"
)
output_csv = os.path.join(DIR, "test_output", self.root_filename + ".csv")
# Do the conversion:
json_survey = pyxform.xls2json.parse_file_to_json(self.path_to_excel_file)
self.assertTrue(
sheet_to_csv(self.path_to_excel_file, output_csv, "external_choices")
)
self.assertFalse(
sheet_to_csv(self.path_to_excel_file, output_csv, "non-existant sheet")
)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(self.output_path)
# Compare with the expected output:
with codecs.open(
expected_output_path, "rb", encoding="utf-8"
) as expected_file:
with codecs.open(
self.output_path, "rb", encoding="utf-8"
) as actual_file:
self.assertXFormEqual(expected_file.read(), actual_file.read())
def runTest(self):
for filename in ["new_cascading_select.xls", "old_cascades.xls",
"cascading_select_test.xls"]:
path_to_excel_file = os.path.join(DIR, "example_xls", filename)
# Get the xform output path:
root_filename, ext = os.path.splitext(filename)
output_path = os.path.join(
DIR, "test_output", root_filename + ".xml")
expected_output_path = os.path.join(
DIR, "test_expected_output", root_filename + ".xml")
# Do the conversion:
json_survey = pyxform.xls2json.parse_file_to_json(
path_to_excel_file)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(output_path)
# Compare with the expected output:
with codecs.open(expected_output_path, 'rb', encoding="utf-8") as\
expected_file:
with codecs.open(output_path, 'rb', encoding="utf-8") as \
actual_file:
self.assertXFormEqual(expected_file.read(),
actual_file.read())
def runTest(self):
for filename in [
"new_cascading_select.xls",
"old_cascades.xls",
"cascading_select_test.xls",
]:
self.get_file_path(filename)
expected_output_path = os.path.join(
DIR, "test_expected_output", self.root_filename + ".xml"
)
# Do the conversion:
json_survey = pyxform.xls2json.parse_file_to_json(self.path_to_excel_file)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(self.output_path)
# Compare with the expected output:
with codecs.open(
expected_output_path, "rb", encoding="utf-8"
) as expected_file:
with codecs.open(
self.output_path, "rb", encoding="utf-8"
) as actual_file:
self.assertXFormEqual(expected_file.read(), actual_file.read())
def runTest(self):
filename = "ODKValidateWarnings.xlsx"
path_to_excel_file = os.path.join(DIR, "bug_example_xls", filename)
# Get the xform output path:
root_filename, ext = os.path.splitext(filename)
output_path = os.path.join(DIR, "test_output", root_filename + ".xml")
# Do the conversion:
warnings = []
json_survey = pyxform.xls2json.parse_file_to_json(path_to_excel_file, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(output_path, warnings=warnings)
def runTest(self):
filename = "cascades_old_with_no_cascade_sheet.xls"
path_to_excel_file = os.path.join(DIR, "bug_example_xls", filename)
# Get the xform output path:
root_filename, ext = os.path.splitext(filename)
output_path = os.path.join(DIR, "test_output", root_filename + ".xml")
# Do the conversion:
warnings = []
with self.assertRaises(pyxform.errors.PyXFormError):
json_survey = pyxform.xls2json.parse_file_to_json(path_to_excel_file, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(output_path, warnings=warnings)
def runTest(self):
filename = "duplicate_columns.xlsx"
path_to_excel_file = os.path.join(DIR, "example_xls", filename)
# Get the xform output path:
root_filename, ext = os.path.splitext(filename)
output_path = os.path.join(DIR, "test_output", root_filename + ".xml")
# Do the conversion:
warnings = []
with self.assertRaises(Exception):
json_survey = pyxform.xls2json.parse_file_to_json(path_to_excel_file, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(output_path, warnings=warnings)
def index(request):
if request.method == 'POST': # If the form has been submitted...
form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data
if form.is_valid(): # All validation rules pass
error = None
warnings = None
filename, ext = os.path.splitext(request.FILES['file'].name)
#Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR)
xml_path = os.path.join(temp_dir, filename + '.xml')
#Init the output xml file.
fo = open(xml_path, "wb+")
fo.close()
try:
#TODO: use the file object directly
xls_path = handle_uploaded_file(request.FILES['file'], temp_dir)
warnings = []
json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(xml_path, warnings=warnings)
except Exception as e:
error = 'Error: ' + str(e)
return render_to_response('upload.html', {
'form': UploadFileForm(),
'xml_path' : '.' + xml_path,
'xml_url' : request.build_absolute_uri('.' + xml_path),
'success': not error,
'error': error,
'warnings': warnings,
'result': True,
})
else:
form = UploadFileForm() # An unbound form
return render_to_response('upload.html', {
'form': form,
})
def runTest(self):
filename = "widgets.xls"
self.get_file_path(filename)
expected_output_path = os.path.join(
DIR, "test_expected_output", self.root_filename + ".xml"
)
# Do the conversion:
warnings = []
json_survey = pyxform.xls2json.parse_file_to_json(
self.path_to_excel_file, warnings=warnings
)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(self.output_path, warnings=warnings)
# print warnings
# Compare with the expected output:
with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file:
with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file:
self.assertXFormEqual(expected_file.read(), actual_file.read())
def runTest(self):
path_to_excel_file = utils.path_to_text_fixture("xlsform_spec_test.xls")
#Get the xform output path:
directory, filename = os.path.split(path_to_excel_file)
root_filename, ext = os.path.splitext(filename)
path_to_xform = os.path.join(directory, root_filename + ".xml")
#Do the conversion:
json_survey = xls2json.parse_file_to_json(path_to_excel_file)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(path_to_xform)
#Compare with the expected output:
expected_path = utils.path_to_text_fixture("spec_test_expected_output.xml")
with codecs.open(expected_path, 'rb', encoding="utf-8") as expected_file:
expected = etree.fromstring(expected_file.read())
result = etree.fromstring(survey.to_xml())
self.assertTrue(xml_compare(expected, result))
def runTest(self):
filename = "default_time_demo.xls"
path_to_excel_file = os.path.join(DIR, "bug_example_xls", filename)
# Get the xform output path:
root_filename, ext = os.path.splitext(filename)
output_path = os.path.join(DIR, "test_output", root_filename + ".xml")
expected_output_path = os.path.join(
DIR, "test_expected_output", root_filename + ".xml"
)
# Do the conversion:
warnings = []
json_survey = pyxform.xls2json.parse_file_to_json(
path_to_excel_file, warnings=warnings
)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(output_path, warnings=warnings)
# print warnings
# Compare with the expected output:
with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file:
with codecs.open(output_path, "rb", encoding="utf-8") as actual_file:
self.assertXFormEqual(expected_file.read(), actual_file.read())
def runTest(self):
filename = "search_and_select.xlsx"
self.get_file_path(filename)
expected_output_path = os.path.join(
DIR, "test_expected_output", self.root_filename + ".xml"
)
# Do the conversion:
warnings = []
json_survey = pyxform.xls2json.parse_file_to_json(
self.path_to_excel_file, default_name="search_and_select", warnings=warnings
)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(self.output_path, warnings=warnings)
# Compare with the expected output:
with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file:
with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file:
self.assertXFormEqual(expected_file.read(), actual_file.read())
# cleanup
os.remove(self.output_path)
def json_workbook(request):
error = None
warningsList = []
#Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR)
form_name = request.POST.get('name', 'form')
output_filename = form_name + '.xml'
out_path = os.path.join(temp_dir, output_filename)
fo = open(out_path, "wb+")
fo.close()
try:
json_survey = xls2json.workbook_to_json(json.loads(request.POST['workbookJson']), form_name=form_name, warnings=warningsList)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(out_path, warnings=warningsList)
except Exception as e:
error = str(e)
return HttpResponse(json.dumps({
'dir': os.path.split(temp_dir)[-1],
'name' : output_filename,
'error': error,
'warnings': warningsList,
}, indent=4), mimetype="application/json")
def runTest(self):
filename = "attribute_columns_test.xlsx"
path_to_excel_file = os.path.join(DIR, "example_xls", filename)
# Get the xform output path:
root_filename, ext = os.path.splitext(filename)
output_path = os.path.join(DIR, "test_output", root_filename + ".xml")
expected_output_path = os.path.join(DIR, "test_expected_output",
root_filename + ".xml")
# Do the conversion:
warnings = []
json_survey = pyxform.xls2json.parse_file_to_json(path_to_excel_file,
warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(output_path, warnings=warnings)
# print warnings
# Compare with the expected output:
with codecs.open(expected_output_path, 'rb', encoding="utf-8") \
as expected_file:
with codecs.open(output_path, 'rb', encoding="utf-8") \
as actual_file:
self.assertMultiLineEqual(expected_file.read(),
actual_file.read())
def runTest(self):
filename = "default_time_demo.xls"
path_to_excel_file = os.path.join(DIR, "bug_example_xls", filename)
# Get the xform output path:
root_filename, ext = os.path.splitext(filename)
output_path = os.path.join(DIR, "test_output", root_filename + ".xml")
expected_output_path = os.path.join(DIR, "test_expected_output",
root_filename + ".xml")
# Do the conversion:
warnings = []
json_survey = pyxform.xls2json.parse_file_to_json(
path_to_excel_file,
default_name="default_time_demo",
warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(output_path, warnings=warnings)
# print warnings
# Compare with the expected output:
with codecs.open(expected_output_path, "rb",
encoding="utf-8") as expected_file:
with codecs.open(output_path, "rb",
encoding="utf-8") as actual_file:
self.assertXFormEqual(expected_file.read(), actual_file.read())
def parse(self):
fields = self.xform_dict['children']
errors = self._validate_fields_are_recognised(fields)
settings_page_errors = self._validate_settings_page_is_not_present(self.xform_dict)
errors = errors.union(settings_page_errors)
choice_errors = self._validate_media_in_choices(fields)
[errors.add(choice_error) for choice_error in choice_errors if choice_error]
choice_name_errors = self._validate_choice_names(fields)
errors = errors.union(set(choice_name_errors))
questions, question_errors, info = self._create_questions(fields)
if question_errors:
errors = errors.union(question_errors)
if not errors and not questions:
errors.add("Uploaded file is empty!")
if errors:
return XlsParserResponse(errors)
_map_unique_id_question_to_select_one(self.xform_dict)
survey = create_survey_element_from_dict(self.xform_dict)
xform = survey.to_xml()
# encoding is added to support ie8
xform = re.sub(r'<\?xml version="1.0"\?>', '<?xml version="1.0" encoding="utf-8"?>', xform)
updated_xform = self.update_xform_with_questionnaire_name(xform)
return XlsParserResponse([], updated_xform, questions, info)
def runTest(self):
for filename in [
"new_cascading_select.xls", "old_cascades.xls",
"cascading_select_test.xls"
]:
self.get_file_path(filename)
expected_output_path = os.path.join(DIR, "test_expected_output",
self.root_filename + ".xml")
# Do the conversion:
json_survey = pyxform.xls2json.parse_file_to_json(
self.path_to_excel_file)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(self.output_path)
# Compare with the expected output:
with codecs.open(expected_output_path, 'rb', encoding="utf-8") as\
expected_file:
with codecs.open(self.output_path, 'rb', encoding="utf-8") as \
actual_file:
self.assertXFormEqual(expected_file.read(),
actual_file.read())
def index(request):
if request.method == 'POST': # If the form has been submitted...
form = UploadFileForm(request.POST,
request.FILES) # A form bound to the POST data
if form.is_valid(): # All validation rules pass
error = None
warnings = None
filename, ext = os.path.splitext(request.FILES['file'].name)
if not (os.access(SERVER_TMP_DIR, os.F_OK)):
os.mkdir(SERVER_TMP_DIR)
#Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR)
xml_path = os.path.join(temp_dir, filename + '.xml')
itemsets_url = None
relpath = os.path.relpath(xml_path, SERVER_TMP_DIR)
#Init the output xml file.
fo = open(xml_path, "wb+")
fo.close()
try:
#TODO: use the file object directly
xls_path = handle_uploaded_file(request.FILES['file'],
temp_dir)
warnings = []
json_survey = xls2json.parse_file_to_json(xls_path,
warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(xml_path, warnings=warnings)
if has_external_choices(json_survey):
# Create a csv for the external choices
itemsets_csv = os.path.join(
os.path.split(xls_path)[0], "itemsets.csv")
relpath_itemsets_csv = os.path.relpath(
itemsets_csv, SERVER_TMP_DIR)
choices_exported = sheet_to_csv(xls_path, itemsets_csv,
"external_choices")
if not choices_exported:
warnings += [
"Could not export itemsets.csv, "
"perhaps the external choices sheet is missing."
]
else:
itemsets_url = request.build_absolute_uri(
'./downloads/' + relpath_itemsets_csv)
except Exception as e:
error = 'Error: ' + str(e)
return render_to_response(
'upload.html', {
'form':
UploadFileForm(),
'xml_path':
request.build_absolute_uri('./downloads/' + relpath),
'xml_url':
request.build_absolute_uri('./downloads/' + relpath),
'itemsets_url':
itemsets_url,
'success':
not error,
'error':
error,
'warnings':
warnings,
'result':
True,
})
else:
form = UploadFileForm() # An unbound form
return render(request, 'upload.html', context={
'form': form,
})
def do_work(pk, token):
ona_api_url = ONA_API_URL
headers = {"Authorization": "Token {}".format(token)}
if ona_api_url.endswith('/'):
ona_api_url = ona_api_url[:-1]
data = requests.get("{}/data/{}".format(ona_api_url, pk),
headers=headers).json()
definition = requests.get("{}/forms/{}/form.json".format(ona_api_url, pk),
headers=headers).json()
xform_survey = create_survey_element_from_dict(definition)
data = generate_export(definition, data, xform_survey=xform_survey)
sections = generate_sections(definition)
td = OrderedDict(title_dictionary(definition['children']))
dict_copy = td.copy()
keys_copy = list(td.keys())
for item in td.keys():
value = td[item]
keys = [k for k in keys_copy if value == dict_copy[k]]
if len(keys) <= 1:
continue
keys.sort()
fill = math.ceil(math.log(len(keys), 10))
# td[item] = "{} ({})".format(value, str(keys.index(item) + 1).zfill(int(fill)))
td[item] = "{} ({})".format(value, item)
for key, data_set in data.iteritems():
section = sections[key]
for s in section:
for d in data_set:
if s['xpath'] in d:
name = d[s['xpath']]
simplified_name = s['xpath'].split('/')[-1]
if s['type'] in ['select', 'select1']:
if s['type'] == 'select1':
option = [
l['label']['English'] if isinstance(
l['label'], dict) else l['label']
for l in s['children']
if isinstance(l, dict) and l['name'] == name
]
if option:
option = option.pop()
d[s['xpath']] = option
elif 'itemset' in s and s['itemset']:
choices = xform_survey.choices.get(
s['itemset'])
if choices:
option = [
l['label']['English'] if isinstance(
l['label'], dict) else l['label']
for l in choices
if isinstance(l, dict)
and l['name'] == name
]
if option:
option = option.pop()
d[s['xpath']] = option
else:
print(name, " not found")
else:
name = name.split(' ')
options = [
l['label']['English'] if isinstance(
l['label'], dict) else l['label']
for l in s['children']
if isinstance(l, dict) and l['name'] in name
]
# print("Options", options)
if options:
d[s['xpath']] = ", ".join(options)
elif 'itemset' in s and s['itemset']:
choices = xform_survey.choices.get(
s['itemset'])
if choices:
options = [
l['label']['English'] if isinstance(
l['label'], dict) else l['label']
for l in choices
if isinstance(l, dict)
and l['name'] == name
]
if options:
d[s['xpath']] = ", ".join(options)
intermediate = d[s['xpath']]
del d[s['xpath']]
if simplified_name in td:
d[td[simplified_name]] = intermediate
else:
d[simplified_name] = intermediate
section_name = data.keys()[0]
return data
def index(request):
if request.method == "POST": # If the form has been submitted...
form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data
if form.is_valid(): # All validation rules pass
error = None
warnings = None
filename, ext = os.path.splitext(request.FILES["file"].name)
if not (os.access(SERVER_TMP_DIR, os.F_OK)):
os.mkdir(SERVER_TMP_DIR)
# Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR)
xml_path = os.path.join(temp_dir, filename + ".xml")
itemsets_url = None
relpath = os.path.relpath(xml_path, SERVER_TMP_DIR)
# Init the output xml file.
fo = open(xml_path, "wb+")
fo.close()
try:
# TODO: use the file object directly
xls_path = handle_uploaded_file(request.FILES["file"], temp_dir)
warnings = []
json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(xml_path, warnings=warnings)
if has_external_choices(json_survey):
# Create a csv for the external choices
itemsets_csv = os.path.join(os.path.split(xls_path)[0], "itemsets.csv")
relpath_itemsets_csv = os.path.relpath(itemsets_csv, SERVER_TMP_DIR)
choices_exported = sheet_to_csv(xls_path, itemsets_csv, "external_choices")
if not choices_exported:
warnings += ["Could not export itemsets.csv, " "perhaps the external choices sheet is missing."]
else:
itemsets_url = request.build_absolute_uri("./downloads/" + relpath_itemsets_csv)
except Exception as e:
error = "Error: " + str(e)
return render_to_response(
"upload.html",
{
"form": UploadFileForm(),
"xml_path": request.build_absolute_uri("./downloads/" + relpath),
"xml_url": request.build_absolute_uri("./downloads/" + relpath),
"itemsets_url": itemsets_url,
"success": not error,
"error": error,
"warnings": warnings,
"result": True,
},
)
else:
form = UploadFileForm() # An unbound form
return render_to_response("upload.html", {"form": form})
