def runTest(self): for filename in ["select_one_external.xlsx"]: path_to_excel_file = os.path.join(DIR, "example_xls", filename) #Get the xform output path: root_filename, ext = os.path.splitext(filename) output_path = os.path.join( DIR, "test_output", root_filename + ".xml") expected_output_path = os.path.join( DIR, "test_expected_output", root_filename + ".xml") output_csv = os.path.join( DIR, "test_output", root_filename + ".csv") #Do the conversion: json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file) self.assertTrue(sheet_to_csv(path_to_excel_file, output_csv, "external_choices")) self.assertFalse(sheet_to_csv(path_to_excel_file, output_csv, "non-existant sheet")) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(output_path) #Compare with the expected output: with codecs.open(expected_output_path, 'rb', encoding="utf-8") as\ expected_file: with codecs.open(output_path, 'rb', encoding="utf-8") as \ actual_file: self.assertMultiLineEqual( expected_file.read(), actual_file.read())
def runTest(self): files_to_test = ["instance_xmlns_test.xls"] for file_to_test in files_to_test: path_to_excel_file = utils.path_to_text_fixture(file_to_test) # Get the xform output path: directory, filename = os.path.split(path_to_excel_file) root_filename, ext = os.path.splitext(filename) path_to_output_xform = os.path.join( directory, root_filename + "_output.xml") path_to_expected_xform = os.path.join( directory, root_filename + ".xml") # Do the conversion: json_survey = xls2json.parse_file_to_json(path_to_excel_file) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(path_to_output_xform) # Compare with the expected output: with codecs.open(path_to_expected_xform, 'rb', encoding="utf-8" ) as expected_file: expected = ETree.fromstring(expected_file.read()) result = ETree.fromstring(survey.to_xml()) def write_line(x): sys.stdout.write(x + "\n") reporter = write_line self.assertTrue(xml_compare( expected, result, reporter=reporter)) os.remove(path_to_output_xform)
def json_workbook(request): error = None warningsList = [] if not (os.access(SERVER_TMP_DIR, os.F_OK)): os.mkdir(SERVER_TMP_DIR) # Make a randomly generated directory to prevent name collisions temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR) form_name = request.POST.get("name", "form") output_filename = form_name + ".xml" out_path = os.path.join(temp_dir, output_filename) if os.access(out_path, os.F_OK): os.remove(out_path) try: json_survey = xls2json.workbook_to_json( json.loads(request.POST["workbookJson"]), form_name=form_name, warnings=warningsList ) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(out_path, warnings=warningsList) except Exception as e: error = str(e) return HttpResponse( json.dumps( {"dir": os.path.split(temp_dir)[-1], "name": output_filename, "error": error, "warnings": warningsList}, indent=4, ), mimetype="application/json", )
def runTest(self): filename = "search_and_select.xlsx" path_to_excel_file = os.path.join(DIR, "example_xls", filename) # Get the xform output path: root_filename, ext = os.path.splitext(filename) output_path = os.path.join(DIR, "test_output", root_filename + ".xml") expected_output_path = os.path.join(DIR, "test_expected_output", root_filename + ".xml") # Do the conversion: warnings = [] json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(output_path, warnings=warnings) # Compare with the expected output: with codecs.open(expected_output_path, 'rb', encoding="utf-8") \ as expected_file: with codecs.open(output_path, 'rb', encoding="utf-8") \ as actual_file: self.assertMultiLineEqual( expected_file.read(), actual_file.read()) # cleanup os.remove(output_path)
def runTest(self): for filename in ["select_one_external.xlsx"]: self.get_file_path(filename) expected_output_path = os.path.join( DIR, "test_expected_output", self.root_filename + ".xml" ) output_csv = os.path.join(DIR, "test_output", self.root_filename + ".csv") # Do the conversion: json_survey = pyxform.xls2json.parse_file_to_json(self.path_to_excel_file) self.assertTrue( sheet_to_csv(self.path_to_excel_file, output_csv, "external_choices") ) self.assertFalse( sheet_to_csv(self.path_to_excel_file, output_csv, "non-existant sheet") ) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(self.output_path) # Compare with the expected output: with codecs.open( expected_output_path, "rb", encoding="utf-8" ) as expected_file: with codecs.open( self.output_path, "rb", encoding="utf-8" ) as actual_file: self.assertXFormEqual(expected_file.read(), actual_file.read())
def runTest(self): for filename in ["new_cascading_select.xls", "old_cascades.xls", "cascading_select_test.xls"]: path_to_excel_file = os.path.join(DIR, "example_xls", filename) # Get the xform output path: root_filename, ext = os.path.splitext(filename) output_path = os.path.join( DIR, "test_output", root_filename + ".xml") expected_output_path = os.path.join( DIR, "test_expected_output", root_filename + ".xml") # Do the conversion: json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(output_path) # Compare with the expected output: with codecs.open(expected_output_path, 'rb', encoding="utf-8") as\ expected_file: with codecs.open(output_path, 'rb', encoding="utf-8") as \ actual_file: self.assertXFormEqual(expected_file.read(), actual_file.read())
def runTest(self): for filename in [ "new_cascading_select.xls", "old_cascades.xls", "cascading_select_test.xls", ]: self.get_file_path(filename) expected_output_path = os.path.join( DIR, "test_expected_output", self.root_filename + ".xml" ) # Do the conversion: json_survey = pyxform.xls2json.parse_file_to_json(self.path_to_excel_file) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(self.output_path) # Compare with the expected output: with codecs.open( expected_output_path, "rb", encoding="utf-8" ) as expected_file: with codecs.open( self.output_path, "rb", encoding="utf-8" ) as actual_file: self.assertXFormEqual(expected_file.read(), actual_file.read())
def runTest(self): filename = "ODKValidateWarnings.xlsx" path_to_excel_file = os.path.join(DIR, "bug_example_xls", filename) # Get the xform output path: root_filename, ext = os.path.splitext(filename) output_path = os.path.join(DIR, "test_output", root_filename + ".xml") # Do the conversion: warnings = [] json_survey = pyxform.xls2json.parse_file_to_json(path_to_excel_file, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(output_path, warnings=warnings)
def runTest(self): filename = "cascades_old_with_no_cascade_sheet.xls" path_to_excel_file = os.path.join(DIR, "bug_example_xls", filename) # Get the xform output path: root_filename, ext = os.path.splitext(filename) output_path = os.path.join(DIR, "test_output", root_filename + ".xml") # Do the conversion: warnings = [] with self.assertRaises(pyxform.errors.PyXFormError): json_survey = pyxform.xls2json.parse_file_to_json(path_to_excel_file, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(output_path, warnings=warnings)
def runTest(self): filename = "duplicate_columns.xlsx" path_to_excel_file = os.path.join(DIR, "example_xls", filename) # Get the xform output path: root_filename, ext = os.path.splitext(filename) output_path = os.path.join(DIR, "test_output", root_filename + ".xml") # Do the conversion: warnings = [] with self.assertRaises(Exception): json_survey = pyxform.xls2json.parse_file_to_json(path_to_excel_file, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(output_path, warnings=warnings)
def index(request): if request.method == 'POST': # If the form has been submitted... form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data if form.is_valid(): # All validation rules pass error = None warnings = None filename, ext = os.path.splitext(request.FILES['file'].name) #Make a randomly generated directory to prevent name collisions temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR) xml_path = os.path.join(temp_dir, filename + '.xml') #Init the output xml file. fo = open(xml_path, "wb+") fo.close() try: #TODO: use the file object directly xls_path = handle_uploaded_file(request.FILES['file'], temp_dir) warnings = [] json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(xml_path, warnings=warnings) except Exception as e: error = 'Error: ' + str(e) return render_to_response('upload.html', { 'form': UploadFileForm(), 'xml_path' : '.' + xml_path, 'xml_url' : request.build_absolute_uri('.' + xml_path), 'success': not error, 'error': error, 'warnings': warnings, 'result': True, }) else: form = UploadFileForm() # An unbound form return render_to_response('upload.html', { 'form': form, })
def runTest(self): filename = "widgets.xls" self.get_file_path(filename) expected_output_path = os.path.join( DIR, "test_expected_output", self.root_filename + ".xml" ) # Do the conversion: warnings = [] json_survey = pyxform.xls2json.parse_file_to_json( self.path_to_excel_file, warnings=warnings ) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(self.output_path, warnings=warnings) # print warnings # Compare with the expected output: with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file: self.assertXFormEqual(expected_file.read(), actual_file.read())
def runTest(self): path_to_excel_file = utils.path_to_text_fixture("xlsform_spec_test.xls") #Get the xform output path: directory, filename = os.path.split(path_to_excel_file) root_filename, ext = os.path.splitext(filename) path_to_xform = os.path.join(directory, root_filename + ".xml") #Do the conversion: json_survey = xls2json.parse_file_to_json(path_to_excel_file) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(path_to_xform) #Compare with the expected output: expected_path = utils.path_to_text_fixture("spec_test_expected_output.xml") with codecs.open(expected_path, 'rb', encoding="utf-8") as expected_file: expected = etree.fromstring(expected_file.read()) result = etree.fromstring(survey.to_xml()) self.assertTrue(xml_compare(expected, result))
def runTest(self): filename = "default_time_demo.xls" path_to_excel_file = os.path.join(DIR, "bug_example_xls", filename) # Get the xform output path: root_filename, ext = os.path.splitext(filename) output_path = os.path.join(DIR, "test_output", root_filename + ".xml") expected_output_path = os.path.join( DIR, "test_expected_output", root_filename + ".xml" ) # Do the conversion: warnings = [] json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file, warnings=warnings ) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(output_path, warnings=warnings) # print warnings # Compare with the expected output: with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: with codecs.open(output_path, "rb", encoding="utf-8") as actual_file: self.assertXFormEqual(expected_file.read(), actual_file.read())
def runTest(self): filename = "search_and_select.xlsx" self.get_file_path(filename) expected_output_path = os.path.join( DIR, "test_expected_output", self.root_filename + ".xml" ) # Do the conversion: warnings = [] json_survey = pyxform.xls2json.parse_file_to_json( self.path_to_excel_file, default_name="search_and_select", warnings=warnings ) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(self.output_path, warnings=warnings) # Compare with the expected output: with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file: self.assertXFormEqual(expected_file.read(), actual_file.read()) # cleanup os.remove(self.output_path)
def json_workbook(request): error = None warningsList = [] #Make a randomly generated directory to prevent name collisions temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR) form_name = request.POST.get('name', 'form') output_filename = form_name + '.xml' out_path = os.path.join(temp_dir, output_filename) fo = open(out_path, "wb+") fo.close() try: json_survey = xls2json.workbook_to_json(json.loads(request.POST['workbookJson']), form_name=form_name, warnings=warningsList) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(out_path, warnings=warningsList) except Exception as e: error = str(e) return HttpResponse(json.dumps({ 'dir': os.path.split(temp_dir)[-1], 'name' : output_filename, 'error': error, 'warnings': warningsList, }, indent=4), mimetype="application/json")
def runTest(self): filename = "attribute_columns_test.xlsx" path_to_excel_file = os.path.join(DIR, "example_xls", filename) # Get the xform output path: root_filename, ext = os.path.splitext(filename) output_path = os.path.join(DIR, "test_output", root_filename + ".xml") expected_output_path = os.path.join(DIR, "test_expected_output", root_filename + ".xml") # Do the conversion: warnings = [] json_survey = pyxform.xls2json.parse_file_to_json(path_to_excel_file, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(output_path, warnings=warnings) # print warnings # Compare with the expected output: with codecs.open(expected_output_path, 'rb', encoding="utf-8") \ as expected_file: with codecs.open(output_path, 'rb', encoding="utf-8") \ as actual_file: self.assertMultiLineEqual(expected_file.read(), actual_file.read())
def runTest(self): filename = "default_time_demo.xls" path_to_excel_file = os.path.join(DIR, "bug_example_xls", filename) # Get the xform output path: root_filename, ext = os.path.splitext(filename) output_path = os.path.join(DIR, "test_output", root_filename + ".xml") expected_output_path = os.path.join(DIR, "test_expected_output", root_filename + ".xml") # Do the conversion: warnings = [] json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file, default_name="default_time_demo", warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(output_path, warnings=warnings) # print warnings # Compare with the expected output: with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: with codecs.open(output_path, "rb", encoding="utf-8") as actual_file: self.assertXFormEqual(expected_file.read(), actual_file.read())
def parse(self): fields = self.xform_dict['children'] errors = self._validate_fields_are_recognised(fields) settings_page_errors = self._validate_settings_page_is_not_present(self.xform_dict) errors = errors.union(settings_page_errors) choice_errors = self._validate_media_in_choices(fields) [errors.add(choice_error) for choice_error in choice_errors if choice_error] choice_name_errors = self._validate_choice_names(fields) errors = errors.union(set(choice_name_errors)) questions, question_errors, info = self._create_questions(fields) if question_errors: errors = errors.union(question_errors) if not errors and not questions: errors.add("Uploaded file is empty!") if errors: return XlsParserResponse(errors) _map_unique_id_question_to_select_one(self.xform_dict) survey = create_survey_element_from_dict(self.xform_dict) xform = survey.to_xml() # encoding is added to support ie8 xform = re.sub(r'<\?xml version="1.0"\?>', '<?xml version="1.0" encoding="utf-8"?>', xform) updated_xform = self.update_xform_with_questionnaire_name(xform) return XlsParserResponse([], updated_xform, questions, info)
def runTest(self): for filename in [ "new_cascading_select.xls", "old_cascades.xls", "cascading_select_test.xls" ]: self.get_file_path(filename) expected_output_path = os.path.join(DIR, "test_expected_output", self.root_filename + ".xml") # Do the conversion: json_survey = pyxform.xls2json.parse_file_to_json( self.path_to_excel_file) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(self.output_path) # Compare with the expected output: with codecs.open(expected_output_path, 'rb', encoding="utf-8") as\ expected_file: with codecs.open(self.output_path, 'rb', encoding="utf-8") as \ actual_file: self.assertXFormEqual(expected_file.read(), actual_file.read())
def index(request): if request.method == 'POST': # If the form has been submitted... form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data if form.is_valid(): # All validation rules pass error = None warnings = None filename, ext = os.path.splitext(request.FILES['file'].name) if not (os.access(SERVER_TMP_DIR, os.F_OK)): os.mkdir(SERVER_TMP_DIR) #Make a randomly generated directory to prevent name collisions temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR) xml_path = os.path.join(temp_dir, filename + '.xml') itemsets_url = None relpath = os.path.relpath(xml_path, SERVER_TMP_DIR) #Init the output xml file. fo = open(xml_path, "wb+") fo.close() try: #TODO: use the file object directly xls_path = handle_uploaded_file(request.FILES['file'], temp_dir) warnings = [] json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(xml_path, warnings=warnings) if has_external_choices(json_survey): # Create a csv for the external choices itemsets_csv = os.path.join( os.path.split(xls_path)[0], "itemsets.csv") relpath_itemsets_csv = os.path.relpath( itemsets_csv, SERVER_TMP_DIR) choices_exported = sheet_to_csv(xls_path, itemsets_csv, "external_choices") if not choices_exported: warnings += [ "Could not export itemsets.csv, " "perhaps the external choices sheet is missing." ] else: itemsets_url = request.build_absolute_uri( './downloads/' + relpath_itemsets_csv) except Exception as e: error = 'Error: ' + str(e) return render_to_response( 'upload.html', { 'form': UploadFileForm(), 'xml_path': request.build_absolute_uri('./downloads/' + relpath), 'xml_url': request.build_absolute_uri('./downloads/' + relpath), 'itemsets_url': itemsets_url, 'success': not error, 'error': error, 'warnings': warnings, 'result': True, }) else: form = UploadFileForm() # An unbound form return render(request, 'upload.html', context={ 'form': form, })
def do_work(pk, token): ona_api_url = ONA_API_URL headers = {"Authorization": "Token {}".format(token)} if ona_api_url.endswith('/'): ona_api_url = ona_api_url[:-1] data = requests.get("{}/data/{}".format(ona_api_url, pk), headers=headers).json() definition = requests.get("{}/forms/{}/form.json".format(ona_api_url, pk), headers=headers).json() xform_survey = create_survey_element_from_dict(definition) data = generate_export(definition, data, xform_survey=xform_survey) sections = generate_sections(definition) td = OrderedDict(title_dictionary(definition['children'])) dict_copy = td.copy() keys_copy = list(td.keys()) for item in td.keys(): value = td[item] keys = [k for k in keys_copy if value == dict_copy[k]] if len(keys) <= 1: continue keys.sort() fill = math.ceil(math.log(len(keys), 10)) # td[item] = "{} ({})".format(value, str(keys.index(item) + 1).zfill(int(fill))) td[item] = "{} ({})".format(value, item) for key, data_set in data.iteritems(): section = sections[key] for s in section: for d in data_set: if s['xpath'] in d: name = d[s['xpath']] simplified_name = s['xpath'].split('/')[-1] if s['type'] in ['select', 'select1']: if s['type'] == 'select1': option = [ l['label']['English'] if isinstance( l['label'], dict) else l['label'] for l in s['children'] if isinstance(l, dict) and l['name'] == name ] if option: option = option.pop() d[s['xpath']] = option elif 'itemset' in s and s['itemset']: choices = xform_survey.choices.get( s['itemset']) if choices: option = [ l['label']['English'] if isinstance( l['label'], dict) else l['label'] for l in choices if isinstance(l, dict) and l['name'] == name ] if option: option = option.pop() d[s['xpath']] = option else: print(name, " not found") else: name = name.split(' ') options = [ l['label']['English'] if isinstance( l['label'], dict) else l['label'] for l in s['children'] if isinstance(l, dict) and l['name'] in name ] # print("Options", options) if options: d[s['xpath']] = ", ".join(options) elif 'itemset' in s and s['itemset']: choices = xform_survey.choices.get( s['itemset']) if choices: options = [ l['label']['English'] if isinstance( l['label'], dict) else l['label'] for l in choices if isinstance(l, dict) and l['name'] == name ] if options: d[s['xpath']] = ", ".join(options) intermediate = d[s['xpath']] del d[s['xpath']] if simplified_name in td: d[td[simplified_name]] = intermediate else: d[simplified_name] = intermediate section_name = data.keys()[0] return data
def index(request): if request.method == "POST": # If the form has been submitted... form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data if form.is_valid(): # All validation rules pass error = None warnings = None filename, ext = os.path.splitext(request.FILES["file"].name) if not (os.access(SERVER_TMP_DIR, os.F_OK)): os.mkdir(SERVER_TMP_DIR) # Make a randomly generated directory to prevent name collisions temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR) xml_path = os.path.join(temp_dir, filename + ".xml") itemsets_url = None relpath = os.path.relpath(xml_path, SERVER_TMP_DIR) # Init the output xml file. fo = open(xml_path, "wb+") fo.close() try: # TODO: use the file object directly xls_path = handle_uploaded_file(request.FILES["file"], temp_dir) warnings = [] json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(xml_path, warnings=warnings) if has_external_choices(json_survey): # Create a csv for the external choices itemsets_csv = os.path.join(os.path.split(xls_path)[0], "itemsets.csv") relpath_itemsets_csv = os.path.relpath(itemsets_csv, SERVER_TMP_DIR) choices_exported = sheet_to_csv(xls_path, itemsets_csv, "external_choices") if not choices_exported: warnings += ["Could not export itemsets.csv, " "perhaps the external choices sheet is missing."] else: itemsets_url = request.build_absolute_uri("./downloads/" + relpath_itemsets_csv) except Exception as e: error = "Error: " + str(e) return render_to_response( "upload.html", { "form": UploadFileForm(), "xml_path": request.build_absolute_uri("./downloads/" + relpath), "xml_url": request.build_absolute_uri("./downloads/" + relpath), "itemsets_url": itemsets_url, "success": not error, "error": error, "warnings": warnings, "result": True, }, ) else: form = UploadFileForm() # An unbound form return render_to_response("upload.html", {"form": form})