def process_xlsform(xls, default_name): """ Process XLSForm file and return the survey dictionary for the XLSForm. """ # FLOW Results package is a JSON file. if xls.name.endswith('json'): return FloipSurvey(xls).survey.to_json_dict() file_object = None if xls.name.endswith('csv'): # a csv file gets closed in pyxform, make a copy xls.seek(0) file_object = BytesIO() file_object.write(xls.read()) file_object.seek(0) xls.seek(0) try: return parse_file_to_json(xls.name, file_object=file_object or xls) except csv.Error as e: if is_newline_error(e): xls.seek(0) file_object = StringIO(u'\n'.join(xls.read().splitlines())) return parse_file_to_json(xls.name, default_name=default_name, file_object=file_object) raise e
def process_xlsform(xls, default_name): """ Process XLSForm file and return the survey dictionary for the XLSForm. """ # FLOW Results package is a JSON file. if xls.name.endswith('json'): return FloipSurvey(xls).survey.to_json_dict() file_object = None if xls.name.endswith('csv'): # a csv file gets closed in pyxform, make a copy xls.seek(0) file_object = BytesIO() file_object.write(xls.read()) file_object.seek(0) xls.seek(0) try: return parse_file_to_json(xls.name, file_object=file_object or xls) except csv.Error as e: if is_newline_error(e): xls.seek(0) file_object = StringIO( u'\n'.join(xls.read().splitlines())) return parse_file_to_json( xls.name, default_name=default_name, file_object=file_object) raise e
def runTest(self): path_to_excel_file = utils.path_to_text_fixture("warnings.xls") warnings = [] xls2json.parse_file_to_json(path_to_excel_file, warnings=warnings) #print '\n'.join(warnings) self.assertEquals(len(warnings), 17, "Found " + str(len(warnings)) + " warnings")
def test_underscore_warnings(self): """Raise warnings incase there are underscores in column names""" warnings = [] parse_file_to_json(utils.path_to_text_fixture("hidden.xls"), warnings=warnings) self.assertGreater(len(warnings), 0) warning = ("Google Sheets submissions don't allow underscores in the " "column name. If you intend to use Google Sheets " "submissions, replace underscores with hyphens in the " "following names: hidden_test") self.assertIn(warning, warnings)
def test_underscore_warnings(self): """Raise warnings incase there are underscores in column names""" warnings = [] parse_file_to_json(utils.path_to_text_fixture("hidden.xls"), warnings=warnings) self.assertGreater(len(warnings), 0) warning = ( "Google Sheets submissions don't allow underscores in the " "column name. If you intend to use Google Sheets " "submissions, replace underscores with hyphens in the " "following names: hidden_test" ) self.assertIn(warning, warnings)
def xls2xform_convert(xlsform_path, xform_path, validate=True, pretty_print=True, enketo=False): warnings = [] json_survey = xls2json.parse_file_to_json(xlsform_path, warnings=warnings) survey = builder.create_survey_element_from_dict(json_survey) # Setting validate to false will cause the form not to be processed by # ODK Validate. # This may be desirable since ODK Validate requires launching a subprocess # that runs some java code. survey.print_xform_to_file( xform_path, validate=validate, pretty_print=pretty_print, warnings=warnings, enketo=enketo, ) output_dir = os.path.split(xform_path)[0] if has_external_choices(json_survey): itemsets_csv = os.path.join(output_dir, "itemsets.csv") choices_exported = sheet_to_csv(xlsform_path, itemsets_csv, "external_choices") if not choices_exported: warnings.append("Could not export itemsets.csv, perhaps the " "external choices sheet is missing.") else: logger.info("External choices csv is located at: %s", itemsets_csv) return warnings
def runTest(self): files_to_test = ["instance_xmlns_test.xls"] for file_to_test in files_to_test: path_to_excel_file = utils.path_to_text_fixture(file_to_test) #Get the xform output path: directory, filename = os.path.split(path_to_excel_file) root_filename, ext = os.path.splitext(filename) path_to_output_xform = os.path.join(directory, root_filename + "_output.xml") path_to_expected_xform = os.path.join(directory, root_filename + ".xml") #Do the conversion: json_survey = xls2json.parse_file_to_json(path_to_excel_file) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(path_to_output_xform) #Compare with the expected output: with codecs.open(path_to_expected_xform, 'rb', encoding="utf-8") as expected_file: expected = etree.fromstring(expected_file.read()) result = etree.fromstring(survey.to_xml()) reporter = lambda x: sys.stdout.write(x + "\n") self.assertTrue( xml_compare(expected, result, reporter=reporter)) os.remove(path_to_output_xform)
def get_survey_dict(csv_name): survey_file = default_storage.open(csv_name, 'rb') survey_dict = parse_file_to_json( survey_file.name, default_name='data', file_object=survey_file) return survey_dict
def xls2xform_convert( xlsform_path, xform_path, validate=True, pretty_print=True, enketo=False ): warnings = [] json_survey = xls2json.parse_file_to_json(xlsform_path, warnings=warnings) survey = builder.create_survey_element_from_dict(json_survey) # Setting validate to false will cause the form not to be processed by # ODK Validate. # This may be desirable since ODK Validate requires launching a subprocess # that runs some java code. survey.print_xform_to_file( xform_path, validate=validate, pretty_print=pretty_print, warnings=warnings, enketo=enketo, ) output_dir = os.path.split(xform_path)[0] if has_external_choices(json_survey): itemsets_csv = os.path.join(output_dir, "itemsets.csv") choices_exported = sheet_to_csv(xlsform_path, itemsets_csv, "external_choices") if not choices_exported: warnings.append( "Could not export itemsets.csv, perhaps the " "external choices sheet is missing." ) else: logger.info("External choices csv is located at: %s", itemsets_csv) return warnings
def parse_xls(file_or_name): if isinstance(file_or_name, str): filename = file_or_name fileobj = None else: fileobj = file_or_name filename = fileobj.name xform_json = parse_file_to_json(filename, file_object=fileobj) def process_fields(root): for field in root['children']: cons = field.get('bind', {}).get('constraint', '') if cons.startswith('wq:'): cons = cons[3:] for ext in WQ_EXTENSIONS: if cons.startswith(ext + "(") and cons.endswith(")"): field['wq:%s' % ext] = cons[len(ext) + 1:-1] if field['type'] in GROUP_TYPES: process_fields(field) if not field['name'] == "meta": field['wq:nested'] = True if field['type'] == 'repeat': field['wq:many'] = True continue elif field['type'] in QTYPES: field['type_info'] = QTYPES[field['type']] else: raise Exception("Unknown field type: %s" % field['type']) process_fields(xform_json) return xform_json
def runTest(self): files_to_test = ["instance_xmlns_test.xls"] for file_to_test in files_to_test: path_to_excel_file = utils.path_to_text_fixture(file_to_test) # Get the xform output path: directory, filename = os.path.split(path_to_excel_file) root_filename, ext = os.path.splitext(filename) path_to_output_xform = os.path.join( directory, root_filename + "_output.xml") path_to_expected_xform = os.path.join( directory, root_filename + ".xml") # Do the conversion: json_survey = xls2json.parse_file_to_json(path_to_excel_file) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(path_to_output_xform) # Compare with the expected output: with codecs.open(path_to_expected_xform, 'rb', encoding="utf-8" ) as expected_file: expected = ETree.fromstring(expected_file.read()) result = ETree.fromstring(survey.to_xml()) def write_line(x): sys.stdout.write(x + "\n") reporter = write_line self.assertTrue(xml_compare( expected, result, reporter=reporter)) os.remove(path_to_output_xform)
def cli(xlsform: BinaryIO) -> None: """xlson - XLSForm to native form JSON.""" survey = create_survey_element_from_dict( parse_file_to_json(xlsform.name, file_object=xlsform) ) form = create_native_form(survey.to_json_dict()) click.echo(json.dumps(form, indent=4))
def create_from_form(self, xls_form=None, original_file=None, project=None): try: with transaction.atomic(): errors = [] instance = self.model( xls_form=xls_form, original_file=original_file, project=project ) json = parse_file_to_json(instance.xls_form.file.name) has_default_language = ( 'default_language' in json and json['default_language'] != 'default' ) if (has_default_language and not check_for_language(json['default_language'])): raise InvalidXLSForm( ["Default language code '{}' unknown".format( json['default_language'] )] ) is_multilingual = multilingual_label_check(json['children']) if is_multilingual and not has_default_language: raise InvalidXLSForm(["Multilingual XLS forms must have " "a default_language setting"]) instance.default_language = json['default_language'] if instance.default_language == 'default': instance.default_language = '' instance.filename = json.get('name') instance.title = json.get('title') instance.id_string = json.get('id_string') survey = create_survey_element_from_dict(json) xml_form = survey.xml() fix_languages(xml_form) instance.save() project.current_questionnaire = instance.id project.save() create_children( children=json.get('children'), errors=errors, project=project, default_language=instance.default_language, kwargs={'questionnaire': instance} ) # all these errors handled by PyXForm so turning off for now # if errors: # raise InvalidXLSForm(errors) return instance except PyXFormError as e: raise InvalidXLSForm([str(e)])
def read_json_from_file(self, filename): try: json_data = xls2json.parse_file_to_json(filename, warnings=[]) self.stdout.write( self.style.SUCCESS('Successfully read xls file %s' % filename)) except IOError: self.stdout.write( self.style.ERROR('Failure reading xls file %s' % filename)) return json_data
def create_from_form(self, xls_form=None, original_file=None, project=None): try: with transaction.atomic(): errors = [] instance = self.model(xls_form=xls_form, original_file=original_file, project=project) json = parse_file_to_json(instance.xls_form.file.name) has_default_language = ('default_language' in json and json['default_language'] != 'default') if (has_default_language and not check_for_language(json['default_language'])): raise InvalidQuestionnaire([ "Default language code '{}' unknown".format( json['default_language']) ]) is_multilingual = multilingual_label_check(json['children']) if is_multilingual and not has_default_language: raise InvalidQuestionnaire([ "Multilingual XLS forms must have a default_language" " setting" ]) instance.default_language = json['default_language'] if instance.default_language == 'default': instance.default_language = '' instance.filename = json.get('name') instance.title = json.get('title') instance.id_string = json.get('id_string') survey = create_survey_element_from_dict(json) xml_form = survey.xml() fix_languages(xml_form) instance.save() project.current_questionnaire = instance.id create_children(children=json.get('children'), errors=errors, project=project, default_language=instance.default_language, kwargs={'questionnaire': instance}) project.save() # all these errors handled by PyXForm so turning off for now # if errors: # raise InvalidQuestionnaire(errors) return instance except PyXFormError as e: raise InvalidQuestionnaire([str(e)])
def __init__(self, path_or_file, questionnaire_name, dbm=None): self.questionnaire_name = questionnaire_name self.dbm = dbm if isinstance(path_or_file, basestring): self._file_object = None path = path_or_file else: self._file_object = path_or_file path = path_or_file.name self.xform_dict = parse_file_to_json(path, file_object=path_or_file)
def save(self, *args, **kwargs): if self.xls: default_name = None \ if not self.pk else self.survey.xml_instance().tagName try: survey_dict = parse_file_to_json(self.xls.name, default_name=default_name, file_object=self.xls) except csv.Error as e: newline_error = u'new-line character seen in unquoted field '\ u'- do you need to open the file in universal-newline '\ u'mode?' if newline_error == unicode(e): self.xls.seek(0) file_obj = StringIO(u'\n'.join( self.xls.read().splitlines())) survey_dict = parse_file_to_json(self.xls.name, default_name=default_name, file_object=file_obj) else: raise e if has_external_choices(survey_dict): self.survey_dict = survey_dict self.has_external_choices = True survey = create_survey_element_from_dict(survey_dict) survey = self._check_version_set(survey) # if form is being replaced, don't check for id_string uniqueness if self.pk is None: survey['id_string'] = self.get_unique_id_string( survey.get('id_string')) self.json = survey.to_json() self.xml = survey.to_xml() self.version = survey.get('version') self.title = survey.get('title') self._mark_start_time_boolean() set_uuid(self) self._set_uuid_in_xml() super(DataDictionary, self).save(*args, **kwargs)
def save(self, *args, **kwargs): if self.xls: default_name = None \ if not self.pk else self.survey.xml_instance().tagName try: survey_dict = parse_file_to_json( self.xls.name, default_name=default_name, file_object=self.xls) except csv.Error as e: newline_error = u'new-line character seen in unquoted field '\ u'- do you need to open the file in universal-newline '\ u'mode?' if newline_error == unicode(e): self.xls.seek(0) file_obj = StringIO( u'\n'.join(self.xls.read().splitlines())) survey_dict = parse_file_to_json( self.xls.name, default_name=default_name, file_object=file_obj) else: raise e if has_external_choices(survey_dict): self.survey_dict = survey_dict self.has_external_choices = True survey = create_survey_element_from_dict(survey_dict) survey = self._check_version_set(survey) # if form is being replaced, don't check for id_string uniqueness if self.pk is None: survey['id_string'] = self.get_unique_id_string( survey.get('id_string')) self.json = survey.to_json() self.xml = survey.to_xml() self.version = survey.get('version') self.title = survey.get('title') self._mark_start_time_boolean() set_uuid(self) self._set_uuid_in_xml() super(DataDictionary, self).save(*args, **kwargs)
def create_from_form(self, xls_form=None, original_file=None, project=None): try: with transaction.atomic(): errors = [] instance = self.model(xls_form=xls_form, original_file=original_file, project=project) json = parse_file_to_json(instance.xls_form.file.name) instance.filename = json.get('name') instance.title = json.get('title') instance.id_string = json.get('id_string') instance.version = int( datetime.utcnow().strftime('%Y%m%d%H%M%S%f')[:-4]) instance.md5_hash = self.get_hash(instance.filename, instance.id_string, instance.version) survey = create_survey_element_from_dict(json) xml_form = survey.xml().toxml() # insert version attr into the xform instance root node xml = self.insert_version_attribute(xml_form, instance.filename, instance.version) name = os.path.join(instance.xml_form.field.upload_to, os.path.basename(instance.filename)) url = instance.xml_form.storage.save('{}.xml'.format(name), xml) instance.xml_form = url instance.save() project.current_questionnaire = instance.id project.save() create_children(children=json.get('children'), errors=errors, project=project, kwargs={'questionnaire': instance}) # all these errors handled by PyXForm so turning off for now # if errors: # raise InvalidXLSForm(errors) return instance except PyXFormError as e: raise InvalidXLSForm([str(e)])
def __init__(self, path_or_file, questionnaire_name, dbm=None, excel_raw_stream=None, file_type=None): self.questionnaire_name = questionnaire_name self.dbm = dbm if excel_raw_stream is not None: self._file_object = excel_raw_stream self.path = 'questionnaire.'+file_type #Used only to deduct the extension internally in pyxform elif isinstance(path_or_file, basestring): self._file_object = None self.path = path_or_file else: self._file_object = path_or_file self.path = path_or_file.name self.xform_dict = parse_file_to_json(self.path, file_object=self._file_object)
def populate_index_fields(apps, schema_editor): Project = apps.get_model('organization', 'Project') Questionnaire = apps.get_model('questionnaires', 'Questionnaire') QuestionGroup = apps.get_model('questionnaires', 'QuestionGroup') Question = apps.get_model('questionnaires', 'Question') def update_question(idx, **kwargs): question = Question.objects.get(**kwargs) question.index = idx question.save() def update_group(idx, **kwargs): group = QuestionGroup.objects.get(**kwargs) group.index = idx group.save() return group.id def update_children(children, questionnaire_id, question_group_id=None): for child, idx in zip(children, itertools.count()): if child['type'] in ['group', 'repeat']: group_id = update_group(idx, questionnaire_id=questionnaire_id, question_group_id=question_group_id, name=child['name']) update_children(child.get('children', []), questionnaire_id, question_group_id=group_id) else: update_question(idx, questionnaire_id=questionnaire_id, question_group_id=question_group_id, name=child['name']) for project in Project.objects.all(): if project.current_questionnaire: questionnaire = Questionnaire.objects.get( id=project.current_questionnaire) if questionnaire.xls_form: try: q_json = parse_file_to_json( questionnaire.xls_form.file.name) update_children( q_json.get('children', []), questionnaire.id) except ClientError: pass
def index(request): if request.method == 'POST': # If the form has been submitted... form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data if form.is_valid(): # All validation rules pass error = None warnings = None filename, ext = os.path.splitext(request.FILES['file'].name) #Make a randomly generated directory to prevent name collisions temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR) xml_path = os.path.join(temp_dir, filename + '.xml') #Init the output xml file. fo = open(xml_path, "wb+") fo.close() try: #TODO: use the file object directly xls_path = handle_uploaded_file(request.FILES['file'], temp_dir) warnings = [] json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(xml_path, warnings=warnings) except Exception as e: error = 'Error: ' + str(e) return render_to_response('upload.html', { 'form': UploadFileForm(), 'xml_path' : '.' + xml_path, 'xml_url' : request.build_absolute_uri('.' + xml_path), 'success': not error, 'error': error, 'warnings': warnings, 'result': True, }) else: form = UploadFileForm() # An unbound form return render_to_response('upload.html', { 'form': form, })
def runTest(self): path_to_excel_file = utils.path_to_text_fixture("xlsform_spec_test.xls") #Get the xform output path: directory, filename = os.path.split(path_to_excel_file) root_filename, ext = os.path.splitext(filename) path_to_xform = os.path.join(directory, root_filename + ".xml") #Do the conversion: json_survey = xls2json.parse_file_to_json(path_to_excel_file) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(path_to_xform) #Compare with the expected output: expected_path = utils.path_to_text_fixture("spec_test_expected_output.xml") with codecs.open(expected_path, 'rb', encoding="utf-8") as expected_file: expected = etree.fromstring(expected_file.read()) result = etree.fromstring(survey.to_xml()) self.assertTrue(xml_compare(expected, result))
def parse_xls(file_or_name): if isinstance(file_or_name, str): filename = file_or_name fileobj = None else: fileobj = file_or_name filename = fileobj.name xform_json = parse_file_to_json(filename, file_object=fileobj, default_name=get_filename(filename)) # Remove 'meta' field from form xform_json['children'] = [ field for field in xform_json['children'] if field['type'] != 'group' or field['name'] != 'meta' ] def process_fields(root): for field in root['children']: cons = field.get('bind', {}).get('constraint', '') if cons.startswith('wq:'): cons = cons[3:] for ext in WQ_EXTENSIONS: if cons.startswith(ext + "(") and cons.endswith(")"): field['wq:%s' % ext] = cons[len(ext) + 1:-1] if field['type'] in GROUP_TYPES: process_fields(field) field['wq:nested'] = True if field['type'] == 'repeat': field['wq:many'] = True continue elif field['type'] in QTYPES: field['type_info'] = QTYPES[field['type']] else: raise Exception("Unknown field type: %s" % field['type']) process_fields(xform_json) return xform_json
def index(request): if request.method == 'POST': # If the form has been submitted... form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data if form.is_valid(): # All validation rules pass error = None warnings = None filename, ext = os.path.splitext(request.FILES['file'].name) if not (os.access(SERVER_TMP_DIR, os.F_OK)): os.mkdir(SERVER_TMP_DIR) #Make a randomly generated directory to prevent name collisions temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR) xml_path = os.path.join(temp_dir, filename + '.xml') itemsets_url = None relpath = os.path.relpath(xml_path, SERVER_TMP_DIR) #Init the output xml file. fo = open(xml_path, "wb+") fo.close() try: #TODO: use the file object directly xls_path = handle_uploaded_file(request.FILES['file'], temp_dir) warnings = [] json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(xml_path, warnings=warnings) if has_external_choices(json_survey): # Create a csv for the external choices itemsets_csv = os.path.join( os.path.split(xls_path)[0], "itemsets.csv") relpath_itemsets_csv = os.path.relpath( itemsets_csv, SERVER_TMP_DIR) choices_exported = sheet_to_csv(xls_path, itemsets_csv, "external_choices") if not choices_exported: warnings += [ "Could not export itemsets.csv, " "perhaps the external choices sheet is missing." ] else: itemsets_url = request.build_absolute_uri( './downloads/' + relpath_itemsets_csv) except Exception as e: error = 'Error: ' + str(e) return render_to_response( 'upload.html', { 'form': UploadFileForm(), 'xml_path': request.build_absolute_uri('./downloads/' + relpath), 'xml_url': request.build_absolute_uri('./downloads/' + relpath), 'itemsets_url': itemsets_url, 'success': not error, 'error': error, 'warnings': warnings, 'result': True, }) else: form = UploadFileForm() # An unbound form return render(request, 'upload.html', context={ 'form': form, })
def reprocess_multilingual_forms(apps, schema_editor): Questionnaire = apps.get_model('questionnaires', 'Questionnaire') for quest in Questionnaire.objects.all(): # Parse XLS form and check multilingual settings. try: json = parse_file_to_json(quest.xls_form.file.name) except: # Skip bad forms. continue has_default_language = ( 'default_language' in json and json['default_language'] != 'default' ) if (has_default_language and not check_for_language(json['default_language'])): raise InvalidXLSForm( ["Default language code '{}' unknown".format( json['default_language'] )] ) is_multilingual = multilingual_label_check(json['children']) if is_multilingual and not has_default_language: raise InvalidXLSForm(["Multilingual XLS forms must have " "a default_language setting"]) # Skip monolingual forms. No changes needed here. if not is_multilingual: continue # Skip "left over" questionnaires. if quest.project.current_questionnaire != quest.pk: continue # Set up default language: fix "default" where it crops up. quest.default_language = json['default_language'] if quest.default_language == 'default': quest.default_language = '' # Recreate XML form with correct language labels. survey = create_survey_element_from_dict(json) xml_form = survey.xml() fix_languages(xml_form) xml_form = xml_form.toxml() # insert version attr into the xform instance root node xml = insert_version_attribute( xml_form, quest.filename, quest.version ) name = os.path.join(quest.xml_form.field.upload_to, os.path.basename(quest.filename)) url = quest.xml_form.storage.save('{}.xml'.format(name), xml) quest.xml_form = url # Store updated default language and XML form information. quest.save() # Update from the top. errors = [] update_children( apps, children=json.get('children'), errors=errors, project=quest.project, default_language=quest.default_language, kwargs={'questionnaire': quest} )
def save(self, *args, **kwargs): skip_xls_read = kwargs.get('skip_xls_read') if self.xls and not skip_xls_read: default_name = None \ if not self.pk else self.survey.xml_instance().tagName try: if self.xls.name.endswith('csv'): # csv file gets closed in pyxform, make a copy self.xls.seek(0) file_object = io.BytesIO() file_object.write(self.xls.read()) file_object.seek(0) self.xls.seek(0) else: file_object = self.xls if self.xls.name.endswith('json'): survey_dict = FloipSurvey(self.xls).survey.to_json_dict() else: survey_dict = parse_file_to_json(self.xls.name, file_object=file_object) except csv.Error as e: newline_error = u'new-line character seen in unquoted field '\ u'- do you need to open the file in universal-newline '\ u'mode?' if newline_error == unicode(e): self.xls.seek(0) file_obj = StringIO(u'\n'.join( self.xls.read().splitlines())) survey_dict = parse_file_to_json(self.xls.name, default_name=default_name, file_object=file_obj) else: raise e if has_external_choices(survey_dict): self.survey_dict = survey_dict self.has_external_choices = True survey = create_survey_element_from_dict(survey_dict) survey = self._check_version_set(survey) if get_columns_with_hxl(survey.get('children')): self.has_hxl_support = True # if form is being replaced, don't check for id_string uniqueness if self.pk is None: new_id_string = self.get_unique_id_string( survey.get('id_string')) self._id_string_changed = \ new_id_string != survey.get('id_string') survey['id_string'] = new_id_string elif self.id_string != survey.get('id_string'): raise XLSFormError( _((u"Your updated form's id_string '%(new_id)s' must match " "the existing forms' id_string '%(old_id)s'." % { 'new_id': survey.get('id_string'), 'old_id': self.id_string }))) elif default_name and default_name != survey.get('name'): survey['name'] = default_name else: survey['id_string'] = self.id_string self.json = survey.to_json() self.xml = survey.to_xml() self.version = survey.get('version') self.last_updated_at = timezone.now() self.title = survey.get('title') self._mark_start_time_boolean() self._set_hash() set_uuid(self) self._set_uuid_in_xml() if 'skip_xls_read' in kwargs: del kwargs['skip_xls_read'] super(DataDictionary, self).save(*args, **kwargs)
def run(self): single_file = True start_message = 'Input file: ' + self.input_file_path + '\n\n' wx.PostEvent(self._notify_window, WorkEvent(start_message)) try: warnings = [] json_survey = xls2json.parse_file_to_json(self.input_file_path, warnings=warnings) survey = builder.create_survey_element_from_dict(json_survey) # need a temp file because print_xform_to_file automatically creates the file temp_dir = tempfile.mkdtemp() survey.print_xform_to_file(temp_dir + str(os.path.sep) + self.file_name + '.xml', validate=self.validate, warnings=warnings) if has_external_choices(json_survey): single_file = False choices_exported = sheet_to_csv(self.input_file_path, temp_dir + str(os.path.sep) + 'itemsets.csv', 'external_choices') if not choices_exported: warnings.append('Could not export itemsets.csv, perhaps the external choices sheet is missing.') if warnings: wx.PostEvent(self._notify_window, WorkEvent('ODK XLSForm Offline Warnings:\n')) # need to add whitespace to beginning to prevent truncation of forms with many warnings. for warning in warnings: # warning = warning.replace('XForm Parse Warning: Warning: ', '').replace(' ', '') wx.PostEvent(self._notify_window, WorkEvent(' ' + warning.strip() + '\n')) wx.PostEvent(self._notify_window, WorkEvent('\n')) if single_file: output_path_test = self.output_path + '.xml' output_path_template = self.output_path + ' ({0}).xml' else: output_path_test = self.output_path output_path_template = self.output_path + ' ({0})' if not self.overwrite and os.path.exists(output_path_test): # find an unused name i = 1 while os.path.exists(output_path_template.format(i)): i += 1 output_path_test = output_path_template.format(i) if single_file: shutil.copyfile(temp_dir + str(os.path.sep) + self.file_name + '.xml', output_path_test) else: if self.overwrite: shutil.rmtree(self.output_path, True) shutil.copytree(temp_dir, output_path_test) finish_message = 'Output file(s): ' + output_path_test + '\n\n' wx.PostEvent(self._notify_window, WorkEvent(finish_message)) except Exception as e: exception_text = str(e) exception_text = exception_text.replace('>> Something broke the parser. See above for a hint.', '') exception_text = exception_text.replace('Result: Invalid', '') exception_text = exception_text.replace('\n\n', '\n') validate_regex = re.compile('ODK Validate') if not (validate_regex.match(exception_text)): wx.PostEvent(self._notify_window, WorkEvent('ODK XLSForm Offline Errors:\n')) wx.PostEvent(self._notify_window, WorkEvent(exception_text.strip() + '\n')) wx.PostEvent(self._notify_window, WorkEvent('\n')) else: wx.PostEvent(self._notify_window, WorkEvent(exception_text.strip() + '\n\n')) # special message to main thread wx.PostEvent(self._notify_window, WorkEvent(main.WORKER_FINISH)) return
def index(request): if request.method == "POST": # If the form has been submitted... form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data if form.is_valid(): # All validation rules pass error = None warnings = None filename, ext = os.path.splitext(request.FILES["file"].name) if not (os.access(SERVER_TMP_DIR, os.F_OK)): os.mkdir(SERVER_TMP_DIR) # Make a randomly generated directory to prevent name collisions temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR) xml_path = os.path.join(temp_dir, filename + ".xml") itemsets_url = None relpath = os.path.relpath(xml_path, SERVER_TMP_DIR) # Init the output xml file. fo = open(xml_path, "wb+") fo.close() try: # TODO: use the file object directly xls_path = handle_uploaded_file(request.FILES["file"], temp_dir) warnings = [] json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings) survey = pyxform.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(xml_path, warnings=warnings) if has_external_choices(json_survey): # Create a csv for the external choices itemsets_csv = os.path.join(os.path.split(xls_path)[0], "itemsets.csv") relpath_itemsets_csv = os.path.relpath(itemsets_csv, SERVER_TMP_DIR) choices_exported = sheet_to_csv(xls_path, itemsets_csv, "external_choices") if not choices_exported: warnings += ["Could not export itemsets.csv, " "perhaps the external choices sheet is missing."] else: itemsets_url = request.build_absolute_uri("./downloads/" + relpath_itemsets_csv) except Exception as e: error = "Error: " + str(e) return render_to_response( "upload.html", { "form": UploadFileForm(), "xml_path": request.build_absolute_uri("./downloads/" + relpath), "xml_url": request.build_absolute_uri("./downloads/" + relpath), "itemsets_url": itemsets_url, "success": not error, "error": error, "warnings": warnings, "result": True, }, ) else: form = UploadFileForm() # An unbound form return render_to_response("upload.html", {"form": form})
def xls2json(xls_path, instance_name="data"): warnings = [] json = parse_file_to_json(xls_path, warnings=warnings) json['name'] = instance_name return (json, warnings)
def populate_additional_fields(apps, schema_editor): Project = apps.get_model('organization', 'Project') Questionnaire = apps.get_model('questionnaires', 'Questionnaire') QuestionGroup = apps.get_model('questionnaires', 'QuestionGroup') Question = apps.get_model('questionnaires', 'Question') def update_question(child, **kwargs): relevant = None required = False bind = child.get('bind') if bind: relevant = bind.get('relevant', None) required = True if bind.get('required', 'no') == 'yes' else False question = Question.objects.get(**kwargs) question.default = child.get('default', None) question.hint = child.get('hint', None) question.relevant = relevant question.required = required question.save() def update_group(child, group=None, **kwargs): group = QuestionGroup.objects.get(**kwargs) relevant = None bind = child.get('bind') if bind: relevant = bind.get('relevant', None) group.relevant = relevant group.type = child['type'] return group.id def update_children(children, questionnaire_id, question_group_id=None): for child in children: if child['type'] in ['group', 'repeat']: group_id = update_group(child, questionnaire_id=questionnaire_id, question_group_id=question_group_id, name=child['name']) update_children(child.get('children', []), questionnaire_id, question_group_id=group_id) else: update_question(child, questionnaire_id=questionnaire_id, question_group_id=question_group_id, name=child['name']) for project in Project.objects.all(): if project.current_questionnaire: questionnaire = Questionnaire.objects.get( id=project.current_questionnaire) if questionnaire.xls_form: try: q_json = parse_file_to_json( questionnaire.xls_form.file.name) update_children( q_json.get('children', []), questionnaire.id) except ClientError: pass
def reprocess_multilingual_forms(apps, schema_editor): Questionnaire = apps.get_model('questionnaires', 'Questionnaire') for quest in Questionnaire.objects.all(): # Parse XLS form and check multilingual settings. try: json = parse_file_to_json(quest.xls_form.file.name) except: # Skip bad forms. continue has_default_language = ( 'default_language' in json and json['default_language'] != 'default' ) if (has_default_language and not check_for_language(json['default_language'])): raise InvalidQuestionnaire( ["Default language code '{}' unknown".format( json['default_language'] )] ) is_multilingual = multilingual_label_check(json['children']) if is_multilingual and not has_default_language: raise InvalidQuestionnaire(["Multilingual XLS forms must have " "a default_language setting"]) # Skip monolingual forms. No changes needed here. if not is_multilingual: continue # Skip "left over" questionnaires. if quest.project.current_questionnaire != quest.pk: continue # Set up default language: fix "default" where it crops up. quest.default_language = json['default_language'] if quest.default_language == 'default': quest.default_language = '' # Recreate XML form with correct language labels. survey = create_survey_element_from_dict(json) xml_form = survey.xml() fix_languages(xml_form) xml_form = xml_form.toxml() # insert version attr into the xform instance root node xml = insert_version_attribute( xml_form, quest.filename, quest.version ) name = os.path.join(quest.xml_form.field.upload_to, os.path.basename(quest.filename)) url = quest.xml_form.storage.save('{}.xml'.format(name), xml) quest.xml_form = url # Store updated default language and XML form information. quest.save() # Update from the top. errors = [] update_children( apps, children=json.get('children'), errors=errors, project=quest.project, default_language=quest.default_language, kwargs={'questionnaire': quest} )