def parse(self):
fields = self.xform_dict['children']
errors = self._validate_fields_are_recognised(fields)
settings_page_errors = self._validate_settings_page_is_not_present()
errors = errors.union(settings_page_errors)
choice_errors = self._validate_media_in_choices(fields)
[errors.add(choice_error) for choice_error in choice_errors if choice_error]
choice_name_errors = self._validate_choice_names(fields)
errors = errors.union(set(choice_name_errors))
self._identify_default_language()
questions, question_errors, info = self._create_questions(fields)
if question_errors:
errors = errors.union(question_errors)
if not errors and not questions:
errors.add("Uploaded file is empty!")
if errors:
return XlsParserResponse(errors)
_map_unique_id_question_to_select_one(self.xform_dict)
survey = create_survey_element_from_dict(self.xform_dict)
itemsets_csv = None
if has_external_choices(self.xform_dict):
itemsets_csv = NamedTemporaryFile(delete=False, suffix=".csv")
choices_exported = self.sheet_to_csv(self.path, itemsets_csv.name, "external_choices")
if not choices_exported:
errors.add("Could not export itemsets.csv, perhaps the external choices sheet is missing.")
else:
print 'External choices csv is located at:', itemsets_csv
xform = survey.to_xml()
# encoding is added to support ie8
xform = re.sub(r'<\?xml version="1.0"\?>', '<?xml version="1.0" encoding="utf-8"?>', xform)
updated_xform = self.update_xform_with_questionnaire_name(xform)
return XlsParserResponse([], updated_xform, questions, info, self._is_multi_language(), itemsets_csv)
def test_choice_name_as_type(self):
filename = "choice_name_as_type.xls"
path_to_excel_file = os.path.join(DIR, "example_xls", filename)
xls_reader = SurveyReader(path_to_excel_file,
default_name="choice_name_as_type")
survey_dict = xls_reader.to_json_dict()
self.assertTrue(has_external_choices(survey_dict))
def xls2xform_convert(xlsform_path,
xform_path,
validate=True,
pretty_print=True,
enketo=False):
warnings = []
json_survey = xls2json.parse_file_to_json(xlsform_path, warnings=warnings)
survey = builder.create_survey_element_from_dict(json_survey)
# Setting validate to false will cause the form not to be processed by
# ODK Validate.
# This may be desirable since ODK Validate requires launching a subprocess
# that runs some java code.
survey.print_xform_to_file(
xform_path,
validate=validate,
pretty_print=pretty_print,
warnings=warnings,
enketo=enketo,
)
output_dir = os.path.split(xform_path)[0]
if has_external_choices(json_survey):
itemsets_csv = os.path.join(output_dir, "itemsets.csv")
choices_exported = sheet_to_csv(xlsform_path, itemsets_csv,
"external_choices")
if not choices_exported:
warnings.append("Could not export itemsets.csv, perhaps the "
"external choices sheet is missing.")
else:
logger.info("External choices csv is located at: %s", itemsets_csv)
return warnings
def xls2xform_convert(
xlsform_path, xform_path, validate=True, pretty_print=True, enketo=False
):
warnings = []
json_survey = xls2json.parse_file_to_json(xlsform_path, warnings=warnings)
survey = builder.create_survey_element_from_dict(json_survey)
# Setting validate to false will cause the form not to be processed by
# ODK Validate.
# This may be desirable since ODK Validate requires launching a subprocess
# that runs some java code.
survey.print_xform_to_file(
xform_path,
validate=validate,
pretty_print=pretty_print,
warnings=warnings,
enketo=enketo,
)
output_dir = os.path.split(xform_path)[0]
if has_external_choices(json_survey):
itemsets_csv = os.path.join(output_dir, "itemsets.csv")
choices_exported = sheet_to_csv(xlsform_path, itemsets_csv, "external_choices")
if not choices_exported:
warnings.append(
"Could not export itemsets.csv, perhaps the "
"external choices sheet is missing."
)
else:
logger.info("External choices csv is located at: %s", itemsets_csv)
return warnings
def xls2zip(work_dir,
xls_path,
xform_name="form.xml",
items_name="itemsets.csv",
zip_name="form.zip",
formatted=False,
validated=True):
xform_path = join(work_dir, xform_name)
items_path = join(work_dir, items_name)
zip_path = join(work_dir, zip_name)
(json, json_warnings) = xls2json(xls_path)
survey = json2survey(json)
xform_warnings = survey2xform(survey,
xform_path,
formatted=formatted,
validated=validated)
items_warnings = []
if has_external_choices(json):
items_warnings = _xls2itemset(xls_path, items_path)
write_zip(zip_path, xform_path, items_path)
return (zip_path, json_warnings + xform_warnings + items_warnings)
def save(self, *args, **kwargs):
skip_xls_read = kwargs.get('skip_xls_read')
if self.xls and not skip_xls_read:
default_name = None \
if not self.pk else self.survey.xml_instance().tagName
survey_dict = process_xlsform(self.xls, default_name)
if has_external_choices(survey_dict):
self.has_external_choices = True
survey = create_survey_element_from_dict(survey_dict)
survey = check_version_set(survey)
if get_columns_with_hxl(survey.get('children')):
self.has_hxl_support = True
# if form is being replaced, don't check for id_string uniqueness
if self.pk is None:
new_id_string = self.get_unique_id_string(
survey.get('id_string'))
self._id_string_changed = \
new_id_string != survey.get('id_string')
survey['id_string'] = new_id_string
# For flow results packages use the user defined id/uuid
if self.xls.name.endswith('json'):
self.uuid = FloipSurvey(self.xls).descriptor.get('id')
if self.uuid:
check_xform_uuid(self.uuid)
elif self.id_string != survey.get('id_string'):
raise XLSFormError(
_((u"Your updated form's id_string '%(new_id)s' must match "
"the existing forms' id_string '%(old_id)s'." % {
'new_id': survey.get('id_string'),
'old_id': self.id_string
})))
elif default_name and default_name != survey.get('name'):
survey['name'] = default_name
else:
survey['id_string'] = self.id_string
self.json = survey.to_json()
self.xml = survey.to_xml()
self.version = survey.get('version')
self.last_updated_at = timezone.now()
self.title = survey.get('title')
self._mark_start_time_boolean()
set_uuid(self)
self._set_uuid_in_xml()
self._set_hash()
if 'skip_xls_read' in kwargs:
del kwargs['skip_xls_read']
super(DataDictionary, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
skip_xls_read = kwargs.get('skip_xls_read')
if self.xls and not skip_xls_read:
default_name = None \
if not self.pk else self.survey.xml_instance().tagName
survey_dict = process_xlsform(self.xls, default_name)
if has_external_choices(survey_dict):
self.has_external_choices = True
survey = create_survey_element_from_dict(survey_dict)
survey = check_version_set(survey)
if get_columns_with_hxl(survey.get('children')):
self.has_hxl_support = True
# if form is being replaced, don't check for id_string uniqueness
if self.pk is None:
new_id_string = self.get_unique_id_string(
survey.get('id_string'))
self._id_string_changed = \
new_id_string != survey.get('id_string')
survey['id_string'] = new_id_string
# For flow results packages use the user defined id/uuid
if self.xls.name.endswith('json'):
self.uuid = FloipSurvey(self.xls).descriptor.get('id')
if self.uuid:
check_xform_uuid(self.uuid)
elif self.id_string != survey.get('id_string'):
raise XLSFormError(_(
(u"Your updated form's id_string '%(new_id)s' must match "
"the existing forms' id_string '%(old_id)s'." % {
'new_id': survey.get('id_string'),
'old_id': self.id_string})))
elif default_name and default_name != survey.get('name'):
survey['name'] = default_name
else:
survey['id_string'] = self.id_string
self.json = survey.to_json()
self.xml = survey.to_xml()
self.version = survey.get('version')
self.last_updated_at = timezone.now()
self.title = survey.get('title')
self._mark_start_time_boolean()
set_uuid(self)
self._set_uuid_in_xml()
self._set_hash()
if 'skip_xls_read' in kwargs:
del kwargs['skip_xls_read']
super(DataDictionary, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
if self.xls:
# check if version is set
excel_reader = SurveyReader(self.xls)
survey_dict = excel_reader.to_json_dict()
if has_external_choices(survey_dict):
self.survey_dict = survey_dict
self.has_external_choices = True
survey = create_survey_element_from_dict(survey_dict)
survey = self._check_version_set(survey)
self.json = survey.to_json()
self.xml = survey.to_xml()
self.version = survey.get('version')
self._mark_start_time_boolean()
set_uuid(self)
self._set_uuid_in_xml()
super(DataDictionary, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
if self.xls:
default_name = None \
if not self.pk else self.survey.xml_instance().tagName
try:
survey_dict = parse_file_to_json(self.xls.name,
default_name=default_name,
file_object=self.xls)
except csv.Error as e:
newline_error = u'new-line character seen in unquoted field '\
u'- do you need to open the file in universal-newline '\
u'mode?'
if newline_error == unicode(e):
self.xls.seek(0)
file_obj = StringIO(u'\n'.join(
self.xls.read().splitlines()))
survey_dict = parse_file_to_json(self.xls.name,
default_name=default_name,
file_object=file_obj)
else:
raise e
if has_external_choices(survey_dict):
self.survey_dict = survey_dict
self.has_external_choices = True
survey = create_survey_element_from_dict(survey_dict)
survey = self._check_version_set(survey)
# if form is being replaced, don't check for id_string uniqueness
if self.pk is None:
survey['id_string'] = self.get_unique_id_string(
survey.get('id_string'))
self.json = survey.to_json()
self.xml = survey.to_xml()
self.version = survey.get('version')
self.title = survey.get('title')
self._mark_start_time_boolean()
set_uuid(self)
self._set_uuid_in_xml()
super(DataDictionary, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
if self.xls:
default_name = None \
if not self.pk else self.survey.xml_instance().tagName
try:
survey_dict = parse_file_to_json(
self.xls.name, default_name=default_name,
file_object=self.xls)
except csv.Error as e:
newline_error = u'new-line character seen in unquoted field '\
u'- do you need to open the file in universal-newline '\
u'mode?'
if newline_error == unicode(e):
self.xls.seek(0)
file_obj = StringIO(
u'\n'.join(self.xls.read().splitlines()))
survey_dict = parse_file_to_json(
self.xls.name, default_name=default_name,
file_object=file_obj)
else:
raise e
if has_external_choices(survey_dict):
self.survey_dict = survey_dict
self.has_external_choices = True
survey = create_survey_element_from_dict(survey_dict)
survey = self._check_version_set(survey)
# if form is being replaced, don't check for id_string uniqueness
if self.pk is None:
survey['id_string'] = self.get_unique_id_string(
survey.get('id_string'))
self.json = survey.to_json()
self.xml = survey.to_xml()
self.version = survey.get('version')
self.title = survey.get('title')
self._mark_start_time_boolean()
set_uuid(self)
self._set_uuid_in_xml()
super(DataDictionary, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
skip_xls_read = kwargs.get('skip_xls_read')
if self.xls and not skip_xls_read:
default_name = None \
if not self.pk else self.survey.xml_instance().tagName
try:
if self.xls.name.endswith('csv'):
# csv file gets closed in pyxform, make a copy
self.xls.seek(0)
file_object = io.BytesIO()
file_object.write(self.xls.read())
file_object.seek(0)
self.xls.seek(0)
else:
file_object = self.xls
if self.xls.name.endswith('json'):
survey_dict = FloipSurvey(self.xls).survey.to_json_dict()
else:
survey_dict = parse_file_to_json(self.xls.name,
file_object=file_object)
except csv.Error as e:
newline_error = u'new-line character seen in unquoted field '\
u'- do you need to open the file in universal-newline '\
u'mode?'
if newline_error == unicode(e):
self.xls.seek(0)
file_obj = StringIO(u'\n'.join(
self.xls.read().splitlines()))
survey_dict = parse_file_to_json(self.xls.name,
default_name=default_name,
file_object=file_obj)
else:
raise e
if has_external_choices(survey_dict):
self.survey_dict = survey_dict
self.has_external_choices = True
survey = create_survey_element_from_dict(survey_dict)
survey = self._check_version_set(survey)
if get_columns_with_hxl(survey.get('children')):
self.has_hxl_support = True
# if form is being replaced, don't check for id_string uniqueness
if self.pk is None:
new_id_string = self.get_unique_id_string(
survey.get('id_string'))
self._id_string_changed = \
new_id_string != survey.get('id_string')
survey['id_string'] = new_id_string
elif self.id_string != survey.get('id_string'):
raise XLSFormError(
_((u"Your updated form's id_string '%(new_id)s' must match "
"the existing forms' id_string '%(old_id)s'." % {
'new_id': survey.get('id_string'),
'old_id': self.id_string
})))
elif default_name and default_name != survey.get('name'):
survey['name'] = default_name
else:
survey['id_string'] = self.id_string
self.json = survey.to_json()
self.xml = survey.to_xml()
self.version = survey.get('version')
self.last_updated_at = timezone.now()
self.title = survey.get('title')
self._mark_start_time_boolean()
self._set_hash()
set_uuid(self)
self._set_uuid_in_xml()
if 'skip_xls_read' in kwargs:
del kwargs['skip_xls_read']
super(DataDictionary, self).save(*args, **kwargs)
def test_choice_name_as_type(self):
filename = "choice_name_as_type.xls"
path_to_excel_file = os.path.join(DIR, "example_xls", filename)
xls_reader = SurveyReader(path_to_excel_file)
survey_dict = xls_reader.to_json_dict()
self.assertTrue(has_external_choices(survey_dict))
argv = sys.argv
if len(argv) < 2:
print __doc__
print "Usage:"
print argv[0] + " path_to_XLSForm"
else:
warnings = []
name, ext = os.path.splitext(os.path.basename(argv[1]))
output_dir = os.path.join(os.path.split(argv[0])[0], "output")
out_file = os.path.join(output_dir, name + ".xml")
if not os.path.exists(output_dir):
os.makedirs(output_dir)
json_survey = pyxform.xls2json.parse_file_to_json(argv[1], warnings=warnings)
survey = pyxform.builder.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(out_file, validate=False)
if has_external_choices(json_survey):
itemsets_csv = os.path.join(output_dir, "itemsets.csv")
choices_exported = sheet_to_csv(argv[1], itemsets_csv, "external_choices")
if not choices_exported:
print "Could not export itemsets.csv, perhaps the external choices sheet is missing."
else:
print "external choices csv is located at:", itemsets_csv
if len(warnings) > 0:
print "Warnings:"
for w in warnings:
print w
print "xform is located at:", out_file
print "Conversion complete!"
except Exception as e:
print e
raw_input("Press Enter to continue...")
def run(self):
single_file = True
start_message = 'Input file: ' + self.input_file_path + '\n\n'
wx.PostEvent(self._notify_window, WorkEvent(start_message))
try:
warnings = []
json_survey = xls2json.parse_file_to_json(self.input_file_path, warnings=warnings)
survey = builder.create_survey_element_from_dict(json_survey)
# need a temp file because print_xform_to_file automatically creates the file
temp_dir = tempfile.mkdtemp()
survey.print_xform_to_file(temp_dir + str(os.path.sep) + self.file_name + '.xml', validate=self.validate,
warnings=warnings)
if has_external_choices(json_survey):
single_file = False
choices_exported = sheet_to_csv(self.input_file_path, temp_dir + str(os.path.sep) + 'itemsets.csv',
'external_choices')
if not choices_exported:
warnings.append('Could not export itemsets.csv, perhaps the external choices sheet is missing.')
if warnings:
wx.PostEvent(self._notify_window, WorkEvent('ODK XLSForm Offline Warnings:\n'))
# need to add whitespace to beginning to prevent truncation of forms with many warnings.
for warning in warnings:
# warning = warning.replace('XForm Parse Warning: Warning: ', '').replace(' ', '')
wx.PostEvent(self._notify_window, WorkEvent(' ' + warning.strip() + '\n'))
wx.PostEvent(self._notify_window, WorkEvent('\n'))
if single_file:
output_path_test = self.output_path + '.xml'
output_path_template = self.output_path + ' ({0}).xml'
else:
output_path_test = self.output_path
output_path_template = self.output_path + ' ({0})'
if not self.overwrite and os.path.exists(output_path_test):
# find an unused name
i = 1
while os.path.exists(output_path_template.format(i)):
i += 1
output_path_test = output_path_template.format(i)
if single_file:
shutil.copyfile(temp_dir + str(os.path.sep) + self.file_name + '.xml', output_path_test)
else:
if self.overwrite:
shutil.rmtree(self.output_path, True)
shutil.copytree(temp_dir, output_path_test)
finish_message = 'Output file(s): ' + output_path_test + '\n\n'
wx.PostEvent(self._notify_window, WorkEvent(finish_message))
except Exception as e:
exception_text = str(e)
exception_text = exception_text.replace('>> Something broke the parser. See above for a hint.', '')
exception_text = exception_text.replace('Result: Invalid', '')
exception_text = exception_text.replace('\n\n', '\n')
validate_regex = re.compile('ODK Validate')
if not (validate_regex.match(exception_text)):
wx.PostEvent(self._notify_window, WorkEvent('ODK XLSForm Offline Errors:\n'))
wx.PostEvent(self._notify_window, WorkEvent(exception_text.strip() + '\n'))
wx.PostEvent(self._notify_window, WorkEvent('\n'))
else:
wx.PostEvent(self._notify_window, WorkEvent(exception_text.strip() + '\n\n'))
# special message to main thread
wx.PostEvent(self._notify_window, WorkEvent(main.WORKER_FINISH))
return
def index(request):
if request.method == 'POST': # If the form has been submitted...
form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data
if form.is_valid(): # All validation rules pass
error = None
warnings = None
filename, ext = os.path.splitext(request.FILES['file'].name)
if not (os.access(DJANGO_TMP_HOME, os.F_OK)):
os.mkdir(DJANGO_TMP_HOME)
#Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(prefix='', dir=DJANGO_TMP_HOME)
xml_path = os.path.join(temp_dir, filename + '.xml')
itemsets_url = None
relpath = os.path.relpath(xml_path, DJANGO_TMP_HOME)
#Init the output xml file.
fo = open(xml_path, "wb+")
fo.close()
try:
#TODO: use the file object directly
xls_path = handle_uploaded_file(request.FILES['file'], temp_dir)
warnings = []
json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(xml_path, warnings=warnings, pretty_print=False)
if has_external_choices(json_survey):
# Create a csv for the external choices
itemsets_csv = os.path.join(os.path.split(xls_path)[0],
"itemsets.csv")
relpath_itemsets_csv = os.path.relpath(itemsets_csv, DJANGO_TMP_HOME)
choices_exported = sheet_to_csv(xls_path, itemsets_csv,
"external_choices")
if not choices_exported:
warnings += ["Could not export itemsets.csv, "
"perhaps the external choices sheet is missing."]
else:
itemsets_url = request.build_absolute_uri('./downloads/' + relpath_itemsets_csv)
except Exception as e:
error = 'Error: ' + str(e)
return render(request, 'upload.html', {
'form': UploadFileForm(),
'xml_path': request.build_absolute_uri('./downloads/' + relpath),
'xml_url': request.build_absolute_uri('./downloads/' + relpath),
'itemsets_url': itemsets_url,
'success': not error,
'error': error,
'warnings': warnings,
'result': True,
})
else:
form = UploadFileForm() # An unbound form
return render(request, 'upload.html', context={
'form': form,
})
print __doc__
print 'Usage:'
print argv[0] + ' path_to_XLSForm'
else:
warnings = []
name, ext = os.path.splitext(os.path.basename(argv[1]))
output_dir = os.path.join(os.path.split(argv[0])[0], 'output')
out_file = os.path.join(output_dir, name + '.xml')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
json_survey = pyxform.xls2json.parse_file_to_json(
argv[1], warnings=warnings)
survey = pyxform.builder.create_survey_element_from_dict(
json_survey)
survey.print_xform_to_file(out_file, validate=False)
if has_external_choices(json_survey):
itemsets_csv = os.path.join(output_dir, "itemsets.csv")
choices_exported = sheet_to_csv(argv[1], itemsets_csv,
"external_choices")
if not choices_exported:
print "Could not export itemsets.csv, perhaps the external choices sheet is missing."
else:
print 'external choices csv is located at:', itemsets_csv
if len(warnings) > 0:
print "Warnings:"
for w in warnings:
print w
print 'xform is located at:', out_file
print 'Conversion complete!'
except Exception as e:
print e