def process_xlsform(xls, default_name):
"""
Process XLSForm file and return the survey dictionary for the XLSForm.
"""
# FLOW Results package is a JSON file.
if xls.name.endswith('json'):
return FloipSurvey(xls).survey.to_json_dict()
file_object = None
if xls.name.endswith('csv'):
# a csv file gets closed in pyxform, make a copy
xls.seek(0)
file_object = BytesIO()
file_object.write(xls.read())
file_object.seek(0)
xls.seek(0)
try:
return parse_file_to_json(xls.name, file_object=file_object or xls)
except csv.Error as e:
if is_newline_error(e):
xls.seek(0)
file_object = StringIO(u'\n'.join(xls.read().splitlines()))
return parse_file_to_json(xls.name,
default_name=default_name,
file_object=file_object)
raise e
def process_xlsform(xls, default_name):
"""
Process XLSForm file and return the survey dictionary for the XLSForm.
"""
# FLOW Results package is a JSON file.
if xls.name.endswith('json'):
return FloipSurvey(xls).survey.to_json_dict()
file_object = None
if xls.name.endswith('csv'):
# a csv file gets closed in pyxform, make a copy
xls.seek(0)
file_object = BytesIO()
file_object.write(xls.read())
file_object.seek(0)
xls.seek(0)
try:
return parse_file_to_json(xls.name, file_object=file_object or xls)
except csv.Error as e:
if is_newline_error(e):
xls.seek(0)
file_object = StringIO(
u'\n'.join(xls.read().splitlines()))
return parse_file_to_json(
xls.name, default_name=default_name, file_object=file_object)
raise e
def runTest(self):
path_to_excel_file = utils.path_to_text_fixture("warnings.xls")
warnings = []
xls2json.parse_file_to_json(path_to_excel_file, warnings=warnings)
#print '\n'.join(warnings)
self.assertEquals(len(warnings), 17, "Found " + str(len(warnings)) + " warnings")
def test_underscore_warnings(self):
"""Raise warnings incase there are underscores in column names"""
warnings = []
parse_file_to_json(utils.path_to_text_fixture("hidden.xls"),
warnings=warnings)
self.assertGreater(len(warnings), 0)
warning = ("Google Sheets submissions don't allow underscores in the "
"column name. If you intend to use Google Sheets "
"submissions, replace underscores with hyphens in the "
"following names: hidden_test")
self.assertIn(warning, warnings)
def test_underscore_warnings(self):
"""Raise warnings incase there are underscores in column names"""
warnings = []
parse_file_to_json(utils.path_to_text_fixture("hidden.xls"), warnings=warnings)
self.assertGreater(len(warnings), 0)
warning = (
"Google Sheets submissions don't allow underscores in the "
"column name. If you intend to use Google Sheets "
"submissions, replace underscores with hyphens in the "
"following names: hidden_test"
)
self.assertIn(warning, warnings)
def xls2xform_convert(xlsform_path,
xform_path,
validate=True,
pretty_print=True,
enketo=False):
warnings = []
json_survey = xls2json.parse_file_to_json(xlsform_path, warnings=warnings)
survey = builder.create_survey_element_from_dict(json_survey)
# Setting validate to false will cause the form not to be processed by
# ODK Validate.
# This may be desirable since ODK Validate requires launching a subprocess
# that runs some java code.
survey.print_xform_to_file(
xform_path,
validate=validate,
pretty_print=pretty_print,
warnings=warnings,
enketo=enketo,
)
output_dir = os.path.split(xform_path)[0]
if has_external_choices(json_survey):
itemsets_csv = os.path.join(output_dir, "itemsets.csv")
choices_exported = sheet_to_csv(xlsform_path, itemsets_csv,
"external_choices")
if not choices_exported:
warnings.append("Could not export itemsets.csv, perhaps the "
"external choices sheet is missing.")
else:
logger.info("External choices csv is located at: %s", itemsets_csv)
return warnings
def runTest(self):
files_to_test = ["instance_xmlns_test.xls"]
for file_to_test in files_to_test:
path_to_excel_file = utils.path_to_text_fixture(file_to_test)
#Get the xform output path:
directory, filename = os.path.split(path_to_excel_file)
root_filename, ext = os.path.splitext(filename)
path_to_output_xform = os.path.join(directory,
root_filename + "_output.xml")
path_to_expected_xform = os.path.join(directory,
root_filename + ".xml")
#Do the conversion:
json_survey = xls2json.parse_file_to_json(path_to_excel_file)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(path_to_output_xform)
#Compare with the expected output:
with codecs.open(path_to_expected_xform, 'rb',
encoding="utf-8") as expected_file:
expected = etree.fromstring(expected_file.read())
result = etree.fromstring(survey.to_xml())
reporter = lambda x: sys.stdout.write(x + "\n")
self.assertTrue(
xml_compare(expected, result, reporter=reporter))
os.remove(path_to_output_xform)
def get_survey_dict(csv_name):
survey_file = default_storage.open(csv_name, 'rb')
survey_dict = parse_file_to_json(
survey_file.name, default_name='data', file_object=survey_file)
return survey_dict
def xls2xform_convert(
xlsform_path, xform_path, validate=True, pretty_print=True, enketo=False
):
warnings = []
json_survey = xls2json.parse_file_to_json(xlsform_path, warnings=warnings)
survey = builder.create_survey_element_from_dict(json_survey)
# Setting validate to false will cause the form not to be processed by
# ODK Validate.
# This may be desirable since ODK Validate requires launching a subprocess
# that runs some java code.
survey.print_xform_to_file(
xform_path,
validate=validate,
pretty_print=pretty_print,
warnings=warnings,
enketo=enketo,
)
output_dir = os.path.split(xform_path)[0]
if has_external_choices(json_survey):
itemsets_csv = os.path.join(output_dir, "itemsets.csv")
choices_exported = sheet_to_csv(xlsform_path, itemsets_csv, "external_choices")
if not choices_exported:
warnings.append(
"Could not export itemsets.csv, perhaps the "
"external choices sheet is missing."
)
else:
logger.info("External choices csv is located at: %s", itemsets_csv)
return warnings
def parse_xls(file_or_name):
if isinstance(file_or_name, str):
filename = file_or_name
fileobj = None
else:
fileobj = file_or_name
filename = fileobj.name
xform_json = parse_file_to_json(filename, file_object=fileobj)
def process_fields(root):
for field in root['children']:
cons = field.get('bind', {}).get('constraint', '')
if cons.startswith('wq:'):
cons = cons[3:]
for ext in WQ_EXTENSIONS:
if cons.startswith(ext + "(") and cons.endswith(")"):
field['wq:%s' % ext] = cons[len(ext) + 1:-1]
if field['type'] in GROUP_TYPES:
process_fields(field)
if not field['name'] == "meta":
field['wq:nested'] = True
if field['type'] == 'repeat':
field['wq:many'] = True
continue
elif field['type'] in QTYPES:
field['type_info'] = QTYPES[field['type']]
else:
raise Exception("Unknown field type: %s" % field['type'])
process_fields(xform_json)
return xform_json
def runTest(self):
files_to_test = ["instance_xmlns_test.xls"]
for file_to_test in files_to_test:
path_to_excel_file = utils.path_to_text_fixture(file_to_test)
# Get the xform output path:
directory, filename = os.path.split(path_to_excel_file)
root_filename, ext = os.path.splitext(filename)
path_to_output_xform = os.path.join(
directory, root_filename + "_output.xml")
path_to_expected_xform = os.path.join(
directory, root_filename + ".xml")
# Do the conversion:
json_survey = xls2json.parse_file_to_json(path_to_excel_file)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(path_to_output_xform)
# Compare with the expected output:
with codecs.open(path_to_expected_xform, 'rb', encoding="utf-8"
) as expected_file:
expected = ETree.fromstring(expected_file.read())
result = ETree.fromstring(survey.to_xml())
def write_line(x): sys.stdout.write(x + "\n")
reporter = write_line
self.assertTrue(xml_compare(
expected, result, reporter=reporter))
os.remove(path_to_output_xform)
def get_survey_dict(csv_name):
survey_file = default_storage.open(csv_name, 'rb')
survey_dict = parse_file_to_json(
survey_file.name, default_name='data', file_object=survey_file)
return survey_dict
def cli(xlsform: BinaryIO) -> None:
"""xlson - XLSForm to native form JSON."""
survey = create_survey_element_from_dict(
parse_file_to_json(xlsform.name, file_object=xlsform)
)
form = create_native_form(survey.to_json_dict())
click.echo(json.dumps(form, indent=4))
def create_from_form(self, xls_form=None, original_file=None,
project=None):
try:
with transaction.atomic():
errors = []
instance = self.model(
xls_form=xls_form,
original_file=original_file,
project=project
)
json = parse_file_to_json(instance.xls_form.file.name)
has_default_language = (
'default_language' in json and
json['default_language'] != 'default'
)
if (has_default_language and
not check_for_language(json['default_language'])):
raise InvalidXLSForm(
["Default language code '{}' unknown".format(
json['default_language']
)]
)
is_multilingual = multilingual_label_check(json['children'])
if is_multilingual and not has_default_language:
raise InvalidXLSForm(["Multilingual XLS forms must have "
"a default_language setting"])
instance.default_language = json['default_language']
if instance.default_language == 'default':
instance.default_language = ''
instance.filename = json.get('name')
instance.title = json.get('title')
instance.id_string = json.get('id_string')
survey = create_survey_element_from_dict(json)
xml_form = survey.xml()
fix_languages(xml_form)
instance.save()
project.current_questionnaire = instance.id
project.save()
create_children(
children=json.get('children'),
errors=errors,
project=project,
default_language=instance.default_language,
kwargs={'questionnaire': instance}
)
# all these errors handled by PyXForm so turning off for now
# if errors:
# raise InvalidXLSForm(errors)
return instance
except PyXFormError as e:
raise InvalidXLSForm([str(e)])
def read_json_from_file(self, filename):
try:
json_data = xls2json.parse_file_to_json(filename, warnings=[])
self.stdout.write(
self.style.SUCCESS('Successfully read xls file %s' % filename))
except IOError:
self.stdout.write(
self.style.ERROR('Failure reading xls file %s' % filename))
return json_data
def create_from_form(self,
xls_form=None,
original_file=None,
project=None):
try:
with transaction.atomic():
errors = []
instance = self.model(xls_form=xls_form,
original_file=original_file,
project=project)
json = parse_file_to_json(instance.xls_form.file.name)
has_default_language = ('default_language' in json and
json['default_language'] != 'default')
if (has_default_language
and not check_for_language(json['default_language'])):
raise InvalidQuestionnaire([
"Default language code '{}' unknown".format(
json['default_language'])
])
is_multilingual = multilingual_label_check(json['children'])
if is_multilingual and not has_default_language:
raise InvalidQuestionnaire([
"Multilingual XLS forms must have a default_language"
" setting"
])
instance.default_language = json['default_language']
if instance.default_language == 'default':
instance.default_language = ''
instance.filename = json.get('name')
instance.title = json.get('title')
instance.id_string = json.get('id_string')
survey = create_survey_element_from_dict(json)
xml_form = survey.xml()
fix_languages(xml_form)
instance.save()
project.current_questionnaire = instance.id
create_children(children=json.get('children'),
errors=errors,
project=project,
default_language=instance.default_language,
kwargs={'questionnaire': instance})
project.save()
# all these errors handled by PyXForm so turning off for now
# if errors:
# raise InvalidQuestionnaire(errors)
return instance
except PyXFormError as e:
raise InvalidQuestionnaire([str(e)])
def __init__(self, path_or_file, questionnaire_name, dbm=None):
self.questionnaire_name = questionnaire_name
self.dbm = dbm
if isinstance(path_or_file, basestring):
self._file_object = None
path = path_or_file
else:
self._file_object = path_or_file
path = path_or_file.name
self.xform_dict = parse_file_to_json(path, file_object=path_or_file)
def save(self, *args, **kwargs):
if self.xls:
default_name = None \
if not self.pk else self.survey.xml_instance().tagName
try:
survey_dict = parse_file_to_json(self.xls.name,
default_name=default_name,
file_object=self.xls)
except csv.Error as e:
newline_error = u'new-line character seen in unquoted field '\
u'- do you need to open the file in universal-newline '\
u'mode?'
if newline_error == unicode(e):
self.xls.seek(0)
file_obj = StringIO(u'\n'.join(
self.xls.read().splitlines()))
survey_dict = parse_file_to_json(self.xls.name,
default_name=default_name,
file_object=file_obj)
else:
raise e
if has_external_choices(survey_dict):
self.survey_dict = survey_dict
self.has_external_choices = True
survey = create_survey_element_from_dict(survey_dict)
survey = self._check_version_set(survey)
# if form is being replaced, don't check for id_string uniqueness
if self.pk is None:
survey['id_string'] = self.get_unique_id_string(
survey.get('id_string'))
self.json = survey.to_json()
self.xml = survey.to_xml()
self.version = survey.get('version')
self.title = survey.get('title')
self._mark_start_time_boolean()
set_uuid(self)
self._set_uuid_in_xml()
super(DataDictionary, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
if self.xls:
default_name = None \
if not self.pk else self.survey.xml_instance().tagName
try:
survey_dict = parse_file_to_json(
self.xls.name, default_name=default_name,
file_object=self.xls)
except csv.Error as e:
newline_error = u'new-line character seen in unquoted field '\
u'- do you need to open the file in universal-newline '\
u'mode?'
if newline_error == unicode(e):
self.xls.seek(0)
file_obj = StringIO(
u'\n'.join(self.xls.read().splitlines()))
survey_dict = parse_file_to_json(
self.xls.name, default_name=default_name,
file_object=file_obj)
else:
raise e
if has_external_choices(survey_dict):
self.survey_dict = survey_dict
self.has_external_choices = True
survey = create_survey_element_from_dict(survey_dict)
survey = self._check_version_set(survey)
# if form is being replaced, don't check for id_string uniqueness
if self.pk is None:
survey['id_string'] = self.get_unique_id_string(
survey.get('id_string'))
self.json = survey.to_json()
self.xml = survey.to_xml()
self.version = survey.get('version')
self.title = survey.get('title')
self._mark_start_time_boolean()
set_uuid(self)
self._set_uuid_in_xml()
super(DataDictionary, self).save(*args, **kwargs)
def create_from_form(self,
xls_form=None,
original_file=None,
project=None):
try:
with transaction.atomic():
errors = []
instance = self.model(xls_form=xls_form,
original_file=original_file,
project=project)
json = parse_file_to_json(instance.xls_form.file.name)
instance.filename = json.get('name')
instance.title = json.get('title')
instance.id_string = json.get('id_string')
instance.version = int(
datetime.utcnow().strftime('%Y%m%d%H%M%S%f')[:-4])
instance.md5_hash = self.get_hash(instance.filename,
instance.id_string,
instance.version)
survey = create_survey_element_from_dict(json)
xml_form = survey.xml().toxml()
# insert version attr into the xform instance root node
xml = self.insert_version_attribute(xml_form,
instance.filename,
instance.version)
name = os.path.join(instance.xml_form.field.upload_to,
os.path.basename(instance.filename))
url = instance.xml_form.storage.save('{}.xml'.format(name),
xml)
instance.xml_form = url
instance.save()
project.current_questionnaire = instance.id
project.save()
create_children(children=json.get('children'),
errors=errors,
project=project,
kwargs={'questionnaire': instance})
# all these errors handled by PyXForm so turning off for now
# if errors:
# raise InvalidXLSForm(errors)
return instance
except PyXFormError as e:
raise InvalidXLSForm([str(e)])
def __init__(self, path_or_file, questionnaire_name, dbm=None, excel_raw_stream=None, file_type=None):
self.questionnaire_name = questionnaire_name
self.dbm = dbm
if excel_raw_stream is not None:
self._file_object = excel_raw_stream
self.path = 'questionnaire.'+file_type #Used only to deduct the extension internally in pyxform
elif isinstance(path_or_file, basestring):
self._file_object = None
self.path = path_or_file
else:
self._file_object = path_or_file
self.path = path_or_file.name
self.xform_dict = parse_file_to_json(self.path, file_object=self._file_object)
def populate_index_fields(apps, schema_editor):
Project = apps.get_model('organization', 'Project')
Questionnaire = apps.get_model('questionnaires', 'Questionnaire')
QuestionGroup = apps.get_model('questionnaires', 'QuestionGroup')
Question = apps.get_model('questionnaires', 'Question')
def update_question(idx, **kwargs):
question = Question.objects.get(**kwargs)
question.index = idx
question.save()
def update_group(idx, **kwargs):
group = QuestionGroup.objects.get(**kwargs)
group.index = idx
group.save()
return group.id
def update_children(children, questionnaire_id, question_group_id=None):
for child, idx in zip(children, itertools.count()):
if child['type'] in ['group', 'repeat']:
group_id = update_group(idx,
questionnaire_id=questionnaire_id,
question_group_id=question_group_id,
name=child['name'])
update_children(child.get('children', []),
questionnaire_id,
question_group_id=group_id)
else:
update_question(idx,
questionnaire_id=questionnaire_id,
question_group_id=question_group_id,
name=child['name'])
for project in Project.objects.all():
if project.current_questionnaire:
questionnaire = Questionnaire.objects.get(
id=project.current_questionnaire)
if questionnaire.xls_form:
try:
q_json = parse_file_to_json(
questionnaire.xls_form.file.name)
update_children(
q_json.get('children', []), questionnaire.id)
except ClientError:
pass
def index(request):
if request.method == 'POST': # If the form has been submitted...
form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data
if form.is_valid(): # All validation rules pass
error = None
warnings = None
filename, ext = os.path.splitext(request.FILES['file'].name)
#Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR)
xml_path = os.path.join(temp_dir, filename + '.xml')
#Init the output xml file.
fo = open(xml_path, "wb+")
fo.close()
try:
#TODO: use the file object directly
xls_path = handle_uploaded_file(request.FILES['file'], temp_dir)
warnings = []
json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(xml_path, warnings=warnings)
except Exception as e:
error = 'Error: ' + str(e)
return render_to_response('upload.html', {
'form': UploadFileForm(),
'xml_path' : '.' + xml_path,
'xml_url' : request.build_absolute_uri('.' + xml_path),
'success': not error,
'error': error,
'warnings': warnings,
'result': True,
})
else:
form = UploadFileForm() # An unbound form
return render_to_response('upload.html', {
'form': form,
})
def runTest(self):
path_to_excel_file = utils.path_to_text_fixture("xlsform_spec_test.xls")
#Get the xform output path:
directory, filename = os.path.split(path_to_excel_file)
root_filename, ext = os.path.splitext(filename)
path_to_xform = os.path.join(directory, root_filename + ".xml")
#Do the conversion:
json_survey = xls2json.parse_file_to_json(path_to_excel_file)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(path_to_xform)
#Compare with the expected output:
expected_path = utils.path_to_text_fixture("spec_test_expected_output.xml")
with codecs.open(expected_path, 'rb', encoding="utf-8") as expected_file:
expected = etree.fromstring(expected_file.read())
result = etree.fromstring(survey.to_xml())
self.assertTrue(xml_compare(expected, result))
def parse_xls(file_or_name):
if isinstance(file_or_name, str):
filename = file_or_name
fileobj = None
else:
fileobj = file_or_name
filename = fileobj.name
xform_json = parse_file_to_json(filename,
file_object=fileobj,
default_name=get_filename(filename))
# Remove 'meta' field from form
xform_json['children'] = [
field for field in xform_json['children']
if field['type'] != 'group' or field['name'] != 'meta'
]
def process_fields(root):
for field in root['children']:
cons = field.get('bind', {}).get('constraint', '')
if cons.startswith('wq:'):
cons = cons[3:]
for ext in WQ_EXTENSIONS:
if cons.startswith(ext + "(") and cons.endswith(")"):
field['wq:%s' % ext] = cons[len(ext) + 1:-1]
if field['type'] in GROUP_TYPES:
process_fields(field)
field['wq:nested'] = True
if field['type'] == 'repeat':
field['wq:many'] = True
continue
elif field['type'] in QTYPES:
field['type_info'] = QTYPES[field['type']]
else:
raise Exception("Unknown field type: %s" % field['type'])
process_fields(xform_json)
return xform_json
def index(request):
if request.method == 'POST': # If the form has been submitted...
form = UploadFileForm(request.POST,
request.FILES) # A form bound to the POST data
if form.is_valid(): # All validation rules pass
error = None
warnings = None
filename, ext = os.path.splitext(request.FILES['file'].name)
if not (os.access(SERVER_TMP_DIR, os.F_OK)):
os.mkdir(SERVER_TMP_DIR)
#Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR)
xml_path = os.path.join(temp_dir, filename + '.xml')
itemsets_url = None
relpath = os.path.relpath(xml_path, SERVER_TMP_DIR)
#Init the output xml file.
fo = open(xml_path, "wb+")
fo.close()
try:
#TODO: use the file object directly
xls_path = handle_uploaded_file(request.FILES['file'],
temp_dir)
warnings = []
json_survey = xls2json.parse_file_to_json(xls_path,
warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(xml_path, warnings=warnings)
if has_external_choices(json_survey):
# Create a csv for the external choices
itemsets_csv = os.path.join(
os.path.split(xls_path)[0], "itemsets.csv")
relpath_itemsets_csv = os.path.relpath(
itemsets_csv, SERVER_TMP_DIR)
choices_exported = sheet_to_csv(xls_path, itemsets_csv,
"external_choices")
if not choices_exported:
warnings += [
"Could not export itemsets.csv, "
"perhaps the external choices sheet is missing."
]
else:
itemsets_url = request.build_absolute_uri(
'./downloads/' + relpath_itemsets_csv)
except Exception as e:
error = 'Error: ' + str(e)
return render_to_response(
'upload.html', {
'form':
UploadFileForm(),
'xml_path':
request.build_absolute_uri('./downloads/' + relpath),
'xml_url':
request.build_absolute_uri('./downloads/' + relpath),
'itemsets_url':
itemsets_url,
'success':
not error,
'error':
error,
'warnings':
warnings,
'result':
True,
})
else:
form = UploadFileForm() # An unbound form
return render(request, 'upload.html', context={
'form': form,
})
def reprocess_multilingual_forms(apps, schema_editor):
Questionnaire = apps.get_model('questionnaires', 'Questionnaire')
for quest in Questionnaire.objects.all():
# Parse XLS form and check multilingual settings.
try:
json = parse_file_to_json(quest.xls_form.file.name)
except:
# Skip bad forms.
continue
has_default_language = (
'default_language' in json and
json['default_language'] != 'default'
)
if (has_default_language and
not check_for_language(json['default_language'])):
raise InvalidXLSForm(
["Default language code '{}' unknown".format(
json['default_language']
)]
)
is_multilingual = multilingual_label_check(json['children'])
if is_multilingual and not has_default_language:
raise InvalidXLSForm(["Multilingual XLS forms must have "
"a default_language setting"])
# Skip monolingual forms. No changes needed here.
if not is_multilingual:
continue
# Skip "left over" questionnaires.
if quest.project.current_questionnaire != quest.pk:
continue
# Set up default language: fix "default" where it crops up.
quest.default_language = json['default_language']
if quest.default_language == 'default':
quest.default_language = ''
# Recreate XML form with correct language labels.
survey = create_survey_element_from_dict(json)
xml_form = survey.xml()
fix_languages(xml_form)
xml_form = xml_form.toxml()
# insert version attr into the xform instance root node
xml = insert_version_attribute(
xml_form, quest.filename, quest.version
)
name = os.path.join(quest.xml_form.field.upload_to,
os.path.basename(quest.filename))
url = quest.xml_form.storage.save('{}.xml'.format(name), xml)
quest.xml_form = url
# Store updated default language and XML form information.
quest.save()
# Update from the top.
errors = []
update_children(
apps,
children=json.get('children'),
errors=errors,
project=quest.project,
default_language=quest.default_language,
kwargs={'questionnaire': quest}
)
def save(self, *args, **kwargs):
skip_xls_read = kwargs.get('skip_xls_read')
if self.xls and not skip_xls_read:
default_name = None \
if not self.pk else self.survey.xml_instance().tagName
try:
if self.xls.name.endswith('csv'):
# csv file gets closed in pyxform, make a copy
self.xls.seek(0)
file_object = io.BytesIO()
file_object.write(self.xls.read())
file_object.seek(0)
self.xls.seek(0)
else:
file_object = self.xls
if self.xls.name.endswith('json'):
survey_dict = FloipSurvey(self.xls).survey.to_json_dict()
else:
survey_dict = parse_file_to_json(self.xls.name,
file_object=file_object)
except csv.Error as e:
newline_error = u'new-line character seen in unquoted field '\
u'- do you need to open the file in universal-newline '\
u'mode?'
if newline_error == unicode(e):
self.xls.seek(0)
file_obj = StringIO(u'\n'.join(
self.xls.read().splitlines()))
survey_dict = parse_file_to_json(self.xls.name,
default_name=default_name,
file_object=file_obj)
else:
raise e
if has_external_choices(survey_dict):
self.survey_dict = survey_dict
self.has_external_choices = True
survey = create_survey_element_from_dict(survey_dict)
survey = self._check_version_set(survey)
if get_columns_with_hxl(survey.get('children')):
self.has_hxl_support = True
# if form is being replaced, don't check for id_string uniqueness
if self.pk is None:
new_id_string = self.get_unique_id_string(
survey.get('id_string'))
self._id_string_changed = \
new_id_string != survey.get('id_string')
survey['id_string'] = new_id_string
elif self.id_string != survey.get('id_string'):
raise XLSFormError(
_((u"Your updated form's id_string '%(new_id)s' must match "
"the existing forms' id_string '%(old_id)s'." % {
'new_id': survey.get('id_string'),
'old_id': self.id_string
})))
elif default_name and default_name != survey.get('name'):
survey['name'] = default_name
else:
survey['id_string'] = self.id_string
self.json = survey.to_json()
self.xml = survey.to_xml()
self.version = survey.get('version')
self.last_updated_at = timezone.now()
self.title = survey.get('title')
self._mark_start_time_boolean()
self._set_hash()
set_uuid(self)
self._set_uuid_in_xml()
if 'skip_xls_read' in kwargs:
del kwargs['skip_xls_read']
super(DataDictionary, self).save(*args, **kwargs)
def run(self):
single_file = True
start_message = 'Input file: ' + self.input_file_path + '\n\n'
wx.PostEvent(self._notify_window, WorkEvent(start_message))
try:
warnings = []
json_survey = xls2json.parse_file_to_json(self.input_file_path, warnings=warnings)
survey = builder.create_survey_element_from_dict(json_survey)
# need a temp file because print_xform_to_file automatically creates the file
temp_dir = tempfile.mkdtemp()
survey.print_xform_to_file(temp_dir + str(os.path.sep) + self.file_name + '.xml', validate=self.validate,
warnings=warnings)
if has_external_choices(json_survey):
single_file = False
choices_exported = sheet_to_csv(self.input_file_path, temp_dir + str(os.path.sep) + 'itemsets.csv',
'external_choices')
if not choices_exported:
warnings.append('Could not export itemsets.csv, perhaps the external choices sheet is missing.')
if warnings:
wx.PostEvent(self._notify_window, WorkEvent('ODK XLSForm Offline Warnings:\n'))
# need to add whitespace to beginning to prevent truncation of forms with many warnings.
for warning in warnings:
# warning = warning.replace('XForm Parse Warning: Warning: ', '').replace(' ', '')
wx.PostEvent(self._notify_window, WorkEvent(' ' + warning.strip() + '\n'))
wx.PostEvent(self._notify_window, WorkEvent('\n'))
if single_file:
output_path_test = self.output_path + '.xml'
output_path_template = self.output_path + ' ({0}).xml'
else:
output_path_test = self.output_path
output_path_template = self.output_path + ' ({0})'
if not self.overwrite and os.path.exists(output_path_test):
# find an unused name
i = 1
while os.path.exists(output_path_template.format(i)):
i += 1
output_path_test = output_path_template.format(i)
if single_file:
shutil.copyfile(temp_dir + str(os.path.sep) + self.file_name + '.xml', output_path_test)
else:
if self.overwrite:
shutil.rmtree(self.output_path, True)
shutil.copytree(temp_dir, output_path_test)
finish_message = 'Output file(s): ' + output_path_test + '\n\n'
wx.PostEvent(self._notify_window, WorkEvent(finish_message))
except Exception as e:
exception_text = str(e)
exception_text = exception_text.replace('>> Something broke the parser. See above for a hint.', '')
exception_text = exception_text.replace('Result: Invalid', '')
exception_text = exception_text.replace('\n\n', '\n')
validate_regex = re.compile('ODK Validate')
if not (validate_regex.match(exception_text)):
wx.PostEvent(self._notify_window, WorkEvent('ODK XLSForm Offline Errors:\n'))
wx.PostEvent(self._notify_window, WorkEvent(exception_text.strip() + '\n'))
wx.PostEvent(self._notify_window, WorkEvent('\n'))
else:
wx.PostEvent(self._notify_window, WorkEvent(exception_text.strip() + '\n\n'))
# special message to main thread
wx.PostEvent(self._notify_window, WorkEvent(main.WORKER_FINISH))
return
def index(request):
if request.method == "POST": # If the form has been submitted...
form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data
if form.is_valid(): # All validation rules pass
error = None
warnings = None
filename, ext = os.path.splitext(request.FILES["file"].name)
if not (os.access(SERVER_TMP_DIR, os.F_OK)):
os.mkdir(SERVER_TMP_DIR)
# Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(dir=SERVER_TMP_DIR)
xml_path = os.path.join(temp_dir, filename + ".xml")
itemsets_url = None
relpath = os.path.relpath(xml_path, SERVER_TMP_DIR)
# Init the output xml file.
fo = open(xml_path, "wb+")
fo.close()
try:
# TODO: use the file object directly
xls_path = handle_uploaded_file(request.FILES["file"], temp_dir)
warnings = []
json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(xml_path, warnings=warnings)
if has_external_choices(json_survey):
# Create a csv for the external choices
itemsets_csv = os.path.join(os.path.split(xls_path)[0], "itemsets.csv")
relpath_itemsets_csv = os.path.relpath(itemsets_csv, SERVER_TMP_DIR)
choices_exported = sheet_to_csv(xls_path, itemsets_csv, "external_choices")
if not choices_exported:
warnings += ["Could not export itemsets.csv, " "perhaps the external choices sheet is missing."]
else:
itemsets_url = request.build_absolute_uri("./downloads/" + relpath_itemsets_csv)
except Exception as e:
error = "Error: " + str(e)
return render_to_response(
"upload.html",
{
"form": UploadFileForm(),
"xml_path": request.build_absolute_uri("./downloads/" + relpath),
"xml_url": request.build_absolute_uri("./downloads/" + relpath),
"itemsets_url": itemsets_url,
"success": not error,
"error": error,
"warnings": warnings,
"result": True,
},
)
else:
form = UploadFileForm() # An unbound form
return render_to_response("upload.html", {"form": form})
def xls2json(xls_path, instance_name="data"):
warnings = []
json = parse_file_to_json(xls_path, warnings=warnings)
json['name'] = instance_name
return (json, warnings)
def populate_additional_fields(apps, schema_editor):
Project = apps.get_model('organization', 'Project')
Questionnaire = apps.get_model('questionnaires', 'Questionnaire')
QuestionGroup = apps.get_model('questionnaires', 'QuestionGroup')
Question = apps.get_model('questionnaires', 'Question')
def update_question(child, **kwargs):
relevant = None
required = False
bind = child.get('bind')
if bind:
relevant = bind.get('relevant', None)
required = True if bind.get('required', 'no') == 'yes' else False
question = Question.objects.get(**kwargs)
question.default = child.get('default', None)
question.hint = child.get('hint', None)
question.relevant = relevant
question.required = required
question.save()
def update_group(child, group=None, **kwargs):
group = QuestionGroup.objects.get(**kwargs)
relevant = None
bind = child.get('bind')
if bind:
relevant = bind.get('relevant', None)
group.relevant = relevant
group.type = child['type']
return group.id
def update_children(children, questionnaire_id, question_group_id=None):
for child in children:
if child['type'] in ['group', 'repeat']:
group_id = update_group(child,
questionnaire_id=questionnaire_id,
question_group_id=question_group_id,
name=child['name'])
update_children(child.get('children', []),
questionnaire_id,
question_group_id=group_id)
else:
update_question(child,
questionnaire_id=questionnaire_id,
question_group_id=question_group_id,
name=child['name'])
for project in Project.objects.all():
if project.current_questionnaire:
questionnaire = Questionnaire.objects.get(
id=project.current_questionnaire)
if questionnaire.xls_form:
try:
q_json = parse_file_to_json(
questionnaire.xls_form.file.name)
update_children(
q_json.get('children', []), questionnaire.id)
except ClientError:
pass
def reprocess_multilingual_forms(apps, schema_editor):
Questionnaire = apps.get_model('questionnaires', 'Questionnaire')
for quest in Questionnaire.objects.all():
# Parse XLS form and check multilingual settings.
try:
json = parse_file_to_json(quest.xls_form.file.name)
except:
# Skip bad forms.
continue
has_default_language = (
'default_language' in json and
json['default_language'] != 'default'
)
if (has_default_language and
not check_for_language(json['default_language'])):
raise InvalidQuestionnaire(
["Default language code '{}' unknown".format(
json['default_language']
)]
)
is_multilingual = multilingual_label_check(json['children'])
if is_multilingual and not has_default_language:
raise InvalidQuestionnaire(["Multilingual XLS forms must have "
"a default_language setting"])
# Skip monolingual forms. No changes needed here.
if not is_multilingual:
continue
# Skip "left over" questionnaires.
if quest.project.current_questionnaire != quest.pk:
continue
# Set up default language: fix "default" where it crops up.
quest.default_language = json['default_language']
if quest.default_language == 'default':
quest.default_language = ''
# Recreate XML form with correct language labels.
survey = create_survey_element_from_dict(json)
xml_form = survey.xml()
fix_languages(xml_form)
xml_form = xml_form.toxml()
# insert version attr into the xform instance root node
xml = insert_version_attribute(
xml_form, quest.filename, quest.version
)
name = os.path.join(quest.xml_form.field.upload_to,
os.path.basename(quest.filename))
url = quest.xml_form.storage.save('{}.xml'.format(name), xml)
quest.xml_form = url
# Store updated default language and XML form information.
quest.save()
# Update from the top.
errors = []
update_children(
apps,
children=json.get('children'),
errors=errors,
project=quest.project,
default_language=quest.default_language,
kwargs={'questionnaire': quest}
)
