def _var_repl_function(self, matchobj, context, use_current=False):
"""
Given a dictionary of xpaths, return a function we can use to
replace ${varname} with the xpath to varname.
"""
name = matchobj.group(1)
intro = "There has been a problem trying to replace ${%s} with the "\
"XPath to the survey element named '%s'." % (name, name)
if name not in self._xpath:
raise PyXFormError(intro +
" There is no survey element with this name.")
if self._xpath[name] is None:
raise PyXFormError(intro + " There are multiple survey elements"
" with this name.")
if context:
# if context xpath and target xpath fall under the same repeat use
# relative xpath referencing.
steps, ref_path = share_same_repeat_parent(self, self._xpath[name],
context.get_xpath())
if steps:
ref_path = ref_path if ref_path.endswith(
name) else "/%s" % name
prefix = " current()/" if use_current else " "
return prefix + "/".join([".."] * steps) + ref_path + " "
return " " + self._xpath[name] + " "
def check(update_info):
"""
Check if the installed release of the validator works.
:type update_info: _UpdateInfo
"""
if not os.path.exists(update_info.installed_path):
message = "\nCheck failed!\n\n" \
"No installed release found."
raise PyXFormError(message)
installed = _UpdateHandler._read_json(
file_path=update_info.installed_path)
if update_info.install_check():
template = "\nCheck success!\n\n" \
"The installed release appears to work.\n\n" \
"Installed release:\n\n{installed}"
message = template.format(
installed=_UpdateHandler._get_release_message(
json_data=installed), )
log.info(message)
return True
else:
template = "\nCheck failed!\n\n" \
"The installed release does not appear to work.\n\n" \
"Installed release:\n\n{installed}"
message = template.format(
installed=_UpdateHandler._get_release_message(
json_data=installed), )
raise PyXFormError(message)
def _var_repl_function(self, matchobj, context, use_current=False):
"""
Given a dictionary of xpaths, return a function we can use to
replace ${varname} with the xpath to varname.
"""
name = matchobj.group(1)
intro = ("There has been a problem trying to replace ${%s} with the "
"XPath to the survey element named '%s'." % (name, name))
if name not in self._xpath:
raise PyXFormError(intro +
" There is no survey element with this name.")
if self._xpath[name] is None:
raise PyXFormError(intro + " There are multiple survey elements"
" with this name.")
if context and not (context["type"] == "calculate" and "indexed-repeat"
in context["bind"]["calculate"]):
xpath, context_xpath = self._xpath[name], context.get_xpath()
# share same root i.e repeat_a from /data/repeat_a/...
if xpath.split("/")[2] == context_xpath.split("/")[2]:
# if context xpath and target xpath fall under the same
# repeat use relative xpath referencing.
steps, ref_path = share_same_repeat_parent(
self, xpath, context_xpath)
if steps:
ref_path = ref_path if ref_path.endswith(
name) else "/%s" % name
prefix = " current()/" if use_current else " "
return prefix + "/".join([".."] * steps) + ref_path + " "
return " " + self._xpath[name] + " "
def _find_download_url(update_info, json_data, file_name):
"""
Find the download URL for the file in the GitHub API JSON response doc.
"""
rel_name = json_data["tag_name"]
files = json_data["assets"]
if len(files) == 0:
raise PyXFormError("No files attached to release '{r}'.\n\n{h}"
"".format(r=rel_name, h=update_info.manual_msg))
file_urls = [
x["browser_download_url"] for x in files if x["name"] == file_name
]
urls_len = len(file_urls)
if 0 == urls_len:
raise PyXFormError(
"No files with the name '{n}' attached to release '{r}'."
"\n\n{h}".format(n=file_name,
r=rel_name,
h=update_info.manual_msg))
elif 1 < urls_len:
raise PyXFormError(
"{c} files with the name '{n}' attached to release '{r}'."
"\n\n{h}".format(c=urls_len,
n=file_name,
r=rel_name,
h=update_info.manual_msg))
else:
return file_urls[0]
def update(update_info, file_name, force=False):
"""
Update to the latest version, using the specified file_name.
:type update_info: _UpdateInfo
:type file_name: str
:type force: bool
"""
if not os.path.exists(update_info.installed_path):
installed = _UpdateHandler._install(update_info=update_info,
file_name=file_name)
latest = installed
else:
installed = _UpdateHandler._read_json(
file_path=update_info.installed_path)
latest = _UpdateHandler._get_latest(update_info=update_info)
if installed["tag_name"] == latest["tag_name"] and not force:
installed_info = _UpdateHandler._get_release_message(
json_data=installed)
latest_info = _UpdateHandler._get_release_message(
json_data=latest)
template = "\nUpdate failed!\n\n" \
"The installed release appears to be the latest. " \
"To update anyway, use the '--force' flag.\n\n" \
"Installed release:\n\n{installed}" \
"Latest release:\n\n{latest}"
message = template.format(installed=installed_info,
latest=latest_info)
raise PyXFormError(message)
else:
_UpdateHandler._install(update_info=update_info,
file_name=file_name)
installed_info = _UpdateHandler._get_release_message(
json_data=installed)
latest_info = _UpdateHandler._get_release_message(json_data=latest)
new_bin_file_path = os.path.join(update_info.bin_new_path,
update_info.validator_basename)
if update_info.install_check(bin_file_path=new_bin_file_path):
_UpdateHandler._replace_old_bin_path(update_info=update_info)
template = "\nUpdate success!\n\n" \
"Install check of the latest release succeeded.\n\n" \
"Latest release:\n\n{latest}"
message = template.format(latest=latest_info)
log.info(message)
return True
else:
template = "\nUpdate failed!\n\n" \
"The latest release does not appear to work. " \
"It is saved here in case it's needed:\n{bin_new}\n\n" \
"The installed release has not been changed.\n\n" \
"Installed release:\n\n{installed}" \
"Latest release:\n\n{latest}"
message = template.format(bin_new=new_bin_file_path,
installed=installed_info,
latest=latest_info)
raise PyXFormError(message)
def process_range_question_type(row):
"""Returns a new row that includes the Range parameters start, end and
step.
Raises PyXFormError when invalid range parameters are used.
"""
def _parameters(parameters):
parts = parameters.split(';')
if len(parts) == 1:
parts = parameters.split(',')
if len(parts) == 1:
parts = parameters.split()
return parts
new_dict = row.copy()
parameters = _parameters(new_dict.get('parameters', ''))
parameters_map = {'start': 'start', 'end': 'end', 'step': 'step'}
defaults = {'start': '1', 'end': '10', 'step': '1'}
params = {}
for param in parameters:
if '=' not in param:
raise PyXFormError("Expecting parameters to be in the form of "
"'start=X end=X step=X'.")
k, v = param.split('=')[:2]
key = parameters_map.get(k.lower().strip())
if key:
params[key] = v.strip()
else:
raise PyXFormError(
"Range has the parameters 'start', 'end' and"
" 'step': '%s' is an invalid parameter." % k)
# set defaults
for key in parameters_map.values():
if key not in params:
params[key] = defaults[key]
try:
has_float = any(
[float(x) and '.' in str(x) for x in params.values()])
except ValueError:
raise PyXFormError("Range parameters 'start', "
"'end' or 'step' must all be numbers.")
else:
# is integer by default, convert to decimal if it has any float values
if has_float:
new_dict['bind'] = new_dict.get('bind', {})
new_dict['bind'].update({'type': 'decimal'})
new_dict['parameters'] = params
return new_dict
def xls_value_from_sheet(sheet, row, column):
value = sheet.cell_value(row, column)
value_type = sheet.cell_type(row, column)
if value is not None and value != "":
try:
return xls_value_to_unicode(value, value_type,
workbook.datemode)
except XLDateAmbiguous:
raise PyXFormError(XL_DATE_AMBIGOUS_MSG %
(sheet.name, column, row))
else:
raise PyXFormError("Empty Value")
def _var_repl_function(self, matchobj):
"""
Given a dictionary of xpaths, return a function we can use to
replace ${varname} with the xpath to varname.
"""
name = matchobj.group(1)
intro = "There has been a problem trying to replace ${%s} with the "\
"XPath to the survey element named '%s'." % (name, name)
if name not in self._xpath:
raise PyXFormError(intro +
" There is no survey element with this name.")
if self._xpath[name] is None:
raise PyXFormError(intro + " There are multiple survey elements"
" with this name.")
return " " + self._xpath[name] + " "
def xml(self):
"""
calls necessary preparation methods, then returns the xml.
"""
self.validate()
self._setup_xpath_dictionary()
for triggering_reference in self.setvalues_by_triggering_ref.keys():
if not (re.match(BRACKETED_TAG_REGEX, triggering_reference)):
raise PyXFormError(
"Only references to other fields are allowed in the 'trigger' column."
)
# try to resolve reference and fail if can't
self.insert_xpaths(triggering_reference, self)
body_kwargs = {}
if hasattr(self, constants.STYLE) and getattr(self, constants.STYLE):
body_kwargs["class"] = getattr(self, constants.STYLE)
nsmap = self.get_nsmap()
return node(
"h:html",
node("h:head", node("h:title", self.title), self.xml_model()),
node("h:body", *self.xml_control(), **body_kwargs),
**nsmap
)
def xls_value_from_sheet(sheet, row, column):
value = sheet.cell_value(row, column)
value_type = sheet.cell_type(row, column)
if value is not None and value != "":
return xls_value_to_unicode(value, value_type)
else:
raise PyXFormError("Empty Value")
def create_survey_element_from_dict(self, d):
"""
Convert from a nested python dictionary/array structure (a json dict I
call it because it corresponds directly with a json object)
to a survey object
"""
if "add_none_option" in d:
self._add_none_option = d["add_none_option"]
if d["type"] in self.SECTION_CLASSES:
section = self._create_section_from_dict(d)
if d["type"] == "survey":
section.setvalues_by_triggering_ref = self.setvalues_by_triggering_ref
return section
elif d["type"] == "loop":
return self._create_loop_from_dict(d)
elif d["type"] == "include":
section_name = d["name"]
if section_name not in self._sections:
raise PyXFormError(
"This section has not been included.",
section_name,
self._sections.keys(),
)
d = self._sections[section_name]
full_survey = self.create_survey_element_from_dict(d)
return full_survey.children
elif d["type"] in ["xml-external", "csv-external"]:
return ExternalInstance(**d)
else:
self._save_trigger_as_setvalue_and_remove_calculate(d)
return self._create_question_from_dict(
d, copy_json_dict(QUESTION_TYPE_DICT), self._add_none_option)
def process_range_question_type(row):
"""Returns a new row that includes the Range parameters start, end and
step.
Raises PyXFormError when invalid range parameters are used.
"""
new_dict = row.copy()
parameters = get_parameters(new_dict.get('parameters', ''),
['start', 'end', 'step'])
parameters_map = {'start': 'start', 'end': 'end', 'step': 'step'}
defaults = {'start': '1', 'end': '10', 'step': '1'}
# set defaults
for key in parameters_map.values():
if key not in parameters:
parameters[key] = defaults[key]
try:
has_float = any(
[float(x) and '.' in str(x) for x in parameters.values()])
except ValueError:
raise PyXFormError("Range parameters 'start', "
"'end' or 'step' must all be numbers.")
else:
# is integer by default, convert to decimal if it has any float values
if has_float:
new_dict['bind'] = new_dict.get('bind', {})
new_dict['bind'].update({'type': 'decimal'})
new_dict['parameters'] = parameters
return new_dict
def create_survey_element_from_dict(self, d):
"""
Convert from a nested python dictionary/array structure (a json dict I
call it because it corresponds directly with a json object)
to a survey object
"""
if u"add_none_option" in d:
self._add_none_option = d[u"add_none_option"]
if d[u"type"] in self.SECTION_CLASSES:
return self._create_section_from_dict(d)
elif d[u"type"] == u"loop":
return self._create_loop_from_dict(d)
elif d[u"type"] == u"include":
section_name = d[u"name"]
if section_name not in self._sections:
raise PyXFormError("This section has not been included.",
section_name, self._sections.keys())
d = self._sections[section_name]
full_survey = self.create_survey_element_from_dict(d)
return full_survey.children
elif d[u"type"] == u"xml-external":
return ExternalInstance(**d)
else:
return self._create_question_from_dict(
d, copy_json_dict(QUESTION_TYPE_DICT), self._add_none_option)
def get_cascading_json(sheet_list, prefix, level):
return_list = []
for row in sheet_list:
if 'stopper' in row:
if row['stopper'] == level:
# last element's name IS the prefix; doesn't need level
return_list[-1]["name"] = prefix
return return_list
else:
continue
elif 'lambda' in row:
def replace_prefix(d, prefix):
for k, v in d.items():
if isinstance(v, basestring):
d[k] = v.replace('$PREFIX$', prefix)
elif isinstance(v, dict):
d[k] = replace_prefix(v, prefix)
elif isinstance(v, list):
d[k] = map(lambda x: replace_prefix(x, prefix), v)
return d
return_list.append(replace_prefix(row['lambda'], prefix))
raise PyXFormError("Found a cascading_select " + level +
", but could not find " + level + "in cascades sheet.")
def _get_bin_paths(update_info, file_path):
"""
Get the mapping of zip file paths to extract paths for the file_name.
The zip file paths are actually glob/fnmatch patterns to find these
files among the files in the zip archive.
"""
_, file_name = os.path.split(file_path)
file_base = os.path.basename(file_name)
if "windows" in file_base:
main_bin = "*validate.exe"
elif "linux" in file_base:
main_bin = "*validate"
elif "macos" in file_base:
main_bin = "*validate"
else:
raise PyXFormError(
"Did not find a supported main binary for file: {p}.\n\n{h}"
"".format(p=file_path, h=update_info.manual_msg))
return [
(main_bin, update_info.validator_basename),
(
"*node_modules/libxmljs-mt/build*/xmljs.node",
"node_modules/libxmljs-mt/build/xmljs.node",
),
(
"*node_modules/libxslt/build*/node-libxslt.node",
"node_modules/libxslt/build/node-libxslt.node",
),
]
def validate(self):
SurveyElement.validate(self)
# make sure that the type of this question exists in the
# question type dictionary.
if self.type not in QUESTION_TYPE_DICT:
raise PyXFormError("Unknown question type '%s'." % self.type)
def _unzip_find_jobs(open_zip_file, bin_paths, out_path):
"""
For each bin file, get the zip file item file name and the output path.
Ignore files that may appear in the __MACOSX info dir, and if two files
have the same destination path and the same CRC then they're probably
duplicate files so only one of them is copied out.
"""
zip_info = open_zip_file.infolist()
zip_jobs = {}
for zip_item in zip_info:
if zip_item.filename.startswith("__MACOSX"):
continue
for file_target in bin_paths:
if fnmatch.fnmatch(zip_item.filename, file_target[0]):
file_out_path = os.path.join(out_path, file_target[1])
maybe_existing_match = zip_jobs.get(file_out_path, None)
if maybe_existing_match is not None:
if maybe_existing_match.CRC == zip_item.CRC:
continue
zip_jobs[file_out_path] = zip_item
if len(bin_paths) != len(zip_jobs.keys()):
raise PyXFormError("Expected {e} zip job files, found: {c}"
"".format(e=len(bin_paths),
c=len(zip_jobs.keys())))
return zip_jobs
def _add_other_option_to_multiple_choice_question(d):
# ideally, we'd just be pulling from children
choice_list = d.get("choices", d.get("children", []))
if len(choice_list) <= 0:
raise PyXFormError("There should be choices for this question.")
other_choice = {"name": "other", "label": "Other"}
if other_choice not in choice_list:
choice_list.append(other_choice)
def _validate_uniqueness_of_section_names(self):
root_node_name = self.name
section_names = []
for element in self.iter_descendants():
if not isinstance(element, Survey) and element.name == root_node_name:
raise PyXFormError(
'The name "%s" is the same as the form name. '
"Use a different section name "
'(or change the form name in the "name" column of the settings sheet).'
% element.name
)
if isinstance(element, Section):
if element.name in section_names:
raise PyXFormError(
"There are two sections with the name %s." % element.name
)
section_names.append(element.name)
def _validate_uniqueness_of_element_names(self):
element_slugs = []
for element in self.children:
if element.name in element_slugs:
raise PyXFormError(
"There are two survey elements named '%s' in the section"
" named '%s'." % (element.name, self.name))
element_slugs.append(element.name)
def _validate_uniqueness_of_section_names(self):
section_names = []
for e in self.iter_descendants():
if isinstance(e, Section):
if e.name in section_names:
raise PyXFormError(
"There are two sections with the name %s." % e.name)
section_names.append(e.name)
def xls_to_dict_normal_sheet(sheet):
def iswhitespace(string):
return isinstance(string, basestring) and len(string.strip()) == 0
# Check for duplicate column headers
column_header_list = list()
for column in range(0, sheet.ncols):
column_header = sheet.cell_value(0, column)
if column_header in column_header_list:
raise PyXFormError("Duplicate column header: %s" %
column_header)
# xls file with 3 columns mostly have a 3 more columns that are
# blank by default or something, skip during check
if column_header is not None:
if not iswhitespace(column_header):
# strip whitespaces from the header
clean_header = re.sub(r"( )+", " ", column_header.strip())
column_header_list.append(clean_header)
result = []
for row in range(1, sheet.nrows):
row_dict = OrderedDict()
for column in range(0, sheet.ncols):
# Changing to cell_value function
# convert to string, in case it is not string
key = "%s" % sheet.cell_value(0, column)
key = key.strip()
value = sheet.cell_value(row, column)
# remove whitespace at the beginning and end of value
if isinstance(value, basestring):
value = value.strip()
value_type = sheet.cell_type(row, column)
if value is not None:
if not iswhitespace(value):
try:
row_dict[key] = xls_value_to_unicode(
value, value_type, workbook.datemode)
except XLDateAmbiguous:
raise PyXFormError(
XL_DATE_AMBIGOUS_MSG %
(sheet.name, column_header, row))
# Taking this condition out so I can get accurate row numbers.
# TODO: Do the same for csvs
# if row_dict != {}:
result.append(row_dict)
return result, _list_to_dict_list(column_header_list)
def _validate_uniqueness_of_element_names(self):
element_slugs = []
for element in self.children:
if any(element.name.lower() == s.lower() for s in element_slugs):
raise PyXFormError(
"There are more than one survey elements named '%s' "
"(case-insensitive) in the section named '%s'." %
(element.name.lower(), self.name))
element_slugs.append(element.name)
def _validate_uniqueness_of_section_names(self):
root_node_name = self.name
section_names = []
for element in self.iter_descendants():
if isinstance(element, Section):
if element.name in section_names:
if element.name == root_node_name:
# The root node name is rarely explictly set; explain
# the problem in a more helpful way (#510)
raise PyXFormError(
'The name "%s" is the same as the form name. '
"Use a different section name "
'(or change the form name in the "name" column of '
"the settings sheet)." % element.name)
raise PyXFormError(
"There are two sections with the name %s." %
element.name)
section_names.append(element.name)
def validate(self):
if not is_valid_xml_tag(self.name):
invalid_char = re.search(INVALID_XFORM_TAG_REGEXP, self.name)
msg = ("The name '{}' is an invalid XML tag, it contains an "
"invalid character '{}'. Names must begin with a letter, "
"colon, or underscore, subsequent characters can include "
"numbers, dashes, and periods".format(
self.name, invalid_char.group(0)))
raise PyXFormError(msg)
def _validate_uniqueness_of_element_names(self):
element_slugs = set()
for element in self.children:
elem_lower = element.name.lower()
if elem_lower in element_slugs:
raise PyXFormError(
"There are more than one survey elements named '%s' "
"(case-insensitive) in the section named '%s'." %
(elem_lower, self.name))
element_slugs.add(elem_lower)
def request_get(url):
"""
Get the response content from URL.
"""
try:
r = Request(url)
r.add_header("Accept", "application/json")
with closing(urlopen(r)) as u:
content = u.read()
if len(content) == 0:
raise PyXFormError("Empty response from URL: '{u}'.".format(u=url))
else:
return content
except HTTPError as e:
raise PyXFormError("Unable to fulfill request. Error code: '{c}'. "
"Reason: '{r}'. URL: '{u}'."
"".format(r=e.reason, c=e.code, u=url))
except URLError as e:
raise PyXFormError("Unable to reach a server. Reason: {r}. "
"URL: {u}".format(r=e.reason, u=url))
def parse_file_to_workbook_dict(path, file_object=None):
"""
Given a xls or csv workbook file use xls2json_backends to create
a python workbook_dict.
workbook_dicts are organized as follows:
{sheetname : [{column_header : column_value_in_array_indexed_row}]}
"""
(filepath, filename) = os.path.split(path)
if not filename:
raise PyXFormError("No filename.")
(shortname, extension) = os.path.splitext(filename)
if not extension:
raise PyXFormError("No extension.")
if extension == ".xls" or extension == ".xlsx":
return xls_to_dict(file_object if file_object is not None else path)
elif extension == ".csv":
return csv_to_dict(file_object if file_object is not None else path)
else:
raise PyXFormError("File was not recognized")
def get_parameters(raw_parameters, allowed_parameters):
parts = raw_parameters.split(';')
if len(parts) == 1:
parts = raw_parameters.split(',')
if len(parts) == 1:
parts = raw_parameters.split()
params = {}
for param in parts:
if '=' not in param:
raise PyXFormError("Expecting parameters to be in the form of "
"'parameter1=value parameter2=value'.")
k, v = param.split('=')[:2]
key = k.lower().strip()
if key in allowed_parameters:
params[key] = v.lower().strip()
else:
raise PyXFormError("Accepted parameters are "
"'%s': '%s' is an invalid parameter." %
(", ".join(allowed_parameters), key))
return params
def _add_none_option_to_select_all_that_apply(d_copy):
choice_list = d_copy.get("choices", d_copy.get("children", []))
if len(choice_list) <= 0:
raise PyXFormError("There should be choices for this question.")
none_choice = {"name": "none", "label": "None"}
if none_choice not in choice_list:
choice_list.append(none_choice)
none_constraint = "(.='none' or not(selected(., 'none')))"
if "bind" not in d_copy:
d_copy["bind"] = {}
if "constraint" in d_copy["bind"]:
d_copy["bind"]["constraint"] += " and " + none_constraint
else:
d_copy["bind"]["constraint"] = none_constraint
