def _load_all_functions(self):
if self.all_functions is None:
self.all_functions = dict()
my_package = '{0}/__init__.py'.format(os.path.dirname(__file__))
verbose_print(
'FunctionSupport loading all function modules near {0}.'.
format(my_package),
indent_after=True)
for importer, modname, ispkg in walk_packages(my_package):
if not ispkg and modname.startswith('hq.hquery.functions.'):
verbose_print(
'Found candidate module {0} -- loading.'.format(
modname))
module = importer.find_module(modname).load_module(modname)
if hasattr(module, 'exports'):
exports = {
name.rstrip('_'): getattr(module, name)
for name in getattr(module, 'exports')
}
verbose_print('Module {0} exports are: {1}'.format(
modname, exports.keys()))
if any(not (isclass(obj) or isfunction(obj))
for obj in exports.values()):
raise RuntimeError(
'Non-class/function export(s) loaded from module {0}'
.format(modname))
self.all_functions.update(exports)
else:
verbose_print(
'Module {0} defined no exports.'.format(modname))
verbose_print('Finished loading function modules.',
outdent_before=True)
def parse_interpolated_string(source, parse_interface):
verbose_print(u'Parsing interpolated string contents `{0}`'.format(source), indent_after=True)
expressions = []
for embedded_expr, embedded_var, literal in clauses_pattern.findall(source):
if embedded_expr:
verbose_print(u'Adding embedded expression: {0}'.format(embedded_expr))
expressions.append(reduce_filters_and_expression(embedded_expr[2:-1], parse_interface))
elif embedded_var:
verbose_print('Adding embedded variable reference: {0}'.format(embedded_var))
expressions.append(parse_interface.parse_in_new_processor(embedded_var))
else:
verbose_print(u'Adding literal string contents `{0}`'.format(literal))
expressions.append(_make_literal_identity_closure(literal))
def evaluate():
chunks = [string_value(exp()) for exp in expressions]
verbose_print(u'Interpolated string evaluation assembling {0} chunks{1}.'.format(
len(chunks),
'' if len(chunks) == 0 else u' ("{0}")'.format(u'", "'.join(chunks)))
)
return ''.join(chunks)
verbose_print(
u'Finished parsing interpolated string `{0}` ({1} chunk(s) found)'.format(debug_dump_long_string(source),
len(expressions)),
outdent_before=True
)
return evaluate
def evaluate(self):
verbose_print('Evaluating union decomposition ({} clauses)'.format(
len(self.mapping_generators)),
indent_after=True)
sequence = make_sequence(self.union_expression())
result = []
for item in sequence:
verbose_print(
lambda: u'Visiting item {0}'.format(debug_dump_anything(item)),
indent_after=True)
with variable_scope():
push_variable('_', make_sequence(item))
if not hasattr(item, 'union_index'):
raise HqueryEvaluationError(
"Union decomposition applied to something that wasn't produced by a union"
)
if item.union_index >= len(self.mapping_generators):
raise HqueryEvaluationError(
"Decomposed union had more clauses than its mapping")
this_result = make_sequence(
self.mapping_generators[item.union_index]())
verbose_print(
'Mapping yielded {0} results for this visit'.format(
len(this_result)))
result = sequence_concat(result, this_result)
verbose_print('Visit finished', outdent_before=True)
verbose_print('Union decomposition completed', outdent_before=True)
return result
def _evaluate_without_iteration(self):
with variable_scope():
self._push_global_variables()
verbose_print('Evaluating return expression.', indent_after=True)
result = self.return_expression()
verbose_print('Return expression produced {0}'.format(str(result)), outdent_before=True)
return result
def evaluate(self):
verbose_print('Evaluating union decomposition ({} clauses)'.format(len(self.mapping_generators)),
indent_after=True)
sequence = make_sequence(self.union_expression())
result = []
for item in sequence:
verbose_print(lambda: u'Visiting item {0}'.format(debug_dump_anything(item)), indent_after=True)
with variable_scope():
push_variable('_', make_sequence(item))
if not hasattr(item, 'union_index'):
raise HqueryEvaluationError(
"Union decomposition applied to something that wasn't produced by a union"
)
if item.union_index >= len(self.mapping_generators):
raise HqueryEvaluationError("Decomposed union had more clauses than its mapping")
this_result = make_sequence(self.mapping_generators[item.union_index]())
verbose_print(
'Mapping yielded {0} results for this visit'.format(
len(this_result)))
result = sequence_concat(result, this_result)
verbose_print('Visit finished', outdent_before=True)
verbose_print('Union decomposition completed', outdent_before=True)
return result
def evaluate():
chunks = [string_value(exp()) for exp in expressions]
verbose_print(u'Interpolated string evaluation assembling {0} chunks{1}.'.format(
len(chunks),
'' if len(chunks) == 0 else u' ("{0}")'.format(u'", "'.join(chunks)))
)
return ''.join(chunks)
def _evaluate_without_iteration(self):
with variable_scope():
self._push_global_variables()
verbose_print('Evaluating return expression.', indent_after=True)
result = self.return_expression()
verbose_print('Return expression produced {0}'.format(str(result)),
outdent_before=True)
return result
def evaluate(hash):
for key, value in hash.items():
if key in tag_names:
if not isinstance(value, list):
verbose_print(
'JSON hash constructor array filter converting attribute "{0}" to array'
.format(key))
hash[key] = [value]
def evaluate(hash):
for key, value in hash.items():
if key in mappings:
verbose_print(
'JSON hash constructor mapping filter converting attribute name "{0}" to "{1}"'
.format(key, value))
hash[mappings[key]] = hash[key]
del hash[key]
def push_context(node, position=1, size=1, preserve_space=None):
msg = u'Pushing (node={0}, position={1}, size={2} on context stack.'
verbose_print(lambda: msg.format(debug_dump_node(node), position, size))
context_stack.append(
ExpressionContext(node=node,
position=position,
size=size,
preserve_space=preserve_space))
def evaluate(self):
verbose_print('Evaluating value expression for constructed hash key "{0}"'.format(self.key), indent_after=True)
value = self.value_fn()
msg = u'Finished evaluating; value of constructed hash key "{0}" is {1}'
verbose_print(lambda: msg.format(self.key, debug_dump_anything(value)), outdent_before=True)
return HashKeyValue(self.key, value)
def evaluate(hash):
for key, value in hash.items():
if key in tag_names:
verbose_print(
'JSON hash constructor number filter converting attribute "{0}" value(s) to numbers'.format(key)
)
if isinstance(value, list):
hash[key] = [number(v).value for v in value]
else:
hash[key] = number(value).value
def value_of_variable(name):
if len(variable_stack) > 0:
for index in range(len(variable_stack) - 1, -1, -1):
if variable_stack[index][NAME] == name:
reverse_index = len(variable_stack) - (index + 1)
verbose_print('Variable "${0}" found on stack (position {1}).'.format(name, reverse_index))
return variable_stack[index][VALUE]
verbose_print('Variable "${0}" NOT FOUND on variable stack.'.format(name))
return None
def evaluate(hash):
for key, value in hash.items():
if key in tag_names:
verbose_print(
'JSON hash constructor number filter converting attribute "{0}" value(s) to numbers'
.format(key))
if isinstance(value, list):
hash[key] = [number(v).value for v in value]
else:
hash[key] = number(value).value
def evaluate(self):
verbose_print('Evaluating FLWOR {0}'.format(self), indent_after=True)
if self.sequence_expression is not None:
result = self._evaluate_iteration()
else:
result = self._evaluate_without_iteration()
verbose_print(lambda: 'FLWOR evaluation completed; returning {0}'.format(debug_dump_anything(result)),
outdent_before=True)
return result
def evaluate(self):
verbose_print('Evaluating FLWOR {0}'.format(self), indent_after=True)
if self.sequence_expression is not None:
result = self._evaluate_iteration()
else:
result = self._evaluate_without_iteration()
verbose_print(lambda: 'FLWOR evaluation completed; returning {0}'.
format(debug_dump_anything(result)),
outdent_before=True)
return result
def evaluate(self):
verbose_print(
'Evaluating value expression for constructed hash key "{0}"'.
format(self.key),
indent_after=True)
value = self.value_fn()
msg = u'Finished evaluating; value of constructed hash key "{0}" is {1}'
verbose_print(lambda: msg.format(self.key, debug_dump_anything(value)),
outdent_before=True)
return HashKeyValue(self.key, value)
def _evaluate_iteration(self):
with variable_scope():
self._push_global_variables()
sequence = make_sequence(self.sequence_expression())
verbose_print(
'Iterating over sequence containing {0} items'.format(
len(sequence)))
result = []
for item in sequence:
verbose_print(lambda: u'Visiting item {0}'.format(
debug_dump_anything(item)),
indent_after=True)
with variable_scope():
push_variable(self.sequence_variable, make_sequence(item))
self._push_iteration_variables()
this_result = make_sequence(self.return_expression())
verbose_print(
'Return clause yielded {0} results for this visit'.
format(len(this_result)))
result = sequence_concat(result, this_result)
verbose_print('Visit finished', outdent_before=True)
return result
def accept_context_node():
context = peek_context()
format_str = u'Evaluating predicate expression for context node at position {0} of {1}: {2}.'
verbose_print(lambda: format_str.format(context.position, context.size, debug_dump_node(context.node)))
value = expression_fn()
if is_number(value):
accept = number(context.position) == value
else:
accept = bool(value)
verbose_print(lambda: u'{0} node {1}'.format('Accepted' if accept else 'Rejected',
debug_dump_node(context.node)))
return [context.node] if accept else []
def _eq_node_set_vs_string(nodes_val, string_val):
string_val = str(string_val)
verbose_print(u'(=) comparing number "{0}" to {1} nodes'.format(string_val, len(nodes_val)))
for node in nodes_val:
node_val_string = string_value(node)
verbose_print(u'(=) node string value "{0}" is{1} equal to "{2}"'.format(
node_val_string,
('' if node_val_string == string_val else ' not'),
string_val))
if node_val_string == string_val:
return True
return False
def _eq_node_set_vs_number(nodes_val, num_val):
verbose_print('(=) comparing number {0} to {1} nodes'.format(num_val, len(nodes_val)))
for node in nodes_val:
node_str_val = string_value(node)
node_num_val = number(node_str_val)
verbose_print('(=) node string value "{0}" is{1} equal to "{2}"'.format(
node_num_val,
(' not' if node_num_val == num_val else ''),
num_val))
if node_num_val == num_val:
return True
return False
def _load_all_functions(self):
if self.all_functions is None:
self.all_functions = dict()
my_package = '{0}/__init__.py'.format(os.path.dirname(__file__))
verbose_print('FunctionSupport loading all function modules near {0}.'.format(my_package),
indent_after=True)
for importer, modname, ispkg in walk_packages(my_package):
if not ispkg and modname.startswith('hq.hquery.functions.'):
verbose_print('Found candidate module {0} -- loading.'.format(modname))
module = importer.find_module(modname).load_module(modname)
if hasattr(module, 'exports'):
exports = {name.rstrip('_'): getattr(module, name) for name in getattr(module, 'exports')}
verbose_print('Module {0} exports are: {1}'.format(modname, exports.keys()))
if any(not (isclass(obj) or isfunction(obj)) for obj in exports.values()):
raise RuntimeError('Non-class/function export(s) loaded from module {0}'.format(modname))
self.all_functions.update(exports)
else:
verbose_print('Module {0} defined no exports.'.format(modname))
verbose_print('Finished loading function modules.', outdent_before=True)
def _evaluate_steps(self, remaining_steps):
step = remaining_steps[0]
verbose_print(lambda: 'Evaluating step {0}'.format(remaining_steps[0]), indent_after=True)
result_set = make_node_set(step.node_test.apply(step.axis, get_context_node()),
reverse=step.axis.is_reverse_order())
verbose_print(lambda: 'Axis and node test produced {0} matching nodes'.format(len(result_set)))
for index, expression_fn in enumerate(step.predicates):
def accept_context_node():
context = peek_context()
format_str = u'Evaluating predicate expression for context node at position {0} of {1}: {2}.'
verbose_print(lambda: format_str.format(context.position, context.size, debug_dump_node(context.node)))
value = expression_fn()
if is_number(value):
accept = number(context.position) == value
else:
accept = bool(value)
verbose_print(lambda: u'{0} node {1}'.format('Accepted' if accept else 'Rejected',
debug_dump_node(context.node)))
return [context.node] if accept else []
verbose_print(lambda: 'Evaluating predicate #{0} against {1} nodes'.format(index + 1, len(result_set)),
indent_after=True)
result_set = evaluate_across_contexts(result_set, accept_context_node)
verbose_print(
lambda: 'Evaluation of predicate #{0} complete; accepted {1} nodes.'.format(index + 1, len(result_set)),
outdent_before=True)
if len(remaining_steps) > 1:
result_set = evaluate_across_contexts(result_set, lambda: self._evaluate_steps(remaining_steps[1:]))
verbose_print(lambda: 'Step evaluation completed; returning {0} nodes.'.format(len(result_set)),
outdent_before=True)
return result_set
def _cmp_value_to_nodes(base_op, first, second):
node_values = set([number(node) for node in second])
first = number(first)
verbose_print('Comparing {0} nodes in node set to value "{1}"'.format(len(node_values), first))
for node_value in node_values:
if base_op(first, node_value):
verbose_print('Comparison succeeded for value "{0}" and node value "{1}'.format(first, node_value))
return True
verbose_print('Comparison failed for all nodes in the node set.')
return False
def _cmp_nodes_to_value(base_op, first, second):
node_values = set([number(node) for node in first])
second = number(second)
verbose_print('Comparing {0} nodes in node set to value {1}'.format(len(node_values), second))
for node_value in node_values:
if base_op(node_value, second):
verbose_print('Comparison succeeded for node value "{0}" and value "{1}"'.format(node_value, second))
return True
verbose_print('Comparison failed for all nodes in the node set.')
return False
def _eq_node_sets(first, second):
first_values = set([string_value(node) for node in first])
second_values = set([string_value(node) for node in second])
verbose_print('Comparing two nodes sets (size {0} and {1}).'.format(len(first_values), len(second_values)))
for first_value in first_values:
if first_value in second_values:
verbose_print(u'Found value "{0}" from first node set in second node set'.format(first_value))
return True
verbose_print('Found no matching nodes between node sets.')
return False
def _evaluate_iteration(self):
with variable_scope():
self._push_global_variables()
sequence = make_sequence(self.sequence_expression())
verbose_print('Iterating over sequence containing {0} items'.format(len(sequence)))
result = []
for item in sequence:
verbose_print(lambda: u'Visiting item {0}'.format(debug_dump_anything(item)), indent_after=True)
with variable_scope():
push_variable(self.sequence_variable, make_sequence(item))
self._push_iteration_variables()
this_result = make_sequence(self.return_expression())
verbose_print('Return clause yielded {0} results for this visit'.format(len(this_result)))
result = sequence_concat(result, this_result)
verbose_print('Visit finished', outdent_before=True)
return result
def _cmp_node_sets(base_op, first, second):
first_values = set([number(node) for node in first])
second_values = set([number(node) for node in second])
verbose_print('Comparing two nodes sets (size {0} and {1}).'.format(len(first_values), len(second_values)))
for first_value in first_values:
for second_value in second_values:
if base_op(first_value, second_value):
msg = 'Comparison succeeded for "{0}" from first node set and "{1}" in second node set'
verbose_print(msg.format(first_value, second_value))
return True
verbose_print('Comparison failed for all nodes in both node sets.')
return False
def evaluate(self):
verbose_print(lambda: 'Evaluating location path {0}'.format(self.debug_dump()), indent_after=True)
if self.absolute:
verbose_print('Switching context to root because this path is absolute.')
results = evaluate_in_context(soup_from_any_tag(get_context_node()),
lambda: self._evaluate_steps(self.steps))
elif self.root_expression is not None:
results = evaluate_across_contexts(self.root_expression(), lambda: self._evaluate_steps(self.steps))
else:
results = self._evaluate_steps(self.steps)
verbose_print('Evaluation completed; location path selected {0} nodes'.format(len(results)),
outdent_before=True)
return make_node_set(results, reverse=False)
def _gab(self, message):
verbose_print('JSON array constructor {0}'.format(message))
def _eq_bool_vs_primitive(bool_val, other_val):
verbose_print('Comparing boolean value {0} with non-node-set value {1} (coerced to {2})'.format(bool_val, other_val, boolean(other_val)))
return bool_val == boolean(other_val)
def _gab(self, message):
verbose_print('JSON hash constructor {0}'.format(message))
def pop_context():
result = context_stack.pop()
msg = u'Popping (node={0}, position={1}, size={2} off of context stack.'
verbose_print(lambda: msg.format(debug_dump_node(result.node), result.
position, result.size))
return result
def push_variable(name, value):
global variable_stack
verbose_print(lambda: u'Pushing variable onto stack: let ${0} := {1}'.format(name, debug_dump_anything(value)))
variable_stack.append((name, value))
def push_context(node, position=1, size=1, preserve_space=None):
msg = u'Pushing (node={0}, position={1}, size={2} on context stack.'
verbose_print(lambda: msg.format(debug_dump_node(node), position, size))
context_stack.append(ExpressionContext(node=node, position=position, size=size, preserve_space=preserve_space))
def pop_context():
result = context_stack.pop()
msg = u'Popping (node={0}, position={1}, size={2} off of context stack.'
verbose_print(lambda: msg.format(debug_dump_node(result.node), result.position, result.size))
return result
def _push_iteration_variables(self):
for let in self.per_iteration_variables:
verbose_print('Evaluating let {0} := <expr>'.format(let[0]))
push_variable(let[0], let[1]())
def _push_global_variables(self):
for let in self.global_variables:
verbose_print('Evaluating let {0} := <expr>'.format(let[0]))
push_variable(let[0], let[1]())
def evaluate(hash):
for key, value in hash.items():
if key in tag_names:
if not isinstance(value, list):
verbose_print('JSON hash constructor array filter converting attribute "{0}" to array'.format(key))
hash[key] = [value]
def _push_global_variables(self):
for let in self.global_variables:
verbose_print('Evaluating let {0} := <expr>'.format(let[0]))
push_variable(let[0], let[1]())
def _push_iteration_variables(self):
for let in self.per_iteration_variables:
verbose_print('Evaluating let {0} := <expr>'.format(let[0]))
push_variable(let[0], let[1]())
def _gab(self, msg, **kwargs):
verbose_print(u'{0} {1}'.format(self, msg), **kwargs)
def evaluate(hash):
for key, value in hash.items():
if key in mappings:
verbose_print('JSON hash constructor mapping filter converting attribute name "{0}" to "{1}"'.format(key, value))
hash[mappings[key]] = hash[key]
del hash[key]
