def _load_all_functions(self): if self.all_functions is None: self.all_functions = dict() my_package = '{0}/__init__.py'.format(os.path.dirname(__file__)) verbose_print( 'FunctionSupport loading all function modules near {0}.'. format(my_package), indent_after=True) for importer, modname, ispkg in walk_packages(my_package): if not ispkg and modname.startswith('hq.hquery.functions.'): verbose_print( 'Found candidate module {0} -- loading.'.format( modname)) module = importer.find_module(modname).load_module(modname) if hasattr(module, 'exports'): exports = { name.rstrip('_'): getattr(module, name) for name in getattr(module, 'exports') } verbose_print('Module {0} exports are: {1}'.format( modname, exports.keys())) if any(not (isclass(obj) or isfunction(obj)) for obj in exports.values()): raise RuntimeError( 'Non-class/function export(s) loaded from module {0}' .format(modname)) self.all_functions.update(exports) else: verbose_print( 'Module {0} defined no exports.'.format(modname)) verbose_print('Finished loading function modules.', outdent_before=True)
def parse_interpolated_string(source, parse_interface): verbose_print(u'Parsing interpolated string contents `{0}`'.format(source), indent_after=True) expressions = [] for embedded_expr, embedded_var, literal in clauses_pattern.findall(source): if embedded_expr: verbose_print(u'Adding embedded expression: {0}'.format(embedded_expr)) expressions.append(reduce_filters_and_expression(embedded_expr[2:-1], parse_interface)) elif embedded_var: verbose_print('Adding embedded variable reference: {0}'.format(embedded_var)) expressions.append(parse_interface.parse_in_new_processor(embedded_var)) else: verbose_print(u'Adding literal string contents `{0}`'.format(literal)) expressions.append(_make_literal_identity_closure(literal)) def evaluate(): chunks = [string_value(exp()) for exp in expressions] verbose_print(u'Interpolated string evaluation assembling {0} chunks{1}.'.format( len(chunks), '' if len(chunks) == 0 else u' ("{0}")'.format(u'", "'.join(chunks))) ) return ''.join(chunks) verbose_print( u'Finished parsing interpolated string `{0}` ({1} chunk(s) found)'.format(debug_dump_long_string(source), len(expressions)), outdent_before=True ) return evaluate
def evaluate(self): verbose_print('Evaluating union decomposition ({} clauses)'.format( len(self.mapping_generators)), indent_after=True) sequence = make_sequence(self.union_expression()) result = [] for item in sequence: verbose_print( lambda: u'Visiting item {0}'.format(debug_dump_anything(item)), indent_after=True) with variable_scope(): push_variable('_', make_sequence(item)) if not hasattr(item, 'union_index'): raise HqueryEvaluationError( "Union decomposition applied to something that wasn't produced by a union" ) if item.union_index >= len(self.mapping_generators): raise HqueryEvaluationError( "Decomposed union had more clauses than its mapping") this_result = make_sequence( self.mapping_generators[item.union_index]()) verbose_print( 'Mapping yielded {0} results for this visit'.format( len(this_result))) result = sequence_concat(result, this_result) verbose_print('Visit finished', outdent_before=True) verbose_print('Union decomposition completed', outdent_before=True) return result
def _evaluate_without_iteration(self): with variable_scope(): self._push_global_variables() verbose_print('Evaluating return expression.', indent_after=True) result = self.return_expression() verbose_print('Return expression produced {0}'.format(str(result)), outdent_before=True) return result
def evaluate(self): verbose_print('Evaluating union decomposition ({} clauses)'.format(len(self.mapping_generators)), indent_after=True) sequence = make_sequence(self.union_expression()) result = [] for item in sequence: verbose_print(lambda: u'Visiting item {0}'.format(debug_dump_anything(item)), indent_after=True) with variable_scope(): push_variable('_', make_sequence(item)) if not hasattr(item, 'union_index'): raise HqueryEvaluationError( "Union decomposition applied to something that wasn't produced by a union" ) if item.union_index >= len(self.mapping_generators): raise HqueryEvaluationError("Decomposed union had more clauses than its mapping") this_result = make_sequence(self.mapping_generators[item.union_index]()) verbose_print( 'Mapping yielded {0} results for this visit'.format( len(this_result))) result = sequence_concat(result, this_result) verbose_print('Visit finished', outdent_before=True) verbose_print('Union decomposition completed', outdent_before=True) return result
def evaluate(): chunks = [string_value(exp()) for exp in expressions] verbose_print(u'Interpolated string evaluation assembling {0} chunks{1}.'.format( len(chunks), '' if len(chunks) == 0 else u' ("{0}")'.format(u'", "'.join(chunks))) ) return ''.join(chunks)
def evaluate(hash): for key, value in hash.items(): if key in tag_names: if not isinstance(value, list): verbose_print( 'JSON hash constructor array filter converting attribute "{0}" to array' .format(key)) hash[key] = [value]
def evaluate(hash): for key, value in hash.items(): if key in mappings: verbose_print( 'JSON hash constructor mapping filter converting attribute name "{0}" to "{1}"' .format(key, value)) hash[mappings[key]] = hash[key] del hash[key]
def push_context(node, position=1, size=1, preserve_space=None): msg = u'Pushing (node={0}, position={1}, size={2} on context stack.' verbose_print(lambda: msg.format(debug_dump_node(node), position, size)) context_stack.append( ExpressionContext(node=node, position=position, size=size, preserve_space=preserve_space))
def evaluate(self): verbose_print('Evaluating value expression for constructed hash key "{0}"'.format(self.key), indent_after=True) value = self.value_fn() msg = u'Finished evaluating; value of constructed hash key "{0}" is {1}' verbose_print(lambda: msg.format(self.key, debug_dump_anything(value)), outdent_before=True) return HashKeyValue(self.key, value)
def evaluate(hash): for key, value in hash.items(): if key in tag_names: verbose_print( 'JSON hash constructor number filter converting attribute "{0}" value(s) to numbers'.format(key) ) if isinstance(value, list): hash[key] = [number(v).value for v in value] else: hash[key] = number(value).value
def value_of_variable(name): if len(variable_stack) > 0: for index in range(len(variable_stack) - 1, -1, -1): if variable_stack[index][NAME] == name: reverse_index = len(variable_stack) - (index + 1) verbose_print('Variable "${0}" found on stack (position {1}).'.format(name, reverse_index)) return variable_stack[index][VALUE] verbose_print('Variable "${0}" NOT FOUND on variable stack.'.format(name)) return None
def evaluate(hash): for key, value in hash.items(): if key in tag_names: verbose_print( 'JSON hash constructor number filter converting attribute "{0}" value(s) to numbers' .format(key)) if isinstance(value, list): hash[key] = [number(v).value for v in value] else: hash[key] = number(value).value
def evaluate(self): verbose_print('Evaluating FLWOR {0}'.format(self), indent_after=True) if self.sequence_expression is not None: result = self._evaluate_iteration() else: result = self._evaluate_without_iteration() verbose_print(lambda: 'FLWOR evaluation completed; returning {0}'.format(debug_dump_anything(result)), outdent_before=True) return result
def evaluate(self): verbose_print('Evaluating FLWOR {0}'.format(self), indent_after=True) if self.sequence_expression is not None: result = self._evaluate_iteration() else: result = self._evaluate_without_iteration() verbose_print(lambda: 'FLWOR evaluation completed; returning {0}'. format(debug_dump_anything(result)), outdent_before=True) return result
def evaluate(self): verbose_print( 'Evaluating value expression for constructed hash key "{0}"'. format(self.key), indent_after=True) value = self.value_fn() msg = u'Finished evaluating; value of constructed hash key "{0}" is {1}' verbose_print(lambda: msg.format(self.key, debug_dump_anything(value)), outdent_before=True) return HashKeyValue(self.key, value)
def _evaluate_iteration(self): with variable_scope(): self._push_global_variables() sequence = make_sequence(self.sequence_expression()) verbose_print( 'Iterating over sequence containing {0} items'.format( len(sequence))) result = [] for item in sequence: verbose_print(lambda: u'Visiting item {0}'.format( debug_dump_anything(item)), indent_after=True) with variable_scope(): push_variable(self.sequence_variable, make_sequence(item)) self._push_iteration_variables() this_result = make_sequence(self.return_expression()) verbose_print( 'Return clause yielded {0} results for this visit'. format(len(this_result))) result = sequence_concat(result, this_result) verbose_print('Visit finished', outdent_before=True) return result
def accept_context_node(): context = peek_context() format_str = u'Evaluating predicate expression for context node at position {0} of {1}: {2}.' verbose_print(lambda: format_str.format(context.position, context.size, debug_dump_node(context.node))) value = expression_fn() if is_number(value): accept = number(context.position) == value else: accept = bool(value) verbose_print(lambda: u'{0} node {1}'.format('Accepted' if accept else 'Rejected', debug_dump_node(context.node))) return [context.node] if accept else []
def _eq_node_set_vs_string(nodes_val, string_val): string_val = str(string_val) verbose_print(u'(=) comparing number "{0}" to {1} nodes'.format(string_val, len(nodes_val))) for node in nodes_val: node_val_string = string_value(node) verbose_print(u'(=) node string value "{0}" is{1} equal to "{2}"'.format( node_val_string, ('' if node_val_string == string_val else ' not'), string_val)) if node_val_string == string_val: return True return False
def _eq_node_set_vs_number(nodes_val, num_val): verbose_print('(=) comparing number {0} to {1} nodes'.format(num_val, len(nodes_val))) for node in nodes_val: node_str_val = string_value(node) node_num_val = number(node_str_val) verbose_print('(=) node string value "{0}" is{1} equal to "{2}"'.format( node_num_val, (' not' if node_num_val == num_val else ''), num_val)) if node_num_val == num_val: return True return False
def _load_all_functions(self): if self.all_functions is None: self.all_functions = dict() my_package = '{0}/__init__.py'.format(os.path.dirname(__file__)) verbose_print('FunctionSupport loading all function modules near {0}.'.format(my_package), indent_after=True) for importer, modname, ispkg in walk_packages(my_package): if not ispkg and modname.startswith('hq.hquery.functions.'): verbose_print('Found candidate module {0} -- loading.'.format(modname)) module = importer.find_module(modname).load_module(modname) if hasattr(module, 'exports'): exports = {name.rstrip('_'): getattr(module, name) for name in getattr(module, 'exports')} verbose_print('Module {0} exports are: {1}'.format(modname, exports.keys())) if any(not (isclass(obj) or isfunction(obj)) for obj in exports.values()): raise RuntimeError('Non-class/function export(s) loaded from module {0}'.format(modname)) self.all_functions.update(exports) else: verbose_print('Module {0} defined no exports.'.format(modname)) verbose_print('Finished loading function modules.', outdent_before=True)
def _evaluate_steps(self, remaining_steps): step = remaining_steps[0] verbose_print(lambda: 'Evaluating step {0}'.format(remaining_steps[0]), indent_after=True) result_set = make_node_set(step.node_test.apply(step.axis, get_context_node()), reverse=step.axis.is_reverse_order()) verbose_print(lambda: 'Axis and node test produced {0} matching nodes'.format(len(result_set))) for index, expression_fn in enumerate(step.predicates): def accept_context_node(): context = peek_context() format_str = u'Evaluating predicate expression for context node at position {0} of {1}: {2}.' verbose_print(lambda: format_str.format(context.position, context.size, debug_dump_node(context.node))) value = expression_fn() if is_number(value): accept = number(context.position) == value else: accept = bool(value) verbose_print(lambda: u'{0} node {1}'.format('Accepted' if accept else 'Rejected', debug_dump_node(context.node))) return [context.node] if accept else [] verbose_print(lambda: 'Evaluating predicate #{0} against {1} nodes'.format(index + 1, len(result_set)), indent_after=True) result_set = evaluate_across_contexts(result_set, accept_context_node) verbose_print( lambda: 'Evaluation of predicate #{0} complete; accepted {1} nodes.'.format(index + 1, len(result_set)), outdent_before=True) if len(remaining_steps) > 1: result_set = evaluate_across_contexts(result_set, lambda: self._evaluate_steps(remaining_steps[1:])) verbose_print(lambda: 'Step evaluation completed; returning {0} nodes.'.format(len(result_set)), outdent_before=True) return result_set
def _cmp_value_to_nodes(base_op, first, second): node_values = set([number(node) for node in second]) first = number(first) verbose_print('Comparing {0} nodes in node set to value "{1}"'.format(len(node_values), first)) for node_value in node_values: if base_op(first, node_value): verbose_print('Comparison succeeded for value "{0}" and node value "{1}'.format(first, node_value)) return True verbose_print('Comparison failed for all nodes in the node set.') return False
def _cmp_nodes_to_value(base_op, first, second): node_values = set([number(node) for node in first]) second = number(second) verbose_print('Comparing {0} nodes in node set to value {1}'.format(len(node_values), second)) for node_value in node_values: if base_op(node_value, second): verbose_print('Comparison succeeded for node value "{0}" and value "{1}"'.format(node_value, second)) return True verbose_print('Comparison failed for all nodes in the node set.') return False
def _eq_node_sets(first, second): first_values = set([string_value(node) for node in first]) second_values = set([string_value(node) for node in second]) verbose_print('Comparing two nodes sets (size {0} and {1}).'.format(len(first_values), len(second_values))) for first_value in first_values: if first_value in second_values: verbose_print(u'Found value "{0}" from first node set in second node set'.format(first_value)) return True verbose_print('Found no matching nodes between node sets.') return False
def _evaluate_iteration(self): with variable_scope(): self._push_global_variables() sequence = make_sequence(self.sequence_expression()) verbose_print('Iterating over sequence containing {0} items'.format(len(sequence))) result = [] for item in sequence: verbose_print(lambda: u'Visiting item {0}'.format(debug_dump_anything(item)), indent_after=True) with variable_scope(): push_variable(self.sequence_variable, make_sequence(item)) self._push_iteration_variables() this_result = make_sequence(self.return_expression()) verbose_print('Return clause yielded {0} results for this visit'.format(len(this_result))) result = sequence_concat(result, this_result) verbose_print('Visit finished', outdent_before=True) return result
def _cmp_node_sets(base_op, first, second): first_values = set([number(node) for node in first]) second_values = set([number(node) for node in second]) verbose_print('Comparing two nodes sets (size {0} and {1}).'.format(len(first_values), len(second_values))) for first_value in first_values: for second_value in second_values: if base_op(first_value, second_value): msg = 'Comparison succeeded for "{0}" from first node set and "{1}" in second node set' verbose_print(msg.format(first_value, second_value)) return True verbose_print('Comparison failed for all nodes in both node sets.') return False
def evaluate(self): verbose_print(lambda: 'Evaluating location path {0}'.format(self.debug_dump()), indent_after=True) if self.absolute: verbose_print('Switching context to root because this path is absolute.') results = evaluate_in_context(soup_from_any_tag(get_context_node()), lambda: self._evaluate_steps(self.steps)) elif self.root_expression is not None: results = evaluate_across_contexts(self.root_expression(), lambda: self._evaluate_steps(self.steps)) else: results = self._evaluate_steps(self.steps) verbose_print('Evaluation completed; location path selected {0} nodes'.format(len(results)), outdent_before=True) return make_node_set(results, reverse=False)
def _gab(self, message): verbose_print('JSON array constructor {0}'.format(message))
def _eq_bool_vs_primitive(bool_val, other_val): verbose_print('Comparing boolean value {0} with non-node-set value {1} (coerced to {2})'.format(bool_val, other_val, boolean(other_val))) return bool_val == boolean(other_val)
def _gab(self, message): verbose_print('JSON hash constructor {0}'.format(message))
def pop_context(): result = context_stack.pop() msg = u'Popping (node={0}, position={1}, size={2} off of context stack.' verbose_print(lambda: msg.format(debug_dump_node(result.node), result. position, result.size)) return result
def push_variable(name, value): global variable_stack verbose_print(lambda: u'Pushing variable onto stack: let ${0} := {1}'.format(name, debug_dump_anything(value))) variable_stack.append((name, value))
def push_context(node, position=1, size=1, preserve_space=None): msg = u'Pushing (node={0}, position={1}, size={2} on context stack.' verbose_print(lambda: msg.format(debug_dump_node(node), position, size)) context_stack.append(ExpressionContext(node=node, position=position, size=size, preserve_space=preserve_space))
def pop_context(): result = context_stack.pop() msg = u'Popping (node={0}, position={1}, size={2} off of context stack.' verbose_print(lambda: msg.format(debug_dump_node(result.node), result.position, result.size)) return result
def _push_iteration_variables(self): for let in self.per_iteration_variables: verbose_print('Evaluating let {0} := <expr>'.format(let[0])) push_variable(let[0], let[1]())
def _push_global_variables(self): for let in self.global_variables: verbose_print('Evaluating let {0} := <expr>'.format(let[0])) push_variable(let[0], let[1]())
def evaluate(hash): for key, value in hash.items(): if key in tag_names: if not isinstance(value, list): verbose_print('JSON hash constructor array filter converting attribute "{0}" to array'.format(key)) hash[key] = [value]
def _gab(self, msg, **kwargs): verbose_print(u'{0} {1}'.format(self, msg), **kwargs)
def evaluate(hash): for key, value in hash.items(): if key in mappings: verbose_print('JSON hash constructor mapping filter converting attribute name "{0}" to "{1}"'.format(key, value)) hash[mappings[key]] = hash[key] del hash[key]