def do_fieldnames(self, parser, group): """This filter finds FieldnameNodes in the tree and applies their fieldname to the next node. """ fnclass = syntax.FieldnameNode if self.removeunknown and parser.schema: # Look for field nodes that aren't in the schema and convert them # to text schema = parser.schema newgroup = group.empty_copy() prev_field_node = None for node in group: if isinstance(node, fnclass) and node.fieldname not in schema: prev_field_node = node continue elif prev_field_node: # If prev_field_node is not None, it contains a field node # that appeared before this node but isn't in the schema, # so we'll convert it to text here if node.has_text: node.text = prev_field_node.original + node.text else: newgroup.append(syntax.to_word(prev_field_node)) prev_field_node = None newgroup.append(node) if prev_field_node: newgroup.append(syntax.to_word(prev_field_node)) group = newgroup newgroup = group.empty_copy() # Iterate backwards through the stream, looking for field-able objects # with field nodes in front of them i = len(group) while i > 0: i -= 1 node = group[i] if isinstance(node, fnclass): # If we see a fieldname node, it must not have been in front # of something fieldable, since we would have already removed # it (since we're iterating backwards), so convert it to text node = syntax.to_word(node) elif isinstance(node, syntax.GroupNode): node = self.do_fieldnames(parser, node) if i > 0 and not node.is_ws() and isinstance(group[i - 1], fnclass): node.set_fieldname(group[i - 1].fieldname, override=False) i -= 1 newgroup.append(node) newgroup.reverse() return newgroup
def do_fieldnames(self, parser, group): """This filter finds FieldnameNodes in the tree and applies their fieldname to the next node. """ fnclass = syntax.FieldnameNode if self.removeunknown and parser.schema: # Look for field nodes that aren't in the schema and convert them # to text schema = parser.schema newgroup = group.empty_copy() prev_field_node = None for node in group: if isinstance(node, fnclass) and node.fieldname not in schema: prev_field_node = node continue elif prev_field_node: # If prev_field_node is not None, it contains a field node # that appeared before this node but isn't in the schema, # so we'll convert it to text here if node.has_text: node.text = prev_field_node.original + node.text else: newgroup.append(syntax.to_word(prev_field_node)) prev_field_node = None newgroup.append(node) if prev_field_node: newgroup.append(syntax.to_word(prev_field_node)) group = newgroup newgroup = group.empty_copy() # Iterate backwards through the stream, looking for field-able objects # with field nodes in front of them i = len(group) while i > 0: i -= 1 node = group[i] if isinstance(node, fnclass): # If we see a fieldname node, it must not have been in front # of something fieldable, since we would have already removed # it (since we're iterating backwards), so convert it to text node = syntax.to_word(node) elif isinstance(node, syntax.GroupNode): node = self.do_fieldnames(parser, node) if i > 0 and not node.is_ws() and isinstance( group[i - 1], fnclass): node.set_fieldname(group[i - 1].fieldname, override=False) i -= 1 newgroup.append(node) newgroup.reverse() return newgroup
def clean_boost(self, parser, group): """This filter finds any BoostNodes in positions where they can't boost the previous node (e.g. at the very beginning, after whitespace, or after another BoostNode) and turns them into WordNodes. """ bnode = self.BoostNode for i, node in enumerate(group): if isinstance(node, bnode): if (not i or not group[i - 1].has_boost): group[i] = syntax.to_word(node) return group
def do_fuzzyterms(self, parser, group): newgroup = group.empty_copy() i = 0 while i < len(group): node = group[i] if i < len(group) - 1 and isinstance(node, syntax.WordNode): nextnode = group[i + 1] if isinstance(nextnode, self.FuzzinessNode): node = self.FuzzyTermNode(node, nextnode.maxdist, nextnode.prefixlength) i += 1 if isinstance(node, self.FuzzinessNode): node = syntax.to_word(node) if isinstance(node, syntax.GroupNode): node = self.do_fuzzyterms(parser, node) newgroup.append(node) i += 1 return newgroup
def do_boost(self, parser, group): """This filter finds BoostNodes and applies the boost to the previous node. """ newgroup = group.empty_copy() for node in group: if isinstance(node, syntax.GroupNode): node = self.do_boost(parser, node) elif isinstance(node, self.BoostNode): if (newgroup and newgroup[-1].has_boost): # Apply the BoostNode's boost to the previous node newgroup[-1].set_boost(node.boost) # Skip adding the BoostNode to the new group continue else: node = syntax.to_word(node) newgroup.append(node) return newgroup