def do_fieldnames(self, parser, group):
"""This filter finds FieldnameNodes in the tree and applies their
fieldname to the next node.
"""
fnclass = syntax.FieldnameNode
if self.removeunknown and parser.schema:
# Look for field nodes that aren't in the schema and convert them
# to text
schema = parser.schema
newgroup = group.empty_copy()
prev_field_node = None
for node in group:
if isinstance(node, fnclass) and node.fieldname not in schema:
prev_field_node = node
continue
elif prev_field_node:
# If prev_field_node is not None, it contains a field node
# that appeared before this node but isn't in the schema,
# so we'll convert it to text here
if node.has_text:
node.text = prev_field_node.original + node.text
else:
newgroup.append(syntax.to_word(prev_field_node))
prev_field_node = None
newgroup.append(node)
if prev_field_node:
newgroup.append(syntax.to_word(prev_field_node))
group = newgroup
newgroup = group.empty_copy()
# Iterate backwards through the stream, looking for field-able objects
# with field nodes in front of them
i = len(group)
while i > 0:
i -= 1
node = group[i]
if isinstance(node, fnclass):
# If we see a fieldname node, it must not have been in front
# of something fieldable, since we would have already removed
# it (since we're iterating backwards), so convert it to text
node = syntax.to_word(node)
elif isinstance(node, syntax.GroupNode):
node = self.do_fieldnames(parser, node)
if i > 0 and not node.is_ws() and isinstance(group[i - 1],
fnclass):
node.set_fieldname(group[i - 1].fieldname, override=False)
i -= 1
newgroup.append(node)
newgroup.reverse()
return newgroup
def do_fieldnames(self, parser, group):
"""This filter finds FieldnameNodes in the tree and applies their
fieldname to the next node.
"""
fnclass = syntax.FieldnameNode
if self.removeunknown and parser.schema:
# Look for field nodes that aren't in the schema and convert them
# to text
schema = parser.schema
newgroup = group.empty_copy()
prev_field_node = None
for node in group:
if isinstance(node, fnclass) and node.fieldname not in schema:
prev_field_node = node
continue
elif prev_field_node:
# If prev_field_node is not None, it contains a field node
# that appeared before this node but isn't in the schema,
# so we'll convert it to text here
if node.has_text:
node.text = prev_field_node.original + node.text
else:
newgroup.append(syntax.to_word(prev_field_node))
prev_field_node = None
newgroup.append(node)
if prev_field_node:
newgroup.append(syntax.to_word(prev_field_node))
group = newgroup
newgroup = group.empty_copy()
# Iterate backwards through the stream, looking for field-able objects
# with field nodes in front of them
i = len(group)
while i > 0:
i -= 1
node = group[i]
if isinstance(node, fnclass):
# If we see a fieldname node, it must not have been in front
# of something fieldable, since we would have already removed
# it (since we're iterating backwards), so convert it to text
node = syntax.to_word(node)
elif isinstance(node, syntax.GroupNode):
node = self.do_fieldnames(parser, node)
if i > 0 and not node.is_ws() and isinstance(
group[i - 1], fnclass):
node.set_fieldname(group[i - 1].fieldname, override=False)
i -= 1
newgroup.append(node)
newgroup.reverse()
return newgroup
def clean_boost(self, parser, group):
"""This filter finds any BoostNodes in positions where they can't boost
the previous node (e.g. at the very beginning, after whitespace, or
after another BoostNode) and turns them into WordNodes.
"""
bnode = self.BoostNode
for i, node in enumerate(group):
if isinstance(node, bnode):
if (not i or not group[i - 1].has_boost):
group[i] = syntax.to_word(node)
return group
def clean_boost(self, parser, group):
"""This filter finds any BoostNodes in positions where they can't boost
the previous node (e.g. at the very beginning, after whitespace, or
after another BoostNode) and turns them into WordNodes.
"""
bnode = self.BoostNode
for i, node in enumerate(group):
if isinstance(node, bnode):
if (not i or not group[i - 1].has_boost):
group[i] = syntax.to_word(node)
return group
def do_fuzzyterms(self, parser, group):
newgroup = group.empty_copy()
i = 0
while i < len(group):
node = group[i]
if i < len(group) - 1 and isinstance(node, syntax.WordNode):
nextnode = group[i + 1]
if isinstance(nextnode, self.FuzzinessNode):
node = self.FuzzyTermNode(node, nextnode.maxdist, nextnode.prefixlength)
i += 1
if isinstance(node, self.FuzzinessNode):
node = syntax.to_word(node)
if isinstance(node, syntax.GroupNode):
node = self.do_fuzzyterms(parser, node)
newgroup.append(node)
i += 1
return newgroup
def do_fuzzyterms(self, parser, group):
newgroup = group.empty_copy()
i = 0
while i < len(group):
node = group[i]
if i < len(group) - 1 and isinstance(node, syntax.WordNode):
nextnode = group[i + 1]
if isinstance(nextnode, self.FuzzinessNode):
node = self.FuzzyTermNode(node, nextnode.maxdist,
nextnode.prefixlength)
i += 1
if isinstance(node, self.FuzzinessNode):
node = syntax.to_word(node)
if isinstance(node, syntax.GroupNode):
node = self.do_fuzzyterms(parser, node)
newgroup.append(node)
i += 1
return newgroup
def do_boost(self, parser, group):
"""This filter finds BoostNodes and applies the boost to the previous
node.
"""
newgroup = group.empty_copy()
for node in group:
if isinstance(node, syntax.GroupNode):
node = self.do_boost(parser, node)
elif isinstance(node, self.BoostNode):
if (newgroup and newgroup[-1].has_boost):
# Apply the BoostNode's boost to the previous node
newgroup[-1].set_boost(node.boost)
# Skip adding the BoostNode to the new group
continue
else:
node = syntax.to_word(node)
newgroup.append(node)
return newgroup
def do_boost(self, parser, group):
"""This filter finds BoostNodes and applies the boost to the previous
node.
"""
newgroup = group.empty_copy()
for node in group:
if isinstance(node, syntax.GroupNode):
node = self.do_boost(parser, node)
elif isinstance(node, self.BoostNode):
if (newgroup and newgroup[-1].has_boost):
# Apply the BoostNode's boost to the previous node
newgroup[-1].set_boost(node.boost)
# Skip adding the BoostNode to the new group
continue
else:
node = syntax.to_word(node)
newgroup.append(node)
return newgroup