def test_duplicates():
q = And([Term("a", u("b")), Term("a", u("b"))])
assert_equal(q.normalize(), Term("a", u("b")))
q = And([Prefix("a", u("b")), Prefix("a", u("b"))])
assert_equal(q.normalize(), Prefix("a", u("b")))
q = And([
Variations("a", u("b")),
And([Variations("a", u("b")),
Term("a", u("b"))])
])
assert_equal(q.normalize(),
And([Variations("a", u("b")),
Term("a", u("b"))]))
q = And(
[Term("a", u("b")),
Prefix("a", u("b")),
Term("a", u("b"), boost=1.1)])
assert_equal(q.normalize(), q)
# Wildcard without * or ? normalizes to Term
q = And([
Wildcard("a", u("b")),
And([Wildcard("a", u("b")), Term("a", u("b"))])
])
assert_equal(q.normalize(), Term("a", u("b")))
def generate_next_ovalid(element_type, element_index):
# Find the largest index for this element type
# Increment it by one
global id_cache_list
global verbose
type_key = "def"
if element_type == "state":
type_key = "ste"
elif element_type == "test":
type_key = "tst"
elif element_type == "object":
type_key = "obj"
elif element_type == "variable":
type_key = "var"
cached_list = id_cache_list[type_key]
if not cached_list or len(cached_list) < 1:
search_term = "oval_*_{0}_*".format(type_key)
index_searcher = element_index.get_searcher()
query = Wildcard("oval_id", search_term)
results = index_searcher.search(query, limit=None)
matching = [hit['oval_id'] for hit in results]
index = find_largest_index(matching) + 1
new_id = "oval:org.cisecurity:{0}:{1}".format(type_key, index)
cached_list = [new_id]
id_cache_list[type_key] = cached_list
else:
index = find_largest_index(cached_list) + 1
new_id = "oval:org.cisecurity:{0}:{1}".format(type_key, index)
cached_list.append(new_id)
return new_id
def test_query_copy_hash():
def do(q1, q2):
q1a = copy.deepcopy(q1)
assert_equal(q1, q1a)
assert_equal(hash(q1), hash(q1a))
assert_not_equal(q1, q2)
do(Term("a", u("b"), boost=1.1), Term("a", u("b"), boost=1.5))
do(And([Term("a", u("b")), Term("c", u("d"))], boost=1.1),
And([Term("a", u("b")), Term("c", u("d"))], boost=1.5))
do(Or([Term("a", u("b"), boost=1.1), Term("c", u("d"))]),
Or([Term("a", u("b"), boost=1.8), Term("c", u("d"))], boost=1.5))
do(DisjunctionMax([Term("a", u("b"), boost=1.8), Term("c", u("d"))]),
DisjunctionMax([Term("a", u("b"), boost=1.1), Term("c", u("d"))],
boost=1.5))
do(Not(Term("a", u("b"), boost=1.1)), Not(Term("a", u("b"), boost=1.5)))
do(Prefix("a", u("b"), boost=1.1), Prefix("a", u("b"), boost=1.5))
do(Wildcard("a", u("b*x?"), boost=1.1), Wildcard("a", u("b*x?"),
boost=1.5))
do(FuzzyTerm("a", u("b"), constantscore=True),
FuzzyTerm("a", u("b"), constantscore=False))
do(FuzzyTerm("a", u("b"), boost=1.1), FuzzyTerm("a", u("b"), boost=1.5))
do(TermRange("a", u("b"), u("c")), TermRange("a", u("b"), u("d")))
do(TermRange("a", None, u("c")), TermRange("a", None, None))
do(TermRange("a", u("b"), u("c"), boost=1.1),
TermRange("a", u("b"), u("c"), boost=1.5))
do(TermRange("a", u("b"), u("c"), constantscore=True),
TermRange("a", u("b"), u("c"), constantscore=False))
do(NumericRange("a", 1, 5), NumericRange("a", 1, 6))
do(NumericRange("a", None, 5), NumericRange("a", None, None))
do(NumericRange("a", 3, 6, boost=1.1), NumericRange("a", 3, 6, boost=1.5))
do(NumericRange("a", 3, 6, constantscore=True),
NumericRange("a", 3, 6, constantscore=False))
# do(DateRange)
do(Variations("a", u("render")), Variations("a", u("renders")))
do(Variations("a", u("render"), boost=1.1),
Variations("a", u("renders"), boost=1.5))
do(Phrase("a", [u("b"), u("c"), u("d")]),
Phrase("a", [u("b"), u("c"), u("e")]))
do(Phrase("a", [u("b"), u("c"), u("d")], boost=1.1),
Phrase("a", [u("b"), u("c"), u("d")], boost=1.5))
do(Phrase("a", [u("b"), u("c"), u("d")], slop=1),
Phrase("a", [u("b"), u("c"), u("d")], slop=2))
# do(Ordered)
do(Every(), Every("a"))
do(Every("a"), Every("b"))
do(Every("a", boost=1.1), Every("a", boost=1.5))
do(NullQuery, Term("a", u("b")))
do(ConstantScoreQuery(Term("a", u("b"))),
ConstantScoreQuery(Term("a", u("c"))))
do(ConstantScoreQuery(Term("a", u("b")), score=2.0),
ConstantScoreQuery(Term("a", u("c")), score=2.1))
do(Require(Term("a", u("b")), Term("c", u("d"))),
Require(Term("a", u("b"), boost=1.1), Term("c", u("d"))))
# do(Require)
# do(AndMaybe)
# do(AndNot)
# do(Otherwise)
do(SpanFirst(Term("a", u("b")), limit=1), SpanFirst(Term("a", u("b")),
limit=2))
do(SpanNear(Term("a", u("b")), Term("c", u("d"))),
SpanNear(Term("a", u("b")), Term("c", u("e"))))
do(SpanNear(Term("a", u("b")), Term("c", u("d")), slop=1),
SpanNear(Term("a", u("b")), Term("c", u("d")), slop=2))
do(SpanNear(Term("a", u("b")), Term("c", u("d")), mindist=1),
SpanNear(Term("a", u("b")), Term("c", u("d")), mindist=2))
do(SpanNear(Term("a", u("b")), Term("c", u("d")), ordered=True),
SpanNear(Term("a", u("b")), Term("c", u("d")), ordered=False))
do(SpanNot(Term("a", u("b")), Term("a", u("c"))),
SpanNot(Term("a", u("b")), Term("a", u("d"))))
do(SpanOr([Term("a", u("b")), Term("a", u("c")), Term("a", u("d"))]),
SpanOr([Term("a", u("b")), Term("a", u("c")), Term("a", u("e"))]))
do(SpanContains(Term("a", u("b")), Term("a", u("c"))),
SpanContains(Term("a", u("b")), Term("a", u("d"))))
def test_wildcard():
_run_query(Or([Wildcard('value', u('*red*')), Wildcard('name', u('*yellow*'))]),
[u("A"), u("C"), u("D"), u("E")])
# Missing
_run_query(Wildcard('value', 'glonk*'), [])
def recurse(self, elem, page_href):
# on first call, elem.tag.name=='page'.
# Descendants (body, div, p, include, page, etc.) are processed by recursing through DOM
# stack is used to detect transclusion loops
page_href_new = elem.get(self.tag_page_href)
if page_href_new:
page_href_new = Iri(page_href_new)
if page_href_new != page_href:
page_href = page_href_new
self.stack.append(page_href)
else:
self.stack.append(None)
else:
self.stack.append(None)
try:
if elem.tag == self.tag_xi_include:
# we have already recursed several levels and found a transclusion: "{{SomePage}}" or similar
# process the transclusion and add it to the DOM. Subsequent recursions will traverse through
# the transclusion's elements.
href = elem.get(self.tag_xi_href)
xpointer = elem.get(self.tag_xi_xpointer)
xp_include_pages = None
xp_include_sort = None
xp_include_items = None
xp_include_skipitems = None
xp_include_heading = None
xp_include_level = None
if xpointer:
xp = XPointer(xpointer)
xp_include = None
xp_namespaces = {}
for entry in xp:
uri = None
name = entry.name.split(':', 1)
if len(name) > 1:
prefix, name = name
uri = xp_namespaces.get(prefix, False)
else:
name = name[0]
if uri is None and name == 'xmlns':
d_prefix, d_uri = entry.data.split('=', 1)
xp_namespaces[d_prefix] = d_uri
elif uri == moin_page.namespace and name == 'include':
xp_include = XPointer(entry.data)
if xp_include:
for entry in xp_include:
name, data = entry.name, entry.data_unescape
if name == 'pages':
xp_include_pages = data
elif name == 'sort':
xp_include_sort = data
elif name == 'items':
xp_include_items = int(data)
elif name == 'skipitems':
xp_include_skipitems = int(data)
elif name == 'heading':
xp_include_heading = data
elif name == 'level':
xp_include_level = data
if href:
# We have a single page to transclude
href = Iri(href)
link = Iri(scheme='wiki', authority='')
if href.scheme == 'wiki':
if href.authority:
raise ValueError(
"can't handle xinclude for non-local authority"
)
else:
path = href.path[1:]
elif href.scheme == 'wiki.local':
page = page_href
path = href.path
if path[0] == '':
# /subitem
tmp = page.path[1:]
tmp.extend(path[1:])
path = tmp
elif path[0] == '..':
# ../sisteritem
path = page.path[1:] + path[1:]
else:
raise ValueError(
"can't handle xinclude for schemes other than wiki or wiki.local"
)
link.path = path
if flaskg.user.may.read(unicode(path)):
page = Item.create(unicode(path))
pages = ((page, link), )
else:
# ACLs prevent user from viewing a transclusion - show message
message = moin_page.p(children=(_(
'Access Denied, transcluded content suppressed.')))
attrib = {html.class_: 'warning'}
div = ET.Element(moin_page.div,
attrib,
children=(message, ))
container = ET.Element(moin_page.body,
children=(div, ))
return [
container, 0
] # replace transclusion with container's child
elif xp_include_pages:
# XXX we currently interpret xp_include_pages as wildcard, but it should be regex
# for compatibility with moin 1.9. whoosh has upcoming regex support, but it is not
# released yet.
if xp_include_pages.startswith('^'):
# get rid of the leading ^ the Include macro needed to get into "regex mode"
xp_include_pages = xp_include_pages[1:]
query = And([
Term(WIKINAME, app.cfg.interwikiname),
Wildcard(NAME_EXACT, xp_include_pages)
])
reverse = xp_include_sort == 'descending'
results = flaskg.storage.search(query,
sortedby=NAME_EXACT,
reverse=reverse,
limit=None)
pagelist = [result[NAME] for result in results]
if xp_include_skipitems is not None:
pagelist = pagelist[xp_include_skipitems:]
if xp_include_items is not None:
pagelist = pagelist[xp_include_items + 1:]
pages = ((Item.create(p),
Iri(scheme='wiki', authority='', path='/' + p))
for p in pagelist)
included_elements = []
for page, p_href in pages:
if p_href.path[0] != '/':
p_href.path = IriPath('/' + '/'.join(p_href.path))
if p_href in self.stack:
# we have a transclusion loop, create an error message showing list of pages forming loop
loop = self.stack[self.stack.index(p_href):]
loop = [
u'{0}'.format(ref.path[1:])
for ref in loop if ref is not None
] + [page.name]
msg = u'Error: Transclusion loop via: ' + u', '.join(
loop)
attrib = {html.class_: 'moin-error'}
strong = ET.Element(moin_page.strong, attrib, (msg, ))
included_elements.append(strong)
continue
# TODO: Is this correct?
if not flaskg.user.may.read(page.name):
continue
if xp_include_heading is not None:
attrib = {self.tag_href: p_href}
children = (xp_include_heading or page.name, )
elem_a = ET.Element(self.tag_a,
attrib,
children=children)
attrib = {
self.tag_outline_level: xp_include_level or '1'
}
elem_h = ET.Element(self.tag_h,
attrib,
children=(elem_a, ))
included_elements.append(elem_h)
page_doc = page.content.internal_representation(
attributes=Arguments(keyword=elem.attrib))
self.recurse(page_doc, page_href)
# The href needs to be an absolute URI, without the prefix "wiki://"
page_doc = mark_item_as_transclusion(page_doc, p_href.path)
included_elements.append(page_doc)
if len(included_elements) > 1:
# use a div as container
result = ET.Element(self.tag_div)
result.extend(included_elements)
elif included_elements:
result = included_elements[0]
else:
result = None
# end of processing for transclusion; the "result" will get inserted into the DOM below
return result
# Traverse the DOM by calling self.recurse with each child of the current elem.
# Starting elem.tag.name=='page'.
container = []
i = 0
while i < len(elem):
child = elem[i]
if isinstance(child, ET.Node):
ret = self.recurse(child, page_href)
if ret:
# Either child or a descendant of child is a transclusion.
# See top of this script for notes on why these DOM adjustments are required.
if isinstance(ret, ET.Node
) and elem.tag.name in NO_BLOCK_CHILDREN:
body = ret[0]
if len(body) == 0:
# the transcluded item is empty, insert an empty span into DOM
attrib = Attributes(ret).convert()
elem[i] = ET.Element(moin_page.span,
attrib=attrib)
elif (isinstance(body[0], ET.Node)
and (len(body) > 1 or body[0].tag.name
not in ('p', 'object', 'a'))):
# Complex case: "some text {{BlockItem}} more text" or "\n{{BlockItem}}\n" where
# the BlockItem body contains multiple p's, a table, preformatted text, etc.
# These block elements cannot be made a child of the current elem, so we create
# a container to replace elem.
# Create nodes to hold any siblings before and after current child (elem[i])
before = copy.deepcopy(elem)
after = copy.deepcopy(elem)
before[:] = elem[0:i]
after[:] = elem[i + 1:]
if len(before):
# there are siblings before transclude, save them in container
container.append(before)
new_trans_ptr = len(container)
# get attributes from page node;
# we expect {class: "moin-transclusion"; data-href: "http://some.org/somepage"}
attrib = Attributes(ret).convert()
# current elem will likely be replaced by container so we need to copy data-lineno attr
if html.data_lineno in elem.attrib:
attrib[html.data_lineno] = elem.attrib[
html.data_lineno]
# make new div node to hold transclusion, copy children, and save in container
div = ET.Element(moin_page.div,
attrib=attrib,
children=body[:])
container.append(
div) # new_trans_ptr is index to this
if len(after):
container.append(after)
if elem.tag.name == 'a':
# invalid input [[MyPage|{{BlockItem}}]],
# best option is to retain A-tag and fail html validation
# TODO: error may not be obvious to user - add error message
elem[i] = div
else:
# move up 1 level in recursion where elem becomes the child and
# is usually replaced by container
return [container, new_trans_ptr]
else:
# default action for inline transclusions or odd things like circular transclusion error messages
classes = child.attrib.get(html.class_,
'').split()
classes += ret.attrib.get(html.class_,
'').split()
ret.attrib[html.class_] = ' '.join(classes)
elem[i] = ret
elif isinstance(ret, types.ListType):
# a container has been returned.
# Note: there are multiple places where a container may be constructed
ret_container, trans_ptr = ret
# trans_ptr points to the transclusion within ret_container.
# Here the transclusion will always contain a block level element
if elem.tag.name in NO_BLOCK_CHILDREN:
# Complex case, transclusion effects grand-parent, great-grand-parent, e.g.:
# "/* comment {{BlockItem}} */" or "text ''italic {{BlockItem}} italic'' text"
# elem is an inline element, build a bigger container to replace elem's parent,
before = copy.deepcopy(elem)
after = copy.deepcopy(elem)
before[:] = elem[0:i] + ret_container[
0:trans_ptr]
after[:] = ret_container[trans_ptr +
1:] + elem[i + 1:]
if len(before):
container.append(before)
new_trans_ptr = len(container)
# child may have classes like "comment" that must be added to transcluded element
classes = child.attrib.get(
moin_page.class_, '').split()
# must use moin_page.class_ above, but use html.class below per html_out.py code
classes += ret_container[trans_ptr].attrib.get(
html.class_, '').split()
ret_container[trans_ptr].attrib[
html.class_] = ' '.join(classes)
container.append(ret_container[trans_ptr]
) # the transclusion
if len(after):
container.append(after)
return [container, new_trans_ptr]
else:
# elem is a block element
for grandchild in child:
if isinstance(
grandchild, ET.Node
) and grandchild.tag.name == u'include':
# the include may have classes that must be added to transcluded element
classes = grandchild.attrib.get(
html.class_, '').split()
classes += ret_container[
trans_ptr].attrib.get(
html.class_, '').split()
ret_container[trans_ptr].attrib[
html.class_] = ' '.join(classes)
# replace child element with the container generated in lower recursion
elem[i:i +
1] = ret_container # elem[i] is the child
else:
# default action for any ret not fitting special cases above,
# e.g. tranclusion is within a table cell
elem[i] = ret
# we are finished with this child, advance to next sibling
i += 1
finally:
self.stack.pop()