def report_failure(self, name, msg, show_traceback=True):
"Report a test failure, with some useful context."
orig_tb = _traceback.extract_tb(_sys.exc_info()[2])
tb = orig_tb
# Move up the traceback until we get to the line in the test
# function which caused the failure.
while tb[-1][2] != "test_" + name:
tb = tb[:-1]
# Display the context in the text function.
filepath, linenum, functionname, text = tb[-1]
filename = _os.path.basename(filepath)
self._out.ensure_space()
self._out.write_colour("#red#FAILED##\n")
if self._verbose > 0:
if self._context is None:
context = ""
else:
context = ", when %s" % self._context
firstline = "%s:%d" % (filename, linenum)
self._out.write("\n%s:%s%s\n" % (firstline, msg, context))
# Display sourcecode lines
lines = open(filepath).readlines()
startline = max(linenum - 3, 0)
endline = min(linenum + 2, len(lines))
for num in range(startline, endline):
if num + 1 == linenum:
self._out.write("->")
else:
self._out.write(" ")
self._out.write("%4d %s\n" % (num + 1, lines[num].rstrip()))
# Display the traceback
if show_traceback:
self._out.write("Traceback (most recent call last):\n")
for line in _traceback.format_list(orig_tb):
self._out.write(line.rstrip() + "\n")
self._out.write("\n")
# Display some information about the xapian version and platform
self._out.write("Xapian version: %s\n" % _xapian.version_string())
try:
import platform
platdesc = "%s %s (%s)" % platform.system_alias(
platform.system(), platform.release(), platform.version()
)
self._out.write("Platform: %s\n" % platdesc)
except:
pass
self._out.write(
'\nWhen reporting this problem, please quote all the preceding lines from\n"%s" onwards.\n\n'
% firstline
)
self._out.flush()
def version_info(self):
"""Get version information about the backend.
"""
return 'xappy %s, xapian %s' % (
xappy.__version__,
xapian.version_string(),
)
def report_failure(self, name, msg, show_traceback=True):
"Report a test failure, with some useful context."
tb = _traceback.extract_tb(_sys.exc_info()[2])
# Move up the traceback until we get to the line in the test
# function which caused the failure.
for line in xrange(1, len(tb) + 1):
if tb[-line][2] == 'test_' + name:
break
# Display the context in the text function.
filepath, linenum, functionname, text = tb[-line]
filename = _os.path.basename(filepath)
self._out.ensure_space()
self._out.write_colour("#red#FAILED##\n")
if self._verbose > 0:
if self._context is None:
context = ''
else:
context = ", when %s" % self._context
firstline = "%s:%d" % (filename, linenum)
self._out.write("\n%s:%s%s\n" % (firstline, msg, context))
# Display sourcecode lines
lines = open(filepath).readlines()
startline = max(linenum - 3, 0)
endline = min(linenum + 2, len(lines))
for num in range(startline, endline):
if num + 1 == linenum:
self._out.write('->')
else:
self._out.write(' ')
self._out.write("%4d %s\n" % (num + 1, lines[num].rstrip()))
# Display the traceback
if show_traceback:
self._out.write("Traceback (most recent call last):\n")
for line in _traceback.format_list(tb):
self._out.write(line.rstrip() + '\n')
self._out.write('\n')
# Display some information about the xapian version and platform
self._out.write("Xapian version: %s\n" % _xapian.version_string())
try:
import platform
platdesc = "%s %s (%s)" % platform.system_alias(
platform.system(), platform.release(), platform.version())
self._out.write("Platform: %s\n" % platdesc)
except:
pass
self._out.write(
'\nWhen reporting this problem, please quote all the preceding lines from\n"%s" onwards.\n\n'
% firstline)
self._out.flush()
def get_status_of_module(module, prefix=None, version=True):
modules = list(sys.modules.keys())
is_enabled = module in modules
if is_enabled:
if module == 'pyme':
from pyme import version
version = "version %s" % version.versionstr
elif module == 'MySQLdb':
from MySQLdb import version_info
version = "version %s" % ".".join(
[str(v) for v in version_info])
elif module == 'pychart':
from pychart import version
version = "version %s" % version.version
elif module == 'sqlite3':
from sqlite3 import version
version = "version %s" % version
elif module == "whoosh":
from whoosh import versionstring
version = "version %s" % versionstring()
elif module == 'xapian':
from xapian import version_string
version = "version %s" % version_string()
else:
if version:
m = __import__(module)
try:
version = "version %s" % m.__version__
except AttributeError:
version = "version unavailable - exception thrown"
else:
version = "version unavailable"
if prefix:
return "%s %s %s enabled: %s" % (prefix, module, version,
is_enabled)
else:
return "Module: %s %s enabled: %s" % (module, version,
is_enabled)
else:
if prefix:
return "%s %s enabled: %s" % (prefix, module, is_enabled)
else:
return "Module: %s enabled: %s" % (module, is_enabled)
# Distutils script for building packaged Xapian python bindings for Win32
from distutils.core import setup
from distutils.command.install import INSTALL_SCHEMES
import xapian
import platform
import glob
description = ("""
Win32 MSVC build of Python %s Bindings for the Xapian Open Source Search Engine Library (https://xapian.org/)
""" % platform.python_version())
for scheme in INSTALL_SCHEMES.values():
scheme['data'] = scheme['purelib']
setup(
name = "xapian-python-bindings for Python %s " % platform.python_version(),
version = xapian.version_string(),
url = 'https://xapian.org/',
maintainer = 'Charlie Hull (with thanks to Alexandre Gauthier)',
maintainer_email = '*****@*****.**',
description = description.strip(),
py_modules = ['xapian'],
packages = ['xapian'],
package_dir = {'xapian': 'xapian'},
package_data = {'xapian': ['_xapian.pyd', 'zlib1.dll','docs/index.html','docs/examples/*.*']},
)
def test_all():
# Test the version number reporting functions give plausible results.
v = "%d.%d.%d" % (xapian.major_version(),
xapian.minor_version(),
xapian.revision())
v2 = xapian.version_string()
expect(v2, v, "Unexpected version output")
stem = xapian.Stem("english")
expect(stem.get_description(), "Xapian::Stem(english)", "Unexpected stem.get_description()")
doc = xapian.Document()
doc.set_data("a\0b")
if doc.get_data() == "a":
raise TestFail("get_data+set_data truncates at a zero byte")
expect(doc.get_data(), "a\0b", "get_data+set_data doesn't transparently handle a zero byte")
doc.set_data("is there anybody out there?")
doc.add_term("XYzzy")
doc.add_posting(stem("is"), 1)
doc.add_posting(stem("there"), 2)
doc.add_posting(stem("anybody"), 3)
doc.add_posting(stem("out"), 4)
doc.add_posting(stem("there"), 5)
db = xapian.inmemory_open()
db.add_document(doc)
expect(db.get_doccount(), 1, "Unexpected db.get_doccount()")
terms = ["smoke", "test", "terms"]
expect_query(xapian.Query(xapian.Query.OP_OR, terms),
"(smoke OR test OR terms)")
query1 = xapian.Query(xapian.Query.OP_PHRASE, ("smoke", "test", "tuple"))
query2 = xapian.Query(xapian.Query.OP_XOR, (xapian.Query("smoke"), query1, "string"))
expect_query(query1, "(smoke PHRASE 3 test PHRASE 3 tuple)")
expect_query(query2, "(smoke XOR (smoke PHRASE 3 test PHRASE 3 tuple) XOR string)")
subqs = ["a", "b"]
expect_query(xapian.Query(xapian.Query.OP_OR, subqs), "(a OR b)")
expect_query(xapian.Query(xapian.Query.OP_VALUE_RANGE, 0, '1', '4'),
"VALUE_RANGE 0 1 4")
# Feature test for Query.__iter__
term_count = 0
for term in query2:
term_count += 1
expect(term_count, 4, "Unexpected number of terms in query2")
enq = xapian.Enquire(db)
enq.set_query(xapian.Query(xapian.Query.OP_OR, "there", "is"))
mset = enq.get_mset(0, 10)
expect(mset.size(), 1, "Unexpected mset.size()")
# Feature test for Enquire.matching_terms(docid)
term_count = 0
for term in enq.matching_terms(mset.get_hit(0)):
term_count += 1
expect(term_count, 2, "Unexpected number of matching terms")
# Feature test for MSet.__iter__
msize = 0
for match in mset:
msize += 1
expect(msize, mset.size(), "Unexpected number of entries in mset")
terms = " ".join(enq.matching_terms(mset.get_hit(0)))
expect(terms, "is there", "Unexpected terms")
# Feature test for ESet.__iter__
rset = xapian.RSet()
rset.add_document(1)
eset = enq.get_eset(10, rset)
term_count = 0
for term in eset:
term_count += 1
expect(term_count, 3, "Unexpected number of expand terms")
# Feature test for Database.__iter__
term_count = 0
for term in db:
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db")
# Feature test for Database.allterms
term_count = 0
for term in db.allterms():
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db.allterms")
# Feature test for Database.postlist
count = 0
for posting in db.postlist("there"):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('there')")
# Feature test for Database.postlist with empty term (alldocspostlist)
count = 0
for posting in db.postlist(""):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('')")
# Feature test for Database.termlist
count = 0
for term in db.termlist(1):
count += 1
expect(count, 5, "Unexpected number of entries in db.termlist(1)")
# Feature test for Database.positionlist
count = 0
for term in db.positionlist(1, "there"):
count += 1
expect(count, 2, "Unexpected number of entries in db.positionlist(1, 'there')")
# Feature test for Document.termlist
count = 0
for term in doc.termlist():
count += 1
expect(count, 5, "Unexpected number of entries in doc.termlist()")
# Feature test for TermIter.skip_to
term = doc.termlist()
term.skip_to('n')
while True:
try:
x = term.next()
except StopIteration:
break
if x.term < 'n':
raise TestFail("TermIter.skip_to didn't skip term '%s'" % x.term)
# Feature test for Document.values
count = 0
for term in doc.values():
count += 1
expect(count, 0, "Unexpected number of entries in doc.values")
# Check exception handling for Xapian::DocNotFoundError
expect_exception(xapian.DocNotFoundError, "Docid 3 not found", db.get_document, 3)
# Check value of OP_ELITE_SET
expect(xapian.Query.OP_ELITE_SET, 10, "Unexpected value for OP_ELITE_SET")
# Feature test for MatchDecider
doc = xapian.Document()
doc.set_data("Two")
doc.add_posting(stem("out"), 1)
doc.add_posting(stem("outside"), 1)
doc.add_posting(stem("source"), 2)
doc.add_value(0, "yes")
db.add_document(doc)
class testmatchdecider(xapian.MatchDecider):
def __call__(self, doc):
return doc.get_value(0) == "yes"
query = xapian.Query(stem("out"))
enquire = xapian.Enquire(db)
enquire.set_query(query)
mset = enquire.get_mset(0, 10, None, testmatchdecider())
expect(mset.size(), 1, "Unexpected number of documents returned by match decider")
expect(mset.get_docid(0), 2, "MatchDecider mset has wrong docid in")
# Feature test for ExpandDecider
class testexpanddecider(xapian.ExpandDecider):
def __call__(self, term):
return (not term.startswith('a'))
enquire = xapian.Enquire(db)
rset = xapian.RSet()
rset.add_document(1)
eset = enquire.get_eset(10, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0, testexpanddecider())
eset_terms = [term[xapian.ESET_TNAME] for term in eset.items]
expect(len(eset_terms), eset.size(), "Unexpected number of terms returned by expand")
if filter(lambda t: t.startswith('a'), eset_terms):
raise TestFail("ExpandDecider was not used")
# Check QueryParser parsing error.
qp = xapian.QueryParser()
expect_exception(xapian.QueryParserError, "Syntax: <expression> AND <expression>", qp.parse_query, "test AND")
# Check QueryParser pure NOT option
qp = xapian.QueryParser()
expect_query(qp.parse_query("NOT test", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
"(<alldocuments> AND_NOT test:(pos=1))")
# Check QueryParser partial option
qp = xapian.QueryParser()
qp.set_database(db)
qp.set_default_op(xapian.Query.OP_AND)
qp.set_stemming_strategy(qp.STEM_SOME)
qp.set_stemmer(xapian.Stem('en'))
expect_query(qp.parse_query("foo o", qp.FLAG_PARTIAL),
"(Zfoo:(pos=1) AND (out:(pos=2) OR outsid:(pos=2) OR Zo:(pos=2)))")
expect_query(qp.parse_query("foo outside", qp.FLAG_PARTIAL),
"(Zfoo:(pos=1) AND Zoutsid:(pos=2))")
# Test supplying unicode strings
expect_query(xapian.Query(xapian.Query.OP_OR, (u'foo', u'bar')),
'(foo OR bar)')
expect_query(xapian.Query(xapian.Query.OP_OR, ('foo', u'bar\xa3')),
'(foo OR bar\xc2\xa3)')
expect_query(xapian.Query(xapian.Query.OP_OR, ('foo', 'bar\xc2\xa3')),
'(foo OR bar\xc2\xa3)')
expect_query(xapian.Query(xapian.Query.OP_OR, u'foo', u'bar'),
'(foo OR bar)')
expect_query(qp.parse_query(u"NOT t\xe9st", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
"(<alldocuments> AND_NOT Zt\xc3\xa9st:(pos=1))")
doc = xapian.Document()
doc.set_data(u"Unicode with an acc\xe9nt")
doc.add_posting(stem(u"out\xe9r"), 1)
expect(doc.get_data(), u"Unicode with an acc\xe9nt".encode('utf-8'))
term = doc.termlist().next().term
expect(term, u"out\xe9r".encode('utf-8'))
# Check simple stopper
stop = xapian.SimpleStopper()
qp.set_stopper(stop)
expect(stop('a'), False)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo:(pos=1) AND Zbar:(pos=2) AND Za:(pos=3))")
stop.add('a')
expect(stop('a'), True)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo:(pos=1) AND Zbar:(pos=2))")
# Feature test for custom Stopper
class my_b_stopper(xapian.Stopper):
def __call__(self, term):
return term == "b"
def get_description(self):
return u"my_b_stopper"
stop = my_b_stopper()
expect(stop.get_description(), u"my_b_stopper")
qp.set_stopper(stop)
expect(stop('a'), False)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo:(pos=1) AND Zbar:(pos=2) AND Za:(pos=3))")
expect(stop('b'), True)
expect_query(qp.parse_query(u"foo bar b", qp.FLAG_BOOLEAN),
"(Zfoo:(pos=1) AND Zbar:(pos=2))")
# Test TermGenerator
termgen = xapian.TermGenerator()
doc = xapian.Document()
termgen.set_document(doc)
termgen.index_text('foo bar baz foo')
expect([(item.term, item.wdf, [pos for pos in item.positer]) for item in doc.termlist()], [('bar', 1, [2]), ('baz', 1, [3]), ('foo', 2, [1, 4])])
# Check DateValueRangeProcessor works
context("checking that DateValueRangeProcessor works")
qp = xapian.QueryParser()
vrpdate = xapian.DateValueRangeProcessor(1, 1, 1960)
qp.add_valuerangeprocessor(vrpdate)
query = qp.parse_query('12/03/99..12/04/01')
expect(str(query), 'Xapian::Query(VALUE_RANGE 1 19991203 20011204)')
# Regression test for bug#193, fixed in 1.0.3.
context("running regression test for bug#193")
vrp = xapian.NumberValueRangeProcessor(0, '$', True)
a = '$10'
b = '20'
slot, a, b = vrp(a, b)
expect(slot, 0)
expect(xapian.sortable_unserialise(a), 10)
expect(xapian.sortable_unserialise(b), 20)
# Regression tests copied from PHP (probably always worked in python, but
# let's check...)
context("running regression tests for issues which were found in PHP")
# PHP overload resolution involving boolean types failed.
enq.set_sort_by_value(1, True)
# Regression test - fixed in 0.9.10.1.
oqparser = xapian.QueryParser()
oquery = oqparser.parse_query("I like tea")
# Regression test for bug#192 - fixed in 1.0.3.
enq.set_cutoff(100)
# Test setting and getting metadata
expect(db.get_metadata('Foo'), '')
db.set_metadata('Foo', 'Foo')
expect(db.get_metadata('Foo'), 'Foo')
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, '')
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.set_metadata, '', 'Foo')
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, '')
# Test OP_SCALE_WEIGHT and corresponding constructor
expect_query(xapian.Query(xapian.Query.OP_SCALE_WEIGHT, xapian.Query('foo'), 5),
"5 * foo")
def getXapianVersion(): import xapian return versionStringToTuple(xapian.version_string())
def test_all():
# Test the version number reporting functions give plausible results.
v = "%d.%d.%d" % (xapian.major_version(), xapian.minor_version(),
xapian.revision())
v2 = xapian.version_string()
expect(v2, v, "Unexpected version output")
# A regexp check would be better, but seems to create a bogus "leak" of -1
# objects in Python 3.
expect(len(xapian.__version__.split('.')), 3,
'xapian.__version__ not X.Y.Z')
expect((xapian.__version__.split('.'))[0], '1',
'xapian.__version__ not "1.Y.Z"')
def access_cvar():
res = xapian.cvar
print("Unhandled constants: ", res)
return res
# Check that SWIG isn't generating cvar (regression test for ticket#297).
#
# Python 3.5 generates a different exception message here to earlier
# versions, so we need a check which matches both.
expect_exception(AttributeError,
lambda msg: msg.find("has no attribute 'cvar'") != -1,
access_cvar)
stem = xapian.Stem(b"english")
expect(str(stem), "Xapian::Stem(english)", "Unexpected str(stem)")
doc = xapian.Document()
doc.set_data(b"a\0b")
if doc.get_data() == b"a":
raise TestFail("get_data+set_data truncates at a zero byte")
expect(doc.get_data(), b"a\0b",
"get_data+set_data doesn't transparently handle a zero byte")
doc.set_data(b"is there anybody out there?")
doc.add_term(b"XYzzy")
doc.add_posting(stem(b"is"), 1)
doc.add_posting(stem(b"there"), 2)
doc.add_posting(stem(b"anybody"), 3)
doc.add_posting(stem(b"out"), 4)
doc.add_posting(stem(b"there"), 5)
db = xapian.WritableDatabase('', xapian.DB_BACKEND_INMEMORY)
db.add_document(doc)
expect(db.get_doccount(), 1, "Unexpected db.get_doccount()")
terms = ["smoke", "test", "terms"]
expect_query(
xapian.Query(xapian.Query.OP_OR, [t.encode('utf-8') for t in terms]),
"(smoke OR test OR terms)")
query1 = xapian.Query(xapian.Query.OP_PHRASE,
(b"smoke", b"test", b"tuple"))
query2 = xapian.Query(xapian.Query.OP_XOR,
(xapian.Query(b"smoke"), query1, b"string"))
expect_query(query1, "(smoke PHRASE 3 test PHRASE 3 tuple)")
expect_query(
query2, "(smoke XOR (smoke PHRASE 3 test PHRASE 3 tuple) XOR string)")
subqs = ["a", "b"]
expect_query(
xapian.Query(xapian.Query.OP_OR, [s.encode('utf-8') for s in subqs]),
"(a OR b)")
expect_query(xapian.Query(xapian.Query.OP_VALUE_RANGE, 0, b'1', b'4'),
"VALUE_RANGE 0 1 4")
# Check database factory functions are wrapped as expected (or not wrapped
# in the first cases):
expect_exception(
AttributeError,
lambda msg: msg.find("has no attribute 'open_stub'") != -1,
lambda: xapian.open_stub(b"nosuchdir/nosuchdb"))
expect_exception(
AttributeError,
lambda msg: msg.find("has no attribute 'open_stub'") != -1,
lambda: xapian.open_stub(b"nosuchdir/nosuchdb", xapian.DB_OPEN))
expect_exception(
xapian.DatabaseOpeningError, None,
lambda: xapian.Database(b"nosuchdir/nosuchdb", xapian.DB_BACKEND_STUB))
expect_exception(
xapian.DatabaseOpeningError, None, lambda: xapian.WritableDatabase(
b"nosuchdir/nosuchdb", xapian.DB_OPEN | xapian.DB_BACKEND_STUB))
expect_exception(
xapian.DatabaseOpeningError, None, lambda: xapian.Database(
b"nosuchdir/nosuchdb", xapian.DB_BACKEND_GLASS))
expect_exception(
xapian.DatabaseCreateError, None, lambda: xapian.WritableDatabase(
b"nosuchdir/nosuchdb", xapian.DB_CREATE | xapian.DB_BACKEND_GLASS))
expect_exception(
xapian.FeatureUnavailableError, None, lambda: xapian.Database(
b"nosuchdir/nosuchdb", xapian.DB_BACKEND_CHERT))
expect_exception(
xapian.FeatureUnavailableError, None, lambda: xapian.WritableDatabase(
b"nosuchdir/nosuchdb", xapian.DB_CREATE | xapian.DB_BACKEND_CHERT))
expect_exception(xapian.NetworkError, None, xapian.remote_open,
b"/bin/false", b"")
expect_exception(xapian.NetworkError, None, xapian.remote_open_writable,
b"/bin/false", b"")
expect_exception(xapian.NetworkError, None, xapian.remote_open,
b"127.0.0.1", 0, 1)
expect_exception(xapian.NetworkError, None, xapian.remote_open_writable,
b"127.0.0.1", 0, 1)
# Check wrapping of MatchAll and MatchNothing:
expect_query(xapian.Query.MatchAll, "<alldocuments>")
expect_query(xapian.Query.MatchNothing, "")
# Feature test for Query.__iter__
term_count = 0
for term in query2:
term_count += 1
expect(term_count, 4, "Unexpected number of terms in query2")
enq = xapian.Enquire(db)
enq.set_query(xapian.Query(xapian.Query.OP_OR, b"there", b"is"))
mset = enq.get_mset(0, 10)
expect(mset.size(), 1, "Unexpected mset.size()")
expect(len(mset), 1, "Unexpected mset.size()")
# Feature test for Enquire.matching_terms(docid)
term_count = 0
for term in enq.matching_terms(mset.get_hit(0)):
term_count += 1
expect(term_count, 2, "Unexpected number of matching terms")
# Feature test for MSet.__iter__
msize = 0
for match in mset:
msize += 1
expect(msize, mset.size(), "Unexpected number of entries in mset")
terms = b" ".join(enq.matching_terms(mset.get_hit(0)))
expect(terms, b"is there", "Unexpected terms")
# Feature test for ESet.__iter__
rset = xapian.RSet()
rset.add_document(1)
eset = enq.get_eset(10, rset)
term_count = 0
for term in eset:
term_count += 1
expect(term_count, 3, "Unexpected number of expand terms")
# Feature test for Database.__iter__
term_count = 0
for term in db:
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db")
# Feature test for Database.allterms
term_count = 0
for term in db.allterms():
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db.allterms")
# Feature test for Database.postlist
count = 0
for posting in db.postlist(b"there"):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('there')")
# Feature test for Database.postlist with empty term (alldocspostlist)
count = 0
for posting in db.postlist(b""):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('')")
# Feature test for Database.termlist
count = 0
for term in db.termlist(1):
count += 1
expect(count, 5, "Unexpected number of entries in db.termlist(1)")
# Feature test for Database.positionlist
count = 0
for term in db.positionlist(1, b"there"):
count += 1
expect(count, 2,
"Unexpected number of entries in db.positionlist(1, 'there')")
# Feature test for Document.termlist
count = 0
for term in doc.termlist():
count += 1
expect(count, 5, "Unexpected number of entries in doc.termlist()")
# Feature test for TermIter.skip_to
term = doc.termlist()
term.skip_to(b'n')
while True:
try:
x = next(term)
except StopIteration:
break
if x.term < b'n':
raise TestFail("TermIter.skip_to didn't skip term '%s'" %
x.term.decode('utf-8'))
# Feature test for Document.values
count = 0
for term in list(doc.values()):
count += 1
expect(count, 0, "Unexpected number of entries in doc.values")
# Check exception handling for Xapian::DocNotFoundError
expect_exception(xapian.DocNotFoundError, "Docid 3 not found",
db.get_document, 3)
# Check value of OP_ELITE_SET
expect(xapian.Query.OP_ELITE_SET, 10, "Unexpected value for OP_ELITE_SET")
# Feature test for MatchDecider
doc = xapian.Document()
doc.set_data(b"Two")
doc.add_posting(stem(b"out"), 1)
doc.add_posting(stem(b"outside"), 1)
doc.add_posting(stem(b"source"), 2)
doc.add_value(0, b"yes")
db.add_document(doc)
class testmatchdecider(xapian.MatchDecider):
def __call__(self, doc):
return doc.get_value(0) == b"yes"
query = xapian.Query(stem(b"out"))
enquire = xapian.Enquire(db)
enquire.set_query(query)
mset = enquire.get_mset(0, 10, None, testmatchdecider())
expect(mset.size(), 1,
"Unexpected number of documents returned by match decider")
expect(mset.get_docid(0), 2, "MatchDecider mset has wrong docid in")
# Feature test for ExpandDecider
class testexpanddecider(xapian.ExpandDecider):
def __call__(self, term):
return (not term.startswith(b'a'))
enquire = xapian.Enquire(db)
rset = xapian.RSet()
rset.add_document(1)
eset = enquire.get_eset(10, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0,
testexpanddecider())
eset_terms = [item.term for item in eset]
expect(len(eset_terms), eset.size(),
"Unexpected number of terms returned by expand")
if [t for t in eset_terms if t.startswith(b'a')]:
raise TestFail("ExpandDecider was not used")
# Check min_wt argument to get_eset() works (new in 1.2.5).
eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ)
expect([i.weight for i in eset][-1] < 1.9, True,
"test get_eset() without min_wt")
eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0,
None, 1.9)
expect([i.weight for i in eset][-1] >= 1.9, True, "test get_eset() min_wt")
# Check QueryParser parsing error.
qp = xapian.QueryParser()
expect_exception(xapian.QueryParserError,
"Syntax: <expression> AND <expression>", qp.parse_query,
b"test AND")
# Check QueryParser pure NOT option
qp = xapian.QueryParser()
expect_query(
qp.parse_query(b"NOT test", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
"(<alldocuments> AND_NOT test@1)")
# Check QueryParser partial option
qp = xapian.QueryParser()
qp.set_database(db)
qp.set_default_op(xapian.Query.OP_AND)
qp.set_stemming_strategy(qp.STEM_SOME)
qp.set_stemmer(xapian.Stem(b'en'))
expect_query(qp.parse_query(b"foo o", qp.FLAG_PARTIAL),
"(Zfoo@1 AND ((SYNONYM WILDCARD OR o) OR Zo@2))")
expect_query(qp.parse_query(b"foo outside", qp.FLAG_PARTIAL),
"(Zfoo@1 AND ((SYNONYM WILDCARD OR outside) OR Zoutsid@2))")
# Test supplying unicode strings
expect_query(xapian.Query(xapian.Query.OP_OR, (b'foo', b'bar')),
'(foo OR bar)')
expect_query(xapian.Query(xapian.Query.OP_OR, (b'foo', b'bar\xa3')),
'(foo OR bar\\xa3)')
expect_query(xapian.Query(xapian.Query.OP_OR, (b'foo', b'bar\xc2\xa3')),
'(foo OR bar\u00a3)')
expect_query(xapian.Query(xapian.Query.OP_OR, b'foo', b'bar'),
'(foo OR bar)')
expect_query(
qp.parse_query(b"NOT t\xe9st", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
"(<alldocuments> AND_NOT Zt\u00e9st@1)")
doc = xapian.Document()
doc.set_data(b"Unicode with an acc\xe9nt")
doc.add_posting(stem(b"out\xe9r"), 1)
expect(doc.get_data(), b"Unicode with an acc\xe9nt")
term = next(doc.termlist()).term
expect(term, b"out\xe9r")
# Check simple stopper
stop = xapian.SimpleStopper()
qp.set_stopper(stop)
expect(stop(b'a'), False)
expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2 AND Za@3)")
stop.add(b'a')
expect(stop(b'a'), True)
expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2)")
# Feature test for custom Stopper
class my_b_stopper(xapian.Stopper):
def __call__(self, term):
return term == b"b"
def get_description(self):
return "my_b_stopper"
stop = my_b_stopper()
expect(stop.get_description(), "my_b_stopper")
qp.set_stopper(stop)
expect(stop(b'a'), False)
expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2 AND Za@3)")
expect(stop(b'b'), True)
expect_query(qp.parse_query(b"foo bar b", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2)")
# Test TermGenerator
termgen = xapian.TermGenerator()
doc = xapian.Document()
termgen.set_document(doc)
termgen.index_text(b'foo bar baz foo')
expect([(item.term, item.wdf, [pos for pos in item.positer])
for item in doc.termlist()], [(b'bar', 1, [2]), (b'baz', 1, [3]),
(b'foo', 2, [1, 4])])
# Check DateValueRangeProcessor works
context("checking that DateValueRangeProcessor works")
qp = xapian.QueryParser()
vrpdate = xapian.DateValueRangeProcessor(1, 1, 1960)
qp.add_valuerangeprocessor(vrpdate)
query = qp.parse_query(b'12/03/99..12/04/01')
expect(str(query), 'Query(VALUE_RANGE 1 19991203 20011204)')
# Regression test for bug#193, fixed in 1.0.3.
context("running regression test for bug#193")
vrp = xapian.NumberValueRangeProcessor(0, b'$', True)
a = '$10'
b = '20'
slot, a, b = vrp(a, b.encode('utf-8'))
expect(slot, 0)
expect(xapian.sortable_unserialise(a), 10)
expect(xapian.sortable_unserialise(b), 20)
# Feature test for xapian.FieldProcessor
context("running feature test for xapian.FieldProcessor")
class testfieldprocessor(xapian.FieldProcessor):
def __call__(self, s):
if s == 'spam':
raise Exception('already spam')
return xapian.Query("spam")
qp.add_prefix('spam', testfieldprocessor())
qp.add_boolean_prefix('boolspam', testfieldprocessor())
query = qp.parse_query('spam:ignored')
expect(str(query), 'Query(spam)')
# FIXME: This doesn't currently work:
# expect_exception(Exception, 'already spam', qp.parse_query, 'spam:spam')
# Regression tests copied from PHP (probably always worked in python, but
# let's check...)
context("running regression tests for issues which were found in PHP")
# PHP overload resolution involving boolean types failed.
enq.set_sort_by_value(1, True)
# Regression test - fixed in 0.9.10.1.
oqparser = xapian.QueryParser()
oquery = oqparser.parse_query(b"I like tea")
# Regression test for bug#192 - fixed in 1.0.3.
enq.set_cutoff(100)
# Test setting and getting metadata
expect(db.get_metadata(b'Foo'), b'')
db.set_metadata(b'Foo', b'Foo')
expect(db.get_metadata(b'Foo'), b'Foo')
expect_exception(xapian.InvalidArgumentError,
"Empty metadata keys are invalid", db.get_metadata, b'')
expect_exception(xapian.InvalidArgumentError,
"Empty metadata keys are invalid", db.set_metadata, b'',
b'Foo')
expect_exception(xapian.InvalidArgumentError,
"Empty metadata keys are invalid", db.get_metadata, b'')
# Test OP_SCALE_WEIGHT and corresponding constructor
expect_query(
xapian.Query(xapian.Query.OP_SCALE_WEIGHT, xapian.Query(b'foo'), 5),
"5 * foo")
def test_all():
# Test the version number reporting functions give plausible results.
v = "%d.%d.%d" % (xapian.major_version(),
xapian.minor_version(),
xapian.revision())
v2 = xapian.version_string()
expect(v2, v, "Unexpected version output")
# A regexp check would be better, but seems to create a bogus "leak" of -1
# objects in Python 3.
expect(len(xapian.__version__.split('.')), 3, 'xapian.__version__ not X.Y.Z')
expect((xapian.__version__.split('.'))[0], '1', 'xapian.__version__ not "1.Y.Z"')
def access_cvar():
res = xapian.cvar
print("Unhandled constants: ", res)
return res
# Check that SWIG isn't generating cvar (regression test for ticket#297).
expect_exception(AttributeError, "'module' object has no attribute 'cvar'",
access_cvar)
stem = xapian.Stem(b"english")
expect(str(stem), "Xapian::Stem(english)", "Unexpected str(stem)")
doc = xapian.Document()
doc.set_data(b"a\0b")
if doc.get_data() == b"a":
raise TestFail("get_data+set_data truncates at a zero byte")
expect(doc.get_data(), b"a\0b", "get_data+set_data doesn't transparently handle a zero byte")
doc.set_data(b"is there anybody out there?")
doc.add_term(b"XYzzy")
doc.add_posting(stem(b"is"), 1)
doc.add_posting(stem(b"there"), 2)
doc.add_posting(stem(b"anybody"), 3)
doc.add_posting(stem(b"out"), 4)
doc.add_posting(stem(b"there"), 5)
db = xapian.inmemory_open()
db.add_document(doc)
expect(db.get_doccount(), 1, "Unexpected db.get_doccount()")
terms = ["smoke", "test", "terms"]
expect_query(xapian.Query(xapian.Query.OP_OR, [t.encode('utf-8') for t in terms]),
"(smoke OR test OR terms)")
query1 = xapian.Query(xapian.Query.OP_PHRASE, (b"smoke", b"test", b"tuple"))
query2 = xapian.Query(xapian.Query.OP_XOR, (xapian.Query(b"smoke"), query1, b"string"))
expect_query(query1, "(smoke PHRASE 3 test PHRASE 3 tuple)")
expect_query(query2, "(smoke XOR (smoke PHRASE 3 test PHRASE 3 tuple) XOR string)")
subqs = ["a", "b"]
expect_query(xapian.Query(xapian.Query.OP_OR, [s.encode('utf-8') for s in subqs]), "(a OR b)")
expect_query(xapian.Query(xapian.Query.OP_VALUE_RANGE, 0, b'1', b'4'),
"VALUE_RANGE 0 1 4")
# Check database factory functions are wrapped as expected (or not wrapped
# in the first cases):
expect_exception(AttributeError, "'module' object has no attribute 'open_stub'",
lambda : xapian.open_stub(b"nosuchdir/nosuchdb"))
expect_exception(AttributeError, "'module' object has no attribute 'open_stub'",
lambda : xapian.open_stub(b"nosuchdir/nosuchdb", xapian.DB_OPEN))
expect_exception(AttributeError, "'module' object has no attribute 'chert_open'",
lambda : xapian.chert_open(b"nosuchdir/nosuchdb"))
expect_exception(AttributeError, "'module' object has no attribute 'chert_open'",
lambda : xapian.chert_open(b"nosuchdir/nosuchdb", xapian.DB_CREATE))
expect_exception(xapian.DatabaseOpeningError, None,
lambda : xapian.Database(b"nosuchdir/nosuchdb", xapian.DB_BACKEND_STUB))
expect_exception(xapian.DatabaseOpeningError, None,
lambda : xapian.WritableDatabase(b"nosuchdir/nosuchdb", xapian.DB_OPEN|xapian.DB_BACKEND_STUB))
expect_exception(xapian.DatabaseOpeningError, None,
lambda : xapian.Database(b"nosuchdir/nosuchdb", xapian.DB_BACKEND_GLASS))
expect_exception(xapian.DatabaseCreateError, None,
lambda : xapian.WritableDatabase(b"nosuchdir/nosuchdb", xapian.DB_CREATE|xapian.DB_BACKEND_GLASS))
expect_exception(xapian.DatabaseOpeningError, None,
lambda : xapian.Database(b"nosuchdir/nosuchdb", xapian.DB_BACKEND_CHERT))
expect_exception(xapian.DatabaseCreateError, None,
lambda : xapian.WritableDatabase(b"nosuchdir/nosuchdb", xapian.DB_CREATE|xapian.DB_BACKEND_CHERT))
expect_exception(xapian.NetworkError, None,
xapian.remote_open, b"/bin/false", b"")
expect_exception(xapian.NetworkError, None,
xapian.remote_open_writable, b"/bin/false", b"")
expect_exception(xapian.NetworkError, None,
xapian.remote_open, b"127.0.0.1", 0, 1)
expect_exception(xapian.NetworkError, None,
xapian.remote_open_writable, b"127.0.0.1", 0, 1)
# Check wrapping of MatchAll and MatchNothing:
expect_query(xapian.Query.MatchAll, "<alldocuments>")
expect_query(xapian.Query.MatchNothing, "")
# Feature test for Query.__iter__
term_count = 0
for term in query2:
term_count += 1
expect(term_count, 4, "Unexpected number of terms in query2")
enq = xapian.Enquire(db)
enq.set_query(xapian.Query(xapian.Query.OP_OR, b"there", b"is"))
mset = enq.get_mset(0, 10)
expect(mset.size(), 1, "Unexpected mset.size()")
expect(len(mset), 1, "Unexpected mset.size()")
# Feature test for Enquire.matching_terms(docid)
term_count = 0
for term in enq.matching_terms(mset.get_hit(0)):
term_count += 1
expect(term_count, 2, "Unexpected number of matching terms")
# Feature test for MSet.__iter__
msize = 0
for match in mset:
msize += 1
expect(msize, mset.size(), "Unexpected number of entries in mset")
terms = b" ".join(enq.matching_terms(mset.get_hit(0)))
expect(terms, b"is there", "Unexpected terms")
# Feature test for ESet.__iter__
rset = xapian.RSet()
rset.add_document(1)
eset = enq.get_eset(10, rset)
term_count = 0
for term in eset:
term_count += 1
expect(term_count, 3, "Unexpected number of expand terms")
# Feature test for Database.__iter__
term_count = 0
for term in db:
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db")
# Feature test for Database.allterms
term_count = 0
for term in db.allterms():
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db.allterms")
# Feature test for Database.postlist
count = 0
for posting in db.postlist(b"there"):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('there')")
# Feature test for Database.postlist with empty term (alldocspostlist)
count = 0
for posting in db.postlist(b""):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('')")
# Feature test for Database.termlist
count = 0
for term in db.termlist(1):
count += 1
expect(count, 5, "Unexpected number of entries in db.termlist(1)")
# Feature test for Database.positionlist
count = 0
for term in db.positionlist(1, b"there"):
count += 1
expect(count, 2, "Unexpected number of entries in db.positionlist(1, 'there')")
# Feature test for Document.termlist
count = 0
for term in doc.termlist():
count += 1
expect(count, 5, "Unexpected number of entries in doc.termlist()")
# Feature test for TermIter.skip_to
term = doc.termlist()
term.skip_to(b'n')
while True:
try:
x = next(term)
except StopIteration:
break
if x.term < b'n':
raise TestFail("TermIter.skip_to didn't skip term '%s'" % x.term.decode('utf-8'))
# Feature test for Document.values
count = 0
for term in list(doc.values()):
count += 1
expect(count, 0, "Unexpected number of entries in doc.values")
# Check exception handling for Xapian::DocNotFoundError
expect_exception(xapian.DocNotFoundError, "Docid 3 not found", db.get_document, 3)
# Check value of OP_ELITE_SET
expect(xapian.Query.OP_ELITE_SET, 10, "Unexpected value for OP_ELITE_SET")
# Feature test for MatchDecider
doc = xapian.Document()
doc.set_data(b"Two")
doc.add_posting(stem(b"out"), 1)
doc.add_posting(stem(b"outside"), 1)
doc.add_posting(stem(b"source"), 2)
doc.add_value(0, b"yes")
db.add_document(doc)
class testmatchdecider(xapian.MatchDecider):
def __call__(self, doc):
return doc.get_value(0) == b"yes"
query = xapian.Query(stem(b"out"))
enquire = xapian.Enquire(db)
enquire.set_query(query)
mset = enquire.get_mset(0, 10, None, testmatchdecider())
expect(mset.size(), 1, "Unexpected number of documents returned by match decider")
expect(mset.get_docid(0), 2, "MatchDecider mset has wrong docid in")
# Feature test for ExpandDecider
class testexpanddecider(xapian.ExpandDecider):
def __call__(self, term):
return (not term.startswith(b'a'))
enquire = xapian.Enquire(db)
rset = xapian.RSet()
rset.add_document(1)
eset = enquire.get_eset(10, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0, testexpanddecider())
eset_terms = [item.term for item in eset]
expect(len(eset_terms), eset.size(), "Unexpected number of terms returned by expand")
if [t for t in eset_terms if t.startswith(b'a')]:
raise TestFail("ExpandDecider was not used")
# Check min_wt argument to get_eset() works (new in 1.2.5).
eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ)
expect([i.weight for i in eset][-1] < 1.9, True, "test get_eset() without min_wt")
eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0, None, 1.9)
expect([i.weight for i in eset][-1] >= 1.9, True, "test get_eset() min_wt")
# Check QueryParser parsing error.
qp = xapian.QueryParser()
expect_exception(xapian.QueryParserError, "Syntax: <expression> AND <expression>", qp.parse_query, b"test AND")
# Check QueryParser pure NOT option
qp = xapian.QueryParser()
expect_query(qp.parse_query(b"NOT test", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
"(<alldocuments> AND_NOT test@1)")
# Check QueryParser partial option
qp = xapian.QueryParser()
qp.set_database(db)
qp.set_default_op(xapian.Query.OP_AND)
qp.set_stemming_strategy(qp.STEM_SOME)
qp.set_stemmer(xapian.Stem(b'en'))
expect_query(qp.parse_query(b"foo o", qp.FLAG_PARTIAL),
"(Zfoo@1 AND ((out@2 SYNONYM outsid@2) OR Zo@2))")
expect_query(qp.parse_query(b"foo outside", qp.FLAG_PARTIAL),
"(Zfoo@1 AND Zoutsid@2)")
# Test supplying unicode strings
expect_query(xapian.Query(xapian.Query.OP_OR, (b'foo', b'bar')),
'(foo OR bar)')
expect_query(xapian.Query(xapian.Query.OP_OR, (b'foo', b'bar\xa3')),
'(foo OR bar\\xa3)')
expect_query(xapian.Query(xapian.Query.OP_OR, (b'foo', b'bar\xc2\xa3')),
'(foo OR bar\u00a3)')
expect_query(xapian.Query(xapian.Query.OP_OR, b'foo', b'bar'),
'(foo OR bar)')
expect_query(qp.parse_query(b"NOT t\xe9st", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
"(<alldocuments> AND_NOT Zt\u00e9st@1)")
doc = xapian.Document()
doc.set_data(b"Unicode with an acc\xe9nt")
doc.add_posting(stem(b"out\xe9r"), 1)
expect(doc.get_data(), b"Unicode with an acc\xe9nt")
term = next(doc.termlist()).term
expect(term, b"out\xe9r")
# Check simple stopper
stop = xapian.SimpleStopper()
qp.set_stopper(stop)
expect(stop(b'a'), False)
expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2 AND Za@3)")
stop.add(b'a')
expect(stop(b'a'), True)
expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2)")
# Feature test for custom Stopper
class my_b_stopper(xapian.Stopper):
def __call__(self, term):
return term == b"b"
def get_description(self):
return "my_b_stopper"
stop = my_b_stopper()
expect(stop.get_description(), "my_b_stopper")
qp.set_stopper(stop)
expect(stop(b'a'), False)
expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2 AND Za@3)")
expect(stop(b'b'), True)
expect_query(qp.parse_query(b"foo bar b", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2)")
# Test TermGenerator
termgen = xapian.TermGenerator()
doc = xapian.Document()
termgen.set_document(doc)
termgen.index_text(b'foo bar baz foo')
expect([(item.term, item.wdf, [pos for pos in item.positer]) for item in doc.termlist()], [(b'bar', 1, [2]), (b'baz', 1, [3]), (b'foo', 2, [1, 4])])
# Check DateValueRangeProcessor works
context("checking that DateValueRangeProcessor works")
qp = xapian.QueryParser()
vrpdate = xapian.DateValueRangeProcessor(1, 1, 1960)
qp.add_valuerangeprocessor(vrpdate)
query = qp.parse_query(b'12/03/99..12/04/01')
expect(str(query), 'Query(0 * VALUE_RANGE 1 19991203 20011204)')
# Regression test for bug#193, fixed in 1.0.3.
context("running regression test for bug#193")
vrp = xapian.NumberValueRangeProcessor(0, b'$', True)
a = '$10'
b = '20'
slot, a, b = vrp(a, b.encode('utf-8'))
expect(slot, 0)
expect(xapian.sortable_unserialise(a), 10)
expect(xapian.sortable_unserialise(b), 20)
# Feature test for xapian.FieldProcessor
context("running feature test for xapian.FieldProcessor")
class testfieldprocessor(xapian.FieldProcessor):
def __call__(self, s):
if s == 'spam':
raise Exception('already spam')
return xapian.Query("spam")
qp.add_prefix('spam', testfieldprocessor())
qp.add_boolean_prefix('boolspam', testfieldprocessor())
query = qp.parse_query('spam:ignored')
expect(str(query), 'Query(spam)')
# FIXME: This doesn't currently work:
# expect_exception(Exception, 'already spam', qp.parse_query, 'spam:spam')
# Regression tests copied from PHP (probably always worked in python, but
# let's check...)
context("running regression tests for issues which were found in PHP")
# PHP overload resolution involving boolean types failed.
enq.set_sort_by_value(1, True)
# Regression test - fixed in 0.9.10.1.
oqparser = xapian.QueryParser()
oquery = oqparser.parse_query(b"I like tea")
# Regression test for bug#192 - fixed in 1.0.3.
enq.set_cutoff(100)
# Test setting and getting metadata
expect(db.get_metadata(b'Foo'), b'')
db.set_metadata(b'Foo', b'Foo')
expect(db.get_metadata(b'Foo'), b'Foo')
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, b'')
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.set_metadata, b'', b'Foo')
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, b'')
# Test OP_SCALE_WEIGHT and corresponding constructor
expect_query(xapian.Query(xapian.Query.OP_SCALE_WEIGHT, xapian.Query(b'foo'), 5),
"5 * foo")
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
r"""_checkxapian.py: Check the version of xapian used.
Raises an ImportError on import if the version used is too old to be used at
all.
"""
__docformat__ = "restructuredtext en"
# The minimum version of xapian required to work at all.
min_xapian_version = (1, 0, 6)
# Dictionary of features we can't support do to them being missing from the
# available version of xapian.
missing_features = {}
import xapian
versions = xapian.major_version(), xapian.minor_version(), xapian.revision()
if versions < min_xapian_version:
raise ImportError("""
Xapian Python bindings installed, but need at least version %d.%d.%d - got %s
""".strip() % tuple(list(min_xapian_version) + [xapian.version_string()]))
if not hasattr(xapian, 'TermCountMatchSpy'):
missing_features['tags'] = 1
if not hasattr(xapian, 'CategorySelectMatchSpy'):
missing_features['facets'] = 1
# Distutils script for building packaged Xapian python bindings for Win32
from distutils.core import setup
from distutils.command.install import INSTALL_SCHEMES
import xapian
import platform
import glob
description = ("""
Win32 MSVC build of Python %s Bindings for the Xapian Open Source Search Engine Library (http://www.xapian.org/)
""" % platform.python_version())
for scheme in INSTALL_SCHEMES.values():
scheme['data'] = scheme['purelib']
setup(
name = "xapian-python-bindings for Python %s " % platform.python_version(),
version = xapian.version_string(),
url = 'http://xapian.org/',
maintainer = 'Charlie Hull (with thanks to Alexandre Gauthier)',
maintainer_email = '*****@*****.**',
description = description.strip(),
py_modules = ['xapian'],
packages = ['xapian'],
package_dir = {'xapian': 'xapian'},
package_data = {'xapian': ['_xapian.pyd', 'zlib1.dll','docs/index.html','docs/examples/*.*']},
)
def getInfo(self):
_ = self.request.getText
request = self.request
buf = StringIO()
row = lambda label, value, buf=buf: buf.write(u'<dt>%s</dt><dd>%s</dd>'
% (label, value))
buf.write(u'<dl>')
row(_('Python Version'), sys.version)
row(
_('MoinMoin Version'),
_('Release %s [Revision %s]') %
(version.release, version.revision))
if not request.user.valid:
# for an anonymous user it ends here.
buf.write(u'</dl>')
return buf.getvalue()
if request.user.isSuperUser():
# superuser gets all page dependent stuff only
try:
import Ft
ftversion = Ft.__version__
except ImportError:
ftversion = None
except AttributeError:
ftversion = 'N/A'
if ftversion:
row(_('4Suite Version'), ftversion)
# TODO add python-xml check and display it
# Get the full pagelist of the wiki
pagelist = request.rootpage.getPageList(user='')
systemPages = []
totalsize = 0
for page in pagelist:
if wikiutil.isSystemPage(request, page):
systemPages.append(page)
totalsize += Page(request, page).size()
row(_('Number of pages'), str(len(pagelist) - len(systemPages)))
row(_('Number of system pages'), str(len(systemPages)))
row(_('Accumulated page sizes'),
self.formatInReadableUnits(totalsize))
data_dir = request.cfg.data_dir
row(
_('Disk usage of %(data_dir)s/pages/') %
{'data_dir': data_dir},
self.formatInReadableUnits(
self.getDirectorySize(os.path.join(data_dir, 'pages'))))
row(
_('Disk usage of %(data_dir)s/') % {'data_dir': data_dir},
self.formatInReadableUnits(self.getDirectorySize(data_dir)))
edlog = editlog.EditLog(request)
row(
_('Entries in edit log'), "%s (%s)" %
(edlog.lines(), self.formatInReadableUnits(edlog.size())))
# This puts a heavy load on the server when the log is large
eventlogger = eventlog.EventLog(request)
row('Event log', self.formatInReadableUnits(eventlogger.size()))
nonestr = _("NONE")
# a valid user gets info about all installed extensions
row(_('Global extension macros'), ', '.join(macro.modules) or nonestr)
row(
_('Local extension macros'),
', '.join(wikiutil.wikiPlugins('macro', self.macro.cfg))
or nonestr)
glob_actions = [
x for x in action.modules if not x in request.cfg.actions_excluded
]
row(_('Global extension actions'), ', '.join(glob_actions) or nonestr)
loc_actions = [
x for x in wikiutil.wikiPlugins('action', self.macro.cfg)
if not x in request.cfg.actions_excluded
]
row(_('Local extension actions'), ', '.join(loc_actions) or nonestr)
row(_('Global parsers'), ', '.join(parser.modules) or nonestr)
row(
_('Local extension parsers'),
', '.join(wikiutil.wikiPlugins('parser', self.macro.cfg))
or nonestr)
try:
import xapian
xapVersion = 'Xapian %s' % xapian.version_string()
except ImportError:
xapian = None
xapVersion = _(
'Xapian and/or Python Xapian bindings not installed')
xapian_enabled = request.cfg.xapian_search
xapState = (_('Disabled'), _('Enabled'))
xapRow = '%s, %s' % (xapState[xapian_enabled], xapVersion)
if xapian and xapian_enabled:
from MoinMoin.search.Xapian.indexing import XapianIndex
idx = XapianIndex(request)
idxState = (_('index unavailable'), _('index available'))
idx_exists = idx.exists()
xapRow += ', %s' % idxState[idx_exists]
if idx_exists:
xapRow += ', %s' % (
_('last modified: %s') %
request.user.getFormattedDateTime(idx.mtime()))
row(_('Xapian search'), xapRow)
if xapian and xapian_enabled:
stems = xapian.Stem.get_available_languages()
row(
_('Stemming for Xapian'),
xapState[request.cfg.xapian_stemming] + " (%s)" %
(stems or nonestr))
try:
from threading import activeCount
t_count = activeCount()
except ImportError:
t_count = None
row(_('Active threads'), t_count or _('N/A'))
buf.write(u'</dl>')
return buf.getvalue()
def test_all():
# Test the version number reporting functions give plausible results.
v = "%d.%d.%d" % (xapian.major_version(), xapian.minor_version(), xapian.revision())
v2 = xapian.version_string()
expect(v2, v, "Unexpected version output")
def access_cvar():
return xapian.cvar
# Check that SWIG isn't generating cvar (regression test for ticket#297).
expect_exception(AttributeError, "'module' object has no attribute 'cvar'", access_cvar)
stem = xapian.Stem("english")
expect(str(stem), "Xapian::Stem(english)", "Unexpected str(stem)")
doc = xapian.Document()
doc.set_data("a\0b")
if doc.get_data() == "a":
raise TestFail("get_data+set_data truncates at a zero byte")
expect(doc.get_data(), "a\0b", "get_data+set_data doesn't transparently handle a zero byte")
doc.set_data("is there anybody out there?")
doc.add_term("XYzzy")
doc.add_posting(stem("is"), 1)
doc.add_posting(stem("there"), 2)
doc.add_posting(stem("anybody"), 3)
doc.add_posting(stem("out"), 4)
doc.add_posting(stem("there"), 5)
db = xapian.inmemory_open()
db.add_document(doc)
expect(db.get_doccount(), 1, "Unexpected db.get_doccount()")
terms = ["smoke", "test", "terms"]
expect_query(xapian.Query(xapian.Query.OP_OR, terms), "(smoke OR test OR terms)")
query1 = xapian.Query(xapian.Query.OP_PHRASE, ("smoke", "test", "tuple"))
query2 = xapian.Query(xapian.Query.OP_XOR, (xapian.Query("smoke"), query1, "string"))
expect_query(query1, "(smoke PHRASE 3 test PHRASE 3 tuple)")
expect_query(query2, "(smoke XOR (smoke PHRASE 3 test PHRASE 3 tuple) XOR string)")
subqs = ["a", "b"]
expect_query(xapian.Query(xapian.Query.OP_OR, subqs), "(a OR b)")
expect_query(xapian.Query(xapian.Query.OP_VALUE_RANGE, 0, "1", "4"), "VALUE_RANGE 0 1 4")
# Check database factory functions are wrapped as expected:
expect_exception(xapian.DatabaseOpeningError, None, xapian.open_stub, "nosuchdir/nosuchdb")
expect_exception(xapian.DatabaseOpeningError, None, xapian.open_stub, "nosuchdir/nosuchdb", xapian.DB_OPEN)
expect_exception(xapian.DatabaseOpeningError, None, xapian.brass_open, "nosuchdir/nosuchdb")
expect_exception(xapian.DatabaseCreateError, None, xapian.brass_open, "nosuchdir/nosuchdb", xapian.DB_CREATE)
expect_exception(xapian.DatabaseOpeningError, None, xapian.chert_open, "nosuchdir/nosuchdb")
expect_exception(xapian.DatabaseCreateError, None, xapian.chert_open, "nosuchdir/nosuchdb", xapian.DB_CREATE)
expect_exception(xapian.NetworkError, None, xapian.remote_open, "/bin/false", "")
expect_exception(xapian.NetworkError, None, xapian.remote_open_writable, "/bin/false", "")
expect_exception(xapian.NetworkError, None, xapian.remote_open, "127.0.0.1", 0, 1)
expect_exception(xapian.NetworkError, None, xapian.remote_open_writable, "127.0.0.1", 0, 1)
# Check wrapping of MatchAll and MatchNothing:
expect_query(xapian.Query.MatchAll, "<alldocuments>")
expect_query(xapian.Query.MatchNothing, "")
# Feature test for Query.__iter__
term_count = 0
for term in query2:
term_count += 1
expect(term_count, 4, "Unexpected number of terms in query2")
enq = xapian.Enquire(db)
enq.set_query(xapian.Query(xapian.Query.OP_OR, "there", "is"))
mset = enq.get_mset(0, 10)
expect(mset.size(), 1, "Unexpected mset.size()")
expect(len(mset), 1, "Unexpected mset.size()")
# Feature test for Enquire.matching_terms(docid)
term_count = 0
for term in enq.matching_terms(mset.get_hit(0)):
term_count += 1
expect(term_count, 2, "Unexpected number of matching terms")
# Feature test for MSet.__iter__
msize = 0
for match in mset:
msize += 1
expect(msize, mset.size(), "Unexpected number of entries in mset")
terms = " ".join(enq.matching_terms(mset.get_hit(0)))
expect(terms, "is there", "Unexpected terms")
# Feature test for ESet.__iter__
rset = xapian.RSet()
rset.add_document(1)
eset = enq.get_eset(10, rset)
term_count = 0
for term in eset:
term_count += 1
expect(term_count, 3, "Unexpected number of expand terms")
# Feature test for Database.__iter__
term_count = 0
for term in db:
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db")
# Feature test for Database.allterms
term_count = 0
for term in db.allterms():
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db.allterms")
# Feature test for Database.postlist
count = 0
for posting in db.postlist("there"):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('there')")
# Feature test for Database.postlist with empty term (alldocspostlist)
count = 0
for posting in db.postlist(""):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('')")
# Feature test for Database.termlist
count = 0
for term in db.termlist(1):
count += 1
expect(count, 5, "Unexpected number of entries in db.termlist(1)")
# Feature test for Database.positionlist
count = 0
for term in db.positionlist(1, "there"):
count += 1
expect(count, 2, "Unexpected number of entries in db.positionlist(1, 'there')")
# Feature test for Document.termlist
count = 0
for term in doc.termlist():
count += 1
expect(count, 5, "Unexpected number of entries in doc.termlist()")
# Feature test for TermIter.skip_to
term = doc.termlist()
term.skip_to("n")
while True:
try:
x = next(term)
except StopIteration:
break
if x.term < "n":
raise TestFail("TermIter.skip_to didn't skip term '%s'" % x.term)
# Feature test for Document.values
count = 0
for term in doc.values():
count += 1
expect(count, 0, "Unexpected number of entries in doc.values")
# Check exception handling for Xapian::DocNotFoundError
expect_exception(xapian.DocNotFoundError, "Docid 3 not found", db.get_document, 3)
# Check value of OP_ELITE_SET
expect(xapian.Query.OP_ELITE_SET, 10, "Unexpected value for OP_ELITE_SET")
# Feature test for MatchDecider
doc = xapian.Document()
doc.set_data("Two")
doc.add_posting(stem("out"), 1)
doc.add_posting(stem("outside"), 1)
doc.add_posting(stem("source"), 2)
doc.add_value(0, "yes")
db.add_document(doc)
class testmatchdecider(xapian.MatchDecider):
def __call__(self, doc):
return doc.get_value(0) == "yes"
query = xapian.Query(stem("out"))
enquire = xapian.Enquire(db)
enquire.set_query(query)
mset = enquire.get_mset(0, 10, None, testmatchdecider())
expect(mset.size(), 1, "Unexpected number of documents returned by match decider")
expect(mset.get_docid(0), 2, "MatchDecider mset has wrong docid in")
# Feature test for ExpandDecider
class testexpanddecider(xapian.ExpandDecider):
def __call__(self, term):
return not term.startswith("a")
enquire = xapian.Enquire(db)
rset = xapian.RSet()
rset.add_document(1)
eset = enquire.get_eset(10, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0, testexpanddecider())
eset_terms = [term[xapian.ESET_TNAME] for term in eset.items]
expect(len(eset_terms), eset.size(), "Unexpected number of terms returned by expand")
if [t for t in eset_terms if t.startswith("a")]:
raise TestFail("ExpandDecider was not used")
# Check min_wt argument to get_eset() works (new in 1.2.5).
eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ)
expect(eset.items[-1][xapian.ESET_WT] < 1.9, True, "test get_eset() without min_wt")
eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0, None, 1.9)
expect(eset.items[-1][xapian.ESET_WT] >= 1.9, True, "test get_eset() min_wt")
# Check QueryParser parsing error.
qp = xapian.QueryParser()
expect_exception(xapian.QueryParserError, "Syntax: <expression> AND <expression>", qp.parse_query, "test AND")
# Check QueryParser pure NOT option
qp = xapian.QueryParser()
expect_query(qp.parse_query("NOT test", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT), "(<alldocuments> AND_NOT test@1)")
# Check QueryParser partial option
qp = xapian.QueryParser()
qp.set_database(db)
qp.set_default_op(xapian.Query.OP_AND)
qp.set_stemming_strategy(qp.STEM_SOME)
qp.set_stemmer(xapian.Stem("en"))
expect_query(qp.parse_query("foo o", qp.FLAG_PARTIAL), "(Zfoo@1 AND ((out@2 SYNONYM outsid@2) OR Zo@2))")
expect_query(qp.parse_query("foo outside", qp.FLAG_PARTIAL), "(Zfoo@1 AND Zoutsid@2)")
# Test supplying unicode strings
expect_query(xapian.Query(xapian.Query.OP_OR, (u"foo", u"bar")), "(foo OR bar)")
expect_query(xapian.Query(xapian.Query.OP_OR, ("foo", u"bar\xa3")), "(foo OR bar\xc2\xa3)")
expect_query(xapian.Query(xapian.Query.OP_OR, ("foo", "bar\xc2\xa3")), "(foo OR bar\xc2\xa3)")
expect_query(xapian.Query(xapian.Query.OP_OR, u"foo", u"bar"), "(foo OR bar)")
expect_query(
qp.parse_query(u"NOT t\xe9st", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT), "(<alldocuments> AND_NOT Zt\xc3\xa9st@1)"
)
doc = xapian.Document()
doc.set_data(u"Unicode with an acc\xe9nt")
doc.add_posting(stem(u"out\xe9r"), 1)
expect(doc.get_data(), u"Unicode with an acc\xe9nt".encode("utf-8"))
term = doc.termlist().next().term
expect(term, u"out\xe9r".encode("utf-8"))
# Check simple stopper
stop = xapian.SimpleStopper()
qp.set_stopper(stop)
expect(stop("a"), False)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN), "(Zfoo@1 AND Zbar@2 AND Za@3)")
stop.add("a")
expect(stop("a"), True)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN), "(Zfoo@1 AND Zbar@2)")
# Feature test for custom Stopper
class my_b_stopper(xapian.Stopper):
def __call__(self, term):
return term == "b"
def get_description(self):
return u"my_b_stopper"
stop = my_b_stopper()
expect(stop.get_description(), u"my_b_stopper")
qp.set_stopper(stop)
expect(stop("a"), False)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN), "(Zfoo@1 AND Zbar@2 AND Za@3)")
expect(stop("b"), True)
expect_query(qp.parse_query(u"foo bar b", qp.FLAG_BOOLEAN), "(Zfoo@1 AND Zbar@2)")
# Test TermGenerator
termgen = xapian.TermGenerator()
doc = xapian.Document()
termgen.set_document(doc)
termgen.index_text("foo bar baz foo")
expect(
[(item.term, item.wdf, [pos for pos in item.positer]) for item in doc.termlist()],
[("bar", 1, [2]), ("baz", 1, [3]), ("foo", 2, [1, 4])],
)
# Check DateValueRangeProcessor works
context("checking that DateValueRangeProcessor works")
qp = xapian.QueryParser()
vrpdate = xapian.DateValueRangeProcessor(1, 1, 1960)
qp.add_valuerangeprocessor(vrpdate)
query = qp.parse_query("12/03/99..12/04/01")
expect(str(query), "Query(0 * VALUE_RANGE 1 19991203 20011204)")
# Regression test for bug#193, fixed in 1.0.3.
context("running regression test for bug#193")
vrp = xapian.NumberValueRangeProcessor(0, "$", True)
a = "$10"
b = "20"
slot, a, b = vrp(a, b)
expect(slot, 0)
expect(xapian.sortable_unserialise(a), 10)
expect(xapian.sortable_unserialise(b), 20)
# Regression tests copied from PHP (probably always worked in python, but
# let's check...)
context("running regression tests for issues which were found in PHP")
# PHP overload resolution involving boolean types failed.
enq.set_sort_by_value(1, True)
# Regression test - fixed in 0.9.10.1.
oqparser = xapian.QueryParser()
oquery = oqparser.parse_query("I like tea")
# Regression test for bug#192 - fixed in 1.0.3.
enq.set_cutoff(100)
# Test setting and getting metadata
expect(db.get_metadata("Foo"), "")
db.set_metadata("Foo", "Foo")
expect(db.get_metadata("Foo"), "Foo")
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, "")
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.set_metadata, "", "Foo")
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, "")
# Test OP_SCALE_WEIGHT and corresponding constructor
expect_query(xapian.Query(xapian.Query.OP_SCALE_WEIGHT, xapian.Query("foo"), 5), "5 * foo")
r"""_checkxapian.py: Check the version of xapian used.
Raises an ImportError on import if the version used is too old to be used at
all.
"""
__docformat__ = "restructuredtext en"
# The minimum version of xapian required to work at all.
min_xapian_version = (1, 0, 6)
# Dictionary of features we can't support do to them being missing from the
# available version of xapian.
missing_features = {}
import xapian
versions = xapian.major_version(), xapian.minor_version(), xapian.revision()
if versions < min_xapian_version:
raise ImportError(
"""
Xapian Python bindings installed, but need at least version %d.%d.%d - got %s
""".strip() %
tuple(list(min_xapian_version) + [xapian.version_string()]))
if not hasattr(xapian, 'TermCountMatchSpy'):
missing_features['tags'] = 1
if not hasattr(xapian, 'CategorySelectMatchSpy'):
missing_features['facets'] = 1
def xapiand_run(data=None, logfile=None, pidfile=None, uid=None, gid=None, umask=0,
working_directory=None, verbosity=1, commit_slots=None, commit_timeout=None,
listener=None, queue_type=None, **options):
global STOPPED
current_thread = threading.current_thread()
tid = current_thread.name.rsplit('-', 1)[-1]
current_thread.name = 'Server-%s' % tid
if pidfile:
create_pidlock(pidfile)
address, _, port = listener.partition(':')
if not port:
port, address = address, ''
port = int(port)
loglevel = ['ERROR', 'WARNING', 'INFO', 'DEBUG'][3 if verbosity == 'v' else int(verbosity)]
log = logging.getLogger()
log.setLevel(loglevel)
if len(log.handlers) < 1:
formatter = logging.Formatter(LOG_FORMAT)
if logfile:
outfile = logging.FileHandler(logfile)
outfile.setFormatter(formatter)
log.addHandler(outfile)
if not pidfile:
console = ColoredStreamHandler(sys.stderr)
console.setFormatter(formatter)
log.addHandler(console)
if not commit_slots:
commit_slots = COMMIT_SLOTS
if commit_timeout is None:
commit_timeout = COMMIT_TIMEOUT
timeout = min(max(int(round(commit_timeout * 0.3)), 1), 3)
queue_class = AVAILABLE_QUEUES.get(queue_type) or AVAILABLE_QUEUES['default']
mode = "with multiple threads and %s commit slots using %s" % (commit_slots, queue_class.__name__)
log.warning("Starting Xapiand Server v%s (xapian v%s) %s [%s] (pid:%s)", version, xapian.version_string(), mode, loglevel, os.getpid())
commit_lock = Semaphore(commit_slots)
timeouts = Obj(
timeout=timeout,
commit=commit_timeout * 1.0,
delayed=commit_timeout * 3.0,
maximum=commit_timeout * 9.0,
)
main_queue = queue.Queue()
databases_pool = DatabasesPool(data=data, log=log)
databases = {}
xapian_server = XapiandServer(
(address, port),
databases_pool=databases_pool,
pool_size=COMMANDS_POOL_SIZE,
main_queue=main_queue,
queue_class=queue_class,
data=data,
log=log
)
gevent.signal(signal.SIGTERM, xapian_server.close)
gevent.signal(signal.SIGINT, xapian_server.close)
log.debug("Starting server...")
try:
xapian_server.start()
except Exception as exc:
log.error("Cannot start server: %s", exc)
sys.exit(-1)
pool_size = WRITERS_POOL_SIZE
pool_size_warning = int(pool_size / 3.0 * 2.0)
writers_pool = ThreadPool(pool_size)
def start_writer(db):
db = build_url(*parse_url(db.strip()))
name = database_name(db)
try:
tq = None
t, tq = databases[db]
if t.ready():
raise KeyError
except KeyError:
queue_name = os.path.join(data, name)
tq = tq or xapian_server.get_queue(queue_name)
pool_used = len(writers_pool)
if not (pool_size_warning - pool_used) % 10:
log.warning("Writers pool is close to be full (%s/%s)", pool_used, pool_size)
elif pool_used == pool_size:
log.error("Writers poll is full! (%s/%s)", pool_used, pool_size)
t = writers_pool.spawn(_writer_loop, databases, databases_pool, db, tq, commit_lock, timeouts, data, log)
databases[db] = (t, tq)
return db, name, t, tq
if queue_class.persistent:
# Initialize seen writers:
writers_file = os.path.join(data, WRITERS_FILE)
with open(writers_file, 'rt') as epfile:
for i, db in enumerate(epfile):
if i == 0:
log.debug("Initializing writers...")
start_writer(db)
log.info("Waiting for commands...")
msg = None
timeout = timeouts.timeout
while not xapian_server.closed:
xapian_cleanup(databases_pool, DATABASE_MAX_LIFE, data=data, log=log)
try:
msg = main_queue.get(True, timeout)
except Queue.Empty:
continue
if not msg:
continue
try:
cmd, endpoints, args = msg
except ValueError:
log.error("Wrong command received!")
continue
for db in endpoints:
db, name, t, tq = start_writer(db)
if cmd != 'INIT':
try:
tq.put((cmd, db, args))
log.debug("Command '%s' forwarded to %s", cmd, name)
except Queue.Full:
log.error("Cannot send command to queue! (2)")
log.debug("Waiting for connected clients to disconnect...")
while True:
if xapian_server.close(max_age=10):
break
if gevent.wait(timeout=3):
break
# Stop queues:
queue_class.STOPPED = STOPPED = time.time()
if queue_class.persistent:
with open(writers_file, 'wt') as epfile:
for db, (t, tq) in databases.items():
if not t.ready():
epfile.write("%s\n" % db)
# Wake up writers:
for t, tq in databases.values():
try:
tq.put(None) # wake up!
except Queue.Full:
log.error("Cannot send command to queue! (1)")
log.debug("Waiting for %s writers...", len(databases))
for t, tq in databases.values():
t.wait()
xapian_cleanup(databases_pool, 0, data=data, log=log)
log.warning("Xapiand Server ended! (pid:%s)", os.getpid())
gevent.wait()
def getInfo(self):
_ = self.request.getText
request = self.request
buf = StringIO()
row = lambda label, value, buf=buf: buf.write(u'<dt>%s</dt><dd>%s</dd>' % (label, value))
buf.write(u'<dl>')
row(_('Python Version'), sys.version)
row(_('MoinMoin Version'), _('Release %s [Revision %s]') % (version.release, version.revision))
if not request.user.valid:
# for an anonymous user it ends here.
buf.write(u'</dl>')
return buf.getvalue()
if request.user.isSuperUser():
# superuser gets all page dependent stuff only
try:
import Ft
ftversion = Ft.__version__
except ImportError:
ftversion = None
except AttributeError:
ftversion = 'N/A'
if ftversion:
row(_('4Suite Version'), ftversion)
# TODO add python-xml check and display it
# Get the full pagelist of the wiki
pagelist = request.rootpage.getPageList(user='')
systemPages = []
totalsize = 0
for page in pagelist:
if wikiutil.isSystemPage(request, page):
systemPages.append(page)
totalsize += Page(request, page).size()
row(_('Number of pages'), str(len(pagelist)-len(systemPages)))
row(_('Number of system pages'), str(len(systemPages)))
row(_('Accumulated page sizes'), self.formatInReadableUnits(totalsize))
data_dir = request.cfg.data_dir
row(_('Disk usage of %(data_dir)s/pages/') % {'data_dir': data_dir},
self.formatInReadableUnits(self.getDirectorySize(os.path.join(data_dir, 'pages'))))
row(_('Disk usage of %(data_dir)s/') % {'data_dir': data_dir},
self.formatInReadableUnits(self.getDirectorySize(data_dir)))
edlog = editlog.EditLog(request)
row(_('Entries in edit log'), "%s (%s)" % (edlog.lines(), self.formatInReadableUnits(edlog.size())))
# This puts a heavy load on the server when the log is large
eventlogger = eventlog.EventLog(request)
row('Event log', self.formatInReadableUnits(eventlogger.size()))
nonestr = _("NONE")
# a valid user gets info about all installed extensions
row(_('Global extension macros'), ', '.join(macro.modules) or nonestr)
row(_('Local extension macros'),
', '.join(wikiutil.wikiPlugins('macro', self.macro.cfg)) or nonestr)
glob_actions = [x for x in action.modules
if not x in request.cfg.actions_excluded]
row(_('Global extension actions'), ', '.join(glob_actions) or nonestr)
loc_actions = [x for x in wikiutil.wikiPlugins('action', self.macro.cfg)
if not x in request.cfg.actions_excluded]
row(_('Local extension actions'), ', '.join(loc_actions) or nonestr)
row(_('Global parsers'), ', '.join(parser.modules) or nonestr)
row(_('Local extension parsers'),
', '.join(wikiutil.wikiPlugins('parser', self.macro.cfg)) or nonestr)
try:
import xapian
xapVersion = 'Xapian %s' % xapian.version_string()
except ImportError:
xapian = None
xapVersion = _('Xapian and/or Python Xapian bindings not installed')
xapian_enabled = request.cfg.xapian_search
xapState = (_('Disabled'), _('Enabled'))
xapRow = '%s, %s' % (xapState[xapian_enabled], xapVersion)
if xapian and xapian_enabled:
from MoinMoin.search.Xapian.indexing import XapianIndex
idx = XapianIndex(request)
idxState = (_('index unavailable'), _('index available'))
idx_exists = idx.exists()
xapRow += ', %s' % idxState[idx_exists]
if idx_exists:
xapRow += ', %s' % (_('last modified: %s') %
request.user.getFormattedDateTime(idx.mtime()))
row(_('Xapian search'), xapRow)
if xapian and xapian_enabled:
stems = xapian.Stem.get_available_languages()
row(_('Stemming for Xapian'), xapState[request.cfg.xapian_stemming] +
" (%s)" % (stems or nonestr))
try:
from threading import activeCount
t_count = activeCount()
except ImportError:
t_count = None
row(_('Active threads'), t_count or _('N/A'))
buf.write(u'</dl>')
return buf.getvalue()
def test_all():
# Test the version number reporting functions give plausible results.
v = "%d.%d.%d" % (xapian.major_version(),
xapian.minor_version(),
xapian.revision())
v2 = xapian.version_string()
expect(v2, v, "Unexpected version output")
# A regexp check would be better, but seems to create a bogus "leak" of -1
# objects in Python 3.
expect(len(xapian.__version__.split('.')), 3, 'xapian.__version__ not X.Y.Z')
expect((xapian.__version__.split('.'))[0], '1', 'xapian.__version__ not "1.Y.Z"')
def access_cvar():
res = xapian.cvar
print "Unhandled constants: ", res
return res
# Check that SWIG isn't generating cvar (regression test for ticket#297).
expect_exception(AttributeError, "'module' object has no attribute 'cvar'",
access_cvar)
stem = xapian.Stem("english")
expect(str(stem), "Xapian::Stem(english)", "Unexpected str(stem)")
doc = xapian.Document()
doc.set_data("a\0b")
if doc.get_data() == "a":
raise TestFail("get_data+set_data truncates at a zero byte")
expect(doc.get_data(), "a\0b", "get_data+set_data doesn't transparently handle a zero byte")
doc.set_data("is there anybody out there?")
doc.add_term("XYzzy")
doc.add_posting(stem("is"), 1)
doc.add_posting(stem("there"), 2)
doc.add_posting(stem("anybody"), 3)
doc.add_posting(stem("out"), 4)
doc.add_posting(stem("there"), 5)
db = xapian.WritableDatabase('', xapian.DB_BACKEND_INMEMORY)
db.add_document(doc)
expect(db.get_doccount(), 1, "Unexpected db.get_doccount()")
terms = ["smoke", "test", "terms"]
expect_query(xapian.Query(xapian.Query.OP_OR, terms),
"(smoke OR test OR terms)")
query1 = xapian.Query(xapian.Query.OP_PHRASE, ("smoke", "test", "tuple"))
query2 = xapian.Query(xapian.Query.OP_XOR, (xapian.Query("smoke"), query1, "string"))
expect_query(query1, "(smoke PHRASE 3 test PHRASE 3 tuple)")
expect_query(query2, "(smoke XOR (smoke PHRASE 3 test PHRASE 3 tuple) XOR string)")
subqs = ["a", "b"]
expect_query(xapian.Query(xapian.Query.OP_OR, subqs), "(a OR b)")
expect_query(xapian.Query(xapian.Query.OP_VALUE_RANGE, 0, '1', '4'),
"VALUE_RANGE 0 1 4")
# Check database factory functions are wrapped as expected:
expect_exception(xapian.DatabaseNotFoundError, None,
xapian.Database, "nosuchdir/nosuchdb", xapian.DB_BACKEND_STUB)
expect_exception(xapian.DatabaseNotFoundError, None,
xapian.WritableDatabase, "nosuchdir/nosuchdb", xapian.DB_OPEN|xapian.DB_BACKEND_STUB)
expect_exception(xapian.NetworkError, None,
xapian.remote_open, "/bin/false", "")
expect_exception(xapian.NetworkError, None,
xapian.remote_open_writable, "/bin/false", "")
expect_exception(xapian.NetworkError, None,
xapian.remote_open, "127.0.0.1", 0, 1)
expect_exception(xapian.NetworkError, None,
xapian.remote_open_writable, "127.0.0.1", 0, 1)
# Check wrapping of MatchAll and MatchNothing:
expect_query(xapian.Query.MatchAll, "<alldocuments>")
expect_query(xapian.Query.MatchNothing, "")
# Feature test for Query.__iter__
term_count = 0
for term in query2:
term_count += 1
expect(term_count, 4, "Unexpected number of terms in query2")
enq = xapian.Enquire(db)
# Check Xapian::BAD_VALUENO is wrapped suitably.
enq.set_collapse_key(xapian.BAD_VALUENO)
enq.set_query(xapian.Query(xapian.Query.OP_OR, "there", "is"))
mset = enq.get_mset(0, 10)
expect(mset.size(), 1, "Unexpected mset.size()")
expect(len(mset), 1, "Unexpected mset.size()")
# Feature test for Enquire.matching_terms(docid)
term_count = 0
for term in enq.matching_terms(mset.get_hit(0)):
term_count += 1
expect(term_count, 2, "Unexpected number of matching terms")
# Feature test for MSet.__iter__
msize = 0
for match in mset:
msize += 1
expect(msize, mset.size(), "Unexpected number of entries in mset")
terms = " ".join(enq.matching_terms(mset.get_hit(0)))
expect(terms, "is there", "Unexpected terms")
# Feature test for ESet.__iter__
rset = xapian.RSet()
rset.add_document(1)
eset = enq.get_eset(10, rset)
term_count = 0
for term in eset:
term_count += 1
expect(term_count, 3, "Unexpected number of expand terms")
# Feature test for Database.__iter__
term_count = 0
for term in db:
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db")
# Feature test for Database.allterms
term_count = 0
for term in db.allterms():
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db.allterms")
# Feature test for Database.postlist
count = 0
for posting in db.postlist("there"):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('there')")
# Feature test for Database.postlist with empty term (alldocspostlist)
count = 0
for posting in db.postlist(""):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('')")
# Feature test for Database.termlist
count = 0
for term in db.termlist(1):
count += 1
expect(count, 5, "Unexpected number of entries in db.termlist(1)")
# Feature test for Database.positionlist
count = 0
for term in db.positionlist(1, "there"):
count += 1
expect(count, 2, "Unexpected number of entries in db.positionlist(1, 'there')")
# Feature test for Document.termlist
count = 0
for term in doc.termlist():
count += 1
expect(count, 5, "Unexpected number of entries in doc.termlist()")
# Feature test for TermIter.skip_to
term = doc.termlist()
term.skip_to('n')
while True:
try:
x = next(term)
except StopIteration:
break
if x.term < 'n':
raise TestFail("TermIter.skip_to didn't skip term '%s'" % x.term)
# Feature test for Document.values
count = 0
for term in doc.values():
count += 1
expect(count, 0, "Unexpected number of entries in doc.values")
# Check exception handling for Xapian::DocNotFoundError
expect_exception(xapian.DocNotFoundError, "Docid 3 not found", db.get_document, 3)
# Check value of OP_ELITE_SET
expect(xapian.Query.OP_ELITE_SET, 10, "Unexpected value for OP_ELITE_SET")
# Feature test for MatchDecider
doc = xapian.Document()
doc.set_data("Two")
doc.add_posting(stem("out"), 1)
doc.add_posting(stem("outside"), 1)
doc.add_posting(stem("source"), 2)
doc.add_value(0, "yes")
db.add_document(doc)
class testmatchdecider(xapian.MatchDecider):
def __call__(self, doc):
return doc.get_value(0) == "yes"
query = xapian.Query(stem("out"))
enquire = xapian.Enquire(db)
enquire.set_query(query)
mset = enquire.get_mset(0, 10, None, testmatchdecider())
expect(mset.size(), 1, "Unexpected number of documents returned by match decider")
expect(mset.get_docid(0), 2, "MatchDecider mset has wrong docid in")
# Feature test for ExpandDecider
class testexpanddecider(xapian.ExpandDecider):
def __call__(self, term):
return (not term.startswith('a'))
enquire = xapian.Enquire(db)
rset = xapian.RSet()
rset.add_document(1)
eset = enquire.get_eset(10, rset, xapian.Enquire.USE_EXACT_TERMFREQ, testexpanddecider())
eset_terms = [item.term for item in eset]
expect(len(eset_terms), eset.size(), "Unexpected number of terms returned by expand")
if [t for t in eset_terms if t.startswith('a')]:
raise TestFail("ExpandDecider was not used")
# Check min_wt argument to get_eset() works (new in 1.2.5).
eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ)
expect([i.weight for i in eset][-1] < 1.9, True, "test get_eset() without min_wt")
eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ, None, 1.9)
expect([i.weight for i in eset][-1] >= 1.9, True, "test get_eset() min_wt")
# Check QueryParser parsing error.
qp = xapian.QueryParser()
expect_exception(xapian.QueryParserError, "Syntax: <expression> AND <expression>", qp.parse_query, "test AND")
# Check QueryParser pure NOT option
qp = xapian.QueryParser()
expect_query(qp.parse_query("NOT test", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
"(0 * <alldocuments> AND_NOT test@1)")
# Check QueryParser partial option
qp = xapian.QueryParser()
qp.set_database(db)
qp.set_default_op(xapian.Query.OP_AND)
qp.set_stemming_strategy(qp.STEM_SOME)
qp.set_stemmer(xapian.Stem('en'))
expect_query(qp.parse_query("foo ox", qp.FLAG_PARTIAL),
"(Zfoo@1 AND (WILDCARD SYNONYM ox OR Zox@2))")
expect_query(qp.parse_query("foo outside", qp.FLAG_PARTIAL),
"(Zfoo@1 AND (WILDCARD SYNONYM outside OR Zoutsid@2))")
# Test supplying unicode strings
expect_query(xapian.Query(xapian.Query.OP_OR, (u'foo', u'bar')),
'(foo OR bar)')
expect_query(xapian.Query(xapian.Query.OP_OR, ('foo', u'bar\xa3')),
'(foo OR bar\xc2\xa3)')
expect_query(xapian.Query(xapian.Query.OP_OR, ('foo', 'bar\xc2\xa3')),
'(foo OR bar\xc2\xa3)')
expect_query(xapian.Query(xapian.Query.OP_OR, u'foo', u'bar'),
'(foo OR bar)')
expect_query(qp.parse_query(u"NOT t\xe9st", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
"(0 * <alldocuments> AND_NOT Zt\xc3\xa9st@1)")
doc = xapian.Document()
doc.set_data(u"Unicode with an acc\xe9nt")
doc.add_posting(stem(u"out\xe9r"), 1)
expect(doc.get_data(), u"Unicode with an acc\xe9nt".encode('utf-8'))
term = doc.termlist().next().term
expect(term, u"out\xe9r".encode('utf-8'))
# Check simple stopper
stop = xapian.SimpleStopper()
qp.set_stopper(stop)
expect(stop('a'), False)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2 AND Za@3)")
stop.add('a')
expect(stop('a'), True)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2)")
# Feature test for custom Stopper
class my_b_stopper(xapian.Stopper):
def __call__(self, term):
return term == "b"
def get_description(self):
return u"my_b_stopper"
stop = my_b_stopper()
expect(stop.get_description(), u"my_b_stopper")
qp.set_stopper(stop)
expect(stop('a'), False)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2 AND Za@3)")
expect(stop('b'), True)
expect_query(qp.parse_query(u"foo bar b", qp.FLAG_BOOLEAN),
"(Zfoo@1 AND Zbar@2)")
# Test SimpleStopper initialised from a file.
try:
srcdir = os.environ['srcdir']
except KeyError:
srcdir = '.'
stop = xapian.SimpleStopper(srcdir + '/../shortstop.list')
expect(stop('a'), True)
expect(stop('am'), False)
expect(stop('an'), True)
expect(stop('the'), True)
expect_exception(xapian.InvalidArgumentError, None, xapian.SimpleStopper, 'nosuchfile')
# Test TermGenerator
termgen = xapian.TermGenerator()
doc = xapian.Document()
termgen.set_document(doc)
termgen.index_text('foo bar baz foo')
expect([(item.term, item.wdf, [pos for pos in item.positer]) for item in doc.termlist()], [('bar', 1, [2]), ('baz', 1, [3]), ('foo', 2, [1, 4])])
# Check DateRangeProcessor works
context("checking that DateRangeProcessor works")
qp = xapian.QueryParser()
rpdate = xapian.DateRangeProcessor(1, xapian.RP_DATE_PREFER_MDY, 1960)
qp.add_rangeprocessor(rpdate)
query = qp.parse_query('12/03/99..12/04/01')
expect(str(query), 'Query(VALUE_RANGE 1 19991203 20011204)')
# Feature test for xapian.FieldProcessor
context("running feature test for xapian.FieldProcessor")
class testfieldprocessor(xapian.FieldProcessor):
def __call__(self, s):
if s == 'spam':
raise Exception('already spam')
return xapian.Query("spam")
qp.add_prefix('spam', testfieldprocessor())
qp.add_boolean_prefix('boolspam', testfieldprocessor())
qp.add_boolean_prefix('boolspam2', testfieldprocessor(), False) # Old-style
qp.add_boolean_prefix('boolspam3', testfieldprocessor(), '')
qp.add_boolean_prefix('boolspam4', testfieldprocessor(), 'group')
qp.add_boolean_prefix('boolspam5', testfieldprocessor(), None)
query = qp.parse_query('spam:ignored')
expect(str(query), 'Query(spam)')
expect_exception(Exception, 'already spam', qp.parse_query, 'spam:spam')
# Regression tests copied from PHP (probably always worked in python, but
# let's check...)
context("running regression tests for issues which were found in PHP")
# PHP overload resolution involving boolean types failed.
enq.set_sort_by_value(1, True)
# Regression test - fixed in 0.9.10.1.
oqparser = xapian.QueryParser()
oquery = oqparser.parse_query("I like tea")
# Regression test for bug fixed in 1.4.4:
# https://bugs.debian.org/849722
oqparser.add_boolean_prefix('tag', 'K', '')
# Make sure other cases also work:
oqparser.add_boolean_prefix('zag', 'XR', False) # Old-style
oqparser.add_boolean_prefix('rag', 'XR', None)
oqparser.add_boolean_prefix('nag', 'XB', '')
oqparser.add_boolean_prefix('bag', 'XB', 'blergh')
oqparser.add_boolean_prefix('gag', 'XB', u'blergh')
oqparser.add_boolean_prefix('jag', 'XB', b'blergh')
# Regression test for bug#192 - fixed in 1.0.3.
enq.set_cutoff(100)
# Test setting and getting metadata
expect(db.get_metadata('Foo'), '')
db.set_metadata('Foo', 'Foo')
expect(db.get_metadata('Foo'), 'Foo')
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, '')
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.set_metadata, '', 'Foo')
expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, '')
# Test OP_SCALE_WEIGHT and corresponding constructor
expect_query(xapian.Query(xapian.Query.OP_SCALE_WEIGHT, xapian.Query('foo'), 5),
"5 * foo")
def test_all():
# Test the version number reporting functions give plausible results.
v = "%d.%d.%d" % (xapian.major_version(), xapian.minor_version(),
xapian.revision())
v2 = xapian.version_string()
expect(v2, v, "Unexpected version output")
def access_cvar():
return xapian.cvar
# Check that SWIG isn't generating cvar (regression test for ticket#297).
expect_exception(AttributeError, "'module' object has no attribute 'cvar'",
access_cvar)
stem = xapian.Stem("english")
expect(str(stem), "Xapian::Stem(english)", "Unexpected str(stem)")
doc = xapian.Document()
doc.set_data("a\0b")
if doc.get_data() == "a":
raise TestFail("get_data+set_data truncates at a zero byte")
expect(doc.get_data(), "a\0b",
"get_data+set_data doesn't transparently handle a zero byte")
doc.set_data("is there anybody out there?")
doc.add_term("XYzzy")
doc.add_posting(stem("is"), 1)
doc.add_posting(stem("there"), 2)
doc.add_posting(stem("anybody"), 3)
doc.add_posting(stem("out"), 4)
doc.add_posting(stem("there"), 5)
db = xapian.inmemory_open()
db.add_document(doc)
expect(db.get_doccount(), 1, "Unexpected db.get_doccount()")
terms = ["smoke", "test", "terms"]
expect_query(xapian.Query(xapian.Query.OP_OR, terms),
"(smoke OR test OR terms)")
query1 = xapian.Query(xapian.Query.OP_PHRASE, ("smoke", "test", "tuple"))
query2 = xapian.Query(xapian.Query.OP_XOR,
(xapian.Query("smoke"), query1, "string"))
expect_query(query1, "(smoke PHRASE 3 test PHRASE 3 tuple)")
expect_query(
query2, "(smoke XOR (smoke PHRASE 3 test PHRASE 3 tuple) XOR string)")
subqs = ["a", "b"]
expect_query(xapian.Query(xapian.Query.OP_OR, subqs), "(a OR b)")
expect_query(xapian.Query(xapian.Query.OP_VALUE_RANGE, 0, '1', '4'),
"VALUE_RANGE 0 1 4")
expect_query(xapian.Query.MatchAll, "<alldocuments>")
expect_query(xapian.Query.MatchNothing, "")
# Feature test for Query.__iter__
term_count = 0
for term in query2:
term_count += 1
expect(term_count, 4, "Unexpected number of terms in query2")
enq = xapian.Enquire(db)
enq.set_query(xapian.Query(xapian.Query.OP_OR, "there", "is"))
mset = enq.get_mset(0, 10)
expect(mset.size(), 1, "Unexpected mset.size()")
expect(len(mset), 1, "Unexpected mset.size()")
# Feature test for Enquire.matching_terms(docid)
term_count = 0
for term in enq.matching_terms(mset.get_hit(0)):
term_count += 1
expect(term_count, 2, "Unexpected number of matching terms")
# Feature test for MSet.__iter__
msize = 0
for match in mset:
msize += 1
expect(msize, mset.size(), "Unexpected number of entries in mset")
terms = " ".join(enq.matching_terms(mset.get_hit(0)))
expect(terms, "is there", "Unexpected terms")
# Feature test for ESet.__iter__
rset = xapian.RSet()
rset.add_document(1)
eset = enq.get_eset(10, rset)
term_count = 0
for term in eset:
term_count += 1
expect(term_count, 3, "Unexpected number of expand terms")
# Feature test for Database.__iter__
term_count = 0
for term in db:
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db")
# Feature test for Database.allterms
term_count = 0
for term in db.allterms():
term_count += 1
expect(term_count, 5, "Unexpected number of terms in db.allterms")
# Feature test for Database.postlist
count = 0
for posting in db.postlist("there"):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('there')")
# Feature test for Database.postlist with empty term (alldocspostlist)
count = 0
for posting in db.postlist(""):
count += 1
expect(count, 1, "Unexpected number of entries in db.postlist('')")
# Feature test for Database.termlist
count = 0
for term in db.termlist(1):
count += 1
expect(count, 5, "Unexpected number of entries in db.termlist(1)")
# Feature test for Database.positionlist
count = 0
for term in db.positionlist(1, "there"):
count += 1
expect(count, 2,
"Unexpected number of entries in db.positionlist(1, 'there')")
# Feature test for Document.termlist
count = 0
for term in doc.termlist():
count += 1
expect(count, 5, "Unexpected number of entries in doc.termlist()")
# Feature test for TermIter.skip_to
term = doc.termlist()
term.skip_to('n')
while True:
try:
x = next(term)
except StopIteration:
break
if x.term < 'n':
raise TestFail("TermIter.skip_to didn't skip term '%s'" % x.term)
# Feature test for Document.values
count = 0
for term in doc.values():
count += 1
expect(count, 0, "Unexpected number of entries in doc.values")
# Check exception handling for Xapian::DocNotFoundError
expect_exception(xapian.DocNotFoundError, "Docid 3 not found",
db.get_document, 3)
# Check value of OP_ELITE_SET
expect(xapian.Query.OP_ELITE_SET, 10, "Unexpected value for OP_ELITE_SET")
# Feature test for MatchDecider
doc = xapian.Document()
doc.set_data("Two")
doc.add_posting(stem("out"), 1)
doc.add_posting(stem("outside"), 1)
doc.add_posting(stem("source"), 2)
doc.add_value(0, "yes")
db.add_document(doc)
class testmatchdecider(xapian.MatchDecider):
def __call__(self, doc):
return doc.get_value(0) == "yes"
query = xapian.Query(stem("out"))
enquire = xapian.Enquire(db)
enquire.set_query(query)
mset = enquire.get_mset(0, 10, None, testmatchdecider())
expect(mset.size(), 1,
"Unexpected number of documents returned by match decider")
expect(mset.get_docid(0), 2, "MatchDecider mset has wrong docid in")
# Feature test for ExpandDecider
class testexpanddecider(xapian.ExpandDecider):
def __call__(self, term):
return (not term.startswith('a'))
enquire = xapian.Enquire(db)
rset = xapian.RSet()
rset.add_document(1)
eset = enquire.get_eset(10, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0,
testexpanddecider())
eset_terms = [term[xapian.ESET_TNAME] for term in eset.items]
expect(len(eset_terms), eset.size(),
"Unexpected number of terms returned by expand")
if [t for t in eset_terms if t.startswith('a')]:
raise TestFail("ExpandDecider was not used")
# Check min_wt argument to get_eset() works (new in 1.2.5).
eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ)
expect(eset.items[-1][xapian.ESET_WT] < 1.9, True,
"test get_eset() without min_wt")
eset = enquire.get_eset(100, rset, xapian.Enquire.USE_EXACT_TERMFREQ, 1.0,
None, 1.9)
expect(eset.items[-1][xapian.ESET_WT] >= 1.9, True,
"test get_eset() min_wt")
# Check QueryParser parsing error.
qp = xapian.QueryParser()
expect_exception(xapian.QueryParserError,
"Syntax: <expression> AND <expression>", qp.parse_query,
"test AND")
# Check QueryParser pure NOT option
qp = xapian.QueryParser()
expect_query(
qp.parse_query("NOT test", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
"(<alldocuments> AND_NOT test:(pos=1))")
# Check QueryParser partial option
qp = xapian.QueryParser()
qp.set_database(db)
qp.set_default_op(xapian.Query.OP_AND)
qp.set_stemming_strategy(qp.STEM_SOME)
qp.set_stemmer(xapian.Stem('en'))
expect_query(
qp.parse_query("foo o", qp.FLAG_PARTIAL),
"(Zfoo:(pos=1) AND ((out:(pos=2) SYNONYM outsid:(pos=2)) OR Zo:(pos=2)))"
)
expect_query(qp.parse_query("foo outside", qp.FLAG_PARTIAL),
"(Zfoo:(pos=1) AND Zoutsid:(pos=2))")
# Test supplying unicode strings
expect_query(xapian.Query(xapian.Query.OP_OR, (u'foo', u'bar')),
'(foo OR bar)')
expect_query(xapian.Query(xapian.Query.OP_OR, ('foo', u'bar\xa3')),
'(foo OR bar\xc2\xa3)')
expect_query(xapian.Query(xapian.Query.OP_OR, ('foo', 'bar\xc2\xa3')),
'(foo OR bar\xc2\xa3)')
expect_query(xapian.Query(xapian.Query.OP_OR, u'foo', u'bar'),
'(foo OR bar)')
expect_query(
qp.parse_query(u"NOT t\xe9st", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT),
"(<alldocuments> AND_NOT Zt\xc3\xa9st:(pos=1))")
doc = xapian.Document()
doc.set_data(u"Unicode with an acc\xe9nt")
doc.add_posting(stem(u"out\xe9r"), 1)
expect(doc.get_data(), u"Unicode with an acc\xe9nt".encode('utf-8'))
term = doc.termlist().next().term
expect(term, u"out\xe9r".encode('utf-8'))
# Check simple stopper
stop = xapian.SimpleStopper()
qp.set_stopper(stop)
expect(stop('a'), False)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo:(pos=1) AND Zbar:(pos=2) AND Za:(pos=3))")
stop.add('a')
expect(stop('a'), True)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo:(pos=1) AND Zbar:(pos=2))")
# Feature test for custom Stopper
class my_b_stopper(xapian.Stopper):
def __call__(self, term):
return term == "b"
def get_description(self):
return u"my_b_stopper"
stop = my_b_stopper()
expect(stop.get_description(), u"my_b_stopper")
qp.set_stopper(stop)
expect(stop('a'), False)
expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN),
"(Zfoo:(pos=1) AND Zbar:(pos=2) AND Za:(pos=3))")
expect(stop('b'), True)
expect_query(qp.parse_query(u"foo bar b", qp.FLAG_BOOLEAN),
"(Zfoo:(pos=1) AND Zbar:(pos=2))")
# Test TermGenerator
termgen = xapian.TermGenerator()
doc = xapian.Document()
termgen.set_document(doc)
termgen.index_text('foo bar baz foo')
expect([(item.term, item.wdf, [pos for pos in item.positer])
for item in doc.termlist()], [('bar', 1, [2]), ('baz', 1, [3]),
('foo', 2, [1, 4])])
# Check DateValueRangeProcessor works
context("checking that DateValueRangeProcessor works")
qp = xapian.QueryParser()
vrpdate = xapian.DateValueRangeProcessor(1, 1, 1960)
qp.add_valuerangeprocessor(vrpdate)
query = qp.parse_query('12/03/99..12/04/01')
expect(str(query), 'Xapian::Query(VALUE_RANGE 1 19991203 20011204)')
# Regression test for bug#193, fixed in 1.0.3.
context("running regression test for bug#193")
vrp = xapian.NumberValueRangeProcessor(0, '$', True)
a = '$10'
b = '20'
slot, a, b = vrp(a, b)
expect(slot, 0)
expect(xapian.sortable_unserialise(a), 10)
expect(xapian.sortable_unserialise(b), 20)
# Regression tests copied from PHP (probably always worked in python, but
# let's check...)
context("running regression tests for issues which were found in PHP")
# PHP overload resolution involving boolean types failed.
enq.set_sort_by_value(1, True)
# Regression test - fixed in 0.9.10.1.
oqparser = xapian.QueryParser()
oquery = oqparser.parse_query("I like tea")
# Regression test for bug#192 - fixed in 1.0.3.
enq.set_cutoff(100)
# Test setting and getting metadata
expect(db.get_metadata('Foo'), '')
db.set_metadata('Foo', 'Foo')
expect(db.get_metadata('Foo'), 'Foo')
expect_exception(xapian.InvalidArgumentError,
"Empty metadata keys are invalid", db.get_metadata, '')
expect_exception(xapian.InvalidArgumentError,
"Empty metadata keys are invalid", db.set_metadata, '',
'Foo')
expect_exception(xapian.InvalidArgumentError,
"Empty metadata keys are invalid", db.get_metadata, '')
# Test OP_SCALE_WEIGHT and corresponding constructor
expect_query(
xapian.Query(xapian.Query.OP_SCALE_WEIGHT, xapian.Query('foo'), 5),
"5 * foo")
