def thread_run(self):
thread_push_log("[IDXQ]")
with self.xom.keyfs.transaction(write=False) as tx:
indexer = get_indexer(self.xom)
searcher = indexer.get_project_ix().searcher()
self.shared_data.queue_projects(iter_projects(self.xom),
tx.at_serial, searcher)
def handler(self, is_from_mirror, serial, indexname, names):
log.debug(
"Got %s projects from %s at serial %s for indexing",
len(names), indexname, serial)
ix = get_indexer(self.xom)
counter = itertools.count()
project_ix = ix.get_project_ix()
main_keys = project_ix.schema.names()
writer = project_ix.writer()
searcher = project_ix.searcher()
try:
with self.xom.keyfs.transaction(write=False) as tx:
stage = self.xom.model.getstage(indexname)
if stage is not None:
for name in names:
data = preprocess_project(
ProjectIndexingInfo(stage=stage, name=name))
# because we use the current transaction, we also
# use the current serial for indexing
ix._update_project(
data, tx.at_serial, counter, main_keys, writer,
searcher=searcher)
count = next(counter)
except Exception:
writer.cancel()
# let the queue handle retries
raise
else:
log.debug("Committing %s new documents to search index." % count)
writer.commit()
def clear_index(argv=None):
if argv is None:
argv = sys.argv
else:
# for tests
argv = [str(x) for x in argv]
pluginmanager = get_pluginmanager()
try:
parser = MyArgumentParser(description="Clear project search index.",
add_help=False)
add_help_option(parser, pluginmanager)
add_configfile_option(parser, pluginmanager)
add_storage_options(parser, pluginmanager)
add_indexer_backend_option(parser, pluginmanager)
config = parseoptions(pluginmanager, argv, parser=parser)
configure_cli_logging(config.args)
xom = xom_from_config(config)
log = xom.log
log.warn("You should stop devpi-server before running this command.")
ix = get_indexer(xom)
ix.delete_index()
log.info(
"Index deleted, start devpi-server again to let the index rebuild automatically."
)
except Fatal as e:
tw = py.io.TerminalWriter(sys.stderr)
tw.line("fatal: %s" % e.args[0], red=True)
return 1
def test_index_projects_arg(monkeypatch, tmpdir):
from devpi_web.main import get_indexer
import devpi_server.main
XOM = devpi_server.main.XOM
xom_container = []
class MyXOM(XOM):
def __init__(self, config):
xom_container.append(self)
XOM.__init__(self, config)
def httpget(self, url, allow_redirects=True, extra_headers=None):
class Response:
def __init__(self):
self.status_code = 200
self.text = """<a href='foo'>foo</a>"""
self.url = url
return Response()
def create_app(self):
# if the webserver is started, we fail
0 / 0
monkeypatch.setattr(devpi_server.main, "XOM", MyXOM)
result = devpi_server.main.main(
["devpi-server", "--serverdir",
str(tmpdir), "--init"])
assert not result
result = devpi_server.main.main([
"devpi-server", "--serverdir",
str(tmpdir), "--recreate-search-index"
])
assert result == 0
assert tmpdir.join('.indices').check()
(xom1, xom2) = xom_container
ix = get_indexer(xom2.config)
result = ix.query_projects('foo')
assert result['info']['found'] == 1
assert result['items'][0]['data'] == {
u'index': u'pypi',
u'name': u'foo',
u'path': u'/root/pypi/foo',
u'type': u'project',
u'user': u'root'
}
def test_search_root_pypi(mapp, testapp, pypistage):
from devpi_web.main import get_indexer
pypistage.mock_simple("pkg1", '<a href="/pkg1-2.6.zip" /a>')
pypistage.mock_simple("pkg2", '')
indexer = get_indexer(mapp.xom.config)
indexer.update_projects([
dict(name=u'pkg1', user=u'root', index=u'pypi'),
dict(name=u'pkg2', user=u'root', index=u'pypi')], clear=True)
r = testapp.xget(200, '/+search?query=pkg')
search_results = r.html.select('.searchresults > dl > dt')
assert len(search_results) == 2
links = search_results[0].findAll('a')
assert sorted((l.text.strip(), l.attrs['href']) for l in links) == [
("pkg1", "http://localhost/root/pypi/pkg1")]
links = search_results[1].findAll('a')
assert sorted((l.text.strip(), l.attrs['href']) for l in links) == [
("pkg2", "http://localhost/root/pypi/pkg2")]
def test_search_root_pypi(mapp, testapp, pypistage):
from devpi_web.main import get_indexer
pypistage.mock_simple("pkg1", '<a href="/pkg1-2.6.zip" /a>')
pypistage.mock_simple("pkg2", '')
indexer = get_indexer(mapp.xom.config)
indexer.update_projects([
dict(name=u'pkg1', user=u'root', index=u'pypi'),
dict(name=u'pkg2', user=u'root', index=u'pypi')], clear=True)
r = testapp.xget(200, '/+search?query=pkg')
search_results = r.html.select('.searchresults > dl > dt')
assert len(search_results) == 2
links = search_results[0].findAll('a')
assert sorted((compareable_text(l.text), l.attrs['href']) for l in links) == [
("pkg1", "http://localhost/root/pypi/pkg1")]
links = search_results[1].findAll('a')
assert sorted((compareable_text(l.text), l.attrs['href']) for l in links) == [
("pkg2", "http://localhost/root/pypi/pkg2")]
def test_index_projects_arg(monkeypatch, tmpdir):
from devpi_web.main import get_indexer
import devpi_server.main
XOM = devpi_server.main.XOM
xom_container = []
def MyXOM(config):
xom = XOM(config)
xom_container.append(xom)
return xom
# provide dummy data to avoid fetching mirror data from pypi
n2s = {u'foo': 1}
n2n = {u'foo': u'foo'}
monkeypatch.setattr(devpi_server.extpypi.PyPIMirror, "init_pypi_mirror",
lambda s, p: None)
monkeypatch.setattr(devpi_server.extpypi.PyPIMirror,
"name2serials",
n2s,
raising=False)
monkeypatch.setattr(devpi_server.extpypi.PyPIMirror,
"normname2name",
n2n,
raising=False)
monkeypatch.setattr(devpi_server.main, "XOM", MyXOM)
# if the webserver is started, we fail
monkeypatch.setattr(devpi_server.main, "wsgi_run", lambda *x: 0 / 0)
devpi_server.main.main(
["devpi-server", "--serverdir",
str(tmpdir), "--index-projects"])
assert tmpdir.join('.indices').check()
(xom, ) = xom_container
ix = get_indexer(xom.config)
result = ix.query_projects('foo')
assert result['info']['found'] == 1
assert result['items'][0]['data'] == {
u'index': u'pypi',
u'name': u'foo',
u'path': u'/root/pypi/foo',
u'type': u'project',
u'user': u'root'
}
def test_search_after_register(mapp, testapp):
from devpi_web.main import get_indexer
indexer_thread = get_indexer(mapp.xom).indexer_thread
mapp.xom.thread_pool.start_one(indexer_thread)
api = mapp.create_and_use()
mapp.set_versiondata(
{
"name": "pkg1",
"version": "2.6",
"description": "foo"
},
waithooks=True)
indexer_thread.wait()
r = testapp.get('/+search?query=foo', expect_errors=False)
links = r.html.select('.searchresults a')
assert [(l.text.strip(), l.attrs['href']) for l in links
] == [("pkg1-2.6", "http://localhost/%s/pkg1/2.6" % api.stagename),
("Description",
"http://localhost/%s/pkg1/2.6#description" % api.stagename)]
mapp.set_versiondata(
{
"name": "pkg1",
"version": "2.7",
"description": "foo"
},
waithooks=True)
indexer_thread.wait()
r = testapp.get('/+search?query=foo', expect_errors=False)
links = r.html.select('.searchresults a')
assert [(l.text.strip(), l.attrs['href']) for l in links
] == [("pkg1-2.7", "http://localhost/%s/pkg1/2.7" % api.stagename),
("Description",
"http://localhost/%s/pkg1/2.7#description" % api.stagename)]
r = testapp.xget(200, '/+search?query=foo')
links = r.html.select('.searchresults a')
assert [(l.text.strip(), l.attrs['href']) for l in links
] == [("pkg1-2.7", "http://localhost/%s/pkg1/2.7" % api.stagename),
("Description",
"http://localhost/%s/pkg1/2.7#description" % api.stagename)]
def test_pip_search(mapp, pypistage, testapp):
from devpi_web.main import get_indexer
from operator import itemgetter
api = mapp.create_and_use(indexconfig=dict(bases=["root/pypi"]))
pypistage.mock_simple("pkg1", '<a href="/pkg1-2.6.zip" /a>')
pypistage.mock_simple("pkg2", '')
# we need to set dummy data, so we can use waithooks
mapp.set_versiondata({"name": "pkg2", "version": "2.7"}, waithooks=True)
# now we can access the indexer directly without causing locking issues
indexer = get_indexer(mapp.xom.config)
indexer.update_projects([
dict(name=u'pkg1', user=u'root', index=u'pypi'),
dict(name=u'pkg2', user=u'root', index=u'pypi')
],
clear=True)
mapp.set_versiondata(
{
"name": "pkg2",
"version": "2.7",
"summary": "foo",
"description": "bar"
},
waithooks=True)
headers = {'Content-Type': 'text/xml'}
body = b"""<?xml version='1.0'?>
<methodCall>
<methodName>search</methodName>
<params>
<param>
<value><struct>
<member>
<name>summary</name>
<value><array><data>
<value><string>pkg</string></value>
</data></array></value>
</member>
<member>
<name>name</name>
<value><array><data>
<value><string>pkg</string></value>
</data></array></value>
</member>
</struct></value>
</param>
<param>
<value><string>or</string></value>
</param>
</params>
</methodCall>
"""
r = testapp.post('/%s/' % api.stagename, body, headers=headers)
assert r.status_code == 200
(data, method) = get_xmlrpc_data(r.body)
assert method is None
items = sorted(data[0], key=itemgetter('_pypi_ordering', 'name'))
assert len(items) == 2
# we only use cached data, so the version is empty
assert items[0]['name'] == 'pkg1'
assert items[0]['summary'] == '[root/pypi]'
assert items[0]['version'] == ''
assert items[1]['name'] == 'pkg2'
assert items[1]['summary'] == '[user1/dev] foo'
assert items[1]['version'] == '2.7'
# without root/pypi, we only get data from the private index
r = mapp.modify_index(api.stagename, indexconfig=dict(bases=()))
r = testapp.post('/%s/' % api.stagename, body, headers=headers)
assert r.status_code == 200
(data, method) = get_xmlrpc_data(r.body)
assert method is None
items = data[0]
assert len(items) == 1
assert items[0]['name'] == 'pkg2'