def thread_run(self): thread_push_log("[IDXQ]") with self.xom.keyfs.transaction(write=False) as tx: indexer = get_indexer(self.xom) searcher = indexer.get_project_ix().searcher() self.shared_data.queue_projects(iter_projects(self.xom), tx.at_serial, searcher)
def handler(self, is_from_mirror, serial, indexname, names): log.debug( "Got %s projects from %s at serial %s for indexing", len(names), indexname, serial) ix = get_indexer(self.xom) counter = itertools.count() project_ix = ix.get_project_ix() main_keys = project_ix.schema.names() writer = project_ix.writer() searcher = project_ix.searcher() try: with self.xom.keyfs.transaction(write=False) as tx: stage = self.xom.model.getstage(indexname) if stage is not None: for name in names: data = preprocess_project( ProjectIndexingInfo(stage=stage, name=name)) # because we use the current transaction, we also # use the current serial for indexing ix._update_project( data, tx.at_serial, counter, main_keys, writer, searcher=searcher) count = next(counter) except Exception: writer.cancel() # let the queue handle retries raise else: log.debug("Committing %s new documents to search index." % count) writer.commit()
def clear_index(argv=None): if argv is None: argv = sys.argv else: # for tests argv = [str(x) for x in argv] pluginmanager = get_pluginmanager() try: parser = MyArgumentParser(description="Clear project search index.", add_help=False) add_help_option(parser, pluginmanager) add_configfile_option(parser, pluginmanager) add_storage_options(parser, pluginmanager) add_indexer_backend_option(parser, pluginmanager) config = parseoptions(pluginmanager, argv, parser=parser) configure_cli_logging(config.args) xom = xom_from_config(config) log = xom.log log.warn("You should stop devpi-server before running this command.") ix = get_indexer(xom) ix.delete_index() log.info( "Index deleted, start devpi-server again to let the index rebuild automatically." ) except Fatal as e: tw = py.io.TerminalWriter(sys.stderr) tw.line("fatal: %s" % e.args[0], red=True) return 1
def test_index_projects_arg(monkeypatch, tmpdir): from devpi_web.main import get_indexer import devpi_server.main XOM = devpi_server.main.XOM xom_container = [] class MyXOM(XOM): def __init__(self, config): xom_container.append(self) XOM.__init__(self, config) def httpget(self, url, allow_redirects=True, extra_headers=None): class Response: def __init__(self): self.status_code = 200 self.text = """<a href='foo'>foo</a>""" self.url = url return Response() def create_app(self): # if the webserver is started, we fail 0 / 0 monkeypatch.setattr(devpi_server.main, "XOM", MyXOM) result = devpi_server.main.main( ["devpi-server", "--serverdir", str(tmpdir), "--init"]) assert not result result = devpi_server.main.main([ "devpi-server", "--serverdir", str(tmpdir), "--recreate-search-index" ]) assert result == 0 assert tmpdir.join('.indices').check() (xom1, xom2) = xom_container ix = get_indexer(xom2.config) result = ix.query_projects('foo') assert result['info']['found'] == 1 assert result['items'][0]['data'] == { u'index': u'pypi', u'name': u'foo', u'path': u'/root/pypi/foo', u'type': u'project', u'user': u'root' }
def test_search_root_pypi(mapp, testapp, pypistage): from devpi_web.main import get_indexer pypistage.mock_simple("pkg1", '<a href="/pkg1-2.6.zip" /a>') pypistage.mock_simple("pkg2", '') indexer = get_indexer(mapp.xom.config) indexer.update_projects([ dict(name=u'pkg1', user=u'root', index=u'pypi'), dict(name=u'pkg2', user=u'root', index=u'pypi')], clear=True) r = testapp.xget(200, '/+search?query=pkg') search_results = r.html.select('.searchresults > dl > dt') assert len(search_results) == 2 links = search_results[0].findAll('a') assert sorted((l.text.strip(), l.attrs['href']) for l in links) == [ ("pkg1", "http://localhost/root/pypi/pkg1")] links = search_results[1].findAll('a') assert sorted((l.text.strip(), l.attrs['href']) for l in links) == [ ("pkg2", "http://localhost/root/pypi/pkg2")]
def test_search_root_pypi(mapp, testapp, pypistage): from devpi_web.main import get_indexer pypistage.mock_simple("pkg1", '<a href="/pkg1-2.6.zip" /a>') pypistage.mock_simple("pkg2", '') indexer = get_indexer(mapp.xom.config) indexer.update_projects([ dict(name=u'pkg1', user=u'root', index=u'pypi'), dict(name=u'pkg2', user=u'root', index=u'pypi')], clear=True) r = testapp.xget(200, '/+search?query=pkg') search_results = r.html.select('.searchresults > dl > dt') assert len(search_results) == 2 links = search_results[0].findAll('a') assert sorted((compareable_text(l.text), l.attrs['href']) for l in links) == [ ("pkg1", "http://localhost/root/pypi/pkg1")] links = search_results[1].findAll('a') assert sorted((compareable_text(l.text), l.attrs['href']) for l in links) == [ ("pkg2", "http://localhost/root/pypi/pkg2")]
def test_index_projects_arg(monkeypatch, tmpdir): from devpi_web.main import get_indexer import devpi_server.main XOM = devpi_server.main.XOM xom_container = [] def MyXOM(config): xom = XOM(config) xom_container.append(xom) return xom # provide dummy data to avoid fetching mirror data from pypi n2s = {u'foo': 1} n2n = {u'foo': u'foo'} monkeypatch.setattr(devpi_server.extpypi.PyPIMirror, "init_pypi_mirror", lambda s, p: None) monkeypatch.setattr(devpi_server.extpypi.PyPIMirror, "name2serials", n2s, raising=False) monkeypatch.setattr(devpi_server.extpypi.PyPIMirror, "normname2name", n2n, raising=False) monkeypatch.setattr(devpi_server.main, "XOM", MyXOM) # if the webserver is started, we fail monkeypatch.setattr(devpi_server.main, "wsgi_run", lambda *x: 0 / 0) devpi_server.main.main( ["devpi-server", "--serverdir", str(tmpdir), "--index-projects"]) assert tmpdir.join('.indices').check() (xom, ) = xom_container ix = get_indexer(xom.config) result = ix.query_projects('foo') assert result['info']['found'] == 1 assert result['items'][0]['data'] == { u'index': u'pypi', u'name': u'foo', u'path': u'/root/pypi/foo', u'type': u'project', u'user': u'root' }
def test_search_after_register(mapp, testapp): from devpi_web.main import get_indexer indexer_thread = get_indexer(mapp.xom).indexer_thread mapp.xom.thread_pool.start_one(indexer_thread) api = mapp.create_and_use() mapp.set_versiondata( { "name": "pkg1", "version": "2.6", "description": "foo" }, waithooks=True) indexer_thread.wait() r = testapp.get('/+search?query=foo', expect_errors=False) links = r.html.select('.searchresults a') assert [(l.text.strip(), l.attrs['href']) for l in links ] == [("pkg1-2.6", "http://localhost/%s/pkg1/2.6" % api.stagename), ("Description", "http://localhost/%s/pkg1/2.6#description" % api.stagename)] mapp.set_versiondata( { "name": "pkg1", "version": "2.7", "description": "foo" }, waithooks=True) indexer_thread.wait() r = testapp.get('/+search?query=foo', expect_errors=False) links = r.html.select('.searchresults a') assert [(l.text.strip(), l.attrs['href']) for l in links ] == [("pkg1-2.7", "http://localhost/%s/pkg1/2.7" % api.stagename), ("Description", "http://localhost/%s/pkg1/2.7#description" % api.stagename)] r = testapp.xget(200, '/+search?query=foo') links = r.html.select('.searchresults a') assert [(l.text.strip(), l.attrs['href']) for l in links ] == [("pkg1-2.7", "http://localhost/%s/pkg1/2.7" % api.stagename), ("Description", "http://localhost/%s/pkg1/2.7#description" % api.stagename)]
def test_pip_search(mapp, pypistage, testapp): from devpi_web.main import get_indexer from operator import itemgetter api = mapp.create_and_use(indexconfig=dict(bases=["root/pypi"])) pypistage.mock_simple("pkg1", '<a href="/pkg1-2.6.zip" /a>') pypistage.mock_simple("pkg2", '') # we need to set dummy data, so we can use waithooks mapp.set_versiondata({"name": "pkg2", "version": "2.7"}, waithooks=True) # now we can access the indexer directly without causing locking issues indexer = get_indexer(mapp.xom.config) indexer.update_projects([ dict(name=u'pkg1', user=u'root', index=u'pypi'), dict(name=u'pkg2', user=u'root', index=u'pypi') ], clear=True) mapp.set_versiondata( { "name": "pkg2", "version": "2.7", "summary": "foo", "description": "bar" }, waithooks=True) headers = {'Content-Type': 'text/xml'} body = b"""<?xml version='1.0'?> <methodCall> <methodName>search</methodName> <params> <param> <value><struct> <member> <name>summary</name> <value><array><data> <value><string>pkg</string></value> </data></array></value> </member> <member> <name>name</name> <value><array><data> <value><string>pkg</string></value> </data></array></value> </member> </struct></value> </param> <param> <value><string>or</string></value> </param> </params> </methodCall> """ r = testapp.post('/%s/' % api.stagename, body, headers=headers) assert r.status_code == 200 (data, method) = get_xmlrpc_data(r.body) assert method is None items = sorted(data[0], key=itemgetter('_pypi_ordering', 'name')) assert len(items) == 2 # we only use cached data, so the version is empty assert items[0]['name'] == 'pkg1' assert items[0]['summary'] == '[root/pypi]' assert items[0]['version'] == '' assert items[1]['name'] == 'pkg2' assert items[1]['summary'] == '[user1/dev] foo' assert items[1]['version'] == '2.7' # without root/pypi, we only get data from the private index r = mapp.modify_index(api.stagename, indexconfig=dict(bases=())) r = testapp.post('/%s/' % api.stagename, body, headers=headers) assert r.status_code == 200 (data, method) = get_xmlrpc_data(r.body) assert method is None items = data[0] assert len(items) == 1 assert items[0]['name'] == 'pkg2'