Exemplo n.º 1
0
def test_replication(tempdir, servers):
    directory = os.path.join(tempdir, 'backup')
    primary = servers.start(servers.ports[0], tempdir)
    sync, update = '--autosync=' + primary.url, '--autoupdate=1'
    secondary = servers.start(servers.ports[1], '-r', directory, sync, update)
    resource = servers.start(servers.ports[2], '-r', directory)
    for args in [('-r', tempdir), (update, tempdir),
                 (update, tempdir, tempdir)]:
        assert subprocess.call(
            (sys.executable, '-m', 'lupyne.server', sync) + args,
            stderr=subprocess.PIPE)
    primary.post('docs', [{}])
    assert primary.post('update') == 1
    assert primary.client.put('update/0').status_code == http.client.NOT_FOUND
    response = resource.client.post(json={'url': primary.url})
    assert response.status_code == http.client.ACCEPTED and sum(
        response.json().values()) == 0
    assert resource.post('update') == 1
    assert resource.post(json={'url': primary.url})
    assert resource.post('update') == 1
    primary.post('docs', [{}])
    assert primary.post('update') == 2
    time.sleep(1.5)
    assert sum(secondary().values()) == 2
    servers.stop(servers.ports[-1])
    root = server.WebSearcher(directory, urls=(primary.url, secondary.url))
    app = server.mount(root)
    root.fields = {}
    assert root.update() == 2
    assert len(root.urls) == 2
    servers.stop(servers.ports[0])
    assert secondary.docs()
    assert secondary.client.post(
        'docs', []).status_code == http.client.METHOD_NOT_ALLOWED
    assert secondary.terms() == []
    assert root.update() == 2
    assert len(root.urls) == 1
    servers.stop(servers.ports[1])
    assert root.update() == 2
    assert len(root.urls) == 0 and isinstance(app.root, server.WebIndexer)
    app.root.close()
    root = server.WebSearcher(directory)
    app = server.mount(root, autoupdate=0.1)
    root.fields, root.autoupdate = {}, 0.1
    cherrypy.config['log.screen'] = servers.config['log.screen']
    cherrypy.engine.state = cherrypy.engine.states.STARTED
    root.monitor.start()  # simulate engine starting
    time.sleep(0.2)
    app.root.indexer.add()
    time.sleep(0.2)
    assert len(app.root.indexer) == len(root.searcher) + 1
    app.root.monitor.unsubscribe()
    del app.root
Exemplo n.º 2
0
def test_replication(tempdir, servers):  # noqa
    "Replication from indexer to searcher."
    directory = os.path.join(tempdir, 'backup')
    sync, update = '--autosync=' + servers.hosts[0], '--autoupdate=1'
    servers.start(servers.ports[0], tempdir),
    servers.start(servers.ports[1], '-r', directory, sync, update),
    servers.start(servers.ports[2], '-r', directory),
    for args in [('-r', tempdir), (update, tempdir), (update, tempdir, tempdir)]:
        assert subprocess.call((sys.executable, '-m', 'lupyne.server', sync) + args, stderr=subprocess.PIPE)
    replicas = client.Replicas(servers.hosts[:2], limit=1)
    replicas.post('/docs', [{}])
    assert replicas.post('/update') == 1
    resource = client.Resource(servers.hosts[2])
    response = resource.call('POST', '/', {'host': servers.hosts[0]})
    assert response.status == httplib.ACCEPTED and sum(response().values()) == 0
    assert resource.post('/update') == 1
    assert resource.post('/', {'host': servers.hosts[0], 'path': '/'})
    assert resource.post('/update') == 1
    replicas.post('/docs', [{}])
    assert replicas.post('/update') == 2
    resource = client.Resource(servers.hosts[1])
    time.sleep(1.1)
    assert sum(resource.get('/').values()) == 2
    servers.stop(servers.ports[-1])
    root = server.WebSearcher(directory, hosts=servers.hosts[:2])
    app = server.mount(root)
    root.fields = {}
    assert root.update() == 2
    assert len(root.hosts) == 2
    servers.stop(servers.ports[0])
    assert replicas.get('/docs')
    assert replicas.call('POST', '/docs', []).status == httplib.METHOD_NOT_ALLOWED
    assert replicas.get('/terms', option='indexed') == []
    assert replicas.call('POST', '/docs', [], retry=True).status == httplib.METHOD_NOT_ALLOWED
    assert root.update() == 2
    assert len(root.hosts) == 1
    servers.stop(servers.ports[1])
    assert root.update() == 2
    assert len(root.hosts) == 0 and isinstance(app.root, server.WebIndexer)
    app.root.close()
    root = server.WebSearcher(directory)
    app = server.mount(root, autoupdate=0.1)
    root.fields, root.autoupdate = {}, 0.1
    cherrypy.config['log.screen'] = servers.config['log.screen']
    cherrypy.engine.state = cherrypy.engine.states.STARTED
    root.monitor.start()  # simulate engine starting
    time.sleep(0.2)
    app.root.indexer.add()
    time.sleep(0.2)
    assert len(app.root.indexer) == len(root.searcher) + 1
    app.root.monitor.unsubscribe()
    del app.root
Exemplo n.º 3
0
 def testInterface(self):
     "Remote reading and writing."
     self.servers += (
         self.start(self.ports[0], self.tempdir, '--autoreload=1', **{'tools.validate.expires': 0, 'tools.validate.max_age': 0}),
         self.start(self.ports[1], self.tempdir, self.tempdir, '--autoupdate=2'), # concurrent searchers
     )
     resource = client.Resource('localhost', self.ports[0])
     assert resource.get('/favicon.ico')
     response = resource.call('GET', '/')
     assert response.status == httplib.OK and response.reason == 'OK' and response.time > 0
     assert response.getheader('content-encoding') == 'gzip' and response.getheader('content-type').startswith('application/json')
     version, modified = response.getheader('etag'), response.getheader('last-modified')
     assert version.strip('W/"').isdigit()
     assert int(response.getheader('age')) >= 0 and response.getheader('cache-control') == 'max-age=0'
     dates = list(map(parsedate, [modified, response.getheader('expires'), response.getheader('date')]))
     assert all(dates) and sorted(dates) == dates
     (directory, count), = response().items()
     assert count == 0 and 'FSDirectory@' in directory
     assert resource.call('HEAD', '/').status == httplib.OK
     with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
         resource.post('/terms')
     with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
         resource.get('/update')
     with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
         resource.post('/update/snapshot')
     with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
         resource.put('/fields')
     with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
         resource.post('/docs/x/y')
     with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
         resource.put('/docs/0')
     with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
         resource.delete('/docs')
     httplib.HTTPConnection.request(resource, 'POST', '/docs', headers={'content-length': '0', 'content-type': 'application/json'})
     assert resource.getresponse().status == httplib.BAD_REQUEST
     httplib.HTTPConnection.request(resource, 'POST', '/docs', headers={'content-length': '0', 'content-type': 'application/x-www-form-urlencoded'})
     assert resource.getresponse().status == httplib.UNSUPPORTED_MEDIA_TYPE
     httplib.HTTPConnection.request(resource, 'GET', '/', headers={'accept': 'text/html'})
     assert resource.getresponse().status == httplib.NOT_ACCEPTABLE
     assert resource.get('/docs') == []
     with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
         resource.get('/docs/0')
     with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
         resource.get('/docs/x/y')
     try:
         assert resource.get('/docs/~')
     except httplib.HTTPException as exc:
         status, reason, body = exc
         assert body['status'] == '404 Not Found'
         assert body['message'].startswith('invalid literal for int')
         assert body['message'] in body['traceback']
     assert resource.get('/fields') == []
     with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
         resource.get('/fields/name')
     assert resource.get('/terms') == []
     assert resource.get('/terms/x') == []
     assert resource.get('/terms/x/:') == []
     assert resource.get('/terms/x/y') == 0
     assert resource.get('/terms/x/y/docs') == []
     assert resource.get('/terms/x/y/docs/counts') == []
     assert resource.get('/terms/x/y/docs/positions') == []
     with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
         resource.get('/terms/x/y/missing')
     with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
         resource.get('/terms/x/y/docs/missing')
     defaults = {'index': 'ANALYZED', 'store': 'NO', 'termvector': 'NO'}
     response = resource.call('PUT', '/fields/text')
     assert response.getheader('etag') is None
     assert response.status == httplib.CREATED and response() == defaults
     response = resource.call('PUT', '/fields/text', {})
     assert response.status == httplib.OK and response() == defaults
     assert resource.put('/fields/name', {'store': True, 'index': 'not_analyzed'})
     assert sorted(resource.get('/fields')) == ['name', 'text']
     assert resource.get('/fields/text')['index'] == 'ANALYZED'
     assert not resource.post('/docs', [{'name': 'sample', 'text': 'hello world'}])
     assert not resource.post('/docs')
     (directory, count), = resource.get('/').items()
     assert count == 1
     assert resource.get('/docs') == []
     result = resource.get('/search?q=text:hello')
     assert math.isnan(result.pop('maxscore'))
     assert result == {'query': 'text:hello', 'count': 0, 'docs': []}
     resource.headers['if-none-match'] = version
     response = resource.call('GET', '/', redirect=True)
     assert response.status == httplib.NOT_MODIFIED and response.getheader('etag') == version
     del resource.headers['if-none-match']
     resource.headers['if-modified-since'] = modified
     assert resource.call('GET', '/').status == httplib.NOT_MODIFIED
     time.sleep(max(0, calendar.timegm(parsedate(modified)) + 1 - time.time()))
     assert resource.post('/update')
     response = resource.call('GET', '/')
     assert response and response.getheader('etag') != version and parsedate(response.getheader('last-modified')) > parsedate(modified)
     resource.headers['if-match'] = version
     assert resource.call('GET', '/docs/0').status == httplib.PRECONDITION_FAILED
     del resource.headers['if-match']
     assert resource.get('/docs') == [0]
     assert resource.get('/docs/0') == resource.get('/docs/name/sample') == {'name': 'sample'}
     assert resource.get('/docs/0', fields='missing') == {'missing': None}
     assert resource.get('/docs/0', fields='', **{'fields.multi': 'missing'}) == {'missing': []}
     assert resource.get('/terms') == ['name', 'text']
     assert resource.get('/terms', option='unindexed') == []
     assert resource.get('/terms/text') == ['hello', 'world']
     assert resource.get('/terms/text/world') == 1
     with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
         resource.get('/terms/text/world~-')
     with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
         resource.get('/terms/text/world~?count=')
     assert resource.get('/terms/text/world?count=1')
     assert resource.get('/terms/text/world/docs') == [0]
     assert resource.get('/terms/text/world/docs/counts') == [[0, 1]]
     assert resource.get('/terms/text/world/docs/positions') == [[0, [1]]]
     with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
         resource.get('/search?count=')
     with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
         resource.get('/search', sort='-x,y')
     with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
         resource.get('/search', count=1, sort='x:str')
     with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
         resource.get('/search', count=1, group='x:str')
     with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
         resource.get('/search', q='')
     with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
         resource.get('/search?q.test=True')
     assert resource.get('/search', count=0) == {'count': 1, 'maxscore': 1.0, 'query': None, 'docs': []}
     assert resource.get('/search', fields='')['docs'] == [{'__id__': 0, '__score__': 1.0}]
     hit, = resource.get('/search', fields='', **{'fields.multi': 'name'})['docs']
     assert hit == {'__id__': 0, 'name': ['sample'], '__score__': 1.0}
     hit, = resource.get('/search', q='name:sample', fields='', hl='name')['docs']
     assert sorted(hit) == ['__highlights__', '__id__', '__score__']
     result = resource.get('/search', q='text:hello')
     assert result == resource.get('/search?q=hello&q.field=text')
     assert result['count'] == resource.get('/search')['count'] == 1
     assert result['query'] == 'text:hello'
     assert 0 < result['maxscore'] < 1
     doc, = result['docs']
     assert sorted(doc) == ['__id__', '__score__', 'name']
     assert doc['__id__'] == 0 and doc['__score__'] > 0 and doc['name'] == 'sample' 
     hit, = resource.get('/search', q='hello world', **{'q.field': ['text', 'body']})['docs']
     assert hit['__id__'] == doc['__id__'] and hit['__score__'] < doc['__score__']
     hit, = resource.get('/search', q='hello world', **{'q.field': 'text', 'q.op': 'and'})['docs']
     assert hit['__id__'] == doc['__id__'] and hit['__score__'] > doc['__score__']
     result = resource.get('/search?q=hello+world&q.field=text^4&q.field=body')
     assert result['query'] == '(body:hello text:hello^4.0) (body:world text:world^4.0)'
     assert resource.get('/search', q='hello', **{'q.field': 'body.title^2.0'})['query'] == 'body.title:hello^2.0'
     hit, = result['docs']
     assert hit['__id__'] == doc['__id__'] and hit['__score__'] > doc['__score__']
     result = resource.get('/search', facets='name', spellcheck=1)
     assert result['facets'] == {'name': {'sample': 1}} and result['spellcheck'] == {}
     resource = client.Resource('localhost', self.ports[-1])
     assert set(resource.get('/').values()) < set([0, 1])
     assert resource.post('/update', ['filters', 'sorters']) == 2
     assert resource.get('/docs') == [0, 1]
     with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
         resource.get('/docs/name/sample')
     with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
         resource.get('/fields')
     resource = client.Resource('localhost', self.ports[0])
     assert not resource.delete('/search', q='sample', **{'q.field': 'name', 'q.type': 'term'})
     assert resource.get('/docs') == [0]
     assert not resource.post('/update', ['expunge'])
     assert resource.get('/docs') == []
     assert not resource.put('/docs/name/sample')
     assert resource.post('/update')
     assert resource.get('/docs/name/sample')
     assert not resource.put('/docs/name/sample')
     assert resource.post('/update')
     assert resource.get('/docs/name/sample')
     with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
         assert resource.put('/fields/name', {'omit': True})
     with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
         resource.put('/docs/name/sample', {'name': 'mismatched'})
     with assertRaises(httplib.HTTPException, httplib.CONFLICT):
         resource.put('/docs/missing/sample')
     assert resource.put('/fields/name', {'store': True, 'index': False, 'omitNorms': True})
     with assertRaises(httplib.HTTPException, httplib.CONFLICT):
         resource.put('/docs/name/sample')
     assert not resource.delete('/docs/missing/sample')
     resource.post('/update')
     with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
         resource.get('/docs/missing/sample')
     assert not resource.delete('/search')
     responses = resource.multicall(('POST', '/docs', [{}]), ('POST', '/update'), ('GET', '/docs'))
     assert responses[0].status == httplib.ACCEPTED and responses[1]() == len(responses[2]()) == 1
     assert resource.post('/', [self.tempdir]).values() == [2]
     with local.assertWarns(DeprecationWarning, UserWarning):
         assert Resource(resource.host, resource.port).call('GET', '/missing', redirect=True)()
     response = resource.call('PUT', '/update/snapshot')
     assert response.status == httplib.CREATED
     assert all(name.startswith('_') or name.startswith('segments_') for name in response())
     assert resource.put('/update/backup') == response()
     with assertRaises(httplib.HTTPException, httplib.CONFLICT):
         resource.put('/update/snapshot')
     assert not resource.delete('/update/snapshot')
     with assertRaises(httplib.HTTPException, httplib.CONFLICT):
         resource.delete('/update/snapshot')
     resource = client.Resource('localhost', self.ports[-1] + 1)
     with assertRaises(socket.error, errno.ECONNREFUSED):
         resource.get('/')
     self.stop(self.servers.pop(0))
     pidfile = os.path.join(self.tempdir, 'pid')
     self.start(self.ports[0], '-dp', pidfile)
     time.sleep(1)
     os.kill(int(open(pidfile).read()), signal.SIGTERM)
     filepath = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lupyne/server.py')
     assert subprocess.call((sys.executable, filepath, '-c', filepath), stderr=subprocess.PIPE)
     assert server.mount(None, '/path', config={'': {}})
     server.init(vmargs=None)
     self.assertRaises(AttributeError, server.start, config=True)