def test_replication(tempdir, servers):
directory = os.path.join(tempdir, 'backup')
primary = servers.start(servers.ports[0], tempdir)
sync, update = '--autosync=' + primary.url, '--autoupdate=1'
secondary = servers.start(servers.ports[1], '-r', directory, sync, update)
resource = servers.start(servers.ports[2], '-r', directory)
for args in [('-r', tempdir), (update, tempdir),
(update, tempdir, tempdir)]:
assert subprocess.call(
(sys.executable, '-m', 'lupyne.server', sync) + args,
stderr=subprocess.PIPE)
primary.post('docs', [{}])
assert primary.post('update') == 1
assert primary.client.put('update/0').status_code == http.client.NOT_FOUND
response = resource.client.post(json={'url': primary.url})
assert response.status_code == http.client.ACCEPTED and sum(
response.json().values()) == 0
assert resource.post('update') == 1
assert resource.post(json={'url': primary.url})
assert resource.post('update') == 1
primary.post('docs', [{}])
assert primary.post('update') == 2
time.sleep(1.5)
assert sum(secondary().values()) == 2
servers.stop(servers.ports[-1])
root = server.WebSearcher(directory, urls=(primary.url, secondary.url))
app = server.mount(root)
root.fields = {}
assert root.update() == 2
assert len(root.urls) == 2
servers.stop(servers.ports[0])
assert secondary.docs()
assert secondary.client.post(
'docs', []).status_code == http.client.METHOD_NOT_ALLOWED
assert secondary.terms() == []
assert root.update() == 2
assert len(root.urls) == 1
servers.stop(servers.ports[1])
assert root.update() == 2
assert len(root.urls) == 0 and isinstance(app.root, server.WebIndexer)
app.root.close()
root = server.WebSearcher(directory)
app = server.mount(root, autoupdate=0.1)
root.fields, root.autoupdate = {}, 0.1
cherrypy.config['log.screen'] = servers.config['log.screen']
cherrypy.engine.state = cherrypy.engine.states.STARTED
root.monitor.start() # simulate engine starting
time.sleep(0.2)
app.root.indexer.add()
time.sleep(0.2)
assert len(app.root.indexer) == len(root.searcher) + 1
app.root.monitor.unsubscribe()
del app.root
def test_replication(tempdir, servers): # noqa
"Replication from indexer to searcher."
directory = os.path.join(tempdir, 'backup')
sync, update = '--autosync=' + servers.hosts[0], '--autoupdate=1'
servers.start(servers.ports[0], tempdir),
servers.start(servers.ports[1], '-r', directory, sync, update),
servers.start(servers.ports[2], '-r', directory),
for args in [('-r', tempdir), (update, tempdir), (update, tempdir, tempdir)]:
assert subprocess.call((sys.executable, '-m', 'lupyne.server', sync) + args, stderr=subprocess.PIPE)
replicas = client.Replicas(servers.hosts[:2], limit=1)
replicas.post('/docs', [{}])
assert replicas.post('/update') == 1
resource = client.Resource(servers.hosts[2])
response = resource.call('POST', '/', {'host': servers.hosts[0]})
assert response.status == httplib.ACCEPTED and sum(response().values()) == 0
assert resource.post('/update') == 1
assert resource.post('/', {'host': servers.hosts[0], 'path': '/'})
assert resource.post('/update') == 1
replicas.post('/docs', [{}])
assert replicas.post('/update') == 2
resource = client.Resource(servers.hosts[1])
time.sleep(1.1)
assert sum(resource.get('/').values()) == 2
servers.stop(servers.ports[-1])
root = server.WebSearcher(directory, hosts=servers.hosts[:2])
app = server.mount(root)
root.fields = {}
assert root.update() == 2
assert len(root.hosts) == 2
servers.stop(servers.ports[0])
assert replicas.get('/docs')
assert replicas.call('POST', '/docs', []).status == httplib.METHOD_NOT_ALLOWED
assert replicas.get('/terms', option='indexed') == []
assert replicas.call('POST', '/docs', [], retry=True).status == httplib.METHOD_NOT_ALLOWED
assert root.update() == 2
assert len(root.hosts) == 1
servers.stop(servers.ports[1])
assert root.update() == 2
assert len(root.hosts) == 0 and isinstance(app.root, server.WebIndexer)
app.root.close()
root = server.WebSearcher(directory)
app = server.mount(root, autoupdate=0.1)
root.fields, root.autoupdate = {}, 0.1
cherrypy.config['log.screen'] = servers.config['log.screen']
cherrypy.engine.state = cherrypy.engine.states.STARTED
root.monitor.start() # simulate engine starting
time.sleep(0.2)
app.root.indexer.add()
time.sleep(0.2)
assert len(app.root.indexer) == len(root.searcher) + 1
app.root.monitor.unsubscribe()
del app.root
def testInterface(self):
"Remote reading and writing."
self.servers += (
self.start(self.ports[0], self.tempdir, '--autoreload=1', **{'tools.validate.expires': 0, 'tools.validate.max_age': 0}),
self.start(self.ports[1], self.tempdir, self.tempdir, '--autoupdate=2'), # concurrent searchers
)
resource = client.Resource('localhost', self.ports[0])
assert resource.get('/favicon.ico')
response = resource.call('GET', '/')
assert response.status == httplib.OK and response.reason == 'OK' and response.time > 0
assert response.getheader('content-encoding') == 'gzip' and response.getheader('content-type').startswith('application/json')
version, modified = response.getheader('etag'), response.getheader('last-modified')
assert version.strip('W/"').isdigit()
assert int(response.getheader('age')) >= 0 and response.getheader('cache-control') == 'max-age=0'
dates = list(map(parsedate, [modified, response.getheader('expires'), response.getheader('date')]))
assert all(dates) and sorted(dates) == dates
(directory, count), = response().items()
assert count == 0 and 'FSDirectory@' in directory
assert resource.call('HEAD', '/').status == httplib.OK
with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
resource.post('/terms')
with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
resource.get('/update')
with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
resource.post('/update/snapshot')
with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
resource.put('/fields')
with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
resource.post('/docs/x/y')
with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
resource.put('/docs/0')
with assertRaises(httplib.HTTPException, httplib.METHOD_NOT_ALLOWED):
resource.delete('/docs')
httplib.HTTPConnection.request(resource, 'POST', '/docs', headers={'content-length': '0', 'content-type': 'application/json'})
assert resource.getresponse().status == httplib.BAD_REQUEST
httplib.HTTPConnection.request(resource, 'POST', '/docs', headers={'content-length': '0', 'content-type': 'application/x-www-form-urlencoded'})
assert resource.getresponse().status == httplib.UNSUPPORTED_MEDIA_TYPE
httplib.HTTPConnection.request(resource, 'GET', '/', headers={'accept': 'text/html'})
assert resource.getresponse().status == httplib.NOT_ACCEPTABLE
assert resource.get('/docs') == []
with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
resource.get('/docs/0')
with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
resource.get('/docs/x/y')
try:
assert resource.get('/docs/~')
except httplib.HTTPException as exc:
status, reason, body = exc
assert body['status'] == '404 Not Found'
assert body['message'].startswith('invalid literal for int')
assert body['message'] in body['traceback']
assert resource.get('/fields') == []
with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
resource.get('/fields/name')
assert resource.get('/terms') == []
assert resource.get('/terms/x') == []
assert resource.get('/terms/x/:') == []
assert resource.get('/terms/x/y') == 0
assert resource.get('/terms/x/y/docs') == []
assert resource.get('/terms/x/y/docs/counts') == []
assert resource.get('/terms/x/y/docs/positions') == []
with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
resource.get('/terms/x/y/missing')
with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
resource.get('/terms/x/y/docs/missing')
defaults = {'index': 'ANALYZED', 'store': 'NO', 'termvector': 'NO'}
response = resource.call('PUT', '/fields/text')
assert response.getheader('etag') is None
assert response.status == httplib.CREATED and response() == defaults
response = resource.call('PUT', '/fields/text', {})
assert response.status == httplib.OK and response() == defaults
assert resource.put('/fields/name', {'store': True, 'index': 'not_analyzed'})
assert sorted(resource.get('/fields')) == ['name', 'text']
assert resource.get('/fields/text')['index'] == 'ANALYZED'
assert not resource.post('/docs', [{'name': 'sample', 'text': 'hello world'}])
assert not resource.post('/docs')
(directory, count), = resource.get('/').items()
assert count == 1
assert resource.get('/docs') == []
result = resource.get('/search?q=text:hello')
assert math.isnan(result.pop('maxscore'))
assert result == {'query': 'text:hello', 'count': 0, 'docs': []}
resource.headers['if-none-match'] = version
response = resource.call('GET', '/', redirect=True)
assert response.status == httplib.NOT_MODIFIED and response.getheader('etag') == version
del resource.headers['if-none-match']
resource.headers['if-modified-since'] = modified
assert resource.call('GET', '/').status == httplib.NOT_MODIFIED
time.sleep(max(0, calendar.timegm(parsedate(modified)) + 1 - time.time()))
assert resource.post('/update')
response = resource.call('GET', '/')
assert response and response.getheader('etag') != version and parsedate(response.getheader('last-modified')) > parsedate(modified)
resource.headers['if-match'] = version
assert resource.call('GET', '/docs/0').status == httplib.PRECONDITION_FAILED
del resource.headers['if-match']
assert resource.get('/docs') == [0]
assert resource.get('/docs/0') == resource.get('/docs/name/sample') == {'name': 'sample'}
assert resource.get('/docs/0', fields='missing') == {'missing': None}
assert resource.get('/docs/0', fields='', **{'fields.multi': 'missing'}) == {'missing': []}
assert resource.get('/terms') == ['name', 'text']
assert resource.get('/terms', option='unindexed') == []
assert resource.get('/terms/text') == ['hello', 'world']
assert resource.get('/terms/text/world') == 1
with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
resource.get('/terms/text/world~-')
with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
resource.get('/terms/text/world~?count=')
assert resource.get('/terms/text/world?count=1')
assert resource.get('/terms/text/world/docs') == [0]
assert resource.get('/terms/text/world/docs/counts') == [[0, 1]]
assert resource.get('/terms/text/world/docs/positions') == [[0, [1]]]
with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
resource.get('/search?count=')
with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
resource.get('/search', sort='-x,y')
with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
resource.get('/search', count=1, sort='x:str')
with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
resource.get('/search', count=1, group='x:str')
with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
resource.get('/search', q='')
with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
resource.get('/search?q.test=True')
assert resource.get('/search', count=0) == {'count': 1, 'maxscore': 1.0, 'query': None, 'docs': []}
assert resource.get('/search', fields='')['docs'] == [{'__id__': 0, '__score__': 1.0}]
hit, = resource.get('/search', fields='', **{'fields.multi': 'name'})['docs']
assert hit == {'__id__': 0, 'name': ['sample'], '__score__': 1.0}
hit, = resource.get('/search', q='name:sample', fields='', hl='name')['docs']
assert sorted(hit) == ['__highlights__', '__id__', '__score__']
result = resource.get('/search', q='text:hello')
assert result == resource.get('/search?q=hello&q.field=text')
assert result['count'] == resource.get('/search')['count'] == 1
assert result['query'] == 'text:hello'
assert 0 < result['maxscore'] < 1
doc, = result['docs']
assert sorted(doc) == ['__id__', '__score__', 'name']
assert doc['__id__'] == 0 and doc['__score__'] > 0 and doc['name'] == 'sample'
hit, = resource.get('/search', q='hello world', **{'q.field': ['text', 'body']})['docs']
assert hit['__id__'] == doc['__id__'] and hit['__score__'] < doc['__score__']
hit, = resource.get('/search', q='hello world', **{'q.field': 'text', 'q.op': 'and'})['docs']
assert hit['__id__'] == doc['__id__'] and hit['__score__'] > doc['__score__']
result = resource.get('/search?q=hello+world&q.field=text^4&q.field=body')
assert result['query'] == '(body:hello text:hello^4.0) (body:world text:world^4.0)'
assert resource.get('/search', q='hello', **{'q.field': 'body.title^2.0'})['query'] == 'body.title:hello^2.0'
hit, = result['docs']
assert hit['__id__'] == doc['__id__'] and hit['__score__'] > doc['__score__']
result = resource.get('/search', facets='name', spellcheck=1)
assert result['facets'] == {'name': {'sample': 1}} and result['spellcheck'] == {}
resource = client.Resource('localhost', self.ports[-1])
assert set(resource.get('/').values()) < set([0, 1])
assert resource.post('/update', ['filters', 'sorters']) == 2
assert resource.get('/docs') == [0, 1]
with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
resource.get('/docs/name/sample')
with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
resource.get('/fields')
resource = client.Resource('localhost', self.ports[0])
assert not resource.delete('/search', q='sample', **{'q.field': 'name', 'q.type': 'term'})
assert resource.get('/docs') == [0]
assert not resource.post('/update', ['expunge'])
assert resource.get('/docs') == []
assert not resource.put('/docs/name/sample')
assert resource.post('/update')
assert resource.get('/docs/name/sample')
assert not resource.put('/docs/name/sample')
assert resource.post('/update')
assert resource.get('/docs/name/sample')
with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
assert resource.put('/fields/name', {'omit': True})
with assertRaises(httplib.HTTPException, httplib.BAD_REQUEST):
resource.put('/docs/name/sample', {'name': 'mismatched'})
with assertRaises(httplib.HTTPException, httplib.CONFLICT):
resource.put('/docs/missing/sample')
assert resource.put('/fields/name', {'store': True, 'index': False, 'omitNorms': True})
with assertRaises(httplib.HTTPException, httplib.CONFLICT):
resource.put('/docs/name/sample')
assert not resource.delete('/docs/missing/sample')
resource.post('/update')
with assertRaises(httplib.HTTPException, httplib.NOT_FOUND):
resource.get('/docs/missing/sample')
assert not resource.delete('/search')
responses = resource.multicall(('POST', '/docs', [{}]), ('POST', '/update'), ('GET', '/docs'))
assert responses[0].status == httplib.ACCEPTED and responses[1]() == len(responses[2]()) == 1
assert resource.post('/', [self.tempdir]).values() == [2]
with local.assertWarns(DeprecationWarning, UserWarning):
assert Resource(resource.host, resource.port).call('GET', '/missing', redirect=True)()
response = resource.call('PUT', '/update/snapshot')
assert response.status == httplib.CREATED
assert all(name.startswith('_') or name.startswith('segments_') for name in response())
assert resource.put('/update/backup') == response()
with assertRaises(httplib.HTTPException, httplib.CONFLICT):
resource.put('/update/snapshot')
assert not resource.delete('/update/snapshot')
with assertRaises(httplib.HTTPException, httplib.CONFLICT):
resource.delete('/update/snapshot')
resource = client.Resource('localhost', self.ports[-1] + 1)
with assertRaises(socket.error, errno.ECONNREFUSED):
resource.get('/')
self.stop(self.servers.pop(0))
pidfile = os.path.join(self.tempdir, 'pid')
self.start(self.ports[0], '-dp', pidfile)
time.sleep(1)
os.kill(int(open(pidfile).read()), signal.SIGTERM)
filepath = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lupyne/server.py')
assert subprocess.call((sys.executable, filepath, '-c', filepath), stderr=subprocess.PIPE)
assert server.mount(None, '/path', config={'': {}})
server.init(vmargs=None)
self.assertRaises(AttributeError, server.start, config=True)