def test_elastic_commit(self):
"""Test the auto_commit_interval attribute."""
docc = {'_id': '3', 'name': 'Waldo'}
# Disable 1s refresh in elasticsearch
# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
disable_refresh_body = {"index" : {"refresh_interval" : "-1"}}
self.elastic_conn.indices.put_settings(index="test",body=disable_refresh_body)
# test cases:
# -1 = no autocommit
# 0 = commit immediately
# x > 0 = commit within x seconds
for autocommit_interval in [None, 0, 1, 2]:
docman = DocManager(elastic_pair,auto_commit_interval=autocommit_interval)
docman.upsert(docc, *TESTARGS)
if autocommit_interval is None:
docman.commit()
else:
# Allow just a little extra time
time.sleep(autocommit_interval + 1)
results = list(self._search())
self.assertEqual(len(results), 1,
"should commit document with "
"auto_commit_interval = %s" % str(
autocommit_interval))
self.assertEqual(results[0]["name"], "Waldo")
docman.stop()
self._remove()
retry_until_ok(self.elastic_conn.indices.refresh, index="")
enable_refresh_body = {"index" : {"refresh_interval" : "1s"}}
self.elastic_conn.indices.put_settings(index="test",body=enable_refresh_body)
def test_auto_send_interval(self):
"""Test the auto_send_interval
auto_send_interval should control the amount of time to wait before
sending (but not committing) buffered operations.
"""
doc = {"_id": "3", "name": "Waldo"}
# test cases:
# None, 0 = no auto send
# x > 0 = send buffered operations within x seconds
for send_interval in [None, 0, 3, 8]:
docman = DocManager(
elastic_pair, autoSendInterval=send_interval, auto_commit_interval=None
)
docman.upsert(doc, *TESTARGS)
if send_interval:
# Allow just a little extra time
time.sleep(send_interval + 2)
results = list(self._search())
self.assertEqual(
len(results),
0,
"documents should not be commited with "
"auto_commit_interval=None and auto_commit_interval=%s"
% (send_interval,),
)
# Commit the possibly sent changes and search again
retry_until_ok(self.elastic_conn.indices.refresh, index="")
results = list(self._search())
if not send_interval:
self.assertEqual(
len(results),
0,
"should not send document with auto_send_interval=%s"
% (send_interval,),
)
else:
self.assertEqual(
len(results),
1,
"should send document with auto_send_interval=%s"
% (send_interval,),
)
self.assertEqual(results[0]["name"], "Waldo")
docman.stop()
self._remove()
retry_until_ok(self.elastic_conn.indices.refresh, index="")
class ElasticsearchTestCase(unittest.TestCase):
"""Base class for all ES TestCases."""
@classmethod
def setUpClass(cls):
cls.elastic_conn = Elasticsearch(hosts=[elastic_pair])
def setUp(self):
# Create target index in elasticsearch
self.elastic_conn.indices.create(index='test', ignore=400)
self.elastic_conn.cluster.health(wait_for_status='yellow',
index='test')
self.elastic_doc = DocManager(elastic_pair, auto_commit_interval=0)
def tearDown(self):
self.elastic_conn.indices.delete(index='test', ignore=404)
self.elastic_doc.stop()
def _search(self, query=None):
query = query or {"match_all": {}}
return self.elastic_doc._stream_search(
index="test", doc_type='test',
body={"query": query}
)
def _count(self):
return self.elastic_conn.count(index='test')['count']
def _remove(self):
bulk_deletes = []
for result in scan(self.elastic_conn,
index="test",
doc_type="test"):
result['_op_type'] = 'delete'
bulk_deletes.append(result)
bulk(self.elastic_conn, bulk_deletes)
def _mappings(self, index='_all'):
mappings = self.elastic_conn.indices.get_mapping(index=index)
if index in mappings:
return list(mappings[index]['mappings'].keys())
return []
def _indices(self):
return list(self.elastic_conn.indices.stats()['indices'].keys())
class ElasticsearchTestCase(unittest.TestCase):
"""Base class for all ES TestCases."""
@classmethod
def setUpClass(cls):
cls.elastic_conn = Elasticsearch(hosts=[elastic_pair])
def setUp(self):
# Create target index in elasticsearch
self.elastic_conn.indices.create(index="test", ignore=400)
self.elastic_conn.cluster.health(wait_for_status="yellow",
index="test")
self.elastic_doc = DocManager(elastic_pair, auto_commit_interval=0)
def tearDown(self):
self.elastic_conn.indices.delete(index="test", ignore=404)
self.elastic_doc.stop()
def _search(self, query=None):
query = query or {"match_all": {}}
return self.elastic_doc._stream_search(index="test",
doc_type="test",
body={"query": query})
def _count(self):
return self.elastic_conn.count(index="test")["count"]
def _remove(self):
bulk_deletes = []
for result in scan(self.elastic_conn, index="test", doc_type="test"):
result["_op_type"] = "delete"
bulk_deletes.append(result)
bulk(self.elastic_conn, bulk_deletes)
def _mappings(self, index="_all"):
mappings = self.elastic_conn.indices.get_mapping(index=index)
if index in mappings:
return list(mappings[index]["mappings"].keys())
return []
def _indices(self):
return list(self.elastic_conn.indices.stats()["indices"].keys())
def test_elastic_commit(self):
"""Test the auto_commit_interval attribute."""
doc = {'_id': '3', 'name': 'Waldo'}
# test cases:
# None = no autocommit
# 0 = commit immediately
# x > 0 = commit within x seconds
for commit_interval in [None, 0, 2, 8]:
docman = DocManager(elastic_pair,
auto_commit_interval=commit_interval)
docman.upsert(doc, *TESTARGS)
if commit_interval:
# Allow just a little extra time
time.sleep(commit_interval + 2)
results = list(self._search())
if commit_interval is None:
self.assertEqual(len(results), 0,
"should not commit document with "
"auto_commit_interval = None")
else:
self.assertEqual(len(results), 1,
"should commit document with "
"auto_commit_interval = %s" % (
commit_interval,))
self.assertEqual(results[0]["name"], "Waldo")
docman.stop()
self._remove()
retry_until_ok(self.elastic_conn.indices.refresh, index="")
def test_elastic_commit(self):
"""Test the auto_commit_interval attribute."""
docc = {'_id': '3', 'name': 'Waldo'}
docman = DocManager(elastic_pair)
# test cases:
# -1 = no autocommit
# 0 = commit immediately
# x > 0 = commit within x seconds
for autocommit_interval in [None, 0, 1, 2]:
docman.auto_commit_interval = autocommit_interval
docman.upsert(docc, *TESTARGS)
if autocommit_interval is None:
docman.commit()
else:
# Allow just a little extra time
time.sleep(autocommit_interval + 1)
results = list(self._search())
self.assertEqual(
len(results), 1, "should commit document with "
"auto_commit_interval = %s" % str(autocommit_interval))
self.assertEqual(results[0]["name"], "Waldo")
self._remove()
docman.stop()
def setUp(self):
"""Start a new Connector for each test."""
super(TestElastic, self).setUp()
try:
os.unlink("oplog.timestamp")
except OSError:
pass
docman = DocManager(elastic_pair)
self.connector = Connector(mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(docman, ),
gridfs_set=['test.test'])
self.conn.test.test.drop()
self.conn.test.test.files.drop()
self.conn.test.test.chunks.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: self._count() == 0)
def test_auto_send_interval(self):
"""Test the auto_send_interval
auto_send_interval should control the amount of time to wait before
sending (but not committing) buffered operations.
"""
doc = {"_id": "3", "name": "Waldo"}
# test cases:
# None, 0 = no auto send
# x > 0 = send buffered operations within x seconds
for send_interval in [None, 0, 3, 8]:
docman = DocManager(elastic_pair,
autoSendInterval=send_interval,
auto_commit_interval=None)
docman.upsert(doc, *TESTARGS)
if send_interval:
# Allow just a little extra time
time.sleep(send_interval + 2)
results = list(self._search())
self.assertEqual(
len(results),
0,
"documents should not be commited with "
"auto_commit_interval=None and auto_commit_interval=%s" %
(send_interval, ),
)
# Commit the possibly sent changes and search again
retry_until_ok(self.elastic_conn.indices.refresh, index="")
results = list(self._search())
if not send_interval:
self.assertEqual(
len(results),
0,
"should not send document with auto_send_interval=%s" %
(send_interval, ),
)
else:
self.assertEqual(
len(results),
1,
"should send document with auto_send_interval=%s" %
(send_interval, ),
)
self.assertEqual(results[0]["name"], "Waldo")
docman.stop()
self._remove()
retry_until_ok(self.elastic_conn.indices.refresh, index="")
def test_elastic_commit(self):
"""Test the auto_commit_interval attribute."""
docc = {'_id': '3', 'name': 'Waldo'}
docman = DocManager(elastic_pair)
# test cases:
# -1 = no autocommit
# 0 = commit immediately
# x > 0 = commit within x seconds
for autocommit_interval in [None, 0, 1, 2]:
docman.auto_commit_interval = autocommit_interval
docman.upsert(docc, *TESTARGS)
if autocommit_interval is None:
docman.commit()
else:
# Allow just a little extra time
time.sleep(autocommit_interval + 1)
results = list(self._search())
self.assertEqual(len(results), 1,
"should commit document with "
"auto_commit_interval = %s" % str(
autocommit_interval))
self.assertEqual(results[0]["name"], "Waldo")
self._remove()
docman.stop()
def setUpClass(cls):
cls.elastic_conn = Elasticsearch(hosts=[elastic_pair])
cls.elastic_doc = DocManager(elastic_pair,
auto_commit_interval=0)
def test_multiple_hosts(self):
elastic_doc = DocManager(elastic_nodes)
self.assertEqual(len(elastic_doc.elastic.transport.hosts), 2)
def test_aws_raises_invalid_configuration(self):
with self.assertRaises(errors.InvalidConfiguration):
DocManager('notimportant', aws={})
def setUp(self):
# Create target index in elasticsearch
self.elastic_conn.indices.create(index='test', ignore=400)
self.elastic_conn.cluster.health(wait_for_status='yellow',
index='test')
self.elastic_doc = DocManager(elastic_pair, auto_commit_interval=0)
def setUp(self):
# Create target index in elasticsearch
self.elastic_conn.indices.create(index="test", ignore=400)
self.elastic_conn.cluster.health(wait_for_status="yellow",
index="test")
self.elastic_doc = DocManager(elastic_pair, auto_commit_interval=0)
