def test_elastic_commit(self): """Test the auto_commit_interval attribute.""" docc = {'_id': '3', 'name': 'Waldo'} # Disable 1s refresh in elasticsearch # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html disable_refresh_body = {"index" : {"refresh_interval" : "-1"}} self.elastic_conn.indices.put_settings(index="test",body=disable_refresh_body) # test cases: # -1 = no autocommit # 0 = commit immediately # x > 0 = commit within x seconds for autocommit_interval in [None, 0, 1, 2]: docman = DocManager(elastic_pair,auto_commit_interval=autocommit_interval) docman.upsert(docc, *TESTARGS) if autocommit_interval is None: docman.commit() else: # Allow just a little extra time time.sleep(autocommit_interval + 1) results = list(self._search()) self.assertEqual(len(results), 1, "should commit document with " "auto_commit_interval = %s" % str( autocommit_interval)) self.assertEqual(results[0]["name"], "Waldo") docman.stop() self._remove() retry_until_ok(self.elastic_conn.indices.refresh, index="") enable_refresh_body = {"index" : {"refresh_interval" : "1s"}} self.elastic_conn.indices.put_settings(index="test",body=enable_refresh_body)
def test_auto_send_interval(self): """Test the auto_send_interval auto_send_interval should control the amount of time to wait before sending (but not committing) buffered operations. """ doc = {"_id": "3", "name": "Waldo"} # test cases: # None, 0 = no auto send # x > 0 = send buffered operations within x seconds for send_interval in [None, 0, 3, 8]: docman = DocManager( elastic_pair, autoSendInterval=send_interval, auto_commit_interval=None ) docman.upsert(doc, *TESTARGS) if send_interval: # Allow just a little extra time time.sleep(send_interval + 2) results = list(self._search()) self.assertEqual( len(results), 0, "documents should not be commited with " "auto_commit_interval=None and auto_commit_interval=%s" % (send_interval,), ) # Commit the possibly sent changes and search again retry_until_ok(self.elastic_conn.indices.refresh, index="") results = list(self._search()) if not send_interval: self.assertEqual( len(results), 0, "should not send document with auto_send_interval=%s" % (send_interval,), ) else: self.assertEqual( len(results), 1, "should send document with auto_send_interval=%s" % (send_interval,), ) self.assertEqual(results[0]["name"], "Waldo") docman.stop() self._remove() retry_until_ok(self.elastic_conn.indices.refresh, index="")
class ElasticsearchTestCase(unittest.TestCase): """Base class for all ES TestCases.""" @classmethod def setUpClass(cls): cls.elastic_conn = Elasticsearch(hosts=[elastic_pair]) def setUp(self): # Create target index in elasticsearch self.elastic_conn.indices.create(index='test', ignore=400) self.elastic_conn.cluster.health(wait_for_status='yellow', index='test') self.elastic_doc = DocManager(elastic_pair, auto_commit_interval=0) def tearDown(self): self.elastic_conn.indices.delete(index='test', ignore=404) self.elastic_doc.stop() def _search(self, query=None): query = query or {"match_all": {}} return self.elastic_doc._stream_search( index="test", doc_type='test', body={"query": query} ) def _count(self): return self.elastic_conn.count(index='test')['count'] def _remove(self): bulk_deletes = [] for result in scan(self.elastic_conn, index="test", doc_type="test"): result['_op_type'] = 'delete' bulk_deletes.append(result) bulk(self.elastic_conn, bulk_deletes) def _mappings(self, index='_all'): mappings = self.elastic_conn.indices.get_mapping(index=index) if index in mappings: return list(mappings[index]['mappings'].keys()) return [] def _indices(self): return list(self.elastic_conn.indices.stats()['indices'].keys())
class ElasticsearchTestCase(unittest.TestCase): """Base class for all ES TestCases.""" @classmethod def setUpClass(cls): cls.elastic_conn = Elasticsearch(hosts=[elastic_pair]) def setUp(self): # Create target index in elasticsearch self.elastic_conn.indices.create(index="test", ignore=400) self.elastic_conn.cluster.health(wait_for_status="yellow", index="test") self.elastic_doc = DocManager(elastic_pair, auto_commit_interval=0) def tearDown(self): self.elastic_conn.indices.delete(index="test", ignore=404) self.elastic_doc.stop() def _search(self, query=None): query = query or {"match_all": {}} return self.elastic_doc._stream_search(index="test", doc_type="test", body={"query": query}) def _count(self): return self.elastic_conn.count(index="test")["count"] def _remove(self): bulk_deletes = [] for result in scan(self.elastic_conn, index="test", doc_type="test"): result["_op_type"] = "delete" bulk_deletes.append(result) bulk(self.elastic_conn, bulk_deletes) def _mappings(self, index="_all"): mappings = self.elastic_conn.indices.get_mapping(index=index) if index in mappings: return list(mappings[index]["mappings"].keys()) return [] def _indices(self): return list(self.elastic_conn.indices.stats()["indices"].keys())
def test_elastic_commit(self): """Test the auto_commit_interval attribute.""" doc = {'_id': '3', 'name': 'Waldo'} # test cases: # None = no autocommit # 0 = commit immediately # x > 0 = commit within x seconds for commit_interval in [None, 0, 2, 8]: docman = DocManager(elastic_pair, auto_commit_interval=commit_interval) docman.upsert(doc, *TESTARGS) if commit_interval: # Allow just a little extra time time.sleep(commit_interval + 2) results = list(self._search()) if commit_interval is None: self.assertEqual(len(results), 0, "should not commit document with " "auto_commit_interval = None") else: self.assertEqual(len(results), 1, "should commit document with " "auto_commit_interval = %s" % ( commit_interval,)) self.assertEqual(results[0]["name"], "Waldo") docman.stop() self._remove() retry_until_ok(self.elastic_conn.indices.refresh, index="")
def test_elastic_commit(self): """Test the auto_commit_interval attribute.""" docc = {'_id': '3', 'name': 'Waldo'} docman = DocManager(elastic_pair) # test cases: # -1 = no autocommit # 0 = commit immediately # x > 0 = commit within x seconds for autocommit_interval in [None, 0, 1, 2]: docman.auto_commit_interval = autocommit_interval docman.upsert(docc, *TESTARGS) if autocommit_interval is None: docman.commit() else: # Allow just a little extra time time.sleep(autocommit_interval + 1) results = list(self._search()) self.assertEqual( len(results), 1, "should commit document with " "auto_commit_interval = %s" % str(autocommit_interval)) self.assertEqual(results[0]["name"], "Waldo") self._remove() docman.stop()
def setUp(self): """Start a new Connector for each test.""" super(TestElastic, self).setUp() try: os.unlink("oplog.timestamp") except OSError: pass docman = DocManager(elastic_pair) self.connector = Connector(mongo_address=self.repl_set.uri, ns_set=['test.test'], doc_managers=(docman, ), gridfs_set=['test.test']) self.conn.test.test.drop() self.conn.test.test.files.drop() self.conn.test.test.chunks.drop() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: self._count() == 0)
def test_auto_send_interval(self): """Test the auto_send_interval auto_send_interval should control the amount of time to wait before sending (but not committing) buffered operations. """ doc = {"_id": "3", "name": "Waldo"} # test cases: # None, 0 = no auto send # x > 0 = send buffered operations within x seconds for send_interval in [None, 0, 3, 8]: docman = DocManager(elastic_pair, autoSendInterval=send_interval, auto_commit_interval=None) docman.upsert(doc, *TESTARGS) if send_interval: # Allow just a little extra time time.sleep(send_interval + 2) results = list(self._search()) self.assertEqual( len(results), 0, "documents should not be commited with " "auto_commit_interval=None and auto_commit_interval=%s" % (send_interval, ), ) # Commit the possibly sent changes and search again retry_until_ok(self.elastic_conn.indices.refresh, index="") results = list(self._search()) if not send_interval: self.assertEqual( len(results), 0, "should not send document with auto_send_interval=%s" % (send_interval, ), ) else: self.assertEqual( len(results), 1, "should send document with auto_send_interval=%s" % (send_interval, ), ) self.assertEqual(results[0]["name"], "Waldo") docman.stop() self._remove() retry_until_ok(self.elastic_conn.indices.refresh, index="")
def test_elastic_commit(self): """Test the auto_commit_interval attribute.""" docc = {'_id': '3', 'name': 'Waldo'} docman = DocManager(elastic_pair) # test cases: # -1 = no autocommit # 0 = commit immediately # x > 0 = commit within x seconds for autocommit_interval in [None, 0, 1, 2]: docman.auto_commit_interval = autocommit_interval docman.upsert(docc, *TESTARGS) if autocommit_interval is None: docman.commit() else: # Allow just a little extra time time.sleep(autocommit_interval + 1) results = list(self._search()) self.assertEqual(len(results), 1, "should commit document with " "auto_commit_interval = %s" % str( autocommit_interval)) self.assertEqual(results[0]["name"], "Waldo") self._remove() docman.stop()
def setUpClass(cls): cls.elastic_conn = Elasticsearch(hosts=[elastic_pair]) cls.elastic_doc = DocManager(elastic_pair, auto_commit_interval=0)
def test_multiple_hosts(self): elastic_doc = DocManager(elastic_nodes) self.assertEqual(len(elastic_doc.elastic.transport.hosts), 2)
def test_aws_raises_invalid_configuration(self): with self.assertRaises(errors.InvalidConfiguration): DocManager('notimportant', aws={})
def setUp(self): # Create target index in elasticsearch self.elastic_conn.indices.create(index='test', ignore=400) self.elastic_conn.cluster.health(wait_for_status='yellow', index='test') self.elastic_doc = DocManager(elastic_pair, auto_commit_interval=0)
def setUp(self): # Create target index in elasticsearch self.elastic_conn.indices.create(index="test", ignore=400) self.elastic_conn.cluster.health(wait_for_status="yellow", index="test") self.elastic_doc = DocManager(elastic_pair, auto_commit_interval=0)