def test_successfully_indexes_and_adds_new(self, db_request, monkeypatch): docs = pretend.stub() def project_docs(db): return docs monkeypatch.setattr(warehouse.search.tasks, "_project_docs", project_docs) task = pretend.stub() es_client = FakeESClient() db_request.registry.update( {"elasticsearch.index": "warehouse", "elasticsearch.shards": 42} ) db_request.registry.settings = { "elasticsearch.url": "http://some.url", "celery.scheduler_url": "redis://redis:6379/0", } monkeypatch.setattr( warehouse.search.tasks.elasticsearch, "Elasticsearch", lambda *a, **kw: es_client, ) monkeypatch.setattr( redis.StrictRedis, "from_url", lambda *a, **kw: pretend.stub(lock=NotLock) ) parallel_bulk = pretend.call_recorder(lambda client, iterable, index: [None]) monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk", parallel_bulk) monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n) reindex(task, db_request) assert parallel_bulk.calls == [ pretend.call(es_client, docs, index="warehouse-cbcbcbcbcb") ] assert es_client.indices.create.calls == [ pretend.call( body={ "settings": { "number_of_shards": 42, "number_of_replicas": 0, "refresh_interval": "-1", } }, wait_for_active_shards=42, index="warehouse-cbcbcbcbcb", ) ] assert es_client.indices.delete.calls == [] assert es_client.indices.aliases == {"warehouse": ["warehouse-cbcbcbcbcb"]} assert es_client.indices.put_settings.calls == [ pretend.call( index="warehouse-cbcbcbcbcb", body={"index": {"number_of_replicas": 0, "refresh_interval": "1s"}}, ) ]
def test_successfully_indexes_and_replaces(self, db_request, monkeypatch): docs = pretend.stub() def project_docs(db): return docs monkeypatch.setattr(warehouse.search.tasks, "_project_docs", project_docs) es_client = FakeESClient() es_client.indices.indices["warehouse-aaaaaaaaaa"] = None es_client.indices.aliases["warehouse"] = ["warehouse-aaaaaaaaaa"] db_engine = pretend.stub() db_request.registry.update( { "elasticsearch.index": "warehouse", "elasticsearch.shards": 42, "sqlalchemy.engine": db_engine, } ) db_request.registry.settings = {"elasticsearch.url": "http://some.url"} monkeypatch.setattr( warehouse.search.tasks.elasticsearch, "Elasticsearch", lambda *a, **kw: es_client, ) parallel_bulk = pretend.call_recorder(lambda client, iterable, index: [None]) monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk", parallel_bulk) monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n) reindex(db_request) assert parallel_bulk.calls == [ pretend.call(es_client, docs, index="warehouse-cbcbcbcbcb") ] assert es_client.indices.create.calls == [ pretend.call( body={ "settings": { "number_of_shards": 42, "number_of_replicas": 0, "refresh_interval": "-1", } }, wait_for_active_shards=42, index="warehouse-cbcbcbcbcb", ) ] assert es_client.indices.delete.calls == [pretend.call("warehouse-aaaaaaaaaa")] assert es_client.indices.aliases == {"warehouse": ["warehouse-cbcbcbcbcb"]} assert es_client.indices.put_settings.calls == [ pretend.call( index="warehouse-cbcbcbcbcb", body={"index": {"number_of_replicas": 0, "refresh_interval": "1s"}}, ) ]
def test_fails_when_raising(self, db_request, monkeypatch): docs = pretend.stub() def project_docs(db): return docs monkeypatch.setattr( warehouse.search.tasks, "_project_docs", project_docs, ) es_client = FakeESClient() db_request.registry.update( { "elasticsearch.index": "warehouse", }, ) db_request.registry.settings = { "elasticsearch.url": "http://some.url", } monkeypatch.setattr( warehouse.search.tasks.elasticsearch, "Elasticsearch", lambda *a, **kw: es_client ) class TestException(Exception): pass def parallel_bulk(client, iterable, index=None): assert client is es_client assert iterable is docs assert index == "warehouse-cbcbcbcbcb" raise TestException monkeypatch.setattr( warehouse.search.tasks, "parallel_bulk", parallel_bulk) monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n) with pytest.raises(TestException): reindex(db_request) assert es_client.indices.delete.calls == [ pretend.call(index='warehouse-cbcbcbcbcb'), ] assert es_client.indices.put_settings.calls == []
def test_fails_when_raising(self, db_request, monkeypatch): docs = pretend.stub() def project_docs(db): return docs monkeypatch.setattr(warehouse.search.tasks, "_project_docs", project_docs) task = pretend.stub() es_client = FakeESClient() db_request.registry.update({"elasticsearch.index": "warehouse"}) db_request.registry.settings = { "elasticsearch.url": "http://some.url", "celery.scheduler_url": "redis://redis:6379/0", } monkeypatch.setattr( warehouse.search.tasks.elasticsearch, "Elasticsearch", lambda *a, **kw: es_client, ) class TestError(Exception): pass def parallel_bulk(client, iterable, index=None): assert client is es_client assert iterable is docs assert index == "warehouse-cbcbcbcbcb" raise TestError monkeypatch.setattr(redis.StrictRedis, "from_url", lambda *a, **kw: pretend.stub(lock=NotLock)) monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk", parallel_bulk) monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n) with pytest.raises(TestError): reindex(task, db_request) assert es_client.indices.delete.calls == [ pretend.call(index="warehouse-cbcbcbcbcb") ] assert es_client.indices.put_settings.calls == []
def test_retry_on_lock(self, db_request, monkeypatch): task = pretend.stub( retry=pretend.call_recorder(pretend.raiser(celery.exceptions.Retry)) ) db_request.registry.settings = {"celery.scheduler_url": "redis://redis:6379/0"} le = redis.exceptions.LockError() monkeypatch.setattr( redis.StrictRedis, "from_url", lambda *a, **kw: pretend.stub(lock=pretend.raiser(le)), ) with pytest.raises(celery.exceptions.Retry): reindex(task, db_request) assert task.retry.calls == [pretend.call(countdown=60, exc=le)]
def test_client_aws(self, db_request, monkeypatch): docs = pretend.stub() def project_docs(db): return docs monkeypatch.setattr(warehouse.search.tasks, "_project_docs", project_docs) task = pretend.stub() aws4auth_stub = pretend.stub() aws4auth = pretend.call_recorder(lambda *a, **kw: aws4auth_stub) es_client = FakeESClient() es_client_init = pretend.call_recorder(lambda *a, **kw: es_client) db_request.registry.update({ "elasticsearch.index": "warehouse", "elasticsearch.shards": 42 }) db_request.registry.settings = { "aws.key_id": "AAAAAAAAAAAAAAAAAA", "aws.secret_key": "deadbeefdeadbeefdeadbeef", "elasticsearch.url": "https://some.url?aws_auth=1®ion=us-east-2", "celery.scheduler_url": "redis://redis:6379/0", } monkeypatch.setattr( warehouse.search.tasks.requests_aws4auth, "AWS4Auth", aws4auth, ) monkeypatch.setattr(warehouse.search.tasks.elasticsearch, "Elasticsearch", es_client_init) monkeypatch.setattr(redis.StrictRedis, "from_url", lambda *a, **kw: pretend.stub(lock=NotLock)) parallel_bulk = pretend.call_recorder( lambda client, iterable, index: [None]) monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk", parallel_bulk) monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n) reindex(task, db_request) assert len(es_client_init.calls) == 1 assert es_client_init.calls[0].kwargs["hosts"] == ["https://some.url"] assert es_client_init.calls[0].kwargs["timeout"] == 30 assert es_client_init.calls[0].kwargs["retry_on_timeout"] is True assert (es_client_init.calls[0].kwargs["connection_class"] == elasticsearch.connection.http_requests.RequestsHttpConnection) assert es_client_init.calls[0].kwargs["http_auth"] == aws4auth_stub assert aws4auth.calls == [ pretend.call("AAAAAAAAAAAAAAAAAA", "deadbeefdeadbeefdeadbeef", "us-east-2", "es") ] assert parallel_bulk.calls == [ pretend.call(es_client, docs, index="warehouse-cbcbcbcbcb") ] assert es_client.indices.create.calls == [ pretend.call( body={ "settings": { "number_of_shards": 42, "number_of_replicas": 0, "refresh_interval": "-1", } }, wait_for_active_shards=42, index="warehouse-cbcbcbcbcb", ) ] assert es_client.indices.delete.calls == [] assert es_client.indices.aliases == { "warehouse": ["warehouse-cbcbcbcbcb"] } assert es_client.indices.put_settings.calls == [ pretend.call( index="warehouse-cbcbcbcbcb", body={ "index": { "number_of_replicas": 0, "refresh_interval": "1s" } }, ) ]
def test_successfully_indexes_and_replaces(self, db_request, monkeypatch): docs = pretend.stub() def project_docs(db): return docs monkeypatch.setattr( warehouse.search.tasks, "_project_docs", project_docs, ) es_client = FakeESClient() es_client.indices.indices["warehouse-aaaaaaaaaa"] = None es_client.indices.aliases["warehouse"] = ["warehouse-aaaaaaaaaa"] db_engine = pretend.stub() db_request.registry.update( { "elasticsearch.client": es_client, "elasticsearch.index": "warehouse", "elasticsearch.shards": 42, "sqlalchemy.engine": db_engine, }, ) parallel_bulk = pretend.call_recorder(lambda client, iterable: [None]) monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk", parallel_bulk) monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n) reindex(db_request) assert parallel_bulk.calls == [pretend.call(es_client, docs)] assert es_client.indices.create.calls == [ pretend.call( body={ 'settings': { 'number_of_shards': 42, 'number_of_replicas': 0, 'refresh_interval': '-1', } }, wait_for_active_shards=42, index='warehouse-cbcbcbcbcb', ) ] assert es_client.indices.delete.calls == [ pretend.call('warehouse-aaaaaaaaaa'), ] assert es_client.indices.aliases == { "warehouse": ["warehouse-cbcbcbcbcb"], } assert es_client.indices.put_settings.calls == [ pretend.call( index='warehouse-cbcbcbcbcb', body={ 'index': { 'number_of_replicas': 0, 'refresh_interval': '1s', }, }, ) ] assert es_client.indices.forcemerge.calls == [ pretend.call(index='warehouse-cbcbcbcbcb') ]
def test_successfully_indexes_and_adds_new(self, db_request, monkeypatch): docs = pretend.stub() def project_docs(db): return docs monkeypatch.setattr( warehouse.search.tasks, "_project_docs", project_docs, ) es_client = FakeESClient() db_request.registry.update({ "elasticsearch.index": "warehouse", "elasticsearch.shards": 42, }) db_request.registry.settings = { "elasticsearch.url": "http://some.url", } monkeypatch.setattr(warehouse.search.tasks.elasticsearch, "Elasticsearch", lambda *a, **kw: es_client) parallel_bulk = pretend.call_recorder(lambda client, iterable: [None]) monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk", parallel_bulk) monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n) reindex(db_request) assert parallel_bulk.calls == [pretend.call(es_client, docs)] assert es_client.indices.create.calls == [ pretend.call( body={ 'settings': { 'number_of_shards': 42, 'number_of_replicas': 0, 'refresh_interval': '-1', } }, wait_for_active_shards=42, index='warehouse-cbcbcbcbcb', ) ] assert es_client.indices.delete.calls == [] assert es_client.indices.aliases == { "warehouse": ["warehouse-cbcbcbcbcb"], } assert es_client.indices.put_settings.calls == [ pretend.call( index='warehouse-cbcbcbcbcb', body={ 'index': { 'number_of_replicas': 0, 'refresh_interval': '1s', }, }, ) ]
def test_successfully_indexes_and_replaces(self, db_request, monkeypatch): docs = pretend.stub() def project_docs(db): return docs monkeypatch.setattr( warehouse.search.tasks, "_project_docs", project_docs, ) es_client = FakeESClient() es_client.indices.indices["warehouse-aaaaaaaaaa"] = None es_client.indices.aliases["warehouse"] = ["warehouse-aaaaaaaaaa"] db_engine = pretend.stub() db_request.registry.update( { "elasticsearch.index": "warehouse", "elasticsearch.shards": 42, "sqlalchemy.engine": db_engine, }, ) db_request.registry.settings = { "elasticsearch.url": "http://some.url", } monkeypatch.setattr( warehouse.search.tasks.elasticsearch, "Elasticsearch", lambda *a, **kw: es_client ) parallel_bulk = pretend.call_recorder( lambda client, iterable, index: [None] ) monkeypatch.setattr( warehouse.search.tasks, "parallel_bulk", parallel_bulk) monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n) reindex(db_request) assert parallel_bulk.calls == [ pretend.call(es_client, docs, index="warehouse-cbcbcbcbcb") ] assert es_client.indices.create.calls == [ pretend.call( body={ 'settings': { 'number_of_shards': 42, 'number_of_replicas': 0, 'refresh_interval': '-1', } }, wait_for_active_shards=42, index='warehouse-cbcbcbcbcb', ) ] assert es_client.indices.delete.calls == [ pretend.call('warehouse-aaaaaaaaaa'), ] assert es_client.indices.aliases == { "warehouse": ["warehouse-cbcbcbcbcb"], } assert es_client.indices.put_settings.calls == [ pretend.call( index='warehouse-cbcbcbcbcb', body={ 'index': { 'number_of_replicas': 0, 'refresh_interval': '1s', }, }, ) ]