def test_successfully_indexes_and_adds_new(self, db_request, monkeypatch):
docs = pretend.stub()
def project_docs(db):
return docs
monkeypatch.setattr(warehouse.search.tasks, "_project_docs", project_docs)
task = pretend.stub()
es_client = FakeESClient()
db_request.registry.update(
{"elasticsearch.index": "warehouse", "elasticsearch.shards": 42}
)
db_request.registry.settings = {
"elasticsearch.url": "http://some.url",
"celery.scheduler_url": "redis://redis:6379/0",
}
monkeypatch.setattr(
warehouse.search.tasks.elasticsearch,
"Elasticsearch",
lambda *a, **kw: es_client,
)
monkeypatch.setattr(
redis.StrictRedis, "from_url", lambda *a, **kw: pretend.stub(lock=NotLock)
)
parallel_bulk = pretend.call_recorder(lambda client, iterable, index: [None])
monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk", parallel_bulk)
monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n)
reindex(task, db_request)
assert parallel_bulk.calls == [
pretend.call(es_client, docs, index="warehouse-cbcbcbcbcb")
]
assert es_client.indices.create.calls == [
pretend.call(
body={
"settings": {
"number_of_shards": 42,
"number_of_replicas": 0,
"refresh_interval": "-1",
}
},
wait_for_active_shards=42,
index="warehouse-cbcbcbcbcb",
)
]
assert es_client.indices.delete.calls == []
assert es_client.indices.aliases == {"warehouse": ["warehouse-cbcbcbcbcb"]}
assert es_client.indices.put_settings.calls == [
pretend.call(
index="warehouse-cbcbcbcbcb",
body={"index": {"number_of_replicas": 0, "refresh_interval": "1s"}},
)
]
def test_successfully_indexes_and_adds_new(self, db_request, monkeypatch):
docs = pretend.stub()
def project_docs(db):
return docs
monkeypatch.setattr(warehouse.search.tasks, "_project_docs", project_docs)
task = pretend.stub()
es_client = FakeESClient()
db_request.registry.update(
{"elasticsearch.index": "warehouse", "elasticsearch.shards": 42}
)
db_request.registry.settings = {
"elasticsearch.url": "http://some.url",
"celery.scheduler_url": "redis://redis:6379/0",
}
monkeypatch.setattr(
warehouse.search.tasks.elasticsearch,
"Elasticsearch",
lambda *a, **kw: es_client,
)
monkeypatch.setattr(
redis.StrictRedis, "from_url", lambda *a, **kw: pretend.stub(lock=NotLock)
)
parallel_bulk = pretend.call_recorder(lambda client, iterable, index: [None])
monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk", parallel_bulk)
monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n)
reindex(task, db_request)
assert parallel_bulk.calls == [
pretend.call(es_client, docs, index="warehouse-cbcbcbcbcb")
]
assert es_client.indices.create.calls == [
pretend.call(
body={
"settings": {
"number_of_shards": 42,
"number_of_replicas": 0,
"refresh_interval": "-1",
}
},
wait_for_active_shards=42,
index="warehouse-cbcbcbcbcb",
)
]
assert es_client.indices.delete.calls == []
assert es_client.indices.aliases == {"warehouse": ["warehouse-cbcbcbcbcb"]}
assert es_client.indices.put_settings.calls == [
pretend.call(
index="warehouse-cbcbcbcbcb",
body={"index": {"number_of_replicas": 0, "refresh_interval": "1s"}},
)
]
def test_successfully_indexes_and_replaces(self, db_request, monkeypatch):
docs = pretend.stub()
def project_docs(db):
return docs
monkeypatch.setattr(warehouse.search.tasks, "_project_docs", project_docs)
es_client = FakeESClient()
es_client.indices.indices["warehouse-aaaaaaaaaa"] = None
es_client.indices.aliases["warehouse"] = ["warehouse-aaaaaaaaaa"]
db_engine = pretend.stub()
db_request.registry.update(
{
"elasticsearch.index": "warehouse",
"elasticsearch.shards": 42,
"sqlalchemy.engine": db_engine,
}
)
db_request.registry.settings = {"elasticsearch.url": "http://some.url"}
monkeypatch.setattr(
warehouse.search.tasks.elasticsearch,
"Elasticsearch",
lambda *a, **kw: es_client,
)
parallel_bulk = pretend.call_recorder(lambda client, iterable, index: [None])
monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk", parallel_bulk)
monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n)
reindex(db_request)
assert parallel_bulk.calls == [
pretend.call(es_client, docs, index="warehouse-cbcbcbcbcb")
]
assert es_client.indices.create.calls == [
pretend.call(
body={
"settings": {
"number_of_shards": 42,
"number_of_replicas": 0,
"refresh_interval": "-1",
}
},
wait_for_active_shards=42,
index="warehouse-cbcbcbcbcb",
)
]
assert es_client.indices.delete.calls == [pretend.call("warehouse-aaaaaaaaaa")]
assert es_client.indices.aliases == {"warehouse": ["warehouse-cbcbcbcbcb"]}
assert es_client.indices.put_settings.calls == [
pretend.call(
index="warehouse-cbcbcbcbcb",
body={"index": {"number_of_replicas": 0, "refresh_interval": "1s"}},
)
]
def test_fails_when_raising(self, db_request, monkeypatch):
docs = pretend.stub()
def project_docs(db):
return docs
monkeypatch.setattr(
warehouse.search.tasks,
"_project_docs",
project_docs,
)
es_client = FakeESClient()
db_request.registry.update(
{
"elasticsearch.index": "warehouse",
},
)
db_request.registry.settings = {
"elasticsearch.url": "http://some.url",
}
monkeypatch.setattr(
warehouse.search.tasks.elasticsearch,
"Elasticsearch",
lambda *a, **kw: es_client
)
class TestException(Exception):
pass
def parallel_bulk(client, iterable, index=None):
assert client is es_client
assert iterable is docs
assert index == "warehouse-cbcbcbcbcb"
raise TestException
monkeypatch.setattr(
warehouse.search.tasks, "parallel_bulk", parallel_bulk)
monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n)
with pytest.raises(TestException):
reindex(db_request)
assert es_client.indices.delete.calls == [
pretend.call(index='warehouse-cbcbcbcbcb'),
]
assert es_client.indices.put_settings.calls == []
def test_fails_when_raising(self, db_request, monkeypatch):
docs = pretend.stub()
def project_docs(db):
return docs
monkeypatch.setattr(warehouse.search.tasks, "_project_docs",
project_docs)
task = pretend.stub()
es_client = FakeESClient()
db_request.registry.update({"elasticsearch.index": "warehouse"})
db_request.registry.settings = {
"elasticsearch.url": "http://some.url",
"celery.scheduler_url": "redis://redis:6379/0",
}
monkeypatch.setattr(
warehouse.search.tasks.elasticsearch,
"Elasticsearch",
lambda *a, **kw: es_client,
)
class TestError(Exception):
pass
def parallel_bulk(client, iterable, index=None):
assert client is es_client
assert iterable is docs
assert index == "warehouse-cbcbcbcbcb"
raise TestError
monkeypatch.setattr(redis.StrictRedis, "from_url",
lambda *a, **kw: pretend.stub(lock=NotLock))
monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk",
parallel_bulk)
monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n)
with pytest.raises(TestError):
reindex(task, db_request)
assert es_client.indices.delete.calls == [
pretend.call(index="warehouse-cbcbcbcbcb")
]
assert es_client.indices.put_settings.calls == []
def test_retry_on_lock(self, db_request, monkeypatch):
task = pretend.stub(
retry=pretend.call_recorder(pretend.raiser(celery.exceptions.Retry))
)
db_request.registry.settings = {"celery.scheduler_url": "redis://redis:6379/0"}
le = redis.exceptions.LockError()
monkeypatch.setattr(
redis.StrictRedis,
"from_url",
lambda *a, **kw: pretend.stub(lock=pretend.raiser(le)),
)
with pytest.raises(celery.exceptions.Retry):
reindex(task, db_request)
assert task.retry.calls == [pretend.call(countdown=60, exc=le)]
def test_retry_on_lock(self, db_request, monkeypatch):
task = pretend.stub(
retry=pretend.call_recorder(pretend.raiser(celery.exceptions.Retry))
)
db_request.registry.settings = {"celery.scheduler_url": "redis://redis:6379/0"}
le = redis.exceptions.LockError()
monkeypatch.setattr(
redis.StrictRedis,
"from_url",
lambda *a, **kw: pretend.stub(lock=pretend.raiser(le)),
)
with pytest.raises(celery.exceptions.Retry):
reindex(task, db_request)
assert task.retry.calls == [pretend.call(countdown=60, exc=le)]
def test_client_aws(self, db_request, monkeypatch):
docs = pretend.stub()
def project_docs(db):
return docs
monkeypatch.setattr(warehouse.search.tasks, "_project_docs",
project_docs)
task = pretend.stub()
aws4auth_stub = pretend.stub()
aws4auth = pretend.call_recorder(lambda *a, **kw: aws4auth_stub)
es_client = FakeESClient()
es_client_init = pretend.call_recorder(lambda *a, **kw: es_client)
db_request.registry.update({
"elasticsearch.index": "warehouse",
"elasticsearch.shards": 42
})
db_request.registry.settings = {
"aws.key_id": "AAAAAAAAAAAAAAAAAA",
"aws.secret_key": "deadbeefdeadbeefdeadbeef",
"elasticsearch.url":
"https://some.url?aws_auth=1®ion=us-east-2",
"celery.scheduler_url": "redis://redis:6379/0",
}
monkeypatch.setattr(
warehouse.search.tasks.requests_aws4auth,
"AWS4Auth",
aws4auth,
)
monkeypatch.setattr(warehouse.search.tasks.elasticsearch,
"Elasticsearch", es_client_init)
monkeypatch.setattr(redis.StrictRedis, "from_url",
lambda *a, **kw: pretend.stub(lock=NotLock))
parallel_bulk = pretend.call_recorder(
lambda client, iterable, index: [None])
monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk",
parallel_bulk)
monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n)
reindex(task, db_request)
assert len(es_client_init.calls) == 1
assert es_client_init.calls[0].kwargs["hosts"] == ["https://some.url"]
assert es_client_init.calls[0].kwargs["timeout"] == 30
assert es_client_init.calls[0].kwargs["retry_on_timeout"] is True
assert (es_client_init.calls[0].kwargs["connection_class"] ==
elasticsearch.connection.http_requests.RequestsHttpConnection)
assert es_client_init.calls[0].kwargs["http_auth"] == aws4auth_stub
assert aws4auth.calls == [
pretend.call("AAAAAAAAAAAAAAAAAA", "deadbeefdeadbeefdeadbeef",
"us-east-2", "es")
]
assert parallel_bulk.calls == [
pretend.call(es_client, docs, index="warehouse-cbcbcbcbcb")
]
assert es_client.indices.create.calls == [
pretend.call(
body={
"settings": {
"number_of_shards": 42,
"number_of_replicas": 0,
"refresh_interval": "-1",
}
},
wait_for_active_shards=42,
index="warehouse-cbcbcbcbcb",
)
]
assert es_client.indices.delete.calls == []
assert es_client.indices.aliases == {
"warehouse": ["warehouse-cbcbcbcbcb"]
}
assert es_client.indices.put_settings.calls == [
pretend.call(
index="warehouse-cbcbcbcbcb",
body={
"index": {
"number_of_replicas": 0,
"refresh_interval": "1s"
}
},
)
]
def test_successfully_indexes_and_replaces(self, db_request, monkeypatch):
docs = pretend.stub()
def project_docs(db):
return docs
monkeypatch.setattr(
warehouse.search.tasks,
"_project_docs",
project_docs,
)
es_client = FakeESClient()
es_client.indices.indices["warehouse-aaaaaaaaaa"] = None
es_client.indices.aliases["warehouse"] = ["warehouse-aaaaaaaaaa"]
db_engine = pretend.stub()
db_request.registry.update(
{
"elasticsearch.client": es_client,
"elasticsearch.index": "warehouse",
"elasticsearch.shards": 42,
"sqlalchemy.engine": db_engine,
}, )
parallel_bulk = pretend.call_recorder(lambda client, iterable: [None])
monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk",
parallel_bulk)
monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n)
reindex(db_request)
assert parallel_bulk.calls == [pretend.call(es_client, docs)]
assert es_client.indices.create.calls == [
pretend.call(
body={
'settings': {
'number_of_shards': 42,
'number_of_replicas': 0,
'refresh_interval': '-1',
}
},
wait_for_active_shards=42,
index='warehouse-cbcbcbcbcb',
)
]
assert es_client.indices.delete.calls == [
pretend.call('warehouse-aaaaaaaaaa'),
]
assert es_client.indices.aliases == {
"warehouse": ["warehouse-cbcbcbcbcb"],
}
assert es_client.indices.put_settings.calls == [
pretend.call(
index='warehouse-cbcbcbcbcb',
body={
'index': {
'number_of_replicas': 0,
'refresh_interval': '1s',
},
},
)
]
assert es_client.indices.forcemerge.calls == [
pretend.call(index='warehouse-cbcbcbcbcb')
]
def test_successfully_indexes_and_adds_new(self, db_request, monkeypatch):
docs = pretend.stub()
def project_docs(db):
return docs
monkeypatch.setattr(
warehouse.search.tasks,
"_project_docs",
project_docs,
)
es_client = FakeESClient()
db_request.registry.update({
"elasticsearch.index": "warehouse",
"elasticsearch.shards": 42,
})
db_request.registry.settings = {
"elasticsearch.url": "http://some.url",
}
monkeypatch.setattr(warehouse.search.tasks.elasticsearch,
"Elasticsearch", lambda *a, **kw: es_client)
parallel_bulk = pretend.call_recorder(lambda client, iterable: [None])
monkeypatch.setattr(warehouse.search.tasks, "parallel_bulk",
parallel_bulk)
monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n)
reindex(db_request)
assert parallel_bulk.calls == [pretend.call(es_client, docs)]
assert es_client.indices.create.calls == [
pretend.call(
body={
'settings': {
'number_of_shards': 42,
'number_of_replicas': 0,
'refresh_interval': '-1',
}
},
wait_for_active_shards=42,
index='warehouse-cbcbcbcbcb',
)
]
assert es_client.indices.delete.calls == []
assert es_client.indices.aliases == {
"warehouse": ["warehouse-cbcbcbcbcb"],
}
assert es_client.indices.put_settings.calls == [
pretend.call(
index='warehouse-cbcbcbcbcb',
body={
'index': {
'number_of_replicas': 0,
'refresh_interval': '1s',
},
},
)
]
def test_successfully_indexes_and_replaces(self, db_request, monkeypatch):
docs = pretend.stub()
def project_docs(db):
return docs
monkeypatch.setattr(
warehouse.search.tasks,
"_project_docs",
project_docs,
)
es_client = FakeESClient()
es_client.indices.indices["warehouse-aaaaaaaaaa"] = None
es_client.indices.aliases["warehouse"] = ["warehouse-aaaaaaaaaa"]
db_engine = pretend.stub()
db_request.registry.update(
{
"elasticsearch.index": "warehouse",
"elasticsearch.shards": 42,
"sqlalchemy.engine": db_engine,
},
)
db_request.registry.settings = {
"elasticsearch.url": "http://some.url",
}
monkeypatch.setattr(
warehouse.search.tasks.elasticsearch,
"Elasticsearch",
lambda *a, **kw: es_client
)
parallel_bulk = pretend.call_recorder(
lambda client, iterable, index: [None]
)
monkeypatch.setattr(
warehouse.search.tasks, "parallel_bulk", parallel_bulk)
monkeypatch.setattr(os, "urandom", lambda n: b"\xcb" * n)
reindex(db_request)
assert parallel_bulk.calls == [
pretend.call(es_client, docs, index="warehouse-cbcbcbcbcb")
]
assert es_client.indices.create.calls == [
pretend.call(
body={
'settings': {
'number_of_shards': 42,
'number_of_replicas': 0,
'refresh_interval': '-1',
}
},
wait_for_active_shards=42,
index='warehouse-cbcbcbcbcb',
)
]
assert es_client.indices.delete.calls == [
pretend.call('warehouse-aaaaaaaaaa'),
]
assert es_client.indices.aliases == {
"warehouse": ["warehouse-cbcbcbcbcb"],
}
assert es_client.indices.put_settings.calls == [
pretend.call(
index='warehouse-cbcbcbcbcb',
body={
'index': {
'number_of_replicas': 0,
'refresh_interval': '1s',
},
},
)
]
