async def test_stream_processor_pull_unexpected_error(run_engine, _,
logger_class,
redis_stream,
redis_cache):
logs.setup_logging()
logger = logger_class.return_value
run_engine.side_effect = Exception
await worker.push(
redis_stream,
123,
"owner",
"repo",
123,
"pull_request",
{"payload": "whatever"},
)
p = worker.StreamProcessor(redis_stream, redis_cache)
await p.consume("stream~owner~123")
await p.consume("stream~owner~123")
# Exception have been logged, redis must be clean
assert len(run_engine.mock_calls) == 2
assert len(logger.error.mock_calls) == 2
assert logger.error.mock_calls[0].args == (
"failed to process pull request", )
assert logger.error.mock_calls[1].args == (
"failed to process pull request", )
assert 1 == (await redis_stream.zcard("streams"))
assert 1 == len(await redis_stream.keys("stream~*"))
assert 0 == len(await redis_stream.hgetall("attempts"))
async def test_consume_good_stream(run_engine, get_install_by_id, redis,
logger_checker):
get_install_by_id.side_effect = fake_install_id
await worker.push(
redis,
12345,
"owner",
"repo",
123,
"pull_request",
{"payload": "whatever"},
)
await worker.push(
redis,
12345,
"owner",
"repo",
123,
"comment",
{"payload": "foobar"},
)
assert 1 == (await redis.zcard("streams"))
assert 1 == len(await redis.keys("stream~*"))
assert 2 == await redis.xlen("stream~12345")
assert 0 == len(await redis.hgetall("attempts"))
p = worker.StreamProcessor(redis)
await p.consume("stream~12345")
assert len(run_engine.mock_calls) == 1
assert run_engine.mock_calls[0] == mock.call(
fake_install_id(12345),
"owner",
"repo",
123,
[
{
"event_type": "pull_request",
"data": {
"payload": "whatever"
}
},
{
"event_type": "comment",
"data": {
"payload": "foobar"
}
},
],
)
# Check redis is empty
assert 0 == (await redis.zcard("streams"))
assert 0 == len(await redis.keys("stream~*"))
assert 0 == len(await redis.hgetall("attempts"))
async def test_consume_good_stream(run_engine, _, redis_stream, redis_cache,
logger_checker):
await worker.push(
redis_stream,
123,
"owner",
"repo",
123,
"pull_request",
{"payload": "whatever"},
)
await worker.push(
redis_stream,
123,
"owner",
"repo",
123,
"comment",
{"payload": "foobar"},
)
assert 1 == (await redis_stream.zcard("streams"))
assert 1 == len(await redis_stream.keys("stream~*"))
assert 2 == await redis_stream.xlen("stream~owner~123")
assert 0 == len(await redis_stream.hgetall("attempts"))
p = worker.StreamProcessor(redis_stream, redis_cache)
await p.consume("stream~owner~123")
assert len(run_engine.mock_calls) == 1
assert run_engine.mock_calls[0] == mock.call(
InstallationMatcher(owner="owner"),
"repo",
123,
[
{
"event_type": "pull_request",
"data": {
"payload": "whatever"
},
"timestamp": mock.ANY,
},
{
"event_type": "comment",
"data": {
"payload": "foobar"
},
"timestamp": mock.ANY,
},
],
)
# Check redis is empty
assert 0 == (await redis_stream.zcard("streams"))
assert 0 == len(await redis_stream.keys("stream~*"))
assert 0 == len(await redis_stream.hgetall("attempts"))
async def test_stream_processor_retrying_after_read_error(run_engine, redis):
response = mock.Mock()
response.json.return_value = {"message": "boom"}
response.status_code = 503
run_engine.side_effect = httpx.ReadError(
"Server disconnected while attempting read",
request=mock.Mock(),
)
p = worker.StreamProcessor(redis)
with pytest.raises(worker.StreamRetry):
await p._run_engine_and_translate_exception_to_retries(
"stream-owner", "owner", "repo", 1234, [])
async def test_stream_processor_retrying_after_read_error(
run_engine, _, redis_stream, redis_cache):
response = mock.Mock()
response.json.return_value = {"message": "boom"}
response.status_code = 503
run_engine.side_effect = httpx.ReadError(
"Server disconnected while attempting read",
request=mock.Mock(),
)
p = worker.StreamProcessor(redis_stream, redis_cache)
installation = context.Installation(123, "owner", {}, None, None)
with pytest.raises(worker.StreamRetry):
async with p._translate_exception_to_retries(installation.stream_name):
await worker.run_engine(installation, "repo", 1234, [])
async def test_stream_processor_date_scheduling(run_engine, _, redis_stream,
redis_cache, logger_checker):
# Don't process it before 2040
with freeze_time("2040-01-01"):
await worker.push(
redis_stream,
123,
"owner1",
"repo",
123,
"pull_request",
{"payload": "whatever"},
)
unwanted_owner_id = "owner1"
with freeze_time("2020-01-01"):
await worker.push(
redis_stream,
123,
"owner2",
"repo",
321,
"pull_request",
{"payload": "foobar"},
)
wanted_owner_id = "owner2"
assert 2 == (await redis_stream.zcard("streams"))
assert 2 == len(await redis_stream.keys("stream~*"))
assert 0 == len(await redis_stream.hgetall("attempts"))
s = worker.StreamSelector(redis_stream, 0, 1)
p = worker.StreamProcessor(redis_stream, redis_cache)
received = []
def fake_engine(installation, repo, pull_number, sources):
received.append(installation.owner_login)
run_engine.side_effect = fake_engine
with freeze_time("2020-01-14"):
stream_name = await s.next_stream()
assert stream_name is not None
await p.consume(stream_name)
assert 1 == (await redis_stream.zcard("streams"))
assert 1 == len(await redis_stream.keys("stream~*"))
assert 0 == len(await redis_stream.hgetall("attempts"))
assert received == [wanted_owner_id]
with freeze_time("2030-01-14"):
stream_name = await s.next_stream()
assert stream_name is None
assert 1 == (await redis_stream.zcard("streams"))
assert 1 == len(await redis_stream.keys("stream~*"))
assert 0 == len(await redis_stream.hgetall("attempts"))
assert received == [wanted_owner_id]
# We are in 2041, we have something todo :)
with freeze_time("2041-01-14"):
stream_name = await s.next_stream()
assert stream_name is not None
await p.consume(stream_name)
assert 0 == (await redis_stream.zcard("streams"))
assert 0 == len(await redis_stream.keys("stream~*"))
assert 0 == len(await redis_stream.hgetall("attempts"))
assert received == [wanted_owner_id, unwanted_owner_id]
async def test_stream_processor_retrying_stream_failure(
run_engine, _, logger, redis_stream, redis_cache):
logs.setup_logging()
response = mock.Mock()
response.json.return_value = {"message": "boom"}
response.status_code = 401
run_engine.side_effect = http.HTTPClientSideError(message="foobar",
request=response.request,
response=response)
await worker.push(
redis_stream,
123,
"owner",
"repo",
123,
"pull_request",
{"payload": "whatever"},
)
await worker.push(
redis_stream,
123,
"owner",
"repo",
123,
"comment",
{"payload": "foobar"},
)
assert 1 == (await redis_stream.zcard("streams"))
assert 1 == len(await redis_stream.keys("stream~*"))
assert 2 == await redis_stream.xlen("stream~owner~123")
assert 0 == len(await redis_stream.hgetall("attempts"))
p = worker.StreamProcessor(redis_stream, redis_cache)
await p.consume("stream~owner~123")
assert len(run_engine.mock_calls) == 1
assert run_engine.mock_calls[0] == mock.call(
InstallationMatcher(owner="owner"),
"repo",
123,
[
{
"event_type": "pull_request",
"data": {
"payload": "whatever"
},
"timestamp": mock.ANY,
},
{
"event_type": "comment",
"data": {
"payload": "foobar"
},
"timestamp": mock.ANY,
},
],
)
# Check stream still there and attempts recorded
assert 1 == (await redis_stream.zcard("streams"))
assert 1 == len(await redis_stream.keys("stream~*"))
assert 1 == len(await redis_stream.hgetall("attempts"))
assert {
b"stream~owner~123": b"1"
} == await redis_stream.hgetall("attempts")
await p.consume("stream~owner~123")
assert len(run_engine.mock_calls) == 2
assert {
b"stream~owner~123": b"2"
} == await redis_stream.hgetall("attempts")
await p.consume("stream~owner~123")
assert len(run_engine.mock_calls) == 3
# Still there
assert 3 == len(logger.info.mock_calls)
assert 0 == len(logger.error.mock_calls)
assert logger.info.mock_calls[0].args == (
"failed to process stream, retrying", )
assert logger.info.mock_calls[1].args == (
"failed to process stream, retrying", )
assert logger.info.mock_calls[2].args == (
"failed to process stream, retrying", )
assert 1 == (await redis_stream.zcard("streams"))
assert 1 == len(await redis_stream.keys("stream~*"))
assert 1 == len(await redis_stream.hgetall("attempts"))
async def test_stream_processor_retrying_pull(run_engine, _, logger_class,
redis_stream, redis_cache):
logs.setup_logging()
logger = logger_class.return_value
# One retries once, the other reaches max_retry
run_engine.side_effect = [
exceptions.MergeableStateUnknown(mock.Mock()),
exceptions.MergeableStateUnknown(mock.Mock()),
mock.Mock(),
exceptions.MergeableStateUnknown(mock.Mock()),
exceptions.MergeableStateUnknown(mock.Mock()),
]
await worker.push(
redis_stream,
123,
"owner",
"repo",
123,
"pull_request",
{"payload": "whatever"},
)
await worker.push(
redis_stream,
123,
"owner",
"repo",
42,
"comment",
{"payload": "foobar"},
)
assert 1 == (await redis_stream.zcard("streams"))
assert 1 == len(await redis_stream.keys("stream~*"))
assert 2 == await redis_stream.xlen("stream~owner~123")
assert 0 == len(await redis_stream.hgetall("attempts"))
p = worker.StreamProcessor(redis_stream, redis_cache)
await p.consume("stream~owner~123")
assert len(run_engine.mock_calls) == 2
assert run_engine.mock_calls == [
mock.call(
InstallationMatcher(owner="owner"),
"repo",
123,
[
{
"event_type": "pull_request",
"data": {
"payload": "whatever"
},
"timestamp": mock.ANY,
},
],
),
mock.call(
InstallationMatcher(owner="owner"),
"repo",
42,
[
{
"event_type": "comment",
"data": {
"payload": "foobar"
},
"timestamp": mock.ANY,
},
],
),
]
# Check stream still there and attempts recorded
assert 1 == (await redis_stream.zcard("streams"))
assert 1 == len(await redis_stream.keys("stream~*"))
assert {
b"pull~owner~repo~42": b"1",
b"pull~owner~repo~123": b"1",
} == await redis_stream.hgetall("attempts")
await p.consume("stream~owner~123")
assert 1 == (await redis_stream.zcard("streams"))
assert 1 == len(await redis_stream.keys("stream~*"))
assert 1 == len(await redis_stream.hgetall("attempts"))
assert len(run_engine.mock_calls) == 4
assert {
b"pull~owner~repo~42": b"2"
} == await redis_stream.hgetall("attempts")
await p.consume("stream~owner~123")
assert len(run_engine.mock_calls) == 5
# Too many retries, everything is gone
assert 3 == len(logger.info.mock_calls)
assert 1 == len(logger.error.mock_calls)
assert logger.info.mock_calls[0].args == (
"failed to process pull request, retrying", )
assert logger.info.mock_calls[1].args == (
"failed to process pull request, retrying", )
assert logger.error.mock_calls[0].args == (
"failed to process pull request, abandoning", )
assert 0 == (await redis_stream.zcard("streams"))
assert 0 == len(await redis_stream.keys("stream~*"))
assert 0 == len(await redis_stream.hgetall("attempts"))
async def test_consume_unexisting_stream(run_engine, _, redis_stream,
redis_cache, logger_checker):
p = worker.StreamProcessor(redis_stream, redis_cache)
await p.consume("stream~notexists~2")
assert len(run_engine.mock_calls) == 0
async def test_stream_processor_date_scheduling(run_engine, get_install_by_id,
redis, logger_checker):
get_install_by_id.side_effect = fake_install_id
# Don't process it before 2040
with freeze_time("2040-01-01"):
await worker.push(
redis,
12345,
"owner",
"repo",
123,
"pull_request",
{"payload": "whatever"},
)
unwanted_installation_id = fake_install_id(12345)
with freeze_time("2020-01-01"):
await worker.push(
redis,
54321,
"owner",
"repo",
321,
"pull_request",
{"payload": "foobar"},
)
wanted_installation_id = fake_install_id(54321)
assert 2 == (await redis.zcard("streams"))
assert 2 == len(await redis.keys("stream~*"))
assert 0 == len(await redis.hgetall("attempts"))
s = worker.StreamSelector(1, redis)
p = worker.StreamProcessor(redis)
received = []
def fake_engine(installation_id, owner, repo, pull_number, sources):
received.append(installation_id)
run_engine.side_effect = fake_engine
with freeze_time("2020-01-14"):
async with s.next_stream() as stream_name:
assert stream_name is not None
await p.consume(stream_name)
assert 1 == (await redis.zcard("streams"))
assert 1 == len(await redis.keys("stream~*"))
assert 0 == len(await redis.hgetall("attempts"))
assert received == [wanted_installation_id]
with freeze_time("2030-01-14"):
async with s.next_stream() as stream_name:
assert stream_name is None
assert 1 == (await redis.zcard("streams"))
assert 1 == len(await redis.keys("stream~*"))
assert 0 == len(await redis.hgetall("attempts"))
assert received == [wanted_installation_id]
# We are in 2041, we have something todo :)
with freeze_time("2041-01-14"):
async with s.next_stream() as stream_name:
assert stream_name is not None
await p.consume(stream_name)
assert 0 == (await redis.zcard("streams"))
assert 0 == len(await redis.keys("stream~*"))
assert 0 == len(await redis.hgetall("attempts"))
assert received == [wanted_installation_id, unwanted_installation_id]
async def test_stream_processor_retrying_stream_failure(
run_engine, get_install_by_id, logger, redis):
get_install_by_id.side_effect = fake_install_id
logs.setup_logging(worker="streams")
response = mock.Mock()
response.json.return_value = {"message": "boom"}
response.status_code = 401
run_engine.side_effect = httpx.HTTPClientSideError(response=response)
await worker.push(
redis,
12345,
"owner",
"repo",
123,
"pull_request",
{"payload": "whatever"},
)
await worker.push(
redis,
12345,
"owner",
"repo",
123,
"comment",
{"payload": "foobar"},
)
assert 1 == (await redis.zcard("streams"))
assert 1 == len(await redis.keys("stream~*"))
assert 2 == await redis.xlen("stream~12345")
assert 0 == len(await redis.hgetall("attempts"))
p = worker.StreamProcessor(redis)
await p.consume("stream~12345")
assert len(run_engine.mock_calls) == 1
assert run_engine.mock_calls[0] == mock.call(
fake_install_id(12345),
"owner",
"repo",
123,
[
{
"event_type": "pull_request",
"data": {
"payload": "whatever"
}
},
{
"event_type": "comment",
"data": {
"payload": "foobar"
}
},
],
)
# Check stream still there and attempts recorded
assert 1 == (await redis.zcard("streams"))
assert 1 == len(await redis.keys("stream~*"))
assert 1 == len(await redis.hgetall("attempts"))
assert {b"stream~12345": b"1"} == await redis.hgetall("attempts")
await p.consume("stream~12345")
assert len(run_engine.mock_calls) == 2
assert {b"stream~12345": b"2"} == await redis.hgetall("attempts")
await p.consume("stream~12345")
assert len(run_engine.mock_calls) == 3
# Still there
assert 3 == len(logger.info.mock_calls)
assert 0 == len(logger.error.mock_calls)
assert logger.info.mock_calls[0].args == (
"failed to process stream, retrying", )
assert logger.info.mock_calls[1].args == (
"failed to process stream, retrying", )
assert logger.info.mock_calls[2].args == (
"failed to process stream, retrying", )
assert 1 == (await redis.zcard("streams"))
assert 1 == len(await redis.keys("stream~*"))
assert 1 == len(await redis.hgetall("attempts"))
async def test_consume_unexisting_stream(run_engine, get_install_by_id, redis,
logger_checker):
get_install_by_id.side_effect = fake_install_id
p = worker.StreamProcessor(redis)
await p.consume("stream~666")
assert len(run_engine.mock_calls) == 0