コード例 #1
0
async def test_stream_processor_pull_unexpected_error(run_engine, _,
                                                      logger_class,
                                                      redis_stream,
                                                      redis_cache):
    logs.setup_logging()
    logger = logger_class.return_value

    run_engine.side_effect = Exception

    await worker.push(
        redis_stream,
        123,
        "owner",
        "repo",
        123,
        "pull_request",
        {"payload": "whatever"},
    )

    p = worker.StreamProcessor(redis_stream, redis_cache)
    await p.consume("stream~owner~123")
    await p.consume("stream~owner~123")

    # Exception have been logged, redis must be clean
    assert len(run_engine.mock_calls) == 2
    assert len(logger.error.mock_calls) == 2
    assert logger.error.mock_calls[0].args == (
        "failed to process pull request", )
    assert logger.error.mock_calls[1].args == (
        "failed to process pull request", )
    assert 1 == (await redis_stream.zcard("streams"))
    assert 1 == len(await redis_stream.keys("stream~*"))
    assert 0 == len(await redis_stream.hgetall("attempts"))
コード例 #2
0
ファイル: test_worker.py プロジェクト: eladb/mergify-engine
async def test_consume_good_stream(run_engine, get_install_by_id, redis,
                                   logger_checker):
    get_install_by_id.side_effect = fake_install_id

    await worker.push(
        redis,
        12345,
        "owner",
        "repo",
        123,
        "pull_request",
        {"payload": "whatever"},
    )
    await worker.push(
        redis,
        12345,
        "owner",
        "repo",
        123,
        "comment",
        {"payload": "foobar"},
    )

    assert 1 == (await redis.zcard("streams"))
    assert 1 == len(await redis.keys("stream~*"))
    assert 2 == await redis.xlen("stream~12345")
    assert 0 == len(await redis.hgetall("attempts"))

    p = worker.StreamProcessor(redis)
    await p.consume("stream~12345")

    assert len(run_engine.mock_calls) == 1
    assert run_engine.mock_calls[0] == mock.call(
        fake_install_id(12345),
        "owner",
        "repo",
        123,
        [
            {
                "event_type": "pull_request",
                "data": {
                    "payload": "whatever"
                }
            },
            {
                "event_type": "comment",
                "data": {
                    "payload": "foobar"
                }
            },
        ],
    )

    # Check redis is empty
    assert 0 == (await redis.zcard("streams"))
    assert 0 == len(await redis.keys("stream~*"))
    assert 0 == len(await redis.hgetall("attempts"))
コード例 #3
0
async def test_consume_good_stream(run_engine, _, redis_stream, redis_cache,
                                   logger_checker):
    await worker.push(
        redis_stream,
        123,
        "owner",
        "repo",
        123,
        "pull_request",
        {"payload": "whatever"},
    )
    await worker.push(
        redis_stream,
        123,
        "owner",
        "repo",
        123,
        "comment",
        {"payload": "foobar"},
    )

    assert 1 == (await redis_stream.zcard("streams"))
    assert 1 == len(await redis_stream.keys("stream~*"))
    assert 2 == await redis_stream.xlen("stream~owner~123")
    assert 0 == len(await redis_stream.hgetall("attempts"))

    p = worker.StreamProcessor(redis_stream, redis_cache)
    await p.consume("stream~owner~123")

    assert len(run_engine.mock_calls) == 1
    assert run_engine.mock_calls[0] == mock.call(
        InstallationMatcher(owner="owner"),
        "repo",
        123,
        [
            {
                "event_type": "pull_request",
                "data": {
                    "payload": "whatever"
                },
                "timestamp": mock.ANY,
            },
            {
                "event_type": "comment",
                "data": {
                    "payload": "foobar"
                },
                "timestamp": mock.ANY,
            },
        ],
    )

    # Check redis is empty
    assert 0 == (await redis_stream.zcard("streams"))
    assert 0 == len(await redis_stream.keys("stream~*"))
    assert 0 == len(await redis_stream.hgetall("attempts"))
コード例 #4
0
async def test_stream_processor_retrying_after_read_error(run_engine, redis):
    response = mock.Mock()
    response.json.return_value = {"message": "boom"}
    response.status_code = 503
    run_engine.side_effect = httpx.ReadError(
        "Server disconnected while attempting read",
        request=mock.Mock(),
    )

    p = worker.StreamProcessor(redis)

    with pytest.raises(worker.StreamRetry):
        await p._run_engine_and_translate_exception_to_retries(
            "stream-owner", "owner", "repo", 1234, [])
コード例 #5
0
async def test_stream_processor_retrying_after_read_error(
        run_engine, _, redis_stream, redis_cache):
    response = mock.Mock()
    response.json.return_value = {"message": "boom"}
    response.status_code = 503
    run_engine.side_effect = httpx.ReadError(
        "Server disconnected while attempting read",
        request=mock.Mock(),
    )

    p = worker.StreamProcessor(redis_stream, redis_cache)

    installation = context.Installation(123, "owner", {}, None, None)
    with pytest.raises(worker.StreamRetry):
        async with p._translate_exception_to_retries(installation.stream_name):
            await worker.run_engine(installation, "repo", 1234, [])
コード例 #6
0
async def test_stream_processor_date_scheduling(run_engine, _, redis_stream,
                                                redis_cache, logger_checker):

    # Don't process it before 2040
    with freeze_time("2040-01-01"):
        await worker.push(
            redis_stream,
            123,
            "owner1",
            "repo",
            123,
            "pull_request",
            {"payload": "whatever"},
        )
        unwanted_owner_id = "owner1"

    with freeze_time("2020-01-01"):
        await worker.push(
            redis_stream,
            123,
            "owner2",
            "repo",
            321,
            "pull_request",
            {"payload": "foobar"},
        )
        wanted_owner_id = "owner2"

    assert 2 == (await redis_stream.zcard("streams"))
    assert 2 == len(await redis_stream.keys("stream~*"))
    assert 0 == len(await redis_stream.hgetall("attempts"))

    s = worker.StreamSelector(redis_stream, 0, 1)
    p = worker.StreamProcessor(redis_stream, redis_cache)

    received = []

    def fake_engine(installation, repo, pull_number, sources):
        received.append(installation.owner_login)

    run_engine.side_effect = fake_engine

    with freeze_time("2020-01-14"):
        stream_name = await s.next_stream()
        assert stream_name is not None
        await p.consume(stream_name)

    assert 1 == (await redis_stream.zcard("streams"))
    assert 1 == len(await redis_stream.keys("stream~*"))
    assert 0 == len(await redis_stream.hgetall("attempts"))
    assert received == [wanted_owner_id]

    with freeze_time("2030-01-14"):
        stream_name = await s.next_stream()
        assert stream_name is None

    assert 1 == (await redis_stream.zcard("streams"))
    assert 1 == len(await redis_stream.keys("stream~*"))
    assert 0 == len(await redis_stream.hgetall("attempts"))
    assert received == [wanted_owner_id]

    # We are in 2041, we have something todo :)
    with freeze_time("2041-01-14"):
        stream_name = await s.next_stream()
        assert stream_name is not None
        await p.consume(stream_name)

    assert 0 == (await redis_stream.zcard("streams"))
    assert 0 == len(await redis_stream.keys("stream~*"))
    assert 0 == len(await redis_stream.hgetall("attempts"))
    assert received == [wanted_owner_id, unwanted_owner_id]
コード例 #7
0
async def test_stream_processor_retrying_stream_failure(
        run_engine, _, logger, redis_stream, redis_cache):
    logs.setup_logging()

    response = mock.Mock()
    response.json.return_value = {"message": "boom"}
    response.status_code = 401
    run_engine.side_effect = http.HTTPClientSideError(message="foobar",
                                                      request=response.request,
                                                      response=response)

    await worker.push(
        redis_stream,
        123,
        "owner",
        "repo",
        123,
        "pull_request",
        {"payload": "whatever"},
    )
    await worker.push(
        redis_stream,
        123,
        "owner",
        "repo",
        123,
        "comment",
        {"payload": "foobar"},
    )

    assert 1 == (await redis_stream.zcard("streams"))
    assert 1 == len(await redis_stream.keys("stream~*"))
    assert 2 == await redis_stream.xlen("stream~owner~123")
    assert 0 == len(await redis_stream.hgetall("attempts"))

    p = worker.StreamProcessor(redis_stream, redis_cache)
    await p.consume("stream~owner~123")

    assert len(run_engine.mock_calls) == 1
    assert run_engine.mock_calls[0] == mock.call(
        InstallationMatcher(owner="owner"),
        "repo",
        123,
        [
            {
                "event_type": "pull_request",
                "data": {
                    "payload": "whatever"
                },
                "timestamp": mock.ANY,
            },
            {
                "event_type": "comment",
                "data": {
                    "payload": "foobar"
                },
                "timestamp": mock.ANY,
            },
        ],
    )

    # Check stream still there and attempts recorded
    assert 1 == (await redis_stream.zcard("streams"))
    assert 1 == len(await redis_stream.keys("stream~*"))
    assert 1 == len(await redis_stream.hgetall("attempts"))

    assert {
        b"stream~owner~123": b"1"
    } == await redis_stream.hgetall("attempts")

    await p.consume("stream~owner~123")
    assert len(run_engine.mock_calls) == 2
    assert {
        b"stream~owner~123": b"2"
    } == await redis_stream.hgetall("attempts")

    await p.consume("stream~owner~123")
    assert len(run_engine.mock_calls) == 3

    # Still there
    assert 3 == len(logger.info.mock_calls)
    assert 0 == len(logger.error.mock_calls)
    assert logger.info.mock_calls[0].args == (
        "failed to process stream, retrying", )
    assert logger.info.mock_calls[1].args == (
        "failed to process stream, retrying", )
    assert logger.info.mock_calls[2].args == (
        "failed to process stream, retrying", )
    assert 1 == (await redis_stream.zcard("streams"))
    assert 1 == len(await redis_stream.keys("stream~*"))
    assert 1 == len(await redis_stream.hgetall("attempts"))
コード例 #8
0
async def test_stream_processor_retrying_pull(run_engine, _, logger_class,
                                              redis_stream, redis_cache):
    logs.setup_logging()
    logger = logger_class.return_value

    # One retries once, the other reaches max_retry
    run_engine.side_effect = [
        exceptions.MergeableStateUnknown(mock.Mock()),
        exceptions.MergeableStateUnknown(mock.Mock()),
        mock.Mock(),
        exceptions.MergeableStateUnknown(mock.Mock()),
        exceptions.MergeableStateUnknown(mock.Mock()),
    ]

    await worker.push(
        redis_stream,
        123,
        "owner",
        "repo",
        123,
        "pull_request",
        {"payload": "whatever"},
    )
    await worker.push(
        redis_stream,
        123,
        "owner",
        "repo",
        42,
        "comment",
        {"payload": "foobar"},
    )

    assert 1 == (await redis_stream.zcard("streams"))
    assert 1 == len(await redis_stream.keys("stream~*"))
    assert 2 == await redis_stream.xlen("stream~owner~123")
    assert 0 == len(await redis_stream.hgetall("attempts"))

    p = worker.StreamProcessor(redis_stream, redis_cache)
    await p.consume("stream~owner~123")

    assert len(run_engine.mock_calls) == 2
    assert run_engine.mock_calls == [
        mock.call(
            InstallationMatcher(owner="owner"),
            "repo",
            123,
            [
                {
                    "event_type": "pull_request",
                    "data": {
                        "payload": "whatever"
                    },
                    "timestamp": mock.ANY,
                },
            ],
        ),
        mock.call(
            InstallationMatcher(owner="owner"),
            "repo",
            42,
            [
                {
                    "event_type": "comment",
                    "data": {
                        "payload": "foobar"
                    },
                    "timestamp": mock.ANY,
                },
            ],
        ),
    ]

    # Check stream still there and attempts recorded
    assert 1 == (await redis_stream.zcard("streams"))
    assert 1 == len(await redis_stream.keys("stream~*"))
    assert {
        b"pull~owner~repo~42": b"1",
        b"pull~owner~repo~123": b"1",
    } == await redis_stream.hgetall("attempts")

    await p.consume("stream~owner~123")
    assert 1 == (await redis_stream.zcard("streams"))
    assert 1 == len(await redis_stream.keys("stream~*"))
    assert 1 == len(await redis_stream.hgetall("attempts"))
    assert len(run_engine.mock_calls) == 4
    assert {
        b"pull~owner~repo~42": b"2"
    } == await redis_stream.hgetall("attempts")

    await p.consume("stream~owner~123")
    assert len(run_engine.mock_calls) == 5

    # Too many retries, everything is gone
    assert 3 == len(logger.info.mock_calls)
    assert 1 == len(logger.error.mock_calls)
    assert logger.info.mock_calls[0].args == (
        "failed to process pull request, retrying", )
    assert logger.info.mock_calls[1].args == (
        "failed to process pull request, retrying", )
    assert logger.error.mock_calls[0].args == (
        "failed to process pull request, abandoning", )
    assert 0 == (await redis_stream.zcard("streams"))
    assert 0 == len(await redis_stream.keys("stream~*"))
    assert 0 == len(await redis_stream.hgetall("attempts"))
コード例 #9
0
async def test_consume_unexisting_stream(run_engine, _, redis_stream,
                                         redis_cache, logger_checker):
    p = worker.StreamProcessor(redis_stream, redis_cache)
    await p.consume("stream~notexists~2")
    assert len(run_engine.mock_calls) == 0
コード例 #10
0
ファイル: test_worker.py プロジェクト: eladb/mergify-engine
async def test_stream_processor_date_scheduling(run_engine, get_install_by_id,
                                                redis, logger_checker):
    get_install_by_id.side_effect = fake_install_id

    # Don't process it before 2040
    with freeze_time("2040-01-01"):
        await worker.push(
            redis,
            12345,
            "owner",
            "repo",
            123,
            "pull_request",
            {"payload": "whatever"},
        )
        unwanted_installation_id = fake_install_id(12345)

    with freeze_time("2020-01-01"):
        await worker.push(
            redis,
            54321,
            "owner",
            "repo",
            321,
            "pull_request",
            {"payload": "foobar"},
        )
        wanted_installation_id = fake_install_id(54321)

    assert 2 == (await redis.zcard("streams"))
    assert 2 == len(await redis.keys("stream~*"))
    assert 0 == len(await redis.hgetall("attempts"))

    s = worker.StreamSelector(1, redis)
    p = worker.StreamProcessor(redis)

    received = []

    def fake_engine(installation_id, owner, repo, pull_number, sources):
        received.append(installation_id)

    run_engine.side_effect = fake_engine

    with freeze_time("2020-01-14"):
        async with s.next_stream() as stream_name:
            assert stream_name is not None
            await p.consume(stream_name)

    assert 1 == (await redis.zcard("streams"))
    assert 1 == len(await redis.keys("stream~*"))
    assert 0 == len(await redis.hgetall("attempts"))
    assert received == [wanted_installation_id]

    with freeze_time("2030-01-14"):
        async with s.next_stream() as stream_name:
            assert stream_name is None

    assert 1 == (await redis.zcard("streams"))
    assert 1 == len(await redis.keys("stream~*"))
    assert 0 == len(await redis.hgetall("attempts"))
    assert received == [wanted_installation_id]

    # We are in 2041, we have something todo :)
    with freeze_time("2041-01-14"):
        async with s.next_stream() as stream_name:
            assert stream_name is not None
            await p.consume(stream_name)

    assert 0 == (await redis.zcard("streams"))
    assert 0 == len(await redis.keys("stream~*"))
    assert 0 == len(await redis.hgetall("attempts"))
    assert received == [wanted_installation_id, unwanted_installation_id]
コード例 #11
0
ファイル: test_worker.py プロジェクト: eladb/mergify-engine
async def test_stream_processor_retrying_stream_failure(
        run_engine, get_install_by_id, logger, redis):
    get_install_by_id.side_effect = fake_install_id
    logs.setup_logging(worker="streams")

    response = mock.Mock()
    response.json.return_value = {"message": "boom"}
    response.status_code = 401
    run_engine.side_effect = httpx.HTTPClientSideError(response=response)

    await worker.push(
        redis,
        12345,
        "owner",
        "repo",
        123,
        "pull_request",
        {"payload": "whatever"},
    )
    await worker.push(
        redis,
        12345,
        "owner",
        "repo",
        123,
        "comment",
        {"payload": "foobar"},
    )

    assert 1 == (await redis.zcard("streams"))
    assert 1 == len(await redis.keys("stream~*"))
    assert 2 == await redis.xlen("stream~12345")
    assert 0 == len(await redis.hgetall("attempts"))

    p = worker.StreamProcessor(redis)
    await p.consume("stream~12345")

    assert len(run_engine.mock_calls) == 1
    assert run_engine.mock_calls[0] == mock.call(
        fake_install_id(12345),
        "owner",
        "repo",
        123,
        [
            {
                "event_type": "pull_request",
                "data": {
                    "payload": "whatever"
                }
            },
            {
                "event_type": "comment",
                "data": {
                    "payload": "foobar"
                }
            },
        ],
    )

    # Check stream still there and attempts recorded
    assert 1 == (await redis.zcard("streams"))
    assert 1 == len(await redis.keys("stream~*"))
    assert 1 == len(await redis.hgetall("attempts"))

    assert {b"stream~12345": b"1"} == await redis.hgetall("attempts")

    await p.consume("stream~12345")
    assert len(run_engine.mock_calls) == 2
    assert {b"stream~12345": b"2"} == await redis.hgetall("attempts")

    await p.consume("stream~12345")
    assert len(run_engine.mock_calls) == 3

    # Still there
    assert 3 == len(logger.info.mock_calls)
    assert 0 == len(logger.error.mock_calls)
    assert logger.info.mock_calls[0].args == (
        "failed to process stream, retrying", )
    assert logger.info.mock_calls[1].args == (
        "failed to process stream, retrying", )
    assert logger.info.mock_calls[2].args == (
        "failed to process stream, retrying", )
    assert 1 == (await redis.zcard("streams"))
    assert 1 == len(await redis.keys("stream~*"))
    assert 1 == len(await redis.hgetall("attempts"))
コード例 #12
0
ファイル: test_worker.py プロジェクト: eladb/mergify-engine
async def test_consume_unexisting_stream(run_engine, get_install_by_id, redis,
                                         logger_checker):
    get_install_by_id.side_effect = fake_install_id
    p = worker.StreamProcessor(redis)
    await p.consume("stream~666")
    assert len(run_engine.mock_calls) == 0