def test_get_running2(client):
"""
Tests getting running containers when some hosts are unavailable
"""
with client.application.app_context():
match = re.compile(r"test-.+")
client_mock = ClientFixture.new([
ContainerFixture.new(name="fastlane-job-123",
container_id="fastlane-job-123")
])
faulty_client = ClientFixture.new([
ContainerFixture.new(name="fastlane-job-456",
container_id="fastlane-job-456")
])
faulty_client.containers.list.side_effect = RuntimeError("failed")
pool_mock = PoolFixture.new(
clients={
"host:1234": ("host", 1234, client_mock),
"host:4567": ("host", 4567, faulty_client),
},
clients_per_regex=[(match, [("host", 1234, client_mock),
("host", 4567, faulty_client)])],
max_running={match: 2},
)
executor = Executor(client.application, pool_mock)
result = executor.get_running_containers()
expect(result).to_include("available")
available = result["available"]
expect(available).to_be_like([{
"host": "host",
"port": 1234,
"available": True,
"blacklisted": False,
"circuit": "closed",
"error": None,
}])
expect(result).to_include("unavailable")
unavailable = result["unavailable"]
expect(unavailable).to_be_like([{
"host": "host",
"port": 4567,
"available": False,
"blacklisted": False,
"circuit": "closed",
"error": "failed",
}])
expect(result).to_include("running")
running = result["running"]
expect(running).to_length(1)
host, port, container_id = running[0]
expect(host).to_equal("host")
expect(port).to_equal(1234)
expect(container_id).to_equal("fastlane-job-123")
def test_get_running1(client):
"""
Tests getting running containers
"""
with client.application.app_context():
containers = [
ContainerFixture.new(name="fastlane-job-123",
container_id="fastlane-job-123")
]
match, pool_mock, client_mock = PoolFixture.new_defaults(
r"test.+", max_running=1, containers=containers)
executor = Executor(client.application, pool_mock)
result = executor.get_running_containers()
expect(result).to_include("available")
available = result["available"]
expect(available).to_length(1)
expect(available[0]).to_equal({
"host": "host",
"port": 1234,
"available": True,
"blacklisted": False,
"circuit": "closed",
"error": None,
})
expect(result).to_include("running")
running = result["running"]
expect(running).to_length(1)
host, port, container_id = running[0]
expect(host).to_equal("host")
expect(port).to_equal(1234)
expect(container_id).to_equal("fastlane-job-123")
def test_remove_done1(client):
"""
Tests removing all defunct containers
"""
with client.application.app_context():
containers = [
ContainerFixture.new(
container_id="fastlane-job-123",
name="defunct-fastlane-job-123",
stdout="stdout",
stderr="stderr",
)
]
_, pool_mock, _ = PoolFixture.new_defaults(r"test.+",
max_running=1,
containers=containers)
JobExecutionFixture.new_defaults(container_id="fastlane-job-123")
executor = Executor(client.application, pool_mock)
result = executor.remove_done()
containers[0].remove.assert_called()
expect(result).to_length(1)
expect(result[0]).to_be_like({
"host": "host:1234",
"name": "defunct-fastlane-job-123",
"id": "fastlane-job-123",
"image": "ubuntu:latest",
})
def test_mark_as_done2(client):
"""
Tests marking a container as done raises HostUnavailableError
when docker host is not available
"""
with client.application.app_context():
containers = [
ContainerFixture.new(
container_id="fastlane-job-123",
name="fastlane-job-123",
stdout="stdout",
stderr="stderr",
)
]
_, pool_mock, _ = PoolFixture.new_defaults(r"test.+",
max_running=1,
containers=containers)
task, job, execution = JobExecutionFixture.new_defaults(
container_id="fastlane-job-123")
executor = Executor(client.application, pool_mock)
containers[0].rename.side_effect = requests.exceptions.ConnectionError(
"failed")
message = "Connection to host host:1234 failed with error: failed"
with expect.error_to_happen(HostUnavailableError, message=message):
executor.mark_as_done(task, job, execution)
def test_mark_as_done1(client):
"""
Tests marking a container as done renames the container
"""
with client.application.app_context():
containers = [
ContainerFixture.new(
container_id="fastlane-job-123",
name="fastlane-job-123",
stdout="stdout",
stderr="stderr",
)
]
_, pool_mock, _ = PoolFixture.new_defaults(r"test.+",
max_running=1,
containers=containers)
task, job, execution = JobExecutionFixture.new_defaults(
container_id="fastlane-job-123")
executor = Executor(client.application, pool_mock)
executor.mark_as_done(task, job, execution)
new_name = f"defunct-fastlane-job-123"
containers[0].rename.assert_called_with(new_name)
def test_validate_max3(client):
"""
Tests validating max current executions returns False
if max concurrent containers already running
"""
app = client.application
with app.app_context():
containers = [
ContainerFixture.new(name="fastlane-job-123"),
ContainerFixture.new(name="fastlane-job-456"),
]
match, pool_mock, client_mock = PoolFixture.new_defaults(
r"test.+", max_running=1, containers=containers)
executor = Executor(app, pool_mock)
result = executor.validate_max_running_executions("test123")
expect(result).to_be_false()
def verify_get_result(client, status, exit_code, stdout, stderr, custom_error,
started_at, finished_at):
app = client.application
with app.app_context():
container_mock = ContainerFixture.new_with_status(
container_id="fastlane-job-123",
name="fastlane-job-123",
status=status,
exit_code=exit_code,
started_at=started_at,
finished_at=finished_at,
custom_error=custom_error,
stdout=stdout,
stderr=stderr,
)
_, pool_mock, _ = PoolFixture.new_defaults(r"test[-].+",
max_running=1,
containers=[container_mock])
executor = Executor(app, pool_mock)
_, job, execution = JobExecutionFixture.new_defaults(
container_id="fastlane-job-123")
result = executor.get_result(job.task, job, execution)
expect(result.status).to_equal(STATUS.get(status))
expect(result.exit_code).to_equal(exit_code)
if stdout is None:
expect(result.log).to_be_empty()
else:
expect(result.log).to_equal(stdout)
if stderr is not None and custom_error != "":
expect(
result.error).to_equal(f"{custom_error}\n\nstderr:\n{stderr}")
else:
if stderr is not None:
expect(result.error).to_equal(stderr)
else:
expect(result.error).to_equal(custom_error)
parsed_started_at = parse(started_at)
expect(result.started_at).to_equal(parsed_started_at)
if finished_at is not None:
parsed_finished_at = parse(finished_at)
else:
parsed_finished_at = finished_at
expect(result.finished_at).to_equal(parsed_finished_at)
def test_validate_max1(client):
"""
Tests validating max current executions for a docker host
"""
app = client.application
with app.app_context():
containers = [ContainerFixture.new(name="fastlane-job-123")]
match, pool_mock, client_mock = PoolFixture.new_defaults(
r"test.+", max_running=1, containers=containers)
executor = Executor(app, pool_mock)
result = executor.validate_max_running_executions("test123")
expect(result).to_be_true()
def test_get_result3(client):
app = client.application
with app.app_context():
container_mock = ContainerFixture.new_with_status(
container_id="fastlane-job-123", name="fastlane-job-123")
_, pool_mock, client_mock = PoolFixture.new_defaults(
r"test[-].+", max_running=1, containers=[container_mock])
client_mock.containers.get.side_effect = requests.exceptions.ConnectionError(
"failed")
executor = Executor(app, pool_mock)
_, job, execution = JobExecutionFixture.new_defaults(
container_id="fastlane-job-123")
msg = "Connection to host host:1234 failed with error: failed"
with expect.error_to_happen(HostUnavailableError, message=msg):
executor.get_result(job.task, job, execution)
def test_get_running3(client):
"""
Tests getting running containers when some hosts are blacklisted
"""
with client.application.app_context():
match = re.compile(r"test-.+")
client_mock = ClientFixture.new([
ContainerFixture.new(name="fastlane-job-123",
container_id="fastlane-job-123")
])
pool_mock = PoolFixture.new(
clients={"host:1234": ("host", 1234, client_mock)},
clients_per_regex=[(match, [("host", 1234, client_mock)])],
max_running={match: 2},
)
executor = Executor(client.application, pool_mock)
result = executor.get_running_containers(
blacklisted_hosts=set(["host:1234"]))
expect(result).to_include("available")
available = result["available"]
expect(available).to_be_empty()
expect(result).to_include("unavailable")
unavailable = result["unavailable"]
expect(unavailable).to_be_like([{
"host": "host",
"port": 1234,
"available": False,
"blacklisted": True,
"circuit": "closed",
"error": "server is blacklisted",
}])
expect(result).to_include("running")
running = result["running"]
expect(running).to_length(0)
def test_stop1(client):
"""
Tests stopping a job stops the container in docker
"""
app = client.application
with app.app_context():
match, pool_mock, client_mock = PoolFixture.new_defaults(r"test[-].+",
max_running=1)
container_mock = ContainerFixture.new_with_status(
name="fastlane-job-1234")
client_mock.containers.get.return_value = container_mock
task, job, execution = JobExecutionFixture.new_defaults(
container_id="fastlane-job-1234")
executor = Executor(app, pool_mock)
executor.stop_job(task, job, execution)
container_mock.stop.assert_called()
def test_stop1(client):
"""
Tests stopping a job stops the container in docker
"""
app = client.application
with app.app_context():
container_mock = ContainerFixture.new_with_status(
name="fastlane-job-1234", container_id="fastlane-job-1234")
_, pool_mock, _ = PoolFixture.new_defaults(r"test[-].+",
max_running=1,
containers=[container_mock])
task, job, execution = JobExecutionFixture.new_defaults(
container_id="fastlane-job-1234")
executor = Executor(app, pool_mock)
result = executor.stop_job(task, job, execution)
container_mock.stop.assert_called()
expect(result).to_be_true()