def test_remove_done1(client):
"""
Tests removing all defunct containers
"""
with client.application.app_context():
containers = [
ContainerFixture.new(
container_id="fastlane-job-123",
name="defunct-fastlane-job-123",
stdout="stdout",
stderr="stderr",
)
]
_, pool_mock, _ = PoolFixture.new_defaults(r"test.+",
max_running=1,
containers=containers)
JobExecutionFixture.new_defaults(container_id="fastlane-job-123")
executor = Executor(client.application, pool_mock)
result = executor.remove_done()
containers[0].remove.assert_called()
expect(result).to_length(1)
expect(result[0]).to_be_like({
"host": "host:1234",
"name": "defunct-fastlane-job-123",
"id": "fastlane-job-123",
"image": "ubuntu:latest",
})
def test_downloading_image2(worker):
"""
Test updating an image when executor raises HostUnavailableError,
the job is re-enqueued and method returns False
"""
app = worker.app.app
with app.app_context():
task, job, execution = JobExecutionFixture.new_defaults()
exec_mock = MagicMock()
exec_mock.update_image.side_effect = HostUnavailableError(
"docker", "9999", "failed"
)
result = job_mod.download_image(
exec_mock, job, execution, job.image, "latest", job.command, app.logger
)
expect(result).to_be_false()
expect(job.metadata["enqueued_id"]).not_to_be_null()
expect(app.redis.zcard(Queue.SCHEDULED_QUEUE_NAME)).to_equal(1)
item = app.redis.zrank(Queue.SCHEDULED_QUEUE_NAME, job.metadata["enqueued_id"])
expect(item).to_equal(0)
def test_validate_expiration3(worker):
"""
Test validating the expiration of a Job returns False if the job
has expiration and has expired. It also tests that the job is marked
as expired with the proper message as error.
"""
app = worker.app.app
with app.app_context():
task, job, execution = JobExecutionFixture.new_defaults()
unow = unix_now()
exp = unow - 10
job.metadata["expiration"] = exp
job.save()
result = job_mod.validate_expiration(job, execution, app.logger)
expect(result).to_be_false()
expect(execution.status).to_equal(JobExecution.Status.expired)
expiration_utc = from_unix(job.metadata["expiration"])
error = (
f"Job was supposed to be done before {expiration_utc.isoformat()}, "
f"but was started at {from_unix(unow).isoformat()}."
)
expect(execution.error).to_equal(error)
expect(execution.finished_at).not_to_be_null()
def test_stop_execution2(client):
"""Test stopping job execution with invalid data"""
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults(exit_code=0,
log="test log")
resp = client.post(
f"/tasks/invalid/jobs/{job.job_id}/executions/{execution.execution_id}/stop/"
)
msg = f"Task (invalid) or Job ({job.job_id}) not found."
expect(resp).to_be_an_error_with(status=404,
msg=msg,
operation="stop_job_execution")
resp = client.post(
f"/tasks/{task.task_id}/jobs/invalid/executions/{execution.execution_id}/stop/"
)
msg = f"Task ({task.task_id}) or Job (invalid) not found."
expect(resp).to_be_an_error_with(status=404,
msg=msg,
operation="stop_job_execution")
resp = client.post(
f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/invalid/stop/"
)
msg = f"Job Execution (invalid) not found in Job ({job.job_id})."
expect(resp).to_be_an_error_with(status=404,
msg=msg,
operation="stop_job_execution")
resp = client.post(
f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/stop"
)
expect(resp.status_code).to_equal(200)
def test_enqueue_missing1(worker):
"""Test self-healing enqueueing missing monitor jobs"""
with worker.app.app.app_context():
app = worker.app.app
app.redis.flushall()
for status in [
JobExecution.Status.enqueued,
JobExecution.Status.pulling,
JobExecution.Status.running,
JobExecution.Status.done,
JobExecution.Status.failed,
]:
_, job, execution = JobExecutionFixture.new_defaults()
execution.status = status
if status == JobExecution.Status.pulling:
monitor_queue = worker.app.app.monitor_queue
enqueued_id = monitor_queue.enqueue_in(
"1s",
Categories.Monitor,
job.task.task_id,
job.job_id,
execution.execution_id,
)
job.metadata["enqueued_id"] = enqueued_id
job.save()
job_mod.enqueue_missing_monitor_jobs(app)
res = app.redis.zcard(Queue.SCHEDULED_QUEUE_NAME)
expect(res).to_equal(2)
def test_run2(client):
"""
Tests that a docker executor raises HostUnavailableError
when host is not available when running a container and
deletes host and port metadata from execution
"""
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults()
_, pool_mock, client_mock = PoolFixture.new_defaults(r"test-.+")
client_mock.containers.run.side_effect = requests.exceptions.ConnectionError(
"failed")
exe = Executor(app=client.application, pool=pool_mock)
msg = "Connection to host host:1234 failed with error: failed"
with expect.error_to_happen(HostUnavailableError, message=msg):
exe.run(
task,
job,
execution,
"mock-image",
"latest",
"command",
blacklisted_hosts=set(),
)
expect(execution.metadata).not_to_include("docker_host")
expect(execution.metadata).not_to_include("docker_port")
def test_mark_as_done1(client):
"""
Tests marking a container as done renames the container
"""
with client.application.app_context():
containers = [
ContainerFixture.new(
container_id="fastlane-job-123",
name="fastlane-job-123",
stdout="stdout",
stderr="stderr",
)
]
_, pool_mock, _ = PoolFixture.new_defaults(r"test.+",
max_running=1,
containers=containers)
task, job, execution = JobExecutionFixture.new_defaults(
container_id="fastlane-job-123")
executor = Executor(client.application, pool_mock)
executor.mark_as_done(task, job, execution)
new_name = f"defunct-fastlane-job-123"
containers[0].rename.assert_called_with(new_name)
def test_mark_as_done2(client):
"""
Tests marking a container as done raises HostUnavailableError
when docker host is not available
"""
with client.application.app_context():
containers = [
ContainerFixture.new(
container_id="fastlane-job-123",
name="fastlane-job-123",
stdout="stdout",
stderr="stderr",
)
]
_, pool_mock, _ = PoolFixture.new_defaults(r"test.+",
max_running=1,
containers=containers)
task, job, execution = JobExecutionFixture.new_defaults(
container_id="fastlane-job-123")
executor = Executor(client.application, pool_mock)
containers[0].rename.side_effect = requests.exceptions.ConnectionError(
"failed")
message = "Connection to host host:1234 failed with error: failed"
with expect.error_to_happen(HostUnavailableError, message=message):
executor.mark_as_done(task, job, execution)
def test_get_execution2(client):
"""Test getting tasks with invalid task returns 404"""
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults()
resp = client.get(
f"/tasks/invalid/jobs/{job.job_id}/executions/{execution.execution_id}/"
)
msg = f"Task (invalid) or Job ({job.job_id}) not found."
expect(resp).to_be_an_error_with(status=404,
msg=msg,
operation="get_job_execution")
resp = client.get(
f"/tasks/{task.task_id}/jobs/invalid/executions/{execution.execution_id}/"
)
msg = f"Task ({task.task_id}) or Job (invalid) not found."
expect(resp).to_be_an_error_with(status=404,
msg=msg,
operation="get_job_execution")
resp = client.get(
f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/invalid/")
msg = f"Job Execution (invalid) not found in job ({job.job_id})."
expect(resp).to_be_an_error_with(status=404,
msg=msg,
operation="get_job_execution")
def test_circuit6(client):
"""
Tests that when marking a container as done with a
docker host that's not accessible, the circuit is open and
a HostUnavailableError is raised
"""
with client.application.app_context():
client.application.config["DOCKER_CIRCUIT_BREAKER_MAX_FAILS"] = 1
task, job, execution = JobExecutionFixture.new_defaults(
docker_host="localhost",
docker_port=4567,
container_id=str(uuid4()))
pool = DockerPool(([None, ["localhost:4567"], 2], ))
executor = Executor(client.application, pool)
expect(executor.get_circuit("localhost:4567").current_state).to_equal(
"closed")
with expect.error_to_happen(HostUnavailableError):
executor.mark_as_done(task, job, execution)
expect(executor.get_circuit("localhost:4567").current_state).to_equal(
"open")
def test_get_execution_logs3(client):
"""Test getting job execution logs with invalid data"""
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults(exit_code=0,
log="test log")
resp = client.get(
f"/tasks/invalid/jobs/{job.job_id}/executions/{execution.execution_id}/logs/"
)
msg = f"Task (invalid) or Job ({job.job_id}) not found."
expect(resp).to_be_an_error_with(
status=404, msg=msg, operation="retrieve_execution_details")
resp = client.get(
f"/tasks/{task.task_id}/jobs/invalid/executions/{execution.execution_id}/logs/"
)
msg = f"Task ({task.task_id}) or Job (invalid) not found."
expect(resp).to_be_an_error_with(
status=404, msg=msg, operation="retrieve_execution_details")
resp = client.get(
f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/invalid/logs/"
)
msg = "No executions found in job with specified arguments."
expect(resp).to_be_an_error_with(
status=400, msg=msg, operation="retrieve_execution_details")
def test_run3(client):
"""
Tests that a docker executor raises RuntimeError if no
docker_host and docker_port available in execution
"""
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults()
_, pool_mock, _ = PoolFixture.new_defaults(r"test-.+")
exe = Executor(app=client.application, pool=pool_mock)
del execution.metadata["docker_host"]
del execution.metadata["docker_port"]
msg = "Can't run job without docker_host and docker_port in execution metadata."
with expect.error_to_happen(RuntimeError, message=msg):
exe.run(
task,
job,
execution,
"mock-image",
"latest",
"command",
blacklisted_hosts=set(),
)
expect(execution.metadata).not_to_include("docker_host")
expect(execution.metadata).not_to_include("docker_port")
def test_get_unfinished_executions(client):
with client.application.app_context():
app = client.application
app.redis.flushall()
for status in [
JobExecution.Status.enqueued,
JobExecution.Status.pulling,
JobExecution.Status.running,
JobExecution.Status.done,
JobExecution.Status.failed,
]:
_, job, execution = JobExecutionFixture.new_defaults()
execution.status = status
job.save()
topic = Job.get_unfinished_executions()
expect(topic).to_length(2)
for (_, execution) in topic:
expect(execution).to_be_instance_of(JobExecution)
expect(
execution.status
in [JobExecution.Status.pulling, JobExecution.Status.running]
).to_be_true()
def test_run1(client):
"""Tests that a docker executor can run containers"""
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults()
match, pool_mock, client_mock = PoolFixture.new_defaults(r"test-.+")
client_mock.containers.run.return_value = MagicMock(id="job_id")
exe = Executor(app=client.application, pool=pool_mock)
exe.run(
task,
job,
execution,
"mock-image",
"latest",
"command",
blacklisted_hosts=set(),
)
expect(execution.metadata).to_include("container_id")
expect(client_mock.containers.run.call_count).to_equal(1)
client_mock.containers.run.assert_called_with(
image=f"mock-image:latest",
environment={},
command="command",
detach=True,
name=f"fastlane-job-{execution.execution_id}",
)
def test_stop_execution1(client):
with client.application.app_context():
# def test_method():
# pass
# scheduler = Scheduler("jobs", connection=client.application.redis)
# scheduler.enqueue_at(datetime(2020, 1, 1), test_method)
# enqueued_jobs = client.application.redis.zrange(
# b"rq:scheduler:scheduled_jobs", 0, -1
# )
# expect(enqueued_jobs).to_length(1)
# enqueued_job_id = enqueued_jobs[0].decode("utf-8")
task, job, execution = JobExecutionFixture.new_defaults(
status=JobExecution.Status.running
)
# job.metadata["enqueued_id"] = enqueued_job_id
job.metadata["retries"] = 3
job.metadata["retry_count"] = 0
job.save()
executor_mock = MagicMock()
client.application.executor = executor_mock
resp = client.post(
f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/stop/"
)
expect(resp.status_code).to_equal(200)
obj = loads(resp.data)
expect(obj).to_be_like(
{
"execution": {
"id": execution.execution_id,
"url": (
f"http://localhost:10000/tasks/{task.task_id}/jobs/"
f"{job.job_id}/executions/{execution.execution_id}/"
),
},
"job": {
"id": job.job_id,
"url": f"http://localhost:10000/tasks/{task.task_id}/jobs/{job.job_id}/",
},
"task": {
"id": task.task_id,
"url": f"http://localhost:10000/tasks/{task.task_id}/",
},
}
)
executor_mock.stop_job.assert_called()
job.reload()
expect(job.metadata["retry_count"]).to_equal(4)
def test_validate_expiration1(worker):
"""
Test validating the expiration of a Job returns True if the job
has no expiration.
"""
app = worker.app.app
with app.app_context():
task, job, execution = JobExecutionFixture.new_defaults()
result = job_mod.validate_expiration(job, execution, app.logger)
expect(result).to_be_true()
def test_job_logs2(client):
"""Tests get job logs fails if invalid input."""
task, job, execution = JobExecutionFixture.new_defaults()
resp1 = client.get(f"/tasks/{task.task_id}/jobs/invalid-id/logs/")
expect(resp1.status_code).to_equal(404)
obj = loads(resp1.data)
expect(obj["error"]).to_equal(
f"Task ({task.task_id}) or Job (invalid-id) not found.")
expect(obj["operation"]).to_equal("retrieve_execution_details")
def test_job_stderr1(client):
"""Tests get job stderr returns log for last execution."""
task, job, execution = JobExecutionFixture.new_defaults()
execution.error = "test error"
execution.status = JobExecution.Status.done
execution.save()
resp1 = client.get(f"/tasks/{task.task_id}/jobs/{job.job_id}/stderr/")
expect(resp1.status_code).to_equal(200)
expect(resp1.data).to_equal("test error")
def test_monitor_job_with_retry2(client):
"""Test monitoring a job for a task that fails stops after max retries"""
with client.application.app_context():
app = client.application
app.redis.flushall()
task, job, execution = JobExecutionFixture.new_defaults()
job.metadata["retries"] = 3
job.metadata["retry_count"] = 3
job.save()
job_id = job.job_id
exec_mock = MagicMock()
exec_mock.get_result.return_value = MagicMock(
exit_code=1, log="".encode("utf-8"), error="error".encode("utf-8"))
client.application.executor = exec_mock
queue = Queue("monitor",
is_async=False,
connection=client.application.redis)
result = queue.enqueue(job_mod.monitor_job, task.task_id, job_id,
execution.execution_id)
worker = SimpleWorker([queue], connection=queue.connection)
worker.work(burst=True)
task.reload()
expect(task.jobs).to_length(1)
job = task.jobs[0]
expect(job.executions).to_length(1)
execution = job.executions[0]
expect(execution.image).to_equal("image")
expect(execution.command).to_equal("command")
hash_key = f"rq:job:{result.id}"
res = app.redis.exists(hash_key)
expect(res).to_be_true()
res = app.redis.hget(hash_key, "status")
expect(res).to_equal("finished")
res = app.redis.hexists(hash_key, "data")
expect(res).to_be_true()
keys = app.redis.keys()
next_job_id = [
key for key in keys if key.decode("utf-8").startswith("rq:job")
and not key.decode("utf-8").endswith(result.id)
]
expect(next_job_id).to_length(0)
def verify_get_result(client, status, exit_code, stdout, stderr, custom_error,
started_at, finished_at):
app = client.application
with app.app_context():
container_mock = ContainerFixture.new_with_status(
container_id="fastlane-job-123",
name="fastlane-job-123",
status=status,
exit_code=exit_code,
started_at=started_at,
finished_at=finished_at,
custom_error=custom_error,
stdout=stdout,
stderr=stderr,
)
_, pool_mock, _ = PoolFixture.new_defaults(r"test[-].+",
max_running=1,
containers=[container_mock])
executor = Executor(app, pool_mock)
_, job, execution = JobExecutionFixture.new_defaults(
container_id="fastlane-job-123")
result = executor.get_result(job.task, job, execution)
expect(result.status).to_equal(STATUS.get(status))
expect(result.exit_code).to_equal(exit_code)
if stdout is None:
expect(result.log).to_be_empty()
else:
expect(result.log).to_equal(stdout)
if stderr is not None and custom_error != "":
expect(
result.error).to_equal(f"{custom_error}\n\nstderr:\n{stderr}")
else:
if stderr is not None:
expect(result.error).to_equal(stderr)
else:
expect(result.error).to_equal(custom_error)
parsed_started_at = parse(started_at)
expect(result.started_at).to_equal(parsed_started_at)
if finished_at is not None:
parsed_finished_at = parse(finished_at)
else:
parsed_finished_at = finished_at
expect(result.finished_at).to_equal(parsed_finished_at)
def test_validate_expiration2(worker):
"""
Test validating the expiration of a Job returns True if the job
has expiration but not expired.
"""
app = worker.app.app
with app.app_context():
task, job, execution = JobExecutionFixture.new_defaults()
job.metadata["expiration"] = unix_now() + 10
job.save()
result = job_mod.validate_expiration(job, execution, app.logger)
expect(result).to_be_true()
def test_get_execution1(client):
"""Test get execution details"""
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults(
status=JobExecution.Status.done
)
resp = client.get(
f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/"
)
expect(resp.status_code).to_equal(200)
task_url = url_for("task.get_task", task_id=str(task.task_id), _external=True)
data = loads(resp.data)
expect(data).to_include("task")
expect(data["task"]).to_include("id")
expect(data["task"]).to_include("url")
expect(data["task"]["id"]).to_equal(task.task_id)
expect(data["task"]["url"]).to_equal(task_url)
job_url = url_for(
"task.get_job",
task_id=str(task.task_id),
job_id=str(job.job_id),
_external=True,
)
expect(data).to_include("job")
expect(data["job"]).to_include("id")
expect(data["job"]).to_include("url")
expect(data["job"]["id"]).to_equal(job.job_id)
expect(data["job"]["url"]).to_equal(job_url)
expect(data["execution"]["createdAt"]).not_to_be_null()
del data["execution"]["createdAt"]
expect(data["execution"]).to_be_like(
{
"command": "command",
"error": None,
"executionId": execution.execution_id,
"exitCode": None,
"finishedAt": None,
"image": "image",
"log": None,
"metadata": execution.metadata,
"requestIPAddress": None,
"startedAt": None,
"status": "done",
}
)
def test_get_execution_stdout1(client):
"""Test getting job execution stdout"""
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults(
exit_code=0,
log="test log",
error="some error",
status=JobExecution.Status.done,
)
resp = client.get(
f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/stdout/"
)
expect(resp.status_code).to_equal(200)
expect(resp.data).to_equal("test log")
def test_get_execution_logs2(client):
"""Test getting job execution logs returns empty if job is running"""
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults(
exit_code=0,
log="test log",
error="some error",
status=JobExecution.Status.running,
)
resp = client.get(
f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/logs/"
)
expect(resp.status_code).to_equal(200)
expect(resp.data).to_be_empty()
def test_pull1(client):
"""Tests that a docker executor can pull images"""
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults()
match, pool_mock, client_mock = PoolFixture.new_defaults(r"test-.+")
exe = Executor(app=client.application, pool=pool_mock)
exe.update_image(task,
job,
execution,
"mock-image",
"latest",
blacklisted_hosts=set())
expect(client_mock.images.pull.call_count).to_equal(1)
client_mock.images.pull.assert_called_with("mock-image", tag="latest")
def test_run_container1(worker):
"""
Test running a container works and returns True
"""
app = worker.app.app
with app.app_context():
task, job, execution = JobExecutionFixture.new_defaults()
exec_mock = MagicMock()
result = job_mod.run_container(
exec_mock, job, execution, job.image, "latest", job.command, app.logger
)
expect(result).to_be_true()
exec_mock.run.assert_called_with(
task, job, execution, job.image, "latest", job.command
)
expect(execution.started_at).not_to_be_null()
expect(execution.status).to_equal(JobExecution.Status.running)
def test_stop2(client):
"""
Tests stopping a job stops fails if container is not in metadata
"""
app = client.application
with app.app_context():
_, pool_mock, _ = PoolFixture.new_defaults(r"test[-].+",
max_running=1,
containers=[])
task, job, execution = JobExecutionFixture.new_defaults()
del execution.metadata["container_id"]
executor = Executor(app, pool_mock)
result = executor.stop_job(task, job, execution)
expect(result).to_be_false()
def test_stop_execution1(client):
with client.application.app_context():
task, job, execution = JobExecutionFixture.new_defaults(
status=JobExecution.Status.running)
job.metadata["retries"] = 3
job.metadata["retry_count"] = 0
job.save()
executor_mock = MagicMock()
client.application.executor = executor_mock
resp = client.post(
f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/stop/"
)
expect(resp.status_code).to_equal(200)
obj = loads(resp.data)
expect(obj).to_be_like({
"execution": {
"id":
execution.execution_id,
"url": (f"http://localhost:10000/tasks/{task.task_id}/jobs/"
f"{job.job_id}/executions/{execution.execution_id}/"),
},
"job": {
"id":
job.job_id,
"url":
f"http://localhost:10000/tasks/{task.task_id}/jobs/{job.job_id}/",
},
"task": {
"id": task.task_id,
"url": f"http://localhost:10000/tasks/{task.task_id}/",
},
})
executor_mock.stop_job.assert_called()
job.reload()
expect(job.metadata["retry_count"]).to_equal(4)
enqueued_jobs = client.application.redis.zcard(
Queue.SCHEDULED_QUEUE_NAME)
expect(enqueued_jobs).to_equal(0)
def test_monitor_job1(worker):
"""Test monitoring a job with result already there"""
app = worker.app.app
with app.app_context():
app.redis.flushall()
task, job, execution = JobExecutionFixture.new_defaults()
execution.status = JobExecution.Status.running
job.save()
job_id = str(job.job_id)
exec_mock = MagicMock()
exec_mock.get_result.return_value = MagicMock(
exit_code=0, log="qwe".encode("utf-8"), error="".encode("utf-8")
)
app.executor = exec_mock
monitor_queue = app.monitor_queue
monitor_queue.enqueue(
Categories.Monitor, task.task_id, job_id, execution.execution_id
)
worker.loop_once()
monitor_queue.enqueue(
Categories.Monitor, task.task_id, job_id, execution.execution_id
)
worker.loop_once()
task.reload()
expect(task.jobs).to_length(1)
job = task.jobs[0]
expect(job.executions).to_length(1)
execution = job.executions[0]
expect(execution.image).to_equal("image")
expect(execution.command).to_equal("command")
expect(execution.status).to_equal(JobExecution.Status.done)
expect(app.redis.zcard(Queue.SCHEDULED_QUEUE_NAME)).to_equal(0)
def test_monitor_job_with_retry(worker):
"""Test monitoring a job for a task that fails"""
app = worker.app.app
with app.app_context():
app.redis.flushall()
task, job, execution = JobExecutionFixture.new_defaults()
execution.status = JobExecution.Status.running
job.metadata["retries"] = 3
job.metadata["retry_count"] = 0
execution.save()
job.save()
job_id = str(job.job_id)
exec_mock = MagicMock()
exec_mock.get_result.return_value = MagicMock(
exit_code=1, log="".encode("utf-8"), error="error".encode("utf-8")
)
app.executor = exec_mock
monitor_queue = app.monitor_queue
monitor_queue.enqueue(
Categories.Monitor, task.task_id, job_id, execution.execution_id
)
worker.loop_once()
task.reload()
expect(task.jobs).to_length(1)
job = task.jobs[0]
expect(job.executions).to_length(2)
execution = job.executions[0]
expect(execution.image).to_equal("image")
expect(execution.command).to_equal("command")
expect(app.redis.zcard(Queue.SCHEDULED_QUEUE_NAME)).to_equal(1)
task.reload()
expect(task.jobs[0].executions[0].status).to_equal(JobExecution.Status.failed)