def test_remove_done1(client): """ Tests removing all defunct containers """ with client.application.app_context(): containers = [ ContainerFixture.new( container_id="fastlane-job-123", name="defunct-fastlane-job-123", stdout="stdout", stderr="stderr", ) ] _, pool_mock, _ = PoolFixture.new_defaults(r"test.+", max_running=1, containers=containers) JobExecutionFixture.new_defaults(container_id="fastlane-job-123") executor = Executor(client.application, pool_mock) result = executor.remove_done() containers[0].remove.assert_called() expect(result).to_length(1) expect(result[0]).to_be_like({ "host": "host:1234", "name": "defunct-fastlane-job-123", "id": "fastlane-job-123", "image": "ubuntu:latest", })
def test_downloading_image2(worker): """ Test updating an image when executor raises HostUnavailableError, the job is re-enqueued and method returns False """ app = worker.app.app with app.app_context(): task, job, execution = JobExecutionFixture.new_defaults() exec_mock = MagicMock() exec_mock.update_image.side_effect = HostUnavailableError( "docker", "9999", "failed" ) result = job_mod.download_image( exec_mock, job, execution, job.image, "latest", job.command, app.logger ) expect(result).to_be_false() expect(job.metadata["enqueued_id"]).not_to_be_null() expect(app.redis.zcard(Queue.SCHEDULED_QUEUE_NAME)).to_equal(1) item = app.redis.zrank(Queue.SCHEDULED_QUEUE_NAME, job.metadata["enqueued_id"]) expect(item).to_equal(0)
def test_validate_expiration3(worker): """ Test validating the expiration of a Job returns False if the job has expiration and has expired. It also tests that the job is marked as expired with the proper message as error. """ app = worker.app.app with app.app_context(): task, job, execution = JobExecutionFixture.new_defaults() unow = unix_now() exp = unow - 10 job.metadata["expiration"] = exp job.save() result = job_mod.validate_expiration(job, execution, app.logger) expect(result).to_be_false() expect(execution.status).to_equal(JobExecution.Status.expired) expiration_utc = from_unix(job.metadata["expiration"]) error = ( f"Job was supposed to be done before {expiration_utc.isoformat()}, " f"but was started at {from_unix(unow).isoformat()}." ) expect(execution.error).to_equal(error) expect(execution.finished_at).not_to_be_null()
def test_stop_execution2(client): """Test stopping job execution with invalid data""" with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults(exit_code=0, log="test log") resp = client.post( f"/tasks/invalid/jobs/{job.job_id}/executions/{execution.execution_id}/stop/" ) msg = f"Task (invalid) or Job ({job.job_id}) not found." expect(resp).to_be_an_error_with(status=404, msg=msg, operation="stop_job_execution") resp = client.post( f"/tasks/{task.task_id}/jobs/invalid/executions/{execution.execution_id}/stop/" ) msg = f"Task ({task.task_id}) or Job (invalid) not found." expect(resp).to_be_an_error_with(status=404, msg=msg, operation="stop_job_execution") resp = client.post( f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/invalid/stop/" ) msg = f"Job Execution (invalid) not found in Job ({job.job_id})." expect(resp).to_be_an_error_with(status=404, msg=msg, operation="stop_job_execution") resp = client.post( f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/stop" ) expect(resp.status_code).to_equal(200)
def test_enqueue_missing1(worker): """Test self-healing enqueueing missing monitor jobs""" with worker.app.app.app_context(): app = worker.app.app app.redis.flushall() for status in [ JobExecution.Status.enqueued, JobExecution.Status.pulling, JobExecution.Status.running, JobExecution.Status.done, JobExecution.Status.failed, ]: _, job, execution = JobExecutionFixture.new_defaults() execution.status = status if status == JobExecution.Status.pulling: monitor_queue = worker.app.app.monitor_queue enqueued_id = monitor_queue.enqueue_in( "1s", Categories.Monitor, job.task.task_id, job.job_id, execution.execution_id, ) job.metadata["enqueued_id"] = enqueued_id job.save() job_mod.enqueue_missing_monitor_jobs(app) res = app.redis.zcard(Queue.SCHEDULED_QUEUE_NAME) expect(res).to_equal(2)
def test_run2(client): """ Tests that a docker executor raises HostUnavailableError when host is not available when running a container and deletes host and port metadata from execution """ with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults() _, pool_mock, client_mock = PoolFixture.new_defaults(r"test-.+") client_mock.containers.run.side_effect = requests.exceptions.ConnectionError( "failed") exe = Executor(app=client.application, pool=pool_mock) msg = "Connection to host host:1234 failed with error: failed" with expect.error_to_happen(HostUnavailableError, message=msg): exe.run( task, job, execution, "mock-image", "latest", "command", blacklisted_hosts=set(), ) expect(execution.metadata).not_to_include("docker_host") expect(execution.metadata).not_to_include("docker_port")
def test_mark_as_done1(client): """ Tests marking a container as done renames the container """ with client.application.app_context(): containers = [ ContainerFixture.new( container_id="fastlane-job-123", name="fastlane-job-123", stdout="stdout", stderr="stderr", ) ] _, pool_mock, _ = PoolFixture.new_defaults(r"test.+", max_running=1, containers=containers) task, job, execution = JobExecutionFixture.new_defaults( container_id="fastlane-job-123") executor = Executor(client.application, pool_mock) executor.mark_as_done(task, job, execution) new_name = f"defunct-fastlane-job-123" containers[0].rename.assert_called_with(new_name)
def test_mark_as_done2(client): """ Tests marking a container as done raises HostUnavailableError when docker host is not available """ with client.application.app_context(): containers = [ ContainerFixture.new( container_id="fastlane-job-123", name="fastlane-job-123", stdout="stdout", stderr="stderr", ) ] _, pool_mock, _ = PoolFixture.new_defaults(r"test.+", max_running=1, containers=containers) task, job, execution = JobExecutionFixture.new_defaults( container_id="fastlane-job-123") executor = Executor(client.application, pool_mock) containers[0].rename.side_effect = requests.exceptions.ConnectionError( "failed") message = "Connection to host host:1234 failed with error: failed" with expect.error_to_happen(HostUnavailableError, message=message): executor.mark_as_done(task, job, execution)
def test_get_execution2(client): """Test getting tasks with invalid task returns 404""" with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults() resp = client.get( f"/tasks/invalid/jobs/{job.job_id}/executions/{execution.execution_id}/" ) msg = f"Task (invalid) or Job ({job.job_id}) not found." expect(resp).to_be_an_error_with(status=404, msg=msg, operation="get_job_execution") resp = client.get( f"/tasks/{task.task_id}/jobs/invalid/executions/{execution.execution_id}/" ) msg = f"Task ({task.task_id}) or Job (invalid) not found." expect(resp).to_be_an_error_with(status=404, msg=msg, operation="get_job_execution") resp = client.get( f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/invalid/") msg = f"Job Execution (invalid) not found in job ({job.job_id})." expect(resp).to_be_an_error_with(status=404, msg=msg, operation="get_job_execution")
def test_circuit6(client): """ Tests that when marking a container as done with a docker host that's not accessible, the circuit is open and a HostUnavailableError is raised """ with client.application.app_context(): client.application.config["DOCKER_CIRCUIT_BREAKER_MAX_FAILS"] = 1 task, job, execution = JobExecutionFixture.new_defaults( docker_host="localhost", docker_port=4567, container_id=str(uuid4())) pool = DockerPool(([None, ["localhost:4567"], 2], )) executor = Executor(client.application, pool) expect(executor.get_circuit("localhost:4567").current_state).to_equal( "closed") with expect.error_to_happen(HostUnavailableError): executor.mark_as_done(task, job, execution) expect(executor.get_circuit("localhost:4567").current_state).to_equal( "open")
def test_get_execution_logs3(client): """Test getting job execution logs with invalid data""" with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults(exit_code=0, log="test log") resp = client.get( f"/tasks/invalid/jobs/{job.job_id}/executions/{execution.execution_id}/logs/" ) msg = f"Task (invalid) or Job ({job.job_id}) not found." expect(resp).to_be_an_error_with( status=404, msg=msg, operation="retrieve_execution_details") resp = client.get( f"/tasks/{task.task_id}/jobs/invalid/executions/{execution.execution_id}/logs/" ) msg = f"Task ({task.task_id}) or Job (invalid) not found." expect(resp).to_be_an_error_with( status=404, msg=msg, operation="retrieve_execution_details") resp = client.get( f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/invalid/logs/" ) msg = "No executions found in job with specified arguments." expect(resp).to_be_an_error_with( status=400, msg=msg, operation="retrieve_execution_details")
def test_run3(client): """ Tests that a docker executor raises RuntimeError if no docker_host and docker_port available in execution """ with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults() _, pool_mock, _ = PoolFixture.new_defaults(r"test-.+") exe = Executor(app=client.application, pool=pool_mock) del execution.metadata["docker_host"] del execution.metadata["docker_port"] msg = "Can't run job without docker_host and docker_port in execution metadata." with expect.error_to_happen(RuntimeError, message=msg): exe.run( task, job, execution, "mock-image", "latest", "command", blacklisted_hosts=set(), ) expect(execution.metadata).not_to_include("docker_host") expect(execution.metadata).not_to_include("docker_port")
def test_get_unfinished_executions(client): with client.application.app_context(): app = client.application app.redis.flushall() for status in [ JobExecution.Status.enqueued, JobExecution.Status.pulling, JobExecution.Status.running, JobExecution.Status.done, JobExecution.Status.failed, ]: _, job, execution = JobExecutionFixture.new_defaults() execution.status = status job.save() topic = Job.get_unfinished_executions() expect(topic).to_length(2) for (_, execution) in topic: expect(execution).to_be_instance_of(JobExecution) expect( execution.status in [JobExecution.Status.pulling, JobExecution.Status.running] ).to_be_true()
def test_run1(client): """Tests that a docker executor can run containers""" with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults() match, pool_mock, client_mock = PoolFixture.new_defaults(r"test-.+") client_mock.containers.run.return_value = MagicMock(id="job_id") exe = Executor(app=client.application, pool=pool_mock) exe.run( task, job, execution, "mock-image", "latest", "command", blacklisted_hosts=set(), ) expect(execution.metadata).to_include("container_id") expect(client_mock.containers.run.call_count).to_equal(1) client_mock.containers.run.assert_called_with( image=f"mock-image:latest", environment={}, command="command", detach=True, name=f"fastlane-job-{execution.execution_id}", )
def test_stop_execution1(client): with client.application.app_context(): # def test_method(): # pass # scheduler = Scheduler("jobs", connection=client.application.redis) # scheduler.enqueue_at(datetime(2020, 1, 1), test_method) # enqueued_jobs = client.application.redis.zrange( # b"rq:scheduler:scheduled_jobs", 0, -1 # ) # expect(enqueued_jobs).to_length(1) # enqueued_job_id = enqueued_jobs[0].decode("utf-8") task, job, execution = JobExecutionFixture.new_defaults( status=JobExecution.Status.running ) # job.metadata["enqueued_id"] = enqueued_job_id job.metadata["retries"] = 3 job.metadata["retry_count"] = 0 job.save() executor_mock = MagicMock() client.application.executor = executor_mock resp = client.post( f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/stop/" ) expect(resp.status_code).to_equal(200) obj = loads(resp.data) expect(obj).to_be_like( { "execution": { "id": execution.execution_id, "url": ( f"http://localhost:10000/tasks/{task.task_id}/jobs/" f"{job.job_id}/executions/{execution.execution_id}/" ), }, "job": { "id": job.job_id, "url": f"http://localhost:10000/tasks/{task.task_id}/jobs/{job.job_id}/", }, "task": { "id": task.task_id, "url": f"http://localhost:10000/tasks/{task.task_id}/", }, } ) executor_mock.stop_job.assert_called() job.reload() expect(job.metadata["retry_count"]).to_equal(4)
def test_validate_expiration1(worker): """ Test validating the expiration of a Job returns True if the job has no expiration. """ app = worker.app.app with app.app_context(): task, job, execution = JobExecutionFixture.new_defaults() result = job_mod.validate_expiration(job, execution, app.logger) expect(result).to_be_true()
def test_job_logs2(client): """Tests get job logs fails if invalid input.""" task, job, execution = JobExecutionFixture.new_defaults() resp1 = client.get(f"/tasks/{task.task_id}/jobs/invalid-id/logs/") expect(resp1.status_code).to_equal(404) obj = loads(resp1.data) expect(obj["error"]).to_equal( f"Task ({task.task_id}) or Job (invalid-id) not found.") expect(obj["operation"]).to_equal("retrieve_execution_details")
def test_job_stderr1(client): """Tests get job stderr returns log for last execution.""" task, job, execution = JobExecutionFixture.new_defaults() execution.error = "test error" execution.status = JobExecution.Status.done execution.save() resp1 = client.get(f"/tasks/{task.task_id}/jobs/{job.job_id}/stderr/") expect(resp1.status_code).to_equal(200) expect(resp1.data).to_equal("test error")
def test_monitor_job_with_retry2(client): """Test monitoring a job for a task that fails stops after max retries""" with client.application.app_context(): app = client.application app.redis.flushall() task, job, execution = JobExecutionFixture.new_defaults() job.metadata["retries"] = 3 job.metadata["retry_count"] = 3 job.save() job_id = job.job_id exec_mock = MagicMock() exec_mock.get_result.return_value = MagicMock( exit_code=1, log="".encode("utf-8"), error="error".encode("utf-8")) client.application.executor = exec_mock queue = Queue("monitor", is_async=False, connection=client.application.redis) result = queue.enqueue(job_mod.monitor_job, task.task_id, job_id, execution.execution_id) worker = SimpleWorker([queue], connection=queue.connection) worker.work(burst=True) task.reload() expect(task.jobs).to_length(1) job = task.jobs[0] expect(job.executions).to_length(1) execution = job.executions[0] expect(execution.image).to_equal("image") expect(execution.command).to_equal("command") hash_key = f"rq:job:{result.id}" res = app.redis.exists(hash_key) expect(res).to_be_true() res = app.redis.hget(hash_key, "status") expect(res).to_equal("finished") res = app.redis.hexists(hash_key, "data") expect(res).to_be_true() keys = app.redis.keys() next_job_id = [ key for key in keys if key.decode("utf-8").startswith("rq:job") and not key.decode("utf-8").endswith(result.id) ] expect(next_job_id).to_length(0)
def verify_get_result(client, status, exit_code, stdout, stderr, custom_error, started_at, finished_at): app = client.application with app.app_context(): container_mock = ContainerFixture.new_with_status( container_id="fastlane-job-123", name="fastlane-job-123", status=status, exit_code=exit_code, started_at=started_at, finished_at=finished_at, custom_error=custom_error, stdout=stdout, stderr=stderr, ) _, pool_mock, _ = PoolFixture.new_defaults(r"test[-].+", max_running=1, containers=[container_mock]) executor = Executor(app, pool_mock) _, job, execution = JobExecutionFixture.new_defaults( container_id="fastlane-job-123") result = executor.get_result(job.task, job, execution) expect(result.status).to_equal(STATUS.get(status)) expect(result.exit_code).to_equal(exit_code) if stdout is None: expect(result.log).to_be_empty() else: expect(result.log).to_equal(stdout) if stderr is not None and custom_error != "": expect( result.error).to_equal(f"{custom_error}\n\nstderr:\n{stderr}") else: if stderr is not None: expect(result.error).to_equal(stderr) else: expect(result.error).to_equal(custom_error) parsed_started_at = parse(started_at) expect(result.started_at).to_equal(parsed_started_at) if finished_at is not None: parsed_finished_at = parse(finished_at) else: parsed_finished_at = finished_at expect(result.finished_at).to_equal(parsed_finished_at)
def test_validate_expiration2(worker): """ Test validating the expiration of a Job returns True if the job has expiration but not expired. """ app = worker.app.app with app.app_context(): task, job, execution = JobExecutionFixture.new_defaults() job.metadata["expiration"] = unix_now() + 10 job.save() result = job_mod.validate_expiration(job, execution, app.logger) expect(result).to_be_true()
def test_get_execution1(client): """Test get execution details""" with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults( status=JobExecution.Status.done ) resp = client.get( f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/" ) expect(resp.status_code).to_equal(200) task_url = url_for("task.get_task", task_id=str(task.task_id), _external=True) data = loads(resp.data) expect(data).to_include("task") expect(data["task"]).to_include("id") expect(data["task"]).to_include("url") expect(data["task"]["id"]).to_equal(task.task_id) expect(data["task"]["url"]).to_equal(task_url) job_url = url_for( "task.get_job", task_id=str(task.task_id), job_id=str(job.job_id), _external=True, ) expect(data).to_include("job") expect(data["job"]).to_include("id") expect(data["job"]).to_include("url") expect(data["job"]["id"]).to_equal(job.job_id) expect(data["job"]["url"]).to_equal(job_url) expect(data["execution"]["createdAt"]).not_to_be_null() del data["execution"]["createdAt"] expect(data["execution"]).to_be_like( { "command": "command", "error": None, "executionId": execution.execution_id, "exitCode": None, "finishedAt": None, "image": "image", "log": None, "metadata": execution.metadata, "requestIPAddress": None, "startedAt": None, "status": "done", } )
def test_get_execution_stdout1(client): """Test getting job execution stdout""" with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults( exit_code=0, log="test log", error="some error", status=JobExecution.Status.done, ) resp = client.get( f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/stdout/" ) expect(resp.status_code).to_equal(200) expect(resp.data).to_equal("test log")
def test_get_execution_logs2(client): """Test getting job execution logs returns empty if job is running""" with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults( exit_code=0, log="test log", error="some error", status=JobExecution.Status.running, ) resp = client.get( f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/logs/" ) expect(resp.status_code).to_equal(200) expect(resp.data).to_be_empty()
def test_pull1(client): """Tests that a docker executor can pull images""" with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults() match, pool_mock, client_mock = PoolFixture.new_defaults(r"test-.+") exe = Executor(app=client.application, pool=pool_mock) exe.update_image(task, job, execution, "mock-image", "latest", blacklisted_hosts=set()) expect(client_mock.images.pull.call_count).to_equal(1) client_mock.images.pull.assert_called_with("mock-image", tag="latest")
def test_run_container1(worker): """ Test running a container works and returns True """ app = worker.app.app with app.app_context(): task, job, execution = JobExecutionFixture.new_defaults() exec_mock = MagicMock() result = job_mod.run_container( exec_mock, job, execution, job.image, "latest", job.command, app.logger ) expect(result).to_be_true() exec_mock.run.assert_called_with( task, job, execution, job.image, "latest", job.command ) expect(execution.started_at).not_to_be_null() expect(execution.status).to_equal(JobExecution.Status.running)
def test_stop2(client): """ Tests stopping a job stops fails if container is not in metadata """ app = client.application with app.app_context(): _, pool_mock, _ = PoolFixture.new_defaults(r"test[-].+", max_running=1, containers=[]) task, job, execution = JobExecutionFixture.new_defaults() del execution.metadata["container_id"] executor = Executor(app, pool_mock) result = executor.stop_job(task, job, execution) expect(result).to_be_false()
def test_stop_execution1(client): with client.application.app_context(): task, job, execution = JobExecutionFixture.new_defaults( status=JobExecution.Status.running) job.metadata["retries"] = 3 job.metadata["retry_count"] = 0 job.save() executor_mock = MagicMock() client.application.executor = executor_mock resp = client.post( f"/tasks/{task.task_id}/jobs/{job.job_id}/executions/{execution.execution_id}/stop/" ) expect(resp.status_code).to_equal(200) obj = loads(resp.data) expect(obj).to_be_like({ "execution": { "id": execution.execution_id, "url": (f"http://localhost:10000/tasks/{task.task_id}/jobs/" f"{job.job_id}/executions/{execution.execution_id}/"), }, "job": { "id": job.job_id, "url": f"http://localhost:10000/tasks/{task.task_id}/jobs/{job.job_id}/", }, "task": { "id": task.task_id, "url": f"http://localhost:10000/tasks/{task.task_id}/", }, }) executor_mock.stop_job.assert_called() job.reload() expect(job.metadata["retry_count"]).to_equal(4) enqueued_jobs = client.application.redis.zcard( Queue.SCHEDULED_QUEUE_NAME) expect(enqueued_jobs).to_equal(0)
def test_monitor_job1(worker): """Test monitoring a job with result already there""" app = worker.app.app with app.app_context(): app.redis.flushall() task, job, execution = JobExecutionFixture.new_defaults() execution.status = JobExecution.Status.running job.save() job_id = str(job.job_id) exec_mock = MagicMock() exec_mock.get_result.return_value = MagicMock( exit_code=0, log="qwe".encode("utf-8"), error="".encode("utf-8") ) app.executor = exec_mock monitor_queue = app.monitor_queue monitor_queue.enqueue( Categories.Monitor, task.task_id, job_id, execution.execution_id ) worker.loop_once() monitor_queue.enqueue( Categories.Monitor, task.task_id, job_id, execution.execution_id ) worker.loop_once() task.reload() expect(task.jobs).to_length(1) job = task.jobs[0] expect(job.executions).to_length(1) execution = job.executions[0] expect(execution.image).to_equal("image") expect(execution.command).to_equal("command") expect(execution.status).to_equal(JobExecution.Status.done) expect(app.redis.zcard(Queue.SCHEDULED_QUEUE_NAME)).to_equal(0)
def test_monitor_job_with_retry(worker): """Test monitoring a job for a task that fails""" app = worker.app.app with app.app_context(): app.redis.flushall() task, job, execution = JobExecutionFixture.new_defaults() execution.status = JobExecution.Status.running job.metadata["retries"] = 3 job.metadata["retry_count"] = 0 execution.save() job.save() job_id = str(job.job_id) exec_mock = MagicMock() exec_mock.get_result.return_value = MagicMock( exit_code=1, log="".encode("utf-8"), error="error".encode("utf-8") ) app.executor = exec_mock monitor_queue = app.monitor_queue monitor_queue.enqueue( Categories.Monitor, task.task_id, job_id, execution.execution_id ) worker.loop_once() task.reload() expect(task.jobs).to_length(1) job = task.jobs[0] expect(job.executions).to_length(2) execution = job.executions[0] expect(execution.image).to_equal("image") expect(execution.command).to_equal("command") expect(app.redis.zcard(Queue.SCHEDULED_QUEUE_NAME)).to_equal(1) task.reload() expect(task.jobs[0].executions[0].status).to_equal(JobExecution.Status.failed)