def test_pipelinerun_delete_one_run( client, celery, pipeline, abortable_async_res, ): # Multiple parameters so that the job consists of multiple pipeline # runs. job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid, parameters=[{}, {}, {}]) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put(f"/api/jobs/{job_uuid}", json={"confirm_draft": True}) pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] # Cancel the first run, leave the other 2. assert (client.delete(f'/api/jobs/{job_uuid}/{pipeline_runs[0]["uuid"]}', ).status_code == 200) pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] assert pipeline_runs[0]["status"] == "ABORTED" assert pipeline_runs[1]["status"] == "PENDING" assert pipeline_runs[2]["status"] == "PENDING" assert client.get( f"/api/jobs/{job_uuid}").get_json()["status"] == "STARTED"
def test_jobdeletion_delete(client, celery, pipeline): job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.delete(f"/api/jobs/cleanup/{job_uuid}") assert client.get(f"/api/jobs/{job_uuid}").status_code == 404
def test_joblist_post( client, pipeline, proj_env_variables, pipe_env_variables, cron_schedule, scheduled_start, ): project = pipeline.project client.put(f"/api/projects/{project.uuid}", json={"env_variables": proj_env_variables}) client.put( f"/api/pipelines/{project.uuid}/{pipeline.uuid}", json={"env_variables": pipe_env_variables}, ) job_spec = create_job_spec(project.uuid, pipeline.uuid, cron_schedule, scheduled_start) resp = client.post("/api/jobs/", json=job_spec) expect_error = ((cron_schedule is not None and scheduled_start is not None) or cron_schedule == "invalid string" or scheduled_start == "invalid string") expected_code = 201 if not expect_error else 500 assert resp.status_code == expected_code if not expect_error: data = resp.get_json() expected_env_vars = {**proj_env_variables, **pipe_env_variables} assert data["env_variables"] == expected_env_vars
def test_pipelinerun_delete_all_runs( client, celery, pipeline, abortable_async_res, ): # Multiple parameters so that the job consists of multiple pipeline # runs. job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid, parameters=[{}, {}, {}]) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put(f"/api/jobs/{job_uuid}", json={"confirm_draft": True}) pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] for run in pipeline_runs: assert (client.delete( f'/api/jobs/{job_uuid}/{run["uuid"]}', ).status_code == 200) pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] assert client.get( f"/api/jobs/{job_uuid}").get_json()["status"] == "SUCCESS"
def test_job_delete_draft_job(client, celery, pipeline, abortable_async_res): job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.delete(f"/api/jobs/{job_uuid}") job = client.get(f"/api/jobs/{job_uuid}").get_json() assert job["status"] == "ABORTED" assert not job["pipeline_runs"]
def test_delete_non_retained_job_pipeline_runs_on_job_run_update_retain_n( max_retained_pipeline_runs, test_app, client, celery, pipeline, abortable_async_res, monkeypatch, mocker, ): # Multiple parameters so that the job consists of multiple pipeline # runs. job_spec = create_job_spec( pipeline.project.uuid, pipeline.uuid, parameters=[{}, {}, {}], max_retained_pipeline_runs=max_retained_pipeline_runs, ) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put( f"/api/jobs/{job_uuid}", json={ "confirm_draft": True, "cron_schedule": "* * * * *" }, ) # Trigger a job run. with test_app.app_context(): with TwoPhaseExecutor(db.session) as tpe: namespace_jobs.RunJob(tpe).transaction(job_uuid) # Set as done the pipeline runs of this job run. pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] for run in pipeline_runs: client.put( f'/api/jobs/{job_uuid}/{run["uuid"]}', json={ "status": "SUCCESS", "finished_time": datetime.datetime.now().isoformat(), }, ) expected_deleted_runs_n = max(0, 3 - max_retained_pipeline_runs) pipeline_runs.sort(key=lambda x: x["pipeline_run_index"]) expected_deleted_run_uuids = set( [run["uuid"] for run in pipeline_runs[:expected_deleted_runs_n]]) deleted_run_uuids = set([ uuid for task in celery.tasks if task[1]["name"] == "app.core.tasks.delete_job_pipeline_run_directories" for uuid in task[1]["kwargs"]["pipeline_run_uuids"] ]) assert expected_deleted_run_uuids == deleted_run_uuids
def test_joblist_get(client, pipeline): job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) client.post("/api/jobs/", json=job_spec) job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) resp = client.post("/api/jobs/", json=job_spec) for query_string, expected_length in [ ({}, 2), ({ "project_uuid": pipeline.project.uuid }, 2), ({ "project_uuid": "proj" }, 0), ]: resp = client.get("/api/jobs/", query_string=query_string) data = resp.get_json() assert len(data["jobs"]) == expected_length for job in data["jobs"]: assert job["env_variables"] is None
def test_pipelinerun_get(client, celery, pipeline): job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put(f"/api/jobs/{job_uuid}", json={"confirm_draft": True}) job = client.get(f"/api/jobs/{job_uuid}").get_json() for run in job["pipeline_runs"]: resp = client.get(f'/api/jobs/{job_uuid}/{run["uuid"]}') assert resp.status_code == 200 run = resp.get_json() assert run["env_variables"] is not None
def test_delete_non_retained_job_pipeline_runs_on_job_run_retain_all( test_app, client, celery, pipeline, abortable_async_res, monkeypatch, mocker): # Multiple parameters so that the job consists of multiple pipeline # runs. job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid, parameters=[{}, {}, {}]) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put( f"/api/jobs/{job_uuid}", json={ "confirm_draft": True, "cron_schedule": "* * * * *" }, ) # Trigger a job run. with test_app.app_context(): with TwoPhaseExecutor(db.session) as tpe: namespace_jobs.RunJob(tpe).transaction(job_uuid) # Set as done the pipeline runs of this job run. pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] for run in pipeline_runs: client.put( f'/api/jobs/{job_uuid}/{run["uuid"]}', json={ "status": "SUCCESS", "finished_time": datetime.datetime.now().isoformat(), }, ) pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] for run in pipeline_runs: assert run["status"] == "SUCCESS" # Trigger another job run. with test_app.app_context(): with TwoPhaseExecutor(db.session) as tpe: namespace_jobs.RunJob(tpe).transaction(job_uuid) # The previously existing pipeline runs should still be there. pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] assert len(pipeline_runs) == 6 assert all([ task[1]["name"] != "app.core.tasks.delete_job_pipeline_run_directories" for task in celery.tasks ])
def test_job_put_revert(client, pipeline, monkeypatch): job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] # Cause an exception so that running the job fails. monkeypatch.setattr(namespace_jobs, "lock_environment_images_for_run", raise_exception_function()) resp = client.put(f"/api/jobs/{job_uuid}", json={"confirm_draft": True}) assert resp.status_code == 500 job = client.get(f"/api/jobs/{job_uuid}").get_json() assert job["status"] == "FAILURE" pipeline_runs = job["pipeline_runs"] for run in pipeline_runs: assert run["status"] == "FAILURE"
def test_job_delete_running_job( client, celery, pipeline, abortable_async_res, ): job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put(f"/api/jobs/{job_uuid}", json={"confirm_draft": True}) client.delete(f"/api/jobs/{job_uuid}") job = client.get(f"/api/jobs/{job_uuid}").get_json() assert job["status"] == "ABORTED" assert job["pipeline_runs"] for run in job["pipeline_runs"]: assert run["status"] == "ABORTED" for step in run["pipeline_steps"]: assert step["status"] == "ABORTED"
def test_job_get_exist(client, pipeline): project = pipeline.project proj_env_variables = {"var1": "1", "var2": "2"} client.put(f"/api/projects/{project.uuid}", json={"env_variables": proj_env_variables}) pipe_env_variables = {"var2": '["hello"]', "var3": "{}"} client.put( f"/api/pipelines/{project.uuid}/{pipeline.uuid}", json={"env_variables": pipe_env_variables}, ) job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] resp = client.get(f"/api/jobs/{job_uuid}") assert resp.status_code == 200 expected_env_vars = {**proj_env_variables, **pipe_env_variables} assert resp.get_json()["env_variables"] == expected_env_vars
def test_pipelinerundeletion_one_run( client, celery, pipeline, abortable_async_res, ): # Multiple parameters so that the job consists of multiple pipeline # runs. job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid, parameters=[{}, {}, {}]) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put(f"/api/jobs/{job_uuid}", json={"confirm_draft": True}) pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] # Cancel the first run, leave the other 2. assert (client.delete( f'/api/jobs/cleanup/{job_uuid}/{pipeline_runs[0]["uuid"]}', ).status_code == 200) celery_task_kwargs = { "project_uuid": pipeline.project.uuid, "pipeline_uuid": pipeline.uuid, "job_uuid": job_uuid, "pipeline_run_uuids": [pipeline_runs[0]["uuid"]], } assert any( [task[1]["kwargs"] == celery_task_kwargs for task in celery.tasks]) pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] assert len(pipeline_runs) == 2 assert pipeline_runs[0]["status"] == "PENDING" assert pipeline_runs[1]["status"] == "PENDING" assert client.get( f"/api/jobs/{job_uuid}").get_json()["status"] == "STARTED"
def test_pipelinerundeletion_all_runs( client, celery, pipeline, abortable_async_res, ): # Multiple parameters so that the job consists of multiple pipeline # runs. job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid, parameters=[{}, {}, {}]) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put(f"/api/jobs/{job_uuid}", json={"confirm_draft": True}) pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] for run in pipeline_runs: assert (client.delete( f'/api/jobs/cleanup/{job_uuid}/{run["uuid"]}', ).status_code == 200 ) for run in pipeline_runs: celery_task_kwargs = { "project_uuid": pipeline.project.uuid, "pipeline_uuid": pipeline.uuid, "job_uuid": job_uuid, "pipeline_run_uuids": [run["uuid"]], } assert any( [task[1]["kwargs"] == celery_task_kwargs for task in celery.tasks]) pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] assert not pipeline_runs assert client.get( f"/api/jobs/{job_uuid}").get_json()["status"] == "SUCCESS"
def test_pipelinerun_set(client, celery, pipeline): # Multiple parameters so that the job consists of multiple pipeline # runs. job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid, parameters=[{}, {}, {}]) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put(f"/api/jobs/{job_uuid}", json={"confirm_draft": True}) pipeline_runs = client.get( f"/api/jobs/{job_uuid}/pipeline_runs").get_json()["pipeline_runs"] print(pipeline_runs) for run in pipeline_runs: client.put( f'/api/jobs/{job_uuid}/{run["uuid"]}', json={ "status": "SUCCESS", "finished_time": datetime.datetime.now().isoformat(), }, ) assert client.get( f"/api/jobs/{job_uuid}").get_json()["status"] == "SUCCESS"
def test_job_put_on_non_draft_non_cronjob(client, pipeline): job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put( f"/api/jobs/{job_uuid}", json={ # So that it won't run immediately. "next_scheduled_time": datetime.datetime.now().isoformat(), "confirm_draft": True, }, ) for property in [ "env_variables", "cron_schedule", "parameters", "next_scheduled_time", "strategy_json", ]: job_update = {property: "test"} resp = client.put(f"/api/jobs/{job_uuid}", json=job_update) assert resp.status_code == 500
def test_job_put_on_non_draft_cronjob(client, pipeline): job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] client.put( f"/api/jobs/{job_uuid}", json={ # So that it won't run immediately. "cron_schedule": "* * * * *", "confirm_draft": True, }, ) for property, value, expected_code in [ ("env_variables", {}, 200), ("cron_schedule", "1 * * * * ", 200), ("parameters", [{}], 200), ("next_scheduled_time", datetime.datetime.now().isoformat(), 500), ("strategy_json", {}, 200), ]: job_update = {property: value} resp = client.put(f"/api/jobs/{job_uuid}", json=job_update) assert resp.status_code == expected_code
def test_job_put_on_draft( client, celery, pipeline, env_variables, cron_schedule, next_scheduled_time, parameters, ): job_spec = create_job_spec(pipeline.project.uuid, pipeline.uuid) job_uuid = client.post("/api/jobs/", json=job_spec).get_json()["uuid"] job_update = { "env_variables": env_variables, "cron_schedule": cron_schedule, "next_scheduled_time": next_scheduled_time, "parameters": parameters, "confirm_draft": True, } resp = client.put(f"/api/jobs/{job_uuid}", json=job_update) expect_error = ((cron_schedule is not None and next_scheduled_time is not None) or cron_schedule == "invalid string" or next_scheduled_time == "invalid string") expected_code = 200 if not expect_error else 500 assert resp.status_code == expected_code # Job that need to run immediately have both set to None, while cron # jobs have a cron_schedule. Jobs that are to be scheduled in the # future won't trigger this condition. runs_now = cron_schedule is not None or next_scheduled_time is None if not expect_error and runs_now: # For a matter of comparison. if env_variables is None: env_variables = {} job = client.get(f"/api/jobs/{job_uuid}").get_json() # Runs of a job that has to start right now should have been # added to celery. if cron_schedule is None: assert celery.tasks assert len(celery.tasks) == len(parameters) for _, task_kwargs in celery.tasks: assert (task_kwargs["kwargs"]["run_config"] ["user_env_variables"] == env_variables) pipeline_runs = client.get(f"/api/jobs/{job_uuid}/pipeline_runs" ).get_json()["pipeline_runs"] assert len(pipeline_runs) == len(parameters) # Note that the reversed is there because the runs are # sorted descendingly. for run, run_parameters in zip(pipeline_runs, reversed(parameters)): assert run["parameters"] == run_parameters assert run["status"] == "PENDING" assert job["status"] == "STARTED" assert job["env_variables"] == env_variables