def test_get_pipeline_run_output(client, pipeline, client_application, mock_execute_pipeline): db.session.commit() result = client.get( "/v1/pipelines/no-id/runs/no-id/console", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 404 # no such pipeline_run_id result = client.get( f"/v1/pipelines/{pipeline.uuid}/runs/no-id/console", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 404 # successfully fetch a pipeline_run pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) pipeline_run.std_out = "stdout" db.session.commit() result = client.get( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/console", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 200 assert result.json == { "std_out": "stdout", "std_err": "", }
def test_remove_pipeline_run(client, pipeline, client_application, mock_execute_pipeline): db.session.commit() pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) db.session.commit() result = client.delete( "/v1/pipelines/no-id/runs/no-id", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 400 # no such pipeline_run_id result = client.delete( f"/v1/pipelines/{pipeline.uuid}/runs/no-id", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 400 # successfully delete a pipeline_run result = client.delete( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}", content_type="application/json", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 200 assert find_pipeline_run(pipeline_run.uuid) is None
def test_start_pipeline_run_bad_state(app, pipeline): pipeline_run = services.create_pipeline_run( pipeline.uuid, {"inputs": [], "callback_url": "http://example.com"}, ) with pytest.raises(ValueError): services.start_pipeline_run(pipeline_run)
def test_find_pipeline_run_is_deleted(app, pipeline, mock_execute_pipeline): pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) # does not return a pipeline run if the pipeline run is soft-deleted pipeline_run.is_deleted = True db.session.commit() assert find_pipeline_run(pipeline_run.uuid) is None
def test_create_pipeline_run_invalid_org(mock_url, app, organization_pipeline, organization_pipeline_input_file): mock_url.return_value = "http://somefileurl.com" json_request = {"inputs": []} with pytest.raises(ValueError): created_pipeline_run = create_pipeline_run("12345", "12345", json_request)
def test_copy_pipeline_run_artifact( urlopen_mock, create_url_mock, upload_stream_mock, pipeline ): create_url_mock.return_value = "http://example.com/presigned" urlopen_mock.return_value = io.BytesIO(b"this is data") another_run = services.create_pipeline_run( pipeline.uuid, VALID_CALLBACK_INPUT, True ) pipeline_run = services.create_pipeline_run( pipeline.uuid, VALID_CALLBACK_INPUT, True ) artifact = PipelineRunArtifact(name="ex.csv", pipeline_run=pipeline_run) db.session.add(artifact) db.session.commit() services.copy_pipeline_run_artifact(artifact, another_run) assert len(another_run.pipeline_run_inputs) == 1 assert another_run.pipeline_run_inputs[0].filename == "ex.csv" assert another_run.pipeline_run_inputs[0].url == "http://example.com/presigned"
def test_update_pipeline_run_state_bad_transition(app, pipeline, mock_execute_pipeline): pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) with pytest.raises(ValueError): services.update_pipeline_run_state( pipeline_run.uuid, { "state": RunStateEnum.COMPLETED.name, }, ) assert len(pipeline_run.pipeline_run_states) == 2
def test_update_pipeline_run_state_bad_state(app, pipeline, mock_execute_pipeline): pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) with pytest.raises(ValidationError): services.update_pipeline_run_state( pipeline_run.uuid, { "state": "fake", }, ) assert len(pipeline_run.pipeline_run_states) == 2
def test_update_pipeline_run_state( client, pipeline, worker_application, mock_execute_pipeline ): db.session.commit() pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) result = client.put( "/v1/pipelines/no-id/runs/no-id/state", content_type="application/json", json={"state": RunStateEnum.RUNNING.name}, headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 404 # no such pipeline_run_id result = client.put( f"/v1/pipelines/{pipeline.uuid}/runs/no-id/state", content_type="application/json", json={"state": RunStateEnum.RUNNING.name}, headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 404 # Bad state result = client.put( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/state", content_type="application/json", json={"state": "badstate"}, headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 400 # Bad state transition result = client.put( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/state", content_type="application/json", json={"state": RunStateEnum.FAILED.name}, headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 400 # successfully fetch a pipeline_run result = client.put( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/state", content_type="application/json", json={"state": RunStateEnum.RUNNING.name}, headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 200 assert len(pipeline_run.pipeline_run_states) == 3 assert pipeline_run.run_state_enum() == RunStateEnum.RUNNING
def test_update_pipeline_run_state_no_callback_url( urlopen_mock, app, monkeypatch, pipeline, mock_execute_pipeline ): pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) pipeline_run.callback_url = "" db.session.commit() services.update_pipeline_run_state( pipeline_run.uuid, { "state": RunStateEnum.RUNNING.name, }, ) assert len(pipeline_run.pipeline_run_states) == 3 assert pipeline_run.run_state_enum() == RunStateEnum.RUNNING assert not urlopen_mock.called
def test_list_pipeline_runs(client, pipeline, client_application, mock_execute_pipeline): db.session.commit() result = client.get( f"/v1/pipelines/no-id/runs", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 404 result = client.get( f"/v1/pipelines/{pipeline.uuid}/runs", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 200 assert result.json == [] # successfully fetch a pipeline_run pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) result = client.get( f"/v1/pipelines/{pipeline.uuid}/runs", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 200 assert result.json == [{ "uuid": pipeline_run.uuid, "sequence": pipeline_run.sequence, "created_at": to_iso8601(pipeline_run.created_at), "inputs": [], "states": [ { "state": RunStateEnum.QUEUED.name, "created_at": to_iso8601(pipeline_run.pipeline_run_states[0].created_at), }, { "state": RunStateEnum.NOT_STARTED.name, "created_at": to_iso8601(pipeline_run.pipeline_run_states[1].created_at), }, ], "artifacts": [], }]
def test_upload_run_artifact_service_valueerror( client, monkeypatch, pipeline, worker_application, mock_execute_pipeline ): db.session.commit() pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) def raise_error(*args, **kwargs): raise ValueError("Test error") monkeypatch.setattr(runs_module, "create_pipeline_run_artifact", raise_error) result = client.post( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/artifacts?name=blah.file", data=b"blahblah", headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 400
def test_upload_run_artifact( upload_stream_mock, client, pipeline, worker_application, mock_execute_pipeline ): db.session.commit() pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) # no 'name' query parameter. result = client.post( "/v1/pipelines/no-id/runs/no-id/artifacts", data=b"blahblah", headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 400 result = client.post( "/v1/pipelines/no-id/runs/no-id/artifacts?name=blah.file", data=b"blahblah", headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 404 # no such pipeline_run_id result = client.post( f"/v1/pipelines/{pipeline.uuid}/runs/no-id/artifacts?name=blah.file", data=b"blahblah", headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 404 # When the filename is too long, an error is returned. result = client.post( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/artifacts?name=" + (256 * "X"), data=b"blahblah", headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 400 result = client.post( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/artifacts?name=blah.file", data=b"blahblah", headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 200
def test_create_pipeline_run_response_error(mock_url, app, organization_pipeline, organization_pipeline_input_file): mock_url.return_value = "http://somefileurl.com" json_response = dict(PIPELINE_RUN_RESPONSE_JSON) responses.add( responses.POST, f"{app.config[WORKFLOW_HOSTNAME]}/v1/pipelines/{organization_pipeline.pipeline_uuid}/runs", status=503, ) with pytest.raises(HTTPError): created_pipeline_run = create_pipeline_run( organization_pipeline.organization_uuid, organization_pipeline.uuid, PIPELINE_RUN_JSON, )
def test_create_pipeline_run_missing_pipeline( mock_url, app, organization_pipeline, organization_pipeline_input_file): mock_url.return_value = "http://somefileurl.com" json_response = dict(PIPELINE_RUN_RESPONSE_JSON) pipeline = OrganizationPipeline.query.order_by( OrganizationPipeline.id.desc()).first() responses.add( responses.POST, f"{app.config[WORKFLOW_HOSTNAME]}/v1/pipelines/{pipeline.pipeline_uuid}/runs", json=json_response, ) with pytest.raises(ValueError): created_pipeline_run = create_pipeline_run(pipeline.organization_uuid, "1234", PIPELINE_RUN_JSON)
def test_update_pipeline_run_output( client, pipeline, worker_application, mock_execute_pipeline ): db.session.commit() pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) result = client.put( "/v1/pipelines/no-id/runs/no-id/console", content_type="application/json", json={ "std_out": "stdout", "std_err": "stderr", }, headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 404 # no such pipeline_run_id result = client.put( f"/v1/pipelines/{pipeline.uuid}/runs/no-id/console", content_type="application/json", json={ "std_out": "stdout", "std_err": "stderr", }, headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 404 # successfully fetch a pipeline_run result = client.put( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/console", content_type="application/json", json={ "std_out": "stdout", "std_err": "stderr", }, headers={ROLES_KEY: worker_application.api_key}, ) assert result.status_code == 200 assert pipeline_run.std_out == "stdout" assert pipeline_run.std_err == "stderr"
def test_update_pipeline_run_state_callback_err( app, monkeypatch, pipeline, mock_execute_pipeline ): pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) # If a callback_url fails for some network reason, the update should still # work: def mock_urlopen(request, timeout): raise URLError("a reason") monkeypatch.setattr(services.urllib_request, "urlopen", mock_urlopen) services.update_pipeline_run_state( pipeline_run.uuid, { "state": RunStateEnum.RUNNING.name, }, ) assert len(pipeline_run.pipeline_run_states) == 3 assert pipeline_run.run_state_enum() == RunStateEnum.RUNNING
def test_create_queued_pipeline_run(app, pipeline): input1 = { "name": "name1.pdf", "url": "https://example.com/name1.pdf", } input2 = { "name": "name2.pdf", "url": "https://example.com/name2.pdf", } pipeline_run = services.create_pipeline_run( pipeline.uuid, {"inputs": [input1, input2], "callback_url": "http://example.com"}, True, ) assert pipeline_run.pipeline == pipeline assert pipeline_run.sequence == 1 assert len(pipeline_run.pipeline_run_inputs) == 2 assert pipeline_run.pipeline_run_inputs[0].filename == input1["name"] assert pipeline_run.pipeline_run_inputs[1].filename == input2["name"] assert len(pipeline_run.pipeline_run_states) == 1 assert pipeline_run.pipeline_run_states[0].code == RunStateEnum.QUEUED
def test_update_pipeline_run_state(app, monkeypatch, pipeline, mock_execute_pipeline): pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) # A callback is made with the callback_url and the correct payload def mock_urlopen(request, timeout): assert request.full_url == pipeline_run.callback_url assert json.loads(request.data.decode()) == { "pipeline_run_uuid": pipeline_run.uuid, "state": RunStateEnum.RUNNING.name, } assert timeout == CALLBACK_TIMEOUT monkeypatch.setattr(services.urllib_request, "urlopen", mock_urlopen) services.update_pipeline_run_state( pipeline_run.uuid, { "state": RunStateEnum.RUNNING.name, }, ) assert len(pipeline_run.pipeline_run_states) == 3 assert pipeline_run.run_state_enum() == RunStateEnum.RUNNING
def create_workflow_run(workflow_uuid, run_json): """ Create a new WorkflowRun """ data = CreateRunSchema().load(run_json) workflow = find_workflow(workflow_uuid) if workflow is None: raise ValueError("no workflow found") workflow_run = WorkflowRun(workflow=workflow) workflow_run.workflow_run_states.append( WorkflowRunState( run_state_type=find_run_state_type(RunStateEnum.NOT_STARTED))) added_run = False for workflow_pipeline in workflow.workflow_pipelines: if workflow_pipeline.is_deleted: continue queue_run = len(workflow_pipeline.source_workflow_pipelines) > 0 no_input_data = {"callback_url": data["callback_url"], "inputs": []} run_data = no_input_data if queue_run else data pipeline_run = create_pipeline_run(workflow_pipeline.pipeline.uuid, run_data, queue_run) workflow_pipeline_run = WorkflowPipelineRun( workflow_run=workflow_run, pipeline_run=pipeline_run, workflow_pipeline=workflow_pipeline, ) db.session.add(workflow_pipeline_run) added_run = True if not added_run: db.session.rollback() raise ValueError("No WorkflowPipelines exist!") db.session.add(workflow_run) db.session.commit() return workflow_run
def test_create_pipeline_run(mock_url, app, organization_pipeline, organization_pipeline_input_file): json_response = dict(PIPELINE_RUN_RESPONSE_JSON) mock_url.return_value = "http://somefileurl.com" pipeline = OrganizationPipeline.query.order_by( OrganizationPipeline.id.desc()).first() responses.add( responses.POST, f"{app.config[WORKFLOW_HOSTNAME]}/v1/pipelines/{pipeline.pipeline_uuid}/runs", json=json_response, ) created_pipeline_run = create_pipeline_run(pipeline.organization_uuid, pipeline.uuid, PIPELINE_RUN_JSON) new_run = OrganizationPipelineRun.query.filter( OrganizationPipelineRun.pipeline_run_uuid == created_pipeline_run["uuid"]).first() assert new_run is not None assert created_pipeline_run == json_response
def test_get_pipeline_run(client, pipeline, client_application, mock_execute_pipeline): db.session.commit() pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) artifact = PipelineRunArtifact(name="test.pdf") artifact.public_url = Mock() artifact.public_url.return_value = "http://fake.example.com/url" pipeline_run.pipeline_run_artifacts.append(artifact) db.session.commit() result = client.get( "/v1/pipelines/no-id/runs/no-id", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 404 # no such pipeline_run_id result = client.get( f"/v1/pipelines/{pipeline.uuid}/runs/no-id", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 404 # successfully fetch a pipeline_run result = client.get( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 200 assert result.json == { "uuid": pipeline_run.uuid, "sequence": pipeline_run.sequence, "created_at": to_iso8601(pipeline_run.created_at), "inputs": [], "states": [ { "state": RunStateEnum.QUEUED.name, "created_at": to_iso8601(pipeline_run.pipeline_run_states[0].created_at), }, { "state": RunStateEnum.NOT_STARTED.name, "created_at": to_iso8601(pipeline_run.pipeline_run_states[1].created_at), }, ], "artifacts": [{ "uuid": artifact.uuid, "name": "test.pdf", "url": "http://fake.example.com/url", }], } # fails if the pipeline is deleted. pipeline.is_deleted = True db.session.commit() result = client.get( f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}", headers={ROLES_KEY: client_application.api_key}, ) assert result.status_code == 404
def test_update_workflow_run_no_workflow(execute_pipeline_mock, app, pipeline): # a pipeline_run not associated with workflow_pipeline_run nothing breaks pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) assert services.update_workflow_run(pipeline_run) is None
def test_delete_pipeline_run(app, pipeline): pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) services.delete_pipeline_run(pipeline.uuid, pipeline_run.uuid) assert pipeline_run.is_deleted
def test_delete_pipeline_run_has_workflow(app, pipeline, workflow, workflow_pipeline): pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) with pytest.raises(ValueError): services.delete_pipeline_run(pipeline.uuid, pipeline_run.uuid) assert not pipeline_run.is_deleted
def test_create_pipeline_bad_input(app): with pytest.raises(ValidationError): pipeline_run = services.create_pipeline_run("no-id", INVALID_CALLBACK_INPUT)
def test_update_pipeline_run_output(app, pipeline, mock_execute_pipeline): pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT) services.update_pipeline_run_output(pipeline_run.uuid, "stdout", "stderr") assert pipeline_run.std_out == "stdout" assert pipeline_run.std_err == "stderr"
def test_create_pipeline_run_no_pipeline(app): with pytest.raises(ValueError): pipeline_run = services.create_pipeline_run("no-id", VALID_CALLBACK_INPUT)