def test_get_pipeline_run_output(client, pipeline, client_application,
mock_execute_pipeline):
db.session.commit()
result = client.get(
"/v1/pipelines/no-id/runs/no-id/console",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 404
# no such pipeline_run_id
result = client.get(
f"/v1/pipelines/{pipeline.uuid}/runs/no-id/console",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 404
# successfully fetch a pipeline_run
pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
pipeline_run.std_out = "stdout"
db.session.commit()
result = client.get(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/console",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 200
assert result.json == {
"std_out": "stdout",
"std_err": "",
}
def test_remove_pipeline_run(client, pipeline, client_application,
mock_execute_pipeline):
db.session.commit()
pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
db.session.commit()
result = client.delete(
"/v1/pipelines/no-id/runs/no-id",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 400
# no such pipeline_run_id
result = client.delete(
f"/v1/pipelines/{pipeline.uuid}/runs/no-id",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 400
# successfully delete a pipeline_run
result = client.delete(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}",
content_type="application/json",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 200
assert find_pipeline_run(pipeline_run.uuid) is None
def test_start_pipeline_run_bad_state(app, pipeline):
pipeline_run = services.create_pipeline_run(
pipeline.uuid,
{"inputs": [], "callback_url": "http://example.com"},
)
with pytest.raises(ValueError):
services.start_pipeline_run(pipeline_run)
def test_find_pipeline_run_is_deleted(app, pipeline, mock_execute_pipeline):
pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
# does not return a pipeline run if the pipeline run is soft-deleted
pipeline_run.is_deleted = True
db.session.commit()
assert find_pipeline_run(pipeline_run.uuid) is None
def test_create_pipeline_run_invalid_org(mock_url, app, organization_pipeline,
organization_pipeline_input_file):
mock_url.return_value = "http://somefileurl.com"
json_request = {"inputs": []}
with pytest.raises(ValueError):
created_pipeline_run = create_pipeline_run("12345", "12345",
json_request)
def test_copy_pipeline_run_artifact(
urlopen_mock, create_url_mock, upload_stream_mock, pipeline
):
create_url_mock.return_value = "http://example.com/presigned"
urlopen_mock.return_value = io.BytesIO(b"this is data")
another_run = services.create_pipeline_run(
pipeline.uuid, VALID_CALLBACK_INPUT, True
)
pipeline_run = services.create_pipeline_run(
pipeline.uuid, VALID_CALLBACK_INPUT, True
)
artifact = PipelineRunArtifact(name="ex.csv", pipeline_run=pipeline_run)
db.session.add(artifact)
db.session.commit()
services.copy_pipeline_run_artifact(artifact, another_run)
assert len(another_run.pipeline_run_inputs) == 1
assert another_run.pipeline_run_inputs[0].filename == "ex.csv"
assert another_run.pipeline_run_inputs[0].url == "http://example.com/presigned"
def test_update_pipeline_run_state_bad_transition(app, pipeline, mock_execute_pipeline):
pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
with pytest.raises(ValueError):
services.update_pipeline_run_state(
pipeline_run.uuid,
{
"state": RunStateEnum.COMPLETED.name,
},
)
assert len(pipeline_run.pipeline_run_states) == 2
def test_update_pipeline_run_state_bad_state(app, pipeline, mock_execute_pipeline):
pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
with pytest.raises(ValidationError):
services.update_pipeline_run_state(
pipeline_run.uuid,
{
"state": "fake",
},
)
assert len(pipeline_run.pipeline_run_states) == 2
def test_update_pipeline_run_state(
client, pipeline, worker_application, mock_execute_pipeline
):
db.session.commit()
pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
result = client.put(
"/v1/pipelines/no-id/runs/no-id/state",
content_type="application/json",
json={"state": RunStateEnum.RUNNING.name},
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 404
# no such pipeline_run_id
result = client.put(
f"/v1/pipelines/{pipeline.uuid}/runs/no-id/state",
content_type="application/json",
json={"state": RunStateEnum.RUNNING.name},
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 404
# Bad state
result = client.put(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/state",
content_type="application/json",
json={"state": "badstate"},
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 400
# Bad state transition
result = client.put(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/state",
content_type="application/json",
json={"state": RunStateEnum.FAILED.name},
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 400
# successfully fetch a pipeline_run
result = client.put(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/state",
content_type="application/json",
json={"state": RunStateEnum.RUNNING.name},
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 200
assert len(pipeline_run.pipeline_run_states) == 3
assert pipeline_run.run_state_enum() == RunStateEnum.RUNNING
def test_update_pipeline_run_state_no_callback_url(
urlopen_mock, app, monkeypatch, pipeline, mock_execute_pipeline
):
pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
pipeline_run.callback_url = ""
db.session.commit()
services.update_pipeline_run_state(
pipeline_run.uuid,
{
"state": RunStateEnum.RUNNING.name,
},
)
assert len(pipeline_run.pipeline_run_states) == 3
assert pipeline_run.run_state_enum() == RunStateEnum.RUNNING
assert not urlopen_mock.called
def test_list_pipeline_runs(client, pipeline, client_application,
mock_execute_pipeline):
db.session.commit()
result = client.get(
f"/v1/pipelines/no-id/runs",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 404
result = client.get(
f"/v1/pipelines/{pipeline.uuid}/runs",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 200
assert result.json == []
# successfully fetch a pipeline_run
pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
result = client.get(
f"/v1/pipelines/{pipeline.uuid}/runs",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 200
assert result.json == [{
"uuid":
pipeline_run.uuid,
"sequence":
pipeline_run.sequence,
"created_at":
to_iso8601(pipeline_run.created_at),
"inputs": [],
"states": [
{
"state":
RunStateEnum.QUEUED.name,
"created_at":
to_iso8601(pipeline_run.pipeline_run_states[0].created_at),
},
{
"state":
RunStateEnum.NOT_STARTED.name,
"created_at":
to_iso8601(pipeline_run.pipeline_run_states[1].created_at),
},
],
"artifacts": [],
}]
def test_upload_run_artifact_service_valueerror(
client, monkeypatch, pipeline, worker_application, mock_execute_pipeline
):
db.session.commit()
pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
def raise_error(*args, **kwargs):
raise ValueError("Test error")
monkeypatch.setattr(runs_module, "create_pipeline_run_artifact", raise_error)
result = client.post(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/artifacts?name=blah.file",
data=b"blahblah",
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 400
def test_upload_run_artifact(
upload_stream_mock, client, pipeline, worker_application, mock_execute_pipeline
):
db.session.commit()
pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
# no 'name' query parameter.
result = client.post(
"/v1/pipelines/no-id/runs/no-id/artifacts",
data=b"blahblah",
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 400
result = client.post(
"/v1/pipelines/no-id/runs/no-id/artifacts?name=blah.file",
data=b"blahblah",
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 404
# no such pipeline_run_id
result = client.post(
f"/v1/pipelines/{pipeline.uuid}/runs/no-id/artifacts?name=blah.file",
data=b"blahblah",
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 404
# When the filename is too long, an error is returned.
result = client.post(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/artifacts?name="
+ (256 * "X"),
data=b"blahblah",
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 400
result = client.post(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/artifacts?name=blah.file",
data=b"blahblah",
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 200
def test_create_pipeline_run_response_error(mock_url, app,
organization_pipeline,
organization_pipeline_input_file):
mock_url.return_value = "http://somefileurl.com"
json_response = dict(PIPELINE_RUN_RESPONSE_JSON)
responses.add(
responses.POST,
f"{app.config[WORKFLOW_HOSTNAME]}/v1/pipelines/{organization_pipeline.pipeline_uuid}/runs",
status=503,
)
with pytest.raises(HTTPError):
created_pipeline_run = create_pipeline_run(
organization_pipeline.organization_uuid,
organization_pipeline.uuid,
PIPELINE_RUN_JSON,
)
def test_create_pipeline_run_missing_pipeline(
mock_url, app, organization_pipeline,
organization_pipeline_input_file):
mock_url.return_value = "http://somefileurl.com"
json_response = dict(PIPELINE_RUN_RESPONSE_JSON)
pipeline = OrganizationPipeline.query.order_by(
OrganizationPipeline.id.desc()).first()
responses.add(
responses.POST,
f"{app.config[WORKFLOW_HOSTNAME]}/v1/pipelines/{pipeline.pipeline_uuid}/runs",
json=json_response,
)
with pytest.raises(ValueError):
created_pipeline_run = create_pipeline_run(pipeline.organization_uuid,
"1234", PIPELINE_RUN_JSON)
def test_update_pipeline_run_output(
client, pipeline, worker_application, mock_execute_pipeline
):
db.session.commit()
pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
result = client.put(
"/v1/pipelines/no-id/runs/no-id/console",
content_type="application/json",
json={
"std_out": "stdout",
"std_err": "stderr",
},
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 404
# no such pipeline_run_id
result = client.put(
f"/v1/pipelines/{pipeline.uuid}/runs/no-id/console",
content_type="application/json",
json={
"std_out": "stdout",
"std_err": "stderr",
},
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 404
# successfully fetch a pipeline_run
result = client.put(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}/console",
content_type="application/json",
json={
"std_out": "stdout",
"std_err": "stderr",
},
headers={ROLES_KEY: worker_application.api_key},
)
assert result.status_code == 200
assert pipeline_run.std_out == "stdout"
assert pipeline_run.std_err == "stderr"
def test_update_pipeline_run_state_callback_err(
app, monkeypatch, pipeline, mock_execute_pipeline
):
pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
# If a callback_url fails for some network reason, the update should still
# work:
def mock_urlopen(request, timeout):
raise URLError("a reason")
monkeypatch.setattr(services.urllib_request, "urlopen", mock_urlopen)
services.update_pipeline_run_state(
pipeline_run.uuid,
{
"state": RunStateEnum.RUNNING.name,
},
)
assert len(pipeline_run.pipeline_run_states) == 3
assert pipeline_run.run_state_enum() == RunStateEnum.RUNNING
def test_create_queued_pipeline_run(app, pipeline):
input1 = {
"name": "name1.pdf",
"url": "https://example.com/name1.pdf",
}
input2 = {
"name": "name2.pdf",
"url": "https://example.com/name2.pdf",
}
pipeline_run = services.create_pipeline_run(
pipeline.uuid,
{"inputs": [input1, input2], "callback_url": "http://example.com"},
True,
)
assert pipeline_run.pipeline == pipeline
assert pipeline_run.sequence == 1
assert len(pipeline_run.pipeline_run_inputs) == 2
assert pipeline_run.pipeline_run_inputs[0].filename == input1["name"]
assert pipeline_run.pipeline_run_inputs[1].filename == input2["name"]
assert len(pipeline_run.pipeline_run_states) == 1
assert pipeline_run.pipeline_run_states[0].code == RunStateEnum.QUEUED
def test_update_pipeline_run_state(app, monkeypatch, pipeline, mock_execute_pipeline):
pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
# A callback is made with the callback_url and the correct payload
def mock_urlopen(request, timeout):
assert request.full_url == pipeline_run.callback_url
assert json.loads(request.data.decode()) == {
"pipeline_run_uuid": pipeline_run.uuid,
"state": RunStateEnum.RUNNING.name,
}
assert timeout == CALLBACK_TIMEOUT
monkeypatch.setattr(services.urllib_request, "urlopen", mock_urlopen)
services.update_pipeline_run_state(
pipeline_run.uuid,
{
"state": RunStateEnum.RUNNING.name,
},
)
assert len(pipeline_run.pipeline_run_states) == 3
assert pipeline_run.run_state_enum() == RunStateEnum.RUNNING
def create_workflow_run(workflow_uuid, run_json):
""" Create a new WorkflowRun """
data = CreateRunSchema().load(run_json)
workflow = find_workflow(workflow_uuid)
if workflow is None:
raise ValueError("no workflow found")
workflow_run = WorkflowRun(workflow=workflow)
workflow_run.workflow_run_states.append(
WorkflowRunState(
run_state_type=find_run_state_type(RunStateEnum.NOT_STARTED)))
added_run = False
for workflow_pipeline in workflow.workflow_pipelines:
if workflow_pipeline.is_deleted:
continue
queue_run = len(workflow_pipeline.source_workflow_pipelines) > 0
no_input_data = {"callback_url": data["callback_url"], "inputs": []}
run_data = no_input_data if queue_run else data
pipeline_run = create_pipeline_run(workflow_pipeline.pipeline.uuid,
run_data, queue_run)
workflow_pipeline_run = WorkflowPipelineRun(
workflow_run=workflow_run,
pipeline_run=pipeline_run,
workflow_pipeline=workflow_pipeline,
)
db.session.add(workflow_pipeline_run)
added_run = True
if not added_run:
db.session.rollback()
raise ValueError("No WorkflowPipelines exist!")
db.session.add(workflow_run)
db.session.commit()
return workflow_run
def test_create_pipeline_run(mock_url, app, organization_pipeline,
organization_pipeline_input_file):
json_response = dict(PIPELINE_RUN_RESPONSE_JSON)
mock_url.return_value = "http://somefileurl.com"
pipeline = OrganizationPipeline.query.order_by(
OrganizationPipeline.id.desc()).first()
responses.add(
responses.POST,
f"{app.config[WORKFLOW_HOSTNAME]}/v1/pipelines/{pipeline.pipeline_uuid}/runs",
json=json_response,
)
created_pipeline_run = create_pipeline_run(pipeline.organization_uuid,
pipeline.uuid,
PIPELINE_RUN_JSON)
new_run = OrganizationPipelineRun.query.filter(
OrganizationPipelineRun.pipeline_run_uuid ==
created_pipeline_run["uuid"]).first()
assert new_run is not None
assert created_pipeline_run == json_response
def test_get_pipeline_run(client, pipeline, client_application,
mock_execute_pipeline):
db.session.commit()
pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
artifact = PipelineRunArtifact(name="test.pdf")
artifact.public_url = Mock()
artifact.public_url.return_value = "http://fake.example.com/url"
pipeline_run.pipeline_run_artifacts.append(artifact)
db.session.commit()
result = client.get(
"/v1/pipelines/no-id/runs/no-id",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 404
# no such pipeline_run_id
result = client.get(
f"/v1/pipelines/{pipeline.uuid}/runs/no-id",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 404
# successfully fetch a pipeline_run
result = client.get(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 200
assert result.json == {
"uuid":
pipeline_run.uuid,
"sequence":
pipeline_run.sequence,
"created_at":
to_iso8601(pipeline_run.created_at),
"inputs": [],
"states": [
{
"state":
RunStateEnum.QUEUED.name,
"created_at":
to_iso8601(pipeline_run.pipeline_run_states[0].created_at),
},
{
"state":
RunStateEnum.NOT_STARTED.name,
"created_at":
to_iso8601(pipeline_run.pipeline_run_states[1].created_at),
},
],
"artifacts": [{
"uuid": artifact.uuid,
"name": "test.pdf",
"url": "http://fake.example.com/url",
}],
}
# fails if the pipeline is deleted.
pipeline.is_deleted = True
db.session.commit()
result = client.get(
f"/v1/pipelines/{pipeline.uuid}/runs/{pipeline_run.uuid}",
headers={ROLES_KEY: client_application.api_key},
)
assert result.status_code == 404
def test_update_workflow_run_no_workflow(execute_pipeline_mock, app, pipeline):
# a pipeline_run not associated with workflow_pipeline_run nothing breaks
pipeline_run = create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
assert services.update_workflow_run(pipeline_run) is None
def test_delete_pipeline_run(app, pipeline):
pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
services.delete_pipeline_run(pipeline.uuid, pipeline_run.uuid)
assert pipeline_run.is_deleted
def test_delete_pipeline_run_has_workflow(app, pipeline, workflow, workflow_pipeline):
pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
with pytest.raises(ValueError):
services.delete_pipeline_run(pipeline.uuid, pipeline_run.uuid)
assert not pipeline_run.is_deleted
def test_create_pipeline_bad_input(app):
with pytest.raises(ValidationError):
pipeline_run = services.create_pipeline_run("no-id", INVALID_CALLBACK_INPUT)
def test_update_pipeline_run_output(app, pipeline, mock_execute_pipeline):
pipeline_run = services.create_pipeline_run(pipeline.uuid, VALID_CALLBACK_INPUT)
services.update_pipeline_run_output(pipeline_run.uuid, "stdout", "stderr")
assert pipeline_run.std_out == "stdout"
assert pipeline_run.std_err == "stderr"
def test_create_pipeline_run_no_pipeline(app):
with pytest.raises(ValueError):
pipeline_run = services.create_pipeline_run("no-id", VALID_CALLBACK_INPUT)
