async def test_publish_transformation_revision_from_workflow_dto(
async_test_client, clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
WorkflowRevisionFrontendDto(**dto_json_workflow_2_publishable).
to_transformation_revision())
dto_json_workflow_2_publish = deepcopy(dto_json_workflow_2_publishable)
dto_json_workflow_2_publish["state"] = "RELEASED"
# print("json inputs",dto_json_workflow_2_publish["inputs"])
# print("json outputs",dto_json_workflow_2_publish["outputs"])
print()
async with async_test_client as ac:
response = await ac.put(
"/api/workflows/" + str(get_uuid_from_seed("workflow 2")),
json=dto_json_workflow_2_publish,
)
assert response.status_code == 201
assert response.json()["state"] == "RELEASED"
assert response.json()["name"] != "new name"
async def get_workflow_revision_by_id(
# pylint: disable=redefined-builtin
id: UUID = Path(
...,
example=UUID("123e4567-e89b-12d3-a456-426614174000"),
),
) -> WorkflowRevisionFrontendDto:
"""Get a single transformation revision of type workflow from the data base.
This endpoint is deprecated and will be removed soon,
use GET /api/transformations/{id} instead.
"""
logger.info("get workflow %s", id)
try:
transformation_revision = read_single_transformation_revision(id)
logger.info("found workflow with id %s", id)
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
if transformation_revision.type != Type.WORKFLOW:
msg = f"DB entry for id {id} does not have type {Type.WORKFLOW}"
logger.error(msg)
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=msg)
workflow_dto = WorkflowRevisionFrontendDto.from_transformation_revision(
transformation_revision)
logger.debug(workflow_dto.json())
return workflow_dto
async def test_execute_for_workflow_dto(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
component_dto = ComponentRevisionFrontendDto(
**dto_json_component_1)
tr_component = component_dto.to_transformation_revision()
tr_component.content = update_code(tr_component)
store_single_transformation_revision(tr_component)
tr_workflow_2 = WorkflowRevisionFrontendDto(
**dto_json_workflow_2_update).to_transformation_revision()
tr_workflow_2.content.inputs[0].name = "wf_input"
tr_workflow_2.content.outputs[0].name = "wf_output"
tr_workflow_2.content.links.append(
Link(
start=Vertex(
operator=None,
connector=Connector.from_io(
tr_workflow_2.content.inputs[0]),
),
end=Vertex(
operator=tr_workflow_2.content.operators[0].id,
connector=tr_workflow_2.content.operators[0].inputs[0],
),
))
tr_workflow_2.content.links.append(
Link(
start=Vertex(
operator=tr_workflow_2.content.operators[0].id,
connector=tr_workflow_2.content.operators[0].
outputs[0],
),
end=Vertex(
operator=None,
connector=Connector.from_io(
tr_workflow_2.content.outputs[0]),
),
))
tr_workflow_2.io_interface.inputs[0].name = "wf_input"
tr_workflow_2.io_interface.outputs[0].name = "wf_output"
store_single_transformation_revision(tr_workflow_2)
update_or_create_nesting(tr_workflow_2)
async with async_test_client as ac:
response = await ac.post(
"/api/workflows/" + str(get_uuid_from_seed("workflow 2")) +
"/execute",
json=dto_json_wiring,
)
assert response.status_code == 200
assert "output_types_by_output_name" in response.json()
async def test_get_all_worfklow_revisions_with_valid_db_entries(
async_test_client, clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
WorkflowRevisionFrontendDto(
**dto_json_workflow_1).to_transformation_revision())
store_single_transformation_revision(
WorkflowRevisionFrontendDto(
**dto_json_workflow_2).to_transformation_revision())
async with async_test_client as ac:
response = await ac.get("/api/workflows/")
assert response.status_code == 200
assert response.json()[0] == dto_json_workflow_1
assert response.json()[1] == dto_json_workflow_2
async def create_workflow_revision(
workflow_dto: WorkflowRevisionFrontendDto,
) -> WorkflowRevisionFrontendDto:
"""Store a transformation revision of type workflow in the data base.
This endpoint is deprecated and will be removed soon,
use POST /api/transformations/ instead.
"""
logger.info("create a new workflow")
try:
transformation_revision = workflow_dto.to_transformation_revision(
documentation=("# New Component/Workflow\n"
"## Description\n"
"## Inputs\n"
"## Outputs\n"
"## Details\n"
"## Examples\n"))
except ValidationError as e:
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=str(e)) from e
try:
store_single_transformation_revision(transformation_revision)
logger.info("created new workflow")
except DBIntegrityError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=str(e)) from e
try:
persisted_transformation_revision = read_single_transformation_revision(
transformation_revision.id)
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
persisted_workflow_dto = WorkflowRevisionFrontendDto.from_transformation_revision(
persisted_transformation_revision)
logger.debug(persisted_workflow_dto.json())
return persisted_workflow_dto
async def test_delete_transformation_revision_from_workflow_dto(
async_test_client, clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
WorkflowRevisionFrontendDto(
**dto_json_workflow_2).to_transformation_revision())
async with async_test_client as ac:
response = await ac.delete(
"/api/workflows/" + str(get_uuid_from_seed("workflow 2")), )
assert response.status_code == 204
async def test_get_workflow_revision_without_content_by_id_with_valid_workflow(
async_test_client, clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
WorkflowRevisionFrontendDto(
**dto_json_workflow_1).to_transformation_revision())
async with async_test_client as ac:
response = await ac.get("/api/workflows/" +
str(get_uuid_from_seed("workflow 1")))
assert response.status_code == 200
assert response.json() == dto_json_workflow_1
async def bind_wiring_to_workflow_revision(
# pylint: disable=redefined-builtin
id: UUID,
wiring_dto: WiringFrontendDto,
) -> WorkflowRevisionFrontendDto:
"""Store or update the test wiring of a transformation revision of type workflow.
This endpoint is deprecated and will be removed soon,
use PUT /api/transformations/{id} instead.
"""
logger.info("bind wiring to workflow %s", id)
try:
transformation_revision = read_single_transformation_revision(id)
logger.info("found workflow with id %s", id)
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
if transformation_revision.type != Type.WORKFLOW:
msg = f"DB entry for id {id} does not have type {Type.WORKFLOW}"
logger.error(msg)
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=msg)
wiring = wiring_dto.to_wiring()
transformation_revision.test_wiring = wiring
try:
persisted_transformation_revision = (
update_or_create_single_transformation_revision(
transformation_revision))
logger.info("bound wiring to workflow %s", id)
except DBIntegrityError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=str(e)) from e
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
persisted_workflow_dto = WorkflowRevisionFrontendDto.from_transformation_revision(
persisted_transformation_revision)
logger.debug(persisted_workflow_dto.json())
return persisted_workflow_dto
async def test_deprecate_transformation_revision_from_workflow_dto(
async_test_client, clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
WorkflowRevisionFrontendDto(
**dto_json_workflow_1).to_transformation_revision())
async with async_test_client as ac:
response = await ac.put(
"/api/workflows/" + str(get_uuid_from_seed("workflow 1")),
json=dto_json_workflow_1_deprecate,
)
assert response.status_code == 201
assert response.json()["state"] == "DISABLED"
assert response.json()["name"] != "new name"
async def get_all_workflow_revisions() -> List[WorkflowRevisionFrontendDto]:
"""Get all transformation revisions of type workflow from the data base.
This endpoint is deprecated and will be removed soon,
use GET /api/transformations/{id} instead.
"""
logger.info("get all workflows")
transformation_revision_list = select_multiple_transformation_revisions(
type=Type.WORKFLOW)
logger.info("got all workflows")
workflow_dto_list = [
WorkflowRevisionFrontendDto.from_transformation_revision(tr)
for tr in transformation_revision_list
]
return workflow_dto_list
async def test_update_wiring(async_test_client, clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
WorkflowRevisionFrontendDto(
**dto_json_workflow_2_update).to_transformation_revision())
async with async_test_client as ac:
response = await ac.put(
"/api/wirings/" + str(get_uuid_from_seed("workflow 2")),
json=dto_json_wiring,
)
assert response.status_code == 200
assert len(response.json()["inputWirings"]) == len(
dto_json_wiring["inputWirings"])
assert len(response.json()["outputWirings"]) == len(
dto_json_wiring["outputWirings"])
async def test_update_transformation_revision_from_workflow_dto(
async_test_client, clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
WorkflowRevisionFrontendDto(
**dto_json_workflow_2).to_transformation_revision())
async with async_test_client as ac:
response = await ac.put(
"/api/workflows/" + str(get_uuid_from_seed("workflow 2")),
json=dto_json_workflow_2_update,
)
assert response.status_code == 201
assert response.json()["operators"][0]["id"] == str(
get_uuid_from_seed("operator"))
assert "name" not in response.json()["inputs"][0]
assert "name" not in response.json()["outputs"][0]
def get_transformation_from_java_backend(id: UUID, type: Type) -> Any:
"""
Loads a single transformation revision together with its documentation based on its id
"""
headers = get_auth_headers()
if type == Type.COMPONENT:
url = posix_urljoin(get_config().hd_backend_api_url, "components",
str(id))
else:
url = posix_urljoin(get_config().hd_backend_api_url, "workflows",
str(id))
# Get transformation revision from old backend
response = requests.get(
url,
verify=get_config().hd_backend_verify_certs,
auth=get_backend_basic_auth() # type: ignore
if get_config().hd_backend_use_basic_auth else None,
headers=headers,
)
logger.info(
"GET %s status code: %i for %s with id %ss",
type,
response.status_code,
type,
str(id),
)
if response.status_code != 200:
msg = (f"COULD NOT GET {type} with id {id}.\n"
f"Response status code {response.status_code} "
f"with response text:\n{response.json()['detail']}")
logger.error(msg)
raise Exception(msg)
revision_json = response.json()
# Get documentation from old backend
doc_response = requests.get(
posix_urljoin(get_config().hd_backend_api_url, "documentations",
str(id)),
verify=get_config().hd_backend_verify_certs,
auth=get_backend_basic_auth() # type: ignore
if get_config().hd_backend_use_basic_auth else None,
headers=headers,
)
logger.info(
"GET documentation status code: %i for %s with id %s",
response.status_code,
type,
str(id),
)
if response.status_code != 200:
msg = (f"COULD NOT GET documentation with id {id}.\n"
f"Response status code {response.status_code} "
f"with response text:\n{response.json()['detail']}")
logger.error(msg)
raise Exception(msg)
doc_text = doc_response.json().get("document", "")
frontend_dto: Union[ComponentRevisionFrontendDto,
WorkflowRevisionFrontendDto]
# Generate transformation revision
if type == Type.COMPONENT:
revision_json["type"] = Type.COMPONENT
frontend_dto = ComponentRevisionFrontendDto(**revision_json, )
else:
frontend_dto = WorkflowRevisionFrontendDto(**revision_json, )
transformation_revision = frontend_dto.to_transformation_revision(
documentation=doc_text)
tr_json = json.loads(transformation_revision.json())
return tr_json
async def update_workflow_revision(
# pylint: disable=redefined-builtin
id: UUID,
updated_workflow_dto: WorkflowRevisionFrontendDto,
) -> WorkflowRevisionFrontendDto:
"""Update or store a transformation revision of type workflow in the data base.
If no DB entry with the provided id is found, it will be created.
Updating a transformation revision is only possible if it is in state DRAFT
or to change the state from RELEASED to DISABLED.
This endpoint is deprecated and will be removed soon,
use PUT /api/transformations/{id} instead.
"""
logger.info("update workflow %s", id)
if id != updated_workflow_dto.id:
msg = (
"The id {id} does not match "
f"the id of the provided workflow revision DTO {updated_workflow_dto.id}"
)
logger.error(msg)
raise HTTPException(status.HTTP_403_FORBIDDEN, detail=msg)
try:
updated_transformation_revision = (
updated_workflow_dto.to_transformation_revision())
except ValidationError as e:
logger.error("The following validation error occured:\n%s", str(e))
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=str(e)) from e
existing_transformation_revision: Optional[TransformationRevision] = None
try:
existing_transformation_revision = read_single_transformation_revision(
id, log_error=False)
logger.info("found transformation revision %s", id)
except DBNotFoundError:
# base/example workflow deployment needs to be able to put
# with an id and either create or update the workflow revision
pass
modifiable, msg = is_modifiable(
existing_transformation_revision,
updated_transformation_revision,
)
if not modifiable:
logger.error(msg)
raise HTTPException(status.HTTP_403_FORBIDDEN, detail=msg)
if existing_transformation_revision is not None:
updated_transformation_revision.documentation = (
existing_transformation_revision.documentation)
updated_transformation_revision.test_wiring = (
existing_transformation_revision.test_wiring)
updated_transformation_revision.released_timestamp = (
existing_transformation_revision.released_timestamp)
updated_transformation_revision = if_applicable_release_or_deprecate(
existing_transformation_revision, updated_transformation_revision)
updated_transformation_revision = update_content(
existing_transformation_revision, updated_transformation_revision)
try:
persisted_transformation_revision = (
update_or_create_single_transformation_revision(
updated_transformation_revision))
logger.info("updated workflow %s", id)
except DBIntegrityError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=str(e)) from e
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
persisted_workflow_dto = WorkflowRevisionFrontendDto.from_transformation_revision(
persisted_transformation_revision)
logger.debug(persisted_workflow_dto.json())
return persisted_workflow_dto