async def test_execute_for_workflow_dto(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
component_dto = ComponentRevisionFrontendDto(
**dto_json_component_1)
tr_component = component_dto.to_transformation_revision()
tr_component.content = update_code(tr_component)
store_single_transformation_revision(tr_component)
tr_workflow_2 = WorkflowRevisionFrontendDto(
**dto_json_workflow_2_update).to_transformation_revision()
tr_workflow_2.content.inputs[0].name = "wf_input"
tr_workflow_2.content.outputs[0].name = "wf_output"
tr_workflow_2.content.links.append(
Link(
start=Vertex(
operator=None,
connector=Connector.from_io(
tr_workflow_2.content.inputs[0]),
),
end=Vertex(
operator=tr_workflow_2.content.operators[0].id,
connector=tr_workflow_2.content.operators[0].inputs[0],
),
))
tr_workflow_2.content.links.append(
Link(
start=Vertex(
operator=tr_workflow_2.content.operators[0].id,
connector=tr_workflow_2.content.operators[0].
outputs[0],
),
end=Vertex(
operator=None,
connector=Connector.from_io(
tr_workflow_2.content.outputs[0]),
),
))
tr_workflow_2.io_interface.inputs[0].name = "wf_input"
tr_workflow_2.io_interface.outputs[0].name = "wf_output"
store_single_transformation_revision(tr_workflow_2)
update_or_create_nesting(tr_workflow_2)
async with async_test_client as ac:
response = await ac.post(
"/api/workflows/" + str(get_uuid_from_seed("workflow 2")) +
"/execute",
json=dto_json_wiring,
)
assert response.status_code == 200
assert "output_types_by_output_name" in response.json()
def test_wrap_component_in_tr_workflow():
component_dto = ComponentRevisionFrontendDto(**valid_component_dto_dict)
tr_component = component_dto.to_transformation_revision()
tr_workflow = tr_component.wrap_component_in_tr_workflow()
assert valid_component_dto_dict["name"] == tr_workflow.name
assert valid_component_dto_dict["category"] == tr_workflow.category
assert valid_component_dto_dict["tag"] == tr_workflow.version_tag
assert valid_component_dto_dict["state"] == tr_workflow.state
assert Type.WORKFLOW == tr_workflow.type
assert 1 == len(tr_workflow.content.operators)
assert valid_component_dto_dict["id"] == str(
tr_workflow.content.operators[0].transformation_id)
assert len(valid_component_dto_dict["inputs"]) == len(
tr_workflow.content.operators[0].inputs)
assert len(valid_component_dto_dict["outputs"]) == len(
tr_workflow.content.operators[0].outputs)
assert len(tr_component.io_interface.inputs) == len(
tr_workflow.content.inputs)
assert len(tr_component.io_interface.outputs) == len(
tr_workflow.content.outputs)
assert len(tr_component.io_interface.inputs) == len(
tr_workflow.io_interface.inputs)
assert len(tr_component.io_interface.outputs) == len(
tr_workflow.io_interface.outputs)
async def test_deprecate_transformation_revision_from_component_dto(
async_test_client, clean_test_db_engine
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
ComponentRevisionFrontendDto(
**dto_json_component_2
).to_transformation_revision()
)
async with async_test_client as ac:
response = await ac.put(
"/api/components/" + str(get_uuid_from_seed("component 2")),
json=dto_json_component_2_deprecate,
)
assert response.status_code == 201
assert response.json()["state"] == "DISABLED"
assert response.json()["name"] != "new name"
assert response.json()["category"] != "Test"
assert len(response.json()["inputs"]) == 0
assert "new comment" not in response.json()["code"]
assert "disabled_timestamp" in response.json()["code"]
assert "released_timestamp" in response.json()["code"]
async def test_update_transformation_revision_from_component_dto(
async_test_client, clean_test_db_engine
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
ComponentRevisionFrontendDto(
**dto_json_component_1
).to_transformation_revision()
)
async with async_test_client as ac:
response = await ac.put(
"/api/components/" + str(get_uuid_from_seed("component 1")),
json=dto_json_component_1_update,
)
assert response.status_code == 201
assert response.json()["name"] == "new name"
assert response.json()["category"] == "Test"
assert response.json()["inputs"][0]["id"] == str(
get_uuid_from_seed("new input")
)
print(response.json()["code"])
assert "new_input" in response.json()["code"]
async def get_component_revision_by_id(
# pylint: disable=redefined-builtin
id: UUID = Path(
...,
example=UUID("123e4567-e89b-12d3-a456-426614174000"),
),
) -> ComponentRevisionFrontendDto:
"""Get a single transformation revision of type component from the data base.
This endpoint is deprecated and will be removed soon,
use GET /api/transformations/{id} instead.
"""
logger.info("get component %s", id)
try:
transformation_revision = read_single_transformation_revision(id)
logger.info("found component with id %s", id)
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
if transformation_revision.type != Type.COMPONENT:
msg = f"DB entry for id {id} does not have type {Type.COMPONENT}"
logger.error(msg)
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=msg)
component_dto = ComponentRevisionFrontendDto.from_transformation_revision(
transformation_revision)
logger.debug(component_dto.json())
return component_dto
def test_to_workflow_node():
component_dto = ComponentRevisionFrontendDto(**valid_component_dto_dict)
tr_component = component_dto.to_transformation_revision()
tr_workflow = tr_component.wrap_component_in_tr_workflow()
nested_transformations = {
tr_workflow.content.operators[0].id: tr_component
}
workflow_node = tr_workflow.to_workflow_node(
uuid4(), nested_nodes(tr_workflow, nested_transformations))
assert len(workflow_node.inputs) == len(valid_component_dto_dict["inputs"])
assert len(workflow_node.outputs) == len(
valid_component_dto_dict["outputs"])
assert len(workflow_node.sub_nodes) == 1
assert len(workflow_node.connections) == 0
assert workflow_node.name == valid_component_dto_dict["name"]
async def create_component_revision(
component_dto: ComponentRevisionFrontendDto,
) -> ComponentRevisionFrontendDto:
"""Store a transformation revision of type component in the data base.
This endpoint is deprecated and will be removed soon,
use POST /api/transformations/ instead.
"""
logger.info("create new component")
try:
transformation_revision = component_dto.to_transformation_revision(
documentation=("# New Component/Workflow\n"
"## Description\n"
"## Inputs\n"
"## Outputs\n"
"## Details\n"
"## Examples\n"))
except ValidationError as e:
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=str(e)) from e
logger.debug("generate code")
transformation_revision.content = update_code(transformation_revision)
logger.debug("generated code:\n%s", component_dto.code)
try:
store_single_transformation_revision(transformation_revision)
logger.info("created new component")
except DBIntegrityError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=str(e)) from e
persisted_transformation_revision = read_single_transformation_revision(
transformation_revision.id)
persisted_component_dto = ComponentRevisionFrontendDto.from_transformation_revision(
persisted_transformation_revision)
logger.debug(persisted_component_dto.json())
return persisted_component_dto
async def test_delete_transformation_revision_from_component_dto(
async_test_client, clean_test_db_engine
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
ComponentRevisionFrontendDto(
**dto_json_component_1
).to_transformation_revision()
)
async with async_test_client as ac:
response = await ac.delete(
"/api/components/" + str(get_uuid_from_seed("component 1")),
)
assert response.status_code == 204
async def bind_wiring_to_component_revision(
# pylint: disable=redefined-builtin
id: UUID,
wiring_dto: WiringFrontendDto,
) -> ComponentRevisionFrontendDto:
"""Store or update the test wiring of a transformation revision of type component.
This endpoint is deprecated and will be removed soon,
use PUT /api/transformations/{id} instead.
"""
logger.info("bind wiring to component %s", id)
try:
transformation_revision = read_single_transformation_revision(id)
logger.info("found component with id %s", id)
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
if transformation_revision.type != Type.COMPONENT:
msg = f"DB entry for id {id} does not have type {Type.COMPONENT}"
logger.error(msg)
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=msg)
wiring = wiring_dto.to_wiring()
transformation_revision.test_wiring = wiring
try:
persisted_transformation_revision = (
update_or_create_single_transformation_revision(
transformation_revision))
logger.info("bound wiring to component %s", id)
except DBIntegrityError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=str(e)) from e
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
persisted_component_dto = ComponentRevisionFrontendDto.from_transformation_revision(
persisted_transformation_revision)
logger.debug(persisted_component_dto.json())
return persisted_component_dto
async def test_execute_for_component_dto(async_test_client, clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
ComponentRevisionFrontendDto(
**valid_component_dto_dict
).to_transformation_revision()
)
async with async_test_client as ac:
response = await ac.post(
"/api/components/" + valid_component_dto_dict["id"] + "/execute",
json=valid_component_dto_dict["wirings"][0],
)
assert response.status_code == 200
assert "output_types_by_output_name" in response.json()
async def test_publish_transformation_revision_from_component_dto(
async_test_client, clean_test_db_engine
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(
ComponentRevisionFrontendDto(
**dto_json_component_1
).to_transformation_revision()
)
async with async_test_client as ac:
response = await ac.put(
"/api/components/" + str(get_uuid_from_seed("component 1")),
json=dto_json_component_1_publish,
)
assert response.status_code == 201
assert response.json()["state"] == "RELEASED"
assert "released_timestamp" in response.json()["code"]
def get_transformation_from_java_backend(id: UUID, type: Type) -> Any:
"""
Loads a single transformation revision together with its documentation based on its id
"""
headers = get_auth_headers()
if type == Type.COMPONENT:
url = posix_urljoin(get_config().hd_backend_api_url, "components",
str(id))
else:
url = posix_urljoin(get_config().hd_backend_api_url, "workflows",
str(id))
# Get transformation revision from old backend
response = requests.get(
url,
verify=get_config().hd_backend_verify_certs,
auth=get_backend_basic_auth() # type: ignore
if get_config().hd_backend_use_basic_auth else None,
headers=headers,
)
logger.info(
"GET %s status code: %i for %s with id %ss",
type,
response.status_code,
type,
str(id),
)
if response.status_code != 200:
msg = (f"COULD NOT GET {type} with id {id}.\n"
f"Response status code {response.status_code} "
f"with response text:\n{response.json()['detail']}")
logger.error(msg)
raise Exception(msg)
revision_json = response.json()
# Get documentation from old backend
doc_response = requests.get(
posix_urljoin(get_config().hd_backend_api_url, "documentations",
str(id)),
verify=get_config().hd_backend_verify_certs,
auth=get_backend_basic_auth() # type: ignore
if get_config().hd_backend_use_basic_auth else None,
headers=headers,
)
logger.info(
"GET documentation status code: %i for %s with id %s",
response.status_code,
type,
str(id),
)
if response.status_code != 200:
msg = (f"COULD NOT GET documentation with id {id}.\n"
f"Response status code {response.status_code} "
f"with response text:\n{response.json()['detail']}")
logger.error(msg)
raise Exception(msg)
doc_text = doc_response.json().get("document", "")
frontend_dto: Union[ComponentRevisionFrontendDto,
WorkflowRevisionFrontendDto]
# Generate transformation revision
if type == Type.COMPONENT:
revision_json["type"] = Type.COMPONENT
frontend_dto = ComponentRevisionFrontendDto(**revision_json, )
else:
frontend_dto = WorkflowRevisionFrontendDto(**revision_json, )
transformation_revision = frontend_dto.to_transformation_revision(
documentation=doc_text)
tr_json = json.loads(transformation_revision.json())
return tr_json
async def update_component_revision(
# pylint: disable=redefined-builtin
id: UUID,
updated_component_dto: ComponentRevisionFrontendDto,
) -> ComponentRevisionFrontendDto:
"""Update or store a transformation revision of type component in the data base.
If no DB entry with the provided id is found, it will be created.
Updating a transformation revision is only possible if it is in state DRAFT
or to change the state from RELEASED to DISABLED.
This endpoint is deprecated and will be removed soon,
use PUT /api/transformations/{id} instead.
"""
logger.info("update component %s", id)
if id != updated_component_dto.id:
msg = (
f"The id {id} does not match "
f"the id of the component revision DTO {updated_component_dto.id}")
logger.error(msg)
raise HTTPException(status.HTTP_403_FORBIDDEN, detail=msg)
try:
updated_transformation_revision = (
updated_component_dto.to_transformation_revision())
except ValidationError as e:
logger.error("The following validation error occured:\n%s", str(e))
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=str(e)) from e
existing_transformation_revision: Optional[TransformationRevision] = None
try:
existing_transformation_revision = read_single_transformation_revision(
id, log_error=False)
logger.info("found transformation revision %s", id)
except DBNotFoundError:
# base/example workflow deployment needs to be able to put
# with an id and either create or update the component revision
pass
modifiable, msg = is_modifiable(
existing_transformation_revision,
updated_transformation_revision,
)
if not modifiable:
logger.error(msg)
raise HTTPException(status.HTTP_403_FORBIDDEN, detail=msg)
if existing_transformation_revision is not None:
updated_transformation_revision.documentation = (
existing_transformation_revision.documentation)
updated_transformation_revision.test_wiring = (
existing_transformation_revision.test_wiring)
updated_transformation_revision.released_timestamp = (
existing_transformation_revision.released_timestamp)
updated_transformation_revision = if_applicable_release_or_deprecate(
existing_transformation_revision, updated_transformation_revision)
updated_transformation_revision = update_content(
existing_transformation_revision, updated_transformation_revision)
try:
persisted_transformation_revision = (
update_or_create_single_transformation_revision(
updated_transformation_revision))
logger.info("updated component %s", id)
except DBIntegrityError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=str(e)) from e
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
persisted_component_dto = ComponentRevisionFrontendDto.from_transformation_revision(
persisted_transformation_revision)
logger.debug(persisted_component_dto.json())
return persisted_component_dto