async def test_execute_latest_for_transformation_revision_no_revision_in_db(
async_test_client, clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
tr_component_1 = TransformationRevision(**tr_json_component_1)
tr_component_1.content = update_code(tr_component_1)
exec_latest_by_group_id_input = ExecLatestByGroupIdInput(
revision_group_id=tr_component_1.revision_group_id,
wiring=tr_component_1.test_wiring,
)
async with async_test_client as ac:
response = await ac.post(
"/api/transformations/execute-latest",
json=json.loads(exec_latest_by_group_id_input.json()),
)
assert response.status_code == 404
assert (
"no released transformation revisions with revision group id"
in response.json()["detail"])
async def create_transformation_revision(
transformation_revision: TransformationRevision,
) -> TransformationRevision:
"""Store a transformation revision in the data base."""
logger.info("create transformation revision %s", transformation_revision.id)
if transformation_revision.type == Type.COMPONENT:
logger.debug("transformation revision has type %s", Type.COMPONENT)
transformation_revision.content = update_code(transformation_revision)
logger.debug("generated code:\n%s", transformation_revision.content)
try:
store_single_transformation_revision(transformation_revision)
logger.info("created transformation revision")
except DBIntegrityError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e
try:
persisted_transformation_revision = read_single_transformation_revision(
transformation_revision.id
)
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
logger.debug(persisted_transformation_revision.json())
return persisted_transformation_revision
def update_content(
existing_transformation_revision: Optional[TransformationRevision],
updated_transformation_revision: TransformationRevision,
) -> TransformationRevision:
if updated_transformation_revision.type == Type.COMPONENT:
updated_transformation_revision.content = update_code(
updated_transformation_revision
)
elif existing_transformation_revision is not None:
assert isinstance(
existing_transformation_revision.content, WorkflowContent
) # hint for mypy
existing_operator_ids: List[UUID] = []
for operator in existing_transformation_revision.content.operators:
existing_operator_ids.append(operator.id)
assert isinstance(
updated_transformation_revision.content, WorkflowContent
) # hint for mypy
for operator in updated_transformation_revision.content.operators:
if (
operator.type == Type.WORKFLOW
and operator.id not in existing_operator_ids
):
operator.state = (
State.DISABLED
if contains_deprecated(operator.transformation_id)
else operator.state
)
return updated_transformation_revision
async def test_execute_for_workflow_dto(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
component_dto = ComponentRevisionFrontendDto(
**dto_json_component_1)
tr_component = component_dto.to_transformation_revision()
tr_component.content = update_code(tr_component)
store_single_transformation_revision(tr_component)
tr_workflow_2 = WorkflowRevisionFrontendDto(
**dto_json_workflow_2_update).to_transformation_revision()
tr_workflow_2.content.inputs[0].name = "wf_input"
tr_workflow_2.content.outputs[0].name = "wf_output"
tr_workflow_2.content.links.append(
Link(
start=Vertex(
operator=None,
connector=Connector.from_io(
tr_workflow_2.content.inputs[0]),
),
end=Vertex(
operator=tr_workflow_2.content.operators[0].id,
connector=tr_workflow_2.content.operators[0].inputs[0],
),
))
tr_workflow_2.content.links.append(
Link(
start=Vertex(
operator=tr_workflow_2.content.operators[0].id,
connector=tr_workflow_2.content.operators[0].
outputs[0],
),
end=Vertex(
operator=None,
connector=Connector.from_io(
tr_workflow_2.content.outputs[0]),
),
))
tr_workflow_2.io_interface.inputs[0].name = "wf_input"
tr_workflow_2.io_interface.outputs[0].name = "wf_output"
store_single_transformation_revision(tr_workflow_2)
update_or_create_nesting(tr_workflow_2)
async with async_test_client as ac:
response = await ac.post(
"/api/workflows/" + str(get_uuid_from_seed("workflow 2")) +
"/execute",
json=dto_json_wiring,
)
assert response.status_code == 200
assert "output_types_by_output_name" in response.json()
def test_update_code():
component = TransformationRevision(
io_interface=IOInterface(inputs=[], outputs=[]),
name="Test Component",
description="A test component",
category="Tests",
id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
version_tag="1.0.0",
state="RELEASED",
type="COMPONENT",
released_timestamp="2019-12-01T12:00:00+00:00",
content=example_code,
test_wiring=[],
)
updated_component = TransformationRevision(
io_interface=IOInterface(inputs=[], outputs=[]),
name="Test Component",
description="A test component",
category="Tests",
id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
version_tag="1.0.1",
state="DRAFT",
type="COMPONENT",
content=example_code,
test_wiring=[],
)
new_code = update_code(updated_component)
assert """return {"z": x+y}""" in new_code
assert "c6eff22c-21c4-43c6-9ae1-b2bdfb944565" in new_code
assert "1.0.0" not in new_code
# test input without both start/stop markers
component.content = ""
new_code = update_code(component)
assert "pass" in new_code
# test input without only stop marker
component.content = "# ***** DO NOT EDIT LINES BELOW *****"
new_code = update_code(component)
# test with async def in function header
component.content = example_code_async
new_code = update_code(component)
assert "async def" in new_code
def test_update_code():
func_body = """return {"z":x + y}"""
test_code = example_code.replace("pass", func_body)
new_code = update_code(test_code, {}, {})
assert func_body in new_code
assert "pass" not in new_code
# test with no code (new code generation)
new_code = update_code(None, {}, {})
assert "pass" in new_code
# test input without both start/stop markers
new_code = update_code(" ", {}, {})
assert "pass" in new_code
# test input without only stop marker
new_code = update_code("# ***** DO NOT EDIT LINES BELOW *****", {}, {})
assert "pass" in new_code
async def test_execute_for_component_without_hetdesrun_imports(
async_test_client, clean_test_db_engine
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
path = (
"./tests/data/components/"
"alerts-from-score_100_38f168ef-cb06-d89c-79b3-0cd823f32e9d"
".json"
)
component_tr_json = load_json(path)
wiring_json = {
"id": "38f168ef-cb06-d89c-79b3-0cd823f32e9d",
"name": "STANDARD-WIRING",
"inputWirings": [
{
"id": "8c249f92-4b81-457e-9371-24204d6b373b",
"workflowInputName": "scores",
"adapterId": "direct_provisioning",
"filters": {
"value": (
"{\n"
' "2020-01-03T08:20:03.000Z": 18.7,\n'
' "2020-01-01T01:15:27.000Z": 42.2,\n'
' "2020-01-03T08:20:04.000Z": 25.9\n'
"}"
)
},
},
{
"id": "0f0f97f7-1f5d-4f5d-be11-7c7b78d02129",
"workflowInputName": "threshold",
"adapterId": "direct_provisioning",
"filters": {"value": "30"},
},
],
"outputWirings": [],
}
tr = TransformationRevision(**component_tr_json)
tr.content = update_code(tr)
assert "COMPONENT_INFO" in tr.content
store_single_transformation_revision(tr)
async with async_test_client as ac:
response = await ac.post(
"/api/components/" + component_tr_json["id"] + "/execute",
json=wiring_json,
)
assert response.status_code == 200
assert "output_types_by_output_name" in response.json()
async def codegen_service(codegen_input: CodeBody,) -> GeneratedCode:
"""Service for generating and updating code stubs"""
logger.info("CODEGEN INPUT JSON:\n%s", model_to_pretty_json_str(codegen_input))
return GeneratedCode(
code=update_code(
codegen_input.code,
{c.name: c.type for c in codegen_input.inputs},
{c.name: c.type for c in codegen_input.outputs},
)
)
async def test_execute_latest_for_transformation_revision_works(
async_test_client, clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
tr_component_1 = TransformationRevision(**tr_json_component_1)
tr_component_1.content = update_code(tr_component_1)
store_single_transformation_revision(tr_component_1)
tr_component_1_new_revision = TransformationRevision(
**tr_json_component_1_new_revision)
tr_component_1_new_revision.content = update_code(
tr_component_1_new_revision)
tr_component_1_new_revision.release()
store_single_transformation_revision(tr_component_1_new_revision)
exec_latest_by_group_id_input = ExecLatestByGroupIdInput(
revision_group_id=tr_component_1.revision_group_id,
wiring=tr_component_1.test_wiring,
job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"),
)
async with async_test_client as ac:
response = await ac.post(
"/api/transformations/execute-latest",
json=json.loads(exec_latest_by_group_id_input.json()),
)
assert response.status_code == 200
resp_data = response.json()
assert "output_types_by_output_name" in resp_data
assert (resp_data["output_types_by_output_name"]["operator_output"]
== "STRING")
assert "job_id" in resp_data
assert UUID(resp_data["job_id"]) == UUID(
"1270547c-b224-461d-9387-e9d9d465bbe1")
async def create_component_revision(
component_dto: ComponentRevisionFrontendDto,
) -> ComponentRevisionFrontendDto:
"""Store a transformation revision of type component in the data base.
This endpoint is deprecated and will be removed soon,
use POST /api/transformations/ instead.
"""
logger.info("create new component")
try:
transformation_revision = component_dto.to_transformation_revision(
documentation=("# New Component/Workflow\n"
"## Description\n"
"## Inputs\n"
"## Outputs\n"
"## Details\n"
"## Examples\n"))
except ValidationError as e:
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=str(e)) from e
logger.debug("generate code")
transformation_revision.content = update_code(transformation_revision)
logger.debug("generated code:\n%s", component_dto.code)
try:
store_single_transformation_revision(transformation_revision)
logger.info("created new component")
except DBIntegrityError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=str(e)) from e
persisted_transformation_revision = read_single_transformation_revision(
transformation_revision.id)
persisted_component_dto = ComponentRevisionFrontendDto.from_transformation_revision(
persisted_transformation_revision)
logger.debug(persisted_component_dto.json())
return persisted_component_dto
async def test_execute_for_transformation_revision(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
tr_component_1 = TransformationRevision(**tr_json_component_1)
tr_component_1.content = update_code(tr_component_1)
store_single_transformation_revision(tr_component_1)
tr_workflow_2 = TransformationRevision(**tr_json_workflow_2_update)
store_single_transformation_revision(tr_workflow_2)
update_or_create_nesting(tr_workflow_2)
exec_by_id_input = ExecByIdInput(
id=tr_workflow_2.id,
wiring=tr_workflow_2.test_wiring,
job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"),
)
async with async_test_client as ac:
response = await ac.post(
"/api/transformations/execute",
json=json.loads(exec_by_id_input.json()),
)
assert response.status_code == 200
resp_data = response.json()
assert "output_types_by_output_name" in resp_data
assert "job_id" in resp_data
assert UUID(resp_data["job_id"]) == UUID(
"1270547c-b224-461d-9387-e9d9d465bbe1")
async def test_execute_for_separate_runtime_container(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
with mock.patch(
"hetdesrun.webservice.config.runtime_config",
is_runtime_service=False,
):
resp_mock = mock.Mock()
resp_mock.status_code = 200
resp_mock.json = mock.Mock(
return_value={
"output_results_by_output_name": {
"wf_output": 100
},
"output_types_by_output_name": {
"wf_output": "INT"
},
"result": "ok",
"job_id": "1270547c-b224-461d-9387-e9d9d465bbe1",
})
with mock.patch(
"hetdesrun.backend.execution.httpx.AsyncClient.post",
return_value=resp_mock,
) as mocked_post:
tr_component_1 = TransformationRevision(
**tr_json_component_1)
tr_component_1.content = update_code(tr_component_1)
store_single_transformation_revision(tr_component_1)
tr_workflow_2 = TransformationRevision(
**tr_json_workflow_2_update)
store_single_transformation_revision(tr_workflow_2)
update_or_create_nesting(tr_workflow_2)
exec_by_id_input = ExecByIdInput(
id=tr_workflow_2.id,
wiring=tr_workflow_2.test_wiring,
job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"),
)
async with async_test_client as ac:
response = await ac.post(
"/api/transformations/execute",
json=json.loads(exec_by_id_input.json()),
)
assert response.status_code == 200
resp_data = response.json()
assert "output_types_by_output_name" in resp_data
assert "job_id" in resp_data
assert UUID(resp_data["job_id"]) == UUID(
"1270547c-b224-461d-9387-e9d9d465bbe1")
mocked_post.assert_called_once()
