def if_applicable_release_or_deprecate(
existing_transformation_revision: Optional[TransformationRevision],
updated_transformation_revision: TransformationRevision,
) -> TransformationRevision:
if existing_transformation_revision is not None:
if (
existing_transformation_revision.state == State.DRAFT
and updated_transformation_revision.state == State.RELEASED
):
logger.info(
"release transformation revision %s",
existing_transformation_revision.id,
)
updated_transformation_revision.release()
# prevent overwriting content during releasing
updated_transformation_revision.content = (
existing_transformation_revision.content
)
if (
existing_transformation_revision.state == State.RELEASED
and updated_transformation_revision.state == State.DISABLED
):
logger.info(
"deprecate transformation revision %s",
existing_transformation_revision.id,
)
updated_transformation_revision = TransformationRevision(
**existing_transformation_revision.dict()
)
updated_transformation_revision.deprecate()
# prevent overwriting content during deprecating
updated_transformation_revision.content = (
existing_transformation_revision.content
)
return updated_transformation_revision
async def test_execute_latest_for_transformation_revision_no_revision_in_db(
async_test_client, clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
tr_component_1 = TransformationRevision(**tr_json_component_1)
tr_component_1.content = update_code(tr_component_1)
exec_latest_by_group_id_input = ExecLatestByGroupIdInput(
revision_group_id=tr_component_1.revision_group_id,
wiring=tr_component_1.test_wiring,
)
async with async_test_client as ac:
response = await ac.post(
"/api/transformations/execute-latest",
json=json.loads(exec_latest_by_group_id_input.json()),
)
assert response.status_code == 404
assert (
"no released transformation revisions with revision group id"
in response.json()["detail"])
async def create_transformation_revision(
transformation_revision: TransformationRevision,
) -> TransformationRevision:
"""Store a transformation revision in the data base."""
logger.info("create transformation revision %s", transformation_revision.id)
if transformation_revision.type == Type.COMPONENT:
logger.debug("transformation revision has type %s", Type.COMPONENT)
transformation_revision.content = update_code(transformation_revision)
logger.debug("generated code:\n%s", transformation_revision.content)
try:
store_single_transformation_revision(transformation_revision)
logger.info("created transformation revision")
except DBIntegrityError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e
try:
persisted_transformation_revision = read_single_transformation_revision(
transformation_revision.id
)
except DBNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
logger.debug(persisted_transformation_revision.json())
return persisted_transformation_revision
def update_content(
existing_transformation_revision: Optional[TransformationRevision],
updated_transformation_revision: TransformationRevision,
) -> TransformationRevision:
if updated_transformation_revision.type == Type.COMPONENT:
updated_transformation_revision.content = update_code(
updated_transformation_revision
)
elif existing_transformation_revision is not None:
assert isinstance(
existing_transformation_revision.content, WorkflowContent
) # hint for mypy
existing_operator_ids: List[UUID] = []
for operator in existing_transformation_revision.content.operators:
existing_operator_ids.append(operator.id)
assert isinstance(
updated_transformation_revision.content, WorkflowContent
) # hint for mypy
for operator in updated_transformation_revision.content.operators:
if (
operator.type == Type.WORKFLOW
and operator.id not in existing_operator_ids
):
operator.state = (
State.DISABLED
if contains_deprecated(operator.transformation_id)
else operator.state
)
return updated_transformation_revision
def test_update_code():
component = TransformationRevision(
io_interface=IOInterface(inputs=[], outputs=[]),
name="Test Component",
description="A test component",
category="Tests",
id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
version_tag="1.0.0",
state="RELEASED",
type="COMPONENT",
released_timestamp="2019-12-01T12:00:00+00:00",
content=example_code,
test_wiring=[],
)
updated_component = TransformationRevision(
io_interface=IOInterface(inputs=[], outputs=[]),
name="Test Component",
description="A test component",
category="Tests",
id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
version_tag="1.0.1",
state="DRAFT",
type="COMPONENT",
content=example_code,
test_wiring=[],
)
new_code = update_code(updated_component)
assert """return {"z": x+y}""" in new_code
assert "c6eff22c-21c4-43c6-9ae1-b2bdfb944565" in new_code
assert "1.0.0" not in new_code
# test input without both start/stop markers
component.content = ""
new_code = update_code(component)
assert "pass" in new_code
# test input without only stop marker
component.content = "# ***** DO NOT EDIT LINES BELOW *****"
new_code = update_code(component)
# test with async def in function header
component.content = example_code_async
new_code = update_code(component)
assert "async def" in new_code
async def test_execute_for_component_without_hetdesrun_imports(
async_test_client, clean_test_db_engine
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
path = (
"./tests/data/components/"
"alerts-from-score_100_38f168ef-cb06-d89c-79b3-0cd823f32e9d"
".json"
)
component_tr_json = load_json(path)
wiring_json = {
"id": "38f168ef-cb06-d89c-79b3-0cd823f32e9d",
"name": "STANDARD-WIRING",
"inputWirings": [
{
"id": "8c249f92-4b81-457e-9371-24204d6b373b",
"workflowInputName": "scores",
"adapterId": "direct_provisioning",
"filters": {
"value": (
"{\n"
' "2020-01-03T08:20:03.000Z": 18.7,\n'
' "2020-01-01T01:15:27.000Z": 42.2,\n'
' "2020-01-03T08:20:04.000Z": 25.9\n'
"}"
)
},
},
{
"id": "0f0f97f7-1f5d-4f5d-be11-7c7b78d02129",
"workflowInputName": "threshold",
"adapterId": "direct_provisioning",
"filters": {"value": "30"},
},
],
"outputWirings": [],
}
tr = TransformationRevision(**component_tr_json)
tr.content = update_code(tr)
assert "COMPONENT_INFO" in tr.content
store_single_transformation_revision(tr)
async with async_test_client as ac:
response = await ac.post(
"/api/components/" + component_tr_json["id"] + "/execute",
json=wiring_json,
)
assert response.status_code == 200
assert "output_types_by_output_name" in response.json()
async def test_execute_latest_for_transformation_revision_works(
async_test_client, clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
tr_component_1 = TransformationRevision(**tr_json_component_1)
tr_component_1.content = update_code(tr_component_1)
store_single_transformation_revision(tr_component_1)
tr_component_1_new_revision = TransformationRevision(
**tr_json_component_1_new_revision)
tr_component_1_new_revision.content = update_code(
tr_component_1_new_revision)
tr_component_1_new_revision.release()
store_single_transformation_revision(tr_component_1_new_revision)
exec_latest_by_group_id_input = ExecLatestByGroupIdInput(
revision_group_id=tr_component_1.revision_group_id,
wiring=tr_component_1.test_wiring,
job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"),
)
async with async_test_client as ac:
response = await ac.post(
"/api/transformations/execute-latest",
json=json.loads(exec_latest_by_group_id_input.json()),
)
assert response.status_code == 200
resp_data = response.json()
assert "output_types_by_output_name" in resp_data
assert (resp_data["output_types_by_output_name"]["operator_output"]
== "STRING")
assert "job_id" in resp_data
assert UUID(resp_data["job_id"]) == UUID(
"1270547c-b224-461d-9387-e9d9d465bbe1")
async def test_execute_for_transformation_revision(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
tr_component_1 = TransformationRevision(**tr_json_component_1)
tr_component_1.content = update_code(tr_component_1)
store_single_transformation_revision(tr_component_1)
tr_workflow_2 = TransformationRevision(**tr_json_workflow_2_update)
store_single_transformation_revision(tr_workflow_2)
update_or_create_nesting(tr_workflow_2)
exec_by_id_input = ExecByIdInput(
id=tr_workflow_2.id,
wiring=tr_workflow_2.test_wiring,
job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"),
)
async with async_test_client as ac:
response = await ac.post(
"/api/transformations/execute",
json=json.loads(exec_by_id_input.json()),
)
assert response.status_code == 200
resp_data = response.json()
assert "output_types_by_output_name" in resp_data
assert "job_id" in resp_data
assert UUID(resp_data["job_id"]) == UUID(
"1270547c-b224-461d-9387-e9d9d465bbe1")
async def test_execute_for_separate_runtime_container(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
with mock.patch(
"hetdesrun.webservice.config.runtime_config",
is_runtime_service=False,
):
resp_mock = mock.Mock()
resp_mock.status_code = 200
resp_mock.json = mock.Mock(
return_value={
"output_results_by_output_name": {
"wf_output": 100
},
"output_types_by_output_name": {
"wf_output": "INT"
},
"result": "ok",
"job_id": "1270547c-b224-461d-9387-e9d9d465bbe1",
})
with mock.patch(
"hetdesrun.backend.execution.httpx.AsyncClient.post",
return_value=resp_mock,
) as mocked_post:
tr_component_1 = TransformationRevision(
**tr_json_component_1)
tr_component_1.content = update_code(tr_component_1)
store_single_transformation_revision(tr_component_1)
tr_workflow_2 = TransformationRevision(
**tr_json_workflow_2_update)
store_single_transformation_revision(tr_workflow_2)
update_or_create_nesting(tr_workflow_2)
exec_by_id_input = ExecByIdInput(
id=tr_workflow_2.id,
wiring=tr_workflow_2.test_wiring,
job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"),
)
async with async_test_client as ac:
response = await ac.post(
"/api/transformations/execute",
json=json.loads(exec_by_id_input.json()),
)
assert response.status_code == 200
resp_data = response.json()
assert "output_types_by_output_name" in resp_data
assert "job_id" in resp_data
assert UUID(resp_data["job_id"]) == UUID(
"1270547c-b224-461d-9387-e9d9d465bbe1")
mocked_post.assert_called_once()