def if_applicable_release_or_deprecate( existing_transformation_revision: Optional[TransformationRevision], updated_transformation_revision: TransformationRevision, ) -> TransformationRevision: if existing_transformation_revision is not None: if ( existing_transformation_revision.state == State.DRAFT and updated_transformation_revision.state == State.RELEASED ): logger.info( "release transformation revision %s", existing_transformation_revision.id, ) updated_transformation_revision.release() # prevent overwriting content during releasing updated_transformation_revision.content = ( existing_transformation_revision.content ) if ( existing_transformation_revision.state == State.RELEASED and updated_transformation_revision.state == State.DISABLED ): logger.info( "deprecate transformation revision %s", existing_transformation_revision.id, ) updated_transformation_revision = TransformationRevision( **existing_transformation_revision.dict() ) updated_transformation_revision.deprecate() # prevent overwriting content during deprecating updated_transformation_revision.content = ( existing_transformation_revision.content ) return updated_transformation_revision
async def test_execute_latest_for_transformation_revision_no_revision_in_db( async_test_client, clean_test_db_engine): patched_session = sessionmaker(clean_test_db_engine) with mock.patch( "hetdesrun.persistence.dbservice.nesting.Session", patched_session, ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", patched_session, ): tr_component_1 = TransformationRevision(**tr_json_component_1) tr_component_1.content = update_code(tr_component_1) exec_latest_by_group_id_input = ExecLatestByGroupIdInput( revision_group_id=tr_component_1.revision_group_id, wiring=tr_component_1.test_wiring, ) async with async_test_client as ac: response = await ac.post( "/api/transformations/execute-latest", json=json.loads(exec_latest_by_group_id_input.json()), ) assert response.status_code == 404 assert ( "no released transformation revisions with revision group id" in response.json()["detail"])
async def create_transformation_revision( transformation_revision: TransformationRevision, ) -> TransformationRevision: """Store a transformation revision in the data base.""" logger.info("create transformation revision %s", transformation_revision.id) if transformation_revision.type == Type.COMPONENT: logger.debug("transformation revision has type %s", Type.COMPONENT) transformation_revision.content = update_code(transformation_revision) logger.debug("generated code:\n%s", transformation_revision.content) try: store_single_transformation_revision(transformation_revision) logger.info("created transformation revision") except DBIntegrityError as e: raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e try: persisted_transformation_revision = read_single_transformation_revision( transformation_revision.id ) except DBNotFoundError as e: raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e logger.debug(persisted_transformation_revision.json()) return persisted_transformation_revision
def update_content( existing_transformation_revision: Optional[TransformationRevision], updated_transformation_revision: TransformationRevision, ) -> TransformationRevision: if updated_transformation_revision.type == Type.COMPONENT: updated_transformation_revision.content = update_code( updated_transformation_revision ) elif existing_transformation_revision is not None: assert isinstance( existing_transformation_revision.content, WorkflowContent ) # hint for mypy existing_operator_ids: List[UUID] = [] for operator in existing_transformation_revision.content.operators: existing_operator_ids.append(operator.id) assert isinstance( updated_transformation_revision.content, WorkflowContent ) # hint for mypy for operator in updated_transformation_revision.content.operators: if ( operator.type == Type.WORKFLOW and operator.id not in existing_operator_ids ): operator.state = ( State.DISABLED if contains_deprecated(operator.transformation_id) else operator.state ) return updated_transformation_revision
def test_update_code(): component = TransformationRevision( io_interface=IOInterface(inputs=[], outputs=[]), name="Test Component", description="A test component", category="Tests", id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", version_tag="1.0.0", state="RELEASED", type="COMPONENT", released_timestamp="2019-12-01T12:00:00+00:00", content=example_code, test_wiring=[], ) updated_component = TransformationRevision( io_interface=IOInterface(inputs=[], outputs=[]), name="Test Component", description="A test component", category="Tests", id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", version_tag="1.0.1", state="DRAFT", type="COMPONENT", content=example_code, test_wiring=[], ) new_code = update_code(updated_component) assert """return {"z": x+y}""" in new_code assert "c6eff22c-21c4-43c6-9ae1-b2bdfb944565" in new_code assert "1.0.0" not in new_code # test input without both start/stop markers component.content = "" new_code = update_code(component) assert "pass" in new_code # test input without only stop marker component.content = "# ***** DO NOT EDIT LINES BELOW *****" new_code = update_code(component) # test with async def in function header component.content = example_code_async new_code = update_code(component) assert "async def" in new_code
async def test_execute_for_component_without_hetdesrun_imports( async_test_client, clean_test_db_engine ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): path = ( "./tests/data/components/" "alerts-from-score_100_38f168ef-cb06-d89c-79b3-0cd823f32e9d" ".json" ) component_tr_json = load_json(path) wiring_json = { "id": "38f168ef-cb06-d89c-79b3-0cd823f32e9d", "name": "STANDARD-WIRING", "inputWirings": [ { "id": "8c249f92-4b81-457e-9371-24204d6b373b", "workflowInputName": "scores", "adapterId": "direct_provisioning", "filters": { "value": ( "{\n" ' "2020-01-03T08:20:03.000Z": 18.7,\n' ' "2020-01-01T01:15:27.000Z": 42.2,\n' ' "2020-01-03T08:20:04.000Z": 25.9\n' "}" ) }, }, { "id": "0f0f97f7-1f5d-4f5d-be11-7c7b78d02129", "workflowInputName": "threshold", "adapterId": "direct_provisioning", "filters": {"value": "30"}, }, ], "outputWirings": [], } tr = TransformationRevision(**component_tr_json) tr.content = update_code(tr) assert "COMPONENT_INFO" in tr.content store_single_transformation_revision(tr) async with async_test_client as ac: response = await ac.post( "/api/components/" + component_tr_json["id"] + "/execute", json=wiring_json, ) assert response.status_code == 200 assert "output_types_by_output_name" in response.json()
async def test_execute_latest_for_transformation_revision_works( async_test_client, clean_test_db_engine): patched_session = sessionmaker(clean_test_db_engine) with mock.patch( "hetdesrun.persistence.dbservice.nesting.Session", patched_session, ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", patched_session, ): tr_component_1 = TransformationRevision(**tr_json_component_1) tr_component_1.content = update_code(tr_component_1) store_single_transformation_revision(tr_component_1) tr_component_1_new_revision = TransformationRevision( **tr_json_component_1_new_revision) tr_component_1_new_revision.content = update_code( tr_component_1_new_revision) tr_component_1_new_revision.release() store_single_transformation_revision(tr_component_1_new_revision) exec_latest_by_group_id_input = ExecLatestByGroupIdInput( revision_group_id=tr_component_1.revision_group_id, wiring=tr_component_1.test_wiring, job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"), ) async with async_test_client as ac: response = await ac.post( "/api/transformations/execute-latest", json=json.loads(exec_latest_by_group_id_input.json()), ) assert response.status_code == 200 resp_data = response.json() assert "output_types_by_output_name" in resp_data assert (resp_data["output_types_by_output_name"]["operator_output"] == "STRING") assert "job_id" in resp_data assert UUID(resp_data["job_id"]) == UUID( "1270547c-b224-461d-9387-e9d9d465bbe1")
async def test_execute_for_transformation_revision(async_test_client, clean_test_db_engine): patched_session = sessionmaker(clean_test_db_engine) with mock.patch( "hetdesrun.persistence.dbservice.nesting.Session", patched_session, ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", patched_session, ): tr_component_1 = TransformationRevision(**tr_json_component_1) tr_component_1.content = update_code(tr_component_1) store_single_transformation_revision(tr_component_1) tr_workflow_2 = TransformationRevision(**tr_json_workflow_2_update) store_single_transformation_revision(tr_workflow_2) update_or_create_nesting(tr_workflow_2) exec_by_id_input = ExecByIdInput( id=tr_workflow_2.id, wiring=tr_workflow_2.test_wiring, job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"), ) async with async_test_client as ac: response = await ac.post( "/api/transformations/execute", json=json.loads(exec_by_id_input.json()), ) assert response.status_code == 200 resp_data = response.json() assert "output_types_by_output_name" in resp_data assert "job_id" in resp_data assert UUID(resp_data["job_id"]) == UUID( "1270547c-b224-461d-9387-e9d9d465bbe1")
async def test_execute_for_separate_runtime_container(async_test_client, clean_test_db_engine): patched_session = sessionmaker(clean_test_db_engine) with mock.patch( "hetdesrun.persistence.dbservice.nesting.Session", patched_session, ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", patched_session, ): with mock.patch( "hetdesrun.webservice.config.runtime_config", is_runtime_service=False, ): resp_mock = mock.Mock() resp_mock.status_code = 200 resp_mock.json = mock.Mock( return_value={ "output_results_by_output_name": { "wf_output": 100 }, "output_types_by_output_name": { "wf_output": "INT" }, "result": "ok", "job_id": "1270547c-b224-461d-9387-e9d9d465bbe1", }) with mock.patch( "hetdesrun.backend.execution.httpx.AsyncClient.post", return_value=resp_mock, ) as mocked_post: tr_component_1 = TransformationRevision( **tr_json_component_1) tr_component_1.content = update_code(tr_component_1) store_single_transformation_revision(tr_component_1) tr_workflow_2 = TransformationRevision( **tr_json_workflow_2_update) store_single_transformation_revision(tr_workflow_2) update_or_create_nesting(tr_workflow_2) exec_by_id_input = ExecByIdInput( id=tr_workflow_2.id, wiring=tr_workflow_2.test_wiring, job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"), ) async with async_test_client as ac: response = await ac.post( "/api/transformations/execute", json=json.loads(exec_by_id_input.json()), ) assert response.status_code == 200 resp_data = response.json() assert "output_types_by_output_name" in resp_data assert "job_id" in resp_data assert UUID(resp_data["job_id"]) == UUID( "1270547c-b224-461d-9387-e9d9d465bbe1") mocked_post.assert_called_once()