async def test_execute_latest_for_transformation_revision_no_revision_in_db( async_test_client, clean_test_db_engine): patched_session = sessionmaker(clean_test_db_engine) with mock.patch( "hetdesrun.persistence.dbservice.nesting.Session", patched_session, ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", patched_session, ): tr_component_1 = TransformationRevision(**tr_json_component_1) tr_component_1.content = update_code(tr_component_1) exec_latest_by_group_id_input = ExecLatestByGroupIdInput( revision_group_id=tr_component_1.revision_group_id, wiring=tr_component_1.test_wiring, ) async with async_test_client as ac: response = await ac.post( "/api/transformations/execute-latest", json=json.loads(exec_latest_by_group_id_input.json()), ) assert response.status_code == 404 assert ( "no released transformation revisions with revision group id" in response.json()["detail"])
async def create_transformation_revision( transformation_revision: TransformationRevision, ) -> TransformationRevision: """Store a transformation revision in the data base.""" logger.info("create transformation revision %s", transformation_revision.id) if transformation_revision.type == Type.COMPONENT: logger.debug("transformation revision has type %s", Type.COMPONENT) transformation_revision.content = update_code(transformation_revision) logger.debug("generated code:\n%s", transformation_revision.content) try: store_single_transformation_revision(transformation_revision) logger.info("created transformation revision") except DBIntegrityError as e: raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e try: persisted_transformation_revision = read_single_transformation_revision( transformation_revision.id ) except DBNotFoundError as e: raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e logger.debug(persisted_transformation_revision.json()) return persisted_transformation_revision
def update_content( existing_transformation_revision: Optional[TransformationRevision], updated_transformation_revision: TransformationRevision, ) -> TransformationRevision: if updated_transformation_revision.type == Type.COMPONENT: updated_transformation_revision.content = update_code( updated_transformation_revision ) elif existing_transformation_revision is not None: assert isinstance( existing_transformation_revision.content, WorkflowContent ) # hint for mypy existing_operator_ids: List[UUID] = [] for operator in existing_transformation_revision.content.operators: existing_operator_ids.append(operator.id) assert isinstance( updated_transformation_revision.content, WorkflowContent ) # hint for mypy for operator in updated_transformation_revision.content.operators: if ( operator.type == Type.WORKFLOW and operator.id not in existing_operator_ids ): operator.state = ( State.DISABLED if contains_deprecated(operator.transformation_id) else operator.state ) return updated_transformation_revision
async def test_execute_for_workflow_dto(async_test_client, clean_test_db_engine): patched_session = sessionmaker(clean_test_db_engine) with mock.patch( "hetdesrun.persistence.dbservice.nesting.Session", patched_session, ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", patched_session, ): component_dto = ComponentRevisionFrontendDto( **dto_json_component_1) tr_component = component_dto.to_transformation_revision() tr_component.content = update_code(tr_component) store_single_transformation_revision(tr_component) tr_workflow_2 = WorkflowRevisionFrontendDto( **dto_json_workflow_2_update).to_transformation_revision() tr_workflow_2.content.inputs[0].name = "wf_input" tr_workflow_2.content.outputs[0].name = "wf_output" tr_workflow_2.content.links.append( Link( start=Vertex( operator=None, connector=Connector.from_io( tr_workflow_2.content.inputs[0]), ), end=Vertex( operator=tr_workflow_2.content.operators[0].id, connector=tr_workflow_2.content.operators[0].inputs[0], ), )) tr_workflow_2.content.links.append( Link( start=Vertex( operator=tr_workflow_2.content.operators[0].id, connector=tr_workflow_2.content.operators[0]. outputs[0], ), end=Vertex( operator=None, connector=Connector.from_io( tr_workflow_2.content.outputs[0]), ), )) tr_workflow_2.io_interface.inputs[0].name = "wf_input" tr_workflow_2.io_interface.outputs[0].name = "wf_output" store_single_transformation_revision(tr_workflow_2) update_or_create_nesting(tr_workflow_2) async with async_test_client as ac: response = await ac.post( "/api/workflows/" + str(get_uuid_from_seed("workflow 2")) + "/execute", json=dto_json_wiring, ) assert response.status_code == 200 assert "output_types_by_output_name" in response.json()
def test_update_code(): component = TransformationRevision( io_interface=IOInterface(inputs=[], outputs=[]), name="Test Component", description="A test component", category="Tests", id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", version_tag="1.0.0", state="RELEASED", type="COMPONENT", released_timestamp="2019-12-01T12:00:00+00:00", content=example_code, test_wiring=[], ) updated_component = TransformationRevision( io_interface=IOInterface(inputs=[], outputs=[]), name="Test Component", description="A test component", category="Tests", id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", version_tag="1.0.1", state="DRAFT", type="COMPONENT", content=example_code, test_wiring=[], ) new_code = update_code(updated_component) assert """return {"z": x+y}""" in new_code assert "c6eff22c-21c4-43c6-9ae1-b2bdfb944565" in new_code assert "1.0.0" not in new_code # test input without both start/stop markers component.content = "" new_code = update_code(component) assert "pass" in new_code # test input without only stop marker component.content = "# ***** DO NOT EDIT LINES BELOW *****" new_code = update_code(component) # test with async def in function header component.content = example_code_async new_code = update_code(component) assert "async def" in new_code
def test_update_code(): func_body = """return {"z":x + y}""" test_code = example_code.replace("pass", func_body) new_code = update_code(test_code, {}, {}) assert func_body in new_code assert "pass" not in new_code # test with no code (new code generation) new_code = update_code(None, {}, {}) assert "pass" in new_code # test input without both start/stop markers new_code = update_code(" ", {}, {}) assert "pass" in new_code # test input without only stop marker new_code = update_code("# ***** DO NOT EDIT LINES BELOW *****", {}, {}) assert "pass" in new_code
async def test_execute_for_component_without_hetdesrun_imports( async_test_client, clean_test_db_engine ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): path = ( "./tests/data/components/" "alerts-from-score_100_38f168ef-cb06-d89c-79b3-0cd823f32e9d" ".json" ) component_tr_json = load_json(path) wiring_json = { "id": "38f168ef-cb06-d89c-79b3-0cd823f32e9d", "name": "STANDARD-WIRING", "inputWirings": [ { "id": "8c249f92-4b81-457e-9371-24204d6b373b", "workflowInputName": "scores", "adapterId": "direct_provisioning", "filters": { "value": ( "{\n" ' "2020-01-03T08:20:03.000Z": 18.7,\n' ' "2020-01-01T01:15:27.000Z": 42.2,\n' ' "2020-01-03T08:20:04.000Z": 25.9\n' "}" ) }, }, { "id": "0f0f97f7-1f5d-4f5d-be11-7c7b78d02129", "workflowInputName": "threshold", "adapterId": "direct_provisioning", "filters": {"value": "30"}, }, ], "outputWirings": [], } tr = TransformationRevision(**component_tr_json) tr.content = update_code(tr) assert "COMPONENT_INFO" in tr.content store_single_transformation_revision(tr) async with async_test_client as ac: response = await ac.post( "/api/components/" + component_tr_json["id"] + "/execute", json=wiring_json, ) assert response.status_code == 200 assert "output_types_by_output_name" in response.json()
async def codegen_service(codegen_input: CodeBody,) -> GeneratedCode: """Service for generating and updating code stubs""" logger.info("CODEGEN INPUT JSON:\n%s", model_to_pretty_json_str(codegen_input)) return GeneratedCode( code=update_code( codegen_input.code, {c.name: c.type for c in codegen_input.inputs}, {c.name: c.type for c in codegen_input.outputs}, ) )
async def test_execute_latest_for_transformation_revision_works( async_test_client, clean_test_db_engine): patched_session = sessionmaker(clean_test_db_engine) with mock.patch( "hetdesrun.persistence.dbservice.nesting.Session", patched_session, ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", patched_session, ): tr_component_1 = TransformationRevision(**tr_json_component_1) tr_component_1.content = update_code(tr_component_1) store_single_transformation_revision(tr_component_1) tr_component_1_new_revision = TransformationRevision( **tr_json_component_1_new_revision) tr_component_1_new_revision.content = update_code( tr_component_1_new_revision) tr_component_1_new_revision.release() store_single_transformation_revision(tr_component_1_new_revision) exec_latest_by_group_id_input = ExecLatestByGroupIdInput( revision_group_id=tr_component_1.revision_group_id, wiring=tr_component_1.test_wiring, job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"), ) async with async_test_client as ac: response = await ac.post( "/api/transformations/execute-latest", json=json.loads(exec_latest_by_group_id_input.json()), ) assert response.status_code == 200 resp_data = response.json() assert "output_types_by_output_name" in resp_data assert (resp_data["output_types_by_output_name"]["operator_output"] == "STRING") assert "job_id" in resp_data assert UUID(resp_data["job_id"]) == UUID( "1270547c-b224-461d-9387-e9d9d465bbe1")
async def create_component_revision( component_dto: ComponentRevisionFrontendDto, ) -> ComponentRevisionFrontendDto: """Store a transformation revision of type component in the data base. This endpoint is deprecated and will be removed soon, use POST /api/transformations/ instead. """ logger.info("create new component") try: transformation_revision = component_dto.to_transformation_revision( documentation=("# New Component/Workflow\n" "## Description\n" "## Inputs\n" "## Outputs\n" "## Details\n" "## Examples\n")) except ValidationError as e: raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) from e logger.debug("generate code") transformation_revision.content = update_code(transformation_revision) logger.debug("generated code:\n%s", component_dto.code) try: store_single_transformation_revision(transformation_revision) logger.info("created new component") except DBIntegrityError as e: raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e persisted_transformation_revision = read_single_transformation_revision( transformation_revision.id) persisted_component_dto = ComponentRevisionFrontendDto.from_transformation_revision( persisted_transformation_revision) logger.debug(persisted_component_dto.json()) return persisted_component_dto
async def test_execute_for_transformation_revision(async_test_client, clean_test_db_engine): patched_session = sessionmaker(clean_test_db_engine) with mock.patch( "hetdesrun.persistence.dbservice.nesting.Session", patched_session, ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", patched_session, ): tr_component_1 = TransformationRevision(**tr_json_component_1) tr_component_1.content = update_code(tr_component_1) store_single_transformation_revision(tr_component_1) tr_workflow_2 = TransformationRevision(**tr_json_workflow_2_update) store_single_transformation_revision(tr_workflow_2) update_or_create_nesting(tr_workflow_2) exec_by_id_input = ExecByIdInput( id=tr_workflow_2.id, wiring=tr_workflow_2.test_wiring, job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"), ) async with async_test_client as ac: response = await ac.post( "/api/transformations/execute", json=json.loads(exec_by_id_input.json()), ) assert response.status_code == 200 resp_data = response.json() assert "output_types_by_output_name" in resp_data assert "job_id" in resp_data assert UUID(resp_data["job_id"]) == UUID( "1270547c-b224-461d-9387-e9d9d465bbe1")
async def test_execute_for_separate_runtime_container(async_test_client, clean_test_db_engine): patched_session = sessionmaker(clean_test_db_engine) with mock.patch( "hetdesrun.persistence.dbservice.nesting.Session", patched_session, ): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", patched_session, ): with mock.patch( "hetdesrun.webservice.config.runtime_config", is_runtime_service=False, ): resp_mock = mock.Mock() resp_mock.status_code = 200 resp_mock.json = mock.Mock( return_value={ "output_results_by_output_name": { "wf_output": 100 }, "output_types_by_output_name": { "wf_output": "INT" }, "result": "ok", "job_id": "1270547c-b224-461d-9387-e9d9d465bbe1", }) with mock.patch( "hetdesrun.backend.execution.httpx.AsyncClient.post", return_value=resp_mock, ) as mocked_post: tr_component_1 = TransformationRevision( **tr_json_component_1) tr_component_1.content = update_code(tr_component_1) store_single_transformation_revision(tr_component_1) tr_workflow_2 = TransformationRevision( **tr_json_workflow_2_update) store_single_transformation_revision(tr_workflow_2) update_or_create_nesting(tr_workflow_2) exec_by_id_input = ExecByIdInput( id=tr_workflow_2.id, wiring=tr_workflow_2.test_wiring, job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"), ) async with async_test_client as ac: response = await ac.post( "/api/transformations/execute", json=json.loads(exec_by_id_input.json()), ) assert response.status_code == 200 resp_data = response.json() assert "output_types_by_output_name" in resp_data assert "job_id" in resp_data assert UUID(resp_data["job_id"]) == UUID( "1270547c-b224-461d-9387-e9d9d465bbe1") mocked_post.assert_called_once()