async def update_project_node_state( app: web.Application, user_id: int, project_id: str, node_id: str, new_state: str ) -> Dict: log.debug( "updating node %s current state in project %s for user %s", node_id, project_id, user_id, ) partial_workbench_data = { node_id: {"state": {"currentStatus": new_state}}, } if RunningState(new_state) in [ RunningState.PUBLISHED, RunningState.PENDING, RunningState.STARTED, ]: partial_workbench_data[node_id]["progress"] = 0 elif RunningState(new_state) in [RunningState.SUCCESS, RunningState.FAILED]: partial_workbench_data[node_id]["progress"] = 100 db = app[APP_PROJECT_DBAPI] updated_project, _ = await db.patch_user_project_workbench( partial_workbench_data=partial_workbench_data, user_id=user_id, project_uuid=project_id, ) updated_project = await add_project_states_for_user( user_id=user_id, project=updated_project, is_template=False, app=app ) return updated_project
async def _assert_and_wait_for_pipeline_state( client: TestClient, project_id: str, expected_state: RunningState, expected_api_response: ExpectedResponse, ): url_project_state = client.app.router["state_project"].url_for( project_id=project_id) assert url_project_state == URL(f"/{API_VTAG}/projects/{project_id}/state") async for attempt in AsyncRetrying( reraise=True, stop=stop_after_delay(120), wait=wait_fixed(5), retry=retry_if_exception_type(AssertionError), ): with attempt: print( f"--> waiting for pipeline to complete with {expected_state=} attempt {attempt.retry_state.attempt_number}..." ) resp = await client.get(f"{url_project_state}") data, error = await assert_status(resp, expected_api_response.ok) assert "state" in data assert "value" in data["state"] received_study_state = RunningState(data["state"]["value"]) print(f"<-- received pipeline state: {received_study_state=}") assert received_study_state == expected_state print( f"--> pipeline completed with state {received_study_state=}! " f"That's great: {json_dumps(attempt.retry_state.retry_object.statistics)}", )
def convert_state_from_state_type_enum_if_needed(cls, v): if isinstance(v, str): # try to convert to a StateType, if it fails the validations will continue # and pydantic will try to convert it to a RunninState later on with suppress(ValueError): v = StateType(v) if isinstance(v, StateType): return RunningState(DB_TO_RUNNING_STATE[StateType(v)]) return v
async def get_pipeline_state( app: web.Application, user_id: PositiveInt, project_id: UUID ) -> RunningState: director2_settings: Directorv2Settings = get_settings(app) backend_url = URL( f"{director2_settings.endpoint}/computations/{project_id}" ).update_query(user_id=user_id) # request to director-v2 try: computation_task_out, _ = await _request_director_v2(app, "GET", backend_url) except _DirectorServiceError: log.warning( "getting pipeline state for project %s failed. state is then %s", project_id, RunningState.UNKNOWN, ) return RunningState.UNKNOWN return RunningState(computation_task_out["state"])
async def test_run_pipeline_and_check_state( client: TestClient, sleeper_service: Dict[str, str], postgres_session: sa.orm.session.Session, logged_user: Dict[str, Any], user_project: Dict[str, Any], fake_workbench_adjacency_list: Dict[str, Any], user_role: UserRole, expected: ExpectedResponse, ): project_id = user_project["uuid"] fake_workbench_payload = user_project["workbench"] url_start = client.app.router["start_pipeline"].url_for( project_id=project_id) assert url_start == URL( f"/{API_VTAG}/computation/pipeline/{project_id}:start") # POST /v0/computation/pipeline/{project_id}:start resp = await client.post(f"{url_start}") data, error = await assert_status(resp, expected.created) if error: return assert "pipeline_id" in data assert data["pipeline_id"] == project_id _assert_db_contents( project_id, postgres_session, fake_workbench_payload, fake_workbench_adjacency_list, check_outputs=False, ) url_project_state = client.app.router["state_project"].url_for( project_id=project_id) assert url_project_state == URL(f"/{API_VTAG}/projects/{project_id}/state") running_state_order_lookup = { RunningState.UNKNOWN: 0, RunningState.NOT_STARTED: 1, RunningState.PUBLISHED: 2, RunningState.PENDING: 3, RunningState.STARTED: 4, RunningState.RETRY: 5, RunningState.SUCCESS: 6, RunningState.FAILED: 6, RunningState.ABORTED: 6, } assert all( # pylint: disable=use-a-generator [k in running_state_order_lookup for k in RunningState.__members__ ]), "there are missing members in the order lookup, please complete!" pipeline_state = RunningState.UNKNOWN start = time.monotonic() async for attempt in AsyncRetrying( reraise=True, stop=stop_after_delay(120), wait=wait_fixed(1), retry=retry_if_exception_type(ValueError), ): with attempt: print( f"--> waiting for pipeline to complete attempt {attempt.retry_state.attempt_number}..." ) resp = await client.get(f"{url_project_state}") data, error = await assert_status(resp, expected.ok) assert "state" in data assert "value" in data["state"] received_study_state = RunningState(data["state"]["value"]) print(f"--> project computation state {received_study_state=}") assert ( running_state_order_lookup[received_study_state] >= running_state_order_lookup[pipeline_state] ), (f"the received state {received_study_state} shall be greater " f"or equal to the previous state {pipeline_state}") assert received_study_state not in [ RunningState.ABORTED, RunningState.FAILED, ], "the pipeline did not end up successfully" pipeline_state = received_study_state if received_study_state != RunningState.SUCCESS: raise ValueError print( f"--> pipeline completed with state {received_study_state=}! That's great: {json_dumps(attempt.retry_state.retry_object.statistics)}", ) assert pipeline_state == RunningState.SUCCESS comp_tasks_in_db: Dict[NodeIdStr, Any] = _get_computational_tasks_from_db( project_id, postgres_session) assert all( # pylint: disable=use-a-generator [t.state == StateType.SUCCESS for t in comp_tasks_in_db.values() ]), ("the individual computational services are not finished! " f"Expected to be completed, got {comp_tasks_in_db=}") # we need to wait until the webserver has updated the projects DB await _assert_and_wait_for_comp_task_states_to_be_transmitted_in_projects( project_id, postgres_session) print( f"<-- pipeline completed successfully in {time.monotonic() - start} seconds" )
def is_pipeline_running(pipeline_state: RunningState) -> bool: return pipeline_state.is_running()
def is_pipeline_stopped(pipeline_state: RunningState) -> bool: return not pipeline_state.is_running()
def convert_state_from_db(db_state: StateType) -> RunningState: return RunningState(DB_TO_RUNNING_STATE[StateType(db_state)])
def convert_state_if_needed(cls, v): if isinstance(v, StateType): return RunningState(DB_TO_RUNNING_STATE[StateType(v)]) return v