async def update_project_node_state(
app: web.Application, user_id: int, project_id: str, node_id: str, new_state: str
) -> Dict:
log.debug(
"updating node %s current state in project %s for user %s",
node_id,
project_id,
user_id,
)
partial_workbench_data = {
node_id: {"state": {"currentStatus": new_state}},
}
if RunningState(new_state) in [
RunningState.PUBLISHED,
RunningState.PENDING,
RunningState.STARTED,
]:
partial_workbench_data[node_id]["progress"] = 0
elif RunningState(new_state) in [RunningState.SUCCESS, RunningState.FAILED]:
partial_workbench_data[node_id]["progress"] = 100
db = app[APP_PROJECT_DBAPI]
updated_project, _ = await db.patch_user_project_workbench(
partial_workbench_data=partial_workbench_data,
user_id=user_id,
project_uuid=project_id,
)
updated_project = await add_project_states_for_user(
user_id=user_id, project=updated_project, is_template=False, app=app
)
return updated_project
async def _assert_and_wait_for_pipeline_state(
client: TestClient,
project_id: str,
expected_state: RunningState,
expected_api_response: ExpectedResponse,
):
url_project_state = client.app.router["state_project"].url_for(
project_id=project_id)
assert url_project_state == URL(f"/{API_VTAG}/projects/{project_id}/state")
async for attempt in AsyncRetrying(
reraise=True,
stop=stop_after_delay(120),
wait=wait_fixed(5),
retry=retry_if_exception_type(AssertionError),
):
with attempt:
print(
f"--> waiting for pipeline to complete with {expected_state=} attempt {attempt.retry_state.attempt_number}..."
)
resp = await client.get(f"{url_project_state}")
data, error = await assert_status(resp, expected_api_response.ok)
assert "state" in data
assert "value" in data["state"]
received_study_state = RunningState(data["state"]["value"])
print(f"<-- received pipeline state: {received_study_state=}")
assert received_study_state == expected_state
print(
f"--> pipeline completed with state {received_study_state=}! "
f"That's great: {json_dumps(attempt.retry_state.retry_object.statistics)}",
)
def convert_state_from_state_type_enum_if_needed(cls, v):
if isinstance(v, str):
# try to convert to a StateType, if it fails the validations will continue
# and pydantic will try to convert it to a RunninState later on
with suppress(ValueError):
v = StateType(v)
if isinstance(v, StateType):
return RunningState(DB_TO_RUNNING_STATE[StateType(v)])
return v
async def get_pipeline_state(
app: web.Application, user_id: PositiveInt, project_id: UUID
) -> RunningState:
director2_settings: Directorv2Settings = get_settings(app)
backend_url = URL(
f"{director2_settings.endpoint}/computations/{project_id}"
).update_query(user_id=user_id)
# request to director-v2
try:
computation_task_out, _ = await _request_director_v2(app, "GET", backend_url)
except _DirectorServiceError:
log.warning(
"getting pipeline state for project %s failed. state is then %s",
project_id,
RunningState.UNKNOWN,
)
return RunningState.UNKNOWN
return RunningState(computation_task_out["state"])
async def test_run_pipeline_and_check_state(
client: TestClient,
sleeper_service: Dict[str, str],
postgres_session: sa.orm.session.Session,
logged_user: Dict[str, Any],
user_project: Dict[str, Any],
fake_workbench_adjacency_list: Dict[str, Any],
user_role: UserRole,
expected: ExpectedResponse,
):
project_id = user_project["uuid"]
fake_workbench_payload = user_project["workbench"]
url_start = client.app.router["start_pipeline"].url_for(
project_id=project_id)
assert url_start == URL(
f"/{API_VTAG}/computation/pipeline/{project_id}:start")
# POST /v0/computation/pipeline/{project_id}:start
resp = await client.post(f"{url_start}")
data, error = await assert_status(resp, expected.created)
if error:
return
assert "pipeline_id" in data
assert data["pipeline_id"] == project_id
_assert_db_contents(
project_id,
postgres_session,
fake_workbench_payload,
fake_workbench_adjacency_list,
check_outputs=False,
)
url_project_state = client.app.router["state_project"].url_for(
project_id=project_id)
assert url_project_state == URL(f"/{API_VTAG}/projects/{project_id}/state")
running_state_order_lookup = {
RunningState.UNKNOWN: 0,
RunningState.NOT_STARTED: 1,
RunningState.PUBLISHED: 2,
RunningState.PENDING: 3,
RunningState.STARTED: 4,
RunningState.RETRY: 5,
RunningState.SUCCESS: 6,
RunningState.FAILED: 6,
RunningState.ABORTED: 6,
}
assert all( # pylint: disable=use-a-generator
[k in running_state_order_lookup for k in RunningState.__members__
]), "there are missing members in the order lookup, please complete!"
pipeline_state = RunningState.UNKNOWN
start = time.monotonic()
async for attempt in AsyncRetrying(
reraise=True,
stop=stop_after_delay(120),
wait=wait_fixed(1),
retry=retry_if_exception_type(ValueError),
):
with attempt:
print(
f"--> waiting for pipeline to complete attempt {attempt.retry_state.attempt_number}..."
)
resp = await client.get(f"{url_project_state}")
data, error = await assert_status(resp, expected.ok)
assert "state" in data
assert "value" in data["state"]
received_study_state = RunningState(data["state"]["value"])
print(f"--> project computation state {received_study_state=}")
assert (
running_state_order_lookup[received_study_state] >=
running_state_order_lookup[pipeline_state]
), (f"the received state {received_study_state} shall be greater "
f"or equal to the previous state {pipeline_state}")
assert received_study_state not in [
RunningState.ABORTED,
RunningState.FAILED,
], "the pipeline did not end up successfully"
pipeline_state = received_study_state
if received_study_state != RunningState.SUCCESS:
raise ValueError
print(
f"--> pipeline completed with state {received_study_state=}! That's great: {json_dumps(attempt.retry_state.retry_object.statistics)}",
)
assert pipeline_state == RunningState.SUCCESS
comp_tasks_in_db: Dict[NodeIdStr, Any] = _get_computational_tasks_from_db(
project_id, postgres_session)
assert all( # pylint: disable=use-a-generator
[t.state == StateType.SUCCESS for t in comp_tasks_in_db.values()
]), ("the individual computational services are not finished! "
f"Expected to be completed, got {comp_tasks_in_db=}")
# we need to wait until the webserver has updated the projects DB
await _assert_and_wait_for_comp_task_states_to_be_transmitted_in_projects(
project_id, postgres_session)
print(
f"<-- pipeline completed successfully in {time.monotonic() - start} seconds"
)
def is_pipeline_running(pipeline_state: RunningState) -> bool:
return pipeline_state.is_running()
def is_pipeline_stopped(pipeline_state: RunningState) -> bool:
return not pipeline_state.is_running()
def convert_state_from_db(db_state: StateType) -> RunningState:
return RunningState(DB_TO_RUNNING_STATE[StateType(db_state)])
def convert_state_if_needed(cls, v):
if isinstance(v, StateType):
return RunningState(DB_TO_RUNNING_STATE[StateType(v)])
return v