async def compute_pipeline_details(
complete_dag: nx.DiGraph, pipeline_dag: nx.DiGraph,
comp_tasks: List[CompTaskAtDB]) -> PipelineDetails:
try:
# FIXME: this problem of cyclic graphs for control loops create all kinds of issues that must be fixed
# first pass, traversing in topological order to correctly get the dependencies, set the nodes states
await _set_computational_nodes_states(complete_dag)
except nx.NetworkXUnfeasible:
# not acyclic
pass
return PipelineDetails(
adjacency_list=nx.to_dict_of_lists(pipeline_dag),
node_states={
node_id: NodeState(
modified=node_data.get(kNODE_MODIFIED_STATE, False),
dependencies=node_data.get(kNODE_DEPENDENCIES_TO_COMPUTE,
set()),
currentStatus=next(
(task.state
for task in comp_tasks if str(task.node_id) == node_id),
RunningState.UNKNOWN,
),
)
for node_id, node_data in complete_dag.nodes.data()
if _is_node_computational(node_data.get("key", ""))
},
)
def fake_workbench_computational_pipeline_details(
fake_workbench_computational_adjacency_file: Path,
fake_workbench_node_states_file: Path,
) -> PipelineDetails:
adjacency_list = json.loads(fake_workbench_computational_adjacency_file.read_text())
node_states = json.loads(fake_workbench_node_states_file.read_text())
return PipelineDetails.parse_obj(
{"adjacency_list": adjacency_list, "node_states": node_states}
)
async def assert_computation_task_out_obj(
client: httpx.AsyncClient,
task_out: ComputationTaskGet,
*,
project: ProjectAtDB,
exp_task_state: RunningState,
exp_pipeline_details: PipelineDetails,
):
assert task_out.id == project.uuid
assert task_out.state == exp_task_state
assert task_out.url == f"{client.base_url}/v2/computations/{project.uuid}"
assert task_out.stop_url == (
f"{client.base_url}/v2/computations/{project.uuid}:stop"
if exp_task_state in [RunningState.PUBLISHED, RunningState.PENDING
] else None)
# check pipeline details contents
assert task_out.pipeline_details.dict() == exp_pipeline_details.dict()
def _convert_to_pipeline_details(
project: ProjectAtDB,
exp_pipeline_adj_list: Dict[int, List[str]],
exp_node_states: Dict[int, Dict[str, Any]],
) -> PipelineDetails:
workbench_node_uuids = list(project.workbench.keys())
converted_adj_list: Dict[NodeID, Dict[NodeID, List[NodeID]]] = {}
for node_key, next_nodes in exp_pipeline_adj_list.items():
converted_adj_list[NodeID(workbench_node_uuids[node_key])] = [
NodeID(workbench_node_uuids[n]) for n in next_nodes
]
converted_node_states: Dict[NodeID, NodeState] = {
NodeID(workbench_node_uuids[n]): NodeState(
modified=s["modified"],
dependencies={
workbench_node_uuids[dep_n]
for dep_n in s["dependencies"]
},
currentStatus=s.get("currentStatus", RunningState.NOT_STARTED),
)
for n, s in exp_node_states.items()
}
return PipelineDetails(adjacency_list=converted_adj_list,
node_states=converted_node_states)
async def test_nodeports_integration(
# pylint: disable=too-many-arguments
minimal_configuration: None,
cleanup_services_and_networks: None,
update_project_workbench_with_comp_tasks: Callable,
async_client: httpx.AsyncClient,
db_manager: DBManager,
user_db: Dict,
current_study: ProjectAtDB,
services_endpoint: Dict[str, URL],
workbench_dynamic_services: Dict[str, Node],
services_node_uuids: ServicesNodeUUIDs,
fake_dy_success: Dict[str, Any],
fake_dy_published: Dict[str, Any],
temp_dir: Path,
mocker: MockerFixture,
) -> None:
"""
Creates a new project with where the following connections
are defined: `sleeper:1.0.0` ->
`dy-static-file-server-dynamic-sidecar:2.0.0` ->
`dy-static-file-server-dynamic-sidecar-compose-spec:2.0.0`.
Both `dy-static-file-server-*` services are able to map the
inputs of the service to the outputs. Both services also
generate an internal state which is to be persisted
between runs.
Execution steps:
1. start all the dynamic services and make sure they are running
2. run the computational pipeline & trigger port retrievals
3. check that the outputs of the `sleeper` are the same as the
outputs of the `dy-static-file-server-dynamic-sidecar-compose-spec``
4. fetch the "state" via `docker/aioboto` for both dynamic services
5. start the dynamic-services and fetch the "state" via
`storage-data_manager API/aioboto` for both dynamic services
6. start the dynamic-services again, fetch the "state" via
`docker/aioboto` for both dynamic services
7. finally check that all states for both dynamic services match
NOTE: when the services are started using S3 as a backend
for saving the state, the state files are recovered via
`aioboto` instead of `docker` or `storage-data_manager API`.
"""
# STEP 1
dynamic_services_urls: Dict[
str, str] = await _wait_for_dynamic_services_to_be_running(
director_v2_client=async_client,
user_id=user_db["id"],
workbench_dynamic_services=workbench_dynamic_services,
current_study=current_study,
)
# STEP 2
response = await create_pipeline(
async_client,
project=current_study,
user_id=user_db["id"],
start_pipeline=True,
expected_response_status_code=status.HTTP_201_CREATED,
)
task_out = ComputationTaskGet.parse_obj(response.json())
# check the contents is correct: a pipeline that just started gets PUBLISHED
await assert_computation_task_out_obj(
async_client,
task_out,
project=current_study,
exp_task_state=RunningState.PUBLISHED,
exp_pipeline_details=PipelineDetails.parse_obj(fake_dy_published),
)
# wait for the computation to start
await assert_and_wait_for_pipeline_status(
async_client,
task_out.url,
user_db["id"],
current_study.uuid,
wait_for_states=[RunningState.STARTED],
)
# wait for the computation to finish (either by failing, success or abort)
task_out = await assert_and_wait_for_pipeline_status(
async_client, task_out.url, user_db["id"], current_study.uuid)
await assert_computation_task_out_obj(
async_client,
task_out,
project=current_study,
exp_task_state=RunningState.SUCCESS,
exp_pipeline_details=PipelineDetails.parse_obj(fake_dy_success),
)
update_project_workbench_with_comp_tasks(str(current_study.uuid))
# Trigger inputs pulling & outputs pushing on dynamic services
# Since there is no webserver monitoring postgres notifications
# trigger the call manually
# dump logs form started containers before retrieve
await _print_dynamic_sidecars_containers_logs_and_get_containers(
dynamic_services_urls)
await _assert_retrieve_completed(
director_v2_client=async_client,
service_uuid=services_node_uuids.dy,
dynamic_services_urls=dynamic_services_urls,
)
await _assert_retrieve_completed(
director_v2_client=async_client,
service_uuid=services_node_uuids.dy_compose_spec,
dynamic_services_urls=dynamic_services_urls,
)
# STEP 3
# pull data via nodeports
# storage config.py resolves env vars at import time, unlike newer settingslib
# configuration. patching the module with the correct url
mocker.patch(
"simcore_sdk.node_ports_common.config.STORAGE_ENDPOINT",
str(services_endpoint["storage"]).replace("http://", ""),
)
mapped_nodeports_values = await _get_mapped_nodeports_values(
user_db["id"], str(current_study.uuid), current_study.workbench,
db_manager)
await _assert_port_values(mapped_nodeports_values, services_node_uuids)
# STEP 4
# pylint: disable=protected-access
app_settings: AppSettings = async_client._transport.app.state.settings
r_clone_settings: RCloneSettings = (
app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.
DYNAMIC_SIDECAR_R_CLONE_SETTINGS)
if app_settings.DIRECTOR_V2_DEV_FEATURES_ENABLED:
await sleep_for(
WAIT_FOR_R_CLONE_VOLUME_TO_SYNC_DATA,
"Waiting for rclone to sync data from the docker volume",
)
dy_path_volume_before = (await _fetch_data_via_aioboto(
r_clone_settings=r_clone_settings,
dir_tag="dy",
temp_dir=temp_dir,
node_id=services_node_uuids.dy,
project_id=current_study.uuid,
) if app_settings.DIRECTOR_V2_DEV_FEATURES_ENABLED else await
_fetch_data_from_container(
dir_tag="dy",
service_uuid=services_node_uuids.dy,
temp_dir=temp_dir))
dy_compose_spec_path_volume_before = (
await _fetch_data_via_aioboto(
r_clone_settings=r_clone_settings,
dir_tag="dy_compose_spec",
temp_dir=temp_dir,
node_id=services_node_uuids.dy_compose_spec,
project_id=current_study.uuid,
) if app_settings.DIRECTOR_V2_DEV_FEATURES_ENABLED else await
_fetch_data_from_container(
dir_tag="dy_compose_spec",
service_uuid=services_node_uuids.dy_compose_spec,
temp_dir=temp_dir,
))
# STEP 5
# stop the services to make sure the data is saved to storage
await asyncio.gather(*(assert_stop_service(
director_v2_client=async_client,
service_uuid=service_uuid,
) for service_uuid in workbench_dynamic_services))
await _wait_for_dy_services_to_fully_stop(async_client)
if app_settings.DIRECTOR_V2_DEV_FEATURES_ENABLED:
await sleep_for(
WAIT_FOR_R_CLONE_VOLUME_TO_SYNC_DATA,
"Waiting for rclone to sync data from the docker volume",
)
dy_path_data_manager_before = (await _fetch_data_via_aioboto(
r_clone_settings=r_clone_settings,
dir_tag="dy",
temp_dir=temp_dir,
node_id=services_node_uuids.dy,
project_id=current_study.uuid,
) if app_settings.DIRECTOR_V2_DEV_FEATURES_ENABLED else await
_fetch_data_via_data_manager(
dir_tag="dy",
user_id=user_db["id"],
project_id=str(current_study.uuid),
service_uuid=services_node_uuids.dy,
temp_dir=temp_dir,
))
dy_compose_spec_path_data_manager_before = (
await _fetch_data_via_aioboto(
r_clone_settings=r_clone_settings,
dir_tag="dy_compose_spec",
temp_dir=temp_dir,
node_id=services_node_uuids.dy_compose_spec,
project_id=current_study.uuid,
) if app_settings.DIRECTOR_V2_DEV_FEATURES_ENABLED else await
_fetch_data_via_data_manager(
dir_tag="dy_compose_spec",
user_id=user_db["id"],
project_id=str(current_study.uuid),
service_uuid=services_node_uuids.dy_compose_spec,
temp_dir=temp_dir,
))
# STEP 6
await _wait_for_dynamic_services_to_be_running(
director_v2_client=async_client,
user_id=user_db["id"],
workbench_dynamic_services=workbench_dynamic_services,
current_study=current_study,
)
dy_path_volume_after = (await _fetch_data_via_aioboto(
r_clone_settings=r_clone_settings,
dir_tag="dy",
temp_dir=temp_dir,
node_id=services_node_uuids.dy,
project_id=current_study.uuid,
) if app_settings.DIRECTOR_V2_DEV_FEATURES_ENABLED else await
_fetch_data_from_container(
dir_tag="dy",
service_uuid=services_node_uuids.dy,
temp_dir=temp_dir))
dy_compose_spec_path_volume_after = (
await _fetch_data_via_aioboto(
r_clone_settings=r_clone_settings,
dir_tag="dy_compose_spec",
temp_dir=temp_dir,
node_id=services_node_uuids.dy_compose_spec,
project_id=current_study.uuid,
) if app_settings.DIRECTOR_V2_DEV_FEATURES_ENABLED else await
_fetch_data_from_container(
dir_tag="dy_compose_spec",
service_uuid=services_node_uuids.dy_compose_spec,
temp_dir=temp_dir,
))
# STEP 7
_assert_same_set(
_get_file_hashes_in_path(dy_path_volume_before),
_get_file_hashes_in_path(dy_path_data_manager_before),
_get_file_hashes_in_path(dy_path_volume_after),
)
_assert_same_set(
_get_file_hashes_in_path(dy_compose_spec_path_volume_before),
_get_file_hashes_in_path(dy_compose_spec_path_data_manager_before),
_get_file_hashes_in_path(dy_compose_spec_path_volume_after),
)