async def test_failed_or_aborted_task_cleans_output_files( scheduler: BaseCompScheduler, minimal_app: FastAPI, user_id: PositiveInt, aiopg_engine: Iterator[aiopg.sa.engine.Engine], # type: ignore mocked_dask_client_send_task: mock.MagicMock, published_project: PublishedProject, state: RunningState, mocked_clean_task_output_fct: mock.MagicMock, mocked_scheduler_task: None, ): # we do have a published project where the comp services are in PUBLISHED state # here we will artifically call the completion handler in the scheduler dask_scheduler = cast(DaskScheduler, scheduler) job_id = generate_dask_job_id( "simcore/service/comp/pytest/fake", "12.34.55", user_id, published_project.project.uuid, published_project.tasks[0].node_id, ) state_event = TaskStateEvent( job_id=job_id, msg=TaskOutputData.parse_obj({"output_1": "some fake data"}).json(), state=state, ) await dask_scheduler._on_task_completed(state_event) await assert_comp_tasks_state( aiopg_engine, published_project.project.uuid, [published_project.tasks[0].node_id], exp_state=state, ) mocked_clean_task_output_fct.assert_called_once()
async def test_parse_output_data( aiopg_engine: aiopg.sa.engine.Engine, # type: ignore published_project: PublishedProject, user_id: UserID, fake_io_schema: Dict[str, Dict[str, str]], fake_task_output_data: TaskOutputData, mocker: MockerFixture, ): # need some fakes set in the DB sleeper_task: CompTaskAtDB = published_project.tasks[1] no_outputs = {} await set_comp_task_outputs(aiopg_engine, sleeper_task.node_id, fake_io_schema, no_outputs) # mock the set_value function so we can test it is called correctly mocked_node_ports_set_value_fct = mocker.patch( "simcore_sdk.node_ports_v2.port.Port.set_value") # test dask_job_id = generate_dask_job_id( sleeper_task.image.name, sleeper_task.image.tag, user_id, published_project.project.uuid, sleeper_task.node_id, ) await parse_output_data(aiopg_engine, dask_job_id, fake_task_output_data) # the FileUrl types are converted to a pure url expected_values = { k: v.url if isinstance(v, FileUrl) else v for k, v in fake_task_output_data.items() } mocked_node_ports_set_value_fct.assert_has_calls( [mock.call(value) for value in expected_values.values()])
def creator(user: Dict[str, Any], project: ProjectAtDB, **overrides_kwargs) -> List[CompTaskAtDB]: created_tasks: List[CompTaskAtDB] = [] for internal_id, (node_id, node_data) in enumerate(project.workbench.items()): task_config = { "project_id": f"{project.uuid}", "node_id": f"{node_id}", "schema": { "inputs": {}, "outputs": {} }, "inputs": { key: json.loads(value.json(by_alias=True, exclude_unset=True)) if isinstance(value, BaseModel) else value for key, value in node_data.inputs.items() } if node_data.inputs else {}, "outputs": { key: json.loads(value.json(by_alias=True, exclude_unset=True)) if isinstance(value, BaseModel) else value for key, value in node_data.outputs.items() } if node_data.outputs else {}, "image": Image(name=node_data.key, tag=node_data.version).dict( # type: ignore by_alias=True, exclude_unset=True), # type: ignore "node_class": to_node_class(node_data.key), "internal_id": internal_id + 1, "submit": datetime.utcnow(), "job_id": generate_dask_job_id( service_key=node_data.key, service_version=node_data.version, user_id=user["id"], project_id=project.uuid, node_id=NodeID(node_id), ), } task_config.update(**overrides_kwargs) with postgres_db.connect() as conn: result = conn.execute(comp_tasks.insert().values( **task_config).returning(sa.literal_column("*"))) new_task = CompTaskAtDB.parse_obj(result.first()) created_tasks.append(new_task) created_task_ids.extend( [t.task_id for t in created_tasks if t.task_id]) return created_tasks
def test_dask_job_id_serialization( service_key: str, service_version: str, user_id: UserID, project_id: ProjectID, node_id: NodeID, ): dask_job_id = generate_dask_job_id(service_key, service_version, user_id, project_id, node_id) ( parsed_service_key, parsed_service_version, parsed_user_id, parsed_project_id, parsed_node_id, ) = parse_dask_job_id(dask_job_id) assert service_key == parsed_service_key assert service_version == parsed_service_version assert user_id == parsed_user_id assert project_id == parsed_project_id assert node_id == parsed_node_id