def test_node_requirements_correctly_convert_to_dask_resources(
req_example: Dict[str, Any]):
node_reqs = NodeRequirements(**req_example)
assert node_reqs
dask_resources = node_reqs.dict(exclude_unset=True, by_alias=True)
# all the dask resources shall be of type: RESOURCE_NAME: VALUE
for resource_key, resource_value in dask_resources.items():
assert isinstance(resource_key, str)
assert isinstance(resource_value, (int, float, str, bool))
async def test_too_many_resources_send_computation_task(
dask_client: DaskClient,
user_id: UserID,
project_id: ProjectID,
node_id: NodeID,
cluster_id: ClusterID,
mocked_node_ports: None,
mocked_user_completed_cb: mock.AsyncMock,
):
# create an image that needs a huge amount of CPU
image = Image(
name="simcore/services/comp/pytest",
tag="1.4.5",
node_requirements=NodeRequirements(
CPU=10000000000000000,
RAM=parse_obj_as(ByteSize, "128 MiB"),
MPI=None,
GPU=None,
),
) # type: ignore
fake_task = {node_id: image}
# let's have a big number of CPUs
with pytest.raises(InsuficientComputationalResourcesError):
await dask_client.send_computation_tasks(
user_id=user_id,
project_id=project_id,
cluster_id=cluster_id,
tasks=fake_task,
callback=mocked_user_completed_cb,
remote_fct=None,
)
mocked_user_completed_cb.assert_not_called()
def _image_to_req_params() -> List[Tuple[Image, Dict[str, Any]]]:
return [
(
Image(
name="simcore/services/comp/pytest",
tag="1.4.5",
node_requirements=NodeRequirements(CPU=1, RAM="128 MiB"),
),
{
"resources": {
"CPU": 1.0,
"RAM": 128 * 1024 * 1024
}
},
),
(
Image(
name="simcore/services/comp/pytest",
tag="1.4.5",
node_requirements=NodeRequirements(CPU=1, GPU=1,
RAM="256 MiB"),
),
{
"resources": {
"CPU": 1.0,
"GPU": 1.0,
"RAM": 256 * 1024 * 1024
},
},
),
(
Image(
name="simcore/services/comp/pytest",
tag="1.4.5",
node_requirements=NodeRequirements(CPU=2, RAM="128 MiB",
MPI=1),
),
{
"resources": {
"CPU": 2.0,
"MPI": 1.0,
"RAM": 128 * 1024 * 1024
},
},
),
]
def test_node_requirements_correctly_convert_to_dask_resources(
req_example: Dict[str, Any]
):
node_reqs = NodeRequirements(**req_example)
assert node_reqs
dask_resources = from_node_reqs_to_dask_resources(node_reqs)
# all the dask resources shall be of type: RESOURCE_NAME: VALUE
for resource_key, resource_value in dask_resources.items():
assert isinstance(resource_key, str)
assert isinstance(resource_value, (int, float, str, bool))
assert resource_value is not None
def cpu_image(node_id: NodeID) -> ImageParams:
image = Image(
name="simcore/services/comp/pytest/cpu_image",
tag="1.5.5",
node_requirements=NodeRequirements(CPU=1, RAM="128 MiB"),
)
return ImageParams(
image=image,
expected_annotations={
"resources": {
"CPU": 1.0,
"RAM": 128 * 1024 * 1024,
}
},
fake_task={node_id: image},
)
def mpi_image(node_id: NodeID) -> ImageParams:
image = Image(
name="simcore/services/comp/pytest/mpi_image",
tag="1.4.5123",
node_requirements=NodeRequirements(CPU=2, RAM="128 MiB", MPI=1),
)
return ImageParams(
image=image,
expected_annotations={
"resources": {
"CPU": 2.0,
"MPI": 1.0,
"RAM": 128 * 1024 * 1024,
},
},
fake_task={node_id: image},
)
def cpu_image(node_id: NodeID) -> ImageParams:
image = Image(
name="simcore/services/comp/pytest/cpu_image",
tag="1.5.5",
node_requirements=NodeRequirements(
CPU=1, RAM=parse_obj_as(ByteSize, "128 MiB"), GPU=None, MPI=None
),
) # type: ignore
return ImageParams(
image=image,
expected_annotations={
"resources": {
"CPU": 1.0,
"RAM": 128 * 1024 * 1024,
}
},
fake_tasks={node_id: image},
)
cluster_id=random.randint(cluster_id + 1, 100),
tasks=fake_task,
callback=mocked_done_callback_fct,
remote_fct=None,
)
assert (len(dask_client._taskid_to_future_map) == 0
), "dask client should not store any future here"
@pytest.mark.parametrize(
"image, expected_annotations",
[(
Image(
name="simcore/services/comp/pytest",
tag="1.4.5",
node_requirements=NodeRequirements(CPU=10000000000000000,
RAM="128 MiB"),
),
{
"resources": {
"CPU": 1.0,
"RAM": 128 * 1024 * 1024
}
},
)],
)
async def test_too_many_resource_send_computation_task(
dask_client: DaskClient,
user_id: UserID,
project_id: ProjectID,
node_id: NodeID,
cluster_id: ClusterID,
@celery_app.task
def mul(x, y):
return x * y
celery_worker.reload()
assert mul.delay(4, 4).get(timeout=10) == 16
@pytest.mark.parametrize(
"image, routing_queue",
[
(
Image(
name="simcore/services/comp/pytest",
tag="1.2.2",
node_requirements=NodeRequirements(CPU=1, RAM="128 MiB"),
),
"cpu",
),
(
Image(
name="simcore/services/comp/pytest",
tag="1.2.2",
node_requirements=NodeRequirements(CPU=1, GPU=1,
RAM="256 MiB"),
),
"gpu",
),
(
Image(
name="simcore/services/comp/pytest",