async def test_clean_task_output_and_log_files_if_invalid(
aiopg_engine: aiopg.sa.engine.Engine, # type: ignore
user_id: UserID,
published_project: PublishedProject,
mocked_node_ports_filemanager_fcts: Dict[str, mock.MagicMock],
entry_exists_returns: bool,
fake_io_schema: Dict[str, Dict[str, str]],
faker: Faker,
):
# since the presigned links for outputs and logs file are created
# BEFORE the task is actually run. In case there is a failure at running
# the task, these entries shall be cleaned up. The way to check this is
# by asking storage if these file really exist. If not they get deleted.
mocked_node_ports_filemanager_fcts[
"entry_exists"
].return_value = entry_exists_returns
sleeper_task = published_project.tasks[1]
# simulate pre-created file links
fake_outputs = {
key: SimCoreFileLink(store=0, path=faker.file_path()).dict(
by_alias=True, exclude_unset=True
)
for key, value_type in fake_io_schema.items()
if value_type["type"] == "data:*/*"
}
await set_comp_task_outputs(
aiopg_engine, sleeper_task.node_id, fake_io_schema, fake_outputs
)
# this should ask for the 2 files + the log file
await clean_task_output_and_log_files_if_invalid(
aiopg_engine,
user_id,
published_project.project.uuid,
published_project.tasks[1].node_id,
)
expected_calls = [
mock.call(
user_id=user_id,
store_id="0",
s3_object=f"{published_project.project.uuid}/{sleeper_task.node_id}/{next(iter(fake_io_schema[key].get('fileToKeyMap', {key:key})))}",
)
for key in fake_outputs.keys()
] + [
mock.call(
user_id=user_id,
store_id="0",
s3_object=f"{published_project.project.uuid}/{sleeper_task.node_id}/{_LOGS_FILE_NAME}",
)
]
mocked_node_ports_filemanager_fcts["entry_exists"].assert_has_calls(expected_calls)
if entry_exists_returns:
mocked_node_ports_filemanager_fcts["delete_file"].assert_not_called()
else:
mocked_node_ports_filemanager_fcts["delete_file"].assert_has_calls(
expected_calls
)
async def test_compute_input_data(
app_with_db: None,
aiopg_engine: aiopg.sa.engine.Engine, # type: ignore
async_client: httpx.AsyncClient,
user_id: UserID,
published_project: PublishedProject,
fake_io_schema: Dict[str, Dict[str, str]],
fake_io_data: Dict[str, Any],
faker: Faker,
mocker: MockerFixture,
):
sleeper_task: CompTaskAtDB = published_project.tasks[1]
# set some fake inputs
fake_inputs = {
key: SimCoreFileLink(store=0, path=faker.file_path()).dict(
by_alias=True, exclude_unset=True
)
if value_type["type"] == "data:*/*"
else fake_io_data[key]
for key, value_type in fake_io_schema.items()
}
await set_comp_task_inputs(
aiopg_engine, sleeper_task.node_id, fake_io_schema, fake_inputs
)
# mock the get_value function so we can test it is called correctly
def return_fake_input_value(*args, **kwargs):
for value, value_type in zip(fake_inputs.values(), fake_io_schema.values()):
if value_type["type"] == "data:*/*":
yield parse_obj_as(AnyUrl, faker.url())
else:
yield value
mocked_node_ports_get_value_fct = mocker.patch(
"simcore_sdk.node_ports_v2.port.Port.get_value",
autospec=True,
side_effect=return_fake_input_value(),
)
computed_input_data = await compute_input_data(
async_client._transport.app,
user_id,
published_project.project.uuid,
sleeper_task.node_id,
)
mocked_node_ports_get_value_fct.assert_has_calls(
[mock.call(mock.ANY) for n in fake_io_data.keys()]
)
assert computed_input_data.keys() == fake_io_data.keys()
def test_union_types_coercion():
# SEE https://pydantic-docs.helpmanual.io/usage/types/#unions
class Func(BaseModel):
input: InputTypes
output: OutputTypes
assert get_origin(InputTypes) is Union
assert get_origin(OutputTypes) is Union
#
# pydantic will attempt to 'match' any of the types defined under Union and will use the first one that matches
# NOTE: it is recommended that, when defining Union annotations, the most specific type is included first and followed by less specific types.
#
assert Func.schema()["properties"]["input"] == {
"title":
"Input",
"anyOf": [
{
"type": "boolean"
},
{
"type": "integer"
},
{
"type": "number"
},
{
"format": "json-string",
"type": "string"
},
{
"type": "string"
},
{
"$ref": "#/definitions/PortLink"
},
{
"$ref": "#/definitions/SimCoreFileLink"
},
{
"$ref": "#/definitions/DatCoreFileLink"
},
{
"$ref": "#/definitions/DownloadLink"
},
{
"type": "array",
"items": {}
},
{
"type": "object"
},
],
}
# integers ------------------------
model = Func.parse_obj({"input": "0", "output": 1})
print(model.json(indent=1))
assert model.input == 0
assert model.output == 1
# numbers and bool ------------------------
model = Func.parse_obj({"input": "0.5", "output": "false"})
print(model.json(indent=1))
assert model.input == 0.5
assert model.output == False
# (undefined) json string vs string ------------------------
model = Func.parse_obj({
"input": '{"w": 42, "z": false}', # NOTE: this is a raw json string
"output": "some/path/or/string",
})
print(model.json(indent=1))
assert model.input == {"w": 42, "z": False}
assert model.output == "some/path/or/string"
# (undefined) json string vs SimCoreFileLink.dict() ------------
assert SimCoreFileLink in get_args(OutputTypes)
example = SimCoreFileLink.parse_obj(
SimCoreFileLink.Config.schema_extra["examples"][0])
model = Func.parse_obj({
"input": '{"w": 42, "z": false}',
"output": example.dict(
exclude_unset=True), # NOTE: this is NOT a raw json string
})
print(model.json(indent=1))
assert model.input == {"w": 42, "z": False}
assert model.output == example
assert isinstance(model.output, SimCoreFileLink)
# json array and objects
model = Func.parse_obj({
"input": {
"w": 42,
"z": False
},
"output": [1, 2, 3, None]
})
print(model.json(indent=1))
assert model.input == {"w": 42, "z": False}
assert model.output == [1, 2, 3, None]
def generate_simcore_file_link() -> Dict[str, Any]:
return SimCoreFileLink(store=0, path=faker.file_path()).dict(
by_alias=True, exclude_unset=True
)
},
"d98878dbcffbb908ee6d96d3ca87cc0c083f75683488c50ce9c945bef0588047",
),
(
{
"inputs": {
"input_int":
12,
"input_bool":
True,
"input_string":
"string",
"input_downloadlink":
DownloadLink(downloadLink="http://httpbin.org/image/jpeg"),
"input_simcorelink":
SimCoreFileLink(store=0, path="/path/to/some/file"),
"input_portlink":
PortLink(nodeUuid=ANOTHER_NODE_ID,
output=ANOTHER_NODE_OUTPUT_KEY),
"input_null":
None,
},
"outputs": {
"output_int":
2,
"output_bool":
False,
"output_string":
"some string",
"output_simcorelink":
SimCoreFileLink(store=0, path="/path/to/some/file"),