def test_create_task_output_from_task_throws_when_there_are_entries(
tmp_path: Path, faker: Faker):
task_output_schema = TaskOutputDataSchema.parse_obj({
"some_output": {
"required": True,
},
})
with pytest.raises(ValueError):
TaskOutputData.from_task_output(
schema=task_output_schema,
output_folder=tmp_path,
output_file_ext=faker.file_name(),
)
def test_create_task_output_from_task_throws_when_there_are_missing_files(
tmp_path: Path, faker: Faker):
task_output_schema = TaskOutputDataSchema.parse_obj({
"required_file_output": {
"required": True,
"url": "s3://some_file_url",
"mapping": "the_output_filename",
},
})
with pytest.raises(ValueError):
TaskOutputData.from_task_output(
schema=task_output_schema,
output_folder=tmp_path,
output_file_ext=faker.file_name(),
)
async def _retrieve_output_data(
self,
task_volumes: TaskSharedVolumes,
integration_version: version.Version,
) -> TaskOutputData:
try:
await self._publish_sidecar_log("Retrieving output data...")
logger.debug(
"following files are located in output folder %s:\n%s",
task_volumes.outputs_folder,
pformat(list(task_volumes.outputs_folder.rglob("*"))),
)
logger.debug(
"following outputs will be searched for:\n%s",
self.output_data_keys.json(indent=1),
)
output_data = TaskOutputData.from_task_output(
self.output_data_keys,
task_volumes.outputs_folder,
"outputs.json"
if integration_version > LEGACY_INTEGRATION_VERSION else
"output.json",
)
upload_tasks = []
for output_params in output_data.values():
if isinstance(output_params, FileUrl):
assert ( # nosec
output_params.file_mapping
), f"{output_params.json(indent=1)} expected resolved in TaskOutputData.from_task_output"
src_path = task_volumes.outputs_folder / output_params.file_mapping
upload_tasks.append(
push_file_to_remote(src_path, output_params.url,
self._publish_sidecar_log))
await asyncio.gather(*upload_tasks)
await self._publish_sidecar_log(
"All the output data were uploaded.")
logger.info("retrieved outputs data:\n%s",
output_data.json(indent=1))
return output_data
except (ValueError, ValidationError) as exc:
raise ServiceBadFormattedOutputError(
service_key=self.service_key,
service_version=self.service_version,
exc=exc,
) from exc
def test_create_task_output_from_task_does_not_throw_when_there_are_optional_entries(
tmp_path: Path, faker: Faker):
task_output_schema = TaskOutputDataSchema.parse_obj({
"some_output": {
"required": False,
},
})
task_output_data = TaskOutputData.from_task_output(
schema=task_output_schema,
output_folder=tmp_path,
output_file_ext=faker.file_name(),
)
assert len(task_output_data) == 0
def test_create_task_output_from_task_does_not_throw_when_there_are_optional_missing_files(
tmp_path: Path, faker: Faker):
task_output_schema = TaskOutputDataSchema.parse_obj({
"optional_file_output": {
"required": False,
"url": "s3://some_file_url",
"mapping": "the_output_filename",
},
})
task_output_data = TaskOutputData.from_task_output(
schema=task_output_schema,
output_folder=tmp_path,
output_file_ext=faker.file_name(),
)
assert len(task_output_data) == 0
def test_create_task_output_from_task_with_optional_fields_as_required(
tmp_path: Path, optional_fields_set: bool, faker: Faker):
for schema_example in TaskOutputDataSchema.Config.schema_extra["examples"]:
task_output_schema = TaskOutputDataSchema.parse_obj(schema_example)
outputs_file_name = _create_fake_outputs(task_output_schema, tmp_path,
optional_fields_set, faker)
task_output_data = TaskOutputData.from_task_output(
schema=task_output_schema,
output_folder=tmp_path,
output_file_ext=outputs_file_name,
)
assert task_output_data
for key, value in task_output_schema.items():
if not value.required and not optional_fields_set:
assert task_output_data.get(key) is None
if value.required or optional_fields_set:
assert task_output_data.get(key) is not None