def test_create_task_output_from_task_throws_when_there_are_entries( tmp_path: Path, faker: Faker): task_output_schema = TaskOutputDataSchema.parse_obj({ "some_output": { "required": True, }, }) with pytest.raises(ValueError): TaskOutputData.from_task_output( schema=task_output_schema, output_folder=tmp_path, output_file_ext=faker.file_name(), )
def test_create_task_output_from_task_throws_when_there_are_missing_files( tmp_path: Path, faker: Faker): task_output_schema = TaskOutputDataSchema.parse_obj({ "required_file_output": { "required": True, "url": "s3://some_file_url", "mapping": "the_output_filename", }, }) with pytest.raises(ValueError): TaskOutputData.from_task_output( schema=task_output_schema, output_folder=tmp_path, output_file_ext=faker.file_name(), )
async def _retrieve_output_data( self, task_volumes: TaskSharedVolumes, integration_version: version.Version, ) -> TaskOutputData: try: await self._publish_sidecar_log("Retrieving output data...") logger.debug( "following files are located in output folder %s:\n%s", task_volumes.outputs_folder, pformat(list(task_volumes.outputs_folder.rglob("*"))), ) logger.debug( "following outputs will be searched for:\n%s", self.output_data_keys.json(indent=1), ) output_data = TaskOutputData.from_task_output( self.output_data_keys, task_volumes.outputs_folder, "outputs.json" if integration_version > LEGACY_INTEGRATION_VERSION else "output.json", ) upload_tasks = [] for output_params in output_data.values(): if isinstance(output_params, FileUrl): assert ( # nosec output_params.file_mapping ), f"{output_params.json(indent=1)} expected resolved in TaskOutputData.from_task_output" src_path = task_volumes.outputs_folder / output_params.file_mapping upload_tasks.append( push_file_to_remote(src_path, output_params.url, self._publish_sidecar_log)) await asyncio.gather(*upload_tasks) await self._publish_sidecar_log( "All the output data were uploaded.") logger.info("retrieved outputs data:\n%s", output_data.json(indent=1)) return output_data except (ValueError, ValidationError) as exc: raise ServiceBadFormattedOutputError( service_key=self.service_key, service_version=self.service_version, exc=exc, ) from exc
def test_create_task_output_from_task_does_not_throw_when_there_are_optional_entries( tmp_path: Path, faker: Faker): task_output_schema = TaskOutputDataSchema.parse_obj({ "some_output": { "required": False, }, }) task_output_data = TaskOutputData.from_task_output( schema=task_output_schema, output_folder=tmp_path, output_file_ext=faker.file_name(), ) assert len(task_output_data) == 0
def test_create_task_output_from_task_does_not_throw_when_there_are_optional_missing_files( tmp_path: Path, faker: Faker): task_output_schema = TaskOutputDataSchema.parse_obj({ "optional_file_output": { "required": False, "url": "s3://some_file_url", "mapping": "the_output_filename", }, }) task_output_data = TaskOutputData.from_task_output( schema=task_output_schema, output_folder=tmp_path, output_file_ext=faker.file_name(), ) assert len(task_output_data) == 0
def test_create_task_output_from_task_with_optional_fields_as_required( tmp_path: Path, optional_fields_set: bool, faker: Faker): for schema_example in TaskOutputDataSchema.Config.schema_extra["examples"]: task_output_schema = TaskOutputDataSchema.parse_obj(schema_example) outputs_file_name = _create_fake_outputs(task_output_schema, tmp_path, optional_fields_set, faker) task_output_data = TaskOutputData.from_task_output( schema=task_output_schema, output_folder=tmp_path, output_file_ext=outputs_file_name, ) assert task_output_data for key, value in task_output_schema.items(): if not value.required and not optional_fields_set: assert task_output_data.get(key) is None if value.required or optional_fields_set: assert task_output_data.get(key) is not None