def save(self, dataset_id: str, dataset: Dataset) -> DatasetSource:
writer = dataset.get_writer()
if writer is None:
raise ValueError(
f'{dataset.dataset_type} does not support artifact persistance'
)
reader, artifacts = writer.write(dataset)
with artifacts.blob_dict() as blobs:
try:
pushed = self.repo.push_artifact(self.ARTIFACT_TYPE,
dataset_id, blobs)
except ArtifactExistsError as e:
raise DatasetExistsError(dataset_id, self, e)
return ArtifactDatasetSource(reader, pushed, dataset.dataset_type)
def dataset_write_read_check(dataset: Dataset,
writer: DatasetWriter = None,
reader_type: Type[DatasetReader] = None,
custom_eq: Callable[[Any, Any], bool] = None,
custom_assert: Callable[[Any, Any], Any] = None):
writer = writer or dataset.get_writer()
reader, artifacts = writer.write(dataset)
if reader_type is not None:
assert isinstance(reader, reader_type)
new = reader.read(artifacts)
assert dataset.dataset_type == new.dataset_type
if custom_assert is not None:
custom_assert(new.data, dataset.data)
else:
if custom_eq is not None:
assert custom_eq(new.data, dataset.data)
else:
assert new.data == dataset.data