def migrate_datasets_pre_v0_3(client): """Migrate datasets from Renku 0.3.x.""" for old_path in dataset_pre_0_3(client): name = str(old_path.parent.relative_to(client.path / 'data')) dataset = Dataset.from_yaml(old_path, client=client) new_path = (client.renku_datasets_path / dataset.uid / client.METADATA) new_path.parent.mkdir(parents=True, exist_ok=True) with client.with_metadata(read_only=True) as meta: for module in client.repo.submodules: if Path(module.url).name == meta.name: module.remove() for file_ in dataset.files: if not Path(file_.path).exists(): expected_path = ( client.path / 'data' / dataset.name / file_.path ) if expected_path.exists(): file_.path = expected_path.relative_to(client.path) dataset.__reference__ = new_path dataset.to_yaml() Path(old_path).unlink() ref = LinkReference.create( client=client, name='datasets/{0}'.format(name), force=True, ) ref.set_reference(new_path)
def test_creators_with_same_email(tmp_path): """Test creators with different names and same email address.""" creators = [Person(name="me", email="*****@*****.**"), Person(name="me2", email="*****@*****.**")] dataset = Dataset(name="dataset", creators=creators) path = tmp_path / "dataset.yml" dataset.__reference__ = path dataset.to_yaml() dataset = Dataset.from_yaml(path) assert 1 == len(dataset.creators) assert dataset.creators[0].name in ["me", "me2"]
def datasets_from_commit(self, commit=None): """Return datasets defined in a commit.""" commit = commit or self.repo.head.commit try: datasets = commit.tree / self.renku_home / self.DATASETS except KeyError: return for tree in datasets: try: blob = tree / self.METADATA except KeyError: continue dataset = Dataset.from_yaml(self.path / Path(blob.path), client=self) dataset.commit = commit yield dataset
def test_dataset_deserialization(client, dataset): """Test Dataset deserialization.""" dataset_ = Dataset.from_yaml(client.get_dataset_path("dataset"), client=client) dataset_types = { "date_created": [datetime.datetime], "creators": [list], "description": [str, type(None)], "files": [list], "identifier": [str], "keywords": [list], } for attribute, type_ in dataset_types.items(): assert type(dataset_.__getattribute__(attribute)) in type_ creator_types = {"email": str, "_id": str, "name": str, "affiliation": str} creator = dataset.creators[0] for attribute, type_ in creator_types.items(): assert type(getattr(creator, attribute)) is type_
def _migrate_datasets_pre_v0_3(client): """Migrate datasets from Renku 0.3.x.""" def _dataset_pre_0_3(client): """Return paths of dataset metadata for pre 0.3.4.""" project_is_pre_0_3 = int(client.project.version) < 2 if project_is_pre_0_3: return (client.path / DATA_DIR).rglob(client.METADATA) return [] for old_path in _dataset_pre_0_3(client): name = str(old_path.parent.relative_to(client.path / DATA_DIR)) dataset = Dataset.from_yaml(old_path, client=client) new_path = (client.renku_datasets_path / dataset.uid / client.METADATA) new_path.parent.mkdir(parents=True, exist_ok=True) with client.with_metadata(read_only=True) as meta: for module in client.repo.submodules: if Path(module.url).name == meta.name: module.remove() for file_ in dataset.files: if not Path(file_.path).exists(): expected_path = (client.path / DATA_DIR / dataset.name / file_.path) if expected_path.exists(): file_.path = expected_path.relative_to(client.path) dataset.__reference__ = new_path.relative_to(client.path) dataset.to_yaml() Path(old_path).unlink() ref = LinkReference.create( client=client, name='datasets/{0}'.format(name), force=True, ) ref.set_reference(new_path)
def test_dataset_deserialization(client, dataset): """Test Dataset deserialization.""" from renku.core.models.datasets import Dataset dataset_ = Dataset.from_yaml(client.get_dataset_path('dataset'), client=client) dataset_types = { 'created': [datetime.datetime], 'creator': [list], 'description': [str, type(None)], 'files': [list], 'identifier': [str], 'keywords': [list], } for attribute, type_ in dataset_types.items(): assert type(dataset_.__getattribute__(attribute)) in type_ creator_types = {'email': str, '_id': str, 'name': str, 'affiliation': str} creator = dataset.creator[0] for attribute, type_ in creator_types.items(): assert type(getattr(creator, attribute)) is type_
def load_dataset_from_path(self, path, commit=None): """Return a dataset from a given path.""" path = Path(path) if not path.is_absolute(): path = self.path / path return Dataset.from_yaml(path, client=self, commit=commit)