def test_data_add(scheme, path, overwrite, error, client, data_file, directory_tree, dataset_responses):
"""Test data import."""
with raises(error):
if path == "temp":
path = str(data_file)
elif path == "tempdir":
path = str(directory_tree)
with client.with_dataset("dataset", create=True) as d:
d.creators = [Person(name="me", email="*****@*****.**", id="me_id")]
client.add_data_to_dataset(d, ["{}{}".format(scheme, path)], overwrite=overwrite)
target_path = os.path.join(DATA_DIR, "dataset", "file")
with open(target_path) as f:
assert f.read() == "1234"
assert d.find_file(target_path)
# check that the imported file is read-only
assert not os.access(target_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
# check the linking
if scheme in ("", "file://"):
shutil.rmtree("./data/dataset")
with client.with_dataset("dataset") as d:
d.creators = [Person(name="me", email="*****@*****.**", id="me_id")]
client.add_data_to_dataset(d, ["{}{}".format(scheme, path)], overwrite=True)
assert os.path.exists(target_path)
def test_creators_with_same_email(tmp_path):
"""Test creators with different names and same email address."""
creators = [Person(name="me", email="*****@*****.**"), Person(name="me2", email="*****@*****.**")]
dataset = Dataset(name="dataset", creators=creators)
path = tmp_path / "dataset.yml"
dataset.__reference__ = path
dataset.to_yaml()
dataset = Dataset.from_yaml(path)
assert 1 == len(dataset.creators)
assert dataset.creators[0].name in ["me", "me2"]
def test_creator_parse(creators, data_file):
"""Test that different options for specifying creators work."""
dataset = Dataset(name="dataset", creators=creators)
creator = Person(name="me", email="*****@*****.**")
assert creator in dataset.creators
# email check
with pytest.raises(ValueError):
Person(name="me", email="meexample.com")
# creators must be a set or list of dicts or Person
with pytest.raises(ValueError):
Dataset(name="dataset", creators=["name"])
def test_creator_parse(creators, data_file):
"""Test that different options for specifying creators work."""
f = DatasetFile(path='file', creator=creators)
creator = Person(name='me', email='*****@*****.**')
assert creator in f.creator
# email check
with pytest.raises(ValueError):
Person(name='me', email='meexample.com')
# creators must be a set or list of dicts or Person
with pytest.raises(ValueError):
f = DatasetFile(path='file', creator=['name'])
def test_project_creator_deserialization(client, project):
"""Check that the correct creator is returned on deserialization."""
from renku.core.models.provenance.agents import Person
# modify the project metadata to change the creator
project = client.project
project.creator = Person(email='*****@*****.**', name='Johnny Doe')
project.to_yaml()
client.repo.git.commit('-a', '--amend', '-C', 'HEAD', '--author',
'Johnny Doe <*****@*****.**>', '--no-verify')
# the project creator should always be the one in the metadata
assert '*****@*****.**' == client.project.creator.email
assert 'Johnny Doe' == client.project.creator.name
assert client.project.creator.label == client.project.creator.name
# Remove the creator from metadata
project = client.project
project.creator = None
project.to_yaml()
client.repo.git.commit('-a', '--amend', '-C', 'HEAD', '--author',
'Jane Doe <*****@*****.**>', '--no-verify')
# now the creator should be the one from the commit
project = Project.from_yaml(client.renku_metadata_path, client=client)
assert '*****@*****.**' == project.creator.email
assert 'Jane Doe' == project.creator.name
assert project.creator.label == project.creator.name
def test_data_add_recursive(directory_tree, client):
"""Test recursive data imports."""
with client.with_dataset("dataset", create=True) as dataset:
dataset.creators = [Person(name="me", email="*****@*****.**", id="me_id")]
client.add_data_to_dataset(dataset, [directory_tree.join("dir2").strpath])
assert os.path.basename(os.path.dirname(dataset.files[0].path)) == "dir2"
def test_data_add_recursive(directory_tree, client):
"""Test recursive data imports."""
with client.with_dataset('dataset', create=True) as dataset:
dataset.creator = [
Person(name='me', email='*****@*****.**', id='me_id')
]
client.add_data_to_dataset(dataset,
[directory_tree.join('dir2').strpath])
assert os.path.basename(os.path.dirname(
dataset.files[0].path)) == 'dir2'
def dataset(client):
"""Create a dataset."""
from renku.core.models.provenance.agents import Person
with client.with_dataset('dataset', create=True) as dataset:
dataset.creator = [
Person(
**{
'affiliation': 'xxx',
'email': '*****@*****.**',
'id': 'me_id',
'name': 'me',
})
]
return dataset
def dataset(client):
"""Create a dataset."""
from renku.core.models.provenance.agents import Person
with client.with_dataset("dataset", create=True) as dataset:
dataset.creators = [
Person(
**{
"affiliation": "xxx",
"email": "*****@*****.**",
"id": "me_id",
"name": "me",
})
]
return dataset
def test_data_add(scheme, path, overwrite, error, client, data_file,
directory_tree, dataset_responses):
"""Test data import."""
with raises(error):
if path == 'temp':
path = str(data_file)
elif path == 'tempdir':
path = str(directory_tree)
with client.with_dataset('dataset', create=True) as d:
d.creator = [Person(name='me', email='*****@*****.**', id='me_id')]
client.add_data_to_dataset(d, ['{}{}'.format(scheme, path)],
overwrite=overwrite)
target_path = os.path.join(DATA_DIR, 'dataset', 'file')
with open(target_path) as f:
assert f.read() == '1234'
assert d.find_file(target_path)
# check that the imported file is read-only
assert not os.access(target_path,
stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
# check the linking
if scheme in ('', 'file://'):
shutil.rmtree('./data/dataset')
with client.with_dataset('dataset') as d:
d.creator = [
Person(name='me', email='*****@*****.**', id='me_id')
]
client.add_data_to_dataset(d, ['{}{}'.format(scheme, path)],
overwrite=True)
assert os.path.exists(target_path)
def test_project_shacl(project, client):
"""Test project metadata structure."""
from renku.core.models.provenance.agents import Person
path = Path(
__file__
).parent.parent.parent / 'fixtures' / 'force_project_shacl.json'
project = client.project
project.creator = Person(email='*****@*****.**', name='Johnny Doe')
g = project.as_jsonld()
rdf = pyld.jsonld.to_rdf(g,
options={
'format': 'application/n-quads',
'produceGeneralizedRdf': False
})
r, _, t = validate_graph(rdf, shacl_path=str(path))
assert r is True, t
r, _, t = validate_graph(rdf)
assert r is True, t
def test_git_repo_import(client, dataset, tmpdir, data_repository):
"""Test an import from a git repository."""
# add data from local repo
client.add_data_to_dataset(dataset, [os.path.join(os.path.dirname(data_repository.git_dir), "dir2")])
path = os.path.join(DATA_DIR, "dataset", "dir2", "file2")
assert os.stat(path)
path = os.path.join("dir2", "file2")
assert dataset.files[0].path.endswith(path)
@pytest.mark.parametrize(
"creators",
[
[Person(name="me", email="*****@*****.**")],
[{"http://schema.org/name": "me", "http://schema.org/email": "*****@*****.**",}],
],
)
def test_creator_parse(creators, data_file):
"""Test that different options for specifying creators work."""
dataset = Dataset(name="dataset", creators=creators)
creator = Person(name="me", email="*****@*****.**")
assert creator in dataset.creators
# email check
with pytest.raises(ValueError):
Person(name="me", email="meexample.com")
# creators must be a set or list of dicts or Person
with pytest.raises(ValueError):
assert os.path.basename(os.path.dirname(
dataset.files[0].path)) == 'dir2'
def test_git_repo_import(client, dataset, tmpdir, data_repository):
"""Test an import from a git repository."""
# add data from local repo
client.add_data_to_dataset(
dataset,
[os.path.join(os.path.dirname(data_repository.git_dir), 'dir2')])
assert os.stat('data/dataset/dir2/file2')
assert dataset.files[0].path.endswith('dir2/file2')
@pytest.mark.parametrize('creators', [
[Person(name='me', email='*****@*****.**')],
[{
'name': 'me',
'email': '*****@*****.**',
}],
])
def test_creator_parse(creators, data_file):
"""Test that different options for specifying creators work."""
f = DatasetFile(path='file', creator=creators)
creator = Person(name='me', email='*****@*****.**')
assert creator in f.creator
# email check
with pytest.raises(ValueError):
Person(name='me', email='meexample.com')
