def test_dataset_creator_email(dataset_metadata):
"""Check that creators without an email are assigned a blank node."""
# modify the dataset metadata to change the creator
dataset = Dataset.from_jsonld(dataset_metadata, client=LocalClient("."),)
dataset.creators[0]._id = "mailto:None"
dataset_broken = Dataset.from_jsonld(dataset.as_jsonld(), client=LocalClient("."))
assert "mailto:None" not in dataset_broken.creators[0]._id
def test_local_client(tmpdir):
"""Test a local client."""
from renku.core.management.client import LocalClient
client = LocalClient(str(tmpdir.mkdir('project')))
assert client.path
assert client.repo is None
def test_dataset_files_empty_metadata(dataset_metadata):
"""Check parsing metadata of dataset files with empty filename."""
dataset = Dataset.from_jsonld(dataset_metadata, client=LocalClient("."),)
files = [file.filename for file in dataset.files if not file.filename]
if files:
assert None in files
def cli(ctx, path, renku_home, use_external_storage):
"""Check common Renku commands used in various situations."""
ctx.obj = LocalClient(
path=path,
renku_home=renku_home,
use_external_storage=use_external_storage,
)
def test_doi_migration(dataset_metadata):
"""Test migration of id with doi."""
dataset = Dataset.from_jsonld(
dataset_metadata,
client=LocalClient('.'),
)
assert is_doi(dataset.identifier)
assert urljoin('https://localhost', 'datasets/' +
quote(dataset.identifier, safe='')) == dataset._id
assert dataset.same_as == urljoin('https://doi.org', dataset.identifier)
def test_calamus(client, dataset_metadata_before_calamus):
"""Check Calamus loads project correctly."""
dataset = Dataset.from_jsonld(dataset_metadata_before_calamus, client=LocalClient("."))
file_ = dataset.find_file("data/dataverse/external/data.txt")
assert file_.external is True
assert "file://../../../../tmp/data.txt" == file_.url
file_ = dataset.find_file("data/dataverse/local/result.csv")
assert file_.external is False
assert "file://../../../../tmp/result.csv" == file_.url
def test_dataset_doi_metadata(dataset_metadata):
"""Check dataset metadata for correct DOI."""
from renku.core.utils.doi import is_doi
dataset = Dataset.from_jsonld(
dataset_metadata,
client=LocalClient('.'),
)
if is_doi(dataset.identifier):
assert urljoin('https://doi.org',
dataset.identifier) == dataset.same_as
assert dataset._id.endswith('datasets/{}'.format(
quote(dataset.identifier, safe='')))
def cli(ctx, path, renku_home, use_external_storage):
"""Check common Renku commands used in various situations."""
renku_path = Path(path) / renku_home
if not renku_path.exists() and not is_allowed_command(ctx):
raise UsageError(('`{0}` is not a renku repository.\n'
'To initialize this as a renku '
'repository use: `renku init`'.format(path)))
ctx.obj = LocalClient(
path=path,
renku_home=renku_home,
use_external_storage=use_external_storage,
)
if ctx.invoked_subcommand not in SAFE_COMMANDS:
check_for_migration()
def cli(ctx, path, external_storage_requested):
"""Check common Renku commands used in various situations."""
renku_path = Path(path) / RENKU_HOME
if not renku_path.exists() and not is_allowed_command(ctx):
raise UsageError(("`{0}` is not a renku repository.\n"
"To initialize this as a renku "
"repository use: `renku init`".format(path)))
ctx.obj = LocalClient(
path=path,
external_storage_requested=external_storage_requested,
)
if path != os.getcwd(
) and ctx.invoked_subcommand not in WARNING_UNPROTECTED_COMMANDS:
click.secho(WARNING +
"Run CLI commands only from project's root directory.\n",
err=True)
def test_calamus(client, dataset_metadata_before_calamus):
"""Check Calamus loads project correctly."""
dataset = Dataset.from_jsonld(dataset_metadata_before_calamus,
client=LocalClient('.'))
assert 'Open Source at Harvard' == dataset.name
assert '51db02ad-3cba-47e2-84d0-5ee5914bd654' == dataset.identifier
assert '51db02ad-3cba-47e2-84d0-5ee5914bd654' == dataset._label
assert 'Harvard University' == dataset.creator[0].affiliation
assert 'Durbin, Philip' == dataset.creator[0].name
assert 'Durbin, Philip' == dataset.creator[0].label
assert dataset.created is None
assert '2019-07-03T00:00:00' == dataset.date_published.isoformat('T')
assert 'The tabular file contains information' in dataset.description
assert 'https://doi.org/10.7910/DVN/TJCLKP' == dataset.same_as.url
assert '3' == dataset.tags[0].name
assert 'Tag 3 created by renku import' == dataset.tags[0].description
assert isinstance(dataset.license, dict)
assert ('https://creativecommons.org/publicdomain/zero/1.0/'
in str(dataset.license))
file_ = dataset.find_file('data/dataverse/IQSS-UNF.json')
assert ('https://dataverse.harvard.edu/api/access/datafile/3371500' ==
file_.url)
assert '2020-06-15T08:37:04.571573+00:00' == file_.added.isoformat('T')
assert 'https://orcid.org/0000-0002-9528-9470' == file_.creator[0]._id
assert file_.based_on is None
file_ = dataset.find_file('data/dataverse/git/index.ipynb')
assert ('https://github.com/SwissDataScienceCenter/r10e-ds-py.git' ==
file_.based_on.url)
assert ('notebooks/index.ipynb@f98325d81c700f4b86ee05c2154e94d43ca068b8' ==
file_.based_on._label)
assert file_.based_on.based_on is None
assert 'mailto:cramakri@' in file_.based_on.creator[0]._id
assert ('https://github.com/SwissDataScienceCenter/r10e-ds-py.git' ==
file_.url)
file_ = dataset.find_file('data/dataverse/external/data.txt')
assert file_.external is True
assert 'file://../../../../tmp/data.txt' == file_.url
file_ = dataset.find_file('data/dataverse/local/result.csv')
assert file_.external is False
assert 'file://../../../../tmp/result.csv' == file_.url
def clone(
url,
path=None,
install_githooks=True,
install_lfs=True,
skip_smudge=True,
recursive=True,
depth=None,
progress=None,
config=None,
raise_git_except=False,
checkout_rev=None,
):
"""Clone Renku project repo, install Git hooks and LFS."""
from renku.core.management.client import LocalClient
path = path or GitURL.parse(url).name
if isinstance(path, Path):
path = str(path)
# Clone the project
if skip_smudge:
os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"
try:
repo = Repo.clone_from(url, path, recursive=recursive, depth=depth, progress=progress)
except GitCommandError as e:
if not raise_git_except:
raise errors.GitError("Cannot clone remote Renku project: {}".format(url)) from e
raise e
remote_refs = [Path(ref.abspath).name for ref in repo.remote().refs]
if checkout_rev in remote_refs:
repo.git.checkout(checkout_rev)
elif checkout_rev:
repo.git.checkout(checkout_rev, b=checkout_rev)
if config:
config_writer = repo.config_writer()
for key, value in config.items():
key_path = key.split(".")
key = key_path.pop()
if not key_path or not key:
raise errors.GitError("Cannot write to config. Section path or key is invalid.")
config_writer.set_value(".".join(key_path), key, value)
config_writer.release()
client = LocalClient(path)
if install_githooks:
install(client=client, force=True)
if install_lfs:
command = ["git", "lfs", "install", "--local", "--force"]
if skip_smudge:
command += ["--skip-smudge"]
try:
repo.git.execute(command=command, with_exceptions=True)
except GitCommandError as e:
raise errors.GitError("Cannot install Git LFS") from e
return repo
def clone(
url,
path=None,
install_githooks=True,
install_lfs=True,
skip_smudge=True,
recursive=True,
depth=None,
progress=None,
config=None,
raise_git_except=False,
):
"""Clone Renku project repo, install Git hooks and LFS."""
from renku.core.management.client import LocalClient
path = path or GitURL.parse(url).name
if isinstance(path, Path):
path = str(path)
# Clone the project
if skip_smudge:
os.environ['GIT_LFS_SKIP_SMUDGE'] = '1'
try:
repo = Repo.clone_from(url,
path,
recursive=recursive,
depth=depth,
progress=progress)
except GitCommandError as e:
if not raise_git_except:
raise errors.GitError(
'Cannot clone remote Renku project: {}'.format(url)) from e
raise e
if config:
config_writer = repo.config_writer()
for key, value in config.items():
key_path = key.split('.')
key = key_path.pop()
if not key_path or not key:
raise errors.GitError(
'Cannot write to config. Section path or key is invalid.')
config_writer.set_value('.'.join(key_path), key, value)
config_writer.release()
client = LocalClient(path)
if install_githooks:
install(client=client, force=True)
if install_lfs:
command = ['git', 'lfs', 'install', '--local', '--force']
if skip_smudge:
command += ['--skip-smudge']
try:
repo.git.execute(command=command, with_exceptions=True)
except GitCommandError as e:
raise errors.GitError('Cannot install Git LFS') from e
return repo