def __attrs_post_init__(self):
"""Initialize computed attributes."""
#: Configure Renku path.
path = Path(self.renku_home)
if not path.is_absolute():
path = self.path / path
path.relative_to(path)
self.renku_path = path
data_dir = self.get_value('renku',
self.DATA_DIR_CONFIG_KEY,
local_only=True)
self.data_dir = data_dir or self.data_dir
self._subclients = {}
self._project = None
super().__attrs_post_init__()
# initialize submodules
if self.repo:
try:
check_output(
['git', 'submodule', 'update', '--init', '--recursive'],
cwd=str(self.path))
except subprocess.CalledProcessError:
pass
def resolve_in_submodules(self, commit, path):
"""Resolve filename in submodules."""
original_path = self.path / path
in_vendor = str(path).startswith('.renku/vendors')
if original_path.is_symlink() or in_vendor:
original_path = Path(
os.path.realpath(os.path.abspath(str(original_path)))
)
for submodule, subclient in self.subclients(commit).items():
if (Path(submodule.path) / Path('.git')).exists():
try:
subpath = original_path.relative_to(subclient.path)
return (
subclient,
subclient.find_previous_commit(
subpath, revision=submodule.hexsha
),
subpath,
)
except ValueError:
pass
return self, commit, path
def test_dataset_shacl(tmpdir, runner, project, client):
"""Test dataset metadata structure."""
force_dataset_path = Path(
__file__
).parent.parent.parent / 'fixtures' / 'force_dataset_shacl.json'
force_datasetfile_path = Path(
__file__
).parent.parent.parent / 'fixtures' / 'force_datasetfile_shacl.json'
force_datasettag_path = Path(
__file__
).parent.parent.parent / 'fixtures' / 'force_datasettag_shacl.json'
runner.invoke(cli, ['dataset', 'create', 'dataset'])
paths = []
for i in range(3):
new_file = tmpdir.join('file_{0}'.format(i))
new_file.write(str(i))
paths.append(str(new_file))
# add data
runner.invoke(
cli,
['dataset', 'add', 'dataset'] + paths,
catch_exceptions=False,
)
runner.invoke(
cli,
['dataset', 'tag', 'dataset', '1.0'],
catch_exceptions=False,
)
with client.with_dataset('dataset') as dataset:
g = dataset.asjsonld()
rdf = pyld.jsonld.to_rdf(g,
options={
'format': 'application/n-quads',
'produceGeneralizedRdf': True
})
r, _, t = validate_graph(rdf, shacl_path=str(force_dataset_path))
assert r is True, t
r, _, t = validate_graph(rdf, shacl_path=str(force_datasetfile_path))
assert r is True, t
r, _, t = validate_graph(rdf, shacl_path=str(force_datasettag_path))
assert r is True, t
r, _, t = validate_graph(rdf)
assert r is True, t
def subclients(self, parent_commit):
"""Return mapping from submodule to client."""
if parent_commit in self._subclients:
return self._subclients[parent_commit]
try:
from git import Submodule
submodules = [
submodule for submodule in Submodule.iter_items(
self.repo, parent_commit=parent_commit)
]
except (RuntimeError, ValueError):
# There are no submodules associated with the given commit.
submodules = []
subclients = {}
for submodule in submodules:
subpath = (self.path / submodule.path).resolve()
is_renku = subpath / Path(self.renku_home)
if subpath.exists() and is_renku.exists():
subclients[submodule] = self.__class__(
path=subpath,
parent=(self, submodule),
)
return subclients
def __attrs_post_init__(self):
"""Initialize computed attributes."""
#: Configure Renku path.
path = Path(self.renku_home)
if not path.is_absolute():
path = self.path / path
path.relative_to(path)
self.renku_path = path
self._subclients = {}
super().__attrs_post_init__()
# initialize submodules
if self.repo:
try:
check_output(
['git', 'submodule', 'update', '--init', '--recursive'],
cwd=str(self.path))
except subprocess.CalledProcessError:
pass
def test_project_shacl(project, client):
"""Test project metadata structure."""
from renku.core.models.provenance.agents import Person
path = Path(
__file__
).parent.parent.parent / 'fixtures' / 'force_project_shacl.json'
project = client.project
project.creator = Person(email='*****@*****.**', name='Johnny Doe')
g = project.as_jsonld()
rdf = pyld.jsonld.to_rdf(g,
options={
'format': 'application/n-quads',
'produceGeneralizedRdf': False
})
r, _, t = validate_graph(rdf, shacl_path=str(path))
assert r is True, t
r, _, t = validate_graph(rdf)
assert r is True, t
def process_commit(self, commit=None, path=None):
"""Build an :class:`~renku.core.models.provenance.activities.Activity`.
:param commit: Commit to process. (default: ``HEAD``)
:param path: Process a specific CWL file.
"""
from renku.core.models.cwl.ascwl import CWLClass
from renku.core.models.provenance.activities import Activity
commit = commit or self.repo.head.commit
if len(commit.parents) > 1:
return Activity(commit=commit, client=self)
if path is None:
for file_ in commit.stats.files.keys():
# Find a process (CommandLineTool or Workflow)
if self.is_cwl(file_):
if path is not None:
# Regular activity since it edits multiple CWL files
return Activity(commit=commit, client=self)
path = file_
if path:
data = (commit.tree / path).data_stream.read()
process = CWLClass.from_cwl(
yaml.safe_load(data), __reference__=Path(path)
)
return process.create_run(
commit=commit,
client=self,
process=process,
path=path,
)
return Activity(commit=commit, client=self)
def import_from_template(self, template_path, metadata, force=False):
"""Render template files from a template directory."""
for file in template_path.glob('**/*'):
destination = self.path / file.relative_to(template_path)
try:
# TODO: notify about the expected variables - code stub:
# with file.open() as fr:
# file_content = fr.read()
# # look for the required keys
# env = Environment()
# parsed = env.parse(file_content)
# variables = meta.find_undeclared_variables(parsed)
# parse file and process it
template = Template(file.read_text())
rendered_content = template.render(metadata)
destination = Path(Template(str(destination)).render(metadata))
destination.write_text(rendered_content)
except IsADirectoryError:
destination.mkdir(parents=True, exist_ok=True)
except TypeError:
shutil.copy(file, destination)
def path_converter(path):
"""Converter for path in PathMixin."""
return Path(path).resolve()