def with_dataset(self, name=None):
"""Yield an editable metadata object for a dataset."""
from renku.models._locals import with_reference
from renku.models.refs import LinkReference
dataset = self.load_dataset(name=name)
if dataset is None:
identifier = str(uuid.uuid4())
path = (self.renku_datasets_path / identifier / self.METADATA)
path.parent.mkdir(parents=True, exist_ok=True)
with with_reference(path):
dataset = Dataset(
identifier=identifier, name=name, client=self
)
if name:
LinkReference.create(client=self, name='datasets/' +
name).set_reference(path)
dataset_path = self.path / self.datadir / dataset.name
dataset_path.mkdir(parents=True, exist_ok=True)
yield dataset
# TODO
# if path is None:
# path = dataset_path / self.METADATA
# if path.exists():
# raise ValueError('Dataset already exists')
dataset.to_yaml()
def datasets(ctx, client):
"""Migrate dataset metadata."""
from renku.models._jsonld import asjsonld
from renku.models.datasets import Dataset
from renku.models.refs import LinkReference
from ._checks.location_datasets import _dataset_metadata_pre_0_3_4
for old_path in _dataset_metadata_pre_0_3_4(client):
dataset = Dataset.from_yaml(old_path, client=client)
name = str(old_path.parent.relative_to(client.path / 'data'))
new_path = (client.renku_datasets_path / dataset.uid / client.METADATA)
new_path.parent.mkdir(parents=True, exist_ok=True)
dataset = dataset.rename_files(lambda key: os.path.relpath(
str(old_path.parent / key), start=str(new_path.parent)))
with new_path.open('w') as fp:
yaml.dump(asjsonld(dataset), fp, default_flow_style=False)
old_path.unlink()
LinkReference.create(client=client,
name='datasets/' + name).set_reference(new_path)
def dataset_path(self, name):
"""Get dataset path from name."""
from renku.models.refs import LinkReference
path = self.renku_datasets_path / name / self.METADATA
if not path.exists():
path = LinkReference(client=self,
name='datasets/' + name).reference
return path
def workflow(ctx, client):
"""List or manage workflows with subcommands."""
if ctx.invoked_subcommand is None:
from renku.models.refs import LinkReference
names = defaultdict(list)
for ref in LinkReference.iter_items(client, common_path='workflows'):
names[ref.reference.name].append(ref.name)
for path in client.workflow_path.glob('*.cwl'):
click.echo('{path}: {names}'.format(
path=path.name,
names=', '.join(
click.style(_deref(name), fg='green')
for name in names[path.name]),
))
def with_dataset(self, name=None):
"""Yield an editable metadata object for a dataset."""
from renku.models.refs import LinkReference
with self.lock:
path = None
dataset = None
if name:
path = self.renku_datasets_path / name / self.METADATA
if not path.exists():
path = LinkReference(client=self,
name='datasets/' + name).reference
if path.exists():
with path.open('r') as f:
source = yaml.load(f) or {}
dataset = Dataset.from_jsonld(source)
if dataset is None:
source = {}
dataset = Dataset(name=name)
path = (self.renku_datasets_path / dataset.identifier.hex /
self.METADATA)
path.parent.mkdir(parents=True, exist_ok=True)
if name:
LinkReference.create(client=self, name='datasets/' +
name).set_reference(path)
dataset_path = self.path / self.datadir / dataset.name
dataset_path.mkdir(parents=True, exist_ok=True)
yield dataset
source.update(**asjsonld(dataset))
# TODO
# if path is None:
# path = dataset_path / self.METADATA
# if path.exists():
# raise ValueError('Dataset already exists')
with path.open('w') as f:
yaml.dump(source, f, default_flow_style=False)
def check_missing_references(client):
"""Find missing references."""
from renku.models.refs import LinkReference
missing = [
ref for ref in LinkReference.iter_items(client)
if not ref.reference.exists()
]
if not missing:
return True
click.secho(WARNING + 'There are missing references.'
'\n (use "git rm <name>" to clean them)\n\n\t' +
'\n\t '.join(
click.style(str(ref.path), fg='yellow') + ' -> ' +
click.style(str(ref.reference), fg='red')
for ref in missing) + '\n')
return False
def remove(client, names):
"""Delete a dataset."""
from renku.models.refs import LinkReference
datasets = {name: client.dataset_path(name) for name in names}
if not datasets:
raise click.BadParameter(
'use dataset name or identifier', param_hint='names'
)
unknown = [
name
for name, path in datasets.items() if not path or not path.exists()
]
if unknown:
raise click.BadParameter(
'unknown datasets ' + ', '.join(unknown), param_hint='names'
)
datasets = set(datasets.values())
with progressbar(
datasets,
label='Removing metadata files'.ljust(30),
item_show_func=lambda item: str(item) if item else '',
) as bar:
for dataset in bar:
if dataset and dataset.exists():
dataset.unlink()
with progressbar(
list(LinkReference.iter_items(client, common_path='datasets')),
label='Removing aliases'.ljust(30),
item_show_func=lambda item: item.name if item else '',
) as bar:
for ref in bar:
if ref.reference in datasets:
ref.delete()
click.secho('OK', fg='green')
def workflow_names(self):
"""Return index of workflow names."""
names = defaultdict(list)
for ref in LinkReference.iter_items(self, common_path='workflows'):
names[str(ref.reference.relative_to(self.path))].append(ref.name)
return names
def remove(client, name):
"""Remove the remote named <name>."""
from renku.models.refs import LinkReference
LinkReference(client=client, name=_ref(name)).delete()
def rename(client, old, new, force):
"""Rename the workflow named <old> to <new>."""
from renku.models.refs import LinkReference
LinkReference(client=client, name=_ref(old)).rename(_ref(new), force=force)
def set_name(client, name, path, force):
"""Sets the <name> for remote <path>."""
from renku.models.refs import LinkReference
LinkReference.create(
client=client, name=_ref(name), force=force
).set_reference(path)
