def list_files(
client,
datasets=None,
creators=None,
include=None,
exclude=None,
format=None,
columns=None
):
"""List files in dataset."""
records = _filter(
client,
short_names=datasets,
creators=creators,
include=include,
exclude=exclude
)
for record in records:
record.title = record.dataset.name
record.short_name = record.dataset.short_name
if format is None:
return records
if format not in DATASETS_FORMATS:
raise UsageError('format not supported')
return DATASET_FILES_FORMATS[format](client, records, columns=columns)
def check_queues(queue_list):
"""Check if listening on specified queues is possible."""
for queue in queue_list:
if queue not in QUEUES:
err_msg = "invalid queue name: {0}\n\n" "valid queue names: \n{1}".format(
queue, "\n".join(QUEUES))
raise UsageError(err_msg)
def default_endpoint(ctx, param, value):
"""Return default endpoint if specified."""
if ctx.resilient_parsing:
return
config = ctx.obj['config']
endpoint = default_endpoint_from_config(config, option=value)
if endpoint is None:
raise UsageError('No default endpoint found.')
return endpoint
def validate_endpoint(ctx, param, value):
"""Validate endpoint."""
try:
config = ctx.obj['config']
except Exception:
return
endpoint = default_endpoint(ctx, param, value)
if endpoint not in config.get('endpoints', {}):
raise UsageError('Unknown endpoint: {0}'.format(endpoint))
return endpoint
def list_datasets(client, revision=None, format=None, columns=None):
"""Handle datasets sub commands."""
if revision is None:
datasets = client.datasets.values()
else:
datasets = client.datasets_from_commit(client.repo.commit(revision))
if format is None:
return list(datasets)
if format not in DATASETS_FORMATS:
raise UsageError('format not supported')
return DATASETS_FORMATS[format](client, datasets, columns=columns)
def cli(ctx, path, renku_home, use_external_storage):
"""Check common Renku commands used in various situations."""
renku_path = Path(path) / renku_home
if not renku_path.exists() and not is_allowed_command(ctx):
raise UsageError(('`{0}` is not a renku repository.\n'
'To initialize this as a renku '
'repository use: `renku init`'.format(path)))
ctx.obj = LocalClient(
path=path,
renku_home=renku_home,
use_external_storage=use_external_storage,
)
if ctx.invoked_subcommand not in SAFE_COMMANDS:
check_for_migration()
def cli(ctx, path, external_storage_requested):
"""Check common Renku commands used in various situations."""
renku_path = Path(path) / RENKU_HOME
if not renku_path.exists() and not is_allowed_command(ctx):
raise UsageError(("`{0}` is not a renku repository.\n"
"To initialize this as a renku "
"repository use: `renku init`".format(path)))
ctx.obj = LocalClient(
path=path,
external_storage_requested=external_storage_requested,
)
if path != os.getcwd(
) and ctx.invoked_subcommand not in WARNING_UNPROTECTED_COMMANDS:
click.secho(WARNING +
"Run CLI commands only from project's root directory.\n",
err=True)
def _add_to_dataset(
client,
urls,
short_name,
external=False,
force=False,
overwrite=False,
create=False,
sources=(),
destination='',
ref=None,
with_metadata=None,
urlscontext=contextlib.nullcontext,
commit_message=None,
extract=False,
all_at_once=False,
destination_names=None,
progress=None,
interactive=False,
total_size=None,
):
"""Add data to a dataset."""
if len(urls) == 0:
raise UsageError('No URL is specified')
if sources and len(urls) > 1:
raise UsageError('Cannot use "--source" with multiple URLs.')
if interactive:
if total_size is None:
total_size = 0
for url in urls:
try:
with requests.get(url, stream=True) as r:
total_size += int(r.headers.get('content-length', 0))
except requests.exceptions.RequestException:
pass
usage = shutil.disk_usage(client.path)
if total_size > usage.free:
mb = 2**20
message = 'Insufficient disk space (required: {:.2f} MB' \
'/available: {:.2f} MB). '.format(
total_size/mb, usage.free/mb
)
raise OperationError(message)
try:
with client.with_dataset(
short_name=short_name, create=create
) as dataset:
with urlscontext(urls) as bar:
warning_messages, messages = client.add_data_to_dataset(
dataset,
bar,
external=external,
force=force,
overwrite=overwrite,
sources=sources,
destination=destination,
ref=ref,
extract=extract,
all_at_once=all_at_once,
destination_names=destination_names,
progress=progress,
)
if messages:
for msg in messages:
click.echo(INFO + msg)
if warning_messages:
for msg in warning_messages:
click.echo(WARNING + msg)
if with_metadata:
for file_ in dataset.files:
file_.creator = with_metadata.creator
file_.based_on = None
# dataset has the correct list of files
with_metadata.files = dataset.files
with_metadata.url = dataset._id
dataset.update_metadata(with_metadata)
dataset.same_as = with_metadata.same_as
except DatasetNotFound:
raise DatasetNotFound(
message='Dataset "{0}" does not exist.\n'
'Use "renku dataset create {0}" to create the dataset or retry '
'"renku dataset add {0}" command with "--create" option for '
'automatic dataset creation.'.format(short_name)
)
except (FileNotFoundError, git.exc.NoSuchPathError) as e:
raise ParameterError(
'Could not find paths/URLs: \n{0}'.format('\n'.join(urls))
) from e
def add_to_dataset(
client,
urls,
short_name,
link=False,
force=False,
create=False,
sources=(),
destination='',
ref=None,
with_metadata=None,
urlscontext=contextlib.nullcontext,
commit_message=None,
extract=False,
all_at_once=False,
progress=None,
):
"""Add data to a dataset."""
if len(urls) == 0:
raise UsageError('No URL is specified')
if (sources or destination) and len(urls) > 1:
raise UsageError(
'Cannot add multiple URLs with --source or --destination')
try:
with client.with_dataset(short_name=short_name,
create=create) as dataset:
with urlscontext(urls) as bar:
warning_message = client.add_data_to_dataset(
dataset,
bar,
link=link,
force=force,
sources=sources,
destination=destination,
ref=ref,
extract=extract,
all_at_once=all_at_once,
progress=progress,
)
if warning_message:
click.echo(WARNING + warning_message)
if with_metadata:
for file_ in dataset.files:
file_.creator = with_metadata.creator
# dataset has the correct list of files
with_metadata.files = dataset.files
dataset.update_metadata(with_metadata)
except DatasetNotFound:
raise DatasetNotFound(
'Dataset "{0}" does not exist.\n'
'Use "renku dataset create {0}" to create the dataset or retry '
'"renku dataset add {0}" command with "--create" option for '
'automatic dataset creation.'.format(short_name))
except (FileNotFoundError, git.exc.NoSuchPathError) as e:
raise ParameterError('Could not find paths/URLs: \n{0}'.format(
'\n'.join(urls))) from e