def download_files(ctx, workflow, filenames,
output_directory, access_token): # noqa: D301
"""Download workspace files.
The `download` command allows to download workspace files. By default, the
files specified in the workflow specification as outputs are downloaded.
You can also specify the individual files you would like to download, see
examples below. Note that downloading directories is not yet supported.
Examples: \n
\t $ reana-client download # download all output files \n
\t $ reana-client download mydata.tmp outputs/myplot.png
"""
from reana_client.api.client import download_file
logging.debug('command: {}'.format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p]))
if not filenames:
reana_spec = load_reana_spec(os.path.join(get_workflow_root(),
'reana.yaml'),
False)
if 'outputs' in reana_spec:
filenames = reana_spec['outputs'].get('files') or []
if workflow:
for file_name in filenames:
try:
binary_file = download_file(workflow,
file_name,
access_token)
logging.info('{0} binary file downloaded ... writing to {1}'.
format(file_name, output_directory))
outputs_file_path = os.path.join(output_directory, file_name)
if not os.path.exists(os.path.dirname(outputs_file_path)):
os.makedirs(os.path.dirname(outputs_file_path))
with open(outputs_file_path, 'wb') as f:
f.write(binary_file)
click.secho(
'File {0} downloaded to {1}.'.format(
file_name, output_directory),
fg='green')
except OSError as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(
click.style('File {0} could not be written.'.
format(file_name),
fg='red'), err=True)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style('File {0} could not be downloaded: {1}'.
format(file_name, e), fg='red'),
err=True)
def download_files(ctx, workflow, filenames, output_directory, access_token):
"""Download workflow workspace file(s)."""
logging.debug('command: {}'.format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p]))
if not access_token:
click.echo(click.style(ERROR_MESSAGES['missing_access_token'],
fg='red'),
err=True)
sys.exit(1)
if not filenames:
reana_spec = load_reana_spec(
os.path.join(get_workflow_root(), 'reana.yaml'), False)
if 'outputs' in reana_spec:
filenames = reana_spec['outputs'].get('files') or []
if workflow:
for file_name in filenames:
try:
binary_file = download_file(workflow, file_name, access_token)
logging.info(
'{0} binary file downloaded ... writing to {1}'.format(
file_name, output_directory))
outputs_file_path = os.path.join(output_directory, file_name)
if not os.path.exists(os.path.dirname(outputs_file_path)):
os.makedirs(os.path.dirname(outputs_file_path))
with open(outputs_file_path, 'wb') as f:
f.write(binary_file)
click.secho('File {0} downloaded to {1}.'.format(
file_name, output_directory),
fg='green')
except OSError as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style(
'File {0} could not be written.'.format(file_name),
fg='red'),
err=True)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style(
'File {0} could not be downloaded: {1}'.format(
file_name, e),
fg='red'),
err=True)
def upload_files(ctx, workflow, filenames, access_token):
"""Upload files and directories to workflow workspace."""
logging.debug('command: {}'.format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p]))
if not access_token:
click.echo(click.style(ERROR_MESSAGES['missing_access_token'],
fg='red'),
err=True)
sys.exit(1)
if not filenames:
reana_spec = load_reana_spec(
os.path.join(get_workflow_root(), 'reana.yaml'), False)
if 'inputs' in reana_spec:
filenames = []
filenames += [
os.path.join(get_workflow_root(), f)
for f in reana_spec['inputs'].get('files') or []
]
filenames += [
os.path.join(get_workflow_root(), d)
for d in reana_spec['inputs'].get('directories') or []
]
if workflow:
for filename in filenames:
try:
response = upload_to_server(workflow, filename, access_token)
for file_ in response:
if file_.startswith('symlink:'):
click.echo(
click.style('Symlink resolved to {}. Uploaded'
' hard copy.'.format(
file_[len('symlink:'):]),
fg='green'))
else:
click.echo(
click.style(
'File {} was successfully uploaded.'.format(
file_),
fg='green'))
except FileNotFoundError as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style(
'File {0} could not be uploaded: {0} does not exist.'.
format(filename),
fg='red'),
err=True)
if 'invoked_by_subcommand' in ctx.parent.__dict__:
sys.exit(1)
except FileUploadError as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style(
'Something went wrong while uploading {0}.\n{1}'.format(
filename, str(e)),
fg='red'),
err=True)
if 'invoked_by_subcommand' in ctx.parent.__dict__:
sys.exit(1)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style(
'Something went wrong while uploading {}'.format(filename),
fg='red'),
err=True)
if 'invoked_by_subcommand' in ctx.parent.__dict__:
sys.exit(1)
def upload_to_server(workflow, paths, access_token):
"""Upload file or directory to REANA server.
Shared e.g. by `code upload` and `inputs upload`.
:param workflow: ID of that Workflow whose workspace should be
used to store the files.
:param paths: Absolute filepath(s) of files to be uploaded.
"""
if not workflow:
raise ValueError(
'Workflow name or id must be provided')
if not paths:
raise ValueError(
'Please provide path(s) to file(s) that '
'should be uploaded to workspace.')
logging.info('Workflow "{}" selected'.format(workflow))
# Check if multiple paths were given and iterate over them
if type(paths) is list or type(paths) is tuple:
for path in paths:
upload_to_server(workflow, path, access_token)
# `paths` points to a single file or directory
else:
path = paths
if '..' in paths.split('/'):
raise FileUploadError('Path cannot contain ".."')
# Check if input is a directory and upload everything
# including subdirectories.
if os.path.isdir(path):
logging.debug("'{}' is a directory.".format(path))
logging.info("Uploading contents of folder '{}' ...".format(path))
for root, dirs, files in os.walk(path, topdown=False):
uploaded_files = []
for next_path in files + dirs:
next_uploaded_files = upload_to_server(
workflow, os.path.join(root, next_path),
access_token)
if next_uploaded_files:
uploaded_files.extend(next_uploaded_files)
return uploaded_files
# Check if input is an absolute path and upload file.
else:
symlink = False
if os.path.islink(path):
path = os.path.realpath(path)
logging.info(
'Symlink {} found, uploading'
' hard copy.'.
format(path))
symlink = True
with open(path, 'rb') as f:
fname = os.path.basename(f.name)
workflow_root = get_workflow_root()
if not path.startswith(workflow_root):
raise FileUploadError(
'Files and directories to be uploaded'
'must be under the workflow root directory.')
# Calculate the path that will store the file
# in the workflow controller, by subtracting
# the workflow root path from the file path
save_path = path.replace(workflow_root, '')
# Remove prepending dirs named "." or as the upload type
while len(save_path.split('/')) > 1 and \
save_path.split('/')[0] == '.':
save_path = "/".join(
save_path.strip("/").split('/')[1:])
logging.debug("'{}' is an absolute filepath."
.format(os.path.basename(fname)))
logging.info("Uploading '{}' ...".format(fname))
try:
response = upload_file(workflow, f, save_path,
access_token)
logging.info("File '{}' was successfully "
"uploaded.".format(fname))
if symlink:
save_path = 'symlink:{}'.format(save_path)
return [save_path]
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
logging.info("Something went wrong while uploading {}"
.format(fname))
def upload_files(ctx, workflow, filenames, access_token): # noqa: D301
"""Upload files and directories to workspace.
The `upload` command allows to upload workflow input files and
directories. The SOURCES argument can be repeated and specifies which files
and directories are to be uploaded, see examples below. The default
behaviour is to upload all input files and directories specified in the
reana.yaml file.
Examples: \n
\t $ reana-client upload -w myanalysis.42 \n
\t $ reana-client upload -w myanalysis.42 code/mycode.py
"""
from reana_client.api.client import upload_to_server
logging.debug('command: {}'.format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p]))
if not filenames:
reana_spec = load_reana_spec(os.path.join(get_workflow_root(),
'reana.yaml'),
False)
if 'inputs' in reana_spec:
filenames = []
filenames += [os.path.join(get_workflow_root(), f)
for f in reana_spec['inputs'].get('files') or []]
filenames += [os.path.join(get_workflow_root(), d)
for d in reana_spec['inputs'].
get('directories') or []]
if workflow:
if filenames:
for filename in filenames:
try:
response = upload_to_server(workflow,
filename,
access_token)
for file_ in response:
if file_.startswith('symlink:'):
click.echo(
click.style('Symlink resolved to {}. Uploaded'
' hard copy.'.
format(file_[len('symlink:'):]),
fg='green'))
else:
click.echo(
click.style('File {} was successfully '
'uploaded.'.format(file_),
fg='green'))
except FileNotFoundError as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(
click.style(
'File {0} could not be uploaded: {0} does not'
' exist.'.format(filename),
fg='red'),
err=True)
if 'invoked_by_subcommand' in ctx.parent.__dict__:
sys.exit(1)
except FileUploadError as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(
click.style(
'Something went wrong while uploading {0}.\n{1}'.
format(filename, str(e)),
fg='red'),
err=True)
if 'invoked_by_subcommand' in ctx.parent.__dict__:
sys.exit(1)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(
click.style(
'Something went wrong while uploading {}'.
format(filename),
fg='red'),
err=True)
if 'invoked_by_subcommand' in ctx.parent.__dict__:
sys.exit(1)
