def take_action(self, parsed_args):
parsed_args = self.preprocess_args(parsed_args)
self.requests_client.setup(API_NAME, SERVICE_VERSION)
self.update_payload(parsed_args)
destination = parsed_args.destination
(storage_system, file_path) = self.parse_url(parsed_args.agave_uri)
headers = self.render_headers(FileStaging, parsed_args)
rec = copy(file_path,
system_id=storage_system,
destination=destination,
permissive=False,
agave=self.tapis_client)
# Fixes issue where the name of the listed file/directory is not
# returned by the files service
if rec['name'] == '.':
rec['name'] = os.path.basename(rec['path'])
# Coerce path to absolute path
rec['path'] = abspath(rec['path'], '/')
data = []
for key in headers:
try:
val = rec[key]
except KeyError:
val = None
data.append(self.render_value(val))
return (tuple(headers), tuple(data))
def _files_list(directory_path,
system_id=DEFAULT_SYSTEM_ID,
limit=DEFAULT_PAGE_SIZE,
offset=0,
root_dir='/',
agave=None):
"""Private function to resiliently list a Tapis files directory
"""
rooted_directory_path = abspath(directory_path, root_dir)
logger.info('_files_list: agave://{}{}'.format(system_id, directory_path))
try:
return agave.files.list(systemId=system_id,
filePath=rooted_directory_path,
limit=limit,
offset=offset)
except Exception as err:
logger.warning('_files_list.error: {}'.format(err))
raise
def _jobs_outputs_list(directory_path,
job_uuid,
limit=DEFAULT_PAGE_SIZE,
offset=0,
root_dir='/',
agave=None):
"""Private function to resiliently list a Tapis job output directory
"""
rooted_directory_path = abspath(directory_path, root_dir)
logger.info('_jobs_outputs_list: agave://{}{}'.format(
job_uuid, directory_path))
try:
return agave.jobs.listOutputs(jobId=job_uuid,
filePath=rooted_directory_path,
limit=limit,
offset=offset)
except Exception as err:
logger.warning('_jobs_outputs_list.error: {}'.format(err))
raise
def stat(file_path,
system_id=DEFAULT_SYSTEM_ID,
root_dir='/',
permissive=False,
agave=None,
**kwargs):
"""Retrieve attributes for a given path on a Tapis storageSystem
Arguments:
file_path (str): The path from which to fetch attributes
system_id (str, optional): The Tapis storageSystem for file_path
root_dir (str, optional): Base path on the storageSystem if file_path is relative
permissive (bool, optional): Whether to raise an Exception on failure
agave (Agave, optional): An active Tapis client
Returns:
dict: A dictionary containing Tapis files API attributes
Raises:
HTTPError: A transport or web services error was encountered
TapisOperationFailed: Some other error prevented the operation
"""
try:
try:
rooted_file_path = abspath(file_path, root_dir)
resp = agave.files.list(filePath=rooted_file_path,
systemId=system_id,
limit=2)[0]
return AttrDict(resp)
except HTTPError as herr:
handle_http_error(herr)
except Exception as err:
raise TapisOperationFailed(
'Exception encountered with stat#files.list()', err)
except Exception as err:
logger.warning(
'Exception encountered in rsrc_exists(): {}'.format(err))
if permissive:
return dict()
else:
raise
def pems_list(file_path,
system_id=DEFAULT_SYSTEM_ID,
limit=DEFAULT_PAGE_SIZE,
offset=0,
root_dir='/',
permissive=False,
agave=None,
**kwargs):
try:
resp = agave.files.listPermissions(systemId=system_id,
filePath=file_path,
limit=limit,
offset=offset)
# This fixes an issue with files.listPermissions where the topmost
# path element is returned as a username. This is probably a service-
# level bug, rather than an AgavePy defect
top_dir = splitall(abspath(file_path, root_dir))[1]
pems = [p for p in resp if p['username'] != top_dir]
return pems
except Exception:
if permissive:
return []
else:
raise
def upload(local_file_path,
system_id,
destination='/',
excludes=None,
includes=None,
force=False,
sync=False,
atomic=False,
progress=True,
agave=None):
(uploaded, skipped, errors, ul_bytes, runtime) = ([], [], [], 0, None)
if excludes is None:
excludes = []
# Compile files to be uploaded
if progress:
print_stderr('Finding file(s) to upload...')
start_time = seconds()
upload_files = _local_walk(local_file_path)
elapsed_walk = seconds() - start_time
msg = 'Found {0} file(s) in {1}s'.format(len(upload_files), elapsed_walk)
logger.debug(msg)
if progress:
print_stderr(msg)
# Filter out excludes
# TODO - make sure abs and relpaths are supported
# TODO - support some kind of wildcard match
# upload_files = [f for f in upload_files_all if f[0] not in excludes]
# if progress:
# print_stderr('Excluding {0} file(s)'.format(
# len(upload_files_all) - len(upload_files)))
# Compute which, if any, remote directories might need to be created
# Note that these directory names will be relative to the destination path
if os.path.isfile(local_file_path):
dir_parent = os.path.dirname(local_file_path)
dir_basename = '.'
else:
dir_parent = os.path.dirname(local_file_path)
dir_basename = os.path.basename(local_file_path)
local_dirs = [
relpath(os.path.dirname(f[0]).replace(dir_parent, ''))
for f in upload_files
]
# Before adding the grandparent to set of created dirs, add the destinations to each upload record
for idx, uf in enumerate(upload_files):
upload_files[idx].append(os.path.join(destination, local_dirs[idx]))
# Remove duplicates as each member of create_dirs represents
# at least one API call
local_dirs.insert(0, dir_basename)
create_dirs = []
for d in local_dirs:
if d not in create_dirs and d not in ('.', ''):
create_dirs.append(d)
# Create the remote directories
# Do this ahead of time (and manually) to avoid relying on Tapis' files
# since that service's path handling behavior can be tricky
for rdir in create_dirs:
if progress:
print_stderr('Creating remote directory "{0}"...'.format(rdir))
makedirs(relpath(rdir),
system_id=system_id,
destination=abspath(destination),
agave=agave)
# Do the actual uploads
start_time_all = seconds()
for ufile in upload_files:
if progress:
print_stderr('Uploading {0}...'.format(ufile[0]))
try:
_upload(ufile[0],
system_id,
destination=ufile[3],
size=ufile[1],
timestamp=ufile[2],
includes=includes,
excludes=excludes,
force=force,
sync=sync,
agave=agave)
# TRack uploaded files
uploaded.append(ufile[0])
# Track cumulative data size
ul_bytes = ul_bytes + ufile[1]
# TODO - implement a separate exception for FileExcluded
except FileExcludedError as fexc:
errors.append(fexc)
skipped.append(ufile[0])
except FileExistsError as fxerr:
if sync or force:
skipped.append(ufile[0])
else:
errors.append(fxerr)
except Exception as exc:
errors.append(exc)
elapsed_download = seconds() - start_time_all
msg = 'Uploaded {0} files in {1}s'.format(len(uploaded), elapsed_download)
logger.debug(msg)
if progress:
print_stderr(msg)
return uploaded, skipped, errors, ul_bytes, elapsed_download