def _download(src,
job_uuid,
size=None,
timestamp=None,
excludes=None,
includes=None,
dest=None,
block_size=4096,
atomic=False,
force=False,
sync=False,
agave=None):
local_filename, tmp_local_filename = _local_temp_filename(
src, dest, atomic)
if isinstance(includes, list) and len(includes) > 0:
if not fnmatches(src, includes):
raise FileExcludedError(
'{0} did not match --include filter'.format(src))
# Check filename is in the excludes list
if isinstance(excludes, list) and len(excludes) > 0:
if fnmatches(src, excludes):
raise FileExcludedError('{0} matched --exclude filter'.format(src))
if not _check_write(
local_filename, size, timestamp, sync=sync, force=force):
raise OutputFileExistsError(
'Local {0} exists and was not different from remote'.format(
local_filename))
try:
client = TaccApiDirectClient(agave)
# jobs/v2/9e74b852-0e1f-4363-8c09-5ab9f5299797-007/outputs/media/20190221t174839.out
url_path = '{0}/outputs/media'.format(job_uuid)
client.setup('jobs', 'v2', url_path)
# raise SystemError(client.build_url(src[1:]))
rsp = client.get_bytes(src[1:])
#rsp = agave.jobs.downloadOutput(filePath=src, jobId=job_uuid)
if isinstance(rsp, dict):
raise TapisOperationFailed("Failed to download {}".format(src))
with open(tmp_local_filename, 'wb') as dest_file:
for block in rsp.iter_content(block_size):
if not block:
break
dest_file.write(block)
if atomic:
try:
os.rename(tmp_local_filename, local_filename)
except Exception as err:
raise IOError('Rename failed after download', err)
except HTTPError as h:
handle_http_error(h)
except (OSError, IOError) as err:
logger.error(str(err))
raise
except Exception as exc:
raise TapisOperationFailed("Download failed: {}".format(exc))
def _download(src,
system_id,
size=None,
timestamp=None,
includes=None,
excludes=None,
dest=None,
block_size=4096,
atomic=False,
force=False,
sync=False,
agave=None):
local_filename, tmp_local_filename = _local_temp_filename(
src, dest, atomic)
# If includes is specified, check filename against it
if not fnmatches(src, includes):
raise FileExcludedError(
'{0} did not match --include filter'.format(src))
# Check filename is in the excludes list
if fnmatches(src, excludes):
raise FileExcludedError('{0} matched --exclude filter'.format(src))
if not _check_write(
local_filename, size, timestamp, force=force, sync=sync):
raise FileExistsError(
'Local {0} exists and was not different from remote.'.format(
local_filename))
try:
rsp = agave.files.download(filePath=src, systemId=system_id)
if isinstance(rsp, dict):
raise TapisOperationFailed("Failed to download {}".format(src))
with open(tmp_local_filename, 'wb') as dest_file:
for block in rsp.iter_content(block_size):
if not block:
break
dest_file.write(block)
if atomic:
try:
os.rename(tmp_local_filename, local_filename)
except Exception as err:
raise IOError('Rename failed after download', err)
except HTTPError as h:
handle_http_error(h)
except (OSError, IOError) as err:
print('Failed writing {}: {}'.format(tmp_local_filename, str(err)))
raise
except Exception as exc:
raise TapisOperationFailed("Download failed: {}".format(exc))
def take_action(self, parsed_args):
parsed_args = self.preprocess_args(parsed_args)
actor_id = ActorIdentifier().get_identifier(parsed_args)
glob_filt = parsed_args.list_filter
results = self.tapis_client.actors.listWorkers(
actorId=actor_id, **self.client_extra_args)
headers = ["workerId", "status"]
records = []
for rec in results:
include = False
if glob_filt is None:
include = True
else:
for k in self.FILTERABLE_KEYS:
if glob_filt in rec[k]:
include = True
elif fnmatches(rec[k], [glob_filt]):
include = True
if include:
record = []
record.append(rec.get('id'))
record.append(rec.get('status'))
if record not in records:
records.append(record)
return (tuple(headers), tuple(records))
def take_action(self, parsed_args):
parsed_args = self.preprocess_args(parsed_args)
self.requests_client.setup(API_NAME, SERVICE_VERSION)
self.update_payload(parsed_args)
results = self.tapis_client.actors.listAliases()
headers = self.render_headers(Alias, parsed_args)
records = []
for rec in results:
include = False
if parsed_args.list_filter is None:
include = True
else:
for k in self.FILTERABLE_KEYS:
if parsed_args.list_filter in rec[k]:
include = True
elif fnmatches(rec[k], [parsed_args.list_filter]):
include = True
if include:
record = []
for key in headers:
val = self.render_value(rec.get(key, None))
record.append(val)
records.append(record)
return (tuple(headers), tuple(records))
def take_action(self, parsed_args):
parsed_args = self.preprocess_args(parsed_args)
actor_id = ActorIdentifier().get_identifier(parsed_args)
# Use the requests_client because AgavePy is not configured
# with the alias-specific nonces endpoint
if parsed_args.is_alias:
api_path = 'aliases/' + actor_id + '/nonces'
self.requests_client.setup(API_NAME, SERVICE_VERSION, api_path)
results = self.requests_client.get()
else:
results = self.tapis_client.actors.listNonces(actorId=actor_id)
headers = self.render_headers(Nonce, parsed_args)
records = []
for rec in results:
include = False
if parsed_args.list_filter is None:
include = True
else:
for k in self.FILTERABLE_KEYS:
if parsed_args.list_filter in rec[k]:
include = True
elif fnmatches(rec[k], [parsed_args.list_filter]):
include = True
if include:
record = []
for key in headers:
val = self.render_value(rec.get(key, None))
record.append(val)
records.append(record)
return (tuple(headers), tuple(records))
def take_action(self, parsed_args):
parsed_args = self.preprocess_args(parsed_args)
actor_id = ActorIdentifier().get_identifier(parsed_args)
results = self.tapis_client.actors.listNonces(actorId=actor_id)
headers = self.render_headers(Nonce, parsed_args)
records = []
for rec in results:
include = False
if parsed_args.list_filter is None:
include = True
else:
for k in self.FILTERABLE_KEYS:
if parsed_args.list_filter in rec[k]:
include = True
elif fnmatches(rec[k], [parsed_args.list_filter]):
include = True
if include:
record = []
for key in headers:
val = self.render_value(rec.get(key, None))
record.append(val)
records.append(record)
return (tuple(headers), tuple(records))
def take_action(self, parsed_args):
parsed_args = self.preprocess_args(parsed_args)
actor_id = ActorIdentifier.get_identifier(self, parsed_args)
results = self.tapis_client.actors.listExecutions(
actorId=actor_id, **self.client_extra_args)
# custom headers to print all the execution id and status for a
# given actor id
execs_result = results.get('executions') # returns a list
headers = ["executionId", "status"]
records = []
for rec in execs_result:
include = False
if parsed_args.list_filter is None:
include = True
else:
for k in self.FILTERABLE_KEYS:
if parsed_args.list_filter in rec[k]:
include = True
elif fnmatches(rec[k], [parsed_args.list_filter]):
include = True
if include:
record = []
record.append(rec.get('id'))
record.append(rec.get('status'))
if record not in records:
records.append(record)
return (tuple(headers), tuple(records))
def _upload(local_file_path,
system_id,
destination='/',
includes=None,
excludes=[],
size=None,
timestamp=None,
block_size=4096,
atomic=False,
force=False,
sync=False,
agave=None):
"""Private wrapper for doing a file upload to Tapis
"""
# If includes is specified, check filename against it
if not fnmatches(local_file_path, includes):
raise FileExcludedError(
'{0} did not match --include filter'.format(local_file_path))
# Check filename is in the excludes list
if fnmatches(local_file_path, excludes):
raise FileExcludedError(
'{0} matched --exclude filter'.format(local_file_path))
# Check size and modtime of remote vs overwrite policy
# For eventual implementation of --sync and --force
if not _check_write(
local_file_path, system_id, size, timestamp, force=force,
sync=sync):
raise FileExistsError(
'Remote {0} exists and was not different from local.'.format(
local_file_path))
try:
agave.files.importData(systemId=system_id,
filePath=destination,
fileToUpload=open(local_file_path, 'rb'))
except HTTPError as h:
handle_http_error(h)
except (OSError, IOError) as err:
logger.error(str(err))
raise
except Exception as exc:
traceback.print_exc(file=sys.stdout)
raise TapisOperationFailed("Upload failed: {0}}".format(exc))
return True