def _backup(self, parsed_args):
"""Backup existing deployed assets
"""
if self.backup:
dep_sys = self.document['deploymentSystem']
dep_path = self.document['deploymentPath']
backup_dep_path = dep_path + '.' + str(seconds())
print_stderr('Backing up agave://{}/{}'.format(dep_sys, dep_path))
start_time = milliseconds()
self.messages.append(
('backup', 'src: agave://{}/{}'.format(dep_sys, dep_path)))
self.messages.append(
('backup', 'dst: agave://{}/{}'.format(dep_sys,
backup_dep_path)))
try:
manage.move(dep_path,
system_id=dep_sys,
destination=backup_dep_path,
agave=self.tapis_client)
print_stderr('Finished ({} msec)'.format(milliseconds() -
start_time))
return True
except Exception as exc:
self.messages.append(('backup', str(exc)))
print_stderr('Failed ({} msec)'.format(milliseconds() -
start_time))
return False
return True
def test_jsoncache():
"""Check that Python2/3 lru_cache functions both work w json serializer
"""
from tapis_cli.hashcache import jsoncache, lru_cache
@jsoncache.mcache(lru_cache(maxsize=256))
def timer_function(duration=1):
sleep(duration)
return 'function_response'
delay = 2
start_01 = seconds()
resp = timer_function(delay)
start_02 = seconds()
resp = timer_function(delay)
end_01 = seconds()
delta_1 = round(start_02 - start_01)
delta_2 = round(end_01 - start_02)
assert delta_1 == delay
assert delta_2 < delay
def upload(local_file_path,
system_id,
destination='/',
excludes=None,
includes=None,
force=False,
sync=False,
atomic=False,
progress=True,
agave=None):
(uploaded, skipped, errors, ul_bytes, runtime) = ([], [], [], 0, None)
if excludes is None:
excludes = []
# Compile files to be uploaded
if progress:
print_stderr('Finding file(s) to upload...')
start_time = seconds()
upload_files = _local_walk(local_file_path)
elapsed_walk = seconds() - start_time
msg = 'Found {0} file(s) in {1}s'.format(len(upload_files), elapsed_walk)
logger.debug(msg)
if progress:
print_stderr(msg)
# Filter out excludes
# TODO - make sure abs and relpaths are supported
# TODO - support some kind of wildcard match
# upload_files = [f for f in upload_files_all if f[0] not in excludes]
# if progress:
# print_stderr('Excluding {0} file(s)'.format(
# len(upload_files_all) - len(upload_files)))
# Compute which, if any, remote directories might need to be created
# Note that these directory names will be relative to the destination path
if os.path.isfile(local_file_path):
dir_parent = os.path.dirname(local_file_path)
dir_basename = '.'
else:
dir_parent = os.path.dirname(local_file_path)
dir_basename = os.path.basename(local_file_path)
local_dirs = [
relpath(os.path.dirname(f[0]).replace(dir_parent, ''))
for f in upload_files
]
# Before adding the grandparent to set of created dirs, add the destinations to each upload record
for idx, uf in enumerate(upload_files):
upload_files[idx].append(os.path.join(destination, local_dirs[idx]))
# Remove duplicates as each member of create_dirs represents
# at least one API call
local_dirs.insert(0, dir_basename)
create_dirs = []
for d in local_dirs:
if d not in create_dirs and d not in ('.', ''):
create_dirs.append(d)
# Create the remote directories
# Do this ahead of time (and manually) to avoid relying on Tapis' files
# since that service's path handling behavior can be tricky
for rdir in create_dirs:
if progress:
print_stderr('Creating remote directory "{0}"...'.format(rdir))
makedirs(relpath(rdir),
system_id=system_id,
destination=abspath(destination),
agave=agave)
# Do the actual uploads
start_time_all = seconds()
for ufile in upload_files:
if progress:
print_stderr('Uploading {0}...'.format(ufile[0]))
try:
_upload(ufile[0],
system_id,
destination=ufile[3],
size=ufile[1],
timestamp=ufile[2],
includes=includes,
excludes=excludes,
force=force,
sync=sync,
agave=agave)
# TRack uploaded files
uploaded.append(ufile[0])
# Track cumulative data size
ul_bytes = ul_bytes + ufile[1]
# TODO - implement a separate exception for FileExcluded
except FileExcludedError as fexc:
errors.append(fexc)
skipped.append(ufile[0])
except FileExistsError as fxerr:
if sync or force:
skipped.append(ufile[0])
else:
errors.append(fxerr)
except Exception as exc:
errors.append(exc)
elapsed_download = seconds() - start_time_all
msg = 'Uploaded {0} files in {1}s'.format(len(uploaded), elapsed_download)
logger.debug(msg)
if progress:
print_stderr(msg)
return uploaded, skipped, errors, ul_bytes, elapsed_download
def download(source,
system_id,
destination='.',
includes=None,
excludes=None,
force=False,
sync=False,
atomic=False,
progress=False,
agave=None):
downloaded, skipped, errors, dl_bytes, runtime = ([], [], [], 0, None)
if excludes is None:
excludes = []
if includes is None:
includes = []
if progress:
print_stderr('Walking remote resource...')
start_time = seconds()
all_targets = walk(source,
system_id=system_id,
dotfiles=True,
recurse=True,
agave=agave)
elapsed_walk = seconds() - start_time
msg = 'Found {0} file(s) in {1}s'.format(len(all_targets), elapsed_walk)
logger.debug(msg)
if progress:
print_stderr(msg)
# Filters that build up list of paths to create and files to download
abs_names = [f['path'] for f in all_targets]
sizes = [f['length'] for f in all_targets]
mods = [datestring_to_epoch(f['lastModified']) for f in all_targets]
# Create local destination paths
dirs = [os.path.dirname(p) for p in abs_names]
if not isfile(source, system_id=system_id, agave=agave):
sub_root = None
if source.endswith('/'):
sub_root = source
else:
sub_root = os.path.dirname(source)
sub_root = re.sub('([/]+)$', '', sub_root)
dirs = [re.sub(sub_root, '', d) for d in dirs]
dest_names = [
os.path.join(destination, relpath(re.sub(sub_root, '', f)))
for f in abs_names
]
dirs = [d for d in dirs if d != sub_root]
make_dirs = [os.path.join(destination, relpath(p)) for p in dirs]
# Create destinations
for dir in make_dirs:
makedirs(dir, exist_ok=True)
else:
sub_root = os.path.dirname(source)
dest_names = [os.path.join(destination, os.path.basename(source))]
# Do the downloads
downloads = [list(a) for a in zip(abs_names, sizes, mods, dest_names)]
start_time_all = seconds()
for src, size, mod, dest in downloads:
if progress:
print_stderr('Downloading {0}...'.format(os.path.basename(src)))
try:
_download(src,
system_id,
size=size,
timestamp=mod,
dest=dest,
includes=includes,
excludes=excludes,
force=force,
sync=sync,
atomic=False,
agave=agave)
downloaded.append(src)
# Track cumulative data size
dl_bytes = dl_bytes + size
except (FileExistsError, FileExcludedError) as fxerr:
if sync or force:
skipped.append(src)
errors.append(fxerr)
else:
errors.append(fxerr)
except Exception as exc:
errors.append(exc)
elapsed_download = seconds() - start_time_all
msg = 'Downloaded {0} files in {1}s'.format(len(abs_names),
elapsed_download)
logger.debug(msg)
if progress:
print_stderr(msg)
return downloaded, skipped, errors, dl_bytes, elapsed_walk + elapsed_download
def download(source,
job_uuid,
destination=None,
excludes=None,
includes=None,
force=False,
sync=False,
atomic=False,
progress=False,
agave=None):
downloaded, skipped, errors, runtime = ([], [], [], None)
if destination is None:
dest_dir = str(job_uuid)
else:
dest_dir = destination
# else:
# includes = [os.path.join('/', i) for i in includes]
if progress:
print_stderr('Walking remote resource...')
start_time = seconds()
# Try to avoid timeouts since walk is already pretty slow
agave.refresh()
all_targets = walk(source,
job_uuid=job_uuid,
dotfiles=True,
recurse=True,
agave=agave)
elapsed_walk = seconds() - start_time
msg = 'Found {0} file(s) in {1}s'.format(len(all_targets), elapsed_walk)
logger.debug(msg)
if progress:
print_stderr(msg)
# Extract absolute names
# Under jobs, paths all begin with /
paths = [f['path'] for f in all_targets]
# Tapis Jobs returns a spurious "null/" at the start of
# each file's path. This is a temporary workaround.
paths = [re.sub('null/', '/', p) for p in paths]
sizes = [f['length'] for f in all_targets]
mods = [datestring_to_epoch(f['lastModified']) for f in all_targets]
# Create local destination paths
dirs = [os.path.dirname(p) for p in paths]
make_dirs = [
os.path.join(dest_dir, relpath(p)) for p in dirs
if p not in ('/', './')
]
# Create destinations
for dir in make_dirs:
makedirs(dir, exist_ok=True)
# Local filenames including destination directory
rel_paths = [os.path.join(dest_dir, relpath(p)) for p in paths]
downloads = [list(a) for a in zip(paths, sizes, mods, rel_paths)]
start_time_all = seconds()
# Try to avoid timeouts since walk is already pretty slow
agave.refresh()
for src, size, mod, dest in downloads:
if progress:
print_stderr('Downloading {0}...'.format(os.path.basename(src)))
try:
# TODO - refresh token is size > threshold
_download(src,
job_uuid,
size=size,
timestamp=mod,
dest=dest,
includes=includes,
excludes=excludes,
atomic=atomic,
force=force,
sync=sync,
agave=agave)
downloaded.append(src)
except FileExcludedError as fexc:
errors.append(fexc)
skipped.append(src)
except OutputFileExistsError as ofe:
if sync or force:
skipped.append(src)
errors.append(ofe)
except Exception as exc:
errors.append(exc)
elapsed_download = seconds() - start_time_all
msg = 'Downloaded {0} files in {1}s'.format(len(paths), elapsed_download)
logger.debug(msg)
if progress:
print_stderr(msg)
return downloaded, skipped, errors, elapsed_walk + elapsed_download