def remote_file_download(yabiusername, uri, is_dir=False):
"""Use a local fifo to download a remote file"""
logger.debug('{0} -> local fifo'.format(uri))
# we need ref to the backend
backend = FSBackend.urifactory(yabiusername, uri)
scheme, parts = uriparse(uri)
try:
# create a fifo, start the write to/read from fifo
fifo = create_fifo('remote_file_download_' + yabiusername + '_' + parts.hostname)
if is_dir:
thread, queue = backend.remote_dir_to_fifo(uri, fifo)
else:
thread, queue = backend.remote_to_fifo(uri, fifo)
infile = open(fifo, "rb")
try:
os.unlink(fifo)
except OSError:
logger.exception("Couldn't delete remote file download fifo")
return infile, thread, queue
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def remote_file_upload(yabiusername, filename, uri):
"""Use a local fifo to upload to a remote file"""
logger.debug('local_fifo -> {0}'.format(uri))
# we need ref to the backend
backend = FSBackend.urifactory(yabiusername, uri)
scheme, parts = uriparse(uri)
# uri for an upload must specify filename. we can't rely on the
# source name as we copy from a fifo with a random name
if not uri.endswith(filename):
if not uri.endswith('/'):
uri = uri + '/'
uri = uri + filename
try:
# create a fifo, start the write to/read from fifo
fifo = create_fifo('remote_file_upload_' + yabiusername + '_' + parts.hostname)
thread, queue = backend.fifo_to_remote(uri, fifo)
outfile = open(fifo, "wb")
try:
os.unlink(fifo)
except OSError:
logger.exception("Couldn't delete remote file upload fifo")
return outfile, queue
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def remote_file_copy(yabiusername, src_uri, dst_uri):
"""Use a local fifo to copy a single file from src_uri to dst_uri"""
logger.debug('remote_file_copy {0} -> {1}'.format(src_uri, dst_uri))
# we need refs to the src and dst backends
src_backend = FSBackend.urifactory(yabiusername, src_uri)
dst_backend = FSBackend.urifactory(yabiusername, dst_uri)
src_scheme, src_parts = uriparse(src_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
# Making sure dst_uri is always a file not a dir
if dst_parts.path.endswith("/"): # Looks like a dir
dst_file_uri = "%s/%s" % (dst_uri.rstrip('/'), src_backend.basename(src_parts.path))
dst_scheme, dst_parts = uriparse(dst_uri)
else:
dst_file_uri = dst_uri
fifo = None
try:
src_stat = src_backend.remote_uri_stat(src_uri)
# create a fifo, start the write to/read from fifo
fifo = create_fifo('remote_file_copy_' + yabiusername + '_' + src_parts.hostname + '_' + dst_parts.hostname)
src_cmd, src_queue = src_backend.remote_to_fifo(src_uri, fifo)
dst_cmd, dst_queue = dst_backend.fifo_to_remote(dst_file_uri, fifo)
src_cmd.join()
dst_cmd.join()
try:
os.unlink(fifo)
except OSError:
pass
src_success = src_queue.get()
dst_success = dst_queue.get()
# check exit status
if not src_success:
raise RetryException('remote_file_copy remote_to_fifo failed')
if not dst_success:
raise RetryException('remote_file_copy fifo_to_remote failed')
if src_stat:
atime = src_stat.get('atime')
mtime = src_stat.get('mtime')
dst_backend.set_remote_uri_times(dst_file_uri, atime, mtime)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
