def _submit_job(self):
exec_scheme, exec_parts = uriparse(self.task.job.exec_backend)
working_scheme, working_parts = uriparse(self.working_output_dir_uri())
self.working_dir = working_parts.path
self.submission_script_name = self.executer.generate_remote_script_name(
)
self.task.job_identifier = self.submission_script_name
self.task.save()
self.task_logger.info("Creating submission script %s" %
self.submission_script_name)
self.submission_script_body = self.get_submission_script(
exec_parts.hostname, self.working_dir)
wrapper_script = self._get_submission_wrapper_script()
self.task_logger.info("Executing script:\n\n%s" % wrapper_script)
exit_code, stdout, stderr = self.executer.exec_script(wrapper_script)
result = self.parser.parse_sub(exit_code, stdout, stderr)
if result.status != result.JOB_SUBMITTED:
self.task_logger.error("Yabi Task Name = %s" %
self._yabi_task_name())
self.task_logger.error("Submission script name = %s" %
self.submission_script_name)
self.task_logger.error("Submission script body = %s" %
self.submission_script_body)
self.task_logger.error("stderr:\n")
lines = "\n".join(stderr)
self.task_logger.error(lines)
return result
def submit_task(self):
"""
For local exec submitting a task executes the task and blocks
the current process. It is not intended for large scale real world usage.
"""
exec_scheme, exec_parts = uriparse(self.task.job.exec_backend)
working_scheme, working_parts = uriparse(self.working_output_dir_uri())
script = self.get_submission_script(exec_parts.hostname,
working_parts.path)
logger.debug('script {0}'.format(script))
script_name = self.create_script(script)
if os.path.exists(working_parts.path):
shutil.rmtree(working_parts.path)
os.makedirs(working_parts.path)
try:
stdout = open(os.path.join(working_parts.path, 'STDOUT.txt'), 'w')
stderr = open(os.path.join(working_parts.path, 'STDERR.txt'), 'w')
logger.debug('Running in {0}'.format(working_parts.path))
args = shlex.split(self.task.command.encode('utf-8'))
def set_remote_id(pid):
self.task.remote_id = pid
self.task.save()
args = [script_name]
status = blocking_execute(args=args,
stderr=stderr,
stdout=stdout,
cwd=working_parts.path,
report_pid_callback=set_remote_id)
if status != 0:
if self.is_aborting():
return None
logger.error('Non zero exit status [{0}]'.format(status))
raise RetryException(
'Local Exec of command "{0}" retuned non-zero code {1}'.
format(" ".join(args), status))
except Exception as exc:
raise RetryException(exc)
finally:
try:
stdout.close()
stderr.close()
except Exception as exc:
logger.error(exc)
try:
os.unlink(script_name)
except:
logger.exception("Couldn't delete script file %s", script_name)
return status
def __init__(self, *args, **kwargs):
ret = Task.__init__(self, *args, **kwargs)
# basic stuff used by both stagein types
self.fsscheme, self.fsbackend_parts = uriparse(self.job.fs_backend)
self.execscheme, self.execbackend_parts = uriparse(self.job.exec_backend)
return ret
def local_copy(self, src_uri, dst_uri):
"""A local copy within this backend."""
logger.debug('local_copy {0} -> {1}'.format(src_uri, dst_uri))
src_scheme, src_parts = uriparse(src_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
try:
shutil.copy2(src_parts.path, dst_parts.path)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def __init__(self, *args, **kwargs):
ret = Task.__init__(self, *args, **kwargs)
# basic stuff used by both stagein types
self.fsscheme, self.fsbackend_parts = uriparse(self.job.fs_backend)
self.execscheme, self.execbackend_parts = uriparse(
self.job.exec_backend)
return ret
def local_copy(self, src_uri, dst_uri):
"""A local copy within this backend."""
logger.debug('local_copy {0} -> {1}'.format(src_uri, dst_uri))
src_scheme, src_parts = uriparse(src_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
try:
shutil.copy2(src_parts.path, dst_parts.path)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def symbolic_link(self, target_uri, link_uri):
"""symbolic link to target_uri called link_uri."""
logger.debug('symbolic_link {0} -> {1}'.format(target_uri, link_uri))
target_scheme, target_parts = uriparse(target_uri)
link_scheme, link_parts = uriparse(link_uri)
target = target_parts.path
try:
if not os.path.exists(target):
raise FileNotFoundError("Source of symbolic link '%s' doesn't exist" % target)
os.symlink(target, link_parts.path)
except OSError as ose:
raise RetryException(ose, traceback.format_exc())
def symbolic_link(self, src_uri, dst_uri):
"""symbolic link to target_uri called link_uri."""
logger.debug("SFTPBackend.symbolic_link: %s => %s", src_uri, dst_uri)
src_scheme, src_parts = uriparse(src_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
logger.debug('{0} -> {1}'.format(src_uri, dst_uri))
executer = create_executer(self.yabiusername, src_uri)
try:
executer.local_symlink(src_parts.path, dst_parts.path)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def symbolic_link(self, src_uri, dst_uri):
"""symbolic link to target_uri called link_uri."""
logger.debug("SFTPBackend.symbolic_link: %s => %s", src_uri, dst_uri)
src_scheme, src_parts = uriparse(src_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
logger.debug("{0} -> {1}".format(src_uri, dst_uri))
executer = create_executer(self.yabiusername, src_uri)
try:
executer.local_symlink(src_parts.path, dst_parts.path)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def submit_task(self):
"""
For local exec submitting a task executes the task and blocks
the current process. It is not intended for large scale real world usage.
"""
exec_scheme, exec_parts = uriparse(self.task.job.exec_backend)
working_scheme, working_parts = uriparse(self.working_output_dir_uri())
script = self.get_submission_script(exec_parts.hostname, working_parts.path)
logger.debug('script {0}'.format(script))
script_name = self.create_script(script)
if os.path.exists(working_parts.path):
shutil.rmtree(working_parts.path)
os.makedirs(working_parts.path)
try:
stdout = open(os.path.join(working_parts.path, 'STDOUT.txt'), 'w')
stderr = open(os.path.join(working_parts.path, 'STDERR.txt'), 'w')
logger.debug('Running in {0}'.format(working_parts.path))
args = shlex.split(self.task.command.encode('utf-8'))
def set_remote_id(pid):
self.task.remote_id = pid
self.task.save()
args = [script_name]
status = blocking_execute(args=args, stderr=stderr, stdout=stdout, cwd=working_parts.path, report_pid_callback=set_remote_id)
if status != 0:
if self.is_aborting():
return None
logger.error('Non zero exit status [{0}]'.format(status))
raise RetryException('Local Exec of command "{0}" retuned non-zero code {1}'.format(" ".join(args), status))
except Exception as exc:
raise RetryException(exc)
finally:
try:
stdout.close()
stderr.close()
except Exception as exc:
logger.error(exc)
try:
os.unlink(script_name)
except:
logger.exception("Couldn't delete script file %s", script_name)
return status
def local_copy(self, src_uri, dst_uri):
"""Copy src_uri to dst_uri on the remote backend"""
logger.debug("SFTPBackend.local_copy: %s => %s", src_uri, dst_uri)
src_scheme, src_parts = uriparse(src_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
logger.debug("{0} -> {1}".format(src_uri, dst_uri))
# Given paramiko does not support local copy, we
# use cp on server via exec backend
executer = create_executer(self.yabiusername, src_uri)
try:
executer.local_copy(src_parts.path, dst_parts.path)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def local_copy(self, src_uri, dst_uri):
"""Copy src_uri to dst_uri on the remote backend"""
logger.debug("SFTPBackend.local_copy: %s => %s", src_uri, dst_uri)
src_scheme, src_parts = uriparse(src_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
logger.debug('{0} -> {1}'.format(src_uri, dst_uri))
# Given paramiko does not support local copy, we
# use cp on server via exec backend
executer = create_executer(self.yabiusername, src_uri)
try:
executer.local_copy(src_parts.path, dst_parts.path)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def symbolic_link(self, target_uri, link_uri):
"""symbolic link to target_uri called link_uri."""
logger.debug('symbolic_link {0} -> {1}'.format(target_uri, link_uri))
target_scheme, target_parts = uriparse(target_uri)
link_scheme, link_parts = uriparse(link_uri)
target = target_parts.path
try:
if not os.path.exists(target):
raise FileNotFoundError(
"Source of symbolic link '%s' doesn't exist" % target)
os.symlink(target, link_parts.path)
except OSError as ose:
raise RetryException(ose, traceback.format_exc())
def local_copy_recursive(self, src_uri, dst_uri):
"""A local copy within this backend."""
logger.debug('local_copy {0} -> {1}'.format(src_uri, dst_uri))
src_scheme, src_parts = uriparse(src_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
try:
for item in os.listdir(src_parts.path):
src = os.path.join(src_parts.path, item)
dst = os.path.join(dst_parts.path, item)
if os.path.isdir(src):
shutil.copytree(src, dst)
else:
shutil.copy2(src, dst)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def local_copy_recursive(self, src_uri, dst_uri):
"""A local copy within this backend."""
logger.debug('local_copy {0} -> {1}'.format(src_uri, dst_uri))
src_scheme, src_parts = uriparse(src_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
try:
for item in os.listdir(src_parts.path):
src = os.path.join(src_parts.path, item)
dst = os.path.join(dst_parts.path, item)
if os.path.isdir(src):
shutil.copytree(src, dst)
else:
shutil.copy2(src, dst)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def get_exec_backendcredential_for_uri(yabiusername, uri):
"""
Looks up a backend credential based on the supplied uri, which should include a username.
Returns bc, will log and reraise ObjectDoesNotExist and MultipleObjectsReturned exceptions if more than one credential
"""
logger.debug('yabiusername: %s uri: %s' % (yabiusername, uri))
# parse the URI into chunks
schema, rest = uriparse(uri)
# enforce Exec scehmas only
from ..backend import ExecBackend
if not ExecBackend.get_backend_cls_for_scheme(schema):
logger.error("get_exec_backendcredential_for_uri was asked to get an fs schema! This is forbidden.")
raise ValueError("Invalid schema in uri passed to get_exec_backendcredential_for_uri: asked for %s" % schema)
path = rest.path
if path != "/":
logger.error("get_exec_backendcredential_for_uri was passed a uri with a path! This is forbidden. Path must be / for exec backends")
raise ValueError("Invalid path in uri passed to get_exec_backendcredential_for_uri: path passed in was: %s" % path)
# get our set of credential candidates
bcs = _get_credential_candidates(yabiusername, schema, rest.username, rest.hostname)
if len(bcs) == 0:
raise ObjectDoesNotExist("Could not find backendcredential")
return bcs[0]
def get_fs_backendcredential_for_uri(yabiusername, uri):
"""
Looks up a backend credential based on the supplied uri, which should include a username.
Returns bc, will log and reraise ObjectDoesNotExist and MultipleObjectsReturned exceptions if more than one credential
"""
logger.debug('yabiusername: %s uri: %s' % (yabiusername, uri))
schema, rest = uriparse(uri)
_enforce_FS_schema_only(schema)
path = _normalise_path(rest.path)
bc_candidates = _get_credential_candidates(yabiusername, schema, rest.username, rest.hostname)
logger.debug("credentials candidates [%s]" % (",".join([str(x) for x in bc_candidates])))
matches_path = partial(_does_path_match_be_cred, path)
matching_bcs = filter(matches_path, bc_candidates)
if len(matching_bcs) == 0:
raise ObjectDoesNotExist("Could not find backendcredential")
cred = _find_be_cred_with_longest_path(matching_bcs)
logger.info("chose cred {0} {1} {2}".format(cred.id, cred.backend.path, cred.homedir))
return cred
def exec_script(self, script):
logger.debug("SSHExex.exec_script...")
logger.debug('script content = {0}'.format(script))
exec_scheme, exec_parts = uriparse(self.uri)
ssh = sshclient(exec_parts.hostname, exec_parts.port, self.credential)
sftp = None
try:
sftp = ssh.open_sftp()
script_name = self.upload_script(sftp, script)
stdin, stdout, stderr = ssh.exec_command(script_name, bufsize=-1, timeout=None, get_pty=False)
stdin.close()
exit_code = stdout.channel.recv_exit_status()
logger.debug("sshclient exec'd script OK")
self.remove_script(sftp, script_name)
return exit_code, stdout.readlines(), stderr.readlines()
except paramiko.SSHException as sshe:
raise RetryException(sshe, traceback.format_exc())
finally:
try:
if sftp is not None:
sftp.close()
if ssh is not None:
ssh.close()
except:
pass
def get_fs_backendcredential_for_uri(yabiusername, uri):
"""
Looks up a backend credential based on the supplied uri, which should include a username.
Returns bc, will log and reraise ObjectDoesNotExist and MultipleObjectsReturned exceptions if more than one credential
"""
logger.debug('yabiusername: %s uri: %s' % (yabiusername, uri))
schema, rest = uriparse(uri)
_enforce_FS_schema_only(schema)
path = _normalise_path(rest.path)
bc_candidates = _get_credential_candidates(yabiusername, schema,
rest.username, rest.hostname)
logger.debug("credentials candidates [%s]" %
(",".join([str(x) for x in bc_candidates])))
matches_path = partial(_does_path_match_be_cred, path)
matching_bcs = filter(matches_path, bc_candidates)
if len(matching_bcs) == 0:
raise ObjectDoesNotExist("Could not find backendcredential")
cred = _find_be_cred_with_longest_path(matching_bcs)
logger.info("chose cred {0} {1} {2}".format(cred.id, cred.backend.path,
cred.homedir))
return cred
def remote_file_upload(yabiusername, filename, uri):
"""Use a local fifo to upload to a remote file"""
logger.debug('local_fifo -> {0}'.format(uri))
# we need ref to the backend
backend = FSBackend.urifactory(yabiusername, uri)
scheme, parts = uriparse(uri)
# uri for an upload must specify filename. we can't rely on the
# source name as we copy from a fifo with a random name
if not uri.endswith(filename):
if not uri.endswith('/'):
uri = uri + '/'
uri = uri + filename
try:
# create a fifo, start the write to/read from fifo
fifo = create_fifo('remote_file_upload_' + yabiusername + '_' + parts.hostname)
thread, queue = backend.fifo_to_remote(uri, fifo)
outfile = open(fifo, "wb")
try:
os.unlink(fifo)
except OSError:
logger.exception("Couldn't delete remote file upload fifo")
return outfile, queue
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def remote_file_download(yabiusername, uri, is_dir=False):
"""Use a local fifo to download a remote file"""
logger.debug('{0} -> local fifo'.format(uri))
# we need ref to the backend
backend = FSBackend.urifactory(yabiusername, uri)
scheme, parts = uriparse(uri)
try:
# create a fifo, start the write to/read from fifo
fifo = create_fifo('remote_file_download_' + yabiusername + '_' + parts.hostname)
if is_dir:
thread, queue = backend.remote_dir_to_fifo(uri, fifo)
else:
thread, queue = backend.remote_to_fifo(uri, fifo)
infile = open(fifo, "rb")
try:
os.unlink(fifo)
except OSError:
logger.exception("Couldn't delete remote file download fifo")
return infile, thread, queue
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def exec_script(self, script):
logger.debug("SSHExex.exec_script...")
logger.debug("script content = {0}".format(script))
exec_scheme, exec_parts = uriparse(self.uri)
ssh = sshclient(exec_parts.hostname, exec_parts.port, self.credential)
sftp = None
try:
sftp = ssh.open_sftp()
script_name = self.upload_script(sftp, script)
stdin, stdout, stderr = ssh.exec_command(script_name, bufsize=-1, timeout=None, get_pty=False)
stdin.close()
exit_code = stdout.channel.recv_exit_status()
logger.debug("sshclient exec'd script OK")
self.remove_script(sftp, script_name)
return exit_code, stdout.readlines(), stderr.readlines()
except paramiko.SSHException as sshe:
raise RetryException(sshe, traceback.format_exc())
finally:
try:
if sftp is not None:
sftp.close()
if ssh is not None:
ssh.close()
except:
pass
def parse_s3_uri(self, uri):
if uri.endswith(DELIMITER):
uri = uri.rstrip(DELIMITER) + DELIMITER
scheme, parts = uriparse(uri)
bucket_name = parts.hostname.split('.')[0]
path = parts.path
return bucket_name, path
def remote_file_copy(yabiusername, src_uri, dst_uri):
"""Use a local fifo to copy a single file from src_uri to dst_uri"""
logger.debug('remote_file_copy {0} -> {1}'.format(src_uri, dst_uri))
# we need refs to the src and dst backends
src_backend = FSBackend.urifactory(yabiusername, src_uri)
dst_backend = FSBackend.urifactory(yabiusername, dst_uri)
src_scheme, src_parts = uriparse(src_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
# Making sure dst_uri is always a file not a dir
if dst_parts.path.endswith("/"): # Looks like a dir
dst_file_uri = "%s/%s" % (dst_uri.rstrip('/'), src_backend.basename(src_parts.path))
dst_scheme, dst_parts = uriparse(dst_uri)
else:
dst_file_uri = dst_uri
fifo = None
try:
src_stat = src_backend.remote_uri_stat(src_uri)
# create a fifo, start the write to/read from fifo
fifo = create_fifo('remote_file_copy_' + yabiusername + '_' + src_parts.hostname + '_' + dst_parts.hostname)
src_cmd, src_queue = src_backend.remote_to_fifo(src_uri, fifo)
dst_cmd, dst_queue = dst_backend.fifo_to_remote(dst_file_uri, fifo)
src_cmd.join()
dst_cmd.join()
try:
os.unlink(fifo)
except OSError:
pass
src_success = src_queue.get()
dst_success = dst_queue.get()
# check exit status
if not src_success:
raise RetryException('remote_file_copy remote_to_fifo failed')
if not dst_success:
raise RetryException('remote_file_copy fifo_to_remote failed')
if src_stat:
atime = src_stat.get('atime')
mtime = src_stat.get('mtime')
dst_backend.set_remote_uri_times(dst_file_uri, atime, mtime)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def download_dir(self, uri, outfile):
logger.debug("SFTPBackend.download_dir: %s => tarball => %s", uri, outfile)
scheme, parts = uriparse(uri)
executer = create_executer(self.yabiusername, uri)
try:
return executer.download_dir_as_tarball(parts.path, outfile)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def parse_s3_uri(self, uri):
if uri.endswith(DELIMITER):
uri = uri.rstrip(DELIMITER) + DELIMITER
scheme, parts = uriparse(uri)
bucket_name = parts.hostname.split('.')[0]
path = parts.path
return bucket_name, path
def download_file(self, uri, outfile):
scheme, parts = uriparse(uri)
return self._sftp_copy(host=parts.hostname,
port=parts.port,
credential=self.cred.credential,
localfo=outfile,
remotepath=parts.path,
copy='get',
hostkey=None)
def download_dir(self, uri, outfile):
logger.debug("SFTPBackend.download_dir: %s => tarball => %s", uri,
outfile)
scheme, parts = uriparse(uri)
executer = create_executer(self.yabiusername, uri)
try:
return executer.download_dir_as_tarball(parts.path, outfile)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def sshclient(self):
exec_scheme, exec_parts = uriparse(self.uri)
ssh = sshclient(exec_parts.hostname, exec_parts.port, self.credential)
try:
yield ssh
finally:
try:
ssh.close()
except:
pass
def sshclient(self):
exec_scheme, exec_parts = uriparse(self.uri)
ssh = sshclient(exec_parts.hostname, exec_parts.port, self.credential)
try:
yield ssh
finally:
try:
ssh.close()
except:
pass
def _convert(self, uri):
"""convert a uri into a full remote path"""
schema, rest = uriparse(uri)
return self.uri_conversion_string % {
'schema': schema,
'hostname': rest.hostname,
'username': rest.username,
'port': rest.port,
'fullpath': rest.path,
'filename': rest.path.rsplit("/", 1)[-1]
}
def download_file(self, uri, outfile):
scheme, parts = uriparse(uri)
return self._sftp_copy(
host=parts.hostname,
port=parts.port,
credential=self.cred.credential,
localfo=outfile,
remotepath=parts.path,
copy="get",
hostkey=None,
)
def _convert(self, uri):
"""convert a uri into a full remote path"""
schema, rest = uriparse(uri)
return self.uri_conversion_string % {
'schema': schema,
'hostname': rest.hostname,
'username': rest.username,
'port': rest.port,
'fullpath': rest.path,
'filename': rest.path.rsplit("/", 1)[-1]
}
def urifactory(yabiusername, uri):
assert(uri)
fsscheme, fsbackend_parts = uriparse(uri)
backend = FSBackend.create_backend_for_scheme(fsscheme)
if backend is None:
raise Exception("No backend can be found for uri %s with fsscheme %s for user %s" % (uri, fsscheme, yabiusername))
backend.yabiusername = yabiusername
backend.cred = fs_credential(yabiusername, uri)
return backend
def mkdir(self, uri):
"""mkdir at uri"""
logger.debug('mkdir {0}'.format(uri))
scheme, parts = uriparse(uri)
if os.path.exists(parts.path) and os.path.isdir(parts.path):
return
try:
os.makedirs(parts.path)
except OSError as ose:
raise RetryException(ose, traceback.format_exc())
def mkdir(self, uri):
"""mkdir at uri"""
logger.debug('mkdir {0}'.format(uri))
scheme, parts = uriparse(uri)
if os.path.exists(parts.path) and os.path.isdir(parts.path):
return
try:
os.makedirs(parts.path)
except OSError as ose:
raise RetryException(ose, traceback.format_exc())
def _submit_job(self):
exec_scheme, exec_parts = uriparse(self.task.job.exec_backend)
working_scheme, working_parts = uriparse(self.working_output_dir_uri())
self.working_dir = working_parts.path
self.submission_script_name = self.executer.generate_remote_script_name()
self.task.job_identifier = self.submission_script_name
self.task.save()
self.task_logger.info("Creating submission script %s" % self.submission_script_name)
self.submission_script_body = self.get_submission_script(exec_parts.hostname, self.working_dir)
wrapper_script = self._get_submission_wrapper_script()
self.task_logger.info("Executing script:\n\n%s" % wrapper_script)
exit_code, stdout, stderr = self.executer.exec_script(wrapper_script)
result = self.parser.parse_sub(exit_code, stdout, stderr)
if result.status != result.JOB_SUBMITTED:
self.task_logger.error("Yabi Task Name = %s" % self._yabi_task_name())
self.task_logger.error("Submission script name = %s" % self.submission_script_name)
self.task_logger.error("Submission script body = %s" % self.submission_script_body)
self.task_logger.error("stderr:\n")
lines = "\n".join(stderr)
self.task_logger.error(lines)
return result
def get_backend_by_uri(uri):
"""
Looks up a backend object purely by uri. Ignored username. Thus diesnt consider credentials.
Just pure hostname/portnumber. Used by HostKey system for find what hostkeys are allowed
"""
schema, rest = uriparse(uri)
netloc = rest.netloc
if ':' in netloc:
host, port = netloc.split(':')
else:
host, port = netloc, None
return Backend.objects.filter(scheme=schema, hostname=host, port=port)
def get_backend_by_uri(uri):
"""
Looks up a backend object purely by uri. Ignored username. Thus diesnt consider credentials.
Just pure hostname/portnumber. Used by HostKey system for find what hostkeys are allowed
"""
schema, rest = uriparse(uri)
netloc = rest.netloc
if ':' in netloc:
host, port = netloc.split(':')
else:
host, port = netloc, None
return Backend.objects.filter(scheme=schema, hostname=host, port=port)
def rm(self, uri):
"""recursively delete a uri"""
scheme, parts = uriparse(uri)
logger.debug("{0}".format(parts.path))
ssh = sshclient(parts.hostname, parts.port, self.cred.credential)
try:
sftp = ssh.open_sftp()
self._rm(sftp, parts.path)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
finally:
try:
if ssh is not None:
ssh.close()
except:
pass
def rm(self, uri):
"""recursively delete a uri"""
scheme, parts = uriparse(uri)
logger.debug('{0}'.format(parts.path))
ssh = sshclient(parts.hostname, parts.port, self.cred.credential)
try:
sftp = ssh.open_sftp()
self._rm(sftp, parts.path)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
finally:
try:
if ssh is not None:
ssh.close()
except:
pass
def set_remote_uri_times(self, uri, atime, mtime):
scheme, parts = uriparse(uri)
remotepath = parts.path
ssh = None
try:
ssh = pool_manager.borrow(parts.hostname, parts.port, self.cred.credential)
sftp = ssh.open_sftp()
sftp.utime(remotepath, (atime, mtime))
except Exception:
logger.exception("Exception while setting times for '%s'", uri)
raise
finally:
if ssh is not None:
sftp.close()
pool_manager.give_back(ssh, parts.hostname, parts.port, self.cred.credential)
def rm(self, uri):
"""rm uri"""
logger.debug('rm {0}'.format(uri))
scheme, parts = uriparse(uri)
logger.debug('{0}'.format(parts.path))
if not os.path.exists(parts.path):
raise Exception('rm target ({0}) is not a file or directory'.format(parts.path))
try:
path = parts.path.rstrip('/')
if os.path.isfile(path) or os.path.islink(path):
os.unlink(path)
elif os.path.isdir(path):
shutil.rmtree(path)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def remote_uri_stat(self, uri):
scheme, parts = uriparse(uri)
remotepath = parts.path
ssh = None
try:
ssh = pool_manager.borrow(parts.hostname, parts.port, self.cred.credential)
sftp = ssh.open_sftp()
stat = sftp.stat(remotepath)
return {"atime": stat.st_atime, "mtime": stat.st_mtime}
except Exception:
logger.exception("Exception while stating '%s'", uri)
raise
finally:
if ssh is not None:
sftp.close()
pool_manager.give_back(ssh, parts.hostname, parts.port, self.cred.credential)
def local_copy_recursive(self, src_uri, dst_uri):
"""recursively copy src_uri to dst_uri on the remote backend"""
logger.debug("SFTPBackend.local_copy_recursive: %s => %s", src_uri, dst_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
dst_path = dst_parts.path
listing = self.ls(src_uri)
executer = create_executer(self.yabiusername, src_uri)
try:
for key in listing:
for listing_file in listing[key]["files"]:
file_path = os.path.join(key, listing_file[0])
executer.local_copy(file_path, dst_path)
for listing_dir in listing[key]["directories"]:
dir_path = os.path.join(key, listing_dir[0])
executer.local_copy(dir_path, dst_path, recursive=True)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def set_remote_uri_times(self, uri, atime, mtime):
scheme, parts = uriparse(uri)
remotepath = parts.path
ssh = None
try:
ssh = pool_manager.borrow(parts.hostname, parts.port,
self.cred.credential)
sftp = ssh.open_sftp()
sftp.utime(remotepath, (atime, mtime))
except Exception:
logger.exception("Exception while setting times for '%s'", uri)
raise
finally:
if ssh is not None:
sftp.close()
pool_manager.give_back(ssh, parts.hostname, parts.port,
self.cred.credential)
def rm(self, uri):
"""rm uri"""
logger.debug('rm {0}'.format(uri))
scheme, parts = uriparse(uri)
logger.debug('{0}'.format(parts.path))
if not os.path.exists(parts.path):
raise Exception(
'rm target ({0}) is not a file or directory'.format(
parts.path))
try:
path = parts.path.rstrip('/')
if os.path.isfile(path) or os.path.islink(path):
os.unlink(path)
elif os.path.isdir(path):
shutil.rmtree(path)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def local_copy_recursive(self, src_uri, dst_uri):
"""recursively copy src_uri to dst_uri on the remote backend"""
logger.debug("SFTPBackend.local_copy_recursive: %s => %s", src_uri,
dst_uri)
dst_scheme, dst_parts = uriparse(dst_uri)
dst_path = dst_parts.path
listing = self.ls(src_uri)
executer = create_executer(self.yabiusername, src_uri)
try:
for key in listing:
for listing_file in listing[key]['files']:
file_path = os.path.join(key, listing_file[0])
executer.local_copy(file_path, dst_path)
for listing_dir in listing[key]['directories']:
dir_path = os.path.join(key, listing_dir[0])
executer.local_copy(dir_path, dst_path, recursive=True)
except Exception as exc:
raise RetryException(exc, traceback.format_exc())
def remote_uri_stat(self, uri):
scheme, parts = uriparse(uri)
remotepath = parts.path
ssh = None
try:
ssh = pool_manager.borrow(parts.hostname, parts.port,
self.cred.credential)
sftp = ssh.open_sftp()
stat = sftp.stat(remotepath)
return {'atime': stat.st_atime, 'mtime': stat.st_mtime}
except Exception:
logger.exception("Exception while stating '%s'", uri)
raise
finally:
if ssh is not None:
sftp.close()
pool_manager.give_back(ssh, parts.hostname, parts.port,
self.cred.credential)
def ls(self, uri):
"""ls at uri"""
self.set_cred(uri)
scheme, parts = uriparse(uri)
ssh = sshclient(parts.hostname, parts.port, self.cred.credential)
try:
sftp = ssh.open_sftp()
results = self._do_ls(sftp, parts.path)
output = {}
output[parts.path] = results
return output
except FileNotFoundError:
return {}
except Exception as exc:
logger.exception("ls: %s" % uri)
raise RetryException(exc, traceback.format_exc())
finally:
try:
if ssh is not None:
ssh.close()
except:
pass
def ls(self, uri):
"""ls at uri"""
self.set_cred(uri)
scheme, parts = uriparse(uri)
ssh = sshclient(parts.hostname, parts.port, self.cred.credential)
try:
sftp = ssh.open_sftp()
results = self._do_ls(sftp, parts.path)
output = {}
output[parts.path] = results
return output
except FileNotFoundError:
return {}
except Exception as exc:
logger.exception("ls: %s" % uri)
raise RetryException(exc, traceback.format_exc())
finally:
try:
if ssh is not None:
ssh.close()
except:
pass
def mkdir(self, uri):
"""mkdir at uri"""
self.set_cred(uri)
scheme, parts = uriparse(uri)
path = parts.path
ssh = sshclient(parts.hostname, parts.port, self.cred.credential)
try:
sftp = ssh.open_sftp()
try:
self._rm(sftp, path)
logger.debug("deleted existing directory %s OK" % path)
except Exception as ex:
logger.debug("could not remove directory %s: %s" % (path, ex))
def full_path(result, d):
previous = result[-1] if result else ""
result.append("%s/%s" % (previous, d))
return result
dirs = [p for p in path.split("/") if p.strip() != '']
dir_full_paths = reduce(full_path, dirs, [])
non_existant_dirs = dropwhile(lambda d: self.path_exists(sftp, d),
dir_full_paths)
for d in non_existant_dirs:
sftp.mkdir(d)
logger.debug("created dir %s OK" % path)
except Exception as exc:
logger.error(exc)
raise RetryException(exc, traceback.format_exc())
finally:
try:
if ssh is not None:
ssh.close()
except:
pass
def mkdir(self, uri):
"""mkdir at uri"""
self.set_cred(uri)
scheme, parts = uriparse(uri)
path = parts.path
ssh = sshclient(parts.hostname, parts.port, self.cred.credential)
try:
sftp = ssh.open_sftp()
try:
self._rm(sftp, path)
logger.debug("deleted existing directory %s OK" % path)
except Exception as ex:
logger.debug("could not remove directory %s: %s" % (path, ex))
def full_path(result, d):
previous = result[-1] if result else ""
result.append("%s/%s" % (previous, d))
return result
dirs = [p for p in path.split("/") if p.strip() != ""]
dir_full_paths = reduce(full_path, dirs, [])
non_existant_dirs = dropwhile(lambda d: self.path_exists(sftp, d), dir_full_paths)
for d in non_existant_dirs:
sftp.mkdir(d)
logger.debug("created dir %s OK" % path)
except Exception as exc:
logger.error(exc)
raise RetryException(exc, traceback.format_exc())
finally:
try:
if ssh is not None:
ssh.close()
except:
pass
def get_exec_backendcredential_for_uri(yabiusername, uri):
"""
Looks up a backend credential based on the supplied uri, which should include a username.
Returns bc, will log and reraise ObjectDoesNotExist and MultipleObjectsReturned exceptions if more than one credential
"""
logger.debug('yabiusername: %s uri: %s' % (yabiusername, uri))
# parse the URI into chunks
schema, rest = uriparse(uri)
# enforce Exec scehmas only
from ..backend import ExecBackend
if not ExecBackend.get_backend_cls_for_scheme(schema):
logger.error(
"get_exec_backendcredential_for_uri was asked to get an fs schema! This is forbidden."
)
raise ValueError(
"Invalid schema in uri passed to get_exec_backendcredential_for_uri: asked for %s"
% schema)
path = rest.path
if path != "/":
logger.error(
"get_exec_backendcredential_for_uri was passed a uri with a path! This is forbidden. Path must be / for exec backends"
)
raise ValueError(
"Invalid path in uri passed to get_exec_backendcredential_for_uri: path passed in was: %s"
% path)
# get our set of credential candidates
bcs = _get_credential_candidates(yabiusername, schema, rest.username,
rest.hostname)
if len(bcs) == 0:
raise ObjectDoesNotExist("Could not find backendcredential")
return bcs[0]
def isdir(self, uri):
"""is the uri a dir?"""
scheme, parts = uriparse(uri)
return os.path.exists(parts.path) and os.path.isdir(parts.path)
def matches_filename(self, filename):
_, parts = uriparse(self.src)
return filename == os.path.basename(parts.path)