def __init__(self, directives_list):
self._in_overwrite = dict()
self._in_append = dict()
self._out_overwrite = dict()
self._out_append = dict()
# each line in directives_list should contain one directive
for directive in directives_list:
if (directive.count('>') > 2) or (directive.count('<') > 2):
msg = "'%s' is not a valid transfer directive string."
raise se.BadParameter(msg)
elif '<<' in directive:
(local, remote) = directive.split('<<')
self._out_append[local.strip()] = remote.strip()
elif '>>' in directive:
(local, remote) = directive.split('>>')
self._in_append[local.strip()] = remote.strip()
elif '<' in directive:
(local, remote) = directive.split('<')
self._out_overwrite[local.strip()] = remote.strip()
elif '>' in directive:
(local, remote) = directive.split('>')
self._in_overwrite[local.strip()] = remote.strip()
else:
msg = "'%s' is not a valid transfer directive string."
raise se.BadParameter(msg)
def __init__(self, directives=None):
self._in_overwrite = list()
self._in_append = list()
self._out_overwrite = list()
self._out_append = list()
if not directives:
directives = {}
for d in directives:
if (d.count('>') > 2) or (d.count('<') > 2):
msg = "'%s' is not a valid transfer d string."
raise se.BadParameter(msg)
elif '>>' in d:
(loc, rem) = d.split('>>')
self._in_append.append([loc.strip(), rem.strip()])
elif '>' in d:
(loc, rem) = d.split('>')
self._in_overwrite.append([loc.strip(), rem.strip()])
elif '<<' in d:
(loc, rem) = d.split('<<')
self._out_append.append([loc.strip(), rem.strip()])
elif '<' in d:
(loc, rem) = d.split('<')
self._out_overwrite.append([loc.strip(), rem.strip()])
else:
msg = "'%s' is not a valid transfer directive string." % d
raise se.BadParameter(msg)
def translate_exception(e, msg=None):
"""
In many cases, we should be able to roughly infer the exception cause
from the error message -- this is centrally done in this method. If
possible, it will return a new exception with a more concise error
message and appropriate exception type.
"""
if not issubclass(e.__class__, se.SagaException):
# we do not touch non-saga exceptions
return e
if not issubclass(e.__class__, se.NoSuccess):
# this seems to have a specific cause already, leave it alone
return e
cmsg = e._plain_message
if msg:
cmsg = "%s (%s)" % (cmsg, msg)
lmsg = cmsg.lower()
if 'could not resolve hostname' in lmsg:
e = se.BadParameter(cmsg)
elif 'connection timed out' in lmsg:
e = se.BadParameter(cmsg)
elif 'connection refused' in lmsg:
e = se.BadParameter(cmsg)
elif 'auth' in lmsg:
e = se.AuthorizationFailed(cmsg)
elif 'pass' in lmsg:
e = se.AuthenticationFailed(cmsg)
elif 'ssh_exchange_identification' in lmsg:
e = se.AuthenticationFailed(
"too frequent login attempts, or sshd misconfiguration: %s" % cmsg)
elif 'denied' in lmsg:
e = se.PermissionDenied(cmsg)
elif 'shared connection' in lmsg:
e = se.NoSuccess("Insufficient system resources: %s" % cmsg)
elif 'pty allocation' in lmsg:
e = se.NoSuccess("Insufficient system resources: %s" % cmsg)
elif 'Connection to master closed' in lmsg:
e = se.NoSuccess(
"Connection failed (insufficient system resources?): %s" % cmsg)
return e
def __init__(self, command, logger=None):
"""
The class constructor, which runs (execvpe) command in a separately
forked process. The bew process will inherit the environment of the
application process.
:type command: string or list of strings
:param command: The given command is what is run as a child, and
fed/drained via pty pipes. If given as string, command is split into an
array of strings, using :func:`shlex.split`.
:type logger: :class:`radical.utils.logger.Logger` instance
:param logger: logger stream to send status messages to.
"""
self.logger = logger
if not self.logger: self.logger = rul.getLogger('saga', 'PTYProcess')
self.logger.debug("PTYProcess init %s" % self)
if isinstance(command, basestring):
command = shlex.split(command)
if not isinstance(command, list):
raise se.BadParameter("PTYProcess expects string or list command")
if len(command) < 1:
raise se.BadParameter("PTYProcess expects non-empty command")
self.rlock = ru.RLock("pty process %s" % command)
self.command = command # list of strings too run()
self.cache = "" # data cache
self.tail = "" # tail of data data cache for error messages
self.child = None # the process as created by subprocess.Popen
self.ptyio = None # the process' io channel, from pty.fork()
self.exit_code = None # child died with code (may be revived)
self.exit_signal = None # child kill by signal (may be revived)
self.recover_max = 3 # TODO: make configure option. This does not
self.recover_attempts = 0 # apply for recovers triggered by gc_timeout!
try:
self.initialize()
except Exception as e:
raise ptye.translate_exception(e, "pty or process creation failed")
def _initialise_context(self, ctx, session=None):
if not isinstance(ctx, saga.Context):
raise TypeError, "item to add is not a saga.Context instance"
# create a deep copy of the context (this keeps _adaptor etc)
ctx_clone = saga.Context(ctx.type)
ctx._attributes_deep_copy(ctx_clone)
if not session:
session = self._session
logger = self._logger
else:
logger = session._logger
# try to initialize that context, i.e. evaluate its attributes and
# infer additional runtime information as needed
# logger.debug ("adding context : %s" % (ctx_clone))
if not session:
logger.warning ("cannot initialize context - no session: %s" \
% (ctx_clone))
else:
try:
ctx_clone._initialize(session)
except se.SagaException as e:
msg = "Cannot add context, initialization failed (%s)" % str(e)
raise se.BadParameter(msg)
return ctx_clone
def acquire (self, spec, ttype=None) :
"""
acquire(desc)
Create a new :class:`saga.resource.Resource` handle for a
resource specified by the description.
:type spec: :class:`Description`
:param spec: specifies the resource
Depending on the `RTYPE` attribute in the description, the
returned resource may be a :class:`saga.resource.Compute`,
:class:`saga.resource.Storage` or :class:`saga.resource.Network`
instance. The returned resource will be in NEW, PENDING or ACTIVE
state.
"""
if isinstance (spec, surl.Url) or \
isinstance (spec, basestring) :
id = surl.Url (spec)
return self._adaptor.acquire_by_id (id, ttype=ttype)
else :
# make sure at least 'executable' is defined
if spec.rtype is None:
raise se.BadParameter ("No resource type defined in resource description")
spec_copy = descr.Description ()
spec._attributes_deep_copy (spec_copy)
return self._adaptor.acquire (spec_copy, ttype=ttype)
def wrap_function(self, *args, **kwargs):
if 'ttype' in kwargs and kwargs['ttype'] != None:
if not kwargs['ttype'] in (st.SYNC, st.ASYNC, st.TASK):
# cannot handle that ttype value, do not call async methods
ttype = kwargs['ttype']
msg = " %s: async %s() called with invalid tasktype (%s)" \
% (self.__class__.__name__, sync_function.__name__, str(ttype))
raise se.BadParameter(msg)
# call async method flavor
try:
async_function_name = "%s_async" % sync_function.__name__
async_function = getattr(self, async_function_name)
except AttributeError:
msg = " %s: async %s() not implemented" \
% (self.__class__.__name__, sync_function.__name__)
raise se.NotImplemented(msg)
else:
# 'self' not needed, getattr() returns member function
return async_function(*args, **kwargs)
# no ttype, or ttype==None -- make sure it's gone, and call default sync
# function
if 'ttype' in kwargs:
del kwargs['ttype']
return sync_function(self, *args, **kwargs)
def raise_type_exception(method, arg0, i, arg, kwname=""):
narg = 0
if arg0 and isinstance(arg0, object):
narg = i
else:
narg = i + 1
stack = extract_stack()
for f in stack:
if 'saga/utils/signatures.py' in f[0]:
break
frame = f
msg = "\nSignature Mismatch\n"
msg += " in function : %s\n" % (frame[2])
msg += " in file : %s +%s\n" % (frame[0], frame[1])
msg += " on line : %s\n" % (frame[3])
msg += " method : %s\n" % (method.__name__)
if not kwname:
msg += " argument : #%s\n" % (narg)
else:
msg += " parameter : %s" % (kwname)
msg += " has incorrect type : %s" % (type_name(arg))
raise se.BadParameter(msg)
def run_job (self, cmd, host=None, ttype=None) :
"""
run_job(cmd, host=None)
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
if not cmd:
raise se.BadParameter('run_job needs a command to run. Duh!')
try:
# lets see if the adaptor implements run_job
return self._adaptor.run_job (cmd, host, ttype=ttype)
except:
# fall back to the default implementation below
pass
# The adaptor has no run_job -- we here provide a generic implementation
# FIXME: split should be more clever and respect POSIX shell syntax.
args = cmd.split()
jd = descr.Description()
jd.executable = args[0]
jd.arguments = args[1:]
job = self.create_job(jd)
job.run()
return job
def __init__ (self, id=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
self._resrc = super (Network, self)
self._resrc.__init__ (id, session, _adaptor, _adaptor_state, _ttype)
if self.rtype != const.NETWORK :
raise se.BadParameter ("Cannot initialize Network resource with %s id" \
% self.rtype)
def __init__ (self, id=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
self._resrc = super (Storage, self)
self._resrc.__init__ (id, session, _adaptor, _adaptor_state, _ttype)
if self.rtype != const.STORAGE :
raise se.BadParameter ("Cannot initialize Storage resource with %s id" \
% self.rtype)
def wrap_function (self, *args, **kwargs) :
my_ttype = None
my_call = None
my_args = ()
if not 'ttype' in kwargs :
msg = " %s: async %s() called with no tasktype" \
% (self.__class__.__name__, cpi_async_function.__name__)
raise se.BadParameter (msg)
ttype = kwargs['ttype']
del kwargs['ttype']
if not ttype in (st.SYNC, st.ASYNC, st.TASK) :
# cannot handle that ttype value, do not call async methods
msg = " %s: async %s() called with invalid tasktype (%s)" \
% (self.__class__.__name__, cpi_async_function.__name__, str(ttype))
raise se.BadParameter (msg)
cpi_sync_function_name = None
cpi_sync_function = None
# find sync method flavor
try :
cpi_sync_function_name = re.sub ("_async$", "", cpi_async_function.__name__)
cpi_sync_function = getattr (self, cpi_sync_function_name)
except AttributeError :
msg = " %s: sync %s() not implemented" \
% (self.__class__.__name__, cpi_sync_function.__name__)
raise se.NotImplemented (msg)
# got the sync call, wrap it in a task
c = { '_call' : cpi_sync_function,
'_args' : args,
'_kwargs' : kwargs } # no ttype!
return st.Task (self, cpi_sync_function_name, c, ttype)
def __init__(self, d=None):
if d:
if const.RTYPE in d and d[const.RTYPE] != const.STORAGE:
raise se.BadParameter ("Cannot create StorageResource with type '%s'" \
% d[const.RTYPE])
self._descr = super(StorageDescription, self)
self._descr.__init__(d)
self.rtype = const.STORAGE
def __init__(self, d=None):
if d:
if const.RTYPE in d and d[const.RTYPE] != const.NETWORK:
raise se.BadParameter ("Cannot create NetworkResource with type '%s'" \
% d[const.RTYPE])
self._descr = super(NetworkDescription, self)
self._descr.__init__()
self.rtype = const.NETWORK
def list (self, pattern, flags) :
if pattern :
raise se.BadParameter ("pattern for list() not supported")
ret = []
if not flags :
ret = self._nsdir.list ()
elif flags == saga.advert.RECURSIVE :
# ------------------------------------------------------------------
def get_kids (path) :
d = saga.advert.Directory (path)
kids = d.list ()
for kid in kids :
kid_url = self._url
kid_url.path = kid
if d.is_dir (kid_url) :
get_kids (kid_url)
ret.append (kid)
# ------------------------------------------------------------------
get_kids (self._url)
else :
raise se.BadParameter ("list() only supports the RECURSIVE flag")
return ret
def __init__(self, call, *args, **kwargs):
if not callable(call):
raise se.BadParameter ("Thread requires a callable to function, not %s" \
% (str(call)))
Thread.__init__(self)
self._call = call
self._args = args
self._kwargs = kwargs
self._state = NEW
self._result = None
self._exception = None
self.daemon = True
def change_dir(self, tgt):
# backup state
orig_url = self._url
try:
if not sumisc.url_is_compatible(tgt, self._url):
raise se.BadParameter("cannot chdir to %s, leaves namespace" %
tgt)
self._url = sumisc.url_make_absolute(tgt, self._url)
self._init_check()
finally:
# restore state on error
self._url = orig_url
def wrap_function (self, *args, **kwargs) :
if 'ttype' in kwargs and kwargs['ttype'] != None :
if not kwargs['ttype'] in (st.SYNC, st.ASYNC, st.TASK) :
# cannot handle that ttype value, do not call async methods
ttype = kwargs['ttype']
msg = " %s: async %s() called with invalid tasktype (%s)" \
% (self.__class__.__name__, sync_function.__name__, str(ttype))
raise se.BadParameter (msg)
# call async method flavor
try :
async_function_name = "%s_async" % sync_function.__name__
async_function = getattr (self, async_function_name)
except AttributeError :
msg = " %s: async %s() not implemented" \
% (self.__class__.__name__, sync_function.__name__)
raise se.NotImplemented (msg)
else :
# 'self' not needed, getattr() returns member function
return async_function (*args, **kwargs)
# no ttype, or ttype==None -- make sure it's gone, and call default sync
# function
if 'ttype' in kwargs :
del kwargs['ttype']
# only some functions will provide metrics, and thus need the _from_task
# parameter -- strip that as well if its not needed
if '_from_task' in kwargs:
if not '_from_task' in inspect.getargspec (sync_function).args:
del(kwargs['_from_task'])
return sync_function (self, *args, **kwargs)
def container_wait(self, tasks, mode, timeout):
if timeout >= 0:
raise se.BadParameter("Cannot handle timeouts > 0")
for task in tasks:
task.wait()
def __init__ (self, id=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
"""
__init__(id=None, session=None)
Create / reconnect to a resource.
:param id: id of the resource
:type id: :class:`saga.Url`
:param session: :class:`saga.Session`
Resource class instances are usually created by calling :func:`acquire`
on the :class:`saga.resource.Manager` class. Already acquired resources
are identified by a string typed identifier. This constructor accepts
such an identifier to create another representation of the same
resource. As the resource itself is new newly acquired, it can be in
any state. In particular, it can be in a final state, and thus be
unusable. Further, the resource may already have expired or failed, and
the information about it may have been purged -- in that case the id
will not be valid any longer, and a :class:`saga.BadParameter` exception
will be raised.
The session parameter is interpreted exactly as the session parameter on
the :class:`saga.resource.Manager` constructor.
"""
# set attribute interface properties
import saga.attributes as sa
self._attributes_extensible (False)
self._attributes_camelcasing (True)
# register properties with the attribute interface
self._attributes_register (const.ID , None, sa.ENUM, sa.SCALAR, sa.READONLY)
self._attributes_register (const.RTYPE , None, sa.ENUM, sa.SCALAR, sa.READONLY)
self._attributes_register (const.STATE , None, sa.ENUM, sa.SCALAR, sa.READONLY)
self._attributes_register (const.STATE_DETAIL, None, sa.STRING, sa.SCALAR, sa.READONLY)
self._attributes_register (const.ACCESS , None, sa.URL, sa.SCALAR, sa.READONLY)
self._attributes_register (const.MANAGER , None, sa.URL, sa.SCALAR, sa.READONLY)
self._attributes_register (const.DESCRIPTION , None, sa.ANY, sa.SCALAR, sa.READONLY)
self._attributes_set_enums (const.STATE, [const.UNKNOWN ,
const.PENDING ,
const.ACTIVE ,
const.CANCELED,
const.EXPIRED ,
const.FAILED ,
const.FINAL ])
self._attributes_set_enums (const.RTYPE, [const.COMPUTE ,
const.STORAGE ,
const.NETWORK ])
self._attributes_set_getter (const.ID , self.get_id )
self._attributes_set_getter (const.RTYPE , self.get_rtype )
self._attributes_set_getter (const.STATE , self.get_state )
self._attributes_set_getter (const.STATE_DETAIL, self.get_state_detail)
self._attributes_set_getter (const.ACCESS , self.get_access )
self._attributes_set_getter (const.MANAGER , self.get_manager )
self._attributes_set_getter (const.DESCRIPTION , self.get_description )
# FIXME: we need the ID to be or to include an URL, as we don't have
# a scheme otherwise, which means we can't select an adaptor. Duh! :-/
# FIXME: documentation for attributes is missing.
# param checks
scheme = None
if not id :
if not 'resource_schema' in _adaptor_state :
raise se.BadParameter ("Cannot initialize resource without id" \
% self.rtype)
else :
scheme = _adaptor_state['resource_schema']
else :
url = surl.Url (id)
scheme = url.scheme.lower ()
if not session :
session = ss.Session (default=True)
self._base = super (Resource, self)
self._base.__init__ (scheme, _adaptor, _adaptor_state,
id, session, ttype=_ttype)
def _initialize(self, session):
# make sure we have can access the key
api = self.get_api()
key = None
pub = None
pwd = None
if api.attribute_exists(saga.context.USER_KEY):
key = api.get_attribute(saga.context.USER_KEY)
if api.attribute_exists(saga.context.USER_CERT):
pub = api.get_attribute(saga.context.USER_CERT)
if api.attribute_exists(saga.context.USER_PASS):
pwd = api.get_attribute(saga.context.USER_PASS)
# either user_key or user_cert should be specified (or both),
# then we complement the other, and convert to/from private
# from/to public keys
if pub and not key:
key = pub
if not key:
# nothing to do, really. This likely means that ssh setup is
# done out-of-band.
return
# convert public key into private key
if key.endswith('.pub'):
if not pub:
pub = key
key = key[:-4]
elif key.endswith('.pem'):
if not pub:
pub = key
else:
if not pub:
pub = key + '.pub'
# update the context with these setting
api.set_attribute(saga.context.USER_KEY, key)
api.set_attribute(saga.context.USER_CERT, pub)
# the private and public keys must exist
if not os.path.exists (key) or \
not os.path.isfile (key) :
raise se.BadParameter("ssh key inaccessible: %s" % (key))
if not os.path.exists (pub) or \
not os.path.isfile (pub) :
raise se.BadParameter("ssh public key inaccessible: %s" % (pub))
try:
fh_key = open(key)
except Exception as e:
raise se.PermissionDenied("ssh key '%s' not readable: %s" %
(key, e))
else:
fh_key.close()
try:
fh_pub = open(pub)
except Exception as e:
raise se.PermissionDenied("ssh public key '%s' not readable: %s" %
(pub, e))
else:
fh_pub.close()
import subprocess
if not subprocess.call(
["sh", "-c", "grep ENCRYPTED %s > /dev/null" % key]):
if pwd:
if subprocess.call([
"sh", "-c",
"ssh-keygen -y -f %s -P %s > /dev/null" % (key, pwd)
]):
raise se.PermissionDenied(
"ssh key '%s' is encrypted, incorrect password" %
(key))
else:
self._logger.error(
"ssh key '%s' is encrypted, unknown password" % (key))
self._logger.info("init SSH context for key at '%s' done" % key)
def set_prompt (self, new_prompt) :
"""
:type new_prompt: string
:param new_prompt: a regular expression matching the shell prompt
The new_prompt regex is expected to be a regular expression with one set
of catching brackets, which MUST return the previous command's exit
status. This method will send a newline to the client, and expects to
find the prompt with the exit value '0'.
As a side effect, this method will discard all previous data on the pty,
thus effectively flushing the pty output.
By encoding the exit value in the command prompt, we safe one roundtrip.
The prompt on Posix compliant shells can be set, for example, via::
PS1='PROMPT-$?->'; export PS1
The newline in the example above allows to nicely anchor the regular
expression, which would look like::
PROMPT-(\d+)->$
The regex is compiled with 're.DOTALL', so the dot character matches
all characters, including line breaks. Be careful not to match more
than the exact prompt -- otherwise, a prompt search will swallow stdout
data. For example, the following regex::
PROMPT-(.+)->$
would capture arbitrary strings, and would thus match *all* of::
PROMPT-0->ls
data/ info
PROMPT-0->
and thus swallow the ls output...
Note that the string match *before* the prompt regex is non-gready -- if
the output contains multiple occurrences of the prompt, only the match
up to the first occurence is returned.
"""
def escape (txt) :
pat = re.compile(r'\x1b[^m]*m')
return pat.sub ('', txt)
with self.pty_shell.rlock :
old_prompt = self.prompt
self.prompt = new_prompt
self.prompt_re = re.compile ("^(.*?)%s\s*$" % self.prompt, re.DOTALL)
retries = 0
triggers = 0
while True :
try :
# make sure we have a non-zero waiting delay (default to
# 1 second)
delay = 10 * self.latency
if not delay :
delay = 1.0
# FIXME: how do we know that _PTY_TIMOUT suffices? In particular if
# we actually need to flush...
fret, match = self.pty_shell.find ([self.prompt], delay)
if fret == None :
retries += 1
if retries > 10 :
self.prompt = old_prompt
raise se.BadParameter ("Cannot use new prompt, parsing failed (10 retries)")
self.pty_shell.write ("\n")
self.logger.debug ("sent prompt trigger again (%d)" % retries)
triggers += 1
continue
# found a match -- lets see if this is working now...
ret, _ = self._eval_prompt (match)
if ret != 0 :
self.prompt = old_prompt
raise se.BadParameter ("could not parse exit value (%s)" \
% match)
# prompt looks valid...
break
except Exception as e :
self.prompt = old_prompt
raise ptye.translate_exception (e, "Could not set shell prompt")
# got a valid prompt -- but we have to sync the output again in
# those cases where we had to use triggers to actually get the
# prompt
if triggers > 0 :
self.run_async (' printf "SYNCHRONIZE_PROMPT\n"')
# FIXME: better timout value?
fret, match = self.pty_shell.find (["SYNCHRONIZE_PROMPT"], timeout=10.0)
if fret == None :
# not find prompt after blocking? BAD! Restart the shell
self.finalize (kill_pty=True)
raise se.NoSuccess ("Could not synchronize prompt detection")
self.find_prompt ()
def create_job (self, job_desc, ttype=None) :
"""
create_job(job_desc)
Create a new job.Job instance from a :class:`~saga.job.Description`. The
resulting job instance is in :data:`~saga.job.NEW` state.
:param job_desc: job description to create the job from
:type job_desc: :data:`saga.job.Description`
:param ttype: |param_ttype|
:rtype: :class:`saga.job.Job` or |rtype_ttype|
create_job() accepts a job description, which described the
application instance to be created by the backend. The create_job()
method is not actually attempting to *run* the job, but merely parses
the job description for syntactic and semantic consistency. The job
returned object is thus not in 'Pending' or 'Running', but rather in
'New' state. The actual submission is performed by calling run() on
the job object.
Example::
# A job.Description object describes the executable/application and its requirements
job_desc = saga.job.Description()
job_desc.executable = '/bin/sleep'
job_desc.arguments = ['10']
job_desc.output = 'myjob.out'
job_desc.error = 'myjob.err'
service = saga.job.Service('local://localhost')
job = service.create_job(job_desc)
# Run the job and wait for it to finish
job.run()
print "Job ID : %s" % (job.job_id)
job.wait()
# Get some info about the job
print "Job State : %s" % (job.state)
print "Exitcode : %s" % (job.exit_code)
service.close()
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
jd_copy = descr.Description()
job_desc._attributes_deep_copy (jd_copy)
# do some sanity checks: if the adaptor has specified a set of supported
# job description attributes, we scan the given description for any
# mismatches, and complain then.
adaptor_info = self._adaptor._adaptor.get_info ()
if 'capabilities' in adaptor_info and \
'jdes_attributes' in adaptor_info['capabilities'] :
# this is the list of key supported by the adaptor. These
# attributes may be set to non-default values
supported_keys = adaptor_info['capabilities']['jdes_attributes']
# use an empty job description to compare default values
jd_default = descr.Description ()
for key in jd_copy.list_attributes () :
val = jd_copy .get_attribute (key)
default = jd_default.get_attribute (key)
# Also, we make string compares case insensitive
if isinstance (val, basestring) : val = val .lower ()
if isinstance (default, basestring) : default = default.lower ()
# supported keys are also valid, as are keys with default or
# None values
if key not in supported_keys and \
val != default and \
val :
msg = "'JobDescription.%s' (%s) is not supported by adaptor %s" \
% (key, val, adaptor_info['name'])
raise se.BadParameter._log (self._logger, msg)
# make sure at least 'executable' is defined
if jd_copy.executable is None:
raise se.BadParameter("No executable defined")
# convert environment to string
if jd_copy.attribute_exists ('Environment') :
for (key, value) in jd_copy.environment.iteritems():
jd_copy.environment[key] = str(value)
return self._adaptor.create_job (jd_copy, ttype=ttype)
def run_sync (self, command, iomode=None, new_prompt=None) :
"""
Run a shell command, and report exit code, stdout and stderr (all three
will be returned in a tuple). The call will block until the command
finishes (more exactly, until we find the prompt again on the shell's
I/O stream), and cannot be interrupted.
:type command: string
:param command: shell command to run.
:type iomode: enum
:param iomode: Defines how stdout and stderr are captured.
:type new_prompt: string
:param new_prompt: regular expression matching the prompt after
command succeeded.
We expect the ``command`` to not to do stdio redirection, as this is we want
to capture that separately. We *do* allow pipes and stdin/stdout
redirection. Note that SEPARATE mode will break if the job is run in
the background
The following iomode values are valid:
* *IGNORE:* both stdout and stderr are discarded, `None` will be
returned for each.
* *MERGED:* both streams will be merged and returned as stdout;
stderr will be `None`. This is the default.
* *SEPARATE:* stdout and stderr will be captured separately, and
returned individually. Note that this will require
at least one more network hop!
* *STDOUT:* only stdout is captured, stderr will be `None`.
* *STDERR:* only stderr is captured, stdout will be `None`.
* *None:* do not perform any redirection -- this is effectively
the same as `MERGED`
If any of the requested output streams does not return any data, an
empty string is returned.
If the command to be run changes the prompt to be expected for the
shell, the ``new_prompt`` parameter MUST contain a regex to match the
new prompt. The same conventions as for set_prompt() hold -- i.e. we
expect the prompt regex to capture the exit status of the process.
"""
with self.pty_shell.rlock :
self._trace ("run sync : %s" % command)
self.pty_shell.flush ()
# we expect the shell to be in 'ground state' when running a syncronous
# command -- thus we can check if the shell is alive before doing so,
# and restart if needed
if not self.pty_shell.alive (recover=True) :
raise se.IncorrectState ("Can't run command -- shell died:\n%s" \
% self.pty_shell.autopsy ())
try :
command = command.strip ()
if command.endswith ('&') :
raise se.BadParameter ("run_sync can only run foreground jobs ('%s')" \
% command)
redir = ""
_err = "/tmp/saga-python.ssh-job.stderr.$$"
if iomode == IGNORE :
redir = " 1>>/dev/null 2>>/dev/null"
if iomode == MERGED :
redir = " 2>&1"
if iomode == SEPARATE :
redir = " 2>%s" % _err
if iomode == STDOUT :
redir = " 2>/dev/null"
if iomode == STDERR :
redir = " 2>&1 1>/dev/null"
if iomode == None :
redir = ""
self.logger.debug ('run_sync: %s%s' % (command, redir))
self.pty_shell.write ( "%s%s\n" % (command, redir))
# If given, switch to new prompt pattern right now...
prompt = self.prompt
if new_prompt :
prompt = new_prompt
# command has been started - now find prompt again.
fret, match = self.pty_shell.find ([prompt], timeout=-1.0) # blocks
if fret == None :
# not find prompt after blocking? BAD! Restart the shell
self.finalize (kill_pty=True)
raise se.IncorrectState ("run_sync failed, no prompt (%s)" % command)
ret, txt = self._eval_prompt (match, new_prompt)
stdout = None
stderr = None
if iomode == None :
iomode = STDOUT
if iomode == IGNORE :
pass
if iomode == MERGED :
stdout = txt
if iomode == STDOUT :
stdout = txt
if iomode == SEPARATE or \
iomode == STDERR :
stdout = txt
self.pty_shell.write (" cat %s\n" % _err)
fret, match = self.pty_shell.find ([self.prompt], timeout=-1.0) # blocks
if fret == None :
# not find prompt after blocking? BAD! Restart the shell
self.finalize (kill_pty=True)
raise se.IncorrectState ("run_sync failed, no prompt (%s)" \
% command)
_ret, _stderr = self._eval_prompt (match)
if _ret :
raise se.IncorrectState ("run_sync failed, no stderr (%s: %s)" \
% (_ret, _stderr))
stderr = _stderr
if iomode == STDERR :
# got stderr in branch above
stdout = None
return (ret, stdout, stderr)
except Exception as e :
raise ptye.translate_exception (e)