class Job(sb.Base, st.Task, sasync.Async):
'''Represents a SAGA job as defined in GFD.90
A 'Job' represents a running application instance, which may consist of one
or more processes. Jobs are created by submitting a Job description to
a Job submission system -- usually a queuing system, or some other service
which spawns jobs on the user's behalf.
Jobs have a unique ID (see get_job_id()), and are stateful entities -- their
'state' attribute changes according to a well defined state model:
A job as returned by job.Service.create(jd) is in 'New' state -- it is not
yet submitted to the job submission backend. Once it was submitted, via
run(), it will enter the 'Pending' state, where it waits to get actually
executed by the backend (e.g. waiting in a queue etc). Once the job is
actually executed, it enters the 'Running' state -- only in that state is
the job actually consuming resources (CPU, memory, ...).
Jobs can leave the 'Running' state in three different ways: they finish
successfully on their own ('Done'), they finish unsuccessfully on their own,
or get canceled by the job management backend ('Failed'), or they get
actively canceled by the user or the application ('Canceled').
The methods defined on the Job object serve two purposes: inspecting the
job's state, and initiating job state transitions.
'''
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(basestring), rus.optional(sab.Base),
rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
_method_type='run',
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
_adaptor`` references the adaptor class instance which created this task
instance.
The ``_method_type`` parameter is flattened into the job constructor to
satisfy the bulk optimization properties of the saga.Task class, whose
interface is implemented by saga.job.Job.
``_method_type`` specifies the SAGA API method which task is
representing. For jobs, that is the 'run' method.
We don't have a create classmethod -- jobs are never constructed by the user
'''
if not _adaptor:
raise se.IncorrectState("saga.job.Job constructor is private")
self._valid = False
# we need to keep _method_type around, for the task interface (see
# :class:`saga.Task`)
self._method_type = _method_type
# We need to specify a schema for adaptor selection -- and
# simply choose the first one the adaptor offers.
schema = _adaptor.get_schemas()[0]
if 'job_schema' in _adaptor_state:
schema = _adaptor_state['job_schema']
self._base = super(Job, self)
self._base.__init__(schema, _adaptor, _adaptor_state, ttype=None)
# set attribute interface properties
self._attributes_allow_private(True)
self._attributes_extensible(False)
self._attributes_camelcasing(True)
# register properties with the attribute interface
self._attributes_register(STATE, UNKNOWN, sa.ENUM, sa.SCALAR,
sa.READONLY)
self._attributes_register(EXIT_CODE, None, sa.INT, sa.SCALAR,
sa.READONLY)
self._attributes_register(CREATED, None, sa.INT, sa.SCALAR,
sa.READONLY)
self._attributes_register(STARTED, None, sa.INT, sa.SCALAR,
sa.READONLY)
self._attributes_register(FINISHED, None, sa.INT, sa.SCALAR,
sa.READONLY)
self._attributes_register(EXECUTION_HOSTS, None, sa.STRING, sa.VECTOR,
sa.READONLY)
self._attributes_register(ID, None, sa.STRING, sa.SCALAR, sa.READONLY)
self._attributes_register(SERVICE_URL, None, sa.URL, sa.SCALAR,
sa.READONLY)
self._attributes_set_enums(STATE, [
UNKNOWN, NEW, PENDING, RUNNING, DONE, FAILED, CANCELED, SUSPENDED
])
self._attributes_set_getter(STATE, self.get_state)
self._attributes_set_getter(ID, self.get_id)
self._attributes_set_getter(EXIT_CODE, self._get_exit_code)
self._attributes_set_getter(CREATED, self._get_created)
self._attributes_set_getter(STARTED, self._get_started)
self._attributes_set_getter(FINISHED, self._get_finished)
self._attributes_set_getter(EXECUTION_HOSTS, self._get_execution_hosts)
self._attributes_set_getter(SERVICE_URL, self._get_service_url)
self._valid = True
# --------------------------------------------------------------------------
#
@rus.takes('Job')
@rus.returns(basestring)
def __str__(self):
"""
__str__()
String representation. Returns the job's ID.
"""
if not self._valid:
return 'no job id'
return str(self.id)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, basestring, st.Task))
def get_id(self, ttype=None):
"""
get_id()
Return the job ID.
"""
id = self._adaptor.get_id(ttype=ttype)
return id
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def get_description(self, ttype=None):
"""
get_description()
Return the job description this job was created from.
The returned description can be used to inspect job properties
(executable name, arguments, etc.). It can also be used to start
identical job instances.
The returned job description will in general reflect the actual state of
the running job, and is not necessarily a simple copy of the job
description which was used to create the job instance. For example, the
environment variables in the returned job description may reflect the
actual environment of the running job instance.
**Example**::
service = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j1 = service.create_job(jd)
j1.run()
j2 = service.create_job(j1.get_description())
j2.run()
service.close()
"""
return self._adaptor.get_description(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((file, st.Task))
def get_stdin(self, ttype=None):
"""
get_stdin()
Return the job's STDIN handle.
"""
return self._adaptor.get_stdin(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((file, st.Task))
def get_stdout(self, ttype=None):
"""
get_stdout()
Return the job's STDOUT handle.
"""
return self._adaptor.get_stdout(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((str, st.Task))
def get_stdout_string(self, ttype=None):
"""
get_stdout_string()
Return the job's STDOUT.
"""
return self._adaptor.get_stdout_string(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((file, st.Task))
def get_stderr(self, ttype=None):
"""
get_stderr()
Return the job's STDERR handle.
ttype: saga.task.type enum
ret: File / saga.Task
"""
return self._adaptor.get_stderr(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((str, st.Task))
def get_stderr_string(self, ttype=None):
"""
get_stderr_string()
Return the job's STDERR.
ttype: saga.task.type enum
ret: string.
"""
return self._adaptor.get_stderr_string(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def suspend(self, ttype=None):
"""
suspend()
Suspend the job.
"""
return self._adaptor.suspend(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def resume(self, ttype=None):
"""
resume()
Resume the job.
"""
return self._adaptor.resume(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def checkpoint(self, ttype=None):
"""
checkpoint()
Checkpoint the job.
"""
return self._adaptor.checkpoint(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', descr.Description,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def migrate(self, jd, ttype=None):
"""
jd: saga.job.Description
ttype: saga.task.type enum
ret: None / saga.Task
"""
return self._adaptor.migrate(jd, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', int, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def signal(self, signum, ttype=None):
"""
signal(signum)
Send a signal to the job.
:param signum: signal to send
:type signum: int
"""
return self._adaptor.signal(signum, ttype=ttype)
id = property(get_id) # string
description = property(get_description) # Description
#stdin = property (get_stdin) # File
stdout = property(get_stdout) # File
stderr = property(get_stderr) # File
#-----------------------------------------------------------------
#
# task methods flattened into job :-/
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def run(self, ttype=None):
"""
run()
Run (start) the job.
Request that the job is being executed by the backend. If the backend
is accepting this run request, the job will move to the 'Pending' or
'Running' state -- otherwise this method will raise an error, and the
job will be moved to 'Failed'.
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
if j.get_state() == saga.job.NEW :
print "new"
else :
print "oops!"
j.run()
if j.get_state() == saga.job.PENDING :
print "pending"
elif j.get_state() == saga.job.RUNNING :
print "running"
else :
print "oops!"
"""
return self._adaptor.run(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', float, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def cancel(self, timeout=None, ttype=None):
"""
cancel(timeout)
Cancel the execution of the job.
:param timeout: `cancel` will return after timeout
:type timeout: float
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
if j.get_state() == saga.job.NEW :
print "new"
else :
print "oops!"
j.run()
if j.get_state() == saga.job.PENDING :
print "pending"
elif j.get_state() == saga.job.RUNNING :
print "running"
else :
print "oops!"
j.cancel()
if j.get_state() == saga.job.CANCELED :
print "canceled"
else :
print "oops!"
"""
return self._adaptor.cancel(timeout, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(float),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def wait(self, timeout=None, ttype=None):
"""
wait(timeout)
:param timeout: `wait` will return after timeout
:type timeout: float
Wait for a running job to finish execution.
The optional timeout parameter specifies the time to wait, and accepts
the following values::
timeout < 0 : wait forever (block) -- same for 'None'
timeout == 0 : wait not at all (non-blocking test)
timeout > 0 : wait for 'timeout' seconds
On a non-negative timeout, the call can thus return even if the job is
not in final state, and the application should check the actual job
state. The default timeout value is 'None' (blocking).
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
if j.get_state() == saga.job.NEW :
print "new"
else :
print "oops!"
j.run()
if j.get_state() == saga.job.PENDING :
print "pending"
elif j.get_state() == saga.job.RUNNING :
print "running"
else :
print "oops!"
j.wait(-1.0)
if j.get_state() == saga.job.DONE :
print "done"
elif j.get_state() == saga.job.FAILED :
print "failed"
else :
print "oops!"
"""
if None == timeout:
timeout = -1.0 # FIXME
return self._adaptor.wait(timeout, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.one_of(UNKNOWN, NEW, PENDING, RUNNING, SUSPENDED, DONE,
FAILED, CANCELED), st.Task))
def get_state(self, ttype=None):
"""
get_state()
Return the current state of the job.
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
if j.get_state() == saga.job.NEW :
print "new"
else :
print "oops!"
j.run()
if j.get_state() == saga.job.PENDING :
print "pending"
elif j.get_state() == saga.job.RUNNING :
print "running"
else :
print "oops!"
"""
return self._adaptor.get_state(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.anything, st.Task))
def get_result(self, ttype=None):
"""
get_result()
"""
return self._adaptor.get_result(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((sb.Base, st.Task))
def get_object(self, ttype=None):
""" :todo: describe me
:note: this will return the job_service which created the job.
"""
return self._adaptor.get_object(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((se.SagaException, st.Task))
def get_exception(self, ttype=None):
""" :todo: describe me
:note: if job failed, that will get an exception describing
why, if that exists. Otherwise, the call returns None.
"""
# FIXME: add CPI
return self._adaptor.get_exception(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job')
@rus.returns(rus.nothing)
def re_raise(self):
""" :todo: describe me
:note: if job failed, that will re-raise an exception describing
why, if that exists. Otherwise, the call does nothing.
"""
self._adaptor.re_raise()
# ----------------------------------------------------------------
#
# attribute getters
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, int, st.Task))
def _get_exit_code(self, ttype=None):
ec = self._adaptor.get_exit_code(ttype=ttype)
if ec in [None, ""]:
return None
else:
# Exit code is always an int. If this 'cast' fails,
# the adaptor is doing something stupid.
return ec
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, float, st.Task))
def _get_created(self, ttype=None):
return self._adaptor.get_created(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, float, st.Task))
def _get_started(self, ttype=None):
return self._adaptor.get_started(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, float, st.Task))
def _get_finished(self, ttype=None):
return self._adaptor.get_finished(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, rus.list_of(basestring), st.Task))
def _get_execution_hosts(self, ttype=None):
return self._adaptor.get_execution_hosts(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, surl.Url, st.Task))
def _get_service_url(self, ttype=None):
return self._adaptor.get_service_url(ttype=ttype)
state = property(get_state) # state enum
result = property(get_result) # result type (None)
object = property(get_object) # object type (job_service)
exception = property(re_raise) # exception type
# ----------------------------------------------------------------
#
ExitCode = property(doc="""
The job's exitcode.
this attribute is only meaningful if the job is in 'Done' or 'Final'
state - for all other job states, this attribute value is undefined.
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
j.run()
j.wait()
if j.get_state() == saga.job.FAILED :
if j.exitcode == "42" :
print "Ah, galaxy bypass error!"
else :
print "oops!"
""")
# ----------------------------------------------------------------
#
JobID = property(doc="""
The job's identifier.
This attribute is equivalent to the value returned by job.get_job_id()
""")
# ----------------------------------------------------------------
#
ServiceURL = property(doc="""
The URL of the :class:`saga.job.Service` instance managing this job.
This attribute is represents the URL under where the job management
service can be contacted which owns the job. The value is equivalent to
the service part of the job_id.
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
if j.serviceurl == "fork://localhost" :
print "yes!"
else :
print "oops!"
""")
class Directory(nsdir.Directory, sa.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, str)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=READ,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
url: saga.Url
flags: flags enum
session: saga.Session
ret: obj
'''
# param checks
if not flags: flags = 0
url = ru.Url(url)
self._nsdirec = super(Directory, self)
self._nsdirec.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# set attribute interface properties
self._attributes_allow_private(True)
self._attributes_camelcasing(True)
self._attributes_extensible(True,
getter=self._attribute_getter,
setter=self._attribute_setter,
lister=self._attribute_lister,
caller=self._attribute_caller)
# register properties with the attribute interface
self._attributes_register(ATTRIBUTE, None, sa.STRING, sa.SCALAR,
sa.READONLY)
self._attributes_register(CHANGE, None, sa.STRING, sa.SCALAR,
sa.READONLY)
self._attributes_register(NEW, None, sa.STRING, sa.SCALAR, sa.READONLY)
self._attributes_register(DELETE, None, sa.STRING, sa.SCALAR,
sa.READONLY)
self._attributes_register(TTL, None, sa.INT, sa.SCALAR, sa.WRITEABLE)
self._attributes_set_setter(TTL, self.set_ttl)
self._attributes_set_getter(TTL, self.get_ttl)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('Directory', rus.optional((ru.Url, str)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=READ, session=None, ttype=None):
'''
url: saga.Url
flags: saga.advert.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
if not flags: flags = 0
_nsdir = super(Directory, cls)
return _nsdir.create(url, flags, session, ttype=ttype)
# --------------------------------------------------------------------------
#
# attribute methods
#
# NOTE: we do not yet pass ttype, as async calls are not yet supported by
# the attribute interface
#
@rus.takes('Directory', str, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.anything, st.Task))
def _attribute_getter(self, key, ttype=None):
return self._adaptor.attribute_getter(key)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', str, rus.anything,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def _attribute_setter(self, key, val, ttype=None):
return self._adaptor.attribute_setter(key, val)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(rus.anything), st.Task))
def _attribute_lister(self, ttype=None):
return self._adaptor.attribute_lister()
# --------------------------------------------------------------------------
#
@rus.takes('Directory', str, int, callable,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.anything, st.Task))
def _attribute_caller(self, key, id, cb, ttype=None):
return self._adaptor.attribute_caller(key, id, cb)
# ----------------------------------------------------------------
#
# advert methods
#
@rus.takes('Directory', (ru.Url, str), float,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def set_ttl(self, tgt=None, ttl=-1.0, ttype=None):
"""
tgt : saga.Url / None
ttl : int
ttype: saga.task.type enum
ret: None / saga.Task
"""
if tgt: return self._adaptor.set_ttl(tgt, ttl, ttype=ttype)
else: return self._adaptor.set_ttl_self(ttl, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, str)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((float, st.Task))
def get_ttl(self, tgt=None, ttype=None):
"""
tgt : saga.Url / None
ttype: saga.task.type enum
ret: int / saga.Task
"""
if tgt: return self._adaptor.get_ttl(tgt, ttype=ttype)
else: return self._adaptor.get_ttl_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional(str), rus.optional(str),
rus.optional((str, object)), rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(ru.Url), st.Task))
def find(self,
name_pattern,
attr_pattern=None,
obj_type=None,
flags=RECURSIVE,
ttype=None):
"""
name_pattern: string
attr_pattern: string
obj_type: string
flags: flags enum
ret: list [saga.Url]
"""
if not flags: flags = 0
if attr_pattern or obj_type:
return self._adaptor.find_adverts(name_pattern,
attr_pattern,
obj_type,
flags,
ttype=ttype)
else:
return self._nsdirec.find(name_pattern, flags, ttype=ttype)
class Message(sa.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('Message', rus.optional(int, rus.nothing)) # size
@rus.returns(rus.nothing)
def __init__(self, size=None):
'''
size: expected size of buffer when used -- informative, not normative
ret: Message
'''
# set attribute interface properties
self._attributes_camelcasing(True)
self._attributes_allow_private(True)
self._attributes_extensible(True,
getter=self._attribute_getter,
setter=self._attribute_setter,
lister=self._attribute_lister,
caller=self._attribute_caller)
# register properties with the attribute interface
self._attributes_register(ID, None, sa.STRING, sa.SCALAR, sa.READONLY)
self._attributes_register(SENDER, None, sa.STRING, sa.SCALAR,
sa.READONLY)
# --------------------------------------------------------------------------
#
# class methods
#
@rus.takes('Message')
@rus.returns(rus.anything)
def get_id(self):
return self._adaptor.get_id()
# --------------------------------------------------------------------------
#
@rus.takes('Message')
@rus.returns(rus.anything)
def get_sender(self):
return self._adaptor.get_sender()
# --------------------------------------------------------------------------
#
# attribute methods
#
@rus.takes('Message', basestring)
@rus.returns(rus.anything)
def _attribute_getter(self, key):
return self._adaptor.attribute_getter(key)
# --------------------------------------------------------------------------
#
@rus.takes('Message', basestring, rus.anything)
@rus.returns(rus.nothing)
def _attribute_setter(self, key, val):
return self._adaptor.attribute_setter(key, val)
# --------------------------------------------------------------------------
#
@rus.takes('Message')
@rus.returns(rus.list_of(rus.anything))
def _attribute_lister(self):
return self._adaptor.attribute_lister()
# --------------------------------------------------------------------------
#
@rus.takes('Message', basestring, int, callable)
@rus.returns(rus.anything)
def _attribute_caller(self, key, id, cb):
return self._adaptor.attribute_caller(key, id, cb)
class Service (sb.Base, sasync.Async) :
"""
The job.Service represents a resource management backend, and as such allows
the creation, submission and management of jobs.
A job.Service represents anything which accepts job creation requests, and
which manages thus created :class:`saga.job.Job` instances. That can be a local shell,
a remote ssh shell, a cluster queuing system, a IaaS backend -- you name it.
The job.Service is identified by an URL, which usually points to the contact
endpoint for that service.
Example::
service = saga.job.Service("fork://localhost")
ids = service.list()
for job_id in ids :
print job_id
j = service.get_job(job_id)
if j.get_state() == saga.job.Job.Pending:
print "pending"
elif j.get_state() == saga.job.Job.Running:
print "running"
else:
print "job is already final!"
service.close()
"""
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
rus.optional ((basestring, surl.Url)),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, rm=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
"""
__init__(rm, session)
Create a new job.Service instance.
:param rm: resource manager URL
:type rm: string or :class:`saga.Url`
:param session: an optional session object with security contexts
:type session: :class:`saga.Session`
:rtype: :class:`saga.job.Service`
"""
# job service instances are resource hogs. Before attempting to create
# a new instance, we attempt to clear out all old instances. There is
# some collateral damage: we cannot run the Python GC over only the
# job.Service instances, but have to run it globally -- however,
# compared to the latency introduced by the job service setup, this
# should be a minor inconvenienve (tm)
try :
import gc
gc.collect ()
except :
pass
# param checks
self.valid = False
url = surl.Url (rm)
if not url.scheme :
url.scheme = 'fork'
if not url.host :
url.host = 'localhost'
if not session :
session = ss.Session (default=True)
scheme = url.scheme.lower ()
self._super = super (Service, self)
self._super.__init__ (scheme, _adaptor, _adaptor_state,
url, session, ttype=_ttype)
self.valid = True
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes ('Service',
rus.optional ((surl.Url, basestring)),
rus.optional (ss.Session),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (st.Task)
def create (cls, rm=None, session=None, ttype=SYNC) :
"""
create(rm=None, session=None)
Create a new job.Service instance asynchronously.
:param rm: resource manager URL
:type rm: string or :class:`saga.Url`
:param session: an optional session object with security contexts
:type session: :class:`saga.Session`
:rtype: :class:`saga.Task`
"""
# param checks
if not session :
session = ss.Session (default=True)
url = surl.Url (rm)
scheme = url.scheme.lower ()
return cls (url, session, _ttype=ttype)._init_task
# --------------------------------------------------------------------------
#
@rus.takes ('Service')
@rus.returns (basestring)
def __str__ (self):
"""
__str__()
String representation. Returns the job service Url.
"""
if self.valid :
return "[%s]" % self.url
return ""
# --------------------------------------------------------------------------
#
@rus.takes ('Service')
@rus.returns (rus.nothing)
def close (self) :
"""
close()
Close the job service instance and disconnect from the (remote)
job service if necessary. Any subsequent calls to a job service
instance after `close()` was called will fail.
Example::
service = saga.job.Service("fork://localhost")
# do something with the 'service' object, create jobs, etc...
service.close()
service.list() # this call will throw an exception
.. warning:: While in principle the job service destructor calls
`close()` automatically when a job service instance goes out of scope,
you **shouldn't rely on it**. Python's garbage collection can be a
bit odd at times, so you should always call `close()` explicitly.
Especially in a **multi-threaded program** this will help to avoid
random errors.
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
self._adaptor.close ()
self.valid = False
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
descr.Description,
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((j.Job, st.Task))
def create_job (self, job_desc, ttype=None) :
"""
create_job(job_desc)
Create a new job.Job instance from a :class:`~saga.job.Description`. The
resulting job instance is in :data:`~saga.job.NEW` state.
:param job_desc: job description to create the job from
:type job_desc: :data:`saga.job.Description`
:param ttype: |param_ttype|
:rtype: :class:`saga.job.Job` or |rtype_ttype|
create_job() accepts a job description, which described the
application instance to be created by the backend. The create_job()
method is not actually attempting to *run* the job, but merely parses
the job description for syntactic and semantic consistency. The job
returned object is thus not in 'Pending' or 'Running', but rather in
'New' state. The actual submission is performed by calling run() on
the job object.
Example::
# A job.Description object describes the executable/application and its requirements
job_desc = saga.job.Description()
job_desc.executable = '/bin/sleep'
job_desc.arguments = ['10']
job_desc.output = 'myjob.out'
job_desc.error = 'myjob.err'
service = saga.job.Service('local://localhost')
job = service.create_job(job_desc)
# Run the job and wait for it to finish
job.run()
print "Job ID : %s" % (job.job_id)
job.wait()
# Get some info about the job
print "Job State : %s" % (job.state)
print "Exitcode : %s" % (job.exit_code)
service.close()
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
jd_copy = descr.Description()
job_desc._attributes_deep_copy (jd_copy)
# do some sanity checks: if the adaptor has specified a set of supported
# job description attributes, we scan the given description for any
# mismatches, and complain then.
adaptor_info = self._adaptor._adaptor.get_info ()
if 'capabilities' in adaptor_info and \
'jdes_attributes' in adaptor_info['capabilities'] :
# this is the list of key supported by the adaptor. These
# attributes may be set to non-default values
supported_keys = adaptor_info['capabilities']['jdes_attributes']
# use an empty job description to compare default values
jd_default = descr.Description ()
for key in jd_copy.list_attributes () :
val = jd_copy .get_attribute (key)
default = jd_default.get_attribute (key)
# Also, we make string compares case insensitive
if isinstance (val, basestring) : val = val .lower ()
if isinstance (default, basestring) : default = default.lower ()
# supported keys are also valid, as are keys with default or
# None values
if key not in supported_keys and \
val != default and \
val :
msg = "'JobDescription.%s' (%s) is not supported by adaptor %s" \
% (key, val, adaptor_info['name'])
raise se.BadParameter._log (self._logger, msg)
# make sure at least 'executable' is defined
if jd_copy.executable is None:
raise se.BadParameter("No executable defined")
# convert environment to string
if jd_copy.attribute_exists ('Environment') :
for (key, value) in jd_copy.environment.iteritems():
jd_copy.environment[key] = str(value)
return self._adaptor.create_job (jd_copy, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
basestring,
rus.optional (basestring),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((j.Job, st.Task))
def run_job (self, cmd, host=None, ttype=None) :
"""
run_job(cmd, host=None)
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
if not cmd:
raise se.BadParameter('run_job needs a command to run. Duh!')
try:
# lets see if the adaptor implements run_job
return self._adaptor.run_job (cmd, host, ttype=ttype)
except:
# fall back to the default implementation below
pass
# The adaptor has no run_job -- we here provide a generic implementation
# FIXME: split should be more clever and respect POSIX shell syntax.
args = cmd.split()
jd = descr.Description()
jd.executable = args[0]
jd.arguments = args[1:]
job = self.create_job(jd)
job.run()
return job
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((rus.list_of (basestring), st.Task))
def list (self, ttype=None) :
"""
list()
Return a list of the jobs that are managed by this Service
instance.
.. seealso::
The :data:`~saga.job.Service.jobs` property and the
:meth:`~saga.job.Service.list` method are semantically
equivalent.
:ttype: |param_ttype|
:rtype: list of :class:`saga.job.Job`
As the job.Service represents a job management backend, list() will
return a list of job IDs for all jobs which are known to the backend,
and which can potentially be accessed and managed by the application.
Example::
service = saga.job.Service("fork://localhost")
ids = service.list()
for job_id in ids :
print job_id
service.close()
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
return self._adaptor.list (ttype=ttype)
jobs = property (list)
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((surl.Url, st.Task))
def get_url (self, ttype=None) :
"""
get_url()
Return the URL this Service instance was created with.
.. seealso::
The :data:`~saga.job.Service.url` property and the
:meth:`~saga.job.Service.get_url` method are semantically
equivalent and only duplicated for convenience.
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
return self._adaptor.get_url (ttype=ttype)
url = property (get_url)
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
basestring,
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((j.Job, st.Task))
def get_job (self, job_id, ttype=None) :
"""
get_job(job_id)
Return the job object for a given job id.
:param job_id: The id of the job to retrieve
:rtype: :class:`saga.job.Job`
Job objects are a local representation of a remote stateful entity.
The job.Service supports to reconnect to those remote entities::
service = saga.job.Service("fork://localhost")
j = service.get_job(my_job_id)
if j.get_state() == saga.job.Job.Pending:
print "pending"
elif j.get_state() == saga.job.Job.Running:
print "running"
else:
print "job is already final!"
service.close()
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
return self._adaptor.get_job (job_id, ttype=ttype)
class Directory(entry.Entry):
'''
Represents a SAGA directory as defined in GFD.90
The saga.namespace.Directory class represents, as the name indicates,
a directory on some (local or remote) namespace. That class offers
a number of operations on that directory, such as listing its contents,
copying entries, or creating subdirectories::
# get a directory handle
dir = saga.namespace.Directory("sftp://localhost/tmp/")
# create a subdir
dir.make_dir ("data/")
# list contents of the directory
entries = dir.list ()
# copy *.dat entries into the subdir
for f in entries :
if f ^ '^.*\.dat$' :
dir.copy (f, "sftp://localhost/tmp/data/")
Implementation note:
^^^^^^^^^^^^^^^^^^^^
The SAGA API Specification (GFD.90) prescribes method overloading on method
signatures, but that is not supported by Python (Python only does method
overwriting). So we implement one generic method version here, and do the
case switching based on the provided parameter set.
'''
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=None,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
:param url: Url of the (remote) entry system directory.
:type url: :class:`saga.Url`
flags: flags enum
session: saga.Session
ret: obj
Construct a new directory object
The specified directory is expected to exist -- otherwise
a DoesNotExist exception is raised. Also, the URL must point to
a directory (not to an entry), otherwise a BadParameter exception is
raised.
Example::
# open some directory
dir = saga.namespace.Directory("sftp://localhost/tmp/")
# and list its contents
entries = dir.list ()
'''
if not flags: flags = 0
self._nsentry = super(Directory, self)
self._nsentry.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=None, session=None, ttype=None):
'''
url: saga.Url
flags: saga.namespace.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
if not flags: flags = 0
_nsentry = super(Directory, cls)
return _nsentry.create(url, flags, session, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((entry.Entry, st.Task))
def open(self, name, flags=None, ttype=None):
'''
name: saga.Url
flags: saga.namespace.flags enum
ttype: saga.task.type enum
ret: saga.namespace.Entry / saga.Task
'''
if not flags: flags = 0
url = ru.Url(name)
return self._adaptor.open(url, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(('Directory', st.Task))
def open_dir(self, path, flags=None, ttype=None):
'''
:param path: name/path of the directory to open
:param flags: directory creation flags
ttype: saga.task.type enum
ret: saga.namespace.Directory / saga.Task
Open and return a new directoy
The call opens and returns a directory at the given location.
Example::
# create a subdir 'data' in /tmp
dir = saga.namespace.Directory("sftp://localhost/tmp/")
data = dir.open_dir ('data/', saga.namespace.Create)
'''
if not flags: flags = 0
return self._adaptor.open_dir(ru.Url(path), flags, ttype=ttype)
# ----------------------------------------------------------------
#
# namespace directory methods
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def make_dir(self, tgt, flags=0, ttype=None):
'''
:param tgt: name/path of the new directory
:param flags: directory creation flags
ttype: saga.task.type enum
ret: None / saga.Task
Create a new directoy
The call creates a directory at the given location.
Example::
# create a subdir 'data' in /tmp
dir = saga.namespace.Directory("sftp://localhost/tmp/")
dir.make_dir ('data/')
'''
if not flags: flags = 0
return self._adaptor.make_dir(ru.Url(tgt), flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def change_dir(self, url, flags=0, ttype=None):
'''
url: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
return self._adaptor.change_dir(url, flags=flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional(basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(ru.Url), st.Task))
def list(self, pattern=None, flags=0, ttype=None):
'''
:param pattern: Entry name pattern (like POSIX 'ls', e.g. '\*.txt')
flags: flags enum
ttype: saga.task.type enum
ret: list [saga.Url] / saga.Task
List the directory's content
The call will return a list of entries and subdirectories within the
directory::
# list contents of the directory
for f in dir.list() :
print f
'''
if not flags: flags = 0
return self._adaptor.list(pattern, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def exists(self, path, ttype=None):
'''
:param path: path of the entry to check
ttype: saga.task.type enum
ret: bool / saga.Task
Returns True if path exists, False otherwise.
Example::
# inspect an entry
dir = saga.namespace.Directory("sftp://localhost/tmp/")
if dir.exists ('data'):
# do something
'''
return self._adaptor.exists(path, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional(basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(ru.Url), st.Task))
def find(self, pattern, flags=c.RECURSIVE, ttype=None):
'''
pattern: string
flags: flags enum
ttype: saga.task.type enum
ret: list [saga.Url] / saga.Task
'''
if not flags: flags = 0
return self._adaptor.find(pattern, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def get_num_entries(self, ttype=None):
'''
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.get_num_entries(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', int, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((ru.Url, st.Task))
def get_entry(self, num, ttype=None):
'''
num: int
ttype: saga.task.type enum
ret: saga.Url / saga.Task
'''
return self._adaptor.get_entry(num, ttype=ttype)
# ----------------------------------------------------------------
#
# methods overloaded from namespace.Entry
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def copy(self, url_1, url_2=None, flags=0, ttype=None):
'''
:param src: path of the entry to copy
:param tgt: absolute URL of target name or directory
url_1: saga.Url
url_2: saga.Url / None
flags: flags enum / None
ttype: saga.task.type enum / None
ret: None / saga.Task
Copy an entry from source to target
The source is copied to the given target directory. The path of the
source can be relative::
# copy an entry
dir = saga.namespace.Directory("sftp://localhost/tmp/")
dir.copy ("./data.bin", "sftp://localhost/tmp/data/")
'''
# FIXME: re-implement the url switching (commented out below)
if not flags: flags = 0
if url_2: return self._adaptor.copy(url_1, url_2, flags, ttype=ttype)
else: return self._nsentry.copy(url_1, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def link(self, url_1, url_2=None, flags=0, ttype=None):
'''
src: saga.Url
tgt: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
if url_2: return self._adaptor.link(url_1, url_2, flags, ttype=ttype)
else: return self._nsentry.link(url_1, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def move(self, url_1, url_2=None, flags=0, ttype=None):
'''
:param src: path of the entry to copy
:param tgt: absolute URL of target directory
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
Move an entry from source to target
The source is moved to the given target directory. The path of the
source can be relative::
# copy an entry
dir = saga.namespace.Directory("sftp://localhost/tmp/")
dir.move ("./data.bin", "sftp://localhost/tmp/data/")
'''
if not flags: flags = 0
if url_2: return self._adaptor.move(url_1, url_2, flags, ttype=ttype)
else: return self._nsentry.move(url_1, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def remove(self, tgt=None, flags=0, ttype=None):
'''
tgt: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
if tgt: return self._adaptor.remove(tgt, flags, ttype=ttype)
else: return self._nsentry.remove(flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_dir(self, tgt=None, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: bool / saga.Task
Returns True if path is a directory, False otherwise.
Example::
# inspect an entry
dir = saga.namespace.Directory("sftp://localhost/tmp/")
if dir.is_dir ('data'):
# do something
'''
if tgt: return self._adaptor.is_dir(tgt, ttype=ttype)
else: return self._nsentry.is_dir(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_entry(self, tgt=None, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: bool / saga.Task
'''
if tgt: return self._adaptor.is_entry(tgt, ttype=ttype)
else: return self._nsentry.is_entry(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_link(self, tgt=None, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: bool / saga.Task
'''
if tgt: return self._adaptor.is_link(tgt, ttype=ttype)
else: return self._nsentry.is_link(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((ru.Url, st.Task))
def read_link(self, tgt=None, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: saga.Url / saga.Task
'''
if tgt: return self._adaptor.read_link(tgt, ttype=ttype)
else: return self._nsentry.read_link(ttype=ttype)
class Container(sbase.SimpleBase, satt.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(rus.nothing)
def __init__(self):
self._base = super(Container, self)
self._base.__init__()
# set attribute interface properties
self._attributes_allow_private(True)
self._attributes_extensible(False)
self._attributes_camelcasing(True)
# register properties with the attribute interface
self._attributes_register(SIZE, 0, satt.INT, satt.SCALAR,
satt.READONLY)
self._attributes_set_getter(SIZE, self.get_size)
self._attributes_register(TASKS, [], satt.ANY, satt.VECTOR,
satt.READONLY)
self._attributes_set_getter(TASKS, self.get_tasks)
self._attributes_register(STATES, [], satt.ENUM, satt.VECTOR,
satt.READONLY)
self._attributes_set_getter(STATES, self.get_states)
self._attributes_set_enums(
STATES, [UNKNOWN, NEW, RUNNING, DONE, FAILED, CANCELED])
# cache for created container instances
self._containers = {}
# --------------------------------------------------------------------------
#
def __str__(self):
ret = "["
for task in self.tasks:
ret += "'%s', " % str(task)
ret += "]"
return ret
# --------------------------------------------------------------------------
#
@rus.takes('Container', Task)
@rus.returns(rus.nothing)
def add(self, task):
if not isinstance(task, Task):
raise se.BadParameter ("Container handles tasks, not %s" \
% (type(task)))
if not task in self.tasks:
self.tasks.append(task)
# --------------------------------------------------------------------------
#
@rus.takes('Container', Task)
@rus.returns(rus.nothing)
def remove(self, task):
if task in self.tasks:
self.tasks.delete(task)
# --------------------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(rus.nothing)
def run(self):
if not len(self.tasks):
# nothing to do
return None
buckets = self._get_buckets()
futures = [] # futures running container ops
# handle all container
for c in buckets['bound']:
# handle all methods
for m in buckets['bound'][c]:
tasks = buckets['bound'][c][m]
m_name = "container_%s" % m
m_handle = None
for (name,
handle) in inspect.getmembers(c,
predicate=inspect.ismethod):
if name == m_name:
m_handle = handle
break
if not handle:
# Hmm, the specified container can't handle the call after
# all -- fall back to the unbound handling
buckets['unbound'] += tasks
else:
# hand off to the container function, in a separate task
futures.append(ru.Future.Run(m_handle, tasks))
# handle tasks not bound to a container
for task in buckets['unbound']:
futures.append(ru.Future.Run(task.run))
# wait for all futures to finish
for future in futures:
if future.isAlive():
future.join()
if future.state == FAILED:
raise se.NoSuccess ("future exception: %s" \
% (future.exception))
# --------------------------------------------------------------------------
#
# --------------------------------------------------------------------------
#
@rus.takes('Container', rus.one_of(ANY, ALL), rus.optional(float))
@rus.returns(rus.list_of(Task))
def wait(self, mode=ALL, timeout=None):
if None == timeout:
timeout = -1.0 # FIXME
if not mode in [ANY, ALL]:
raise se.BadParameter(
"wait mode must be saga.task.ANY or saga.task.ALL")
if type(timeout) not in [int, long, float]:
raise se.BadParameter(
"wait timeout must be a floating point number (or integer)")
if not len(self.tasks):
# nothing to do
return None
if mode == ALL:
return self._wait_all(timeout)
else:
return self._wait_any(timeout)
# --------------------------------------------------------------------------
#
@rus.takes('Container', float)
@rus.returns(rus.list_of(Task))
def _wait_any(self, timeout):
buckets = self._get_buckets()
futures = [] # futures running container ops
# handle all tasks bound to containers
for c in buckets['bound']:
# handle all methods -- all go to the same 'container_wait' though)
tasks = []
for m in buckets['bound'][c]:
tasks += buckets['bound'][c][m]
futures.append(ru.Future.Run(c.container_wait, tasks, ANY,
timeout))
# handle all tasks not bound to containers
for task in buckets['unbound']:
futures.append(ru.Future.Run(task.wait, timeout))
# mode == ANY: we need to watch our futures, and whenever one
# returns, and declare success. Note that we still need to get the
# finished task from the 'winner'-future -- we do that via a Queue
# object. Note also that looser futures are not canceled, but left
# running (FIXME: consider sending a signal at least)
timeout = 0.01 # seconds, heuristic :-/
for future in futures:
future.join(timeout)
if future.state == FAILED:
raise future.exception
if not future.isAlive():
# future indeed finished -- dig return value from this
# futures queue
result = future.result
# ignore other futures, and simply declare success
return result
# --------------------------------------------------------------------------
#
@rus.takes('Container', float)
@rus.returns(rus.list_of(Task))
def _wait_all(self, timeout):
# this method should actually be symmetric to _wait_any, and could
# almost be mapped to it, but the code below is a kind of optimization
# (does not need futures, thus simpler code).
buckets = self._get_buckets()
ret = None
# handle all tasks bound to containers
for c in buckets['bound']:
# handle all methods -- all go to the same 'container_wait' though)
tasks = []
for m in buckets['bound'][c]:
tasks += buckets['bound'][c][m]
# TODO: this is semantically not correct: timeout is applied
# n times...
c.container_wait(tasks, ALL, timeout)
ret = tasks[0]
# handle all tasks not bound to containers
for task in buckets['unbound']:
task.wait()
ret = task
# all done - return random task (first from last container, or last
# unbound task)
# FIXME: that task should be removed from the task container
return ret
# --------------------------------------------------------------------------
#
@rus.takes('Container', rus.optional(float))
@rus.returns(rus.nothing)
def cancel(self, timeout=None):
if None == timeout:
timeout = -1.0 # FIXME
buckets = self._get_buckets()
futures = [] # futures running container ops
# handle all tasks bound to containers
for c in buckets['bound']:
# handle all methods -- all go to the same 'container_cancel' though)
tasks = []
for m in buckets['bound'][c]:
tasks += buckets['bound'][c][m]
futures.append(ru.Future.Run(c.container_cancel, tasks, timeout))
# handle all tasks not bound to containers
for task in buckets['unbound']:
futures.append(ru.Future.Run(task.cancel, timeout))
for future in futures:
future.join()
# ----------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(int)
def get_size(self):
return len(self.tasks)
# --------------------------------------------------------------------------
#
@rus.takes('Container', 'basestring')
@rus.returns(Task)
def get_task(self, id):
# FIXME: this should not be a search, but a lookup
if not id:
raise se.NoSuccess("Lookup requires non-empty id (not '%s')" % id)
for t in self.tasks:
if t.id == id:
return t
raise se.NoSuccess("task '%s' not found in container" % id)
# --------------------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(rus.list_of(Task))
def get_tasks(self):
return self.tasks
# --------------------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(
rus.list_of(rus.one_of(UNKNOWN, NEW, RUNNING, DONE, FAILED, CANCELED)))
def get_states(self):
buckets = self._get_buckets()
futures = [] # futures running container ops
# handle all tasks bound to containers
for c in buckets['bound']:
# handle all methods -- all go to the same 'container_get_states' though)
tasks = []
for m in buckets['bound'][c]:
tasks += buckets['bound'][c][m]
futures.append(ru.Future.Run(c.container_get_states, tasks))
# handle all tasks not bound to containers
for task in buckets['unbound']:
futures.append(ru.Future.Run(task.get_state))
# We still need to get the states from all futures.
# FIXME: order
states = []
for future in futures:
future.join()
if future.state == FAILED:
raise future.exception
# FIXME: what about ordering tasks / states?
res = future.result
if res != None:
states += res
return states
# ----------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(dict)
def _get_buckets(self):
# collective container ops: walk through the task list, and sort into
# buckets of tasks which have (a) the same task._container, or if that
# is not set, the same class type (for which one container instance is
# created). All tasks were neither is available are handled one-by-one
buckets = {}
buckets['unbound'] = [] # no container adaptor for these [tasks]
buckets['bound'] = {} # dict of container adaptors [tasks]
for task in self.tasks:
if task._adaptor and task._adaptor._container:
# the task's adaptor has a valid associated container class
# which can handle the container ops - great!
c = task._adaptor._container
m = task._method_type
if not c in buckets['bound']:
buckets['bound'][c] = {}
if not m in buckets['bound'][c]:
buckets['bound'][c][m] = []
buckets['bound'][c][m].append(task)
else:
# we have no container to handle this task -- so
# put it into the fallback list
buckets['unbound'].append(task)
return buckets
def create (cls, url_in=None, session=None, ttype=sc.SYNC) :
"""
This is the asynchronous class constructor, returning
a :class:`saga:Task` instance. For details on the accepted parameters,
please see the description of :func:`__init__`.
"""
return cls (url_in, session, _ttype=ttype)._init_task
# --------------------------------------------------------------------------
#
@rus.takes ('Manager',
rus.optional (rus.one_of (c.COMPUTE, c.STORAGE, c.NETWORK)),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((rus.list_of (basestring), st.Task))
def list (self, rtype=None, ttype=None) :
"""
list(rtype=None)
List known resource instances (which can be acquired).
Returns a list of IDs.
:type rtype: None or enum (COMPUTE | STORAGE | NETWORK)
:param rtype: filter for one or more resource types
"""
return self._adaptor.list (rtype, ttype=ttype)
# --------------------------------------------------------------------------
class LogicalFile(nsentry.Entry, sa.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=c.READ,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
__init__(url=None, flags=READ, session=None)
url: saga.Url
flags: flags enum
session: saga.Session
ret: obj
'''
# param checks
if not flags: flags = 0
url = ru.Url(url)
self._nsentry = super(LogicalFile, self)
self._nsentry.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('LogicalFile', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=c.READ, session=None, ttype=None):
'''
url: saga.Url
flags: saga.replica.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
if not flags: flags = 0
_nsentry = super(LogicalFile, cls)
return _nsentry.create(url, flags, session, ttype=ttype)
# ----------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_file(self, ttype=None):
'''
is_file()
ttype: saga.task.type enum
ret: bool / saga.Task
'''
return self.is_entry(ttype=ttype)
# --------------------------------------------------------------------------
#
# replica methods
#
@rus.takes('LogicalFile', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def get_size(self, ttype=None):
'''
get_size()
Return the size of the file.
ttype: saga.task.type enum
ret: int / saga.Task
Returns the size of the physical file represented by this logical file
(in bytes)
Example::
# get a file handle
lf = saga.replica.LogicalFile("irods://localhost/tmp/data.bin")
# print the logical file's size
print lf.get_size ()
'''
return self._adaptor.get_size_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def add_location(self, name, ttype=None):
'''
add_location(name)
Add a physical location.
name: saga.Url
ttype: saga.task.type enum
ret: None / saga.Task
'''
return self._adaptor.add_location(name, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def remove_location(self, name, ttype=None):
'''
remove_location(name)
Remove a physical location.
name: saga.Url
ttype: saga.task.type enum
ret: None / saga.Task
'''
return self._adaptor.remove_location(name, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional((ru.Url, basestring)),
rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def update_location(self, old, new, ttype=None):
'''
update_location(old, new)
Updates a physical location.
old: saga.Url
new: saga.Url
ttype: saga.task.type enum
ret: None / saga.Task
'''
return self._adaptor.update_location(old, new, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(ru.Url), st.Task))
def list_locations(self, ttype=None):
'''
list_locations()
List all physical locations of a logical file.
ttype: saga.task.type enum
ret: list [saga.Url] / saga.Task
'''
return self._adaptor.list_locations(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def replicate(self, name, flags=None, ttype=None):
'''
replicate(name)
Replicate a logical file.
name: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
return self._adaptor.replicate(name, flags, ttype=ttype)
# --------------------------------------------------------------------------
# non-GFD.90
#
@rus.takes('LogicalFile', (ru.Url, basestring),
rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def upload(self, name, tgt=None, flags=None, ttype=None):
'''
upload(name, tgt=None, flags=None)
Upload a physical file.
name: saga.Url
tgt: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
return self._adaptor.upload(name, tgt, flags, ttype=ttype)
# --------------------------------------------------------------------------
# non-GFD.90
#
@rus.takes('LogicalFile', (ru.Url, basestring),
rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def download(self, name, src=None, flags=None, ttype=None):
'''
download(name, src=None, flags=None)
Download a physical file.
name: saga.Url
src: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
return self._adaptor.download(name, src, flags, ttype=ttype)
def create (cls, url_in=None, session=None, ttype=sc.SYNC) :
"""
This is the asynchronous class constructor, returning
a :class:`saga:Task` instance. For details on the accepted parameters,
please see the description of :func:`__init__`.
"""
return cls (url_in, session, _ttype=ttype)._init_task
# --------------------------------------------------------------------------
#
@rus.takes ('Manager',
rus.optional (rus.one_of (COMPUTE, STORAGE, NETWORK)),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((rus.list_of (basestring), st.Task))
def list (self, rtype=None, ttype=None) :
"""
list(rtype=None)
List known resource instances (which can be acquired).
Returns a list of IDs.
:type rtype: None or enum (COMPUTE | STORAGE | NETWORK)
:param rtype: filter for one or more resource types
"""
return self._adaptor.list (rtype, ttype=ttype)
# --------------------------------------------------------------------------
class Session(saga.base.SimpleBase):
"""A SAGA Session object as defined in GFD.90.
A SAGA session has the purpose of scoping the use of security credentials
for remote operations. In other words, a session instance acts as
a container for security Context instances -- SAGA objects (such as
job.Service or filesystem.File) created in that session will then use
exactly the security contexts from that session (and no others).
That way, the session serves two purposes: (1) it helps SAGA to decide
which security mechanism should be used for what interaction, and (2) it
helps SAGA to find security credentials which would be difficult to pick up
automatically.
The use of a session is as follows:
Example::
# define an ssh context
c = saga.Context('ssh')
c.user_cert = '$HOME/.ssh/special_id_rsa.pub'
c.user_key = '$HOME/.ssh/special_id_rsa'
# add it to a session
s = saga.Session
s.add_context(c)
# create a job service in this session -- that job service can now
# *only* use that ssh context.
j = saga.job.Service('ssh://remote.host.net/', s)
The session argument to the L{job.Service} constructor is fully optional --
if left out, SAGA will use default session, which picks up some default
contexts as described above -- that will suffice for the majority of use
cases.
A session instance exposes a `context` property, which is a list of
authentication contexts managed by this session. As the contexts and the
session are stateless, it is safe to modify this list as needed.
"""
# FIXME: session deep copy not implemented
# --------------------------------------------------------------------------
#
@rus.takes('Session', rus.optional(bool))
@rus.returns(rus.nothing)
def __init__(self, default=True):
"""
default: bool
ret: None
"""
simple_base = super(Session, self)
simple_base.__init__()
self._logger = ru.get_logger('radical.saga')
# if the default session is expected, we point our context list to the
# shared list of the default session singleton. Otherwise, we create
# a private list which is not populated.
# a session also has a lease manager, for adaptors in this session to use.
if default:
default_session = DefaultSession()
self.contexts = copy.deepcopy(default_session.contexts)
self._lease_manager = default_session._lease_manager
else:
self.contexts = _ContextList(session=self)
# FIXME: at the moment, the lease manager is owned by the session.
# Howevwer, the pty layer is the main user of the lease manager,
# and we thus keep the lease manager options in the pty subsection.
# So here we are, in the session, evaluating the pty config options...
config = self.get_config('saga.utils.pty')
self._lease_manager = ru.LeaseManager(
max_pool_size=config['connection_pool_size'].get_value(),
max_pool_wait=config['connection_pool_wait'].get_value(),
max_obj_age=config['connection_pool_ttl'].get_value())
# ----------------------------------------------------------------
#
@rus.takes('Session')
@rus.returns(basestring)
def __str__(self):
"""String represenation."""
return "Registered contexts: %s" % (str(self.contexts))
# ----------------------------------------------------------------
#
@rus.takes('Session', saga.context.Context)
@rus.returns(rus.nothing)
def add_context(self, ctx):
"""
ctx: saga.Context
ret: None
Add a security L{Context} to the session.
It is encouraged to use the L{contexts} property instead.
"""
return self.contexts.insert(0, ctx=ctx, session=self)
# ----------------------------------------------------------------
#
@rus.takes('Session', saga.context.Context)
@rus.returns(rus.nothing)
def remove_context(self, ctx):
"""
ctx: saga.Context
ret: None
Remove a security L{Context} from the session.
It is encouraged to use the L{contexts} property instead.
"""
if ctx in self.contexts:
self.contexts.remove(ctx)
# ----------------------------------------------------------------
#
@rus.takes('Session')
@rus.returns(rus.list_of(saga.context.Context))
def list_contexts(self):
"""
ret: list[saga.Context]
Retrieve all L{Context} objects attached to the session.
It is encouraged to use the L{contexts} property instead.
"""
return self.contexts
# ----------------------------------------------------------------
#
@rus.takes('Session')
@rus.returns(dict)
def get_config(self, section=""):
"""
ret: radical.utils.Configuration
Return the session configuration (optional a specific section).
"""
return saga.engine.engine.Engine().get_config(section)
class LogicalDirectory(nsdir.Directory, sa.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('LogicalDirectory', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=c.READ,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
__init__(url, flags=READ, session=None)
Create a new Logical Directory instance.
url: saga.Url
flags: flags enum
session: saga.Session
ret: obj
'''
# param checks
if not flags: flags = 0
url = ru.Url(url)
self._nsdirec = super(LogicalDirectory, self)
self._nsdirec.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url, flags=c.READ, session=None, ttype=None):
'''
url: saga.Url
flags: saga.replica.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
if not flags: flags = 0
_nsdirec = super(LogicalDirectory, cls)
return _nsdirec.create(url, flags, session, ttype=ttype)
@rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_file(self, tgt=None, ttype=None):
'''
is_file(tgt=None)
tgt: saga.Url / string
ttype: saga.task.type enum
ret: bool / saga.Task
'''
if tgt: return self._adaptor.is_file(tgt, ttype=ttype)
else: return self._nsdirec.is_entry_self(ttype=ttype)
@rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(('LogicalFile', st.Task))
def open(self, tgt, flags=c.READ, ttype=None):
'''
open(tgt, flags=READ)
tgt: saga.Url
flags: saga.namespace.flags enum
ttype: saga.task.type enum
ret: saga.namespace.Entry / saga.Task
'''
if not flags: flags = 0
tgt_url = ru.Url(tgt)
return self._adaptor.open(tgt_url, flags, ttype=ttype)
# ----------------------------------------------------------------
#
@rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(('LogicalDirectory', st.Task))
def open_dir(self, tgt, flags=c.READ, ttype=None):
'''
open_dir(tgt, flags=READ)
:param tgt: name/path of the directory to open
:param flags: directory creation flags
ttype: saga.task.type enum
ret: saga.namespace.Directory / saga.Task
Open and return a new directoy
The call opens and returns a directory at the given location.
Example::
# create a subdir 'data' in /tmp
dir = saga.namespace.Directory("sftp://localhost/tmp/")
data = dir.open_dir ('data/', saga.namespace.Create)
'''
if not flags: flags = 0
tgt_url = ru.Url(tgt)
return self._adaptor.open_dir(tgt_url, flags, ttype=ttype)
# ----------------------------------------------------------------
#
# replica methods
#
# --------------------------------------------------------------------------
#
@rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def get_size(self, tgt, ttype=None):
'''
get_size(tgt)
tgt: logical file to get size for
ttype: saga.task.type enum
ret: int / saga.Task
Returns the size of the physical file represented by the given logical
file (in bytes)
Example::
# get a logical directory handle
lf = saga.replica.LogicalFile("irods://localhost/tmp/data/")
# print a logical file's size
print lf.get_size ('data.dat')
'''
tgt_url = ru.Url(tgt)
return self._adaptor.get_size(tgt_url, ttype=ttype)
# ----------------------------------------------------------------
#
@rus.takes('LogicalDirectory', rus.optional(basestring),
rus.optional(basestring), rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(ru.Url), st.Task))
def find(self,
name_pattern,
attr_pattern=None,
flags=c.RECURSIVE,
ttype=None):
'''
find(name_pattern, attr_pattern=None, flags=RECURSIVE)
name_pattern: string
attr_pattern: string
flags: flags enum
ttype: saga.task.type enum
ret: list [saga.Url] / saga.Task
'''
if not flags: flags = 0
if attr_pattern:
return self._adaptor.find_replicas(name_pattern,
attr_pattern,
flags,
ttype=ttype)
else:
return self._nsdirec.find(name_pattern, flags, ttype=ttype)
class File(nsentry.Entry):
"""
Represents a local or remote file.
The saga.filesystem.File class represents, as the name indicates,
a file on some (local or remote) filesystem. That class offers
a number of operations on that file, such as copy, move and remove::
# get a file handle
file = saga.filesystem.File("sftp://localhost/tmp/data/data.bin")
# copy the file
file.copy ("sftp://localhost/tmp/data/data.bak")
# move the file
file.move ("sftp://localhost/tmp/data/data.new")
"""
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=READ,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
"""
__init__(url, flags=READ, session)
Construct a new file object
:param url: Url of the (remote) file
:type url: :class:`saga.Url`
:fgs: :ref:`filesystemflags`
:param session: :class:`saga.Session`
The specified file is expected to exist -- otherwise a DoesNotExist
exception is raised. Also, the URL must point to a file (not to
a directory), otherwise a BadParameter exception is raised.
Example::
# get a file handle
file = saga.filesystem.File("sftp://localhost/tmp/data/data.bin")
# print the file's size
print file.get_size ()
"""
# param checks
if not flags: flags = 0
url = ru.Url(url)
if not url.schema:
url.schema = 'file'
if not url.host:
url.host = 'localhost'
self._nsentry = super(File, self)
self._nsentry.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('File', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=READ, session=None, ttype=None):
"""
create(url, flags, session)
url: saga.Url
flags: saga.replica.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
"""
if not flags: flags = 0
_nsentry = super(File, cls)
return _nsentry.create(url, flags, session, ttype=ttype)
# ----------------------------------------------------------------
#
# filesystem methods
#
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_file(self, ttype=None):
"""
is_file()
Returns `True` if instance points to a file, `False` otherwise.
"""
return self._adaptor.is_file_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def get_size(self, ttype=None):
'''
get_size()
Returns the size (in bytes) of a file.
Example::
# get a file handle
file = saga.filesystem.File("sftp://localhost/tmp/data/data.bin")
# print the file's size
print file.get_size ()
'''
return self._adaptor.get_size_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional(int),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def read(self, size=None, ttype=None):
'''
size : int
ttype: saga.task.type enum
ret: string / bytearray / saga.Task
'''
return self._adaptor.read(size, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional(bool))
@rus.returns(st.Task)
def close(self, kill=True, ttype=None):
'''
kill : bool
ttype: saga.task.type enum
ret: string / bytearray / saga.Task
'''
return self._adaptor.close()
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def write(self, data, ttype=None):
'''
data : string / bytearray
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.write(data, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', int, rus.optional(rus.one_of(START, CURRENT, END)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def seek(self, offset, whence=START, ttype=None):
'''
offset: int
whence: seek_mode enum
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.seek(offset, whence, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.list_of(rus.tuple_of(int)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def read_v(self, iovecs, ttype=None):
'''
iovecs: list [tuple (int, int)]
ttype: saga.task.type enum
ret: list [bytearray] / saga.Task
'''
return self._adaptor.read_v(iovecs, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.list_of(rus.tuple_of((int, basestring))),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(int), st.Task))
def write_v(self, data, ttype=None):
'''
data: list [tuple (int, string / bytearray)]
ttype: saga.task.type enum
ret: list [int] / saga.Task
'''
return self._adaptor.write_v(data, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def size_p(self, pattern, ttype=None):
'''
pattern: string
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.size_p(pattern, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def read_p(self, pattern, ttype=None):
'''
pattern: string
ttype: saga.task.type enum
ret: string / bytearray / saga.Task
'''
return self._adaptor.read_p(pattern, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, basestring,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def write_p(self, pattern, data, ttype=None):
'''
pattern: string
data: string / bytearray
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.write_p(pattern, data, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(basestring), st.Task))
def modes_e(self, ttype=None):
'''
ttype: saga.task.type enum
ret: list [string] / saga.Task
'''
return self._adaptor.modes_e(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, basestring,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def size_e(self, emode, spec, ttype=None):
'''
emode: string
spec: string
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.size_e(emode, spec, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, basestring,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def read_e(self, emode, spec, ttype=None):
'''
emode: string
spec: string
ttype: saga.task.type enum
ret: bytearray / saga.Task
'''
return self._adaptor.read_e(emode, spec, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, basestring, basestring,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def write_e(self, emode, spec, data, ttype=None):
'''
emode: string
spec: string
data: string / bytearray
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.read_e(emode, spec, data, ttype=ttype)
size = property(get_size) # int
modes_e = property(modes_e) # list [string]
class Entry(nsentry.Entry, sa.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=READ,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
url: saga.Url
flags: flags enum
session: saga.Session
ret: obj
'''
# param checks
url = ru.Url(url)
self._nsentry = super(Entry, self)
self._nsentry.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# set attribute interface properties
self._attributes_allow_private(True)
self._attributes_camelcasing(True)
self._attributes_extensible(True,
getter=self._attribute_getter,
setter=self._attribute_setter,
lister=self._attribute_lister,
caller=self._attribute_caller)
# register properties with the attribute interface
self._attributes_register(ATTRIBUTE, None, sa.STRING, sa.SCALAR,
sa.READONLY)
self._attributes_register(OBJECT, None, sa.ANY, sa.SCALAR, sa.READONLY)
self._attributes_register(TTL, None, sa.INT, sa.SCALAR, sa.WRITEABLE)
self._attributes_set_setter(TTL, self.set_ttl)
self._attributes_set_getter(TTL, self.get_ttl)
self._attributes_set_setter(OBJECT, self.store_object)
self._attributes_set_getter(OBJECT, self.retrieve_object)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('Entry', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=READ, session=None, ttype=None):
'''
url: saga.Url
flags: saga.advert.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
if not flags: flags = 0
_nsentry = super(Entry, cls)
return _nsentry.create(url, flags, session, ttype=ttype)
# --------------------------------------------------------------------------
#
# attribute methods
#
# NOTE: we do not yet pass ttype, as async calls are not yet supported by
# the attribute interface
#
@rus.takes('Entry', basestring, rus.optional(rus.one_of(SYNC, ASYNC,
TASK)))
@rus.returns((rus.anything, st.Task))
def _attribute_getter(self, key, ttype=None):
return self._adaptor.attribute_getter(key)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', basestring, rus.anything,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def _attribute_setter(self, key, val, ttype=None):
return self._adaptor.attribute_setter(key, val)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(rus.anything), st.Task))
def _attribute_lister(self, ttype=None):
return self._adaptor.attribute_lister()
# --------------------------------------------------------------------------
#
@rus.takes('Entry', basestring, int, callable,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.anything, st.Task))
def _attribute_caller(self, key, id, cb, ttype=None):
return self._adaptor.attribute_caller(key, id, cb)
# --------------------------------------------------------------------------
#
# advert methods
#
@rus.takes('Entry', float, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def set_ttl(self, ttl=-1.0, ttype=None):
"""
ttl : int
ttype: saga.task.type enum
ret: None / saga.Task
"""
return self._adaptor.set_ttl(ttl, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((float, st.Task))
def get_ttl(self, ttype=None):
"""
ttype: saga.task.type enum
ret: int / saga.Task
"""
return self._adaptor.get_ttl(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', object, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def store_object(self, object, ttype=None):
"""
object : <object type>
ttype: saga.task.type enum
ret: None / saga.Task
"""
return self._adaptor.store_object(object, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((object, st.Task))
def retrieve_object(self, ttype=None):
"""
ttype: saga.task.type enum
ret: any / saga.Task
"""
return self._adaptor.retrieve_object(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def delete_object(self, ttype=None):
"""
ttype: saga.task.type enum
ret: None / saga.Task
"""
return self._adaptor.delete_object(ttype=ttype)
class Monitorable(sa.Attributes):
# --------------------------------------------------------------------------
#
def __init__(self):
self._attr = super(Monitorable, self)
self._attr.__init__()
# --------------------------------------------------------------------------
#
# since we have no means to call the saga.Base constructor explicitly (we
# don't inherit from it), we have to rely that the classes which implement
# the Monitorable interface are correctly calling the Base constructure --
# otherwise we won't have an self._adaptor to talk to...
#
# This helper method checks the existence of self._adaptor, and should be
# used before each call forwarding.
#
def _check(self):
if not hasattr(self, '_adaptor'):
raise se.IncorrectState("object is not fully initialized")
# --------------------------------------------------------------------------
#
@rus.takes('Monitorable')
@rus.returns(rus.list_of(basestring))
def list_metrics(self):
self._check()
return self._adaptor.list_metrics()
# --------------------------------------------------------------------------
#
# Metrics are not implemented in RADICAL-SAGA
#
# @rus.takes ('Monitorable', basestring)
# @rus.returns ('Metric')
# def get_metric (name) :
#
# self._check ()
# return self._adaptor.get_metric (name)
# --------------------------------------------------------------------------
#
@rus.takes('Monitorable', basestring, rus.one_of('saga.Callback',
callable))
@rus.returns(int)
def add_callback(self, name, cb):
self._check()
return self._adaptor.add_callback(name, cb)
# --------------------------------------------------------------------------
#
@rus.takes('Monitorable', int)
@rus.returns(rus.nothing)
def remove_callback(self, cookie):
self._check()
return self._adaptor.remove_callback(cookie)
class Manager(sb.Base, sasync.Async):
"""
In the context of RADICAL-SAGA, a *ResourceManager* is a service which
asserts control over a set of resources. That manager can, on request,
render control over subsets of those resources (resource slices) to an
application.
This :class:`Manager` class represents the contact point to such
ResourceManager instances -- the application can thus acquire compute, data
or network resources, according to some resource specification, for a bound
or unbound amount of time.
"""
# --------------------------------------------------------------------------
#
@rus.takes('Manager', rus.optional(str, ru.Url), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
"""
__init__(url)
Create a new Manager instance. Connect to a remote resource management endpoint.
:type url: :class:`saga.Url`
:param url: resource management endpoint
"""
# param checks
_url = ru.Url(url)
scheme = _url.scheme.lower()
if not session:
session = ss.Session(default=True)
self._base = super(Manager, self)
self._base.__init__(scheme,
_adaptor,
_adaptor_state,
_url,
session,
ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('Manager', rus.optional((ru.Url, str)),
rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url_in=None, session=None, ttype=sc.SYNC):
"""
This is the asynchronous class constructor, returning
a :class:`saga:Task` instance. For details on the accepted parameters,
please see the description of :func:`__init__`.
"""
return cls(url_in, session, _ttype=ttype)._init_task
# --------------------------------------------------------------------------
#
@rus.takes('Manager',
rus.optional(rus.one_of(c.COMPUTE, c.STORAGE, c.NETWORK)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(str), st.Task))
def list(self, rtype=None, ttype=None):
"""
list(rtype=None)
List known resource instances (which can be acquired).
Returns a list of IDs.
:type rtype: None or enum (COMPUTE | STORAGE | NETWORK)
:param rtype: filter for one or more resource types
"""
return self._adaptor.list(rtype, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager', rus.optional(str),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((descr.Description, st.Task))
def get_description(self, rid, ttype=None):
"""
get_description(rid)
Get the resource :class:`Description` for the specified resource.
:type rid: str
:param rid: identifies the resource to be described.
"""
# TODO / NOTE: if rid is None, should we return a description of
# the managed resources?
return self._adaptor.get_description(id, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager',
rus.optional(rus.one_of(c.COMPUTE, c.STORAGE, c.NETWORK)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(str), st.Task))
def list_templates(self, rtype=None, ttype=None):
"""
list_templates(rtype=None)
List template names available for the specified resource type(s).
Returns a list of strings.
:type rtype: None or enum (COMPUTE | STORAGE | NETWORK)
:param rtype: filter for one or more resource types
"""
return self._adaptor.list_templates(rtype, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager', rus.optional(str),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((descr.Description, st.Task))
def get_template(self, name, ttype=None):
"""
get_template(name)
Get a :class:`Description` for the specified template.
:type name: str
:param name: specifies the name of the template
The returned resource description instance may not have all attributes
filled, and may in fact not sufficiently complete to allow for
successful resource acquisition. The only guaranteed attribute in the
returned description is `TEMPLATE`, containing the very template id
specified in the call parameters.
"""
return self._adaptor.get_template(name, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager',
rus.optional(rus.one_of(c.COMPUTE, c.STORAGE, c.NETWORK)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(str), st.Task))
def list_images(self, rtype=None, ttype=None):
"""
list_images(rtype=None)
List image names available for the specified resource type(s).
Returns a list of strings.
:type rtype: None or enum (COMPUTE | STORAGE | NETWORK)
:param rtype: filter for one or more resource types
"""
return self._adaptor.list_images(rtype, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager', str, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((dict, st.Task))
def get_image(self, name, ttype=None):
"""
get_image(name)
Get a description string for the specified image.
:type name: str
:param name: specifies the image name
"""
return self._adaptor.get_image(name, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager', (str, ru.Url, descr.Description),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((resrc.Resource, st.Task))
def acquire(self, spec, ttype=None):
"""
acquire(desc)
Create a new :class:`saga.resource.Resource` handle for a
resource specified by the description.
:type spec: :class:`Description` or Url
:param spec: specifies the resource
Depending on the `RTYPE` attribute in the description, the
returned resource may be a :class:`saga.resource.Compute`,
:class:`saga.resource.Storage` or :class:`saga.resource.Network`
instance.
If the `spec` parameter is
"""
if isinstance (spec, ru.Url) or \
isinstance (spec, str) :
return self._adaptor.acquire_by_id(spec, ttype=ttype)
else:
# make sure at least 'executable' is defined
if spec.rtype is None:
raise se.BadParameter("resource type undefined in description")
# spec_copy = descr.Description ()
# spec._attributes_deep_copy (spec_copy)
return self._adaptor.acquire(spec, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager', str, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def destroy(self, rid, ttype=None):
"""
destroy(rid)
Destroy / release a resource.
:type rid : string
:param rid : identifies the resource to be released
"""
return self._adaptor.destroy(rid, ttype=ttype)
