class Storage (Resource) :
"""
A Storage resource is a resource which has storage capabilities, i.e. the
ability to persistently store, organize and retrieve data. As such, the
'Access' attribute of the storage resource (a URL) can be used to create
a :class:`saga.filesystem.Directory` instance to manage the resource's data
space.
"""
# --------------------------------------------------------------------------
#
@rus.takes ('StorageResource',
rus.optional (basestring),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, id=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
self._resrc = super (Storage, self)
self._resrc.__init__ (id, session, _adaptor, _adaptor_state, _ttype)
if self.rtype != c.STORAGE :
raise se.BadParameter ("Cannot init Storage resource type %s"
% self.rtype)
class Self (Job) :
# --------------------------------------------------------------------------
#
@rus.takes ('Self',
rus.optional (basestring),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, _method_type='run', _adaptor=None, _adaptor_state={}, _ttype=None) :
self._base = super (Job, self)
self._base.__init__ (_method_type, _adaptor, _adaptor_state, _ttype=_ttype)
class Network (Resource) :
"""
A Network resource is a resource which has network capabilities.
"""
# --------------------------------------------------------------------------
#
@rus.takes ('NetworkResource',
rus.optional (basestring),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, id=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
self._resrc = super (Network, self)
self._resrc.__init__ (id, session, _adaptor, _adaptor_state, _ttype)
if self.rtype != c.NETWORK :
raise se.BadParameter ("Cannot init Network resource type %s"
% self.rtype)
and the resource manager can retract control from the application because
the agreed time duration has passed -- this is represented by the `EXPIRED`
state.
"""
# FIXME:
# - we don't use PENDING like this, yet
# - include state diagram (also for jobs btw)
# --------------------------------------------------------------------------
#
@rus.takes ('Resource',
rus.optional (basestring),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, id=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
"""
__init__(id=None, session=None)
Create / reconnect to a resource.
:param id: id of the resource
:type id: :class:`saga.Url`
:param session: :class:`saga.Session`
Resource class instances are usually created by calling :func:`acquire`
on the :class:`saga.resource.Manager` class. Already acquired resources
control over subsets of those resources (resource slices) to an application.
This :class:`Manager` class represents the contact point to such
ResourceManager instances -- the application can thus acquire compute, data
or network resources, according to some resource specification, for a bound
or unbound amount of time.
"""
# --------------------------------------------------------------------------
#
@rus.takes ('Manager',
rus.optional (basestring, surl.Url),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, url=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
"""
__init__(url)
Create a new Manager instance. Connect to a remote resource management endpoint.
:type url: :class:`saga.Url`
:param url: resource management endpoint
"""
# param checks
_url = surl.Url(url)
scheme = _url.scheme.lower()
class Entry(sb.Base, sasync.Async):
'''
Represents a SAGA namespace entry as defined in GFD.90
The saga.namespace.Entry class represents, as the name indicates,
an entry in some (local or remote) namespace. That class offers
a number of operations on that entry, such as copy, move and remove::
# get an entry handle
entry = saga.namespace.Entry ("sftp://localhost/tmp/data/data.bin")
# copy the entry
entry.copy ("sftp://localhost/tmp/data/data.bak")
# move the entry
entry.move ("sftp://localhost/tmp/data/data.new")
'''
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional((ru.Url, str)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=None,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
:param url: Url of the (remote) entry
:type url: :class:`saga.Url`
flags: flags enum
session: saga.Session
ret: obj
Construct a new entry object
The specified entry is expected to exist -- otherwise a DoesNotExist
exception is raised. Also, the URL must point to an entry (not to
a directory), otherwise a BadParameter exception is raised.
Example::
# get an entry handle
entry = saga.namespace.Entry("sftp://localhost/tmp/data/data.bin")
# print the entry's url
print(entry.get_url ())
'''
self._session = session
self._is_recursive = False # recursion guard (FIXME: NOT THREAD SAFE)
# param checks
if not session:
session = ss.Session(default=True)
if not flags: flags = 0
url = ru.Url(url)
scheme = url.scheme.lower()
self._base = super(Entry, self)
self._base.__init__(scheme,
_adaptor,
_adaptor_state,
url,
flags,
session,
ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('Entry', rus.optional((ru.Url, str)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=None, session=None, ttype=None):
'''
url: saga.Url
flags: saga.namespace.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
# param checks
if not flags: flags = 0
if not session:
session = ss.Session(default=True)
return cls(url, flags, session, _ttype=ttype)._init_task
# ----------------------------------------------------------------
#
# namespace entry methods
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((ru.Url, st.Task))
def get_url(self, ttype=None):
'''
ttype: saga.task.type enum
ret: saga.Url / saga.Task
Return the complete url pointing to the entry.
The call will return the complete url pointing to
this entry as a saga.Url object::
# print URL of an entry
entry = saga.namespace.Entry("sftp://localhost/etc/passwd")
print(entry.get_url())
'''
return self._adaptor.get_url(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((str, st.Task))
def get_cwd(self, ttype=None):
'''
ttype: saga.task.type enum
ret: string / saga.Task
'''
return self._adaptor.get_cwd(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((str, st.Task))
def get_name(self, ttype=None):
'''
ttype: saga.task.type enum
ret: string / saga.Task
'''
return self._adaptor.get_name(ttype=ttype)
# ----------------------------------------------------------------
#
# namespace entry / directory methods
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_dir(self, ttype=None):
'''
ttype: saga.task.type enum
ret: bool / saga.Task
Returns True if path is a directory, False otherwise.
Example::
# inspect an entry
dir = saga.namespace.Directory("sftp://localhost/tmp/")
if dir.is_dir ('data'):
# do something
'''
return self._adaptor.is_dir_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_entry(self, ttype=None):
'''
ttype: saga.task.type enum
ret: bool / saga.Task
'''
return self._adaptor.is_entry_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_link(self, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: bool / saga.Task
'''
return self._adaptor.is_link_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((ru.Url, st.Task))
def read_link(self, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: saga.Url / saga.Task
'''
return self._adaptor.read_link_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', (ru.Url, str), rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def copy(self, tgt, flags=0, ttype=None):
'''
tgt: saga.Url
flags: enum flags
ttype: saga.task.type enum
ret: None / saga.Task
Copy the entry to another location
:param target: Url of the copy target.
:param flags: Flags to use for the operation.
The entry is copied to the given target location. The target URL must
be an absolute path, and can be a target entry name or target
directory name. If the target entry exists, it is overwritten::
# copy an entry
entry = saga.namespace.Entry("sftp://localhost/tmp/data/data.bin")
entry.copy ("sftp://localhost/tmp/data/data.bak")
'''
# parameter checks
if not flags: flags = 0
return self._adaptor.copy_self(tgt, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', (ru.Url, str), rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def link(self, tgt, flags=0, ttype=None):
'''
tgt: saga.Url
flags: enum flags
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
return self._adaptor.link_self(tgt, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', (ru.Url, str), rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def move(self, tgt, flags=0, ttype=None):
'''
:param target: Url of the move target.
:param flags: Flags to use for the operation.
ttype: saga.task.type enum
ret: None / saga.Task
Move the entry to another location
The entry is copied to the given target location. The target URL must
be an absolute path, and can be a target entry name or target
directory name. If the target entry exists, it is overwritten::
# copy an entry
entry = rs.namespace.Directory("sftp://localhost/tmp/data/data.bin")
entry.move ("sftp://localhost/tmp/data/data.bak")
'''
if not flags: flags = 0
return self._adaptor.move_self(tgt, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def remove(self, flags=0, ttype=None):
'''
:param flags: Flags to use for the operation.
ttype: saga.task.type enum
ret: None / saga.Task
Reove the entry.
The entry is removed, and this object instance is then invalid for
further operations.
# remove an entry
entry = rs.namespace.Directory("sftp://localhost/tmp/data/data.bin")
entry.remove ()
'''
if not flags: flags = 0
return self._adaptor.remove_self(flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(float),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def close(self, timeout=None, ttype=None):
'''
timeout: float
ttype: saga.task.type enum
ret: None / saga.Task
'''
return self._adaptor.close(timeout, ttype=ttype)
# --------------------------------------------------------------------------
#
url = property(get_url) # saga.Url
cwd = property(get_cwd) # string
name = property(get_name) # string
class LogicalDirectory(nsdir.Directory, sa.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('LogicalDirectory', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=c.READ,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
__init__(url, flags=READ, session=None)
Create a new Logical Directory instance.
url: saga.Url
flags: flags enum
session: saga.Session
ret: obj
'''
# param checks
if not flags: flags = 0
url = ru.Url(url)
self._nsdirec = super(LogicalDirectory, self)
self._nsdirec.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url, flags=c.READ, session=None, ttype=None):
'''
url: saga.Url
flags: saga.replica.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
if not flags: flags = 0
_nsdirec = super(LogicalDirectory, cls)
return _nsdirec.create(url, flags, session, ttype=ttype)
@rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_file(self, tgt=None, ttype=None):
'''
is_file(tgt=None)
tgt: saga.Url / string
ttype: saga.task.type enum
ret: bool / saga.Task
'''
if tgt: return self._adaptor.is_file(tgt, ttype=ttype)
else: return self._nsdirec.is_entry_self(ttype=ttype)
@rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(('LogicalFile', st.Task))
def open(self, tgt, flags=c.READ, ttype=None):
'''
open(tgt, flags=READ)
tgt: saga.Url
flags: saga.namespace.flags enum
ttype: saga.task.type enum
ret: saga.namespace.Entry / saga.Task
'''
if not flags: flags = 0
tgt_url = ru.Url(tgt)
return self._adaptor.open(tgt_url, flags, ttype=ttype)
# ----------------------------------------------------------------
#
@rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(('LogicalDirectory', st.Task))
def open_dir(self, tgt, flags=c.READ, ttype=None):
'''
open_dir(tgt, flags=READ)
:param tgt: name/path of the directory to open
:param flags: directory creation flags
ttype: saga.task.type enum
ret: saga.namespace.Directory / saga.Task
Open and return a new directoy
The call opens and returns a directory at the given location.
Example::
# create a subdir 'data' in /tmp
dir = saga.namespace.Directory("sftp://localhost/tmp/")
data = dir.open_dir ('data/', saga.namespace.Create)
'''
if not flags: flags = 0
tgt_url = ru.Url(tgt)
return self._adaptor.open_dir(tgt_url, flags, ttype=ttype)
# ----------------------------------------------------------------
#
# replica methods
#
# --------------------------------------------------------------------------
#
@rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def get_size(self, tgt, ttype=None):
'''
get_size(tgt)
tgt: logical file to get size for
ttype: saga.task.type enum
ret: int / saga.Task
Returns the size of the physical file represented by the given logical
file (in bytes)
Example::
# get a logical directory handle
lf = saga.replica.LogicalFile("irods://localhost/tmp/data/")
# print a logical file's size
print lf.get_size ('data.dat')
'''
tgt_url = ru.Url(tgt)
return self._adaptor.get_size(tgt_url, ttype=ttype)
# ----------------------------------------------------------------
#
@rus.takes('LogicalDirectory', rus.optional(basestring),
rus.optional(basestring), rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(ru.Url), st.Task))
def find(self,
name_pattern,
attr_pattern=None,
flags=c.RECURSIVE,
ttype=None):
'''
find(name_pattern, attr_pattern=None, flags=RECURSIVE)
name_pattern: string
attr_pattern: string
flags: flags enum
ttype: saga.task.type enum
ret: list [saga.Url] / saga.Task
'''
if not flags: flags = 0
if attr_pattern:
return self._adaptor.find_replicas(name_pattern,
attr_pattern,
flags,
ttype=ttype)
else:
return self._nsdirec.find(name_pattern, flags, ttype=ttype)
class Directory (nsdir.Directory) :
"""
Represents a (remote) directory.
The saga.filesystem.Directory class represents, as the name indicates,
a directory on some (local or remote) filesystem. That class offers
a number of operations on that directory, such as listing its contents,
copying files, or creating subdirectories::
# get a directory handle
dir = saga.filesystem.Directory("sftp://localhost/tmp/")
# create a subdir
dir.make_dir ("data/")
# list contents of the directory
files = dir.list ()
# copy *.dat files into the subdir
for f in files :
if f ^ '^.*\.dat$' :
dir.copy (f, "sftp://localhost/tmp/data/")
"""
# --------------------------------------------------------------------------
#
@rus.takes ('Directory',
rus.optional ((ru.Url, basestring)),
rus.optional (int, rus.nothing),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, url=None, flags=READ, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
"""
__init__(url, flags=READ, session)
Construct a new directory object
:param url: Url of the (remote) directory
:type url: :class:`saga.Url`
:param flags: :ref:`filesystemflags`
:param session: :class:`saga.Session`
The specified directory is expected to exist -- otherwise
a DoesNotExist exception is raised. Also, the URL must point to
a directory (not to a file), otherwise a BadParameter exception is
raised.
Example::
# open some directory
dir = saga.filesystem.Directory("sftp://localhost/tmp/")
# and list its contents
files = dir.list ()
"""
# param checks
if not flags : flags = 0
url = ru.Url (url)
if not url.schema :
url.schema = 'file'
if not url.host :
url.host = 'localhost'
self._nsdirec = super (Directory, self)
self._nsdirec.__init__ (url, flags, session,
_adaptor, _adaptor_state, _ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes ('Directory',
rus.optional ((ru.Url, basestring)),
rus.optional (int, rus.nothing),
rus.optional (ss.Session),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (st.Task)
def create (cls, url=None, flags=READ, session=None, ttype=None) :
"""
url: saga.Url
flags: saga.replica.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
"""
if not flags : flags = 0
_nsdir = super (Directory, cls)
return _nsdir.create (url, flags, session, ttype=ttype)
# ----------------------------------------------------------------
#
# filesystem directory methods
#
@rus.takes ('Directory',
(ru.Url, basestring),
rus.optional (int, rus.nothing),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (('File', st.Task))
def open (self, path, flags=READ, ttype=None) :
"""
open(path, flags=READ)
Open a file in the directory instance namespace. Returns
a new file object.
:param path: The name/path of the file to open
:type path: str()
:param flags: :ref:`filesystemflags`
"""
if not flags : flags = 0
return self._adaptor.open (path, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes ('Directory',
(ru.Url, basestring),
rus.optional (int, rus.nothing),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (('Directory', st.Task))
def open_dir (self, path, flags=READ, ttype=None) :
"""
open_dir(path, flags=READ)
Open a directory in the directory instance namespace. Returns
a new directory object.
:param path: The name/path of the directory to open
:type path: str()
:param flags: :ref:`filesystemflags`
Example::
# create a subdir 'data' in /tmp
dir = saga.namespace.Directory("sftp://localhost/tmp/")
data = dir.open_dir ('data/', saga.namespace.Create)
"""
if not flags : flags = 0
return self._adaptor.open_dir (path, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes ('Directory',
rus.optional ((ru.Url, basestring)),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((int, st.Task))
def get_size (self, path=None, ttype=None) :
"""
get_size(path=None)
Return the size of the directory itself or the entry pointed to by `path`.
:param path: (Optional) name/path of an entry
:type path: str()
Returns the size of a file or directory (in bytes)
Example::
# inspect a file for its size
dir = saga.filesystem.Directory("sftp://localhost/tmp/")
size = dir.get_size ('data/data.bin')
print size
"""
if path : return self._adaptor.get_size (path, ttype=ttype)
else : return self._adaptor.get_size_self ( ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes ('Directory',
rus.optional (bool))
@rus.returns (st.Task)
def close (self, kill=True, ttype=None) :
'''
kill : bool
ttype: saga.task.type enum
ret: string / bytearray / saga.Task
'''
return self._adaptor.close ()
# --------------------------------------------------------------------------
#
@rus.takes ('Directory',
rus.optional ((ru.Url, basestring)),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((bool, st.Task))
def is_file (self, path=None, ttype=None) :
"""
is_file(path=None)
Returns `True` if entry points to a file, `False` otherwise. If `path`
is not none, the entry pointed to by `path` is inspected instead of the
directory object itself.
:param path: (Optional) name/path of an entry
:type path: str()
"""
if path : return self._adaptor.is_file (path, ttype=ttype)
else : return self._adaptor.is_file_self ( ttype=ttype)
size = property (get_size) # int
class Job(sb.Base, st.Task, sasync.Async):
'''Represents a SAGA job as defined in GFD.90
A 'Job' represents a running application instance, which may consist of one
or more processes. Jobs are created by submitting a Job description to
a Job submission system -- usually a queuing system, or some other service
which spawns jobs on the user's behalf.
Jobs have a unique ID (see get_job_id()), and are stateful entities -- their
'state' attribute changes according to a well defined state model:
A job as returned by job.Service.create(jd) is in 'New' state -- it is not
yet submitted to the job submission backend. Once it was submitted, via
run(), it will enter the 'Pending' state, where it waits to get actually
executed by the backend (e.g. waiting in a queue etc). Once the job is
actually executed, it enters the 'Running' state -- only in that state is
the job actually consuming resources (CPU, memory, ...).
Jobs can leave the 'Running' state in three different ways: they finish
successfully on their own ('Done'), they finish unsuccessfully on their own,
or get canceled by the job management backend ('Failed'), or they get
actively canceled by the user or the application ('Canceled').
The methods defined on the Job object serve two purposes: inspecting the
job's state, and initiating job state transitions.
'''
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(basestring), rus.optional(sab.Base),
rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
_method_type='run',
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
_adaptor`` references the adaptor class instance which created this task
instance.
The ``_method_type`` parameter is flattened into the job constructor to
satisfy the bulk optimization properties of the saga.Task class, whose
interface is implemented by saga.job.Job.
``_method_type`` specifies the SAGA API method which task is
representing. For jobs, that is the 'run' method.
We don't have a create classmethod -- jobs are never constructed by the user
'''
if not _adaptor:
raise se.IncorrectState("saga.job.Job constructor is private")
self._valid = False
# we need to keep _method_type around, for the task interface (see
# :class:`saga.Task`)
self._method_type = _method_type
# We need to specify a schema for adaptor selection -- and
# simply choose the first one the adaptor offers.
schema = _adaptor.get_schemas()[0]
if 'job_schema' in _adaptor_state:
schema = _adaptor_state['job_schema']
self._base = super(Job, self)
self._base.__init__(schema, _adaptor, _adaptor_state, ttype=None)
# set attribute interface properties
self._attributes_allow_private(True)
self._attributes_extensible(False)
self._attributes_camelcasing(True)
# register properties with the attribute interface
self._attributes_register(STATE, UNKNOWN, sa.ENUM, sa.SCALAR,
sa.READONLY)
self._attributes_register(EXIT_CODE, None, sa.INT, sa.SCALAR,
sa.READONLY)
self._attributes_register(CREATED, None, sa.INT, sa.SCALAR,
sa.READONLY)
self._attributes_register(STARTED, None, sa.INT, sa.SCALAR,
sa.READONLY)
self._attributes_register(FINISHED, None, sa.INT, sa.SCALAR,
sa.READONLY)
self._attributes_register(EXECUTION_HOSTS, None, sa.STRING, sa.VECTOR,
sa.READONLY)
self._attributes_register(ID, None, sa.STRING, sa.SCALAR, sa.READONLY)
self._attributes_register(SERVICE_URL, None, sa.URL, sa.SCALAR,
sa.READONLY)
self._attributes_set_enums(STATE, [
UNKNOWN, NEW, PENDING, RUNNING, DONE, FAILED, CANCELED, SUSPENDED
])
self._attributes_set_getter(STATE, self.get_state)
self._attributes_set_getter(ID, self.get_id)
self._attributes_set_getter(EXIT_CODE, self._get_exit_code)
self._attributes_set_getter(CREATED, self._get_created)
self._attributes_set_getter(STARTED, self._get_started)
self._attributes_set_getter(FINISHED, self._get_finished)
self._attributes_set_getter(EXECUTION_HOSTS, self._get_execution_hosts)
self._attributes_set_getter(SERVICE_URL, self._get_service_url)
self._valid = True
# --------------------------------------------------------------------------
#
@rus.takes('Job')
@rus.returns(basestring)
def __str__(self):
"""
__str__()
String representation. Returns the job's ID.
"""
if not self._valid:
return 'no job id'
return str(self.id)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, basestring, st.Task))
def get_id(self, ttype=None):
"""
get_id()
Return the job ID.
"""
id = self._adaptor.get_id(ttype=ttype)
return id
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def get_description(self, ttype=None):
"""
get_description()
Return the job description this job was created from.
The returned description can be used to inspect job properties
(executable name, arguments, etc.). It can also be used to start
identical job instances.
The returned job description will in general reflect the actual state of
the running job, and is not necessarily a simple copy of the job
description which was used to create the job instance. For example, the
environment variables in the returned job description may reflect the
actual environment of the running job instance.
**Example**::
service = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j1 = service.create_job(jd)
j1.run()
j2 = service.create_job(j1.get_description())
j2.run()
service.close()
"""
return self._adaptor.get_description(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((file, st.Task))
def get_stdin(self, ttype=None):
"""
get_stdin()
Return the job's STDIN handle.
"""
return self._adaptor.get_stdin(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((file, st.Task))
def get_stdout(self, ttype=None):
"""
get_stdout()
Return the job's STDOUT handle.
"""
return self._adaptor.get_stdout(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((str, st.Task))
def get_stdout_string(self, ttype=None):
"""
get_stdout_string()
Return the job's STDOUT.
"""
return self._adaptor.get_stdout_string(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((file, st.Task))
def get_stderr(self, ttype=None):
"""
get_stderr()
Return the job's STDERR handle.
ttype: saga.task.type enum
ret: File / saga.Task
"""
return self._adaptor.get_stderr(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((str, st.Task))
def get_stderr_string(self, ttype=None):
"""
get_stderr_string()
Return the job's STDERR.
ttype: saga.task.type enum
ret: string.
"""
return self._adaptor.get_stderr_string(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def suspend(self, ttype=None):
"""
suspend()
Suspend the job.
"""
return self._adaptor.suspend(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def resume(self, ttype=None):
"""
resume()
Resume the job.
"""
return self._adaptor.resume(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def checkpoint(self, ttype=None):
"""
checkpoint()
Checkpoint the job.
"""
return self._adaptor.checkpoint(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', descr.Description,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def migrate(self, jd, ttype=None):
"""
jd: saga.job.Description
ttype: saga.task.type enum
ret: None / saga.Task
"""
return self._adaptor.migrate(jd, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', int, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def signal(self, signum, ttype=None):
"""
signal(signum)
Send a signal to the job.
:param signum: signal to send
:type signum: int
"""
return self._adaptor.signal(signum, ttype=ttype)
id = property(get_id) # string
description = property(get_description) # Description
#stdin = property (get_stdin) # File
stdout = property(get_stdout) # File
stderr = property(get_stderr) # File
#-----------------------------------------------------------------
#
# task methods flattened into job :-/
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def run(self, ttype=None):
"""
run()
Run (start) the job.
Request that the job is being executed by the backend. If the backend
is accepting this run request, the job will move to the 'Pending' or
'Running' state -- otherwise this method will raise an error, and the
job will be moved to 'Failed'.
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
if j.get_state() == saga.job.NEW :
print "new"
else :
print "oops!"
j.run()
if j.get_state() == saga.job.PENDING :
print "pending"
elif j.get_state() == saga.job.RUNNING :
print "running"
else :
print "oops!"
"""
return self._adaptor.run(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', float, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def cancel(self, timeout=None, ttype=None):
"""
cancel(timeout)
Cancel the execution of the job.
:param timeout: `cancel` will return after timeout
:type timeout: float
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
if j.get_state() == saga.job.NEW :
print "new"
else :
print "oops!"
j.run()
if j.get_state() == saga.job.PENDING :
print "pending"
elif j.get_state() == saga.job.RUNNING :
print "running"
else :
print "oops!"
j.cancel()
if j.get_state() == saga.job.CANCELED :
print "canceled"
else :
print "oops!"
"""
return self._adaptor.cancel(timeout, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(float),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def wait(self, timeout=None, ttype=None):
"""
wait(timeout)
:param timeout: `wait` will return after timeout
:type timeout: float
Wait for a running job to finish execution.
The optional timeout parameter specifies the time to wait, and accepts
the following values::
timeout < 0 : wait forever (block) -- same for 'None'
timeout == 0 : wait not at all (non-blocking test)
timeout > 0 : wait for 'timeout' seconds
On a non-negative timeout, the call can thus return even if the job is
not in final state, and the application should check the actual job
state. The default timeout value is 'None' (blocking).
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
if j.get_state() == saga.job.NEW :
print "new"
else :
print "oops!"
j.run()
if j.get_state() == saga.job.PENDING :
print "pending"
elif j.get_state() == saga.job.RUNNING :
print "running"
else :
print "oops!"
j.wait(-1.0)
if j.get_state() == saga.job.DONE :
print "done"
elif j.get_state() == saga.job.FAILED :
print "failed"
else :
print "oops!"
"""
if None == timeout:
timeout = -1.0 # FIXME
return self._adaptor.wait(timeout, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.one_of(UNKNOWN, NEW, PENDING, RUNNING, SUSPENDED, DONE,
FAILED, CANCELED), st.Task))
def get_state(self, ttype=None):
"""
get_state()
Return the current state of the job.
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
if j.get_state() == saga.job.NEW :
print "new"
else :
print "oops!"
j.run()
if j.get_state() == saga.job.PENDING :
print "pending"
elif j.get_state() == saga.job.RUNNING :
print "running"
else :
print "oops!"
"""
return self._adaptor.get_state(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.anything, st.Task))
def get_result(self, ttype=None):
"""
get_result()
"""
return self._adaptor.get_result(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((sb.Base, st.Task))
def get_object(self, ttype=None):
""" :todo: describe me
:note: this will return the job_service which created the job.
"""
return self._adaptor.get_object(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((se.SagaException, st.Task))
def get_exception(self, ttype=None):
""" :todo: describe me
:note: if job failed, that will get an exception describing
why, if that exists. Otherwise, the call returns None.
"""
# FIXME: add CPI
return self._adaptor.get_exception(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job')
@rus.returns(rus.nothing)
def re_raise(self):
""" :todo: describe me
:note: if job failed, that will re-raise an exception describing
why, if that exists. Otherwise, the call does nothing.
"""
self._adaptor.re_raise()
# ----------------------------------------------------------------
#
# attribute getters
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, int, st.Task))
def _get_exit_code(self, ttype=None):
ec = self._adaptor.get_exit_code(ttype=ttype)
if ec in [None, ""]:
return None
else:
# Exit code is always an int. If this 'cast' fails,
# the adaptor is doing something stupid.
return ec
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, float, st.Task))
def _get_created(self, ttype=None):
return self._adaptor.get_created(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, float, st.Task))
def _get_started(self, ttype=None):
return self._adaptor.get_started(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, float, st.Task))
def _get_finished(self, ttype=None):
return self._adaptor.get_finished(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, rus.list_of(basestring), st.Task))
def _get_execution_hosts(self, ttype=None):
return self._adaptor.get_execution_hosts(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, surl.Url, st.Task))
def _get_service_url(self, ttype=None):
return self._adaptor.get_service_url(ttype=ttype)
state = property(get_state) # state enum
result = property(get_result) # result type (None)
object = property(get_object) # object type (job_service)
exception = property(re_raise) # exception type
# ----------------------------------------------------------------
#
ExitCode = property(doc="""
The job's exitcode.
this attribute is only meaningful if the job is in 'Done' or 'Final'
state - for all other job states, this attribute value is undefined.
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
j.run()
j.wait()
if j.get_state() == saga.job.FAILED :
if j.exitcode == "42" :
print "Ah, galaxy bypass error!"
else :
print "oops!"
""")
# ----------------------------------------------------------------
#
JobID = property(doc="""
The job's identifier.
This attribute is equivalent to the value returned by job.get_job_id()
""")
# ----------------------------------------------------------------
#
ServiceURL = property(doc="""
The URL of the :class:`saga.job.Service` instance managing this job.
This attribute is represents the URL under where the job management
service can be contacted which owns the job. The value is equivalent to
the service part of the job_id.
**Example**::
js = saga.job.Service("fork://localhost")
jd = saga.job.Description ()
jd.executable = '/bin/date'
j = js.create_job(jd)
if j.serviceurl == "fork://localhost" :
print "yes!"
else :
print "oops!"
""")
state.
"""
# FIXME:
# - we don't use PENDING like this, yet
# - include state diagram (also for jobs btw)
# --------------------------------------------------------------------------
#
@rus.takes(
"Resource",
rus.optional(basestring),
rus.optional(ss.Session),
rus.optional(sab.Base),
rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)),
)
@rus.returns(rus.nothing)
def __init__(self, id=None, session=None, _adaptor=None, _adaptor_state={}, _ttype=None):
"""
__init__(id=None, session=None)
Create / reconnect to a resource.
:param id: id of the resource
:type id: :class:`saga.Url`
:param session: :class:`saga.Session`
Resource class instances are usually created by calling :func:`acquire`
on the :class:`saga.resource.Manager` class. Already acquired resources
class Directory(entry.Entry):
'''
Represents a SAGA directory as defined in GFD.90
The saga.namespace.Directory class represents, as the name indicates,
a directory on some (local or remote) namespace. That class offers
a number of operations on that directory, such as listing its contents,
copying entries, or creating subdirectories::
# get a directory handle
dir = saga.namespace.Directory("sftp://localhost/tmp/")
# create a subdir
dir.make_dir ("data/")
# list contents of the directory
entries = dir.list ()
# copy *.dat entries into the subdir
for f in entries :
if f ^ '^.*\.dat$' :
dir.copy (f, "sftp://localhost/tmp/data/")
Implementation note:
^^^^^^^^^^^^^^^^^^^^
The SAGA API Specification (GFD.90) prescribes method overloading on method
signatures, but that is not supported by Python (Python only does method
overwriting). So we implement one generic method version here, and do the
case switching based on the provided parameter set.
'''
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=None,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
:param url: Url of the (remote) entry system directory.
:type url: :class:`saga.Url`
flags: flags enum
session: saga.Session
ret: obj
Construct a new directory object
The specified directory is expected to exist -- otherwise
a DoesNotExist exception is raised. Also, the URL must point to
a directory (not to an entry), otherwise a BadParameter exception is
raised.
Example::
# open some directory
dir = saga.namespace.Directory("sftp://localhost/tmp/")
# and list its contents
entries = dir.list ()
'''
if not flags: flags = 0
self._nsentry = super(Directory, self)
self._nsentry.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=None, session=None, ttype=None):
'''
url: saga.Url
flags: saga.namespace.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
if not flags: flags = 0
_nsentry = super(Directory, cls)
return _nsentry.create(url, flags, session, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((entry.Entry, st.Task))
def open(self, name, flags=None, ttype=None):
'''
name: saga.Url
flags: saga.namespace.flags enum
ttype: saga.task.type enum
ret: saga.namespace.Entry / saga.Task
'''
if not flags: flags = 0
url = ru.Url(name)
return self._adaptor.open(url, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(('Directory', st.Task))
def open_dir(self, path, flags=None, ttype=None):
'''
:param path: name/path of the directory to open
:param flags: directory creation flags
ttype: saga.task.type enum
ret: saga.namespace.Directory / saga.Task
Open and return a new directoy
The call opens and returns a directory at the given location.
Example::
# create a subdir 'data' in /tmp
dir = saga.namespace.Directory("sftp://localhost/tmp/")
data = dir.open_dir ('data/', saga.namespace.Create)
'''
if not flags: flags = 0
return self._adaptor.open_dir(ru.Url(path), flags, ttype=ttype)
# ----------------------------------------------------------------
#
# namespace directory methods
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def make_dir(self, tgt, flags=0, ttype=None):
'''
:param tgt: name/path of the new directory
:param flags: directory creation flags
ttype: saga.task.type enum
ret: None / saga.Task
Create a new directoy
The call creates a directory at the given location.
Example::
# create a subdir 'data' in /tmp
dir = saga.namespace.Directory("sftp://localhost/tmp/")
dir.make_dir ('data/')
'''
if not flags: flags = 0
return self._adaptor.make_dir(ru.Url(tgt), flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def change_dir(self, url, flags=0, ttype=None):
'''
url: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
return self._adaptor.change_dir(url, flags=flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional(basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(ru.Url), st.Task))
def list(self, pattern=None, flags=0, ttype=None):
'''
:param pattern: Entry name pattern (like POSIX 'ls', e.g. '\*.txt')
flags: flags enum
ttype: saga.task.type enum
ret: list [saga.Url] / saga.Task
List the directory's content
The call will return a list of entries and subdirectories within the
directory::
# list contents of the directory
for f in dir.list() :
print f
'''
if not flags: flags = 0
return self._adaptor.list(pattern, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def exists(self, path, ttype=None):
'''
:param path: path of the entry to check
ttype: saga.task.type enum
ret: bool / saga.Task
Returns True if path exists, False otherwise.
Example::
# inspect an entry
dir = saga.namespace.Directory("sftp://localhost/tmp/")
if dir.exists ('data'):
# do something
'''
return self._adaptor.exists(path, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional(basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(ru.Url), st.Task))
def find(self, pattern, flags=c.RECURSIVE, ttype=None):
'''
pattern: string
flags: flags enum
ttype: saga.task.type enum
ret: list [saga.Url] / saga.Task
'''
if not flags: flags = 0
return self._adaptor.find(pattern, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def get_num_entries(self, ttype=None):
'''
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.get_num_entries(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', int, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((ru.Url, st.Task))
def get_entry(self, num, ttype=None):
'''
num: int
ttype: saga.task.type enum
ret: saga.Url / saga.Task
'''
return self._adaptor.get_entry(num, ttype=ttype)
# ----------------------------------------------------------------
#
# methods overloaded from namespace.Entry
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def copy(self, url_1, url_2=None, flags=0, ttype=None):
'''
:param src: path of the entry to copy
:param tgt: absolute URL of target name or directory
url_1: saga.Url
url_2: saga.Url / None
flags: flags enum / None
ttype: saga.task.type enum / None
ret: None / saga.Task
Copy an entry from source to target
The source is copied to the given target directory. The path of the
source can be relative::
# copy an entry
dir = saga.namespace.Directory("sftp://localhost/tmp/")
dir.copy ("./data.bin", "sftp://localhost/tmp/data/")
'''
# FIXME: re-implement the url switching (commented out below)
if not flags: flags = 0
if url_2: return self._adaptor.copy(url_1, url_2, flags, ttype=ttype)
else: return self._nsentry.copy(url_1, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def link(self, url_1, url_2=None, flags=0, ttype=None):
'''
src: saga.Url
tgt: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
if url_2: return self._adaptor.link(url_1, url_2, flags, ttype=ttype)
else: return self._nsentry.link(url_1, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def move(self, url_1, url_2=None, flags=0, ttype=None):
'''
:param src: path of the entry to copy
:param tgt: absolute URL of target directory
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
Move an entry from source to target
The source is moved to the given target directory. The path of the
source can be relative::
# copy an entry
dir = saga.namespace.Directory("sftp://localhost/tmp/")
dir.move ("./data.bin", "sftp://localhost/tmp/data/")
'''
if not flags: flags = 0
if url_2: return self._adaptor.move(url_1, url_2, flags, ttype=ttype)
else: return self._nsentry.move(url_1, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def remove(self, tgt=None, flags=0, ttype=None):
'''
tgt: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
if tgt: return self._adaptor.remove(tgt, flags, ttype=ttype)
else: return self._nsentry.remove(flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_dir(self, tgt=None, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: bool / saga.Task
Returns True if path is a directory, False otherwise.
Example::
# inspect an entry
dir = saga.namespace.Directory("sftp://localhost/tmp/")
if dir.is_dir ('data'):
# do something
'''
if tgt: return self._adaptor.is_dir(tgt, ttype=ttype)
else: return self._nsentry.is_dir(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_entry(self, tgt=None, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: bool / saga.Task
'''
if tgt: return self._adaptor.is_entry(tgt, ttype=ttype)
else: return self._nsentry.is_entry(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_link(self, tgt=None, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: bool / saga.Task
'''
if tgt: return self._adaptor.is_link(tgt, ttype=ttype)
else: return self._nsentry.is_link(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((ru.Url, st.Task))
def read_link(self, tgt=None, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: saga.Url / saga.Task
'''
if tgt: return self._adaptor.read_link(tgt, ttype=ttype)
else: return self._nsentry.read_link(ttype=ttype)
class Container(sbase.SimpleBase, satt.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(rus.nothing)
def __init__(self):
self._base = super(Container, self)
self._base.__init__()
# set attribute interface properties
self._attributes_allow_private(True)
self._attributes_extensible(False)
self._attributes_camelcasing(True)
# register properties with the attribute interface
self._attributes_register(SIZE, 0, satt.INT, satt.SCALAR,
satt.READONLY)
self._attributes_set_getter(SIZE, self.get_size)
self._attributes_register(TASKS, [], satt.ANY, satt.VECTOR,
satt.READONLY)
self._attributes_set_getter(TASKS, self.get_tasks)
self._attributes_register(STATES, [], satt.ENUM, satt.VECTOR,
satt.READONLY)
self._attributes_set_getter(STATES, self.get_states)
self._attributes_set_enums(
STATES, [UNKNOWN, NEW, RUNNING, DONE, FAILED, CANCELED])
# cache for created container instances
self._containers = {}
# --------------------------------------------------------------------------
#
def __str__(self):
ret = "["
for task in self.tasks:
ret += "'%s', " % str(task)
ret += "]"
return ret
# --------------------------------------------------------------------------
#
@rus.takes('Container', Task)
@rus.returns(rus.nothing)
def add(self, task):
if not isinstance(task, Task):
raise se.BadParameter ("Container handles tasks, not %s" \
% (type(task)))
if not task in self.tasks:
self.tasks.append(task)
# --------------------------------------------------------------------------
#
@rus.takes('Container', Task)
@rus.returns(rus.nothing)
def remove(self, task):
if task in self.tasks:
self.tasks.delete(task)
# --------------------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(rus.nothing)
def run(self):
if not len(self.tasks):
# nothing to do
return None
buckets = self._get_buckets()
futures = [] # futures running container ops
# handle all container
for c in buckets['bound']:
# handle all methods
for m in buckets['bound'][c]:
tasks = buckets['bound'][c][m]
m_name = "container_%s" % m
m_handle = None
for (name,
handle) in inspect.getmembers(c,
predicate=inspect.ismethod):
if name == m_name:
m_handle = handle
break
if not handle:
# Hmm, the specified container can't handle the call after
# all -- fall back to the unbound handling
buckets['unbound'] += tasks
else:
# hand off to the container function, in a separate task
futures.append(ru.Future.Run(m_handle, tasks))
# handle tasks not bound to a container
for task in buckets['unbound']:
futures.append(ru.Future.Run(task.run))
# wait for all futures to finish
for future in futures:
if future.isAlive():
future.join()
if future.state == FAILED:
raise se.NoSuccess ("future exception: %s" \
% (future.exception))
# --------------------------------------------------------------------------
#
# --------------------------------------------------------------------------
#
@rus.takes('Container', rus.one_of(ANY, ALL), rus.optional(float))
@rus.returns(rus.list_of(Task))
def wait(self, mode=ALL, timeout=None):
if None == timeout:
timeout = -1.0 # FIXME
if not mode in [ANY, ALL]:
raise se.BadParameter(
"wait mode must be saga.task.ANY or saga.task.ALL")
if type(timeout) not in [int, long, float]:
raise se.BadParameter(
"wait timeout must be a floating point number (or integer)")
if not len(self.tasks):
# nothing to do
return None
if mode == ALL:
return self._wait_all(timeout)
else:
return self._wait_any(timeout)
# --------------------------------------------------------------------------
#
@rus.takes('Container', float)
@rus.returns(rus.list_of(Task))
def _wait_any(self, timeout):
buckets = self._get_buckets()
futures = [] # futures running container ops
# handle all tasks bound to containers
for c in buckets['bound']:
# handle all methods -- all go to the same 'container_wait' though)
tasks = []
for m in buckets['bound'][c]:
tasks += buckets['bound'][c][m]
futures.append(ru.Future.Run(c.container_wait, tasks, ANY,
timeout))
# handle all tasks not bound to containers
for task in buckets['unbound']:
futures.append(ru.Future.Run(task.wait, timeout))
# mode == ANY: we need to watch our futures, and whenever one
# returns, and declare success. Note that we still need to get the
# finished task from the 'winner'-future -- we do that via a Queue
# object. Note also that looser futures are not canceled, but left
# running (FIXME: consider sending a signal at least)
timeout = 0.01 # seconds, heuristic :-/
for future in futures:
future.join(timeout)
if future.state == FAILED:
raise future.exception
if not future.isAlive():
# future indeed finished -- dig return value from this
# futures queue
result = future.result
# ignore other futures, and simply declare success
return result
# --------------------------------------------------------------------------
#
@rus.takes('Container', float)
@rus.returns(rus.list_of(Task))
def _wait_all(self, timeout):
# this method should actually be symmetric to _wait_any, and could
# almost be mapped to it, but the code below is a kind of optimization
# (does not need futures, thus simpler code).
buckets = self._get_buckets()
ret = None
# handle all tasks bound to containers
for c in buckets['bound']:
# handle all methods -- all go to the same 'container_wait' though)
tasks = []
for m in buckets['bound'][c]:
tasks += buckets['bound'][c][m]
# TODO: this is semantically not correct: timeout is applied
# n times...
c.container_wait(tasks, ALL, timeout)
ret = tasks[0]
# handle all tasks not bound to containers
for task in buckets['unbound']:
task.wait()
ret = task
# all done - return random task (first from last container, or last
# unbound task)
# FIXME: that task should be removed from the task container
return ret
# --------------------------------------------------------------------------
#
@rus.takes('Container', rus.optional(float))
@rus.returns(rus.nothing)
def cancel(self, timeout=None):
if None == timeout:
timeout = -1.0 # FIXME
buckets = self._get_buckets()
futures = [] # futures running container ops
# handle all tasks bound to containers
for c in buckets['bound']:
# handle all methods -- all go to the same 'container_cancel' though)
tasks = []
for m in buckets['bound'][c]:
tasks += buckets['bound'][c][m]
futures.append(ru.Future.Run(c.container_cancel, tasks, timeout))
# handle all tasks not bound to containers
for task in buckets['unbound']:
futures.append(ru.Future.Run(task.cancel, timeout))
for future in futures:
future.join()
# ----------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(int)
def get_size(self):
return len(self.tasks)
# --------------------------------------------------------------------------
#
@rus.takes('Container', 'basestring')
@rus.returns(Task)
def get_task(self, id):
# FIXME: this should not be a search, but a lookup
if not id:
raise se.NoSuccess("Lookup requires non-empty id (not '%s')" % id)
for t in self.tasks:
if t.id == id:
return t
raise se.NoSuccess("task '%s' not found in container" % id)
# --------------------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(rus.list_of(Task))
def get_tasks(self):
return self.tasks
# --------------------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(
rus.list_of(rus.one_of(UNKNOWN, NEW, RUNNING, DONE, FAILED, CANCELED)))
def get_states(self):
buckets = self._get_buckets()
futures = [] # futures running container ops
# handle all tasks bound to containers
for c in buckets['bound']:
# handle all methods -- all go to the same 'container_get_states' though)
tasks = []
for m in buckets['bound'][c]:
tasks += buckets['bound'][c][m]
futures.append(ru.Future.Run(c.container_get_states, tasks))
# handle all tasks not bound to containers
for task in buckets['unbound']:
futures.append(ru.Future.Run(task.get_state))
# We still need to get the states from all futures.
# FIXME: order
states = []
for future in futures:
future.join()
if future.state == FAILED:
raise future.exception
# FIXME: what about ordering tasks / states?
res = future.result
if res != None:
states += res
return states
# ----------------------------------------------------------------
#
@rus.takes('Container')
@rus.returns(dict)
def _get_buckets(self):
# collective container ops: walk through the task list, and sort into
# buckets of tasks which have (a) the same task._container, or if that
# is not set, the same class type (for which one container instance is
# created). All tasks were neither is available are handled one-by-one
buckets = {}
buckets['unbound'] = [] # no container adaptor for these [tasks]
buckets['bound'] = {} # dict of container adaptors [tasks]
for task in self.tasks:
if task._adaptor and task._adaptor._container:
# the task's adaptor has a valid associated container class
# which can handle the container ops - great!
c = task._adaptor._container
m = task._method_type
if not c in buckets['bound']:
buckets['bound'][c] = {}
if not m in buckets['bound'][c]:
buckets['bound'][c][m] = []
buckets['bound'][c][m].append(task)
else:
# we have no container to handle this task -- so
# put it into the fallback list
buckets['unbound'].append(task)
return buckets
and the resource manager can retract control from the application because
the agreed time duration has passed -- this is represented by the `EXPIRED`
state.
"""
# FIXME:
# - we don't use PENDING like this, yet
# - include state diagram (also for jobs btw)
# --------------------------------------------------------------------------
#
@rus.takes ('Resource',
rus.optional (basestring),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, id=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
"""
__init__(id=None, session=None)
Create / reconnect to a resource.
:param id: id of the resource
:type id: :class:`saga.Url`
:param session: :class:`saga.Session`
Resource class instances are usually created by calling :func:`acquire`
on the :class:`saga.resource.Manager` class. Already acquired resources
application.
This :class:`Manager` class represents the contact point to such
ResourceManager instances -- the application can thus acquire compute, data
or network resources, according to some resource specification, for a bound
or unbound amount of time.
"""
# --------------------------------------------------------------------------
#
@rus.takes ('Manager',
rus.optional (basestring, ru.Url),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, url=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
"""
__init__(url)
Create a new Manager instance. Connect to a remote resource management endpoint.
:type url: :class:`saga.Url`
:param url: resource management endpoint
"""
# param checks
_url = ru.Url(url)
scheme = _url.scheme.lower()
and the resource manager can retract control from the application because
the agreed time duration has passed -- this is represented by the `EXPIRED`
state.
"""
# FIXME:
# - we don't use PENDING like this, yet
# - include state diagram (also for jobs btw)
# --------------------------------------------------------------------------
#
@rus.takes ('Resource',
rus.optional (basestring),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, id=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
"""
__init__(id=None, session=None)
Create / reconnect to a resource.
:param id: id of the resource
:type id: :class:`saga.Url`
:param session: :class:`saga.Session`
Resource class instances are usually created by calling :func:`acquire`
on the :class:`saga.resource.Manager` class. Already acquired resources
class Entry(nsentry.Entry, sa.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=READ,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
url: saga.Url
flags: flags enum
session: saga.Session
ret: obj
'''
# param checks
url = ru.Url(url)
self._nsentry = super(Entry, self)
self._nsentry.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# set attribute interface properties
self._attributes_allow_private(True)
self._attributes_camelcasing(True)
self._attributes_extensible(True,
getter=self._attribute_getter,
setter=self._attribute_setter,
lister=self._attribute_lister,
caller=self._attribute_caller)
# register properties with the attribute interface
self._attributes_register(ATTRIBUTE, None, sa.STRING, sa.SCALAR,
sa.READONLY)
self._attributes_register(OBJECT, None, sa.ANY, sa.SCALAR, sa.READONLY)
self._attributes_register(TTL, None, sa.INT, sa.SCALAR, sa.WRITEABLE)
self._attributes_set_setter(TTL, self.set_ttl)
self._attributes_set_getter(TTL, self.get_ttl)
self._attributes_set_setter(OBJECT, self.store_object)
self._attributes_set_getter(OBJECT, self.retrieve_object)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('Entry', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=READ, session=None, ttype=None):
'''
url: saga.Url
flags: saga.advert.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
if not flags: flags = 0
_nsentry = super(Entry, cls)
return _nsentry.create(url, flags, session, ttype=ttype)
# --------------------------------------------------------------------------
#
# attribute methods
#
# NOTE: we do not yet pass ttype, as async calls are not yet supported by
# the attribute interface
#
@rus.takes('Entry', basestring, rus.optional(rus.one_of(SYNC, ASYNC,
TASK)))
@rus.returns((rus.anything, st.Task))
def _attribute_getter(self, key, ttype=None):
return self._adaptor.attribute_getter(key)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', basestring, rus.anything,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def _attribute_setter(self, key, val, ttype=None):
return self._adaptor.attribute_setter(key, val)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(rus.anything), st.Task))
def _attribute_lister(self, ttype=None):
return self._adaptor.attribute_lister()
# --------------------------------------------------------------------------
#
@rus.takes('Entry', basestring, int, callable,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.anything, st.Task))
def _attribute_caller(self, key, id, cb, ttype=None):
return self._adaptor.attribute_caller(key, id, cb)
# --------------------------------------------------------------------------
#
# advert methods
#
@rus.takes('Entry', float, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def set_ttl(self, ttl=-1.0, ttype=None):
"""
ttl : int
ttype: saga.task.type enum
ret: None / saga.Task
"""
return self._adaptor.set_ttl(ttl, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((float, st.Task))
def get_ttl(self, ttype=None):
"""
ttype: saga.task.type enum
ret: int / saga.Task
"""
return self._adaptor.get_ttl(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', object, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def store_object(self, object, ttype=None):
"""
object : <object type>
ttype: saga.task.type enum
ret: None / saga.Task
"""
return self._adaptor.store_object(object, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((object, st.Task))
def retrieve_object(self, ttype=None):
"""
ttype: saga.task.type enum
ret: any / saga.Task
"""
return self._adaptor.retrieve_object(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def delete_object(self, ttype=None):
"""
ttype: saga.task.type enum
ret: None / saga.Task
"""
return self._adaptor.delete_object(ttype=ttype)
class Monitorable(sa.Attributes):
# --------------------------------------------------------------------------
#
def __init__(self):
self._attr = super(Monitorable, self)
self._attr.__init__()
# --------------------------------------------------------------------------
#
# since we have no means to call the saga.Base constructor explicitly (we
# don't inherit from it), we have to rely that the classes which implement
# the Monitorable interface are correctly calling the Base constructure --
# otherwise we won't have an self._adaptor to talk to...
#
# This helper method checks the existence of self._adaptor, and should be
# used before each call forwarding.
#
def _check(self):
if not hasattr(self, '_adaptor'):
raise se.IncorrectState("object is not fully initialized")
# --------------------------------------------------------------------------
#
@rus.takes('Monitorable')
@rus.returns(rus.list_of(basestring))
def list_metrics(self):
self._check()
return self._adaptor.list_metrics()
# --------------------------------------------------------------------------
#
# Metrics are not implemented in RADICAL-SAGA
#
# @rus.takes ('Monitorable', basestring)
# @rus.returns ('Metric')
# def get_metric (name) :
#
# self._check ()
# return self._adaptor.get_metric (name)
# --------------------------------------------------------------------------
#
@rus.takes('Monitorable', basestring, rus.one_of('saga.Callback',
callable))
@rus.returns(int)
def add_callback(self, name, cb):
self._check()
return self._adaptor.add_callback(name, cb)
# --------------------------------------------------------------------------
#
@rus.takes('Monitorable', int)
@rus.returns(rus.nothing)
def remove_callback(self, cookie):
self._check()
return self._adaptor.remove_callback(cookie)
class Manager(sb.Base, sasync.Async):
"""
In the context of RADICAL-SAGA, a *ResourceManager* is a service which
asserts control over a set of resources. That manager can, on request,
render control over subsets of those resources (resource slices) to an
application.
This :class:`Manager` class represents the contact point to such
ResourceManager instances -- the application can thus acquire compute, data
or network resources, according to some resource specification, for a bound
or unbound amount of time.
"""
# --------------------------------------------------------------------------
#
@rus.takes('Manager', rus.optional(str, ru.Url), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
"""
__init__(url)
Create a new Manager instance. Connect to a remote resource management endpoint.
:type url: :class:`saga.Url`
:param url: resource management endpoint
"""
# param checks
_url = ru.Url(url)
scheme = _url.scheme.lower()
if not session:
session = ss.Session(default=True)
self._base = super(Manager, self)
self._base.__init__(scheme,
_adaptor,
_adaptor_state,
_url,
session,
ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('Manager', rus.optional((ru.Url, str)),
rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url_in=None, session=None, ttype=sc.SYNC):
"""
This is the asynchronous class constructor, returning
a :class:`saga:Task` instance. For details on the accepted parameters,
please see the description of :func:`__init__`.
"""
return cls(url_in, session, _ttype=ttype)._init_task
# --------------------------------------------------------------------------
#
@rus.takes('Manager',
rus.optional(rus.one_of(c.COMPUTE, c.STORAGE, c.NETWORK)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(str), st.Task))
def list(self, rtype=None, ttype=None):
"""
list(rtype=None)
List known resource instances (which can be acquired).
Returns a list of IDs.
:type rtype: None or enum (COMPUTE | STORAGE | NETWORK)
:param rtype: filter for one or more resource types
"""
return self._adaptor.list(rtype, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager', rus.optional(str),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((descr.Description, st.Task))
def get_description(self, rid, ttype=None):
"""
get_description(rid)
Get the resource :class:`Description` for the specified resource.
:type rid: str
:param rid: identifies the resource to be described.
"""
# TODO / NOTE: if rid is None, should we return a description of
# the managed resources?
return self._adaptor.get_description(id, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager',
rus.optional(rus.one_of(c.COMPUTE, c.STORAGE, c.NETWORK)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(str), st.Task))
def list_templates(self, rtype=None, ttype=None):
"""
list_templates(rtype=None)
List template names available for the specified resource type(s).
Returns a list of strings.
:type rtype: None or enum (COMPUTE | STORAGE | NETWORK)
:param rtype: filter for one or more resource types
"""
return self._adaptor.list_templates(rtype, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager', rus.optional(str),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((descr.Description, st.Task))
def get_template(self, name, ttype=None):
"""
get_template(name)
Get a :class:`Description` for the specified template.
:type name: str
:param name: specifies the name of the template
The returned resource description instance may not have all attributes
filled, and may in fact not sufficiently complete to allow for
successful resource acquisition. The only guaranteed attribute in the
returned description is `TEMPLATE`, containing the very template id
specified in the call parameters.
"""
return self._adaptor.get_template(name, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager',
rus.optional(rus.one_of(c.COMPUTE, c.STORAGE, c.NETWORK)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(str), st.Task))
def list_images(self, rtype=None, ttype=None):
"""
list_images(rtype=None)
List image names available for the specified resource type(s).
Returns a list of strings.
:type rtype: None or enum (COMPUTE | STORAGE | NETWORK)
:param rtype: filter for one or more resource types
"""
return self._adaptor.list_images(rtype, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager', str, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((dict, st.Task))
def get_image(self, name, ttype=None):
"""
get_image(name)
Get a description string for the specified image.
:type name: str
:param name: specifies the image name
"""
return self._adaptor.get_image(name, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager', (str, ru.Url, descr.Description),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((resrc.Resource, st.Task))
def acquire(self, spec, ttype=None):
"""
acquire(desc)
Create a new :class:`saga.resource.Resource` handle for a
resource specified by the description.
:type spec: :class:`Description` or Url
:param spec: specifies the resource
Depending on the `RTYPE` attribute in the description, the
returned resource may be a :class:`saga.resource.Compute`,
:class:`saga.resource.Storage` or :class:`saga.resource.Network`
instance.
If the `spec` parameter is
"""
if isinstance (spec, ru.Url) or \
isinstance (spec, str) :
return self._adaptor.acquire_by_id(spec, ttype=ttype)
else:
# make sure at least 'executable' is defined
if spec.rtype is None:
raise se.BadParameter("resource type undefined in description")
# spec_copy = descr.Description ()
# spec._attributes_deep_copy (spec_copy)
return self._adaptor.acquire(spec, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Manager', str, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def destroy(self, rid, ttype=None):
"""
destroy(rid)
Destroy / release a resource.
:type rid : string
:param rid : identifies the resource to be released
"""
return self._adaptor.destroy(rid, ttype=ttype)
"""
In the context of SAGA-Python, a *ResourceManager* is a service which asserts
control over a set of resources. That manager can, on request, render
control over subsets of those resources (resource slices) to an application.
This :class:`Manager` class represents the contact point to such
ResourceManager instances -- the application can thus acquire compute, data
or network resources, according to some resource specification, for a bound
or unbound amount of time.
"""
# --------------------------------------------------------------------------
#
@rus.takes('Manager', rus.optional(basestring, surl.Url),
rus.optional(ss.Session), rus.optional(sab.Base),
rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
"""
__init__(url)
Create a new Manager instance. Connect to a remote resource management endpoint.
:type url: :class:`saga.Url`
:param url: resource management endpoint
"""
class LogicalFile(nsentry.Entry, sa.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=c.READ,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
__init__(url=None, flags=READ, session=None)
url: saga.Url
flags: flags enum
session: saga.Session
ret: obj
'''
# param checks
if not flags: flags = 0
url = ru.Url(url)
self._nsentry = super(LogicalFile, self)
self._nsentry.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('LogicalFile', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=c.READ, session=None, ttype=None):
'''
url: saga.Url
flags: saga.replica.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
if not flags: flags = 0
_nsentry = super(LogicalFile, cls)
return _nsentry.create(url, flags, session, ttype=ttype)
# ----------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_file(self, ttype=None):
'''
is_file()
ttype: saga.task.type enum
ret: bool / saga.Task
'''
return self.is_entry(ttype=ttype)
# --------------------------------------------------------------------------
#
# replica methods
#
@rus.takes('LogicalFile', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def get_size(self, ttype=None):
'''
get_size()
Return the size of the file.
ttype: saga.task.type enum
ret: int / saga.Task
Returns the size of the physical file represented by this logical file
(in bytes)
Example::
# get a file handle
lf = saga.replica.LogicalFile("irods://localhost/tmp/data.bin")
# print the logical file's size
print lf.get_size ()
'''
return self._adaptor.get_size_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def add_location(self, name, ttype=None):
'''
add_location(name)
Add a physical location.
name: saga.Url
ttype: saga.task.type enum
ret: None / saga.Task
'''
return self._adaptor.add_location(name, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def remove_location(self, name, ttype=None):
'''
remove_location(name)
Remove a physical location.
name: saga.Url
ttype: saga.task.type enum
ret: None / saga.Task
'''
return self._adaptor.remove_location(name, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional((ru.Url, basestring)),
rus.optional((ru.Url, basestring)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def update_location(self, old, new, ttype=None):
'''
update_location(old, new)
Updates a physical location.
old: saga.Url
new: saga.Url
ttype: saga.task.type enum
ret: None / saga.Task
'''
return self._adaptor.update_location(old, new, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(ru.Url), st.Task))
def list_locations(self, ttype=None):
'''
list_locations()
List all physical locations of a logical file.
ttype: saga.task.type enum
ret: list [saga.Url] / saga.Task
'''
return self._adaptor.list_locations(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('LogicalFile', (ru.Url, basestring),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def replicate(self, name, flags=None, ttype=None):
'''
replicate(name)
Replicate a logical file.
name: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
return self._adaptor.replicate(name, flags, ttype=ttype)
# --------------------------------------------------------------------------
# non-GFD.90
#
@rus.takes('LogicalFile', (ru.Url, basestring),
rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def upload(self, name, tgt=None, flags=None, ttype=None):
'''
upload(name, tgt=None, flags=None)
Upload a physical file.
name: saga.Url
tgt: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
return self._adaptor.upload(name, tgt, flags, ttype=ttype)
# --------------------------------------------------------------------------
# non-GFD.90
#
@rus.takes('LogicalFile', (ru.Url, basestring),
rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def download(self, name, src=None, flags=None, ttype=None):
'''
download(name, src=None, flags=None)
Download a physical file.
name: saga.Url
src: saga.Url
flags: flags enum
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
return self._adaptor.download(name, src, flags, ttype=ttype)
control over subsets of those resources (resource slices) to an application.
This :class:`Manager` class represents the contact point to such
ResourceManager instances -- the application can thus acquire compute, data
or network resources, according to some resource specification, for a bound
or unbound amount of time.
"""
# --------------------------------------------------------------------------
#
@rus.takes ('Manager',
rus.optional (basestring, surl.Url),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, url=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
"""
__init__(url)
Create a new Manager instance. Connect to a remote resource management endpoint.
:type url: :class:`saga.Url`
:param url: resource management endpoint
"""
# param checks
_url = surl.Url(url)
scheme = _url.scheme.lower()
class Task(sbase.SimpleBase, satt.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('Task', sacb.CPIBase, basestring, dict,
rus.one_of(SYNC, ASYNC, TASK))
@rus.returns(rus.nothing)
def __init__(self, _adaptor, _method_type, _method_context, _ttype):
"""
This saga.Task constructor is private.
``_adaptor`` references the adaptor class instance from which this
task was created via an asynchronous function. Note that the API level
object instance can be inferred via ``_adaptor.get_api ()``. Further, the
adaptor will reference an _adaptor._container class, which will be
considered the target for bulk operations for this task.
``_method_type`` specifies the SAGA API method which task is
representing. For example, for the following code::
d = saga.filesystem.Directory ("file:///")
t = d.copy ('/etc/passwd', '/tmp/passwd.bak', saga.task.ASYNC)
The resulting task ``t`` would represent the *'copy'* method. This is
required to forward :class:`saga.task.Container` calls to the correct
bulk method, in this case ``container_copy()``.
``_method_context`` describes the context in which the task method is
running. It is up to the creator of the task to provide that context --
in general, it will at least include method parameters.
``ttype`` determines in what state the constructor will leave the task:
``DONE`` for ``ttype=SYNC``, ``RUNNING`` for ``ttype=ASYNC`` and ``NEW``
for ``ttype=TASK``.
If the ``_method_context`` has *exactly* three elements, names
``_call``, ``args`` and ``kwargs``, then the created task will wrap
a :class:`ru.Future` with that ``_call (*_args, **kwargs)``.
"""
self._base = super(Task, self)
self._base.__init__()
self._future = None
self._ttype = _ttype
self._adaptor = _adaptor
self._method_type = _method_type
self._method_context = _method_context
# set attribute interface properties
self._attributes_extensible(False)
self._attributes_allow_private(True)
self._attributes_camelcasing(True)
# register properties with the attribute interface
self._attributes_register(RESULT, None, satt.ANY, satt.SCALAR,
satt.READONLY)
self._attributes_set_getter(RESULT, self.get_result)
self._attributes_set_setter(RESULT, self._set_result)
self._attributes_register(EXCEPTION, None, satt.ANY, satt.SCALAR,
satt.READONLY)
self._attributes_set_getter(EXCEPTION, self.get_exception)
self._attributes_set_setter(EXCEPTION, self._set_exception)
self._attributes_register(STATE, UNKNOWN, satt.ENUM, satt.SCALAR,
satt.READONLY)
self._attributes_set_enums(
STATE, [UNKNOWN, NEW, RUNNING, DONE, FAILED, CANCELED])
self._attributes_set_getter(STATE, self.get_state)
self._attributes_set_setter(STATE, self._set_state)
self._set_state(NEW)
# check if this task is supposed to wrap a callable in a future
if '_call' in self._method_context:
call = self._method_context['call']
args = self._method_context.get('_args', list())
kwargs = self._method_context.get('_kwargs', dict())
# if the called function expects a task handle, provide it.
if '_from_task' in inspect.getargspec(call)[0]:
if not '_from_task' in kwargs:
kwargs['_from_task'] = self
self._future = ru.Future(call=call, args=args, kwargs=kwargs)
# ensure task goes into the correct state
if self._ttype == SYNC:
self.run()
self.wait()
elif self._ttype == ASYNC:
self.run()
elif self._ttype == TASK:
pass
# --------------------------------------------------------------------------
#
@rus.takes('Task')
@rus.returns(rus.nothing)
def run(self):
if self._future:
self._future.run()
else:
# FIXME: make sure task_run exists. Should be part of the CPI!
self._adaptor.task_run(self)
# --------------------------------------------------------------------------
#
@rus.takes('Task', rus.optional(float))
@rus.returns(bool)
def wait(self, timeout=None):
if None == timeout:
timeout = -1.0 # FIXME
if self._future:
self._future.wait(timeout) # FIXME: timeout?!
self._set_state(self._future.state)
else:
# FIXME: make sure task_wait exists. Should be part of the CPI!
self._adaptor.task_wait(self, timeout)
# ----------------------------------------------------------------
#
@rus.takes('Task', float)
@rus.returns(rus.nothing)
def cancel(self):
if self._future:
self._future.cancel()
self._set_state(CANCELED)
else:
# FIXME: make sure task_cancel exists. Should be part of the CPI!
self._adaptor.task_cancel(self)
# --------------------------------------------------------------------------
#
@rus.takes('Task', rus.one_of(UNKNOWN, NEW, RUNNING, DONE, FAILED,
CANCELED))
@rus.returns(rus.nothing)
def _set_state(self, state):
if not state in [UNKNOWN, NEW, RUNNING, DONE, FAILED, CANCELED]:
raise se.BadParameter("attempt to set invalid task state '%s'" %
state)
self._attributes_i_set(self._attributes_t_underscore(STATE),
state,
force=True)
# --------------------------------------------------------------------------
#
@rus.takes('Task')
@rus.returns(rus.one_of(UNKNOWN, NEW, RUNNING, DONE, FAILED, CANCELED))
def get_state(self):
if self._future:
self._set_state(self._future.state)
return self.state
# --------------------------------------------------------------------------
#
@rus.takes('Task', rus.anything)
@rus.returns(rus.nothing)
def _set_result(self, result):
self._attributes_i_set(self._attributes_t_underscore(RESULT),
result,
force=True)
self._attributes_i_set(self._attributes_t_underscore(STATE),
DONE,
force=True)
# --------------------------------------------------------------------------
#
@rus.takes('Task')
@rus.returns(rus.anything)
def get_result(self):
if not self.state in [DONE, FAILED, CANCELED]:
self.wait()
assert (self.state in [DONE, FAILED, CANCELED])
if self.state == FAILED:
self.re_raise()
return
if self.state == CANCELED:
raise se.IncorrectState(
"task.get_result() cannot be called on cancelled tasks")
if self.state == DONE:
if self._future:
self._set_result(self._future.result)
return self.result
# --------------------------------------------------------------------------
#
@rus.takes('Task', basestring, rus.anything)
@rus.returns(rus.nothing)
def _set_metric(self, metric, value):
self._attributes_i_set(self._attributes_t_underscore(metric),
value,
force=True)
# --------------------------------------------------------------------------
#
@rus.takes('Task', se.SagaException)
@rus.returns(rus.nothing)
def _set_exception(self, e):
self._attributes_i_set(self._attributes_t_underscore(EXCEPTION),
e,
force=True)
# --------------------------------------------------------------------------
#
@rus.takes('Task')
@rus.returns(se.SagaException)
def get_exception(self):
if self._future:
self._set_exception(self._future.exception)
return self.exception
# --------------------------------------------------------------------------
#
@rus.takes('Task')
@rus.returns(rus.nothing)
def re_raise(self):
if self.exception:
raise self.exception
class Entry(sb.Base, sasync.Async):
'''
Represents a SAGA namespace entry as defined in GFD.90
The saga.namespace.Entry class represents, as the name indicates,
an entry in some (local or remote) namespace. That class offers
a number of operations on that entry, such as copy, move and remove::
# get an entry handle
entry = saga.namespace.Entry ("sftp://localhost/tmp/data/data.bin")
# copy the entry
entry.copy ("sftp://localhost/tmp/data/data.bak")
# move the entry
entry.move ("sftp://localhost/tmp/data/data.new")
'''
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional((surl.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=None,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
:param url: Url of the (remote) entry
:type url: :class:`saga.Url`
flags: flags enum
session: saga.Session
ret: obj
Construct a new entry object
The specified entry is expected to exist -- otherwise a DoesNotExist
exception is raised. Also, the URL must point to an entry (not to
a directory), otherwise a BadParameter exception is raised.
Example::
# get an entry handle
entry = saga.namespace.Entry("sftp://localhost/tmp/data/data.bin")
# print the entry's url
print entry.get_url ()
'''
self._session = session
self._is_recursive = False # recursion guard (FIXME: NOT THREAD SAFE)
# param checks
if not session:
session = ss.Session(default=True)
if not flags: flags = 0
url = surl.Url(url)
scheme = url.scheme.lower()
self._base = super(Entry, self)
self._base.__init__(scheme,
_adaptor,
_adaptor_state,
url,
flags,
session,
ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('Entry', rus.optional((surl.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=None, session=None, ttype=None):
'''
url: saga.Url
flags: saga.namespace.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
# param checks
if not flags: flags = 0
if not session:
session = ss.Session(default=True)
return cls(url, flags, session, _ttype=ttype)._init_task
# ----------------------------------------------------------------
#
# namespace entry methods
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((surl.Url, st.Task))
def get_url(self, ttype=None):
'''
ttype: saga.task.type enum
ret: saga.Url / saga.Task
Return the complete url pointing to the entry.
The call will return the complete url pointing to
this entry as a saga.Url object::
# print URL of an entry
entry = saga.namespace.Entry("sftp://localhost/etc/passwd")
print entry.get_url()
'''
return self._adaptor.get_url(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def get_cwd(self, ttype=None):
'''
ttype: saga.task.type enum
ret: string / saga.Task
'''
return self._adaptor.get_cwd(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def get_name(self, ttype=None):
'''
ttype: saga.task.type enum
ret: string / saga.Task
'''
return self._adaptor.get_name(ttype=ttype)
# ----------------------------------------------------------------
#
# namespace entry / directory methods
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_dir(self, ttype=None):
'''
ttype: saga.task.type enum
ret: bool / saga.Task
Returns True if path is a directory, False otherwise.
Example::
# inspect an entry
dir = saga.namespace.Directory("sftp://localhost/tmp/")
if dir.is_dir ('data'):
# do something
'''
return self._adaptor.is_dir_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_entry(self, ttype=None):
'''
ttype: saga.task.type enum
ret: bool / saga.Task
'''
return self._adaptor.is_entry_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_link(self, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: bool / saga.Task
'''
return self._adaptor.is_link_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((surl.Url, st.Task))
def read_link(self, ttype=None):
'''
tgt: saga.Url / None
ttype: saga.task.type enum
ret: saga.Url / saga.Task
'''
return self._adaptor.read_link_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', (surl.Url, basestring), rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def copy(self, tgt, flags=0, ttype=None):
'''
tgt: saga.Url
flags: enum flags
ttype: saga.task.type enum
ret: None / saga.Task
Copy the entry to another location
:param target: Url of the copy target.
:param flags: Flags to use for the operation.
The entry is copied to the given target location. The target URL must
be an absolute path, and can be a target entry name or target
directory name. If the target entry exists, it is overwritten::
# copy an entry
entry = saga.namespace.Directory("sftp://localhost/tmp/data/data.bin")
entry.copy ("sftp://localhost/tmp/data/data.bak")
'''
# parameter checks
if not flags: flags = 0
tgt_url = surl.Url(tgt) # ensure valid and typed Url
# async ops don't deserve a fallback (yet)
if ttype != None:
return self._adaptor.copy_self(tgt_url, flags, ttype=ttype)
# we have only sync calls here - attempt a normal call to the bound
# adaptor first (doh!)
ret = self._adaptor.copy_self(tgt_url, flags, ttype=ttype)
try:
True
except se.SagaException as e:
# if we don't have a scheme for tgt, all is in vain (adaptor
# should have handled a relative path...)
if not tgt_url.scheme:
raise e
# So, the adaptor bound to the src URL did not manage to copy the # entry.
# If the tgt has a scheme set, we try again with other matching # entry
# adaptors, by setting (a copy of) the *src* URL to the same scheme,
# in the hope that other adaptors can copy from localhost.
#
# In principle that mechanism can also be used for remote copies, but
# URL translation is way more fragile in those cases...
# check recursion guard
if self._is_recursive:
self._logger.debug("fallback recursion detected - abort")
else:
# activate recursion guard
self._is_recursive += 1
import saga.engine
engine = saga.engine.Engine()
# find applicable adaptors we could fall back to, i.e. which
# support the tgt schema
adaptor_names = engine.find_adaptors('saga.namespace.Entry',
tgt_url.scheme)
self._logger.debug("try fallback copy to these adaptors: %s" %
adaptor_names)
# build a new src url, by switching to the target schema
tmp_url = self.get_url()
tmp_url.scheme = tgt_url.scheme
for adaptor_name in adaptor_names:
try:
self._logger.info("try fallback copy to %s" %
adaptor_name)
adaptor_instance = engine.get_adaptor(adaptor_name)
# get an tgt-scheme'd adaptor for the new src url, and try copy again
adaptor = engine.bind_adaptor(self,
'saga.namespace.Entry',
tgt_url.scheme,
adaptor_instance)
adaptor.init_instance({}, tmp_url, None, self._session)
tmp = Entry(tmp_url,
None,
self._session,
_adaptor=adaptor_instance)
ret = tmp.copy(tgt_url, flags)
# release recursion guard
self._is_recursive -= 1
# if nothing raised an exception so far, we are done.
return
except se.SagaException as e:
self._logger.info("fallback failed: %s" % e)
# didn't work, ignore this adaptor
pass
# if all was in vain, we rethrow the original exception
self._is_recursive -= 1
raise e
# --------------------------------------------------------------------------
#
@rus.takes('Entry', (surl.Url, basestring), rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def link(self, tgt, flags=0, ttype=None):
'''
tgt: saga.Url
flags: enum flags
ttype: saga.task.type enum
ret: None / saga.Task
'''
if not flags: flags = 0
return self._adaptor.link_self(tgt, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', (surl.Url, basestring), rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def move(self, tgt, flags=0, ttype=None):
'''
:param target: Url of the move target.
:param flags: Flags to use for the operation.
ttype: saga.task.type enum
ret: None / saga.Task
Move the entry to another location
The entry is copied to the given target location. The target URL must
be an absolute path, and can be a target entry name or target
directory name. If the target entry exists, it is overwritten::
# copy an entry
entry = saga.namespace.Directory("sftp://localhost/tmp/data/data.bin")
entry.move ("sftp://localhost/tmp/data/data.bak")
'''
if not flags: flags = 0
return self._adaptor.move_self(tgt, flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def remove(self, flags=0, ttype=None):
'''
:param flags: Flags to use for the operation.
ttype: saga.task.type enum
ret: None / saga.Task
Reove the entry.
The entry is removed, and this object instance is then invalid for
further operations.
# remove an entry
entry = saga.namespace.Directory("sftp://localhost/tmp/data/data.bin")
entry.remove ()
'''
if not flags: flags = 0
return self._adaptor.remove_self(flags, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Entry', rus.optional(float),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def close(self, timeout=None, ttype=None):
'''
timeout: float
ttype: saga.task.type enum
ret: None / saga.Task
'''
return self._adaptor.close(timeout, ttype=ttype)
# --------------------------------------------------------------------------
#
url = property(get_url) # saga.Url
cwd = property(get_cwd) # string
name = property(get_name) # string
class Service (sb.Base, sasync.Async) :
"""
The job.Service represents a resource management backend, and as such allows
the creation, submission and management of jobs.
A job.Service represents anything which accepts job creation requests, and
which manages thus created :class:`saga.job.Job` instances. That can be a local shell,
a remote ssh shell, a cluster queuing system, a IaaS backend -- you name it.
The job.Service is identified by an URL, which usually points to the contact
endpoint for that service.
Example::
service = saga.job.Service("fork://localhost")
ids = service.list()
for job_id in ids :
print job_id
j = service.get_job(job_id)
if j.get_state() == saga.job.Job.Pending:
print "pending"
elif j.get_state() == saga.job.Job.Running:
print "running"
else:
print "job is already final!"
service.close()
"""
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
rus.optional ((basestring, surl.Url)),
rus.optional (ss.Session),
rus.optional (sab.Base),
rus.optional (dict),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (rus.nothing)
def __init__ (self, rm=None, session=None,
_adaptor=None, _adaptor_state={}, _ttype=None) :
"""
__init__(rm, session)
Create a new job.Service instance.
:param rm: resource manager URL
:type rm: string or :class:`saga.Url`
:param session: an optional session object with security contexts
:type session: :class:`saga.Session`
:rtype: :class:`saga.job.Service`
"""
# job service instances are resource hogs. Before attempting to create
# a new instance, we attempt to clear out all old instances. There is
# some collateral damage: we cannot run the Python GC over only the
# job.Service instances, but have to run it globally -- however,
# compared to the latency introduced by the job service setup, this
# should be a minor inconvenienve (tm)
try :
import gc
gc.collect ()
except :
pass
# param checks
self.valid = False
url = surl.Url (rm)
if not url.scheme :
url.scheme = 'fork'
if not url.host :
url.host = 'localhost'
if not session :
session = ss.Session (default=True)
scheme = url.scheme.lower ()
self._super = super (Service, self)
self._super.__init__ (scheme, _adaptor, _adaptor_state,
url, session, ttype=_ttype)
self.valid = True
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes ('Service',
rus.optional ((surl.Url, basestring)),
rus.optional (ss.Session),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns (st.Task)
def create (cls, rm=None, session=None, ttype=SYNC) :
"""
create(rm=None, session=None)
Create a new job.Service instance asynchronously.
:param rm: resource manager URL
:type rm: string or :class:`saga.Url`
:param session: an optional session object with security contexts
:type session: :class:`saga.Session`
:rtype: :class:`saga.Task`
"""
# param checks
if not session :
session = ss.Session (default=True)
url = surl.Url (rm)
scheme = url.scheme.lower ()
return cls (url, session, _ttype=ttype)._init_task
# --------------------------------------------------------------------------
#
@rus.takes ('Service')
@rus.returns (basestring)
def __str__ (self):
"""
__str__()
String representation. Returns the job service Url.
"""
if self.valid :
return "[%s]" % self.url
return ""
# --------------------------------------------------------------------------
#
@rus.takes ('Service')
@rus.returns (rus.nothing)
def close (self) :
"""
close()
Close the job service instance and disconnect from the (remote)
job service if necessary. Any subsequent calls to a job service
instance after `close()` was called will fail.
Example::
service = saga.job.Service("fork://localhost")
# do something with the 'service' object, create jobs, etc...
service.close()
service.list() # this call will throw an exception
.. warning:: While in principle the job service destructor calls
`close()` automatically when a job service instance goes out of scope,
you **shouldn't rely on it**. Python's garbage collection can be a
bit odd at times, so you should always call `close()` explicitly.
Especially in a **multi-threaded program** this will help to avoid
random errors.
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
self._adaptor.close ()
self.valid = False
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
descr.Description,
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((j.Job, st.Task))
def create_job (self, job_desc, ttype=None) :
"""
create_job(job_desc)
Create a new job.Job instance from a :class:`~saga.job.Description`. The
resulting job instance is in :data:`~saga.job.NEW` state.
:param job_desc: job description to create the job from
:type job_desc: :data:`saga.job.Description`
:param ttype: |param_ttype|
:rtype: :class:`saga.job.Job` or |rtype_ttype|
create_job() accepts a job description, which described the
application instance to be created by the backend. The create_job()
method is not actually attempting to *run* the job, but merely parses
the job description for syntactic and semantic consistency. The job
returned object is thus not in 'Pending' or 'Running', but rather in
'New' state. The actual submission is performed by calling run() on
the job object.
Example::
# A job.Description object describes the executable/application and its requirements
job_desc = saga.job.Description()
job_desc.executable = '/bin/sleep'
job_desc.arguments = ['10']
job_desc.output = 'myjob.out'
job_desc.error = 'myjob.err'
service = saga.job.Service('local://localhost')
job = service.create_job(job_desc)
# Run the job and wait for it to finish
job.run()
print "Job ID : %s" % (job.job_id)
job.wait()
# Get some info about the job
print "Job State : %s" % (job.state)
print "Exitcode : %s" % (job.exit_code)
service.close()
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
jd_copy = descr.Description()
job_desc._attributes_deep_copy (jd_copy)
# do some sanity checks: if the adaptor has specified a set of supported
# job description attributes, we scan the given description for any
# mismatches, and complain then.
adaptor_info = self._adaptor._adaptor.get_info ()
if 'capabilities' in adaptor_info and \
'jdes_attributes' in adaptor_info['capabilities'] :
# this is the list of key supported by the adaptor. These
# attributes may be set to non-default values
supported_keys = adaptor_info['capabilities']['jdes_attributes']
# use an empty job description to compare default values
jd_default = descr.Description ()
for key in jd_copy.list_attributes () :
val = jd_copy .get_attribute (key)
default = jd_default.get_attribute (key)
# Also, we make string compares case insensitive
if isinstance (val, basestring) : val = val .lower ()
if isinstance (default, basestring) : default = default.lower ()
# supported keys are also valid, as are keys with default or
# None values
if key not in supported_keys and \
val != default and \
val :
msg = "'JobDescription.%s' (%s) is not supported by adaptor %s" \
% (key, val, adaptor_info['name'])
raise se.BadParameter._log (self._logger, msg)
# make sure at least 'executable' is defined
if jd_copy.executable is None:
raise se.BadParameter("No executable defined")
# convert environment to string
if jd_copy.attribute_exists ('Environment') :
for (key, value) in jd_copy.environment.iteritems():
jd_copy.environment[key] = str(value)
return self._adaptor.create_job (jd_copy, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
basestring,
rus.optional (basestring),
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((j.Job, st.Task))
def run_job (self, cmd, host=None, ttype=None) :
"""
run_job(cmd, host=None)
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
if not cmd:
raise se.BadParameter('run_job needs a command to run. Duh!')
try:
# lets see if the adaptor implements run_job
return self._adaptor.run_job (cmd, host, ttype=ttype)
except:
# fall back to the default implementation below
pass
# The adaptor has no run_job -- we here provide a generic implementation
# FIXME: split should be more clever and respect POSIX shell syntax.
args = cmd.split()
jd = descr.Description()
jd.executable = args[0]
jd.arguments = args[1:]
job = self.create_job(jd)
job.run()
return job
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((rus.list_of (basestring), st.Task))
def list (self, ttype=None) :
"""
list()
Return a list of the jobs that are managed by this Service
instance.
.. seealso::
The :data:`~saga.job.Service.jobs` property and the
:meth:`~saga.job.Service.list` method are semantically
equivalent.
:ttype: |param_ttype|
:rtype: list of :class:`saga.job.Job`
As the job.Service represents a job management backend, list() will
return a list of job IDs for all jobs which are known to the backend,
and which can potentially be accessed and managed by the application.
Example::
service = saga.job.Service("fork://localhost")
ids = service.list()
for job_id in ids :
print job_id
service.close()
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
return self._adaptor.list (ttype=ttype)
jobs = property (list)
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((surl.Url, st.Task))
def get_url (self, ttype=None) :
"""
get_url()
Return the URL this Service instance was created with.
.. seealso::
The :data:`~saga.job.Service.url` property and the
:meth:`~saga.job.Service.get_url` method are semantically
equivalent and only duplicated for convenience.
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
return self._adaptor.get_url (ttype=ttype)
url = property (get_url)
# --------------------------------------------------------------------------
#
@rus.takes ('Service',
basestring,
rus.optional (rus.one_of (SYNC, ASYNC, TASK)))
@rus.returns ((j.Job, st.Task))
def get_job (self, job_id, ttype=None) :
"""
get_job(job_id)
Return the job object for a given job id.
:param job_id: The id of the job to retrieve
:rtype: :class:`saga.job.Job`
Job objects are a local representation of a remote stateful entity.
The job.Service supports to reconnect to those remote entities::
service = saga.job.Service("fork://localhost")
j = service.get_job(my_job_id)
if j.get_state() == saga.job.Job.Pending:
print "pending"
elif j.get_state() == saga.job.Job.Running:
print "running"
else:
print "job is already final!"
service.close()
"""
if not self.valid :
raise se.IncorrectState ("This instance was already closed.")
return self._adaptor.get_job (job_id, ttype=ttype)
class File(nsentry.Entry):
"""
Represents a local or remote file.
The saga.filesystem.File class represents, as the name indicates,
a file on some (local or remote) filesystem. That class offers
a number of operations on that file, such as copy, move and remove::
# get a file handle
file = saga.filesystem.File("sftp://localhost/tmp/data/data.bin")
# copy the file
file.copy ("sftp://localhost/tmp/data/data.bak")
# move the file
file.move ("sftp://localhost/tmp/data/data.new")
"""
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=READ,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
"""
__init__(url, flags=READ, session)
Construct a new file object
:param url: Url of the (remote) file
:type url: :class:`saga.Url`
:fgs: :ref:`filesystemflags`
:param session: :class:`saga.Session`
The specified file is expected to exist -- otherwise a DoesNotExist
exception is raised. Also, the URL must point to a file (not to
a directory), otherwise a BadParameter exception is raised.
Example::
# get a file handle
file = saga.filesystem.File("sftp://localhost/tmp/data/data.bin")
# print the file's size
print file.get_size ()
"""
# param checks
if not flags: flags = 0
url = ru.Url(url)
if not url.schema:
url.schema = 'file'
if not url.host:
url.host = 'localhost'
self._nsentry = super(File, self)
self._nsentry.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('File', rus.optional((ru.Url, basestring)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=READ, session=None, ttype=None):
"""
create(url, flags, session)
url: saga.Url
flags: saga.replica.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
"""
if not flags: flags = 0
_nsentry = super(File, cls)
return _nsentry.create(url, flags, session, ttype=ttype)
# ----------------------------------------------------------------
#
# filesystem methods
#
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((bool, st.Task))
def is_file(self, ttype=None):
"""
is_file()
Returns `True` if instance points to a file, `False` otherwise.
"""
return self._adaptor.is_file_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def get_size(self, ttype=None):
'''
get_size()
Returns the size (in bytes) of a file.
Example::
# get a file handle
file = saga.filesystem.File("sftp://localhost/tmp/data/data.bin")
# print the file's size
print file.get_size ()
'''
return self._adaptor.get_size_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional(int),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def read(self, size=None, ttype=None):
'''
size : int
ttype: saga.task.type enum
ret: string / bytearray / saga.Task
'''
return self._adaptor.read(size, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional(bool))
@rus.returns(st.Task)
def close(self, kill=True, ttype=None):
'''
kill : bool
ttype: saga.task.type enum
ret: string / bytearray / saga.Task
'''
return self._adaptor.close()
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def write(self, data, ttype=None):
'''
data : string / bytearray
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.write(data, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', int, rus.optional(rus.one_of(START, CURRENT, END)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def seek(self, offset, whence=START, ttype=None):
'''
offset: int
whence: seek_mode enum
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.seek(offset, whence, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.list_of(rus.tuple_of(int)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def read_v(self, iovecs, ttype=None):
'''
iovecs: list [tuple (int, int)]
ttype: saga.task.type enum
ret: list [bytearray] / saga.Task
'''
return self._adaptor.read_v(iovecs, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.list_of(rus.tuple_of((int, basestring))),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(int), st.Task))
def write_v(self, data, ttype=None):
'''
data: list [tuple (int, string / bytearray)]
ttype: saga.task.type enum
ret: list [int] / saga.Task
'''
return self._adaptor.write_v(data, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def size_p(self, pattern, ttype=None):
'''
pattern: string
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.size_p(pattern, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def read_p(self, pattern, ttype=None):
'''
pattern: string
ttype: saga.task.type enum
ret: string / bytearray / saga.Task
'''
return self._adaptor.read_p(pattern, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, basestring,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def write_p(self, pattern, data, ttype=None):
'''
pattern: string
data: string / bytearray
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.write_p(pattern, data, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(basestring), st.Task))
def modes_e(self, ttype=None):
'''
ttype: saga.task.type enum
ret: list [string] / saga.Task
'''
return self._adaptor.modes_e(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, basestring,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def size_e(self, emode, spec, ttype=None):
'''
emode: string
spec: string
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.size_e(emode, spec, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, basestring,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((basestring, st.Task))
def read_e(self, emode, spec, ttype=None):
'''
emode: string
spec: string
ttype: saga.task.type enum
ret: bytearray / saga.Task
'''
return self._adaptor.read_e(emode, spec, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('File', basestring, basestring, basestring,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((int, st.Task))
def write_e(self, emode, spec, data, ttype=None):
'''
emode: string
spec: string
data: string / bytearray
ttype: saga.task.type enum
ret: int / saga.Task
'''
return self._adaptor.read_e(emode, spec, data, ttype=ttype)
size = property(get_size) # int
modes_e = property(modes_e) # list [string]
class Directory(nsdir.Directory, sa.Attributes):
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, str)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
url=None,
flags=READ,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
'''
url: saga.Url
flags: flags enum
session: saga.Session
ret: obj
'''
# param checks
if not flags: flags = 0
url = ru.Url(url)
self._nsdirec = super(Directory, self)
self._nsdirec.__init__(url,
flags,
session,
_adaptor,
_adaptor_state,
_ttype=_ttype)
# set attribute interface properties
self._attributes_allow_private(True)
self._attributes_camelcasing(True)
self._attributes_extensible(True,
getter=self._attribute_getter,
setter=self._attribute_setter,
lister=self._attribute_lister,
caller=self._attribute_caller)
# register properties with the attribute interface
self._attributes_register(ATTRIBUTE, None, sa.STRING, sa.SCALAR,
sa.READONLY)
self._attributes_register(CHANGE, None, sa.STRING, sa.SCALAR,
sa.READONLY)
self._attributes_register(NEW, None, sa.STRING, sa.SCALAR, sa.READONLY)
self._attributes_register(DELETE, None, sa.STRING, sa.SCALAR,
sa.READONLY)
self._attributes_register(TTL, None, sa.INT, sa.SCALAR, sa.WRITEABLE)
self._attributes_set_setter(TTL, self.set_ttl)
self._attributes_set_getter(TTL, self.get_ttl)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('Directory', rus.optional((ru.Url, str)),
rus.optional(int, rus.nothing), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, url=None, flags=READ, session=None, ttype=None):
'''
url: saga.Url
flags: saga.advert.flags enum
session: saga.Session
ttype: saga.task.type enum
ret: saga.Task
'''
if not flags: flags = 0
_nsdir = super(Directory, cls)
return _nsdir.create(url, flags, session, ttype=ttype)
# --------------------------------------------------------------------------
#
# attribute methods
#
# NOTE: we do not yet pass ttype, as async calls are not yet supported by
# the attribute interface
#
@rus.takes('Directory', str, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.anything, st.Task))
def _attribute_getter(self, key, ttype=None):
return self._adaptor.attribute_getter(key)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', str, rus.anything,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def _attribute_setter(self, key, val, ttype=None):
return self._adaptor.attribute_setter(key, val)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(rus.anything), st.Task))
def _attribute_lister(self, ttype=None):
return self._adaptor.attribute_lister()
# --------------------------------------------------------------------------
#
@rus.takes('Directory', str, int, callable,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.anything, st.Task))
def _attribute_caller(self, key, id, cb, ttype=None):
return self._adaptor.attribute_caller(key, id, cb)
# ----------------------------------------------------------------
#
# advert methods
#
@rus.takes('Directory', (ru.Url, str), float,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def set_ttl(self, tgt=None, ttl=-1.0, ttype=None):
"""
tgt : saga.Url / None
ttl : int
ttype: saga.task.type enum
ret: None / saga.Task
"""
if tgt: return self._adaptor.set_ttl(tgt, ttl, ttype=ttype)
else: return self._adaptor.set_ttl_self(ttl, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional((ru.Url, str)),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((float, st.Task))
def get_ttl(self, tgt=None, ttype=None):
"""
tgt : saga.Url / None
ttype: saga.task.type enum
ret: int / saga.Task
"""
if tgt: return self._adaptor.get_ttl(tgt, ttype=ttype)
else: return self._adaptor.get_ttl_self(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Directory', rus.optional(str), rus.optional(str),
rus.optional((str, object)), rus.optional(int, rus.nothing),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.list_of(ru.Url), st.Task))
def find(self,
name_pattern,
attr_pattern=None,
obj_type=None,
flags=RECURSIVE,
ttype=None):
"""
name_pattern: string
attr_pattern: string
obj_type: string
flags: flags enum
ret: list [saga.Url]
"""
if not flags: flags = 0
if attr_pattern or obj_type:
return self._adaptor.find_adverts(name_pattern,
attr_pattern,
obj_type,
flags,
ttype=ttype)
else:
return self._nsdirec.find(name_pattern, flags, ttype=ttype)
class Resource(sb.Base, sa.Attributes, sasync.Async):
"""
A :class:`Resource` class instance represents a specific slice of resource
which is, if in `RUNNING` state, under the applications control and ready to
serve usage requests. The type of accepted usage requests depends on the
specific resource types (job execution for :class:`saga.resource.Compute`,
data storage for :class:`saga.resource.Storage`, and network connectivity
for :class:`saga.resource.Network`. The exact mechanism how those usage
requests are communicated are not part of the resource's class interface,
but are instead served by other RADICAL-SAGA classes -- typically those are
:class:`saga.job.Service` for Compute resources, and
:class:`saga.filesystem.Directory` for Storage resources (Network resources
provide implicit connectivity, but do not have explicit, public entry points
to request usage.
The process of resource acquisition is performed by a *ResourceManager*,
represented by a :class:`saga.resource.Manager` instance. The semantics of
the acquisition process is defined as the act of moving a slice (subset) of
the resources managed by the resource manager under the control of the
requesting application (i.e. under user control), to use as needed. The
type and property of the resource slice to be acquired and the time and
duration over which the resource will be made available to the application
are specified in a :class:`saga.resource.Description`, to be supplied when
acquiring a resource.
The exact backend semantics on *how* a resource slice is provisioned to the
application is up to the resource manager backend -- this can be as simple
as providing a job submission endpoint to a classic HPC resource, and as
complex as instantiating a pilot job or pilot data container, or reserving
a network fiber on demand, or instantiating a virtual machine -- the result
will, from the application's perspective, indistinguishable: a resource
slice is made available for the execution of usage requests (tasks,
workload, jobs, ...).
Resources are stateful: when acquired from a resource manager, they are
typically in `NEW` state, and will become `ACTIVE` once they are provisioned
to the application and can serve usage requests. Some resources may go
through an intermediate state, `PENDING`, when they are about to become
active at some point, and usage requests can already be submitted -- those
usage requests will not be executed until the resources enters the `ACTIVE`
state. The resource can be release from application control in three
different ways: they can be actively be destroyed by the application, and
will then enter the `CANCELED` state; they can internally cease to function
and become unable to serve usage requests, represented by a `FAILED` state,
and the resource manager can retract control from the application because
the agreed time duration has passed -- this is represented by the `EXPIRED`
state.
"""
# FIXME:
# - we don't use PENDING like this, yet
# - include state diagram (also for jobs btw)
# --------------------------------------------------------------------------
#
@rus.takes('Resource', rus.optional(str), rus.optional(ss.Session),
rus.optional(sab.Base), rus.optional(dict),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(rus.nothing)
def __init__(self,
id=None,
session=None,
_adaptor=None,
_adaptor_state={},
_ttype=None):
"""
__init__(id=None, session=None)
Create / reconnect to a resource.
:param id: id of the resource
:type id: :class:`saga.Url`
:param session: :class:`saga.Session`
Resource class instances are usually created by calling :func:`acquire`
on the :class:`saga.resource.Manager` class. Already acquired resources
are identified by a string typed identifier. This constructor accepts
such an identifier to create another representation of the same
resource. As the resource itself is new newly acquired, it can be in
any state. In particular, it can be in a final state, and thus be
unusable. Further, the resource may already have expired or failed, and
the information about it may have been purged -- in that case the id
will not be valid any longer, and a :class:`saga.BadParameter` exception
will be raised.
The session parameter is interpreted exactly as the session parameter on
the :class:`saga.resource.Manager` constructor.
"""
# set attribute interface properties
import radical.saga.attributes as sa
self._attributes_extensible(False)
self._attributes_camelcasing(True)
# register properties with the attribute interface
self._attributes_register(c.ID, None, sa.ENUM, sa.SCALAR, sa.READONLY)
self._attributes_register(c.RTYPE, None, sa.ENUM, sa.SCALAR,
sa.READONLY)
self._attributes_register(c.STATE, None, sa.ENUM, sa.SCALAR,
sa.READONLY)
self._attributes_register(c.STATE_DETAIL, None, sa.STRING, sa.SCALAR,
sa.READONLY)
self._attributes_register(c.ACCESS, None, sa.URL, sa.SCALAR,
sa.READONLY)
self._attributes_register(c.MANAGER, None, sa.URL, sa.SCALAR,
sa.READONLY)
self._attributes_register(c.DESCRIPTION, None, sa.ANY, sa.SCALAR,
sa.READONLY)
self._attributes_set_enums(c.STATE, [
c.UNKNOWN, c.PENDING, c.ACTIVE, c.CANCELED, c.EXPIRED, c.FAILED,
c.FINAL
])
self._attributes_set_enums(c.RTYPE, [c.COMPUTE, c.STORAGE, c.NETWORK])
self._attributes_set_getter(c.ID, self.get_id)
self._attributes_set_getter(c.RTYPE, self.get_rtype)
self._attributes_set_getter(c.STATE, self.get_state)
self._attributes_set_getter(c.STATE_DETAIL, self.get_state_detail)
self._attributes_set_getter(c.ACCESS, self.get_access)
self._attributes_set_getter(c.MANAGER, self.get_manager)
self._attributes_set_getter(c.DESCRIPTION, self.get_description)
# FIXME: we need the ID to be or to include an URL, as we don't have
# a scheme otherwise, which means we can't select an adaptor. Duh! :-/
# FIXME: documentation for attributes is missing.
# param checks
scheme = None
if not id:
if 'resource_schema' not in _adaptor_state:
raise se.BadParameter("Cannot initialize resource without id" %
self.rtype)
else:
scheme = _adaptor_state['resource_schema']
else:
# ID is formatted as '[manager-url]-[resource-id]'
import parse # FIXME: use regex to reduce number of dependencies
res = parse.parse('[{}]-[{}]', id)
url = ru.Url(res[0])
scheme = url.scheme.lower()
if not session:
session = ss.Session(default=True)
self._base = super(Resource, self)
self._base.__init__(scheme,
_adaptor,
_adaptor_state,
id,
session,
ttype=_ttype)
# --------------------------------------------------------------------------
#
@classmethod
@rus.takes('resource', rus.optional(str), rus.optional(ss.Session),
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns(st.Task)
def create(cls, id=None, session=None, ttype=sc.SYNC):
"""
This is the asynchronous class constructor, returning
a :class:`saga:Task` instance. For details on the accepted parameters,
please see the description of :func:`__init__`.
"""
return cls(id, session, _ttype=ttype)._init_task
# --------------------------------------------------------------------------
#
@rus.takes('Resource', descr.Description,
rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def reconfig(self, descr, ttype=None):
"""
reconfig(descr)
A resource is acquired according to a resource description, i.e. to
a specific set of attributes. At some point in time, while the
resource is running, the application requirements on the resource may
have changed -- in that case, the application can request to change the
resource's configuration on the fly.
This method cannot be used to change the type of the resource. Backends
may or may not support this operation -- if not,
a :class:`saga.NotImplemented` exception is raised. If the method is
supported, , then the semantics of the method is equivalent to the
semantics of the :func:`acquire` call on the
:class:`saga.resource.Manager` class.
"""
return self._adaptor.reconfig(descr, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Resource', str, rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, st.Task))
def destroy(self, ttype=None):
"""
destroy()
The semantics of this method is equivalent to the semantics of the
:func:`destroy` call on the :class:`saga.resource.Manager` class.
"""
return self._adaptor.destroy(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Resource',
rus.optional(
rus.one_of(c.UNKNOWN, c.NEW, c.PENDING, c.ACTIVE, c.DONE,
c.FAILED, c.EXPIRED, c.CANCELED, c.FINAL)),
rus.optional(float), rus.optional(rus.one_of(SYNC, ASYNC,
TASK)))
@rus.returns((rus.nothing, st.Task))
def wait(self, state=c.FINAL, timeout=None, ttype=None):
"""
wait(state=FINAL, timeout=None)
Wait for a resource to enter a specific state.
:param state: resource state to wait for (UNKNOWN, NEW, PENDING, ACTIVE,
DONE, FAILED, EXPIRED, CANCELED, FINAL)
:type state: float
:param state: time to block while waiting.
This method will block until the resource entered the specified state,
or until `timeout` seconds have passed -- whichever occurs earlier. If
the resource is in a final state, the call will raise and
:class:`saga.IncorrectState` exception when asked to wait for any
non-final state.
A negative `timeout` value represents an indefinit timeout.
"""
# FIXME:
# - right now, we can not put a resource in a `TaskContainer`, because
# it is not a `Task`. We need to either reconsider the class
# hierarchy, and then inherit a `ResourceContainer` from
# `TaskContainer` (see job); or we implement a `ResourceContainer`
# from scratch...
return self._adaptor.wait(state, timeout, ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Resource', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, str, st.Task))
def get_id(self, ttype=None):
"""
get_id()
Return the resource ID.
"""
return self._adaptor.get_id(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Resource', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.one_of(c.COMPUTE, c.STORAGE, c.NETWORK), st.Task))
def get_rtype(self, ttype=None):
"""
get_rtype()
Return the resource type.
"""
return self._adaptor.get_rtype(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Resource', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.one_of(c.UNKNOWN, c.NEW, c.PENDING, c.ACTIVE, c.CANCELED,
c.EXPIRED, c.DONE, c.FAILED, c.FINAL), st.Task))
def get_state(self, ttype=None):
"""
get_state()
Return the state of the resource.
"""
return self._adaptor.get_state(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Resource', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, str, st.Task))
def get_state_detail(self, ttype=None):
"""
get_state_detail()
Return the state details (backend specific) of the resource.
"""
return self._adaptor.get_state_detail(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Resource', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, str, st.Task))
def get_access(self, ttype=None):
"""
get_access()
Return the resource access Url.
"""
return self._adaptor.get_access(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Resource', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((str, st.Task))
def get_manager(self, ttype=None):
"""
get_manager()
Return the manager instance that was used to acquire this resource.
"""
return self._adaptor.get_manager(ttype=ttype)
# --------------------------------------------------------------------------
#
@rus.takes('Resource', rus.optional(rus.one_of(SYNC, ASYNC, TASK)))
@rus.returns((rus.nothing, descr.Description, st.Task))
def get_description(self, ttype=None):
"""
get_description()
Return the description that was used to aquire this resource.
"""
return self._adaptor.get_description(ttype=ttype)