class Job(sb.Base, st.Task, sasync.Async): '''Represents a SAGA job as defined in GFD.90 A 'Job' represents a running application instance, which may consist of one or more processes. Jobs are created by submitting a Job description to a Job submission system -- usually a queuing system, or some other service which spawns jobs on the user's behalf. Jobs have a unique ID (see get_job_id()), and are stateful entities -- their 'state' attribute changes according to a well defined state model: A job as returned by job.Service.create(jd) is in 'New' state -- it is not yet submitted to the job submission backend. Once it was submitted, via run(), it will enter the 'Pending' state, where it waits to get actually executed by the backend (e.g. waiting in a queue etc). Once the job is actually executed, it enters the 'Running' state -- only in that state is the job actually consuming resources (CPU, memory, ...). Jobs can leave the 'Running' state in three different ways: they finish successfully on their own ('Done'), they finish unsuccessfully on their own, or get canceled by the job management backend ('Failed'), or they get actively canceled by the user or the application ('Canceled'). The methods defined on the Job object serve two purposes: inspecting the job's state, and initiating job state transitions. ''' # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(basestring), rus.optional(sab.Base), rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(rus.nothing) def __init__(self, _method_type='run', _adaptor=None, _adaptor_state={}, _ttype=None): ''' _adaptor`` references the adaptor class instance which created this task instance. The ``_method_type`` parameter is flattened into the job constructor to satisfy the bulk optimization properties of the saga.Task class, whose interface is implemented by saga.job.Job. ``_method_type`` specifies the SAGA API method which task is representing. For jobs, that is the 'run' method. We don't have a create classmethod -- jobs are never constructed by the user ''' if not _adaptor: raise se.IncorrectState("saga.job.Job constructor is private") self._valid = False # we need to keep _method_type around, for the task interface (see # :class:`saga.Task`) self._method_type = _method_type # We need to specify a schema for adaptor selection -- and # simply choose the first one the adaptor offers. schema = _adaptor.get_schemas()[0] if 'job_schema' in _adaptor_state: schema = _adaptor_state['job_schema'] self._base = super(Job, self) self._base.__init__(schema, _adaptor, _adaptor_state, ttype=None) # set attribute interface properties self._attributes_allow_private(True) self._attributes_extensible(False) self._attributes_camelcasing(True) # register properties with the attribute interface self._attributes_register(STATE, UNKNOWN, sa.ENUM, sa.SCALAR, sa.READONLY) self._attributes_register(EXIT_CODE, None, sa.INT, sa.SCALAR, sa.READONLY) self._attributes_register(CREATED, None, sa.INT, sa.SCALAR, sa.READONLY) self._attributes_register(STARTED, None, sa.INT, sa.SCALAR, sa.READONLY) self._attributes_register(FINISHED, None, sa.INT, sa.SCALAR, sa.READONLY) self._attributes_register(EXECUTION_HOSTS, None, sa.STRING, sa.VECTOR, sa.READONLY) self._attributes_register(ID, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register(SERVICE_URL, None, sa.URL, sa.SCALAR, sa.READONLY) self._attributes_set_enums(STATE, [ UNKNOWN, NEW, PENDING, RUNNING, DONE, FAILED, CANCELED, SUSPENDED ]) self._attributes_set_getter(STATE, self.get_state) self._attributes_set_getter(ID, self.get_id) self._attributes_set_getter(EXIT_CODE, self._get_exit_code) self._attributes_set_getter(CREATED, self._get_created) self._attributes_set_getter(STARTED, self._get_started) self._attributes_set_getter(FINISHED, self._get_finished) self._attributes_set_getter(EXECUTION_HOSTS, self._get_execution_hosts) self._attributes_set_getter(SERVICE_URL, self._get_service_url) self._valid = True # -------------------------------------------------------------------------- # @rus.takes('Job') @rus.returns(basestring) def __str__(self): """ __str__() String representation. Returns the job's ID. """ if not self._valid: return 'no job id' return str(self.id) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, basestring, st.Task)) def get_id(self, ttype=None): """ get_id() Return the job ID. """ id = self._adaptor.get_id(ttype=ttype) return id # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((basestring, st.Task)) def get_description(self, ttype=None): """ get_description() Return the job description this job was created from. The returned description can be used to inspect job properties (executable name, arguments, etc.). It can also be used to start identical job instances. The returned job description will in general reflect the actual state of the running job, and is not necessarily a simple copy of the job description which was used to create the job instance. For example, the environment variables in the returned job description may reflect the actual environment of the running job instance. **Example**:: service = saga.job.Service("fork://localhost") jd = saga.job.Description () jd.executable = '/bin/date' j1 = service.create_job(jd) j1.run() j2 = service.create_job(j1.get_description()) j2.run() service.close() """ return self._adaptor.get_description(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((file, st.Task)) def get_stdin(self, ttype=None): """ get_stdin() Return the job's STDIN handle. """ return self._adaptor.get_stdin(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((file, st.Task)) def get_stdout(self, ttype=None): """ get_stdout() Return the job's STDOUT handle. """ return self._adaptor.get_stdout(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((str, st.Task)) def get_stdout_string(self, ttype=None): """ get_stdout_string() Return the job's STDOUT. """ return self._adaptor.get_stdout_string(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((file, st.Task)) def get_stderr(self, ttype=None): """ get_stderr() Return the job's STDERR handle. ttype: saga.task.type enum ret: File / saga.Task """ return self._adaptor.get_stderr(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((str, st.Task)) def get_stderr_string(self, ttype=None): """ get_stderr_string() Return the job's STDERR. ttype: saga.task.type enum ret: string. """ return self._adaptor.get_stderr_string(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def suspend(self, ttype=None): """ suspend() Suspend the job. """ return self._adaptor.suspend(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def resume(self, ttype=None): """ resume() Resume the job. """ return self._adaptor.resume(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def checkpoint(self, ttype=None): """ checkpoint() Checkpoint the job. """ return self._adaptor.checkpoint(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', descr.Description, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def migrate(self, jd, ttype=None): """ jd: saga.job.Description ttype: saga.task.type enum ret: None / saga.Task """ return self._adaptor.migrate(jd, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', int, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def signal(self, signum, ttype=None): """ signal(signum) Send a signal to the job. :param signum: signal to send :type signum: int """ return self._adaptor.signal(signum, ttype=ttype) id = property(get_id) # string description = property(get_description) # Description #stdin = property (get_stdin) # File stdout = property(get_stdout) # File stderr = property(get_stderr) # File #----------------------------------------------------------------- # # task methods flattened into job :-/ # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def run(self, ttype=None): """ run() Run (start) the job. Request that the job is being executed by the backend. If the backend is accepting this run request, the job will move to the 'Pending' or 'Running' state -- otherwise this method will raise an error, and the job will be moved to 'Failed'. **Example**:: js = saga.job.Service("fork://localhost") jd = saga.job.Description () jd.executable = '/bin/date' j = js.create_job(jd) if j.get_state() == saga.job.NEW : print "new" else : print "oops!" j.run() if j.get_state() == saga.job.PENDING : print "pending" elif j.get_state() == saga.job.RUNNING : print "running" else : print "oops!" """ return self._adaptor.run(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', float, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def cancel(self, timeout=None, ttype=None): """ cancel(timeout) Cancel the execution of the job. :param timeout: `cancel` will return after timeout :type timeout: float **Example**:: js = saga.job.Service("fork://localhost") jd = saga.job.Description () jd.executable = '/bin/date' j = js.create_job(jd) if j.get_state() == saga.job.NEW : print "new" else : print "oops!" j.run() if j.get_state() == saga.job.PENDING : print "pending" elif j.get_state() == saga.job.RUNNING : print "running" else : print "oops!" j.cancel() if j.get_state() == saga.job.CANCELED : print "canceled" else : print "oops!" """ return self._adaptor.cancel(timeout, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(float), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((bool, st.Task)) def wait(self, timeout=None, ttype=None): """ wait(timeout) :param timeout: `wait` will return after timeout :type timeout: float Wait for a running job to finish execution. The optional timeout parameter specifies the time to wait, and accepts the following values:: timeout < 0 : wait forever (block) -- same for 'None' timeout == 0 : wait not at all (non-blocking test) timeout > 0 : wait for 'timeout' seconds On a non-negative timeout, the call can thus return even if the job is not in final state, and the application should check the actual job state. The default timeout value is 'None' (blocking). **Example**:: js = saga.job.Service("fork://localhost") jd = saga.job.Description () jd.executable = '/bin/date' j = js.create_job(jd) if j.get_state() == saga.job.NEW : print "new" else : print "oops!" j.run() if j.get_state() == saga.job.PENDING : print "pending" elif j.get_state() == saga.job.RUNNING : print "running" else : print "oops!" j.wait(-1.0) if j.get_state() == saga.job.DONE : print "done" elif j.get_state() == saga.job.FAILED : print "failed" else : print "oops!" """ if None == timeout: timeout = -1.0 # FIXME return self._adaptor.wait(timeout, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.one_of(UNKNOWN, NEW, PENDING, RUNNING, SUSPENDED, DONE, FAILED, CANCELED), st.Task)) def get_state(self, ttype=None): """ get_state() Return the current state of the job. **Example**:: js = saga.job.Service("fork://localhost") jd = saga.job.Description () jd.executable = '/bin/date' j = js.create_job(jd) if j.get_state() == saga.job.NEW : print "new" else : print "oops!" j.run() if j.get_state() == saga.job.PENDING : print "pending" elif j.get_state() == saga.job.RUNNING : print "running" else : print "oops!" """ return self._adaptor.get_state(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.anything, st.Task)) def get_result(self, ttype=None): """ get_result() """ return self._adaptor.get_result(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((sb.Base, st.Task)) def get_object(self, ttype=None): """ :todo: describe me :note: this will return the job_service which created the job. """ return self._adaptor.get_object(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((se.SagaException, st.Task)) def get_exception(self, ttype=None): """ :todo: describe me :note: if job failed, that will get an exception describing why, if that exists. Otherwise, the call returns None. """ # FIXME: add CPI return self._adaptor.get_exception(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job') @rus.returns(rus.nothing) def re_raise(self): """ :todo: describe me :note: if job failed, that will re-raise an exception describing why, if that exists. Otherwise, the call does nothing. """ self._adaptor.re_raise() # ---------------------------------------------------------------- # # attribute getters # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, int, st.Task)) def _get_exit_code(self, ttype=None): ec = self._adaptor.get_exit_code(ttype=ttype) if ec in [None, ""]: return None else: # Exit code is always an int. If this 'cast' fails, # the adaptor is doing something stupid. return ec # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, float, st.Task)) def _get_created(self, ttype=None): return self._adaptor.get_created(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, float, st.Task)) def _get_started(self, ttype=None): return self._adaptor.get_started(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, float, st.Task)) def _get_finished(self, ttype=None): return self._adaptor.get_finished(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, rus.list_of(basestring), st.Task)) def _get_execution_hosts(self, ttype=None): return self._adaptor.get_execution_hosts(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Job', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, surl.Url, st.Task)) def _get_service_url(self, ttype=None): return self._adaptor.get_service_url(ttype=ttype) state = property(get_state) # state enum result = property(get_result) # result type (None) object = property(get_object) # object type (job_service) exception = property(re_raise) # exception type # ---------------------------------------------------------------- # ExitCode = property(doc=""" The job's exitcode. this attribute is only meaningful if the job is in 'Done' or 'Final' state - for all other job states, this attribute value is undefined. **Example**:: js = saga.job.Service("fork://localhost") jd = saga.job.Description () jd.executable = '/bin/date' j = js.create_job(jd) j.run() j.wait() if j.get_state() == saga.job.FAILED : if j.exitcode == "42" : print "Ah, galaxy bypass error!" else : print "oops!" """) # ---------------------------------------------------------------- # JobID = property(doc=""" The job's identifier. This attribute is equivalent to the value returned by job.get_job_id() """) # ---------------------------------------------------------------- # ServiceURL = property(doc=""" The URL of the :class:`saga.job.Service` instance managing this job. This attribute is represents the URL under where the job management service can be contacted which owns the job. The value is equivalent to the service part of the job_id. **Example**:: js = saga.job.Service("fork://localhost") jd = saga.job.Description () jd.executable = '/bin/date' j = js.create_job(jd) if j.serviceurl == "fork://localhost" : print "yes!" else : print "oops!" """)
class Directory(nsdir.Directory, sa.Attributes): # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional((ru.Url, str)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(sab.Base), rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(rus.nothing) def __init__(self, url=None, flags=READ, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): ''' url: saga.Url flags: flags enum session: saga.Session ret: obj ''' # param checks if not flags: flags = 0 url = ru.Url(url) self._nsdirec = super(Directory, self) self._nsdirec.__init__(url, flags, session, _adaptor, _adaptor_state, _ttype=_ttype) # set attribute interface properties self._attributes_allow_private(True) self._attributes_camelcasing(True) self._attributes_extensible(True, getter=self._attribute_getter, setter=self._attribute_setter, lister=self._attribute_lister, caller=self._attribute_caller) # register properties with the attribute interface self._attributes_register(ATTRIBUTE, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register(CHANGE, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register(NEW, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register(DELETE, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register(TTL, None, sa.INT, sa.SCALAR, sa.WRITEABLE) self._attributes_set_setter(TTL, self.set_ttl) self._attributes_set_getter(TTL, self.get_ttl) # -------------------------------------------------------------------------- # @classmethod @rus.takes('Directory', rus.optional((ru.Url, str)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(st.Task) def create(cls, url=None, flags=READ, session=None, ttype=None): ''' url: saga.Url flags: saga.advert.flags enum session: saga.Session ttype: saga.task.type enum ret: saga.Task ''' if not flags: flags = 0 _nsdir = super(Directory, cls) return _nsdir.create(url, flags, session, ttype=ttype) # -------------------------------------------------------------------------- # # attribute methods # # NOTE: we do not yet pass ttype, as async calls are not yet supported by # the attribute interface # @rus.takes('Directory', str, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.anything, st.Task)) def _attribute_getter(self, key, ttype=None): return self._adaptor.attribute_getter(key) # -------------------------------------------------------------------------- # @rus.takes('Directory', str, rus.anything, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def _attribute_setter(self, key, val, ttype=None): return self._adaptor.attribute_setter(key, val) # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(rus.anything), st.Task)) def _attribute_lister(self, ttype=None): return self._adaptor.attribute_lister() # -------------------------------------------------------------------------- # @rus.takes('Directory', str, int, callable, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.anything, st.Task)) def _attribute_caller(self, key, id, cb, ttype=None): return self._adaptor.attribute_caller(key, id, cb) # ---------------------------------------------------------------- # # advert methods # @rus.takes('Directory', (ru.Url, str), float, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def set_ttl(self, tgt=None, ttl=-1.0, ttype=None): """ tgt : saga.Url / None ttl : int ttype: saga.task.type enum ret: None / saga.Task """ if tgt: return self._adaptor.set_ttl(tgt, ttl, ttype=ttype) else: return self._adaptor.set_ttl_self(ttl, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional((ru.Url, str)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((float, st.Task)) def get_ttl(self, tgt=None, ttype=None): """ tgt : saga.Url / None ttype: saga.task.type enum ret: int / saga.Task """ if tgt: return self._adaptor.get_ttl(tgt, ttype=ttype) else: return self._adaptor.get_ttl_self(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional(str), rus.optional(str), rus.optional((str, object)), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(ru.Url), st.Task)) def find(self, name_pattern, attr_pattern=None, obj_type=None, flags=RECURSIVE, ttype=None): """ name_pattern: string attr_pattern: string obj_type: string flags: flags enum ret: list [saga.Url] """ if not flags: flags = 0 if attr_pattern or obj_type: return self._adaptor.find_adverts(name_pattern, attr_pattern, obj_type, flags, ttype=ttype) else: return self._nsdirec.find(name_pattern, flags, ttype=ttype)
class Message(sa.Attributes): # -------------------------------------------------------------------------- # @rus.takes('Message', rus.optional(int, rus.nothing)) # size @rus.returns(rus.nothing) def __init__(self, size=None): ''' size: expected size of buffer when used -- informative, not normative ret: Message ''' # set attribute interface properties self._attributes_camelcasing(True) self._attributes_allow_private(True) self._attributes_extensible(True, getter=self._attribute_getter, setter=self._attribute_setter, lister=self._attribute_lister, caller=self._attribute_caller) # register properties with the attribute interface self._attributes_register(ID, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register(SENDER, None, sa.STRING, sa.SCALAR, sa.READONLY) # -------------------------------------------------------------------------- # # class methods # @rus.takes('Message') @rus.returns(rus.anything) def get_id(self): return self._adaptor.get_id() # -------------------------------------------------------------------------- # @rus.takes('Message') @rus.returns(rus.anything) def get_sender(self): return self._adaptor.get_sender() # -------------------------------------------------------------------------- # # attribute methods # @rus.takes('Message', basestring) @rus.returns(rus.anything) def _attribute_getter(self, key): return self._adaptor.attribute_getter(key) # -------------------------------------------------------------------------- # @rus.takes('Message', basestring, rus.anything) @rus.returns(rus.nothing) def _attribute_setter(self, key, val): return self._adaptor.attribute_setter(key, val) # -------------------------------------------------------------------------- # @rus.takes('Message') @rus.returns(rus.list_of(rus.anything)) def _attribute_lister(self): return self._adaptor.attribute_lister() # -------------------------------------------------------------------------- # @rus.takes('Message', basestring, int, callable) @rus.returns(rus.anything) def _attribute_caller(self, key, id, cb): return self._adaptor.attribute_caller(key, id, cb)
class Service (sb.Base, sasync.Async) : """ The job.Service represents a resource management backend, and as such allows the creation, submission and management of jobs. A job.Service represents anything which accepts job creation requests, and which manages thus created :class:`saga.job.Job` instances. That can be a local shell, a remote ssh shell, a cluster queuing system, a IaaS backend -- you name it. The job.Service is identified by an URL, which usually points to the contact endpoint for that service. Example:: service = saga.job.Service("fork://localhost") ids = service.list() for job_id in ids : print job_id j = service.get_job(job_id) if j.get_state() == saga.job.Job.Pending: print "pending" elif j.get_state() == saga.job.Job.Running: print "running" else: print "job is already final!" service.close() """ # -------------------------------------------------------------------------- # @rus.takes ('Service', rus.optional ((basestring, surl.Url)), rus.optional (ss.Session), rus.optional (sab.Base), rus.optional (dict), rus.optional (rus.one_of (SYNC, ASYNC, TASK))) @rus.returns (rus.nothing) def __init__ (self, rm=None, session=None, _adaptor=None, _adaptor_state={}, _ttype=None) : """ __init__(rm, session) Create a new job.Service instance. :param rm: resource manager URL :type rm: string or :class:`saga.Url` :param session: an optional session object with security contexts :type session: :class:`saga.Session` :rtype: :class:`saga.job.Service` """ # job service instances are resource hogs. Before attempting to create # a new instance, we attempt to clear out all old instances. There is # some collateral damage: we cannot run the Python GC over only the # job.Service instances, but have to run it globally -- however, # compared to the latency introduced by the job service setup, this # should be a minor inconvenienve (tm) try : import gc gc.collect () except : pass # param checks self.valid = False url = surl.Url (rm) if not url.scheme : url.scheme = 'fork' if not url.host : url.host = 'localhost' if not session : session = ss.Session (default=True) scheme = url.scheme.lower () self._super = super (Service, self) self._super.__init__ (scheme, _adaptor, _adaptor_state, url, session, ttype=_ttype) self.valid = True # -------------------------------------------------------------------------- # @classmethod @rus.takes ('Service', rus.optional ((surl.Url, basestring)), rus.optional (ss.Session), rus.optional (rus.one_of (SYNC, ASYNC, TASK))) @rus.returns (st.Task) def create (cls, rm=None, session=None, ttype=SYNC) : """ create(rm=None, session=None) Create a new job.Service instance asynchronously. :param rm: resource manager URL :type rm: string or :class:`saga.Url` :param session: an optional session object with security contexts :type session: :class:`saga.Session` :rtype: :class:`saga.Task` """ # param checks if not session : session = ss.Session (default=True) url = surl.Url (rm) scheme = url.scheme.lower () return cls (url, session, _ttype=ttype)._init_task # -------------------------------------------------------------------------- # @rus.takes ('Service') @rus.returns (basestring) def __str__ (self): """ __str__() String representation. Returns the job service Url. """ if self.valid : return "[%s]" % self.url return "" # -------------------------------------------------------------------------- # @rus.takes ('Service') @rus.returns (rus.nothing) def close (self) : """ close() Close the job service instance and disconnect from the (remote) job service if necessary. Any subsequent calls to a job service instance after `close()` was called will fail. Example:: service = saga.job.Service("fork://localhost") # do something with the 'service' object, create jobs, etc... service.close() service.list() # this call will throw an exception .. warning:: While in principle the job service destructor calls `close()` automatically when a job service instance goes out of scope, you **shouldn't rely on it**. Python's garbage collection can be a bit odd at times, so you should always call `close()` explicitly. Especially in a **multi-threaded program** this will help to avoid random errors. """ if not self.valid : raise se.IncorrectState ("This instance was already closed.") self._adaptor.close () self.valid = False # -------------------------------------------------------------------------- # @rus.takes ('Service', descr.Description, rus.optional (rus.one_of (SYNC, ASYNC, TASK))) @rus.returns ((j.Job, st.Task)) def create_job (self, job_desc, ttype=None) : """ create_job(job_desc) Create a new job.Job instance from a :class:`~saga.job.Description`. The resulting job instance is in :data:`~saga.job.NEW` state. :param job_desc: job description to create the job from :type job_desc: :data:`saga.job.Description` :param ttype: |param_ttype| :rtype: :class:`saga.job.Job` or |rtype_ttype| create_job() accepts a job description, which described the application instance to be created by the backend. The create_job() method is not actually attempting to *run* the job, but merely parses the job description for syntactic and semantic consistency. The job returned object is thus not in 'Pending' or 'Running', but rather in 'New' state. The actual submission is performed by calling run() on the job object. Example:: # A job.Description object describes the executable/application and its requirements job_desc = saga.job.Description() job_desc.executable = '/bin/sleep' job_desc.arguments = ['10'] job_desc.output = 'myjob.out' job_desc.error = 'myjob.err' service = saga.job.Service('local://localhost') job = service.create_job(job_desc) # Run the job and wait for it to finish job.run() print "Job ID : %s" % (job.job_id) job.wait() # Get some info about the job print "Job State : %s" % (job.state) print "Exitcode : %s" % (job.exit_code) service.close() """ if not self.valid : raise se.IncorrectState ("This instance was already closed.") jd_copy = descr.Description() job_desc._attributes_deep_copy (jd_copy) # do some sanity checks: if the adaptor has specified a set of supported # job description attributes, we scan the given description for any # mismatches, and complain then. adaptor_info = self._adaptor._adaptor.get_info () if 'capabilities' in adaptor_info and \ 'jdes_attributes' in adaptor_info['capabilities'] : # this is the list of key supported by the adaptor. These # attributes may be set to non-default values supported_keys = adaptor_info['capabilities']['jdes_attributes'] # use an empty job description to compare default values jd_default = descr.Description () for key in jd_copy.list_attributes () : val = jd_copy .get_attribute (key) default = jd_default.get_attribute (key) # Also, we make string compares case insensitive if isinstance (val, basestring) : val = val .lower () if isinstance (default, basestring) : default = default.lower () # supported keys are also valid, as are keys with default or # None values if key not in supported_keys and \ val != default and \ val : msg = "'JobDescription.%s' (%s) is not supported by adaptor %s" \ % (key, val, adaptor_info['name']) raise se.BadParameter._log (self._logger, msg) # make sure at least 'executable' is defined if jd_copy.executable is None: raise se.BadParameter("No executable defined") # convert environment to string if jd_copy.attribute_exists ('Environment') : for (key, value) in jd_copy.environment.iteritems(): jd_copy.environment[key] = str(value) return self._adaptor.create_job (jd_copy, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes ('Service', basestring, rus.optional (basestring), rus.optional (rus.one_of (SYNC, ASYNC, TASK))) @rus.returns ((j.Job, st.Task)) def run_job (self, cmd, host=None, ttype=None) : """ run_job(cmd, host=None) """ if not self.valid : raise se.IncorrectState ("This instance was already closed.") if not cmd: raise se.BadParameter('run_job needs a command to run. Duh!') try: # lets see if the adaptor implements run_job return self._adaptor.run_job (cmd, host, ttype=ttype) except: # fall back to the default implementation below pass # The adaptor has no run_job -- we here provide a generic implementation # FIXME: split should be more clever and respect POSIX shell syntax. args = cmd.split() jd = descr.Description() jd.executable = args[0] jd.arguments = args[1:] job = self.create_job(jd) job.run() return job # -------------------------------------------------------------------------- # @rus.takes ('Service', rus.optional (rus.one_of (SYNC, ASYNC, TASK))) @rus.returns ((rus.list_of (basestring), st.Task)) def list (self, ttype=None) : """ list() Return a list of the jobs that are managed by this Service instance. .. seealso:: The :data:`~saga.job.Service.jobs` property and the :meth:`~saga.job.Service.list` method are semantically equivalent. :ttype: |param_ttype| :rtype: list of :class:`saga.job.Job` As the job.Service represents a job management backend, list() will return a list of job IDs for all jobs which are known to the backend, and which can potentially be accessed and managed by the application. Example:: service = saga.job.Service("fork://localhost") ids = service.list() for job_id in ids : print job_id service.close() """ if not self.valid : raise se.IncorrectState ("This instance was already closed.") return self._adaptor.list (ttype=ttype) jobs = property (list) # -------------------------------------------------------------------------- # @rus.takes ('Service', rus.optional (rus.one_of (SYNC, ASYNC, TASK))) @rus.returns ((surl.Url, st.Task)) def get_url (self, ttype=None) : """ get_url() Return the URL this Service instance was created with. .. seealso:: The :data:`~saga.job.Service.url` property and the :meth:`~saga.job.Service.get_url` method are semantically equivalent and only duplicated for convenience. """ if not self.valid : raise se.IncorrectState ("This instance was already closed.") return self._adaptor.get_url (ttype=ttype) url = property (get_url) # -------------------------------------------------------------------------- # @rus.takes ('Service', basestring, rus.optional (rus.one_of (SYNC, ASYNC, TASK))) @rus.returns ((j.Job, st.Task)) def get_job (self, job_id, ttype=None) : """ get_job(job_id) Return the job object for a given job id. :param job_id: The id of the job to retrieve :rtype: :class:`saga.job.Job` Job objects are a local representation of a remote stateful entity. The job.Service supports to reconnect to those remote entities:: service = saga.job.Service("fork://localhost") j = service.get_job(my_job_id) if j.get_state() == saga.job.Job.Pending: print "pending" elif j.get_state() == saga.job.Job.Running: print "running" else: print "job is already final!" service.close() """ if not self.valid : raise se.IncorrectState ("This instance was already closed.") return self._adaptor.get_job (job_id, ttype=ttype)
class Directory(entry.Entry): ''' Represents a SAGA directory as defined in GFD.90 The saga.namespace.Directory class represents, as the name indicates, a directory on some (local or remote) namespace. That class offers a number of operations on that directory, such as listing its contents, copying entries, or creating subdirectories:: # get a directory handle dir = saga.namespace.Directory("sftp://localhost/tmp/") # create a subdir dir.make_dir ("data/") # list contents of the directory entries = dir.list () # copy *.dat entries into the subdir for f in entries : if f ^ '^.*\.dat$' : dir.copy (f, "sftp://localhost/tmp/data/") Implementation note: ^^^^^^^^^^^^^^^^^^^^ The SAGA API Specification (GFD.90) prescribes method overloading on method signatures, but that is not supported by Python (Python only does method overwriting). So we implement one generic method version here, and do the case switching based on the provided parameter set. ''' # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(sab.Base), rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(rus.nothing) def __init__(self, url=None, flags=None, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): ''' :param url: Url of the (remote) entry system directory. :type url: :class:`saga.Url` flags: flags enum session: saga.Session ret: obj Construct a new directory object The specified directory is expected to exist -- otherwise a DoesNotExist exception is raised. Also, the URL must point to a directory (not to an entry), otherwise a BadParameter exception is raised. Example:: # open some directory dir = saga.namespace.Directory("sftp://localhost/tmp/") # and list its contents entries = dir.list () ''' if not flags: flags = 0 self._nsentry = super(Directory, self) self._nsentry.__init__(url, flags, session, _adaptor, _adaptor_state, _ttype=_ttype) # -------------------------------------------------------------------------- # @classmethod @rus.takes('Directory', rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(st.Task) def create(cls, url=None, flags=None, session=None, ttype=None): ''' url: saga.Url flags: saga.namespace.flags enum session: saga.Session ttype: saga.task.type enum ret: saga.Task ''' if not flags: flags = 0 _nsentry = super(Directory, cls) return _nsentry.create(url, flags, session, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', (ru.Url, basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((entry.Entry, st.Task)) def open(self, name, flags=None, ttype=None): ''' name: saga.Url flags: saga.namespace.flags enum ttype: saga.task.type enum ret: saga.namespace.Entry / saga.Task ''' if not flags: flags = 0 url = ru.Url(name) return self._adaptor.open(url, flags, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', (ru.Url, basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(('Directory', st.Task)) def open_dir(self, path, flags=None, ttype=None): ''' :param path: name/path of the directory to open :param flags: directory creation flags ttype: saga.task.type enum ret: saga.namespace.Directory / saga.Task Open and return a new directoy The call opens and returns a directory at the given location. Example:: # create a subdir 'data' in /tmp dir = saga.namespace.Directory("sftp://localhost/tmp/") data = dir.open_dir ('data/', saga.namespace.Create) ''' if not flags: flags = 0 return self._adaptor.open_dir(ru.Url(path), flags, ttype=ttype) # ---------------------------------------------------------------- # # namespace directory methods # @rus.takes('Directory', (ru.Url, basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def make_dir(self, tgt, flags=0, ttype=None): ''' :param tgt: name/path of the new directory :param flags: directory creation flags ttype: saga.task.type enum ret: None / saga.Task Create a new directoy The call creates a directory at the given location. Example:: # create a subdir 'data' in /tmp dir = saga.namespace.Directory("sftp://localhost/tmp/") dir.make_dir ('data/') ''' if not flags: flags = 0 return self._adaptor.make_dir(ru.Url(tgt), flags, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', (ru.Url, basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def change_dir(self, url, flags=0, ttype=None): ''' url: saga.Url flags: flags enum ttype: saga.task.type enum ret: None / saga.Task ''' if not flags: flags = 0 return self._adaptor.change_dir(url, flags=flags, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional(basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(ru.Url), st.Task)) def list(self, pattern=None, flags=0, ttype=None): ''' :param pattern: Entry name pattern (like POSIX 'ls', e.g. '\*.txt') flags: flags enum ttype: saga.task.type enum ret: list [saga.Url] / saga.Task List the directory's content The call will return a list of entries and subdirectories within the directory:: # list contents of the directory for f in dir.list() : print f ''' if not flags: flags = 0 return self._adaptor.list(pattern, flags, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', (ru.Url, basestring), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((bool, st.Task)) def exists(self, path, ttype=None): ''' :param path: path of the entry to check ttype: saga.task.type enum ret: bool / saga.Task Returns True if path exists, False otherwise. Example:: # inspect an entry dir = saga.namespace.Directory("sftp://localhost/tmp/") if dir.exists ('data'): # do something ''' return self._adaptor.exists(path, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional(basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(ru.Url), st.Task)) def find(self, pattern, flags=c.RECURSIVE, ttype=None): ''' pattern: string flags: flags enum ttype: saga.task.type enum ret: list [saga.Url] / saga.Task ''' if not flags: flags = 0 return self._adaptor.find(pattern, flags, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((int, st.Task)) def get_num_entries(self, ttype=None): ''' ttype: saga.task.type enum ret: int / saga.Task ''' return self._adaptor.get_num_entries(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', int, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((ru.Url, st.Task)) def get_entry(self, num, ttype=None): ''' num: int ttype: saga.task.type enum ret: saga.Url / saga.Task ''' return self._adaptor.get_entry(num, ttype=ttype) # ---------------------------------------------------------------- # # methods overloaded from namespace.Entry # @rus.takes('Directory', (ru.Url, basestring), rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def copy(self, url_1, url_2=None, flags=0, ttype=None): ''' :param src: path of the entry to copy :param tgt: absolute URL of target name or directory url_1: saga.Url url_2: saga.Url / None flags: flags enum / None ttype: saga.task.type enum / None ret: None / saga.Task Copy an entry from source to target The source is copied to the given target directory. The path of the source can be relative:: # copy an entry dir = saga.namespace.Directory("sftp://localhost/tmp/") dir.copy ("./data.bin", "sftp://localhost/tmp/data/") ''' # FIXME: re-implement the url switching (commented out below) if not flags: flags = 0 if url_2: return self._adaptor.copy(url_1, url_2, flags, ttype=ttype) else: return self._nsentry.copy(url_1, flags, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', (ru.Url, basestring), rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def link(self, url_1, url_2=None, flags=0, ttype=None): ''' src: saga.Url tgt: saga.Url flags: flags enum ttype: saga.task.type enum ret: None / saga.Task ''' if not flags: flags = 0 if url_2: return self._adaptor.link(url_1, url_2, flags, ttype=ttype) else: return self._nsentry.link(url_1, flags, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', (ru.Url, basestring), rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def move(self, url_1, url_2=None, flags=0, ttype=None): ''' :param src: path of the entry to copy :param tgt: absolute URL of target directory flags: flags enum ttype: saga.task.type enum ret: None / saga.Task Move an entry from source to target The source is moved to the given target directory. The path of the source can be relative:: # copy an entry dir = saga.namespace.Directory("sftp://localhost/tmp/") dir.move ("./data.bin", "sftp://localhost/tmp/data/") ''' if not flags: flags = 0 if url_2: return self._adaptor.move(url_1, url_2, flags, ttype=ttype) else: return self._nsentry.move(url_1, flags, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', (ru.Url, basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def remove(self, tgt=None, flags=0, ttype=None): ''' tgt: saga.Url flags: flags enum ttype: saga.task.type enum ret: None / saga.Task ''' if not flags: flags = 0 if tgt: return self._adaptor.remove(tgt, flags, ttype=ttype) else: return self._nsentry.remove(flags, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional((ru.Url, basestring)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((bool, st.Task)) def is_dir(self, tgt=None, ttype=None): ''' tgt: saga.Url / None ttype: saga.task.type enum ret: bool / saga.Task Returns True if path is a directory, False otherwise. Example:: # inspect an entry dir = saga.namespace.Directory("sftp://localhost/tmp/") if dir.is_dir ('data'): # do something ''' if tgt: return self._adaptor.is_dir(tgt, ttype=ttype) else: return self._nsentry.is_dir(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional((ru.Url, basestring)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((bool, st.Task)) def is_entry(self, tgt=None, ttype=None): ''' tgt: saga.Url / None ttype: saga.task.type enum ret: bool / saga.Task ''' if tgt: return self._adaptor.is_entry(tgt, ttype=ttype) else: return self._nsentry.is_entry(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional((ru.Url, basestring)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((bool, st.Task)) def is_link(self, tgt=None, ttype=None): ''' tgt: saga.Url / None ttype: saga.task.type enum ret: bool / saga.Task ''' if tgt: return self._adaptor.is_link(tgt, ttype=ttype) else: return self._nsentry.is_link(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Directory', rus.optional((ru.Url, basestring)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((ru.Url, st.Task)) def read_link(self, tgt=None, ttype=None): ''' tgt: saga.Url / None ttype: saga.task.type enum ret: saga.Url / saga.Task ''' if tgt: return self._adaptor.read_link(tgt, ttype=ttype) else: return self._nsentry.read_link(ttype=ttype)
class Container(sbase.SimpleBase, satt.Attributes): # -------------------------------------------------------------------------- # @rus.takes('Container') @rus.returns(rus.nothing) def __init__(self): self._base = super(Container, self) self._base.__init__() # set attribute interface properties self._attributes_allow_private(True) self._attributes_extensible(False) self._attributes_camelcasing(True) # register properties with the attribute interface self._attributes_register(SIZE, 0, satt.INT, satt.SCALAR, satt.READONLY) self._attributes_set_getter(SIZE, self.get_size) self._attributes_register(TASKS, [], satt.ANY, satt.VECTOR, satt.READONLY) self._attributes_set_getter(TASKS, self.get_tasks) self._attributes_register(STATES, [], satt.ENUM, satt.VECTOR, satt.READONLY) self._attributes_set_getter(STATES, self.get_states) self._attributes_set_enums( STATES, [UNKNOWN, NEW, RUNNING, DONE, FAILED, CANCELED]) # cache for created container instances self._containers = {} # -------------------------------------------------------------------------- # def __str__(self): ret = "[" for task in self.tasks: ret += "'%s', " % str(task) ret += "]" return ret # -------------------------------------------------------------------------- # @rus.takes('Container', Task) @rus.returns(rus.nothing) def add(self, task): if not isinstance(task, Task): raise se.BadParameter ("Container handles tasks, not %s" \ % (type(task))) if not task in self.tasks: self.tasks.append(task) # -------------------------------------------------------------------------- # @rus.takes('Container', Task) @rus.returns(rus.nothing) def remove(self, task): if task in self.tasks: self.tasks.delete(task) # -------------------------------------------------------------------------- # @rus.takes('Container') @rus.returns(rus.nothing) def run(self): if not len(self.tasks): # nothing to do return None buckets = self._get_buckets() futures = [] # futures running container ops # handle all container for c in buckets['bound']: # handle all methods for m in buckets['bound'][c]: tasks = buckets['bound'][c][m] m_name = "container_%s" % m m_handle = None for (name, handle) in inspect.getmembers(c, predicate=inspect.ismethod): if name == m_name: m_handle = handle break if not handle: # Hmm, the specified container can't handle the call after # all -- fall back to the unbound handling buckets['unbound'] += tasks else: # hand off to the container function, in a separate task futures.append(ru.Future.Run(m_handle, tasks)) # handle tasks not bound to a container for task in buckets['unbound']: futures.append(ru.Future.Run(task.run)) # wait for all futures to finish for future in futures: if future.isAlive(): future.join() if future.state == FAILED: raise se.NoSuccess ("future exception: %s" \ % (future.exception)) # -------------------------------------------------------------------------- # # -------------------------------------------------------------------------- # @rus.takes('Container', rus.one_of(ANY, ALL), rus.optional(float)) @rus.returns(rus.list_of(Task)) def wait(self, mode=ALL, timeout=None): if None == timeout: timeout = -1.0 # FIXME if not mode in [ANY, ALL]: raise se.BadParameter( "wait mode must be saga.task.ANY or saga.task.ALL") if type(timeout) not in [int, long, float]: raise se.BadParameter( "wait timeout must be a floating point number (or integer)") if not len(self.tasks): # nothing to do return None if mode == ALL: return self._wait_all(timeout) else: return self._wait_any(timeout) # -------------------------------------------------------------------------- # @rus.takes('Container', float) @rus.returns(rus.list_of(Task)) def _wait_any(self, timeout): buckets = self._get_buckets() futures = [] # futures running container ops # handle all tasks bound to containers for c in buckets['bound']: # handle all methods -- all go to the same 'container_wait' though) tasks = [] for m in buckets['bound'][c]: tasks += buckets['bound'][c][m] futures.append(ru.Future.Run(c.container_wait, tasks, ANY, timeout)) # handle all tasks not bound to containers for task in buckets['unbound']: futures.append(ru.Future.Run(task.wait, timeout)) # mode == ANY: we need to watch our futures, and whenever one # returns, and declare success. Note that we still need to get the # finished task from the 'winner'-future -- we do that via a Queue # object. Note also that looser futures are not canceled, but left # running (FIXME: consider sending a signal at least) timeout = 0.01 # seconds, heuristic :-/ for future in futures: future.join(timeout) if future.state == FAILED: raise future.exception if not future.isAlive(): # future indeed finished -- dig return value from this # futures queue result = future.result # ignore other futures, and simply declare success return result # -------------------------------------------------------------------------- # @rus.takes('Container', float) @rus.returns(rus.list_of(Task)) def _wait_all(self, timeout): # this method should actually be symmetric to _wait_any, and could # almost be mapped to it, but the code below is a kind of optimization # (does not need futures, thus simpler code). buckets = self._get_buckets() ret = None # handle all tasks bound to containers for c in buckets['bound']: # handle all methods -- all go to the same 'container_wait' though) tasks = [] for m in buckets['bound'][c]: tasks += buckets['bound'][c][m] # TODO: this is semantically not correct: timeout is applied # n times... c.container_wait(tasks, ALL, timeout) ret = tasks[0] # handle all tasks not bound to containers for task in buckets['unbound']: task.wait() ret = task # all done - return random task (first from last container, or last # unbound task) # FIXME: that task should be removed from the task container return ret # -------------------------------------------------------------------------- # @rus.takes('Container', rus.optional(float)) @rus.returns(rus.nothing) def cancel(self, timeout=None): if None == timeout: timeout = -1.0 # FIXME buckets = self._get_buckets() futures = [] # futures running container ops # handle all tasks bound to containers for c in buckets['bound']: # handle all methods -- all go to the same 'container_cancel' though) tasks = [] for m in buckets['bound'][c]: tasks += buckets['bound'][c][m] futures.append(ru.Future.Run(c.container_cancel, tasks, timeout)) # handle all tasks not bound to containers for task in buckets['unbound']: futures.append(ru.Future.Run(task.cancel, timeout)) for future in futures: future.join() # ---------------------------------------------------------------- # @rus.takes('Container') @rus.returns(int) def get_size(self): return len(self.tasks) # -------------------------------------------------------------------------- # @rus.takes('Container', 'basestring') @rus.returns(Task) def get_task(self, id): # FIXME: this should not be a search, but a lookup if not id: raise se.NoSuccess("Lookup requires non-empty id (not '%s')" % id) for t in self.tasks: if t.id == id: return t raise se.NoSuccess("task '%s' not found in container" % id) # -------------------------------------------------------------------------- # @rus.takes('Container') @rus.returns(rus.list_of(Task)) def get_tasks(self): return self.tasks # -------------------------------------------------------------------------- # @rus.takes('Container') @rus.returns( rus.list_of(rus.one_of(UNKNOWN, NEW, RUNNING, DONE, FAILED, CANCELED))) def get_states(self): buckets = self._get_buckets() futures = [] # futures running container ops # handle all tasks bound to containers for c in buckets['bound']: # handle all methods -- all go to the same 'container_get_states' though) tasks = [] for m in buckets['bound'][c]: tasks += buckets['bound'][c][m] futures.append(ru.Future.Run(c.container_get_states, tasks)) # handle all tasks not bound to containers for task in buckets['unbound']: futures.append(ru.Future.Run(task.get_state)) # We still need to get the states from all futures. # FIXME: order states = [] for future in futures: future.join() if future.state == FAILED: raise future.exception # FIXME: what about ordering tasks / states? res = future.result if res != None: states += res return states # ---------------------------------------------------------------- # @rus.takes('Container') @rus.returns(dict) def _get_buckets(self): # collective container ops: walk through the task list, and sort into # buckets of tasks which have (a) the same task._container, or if that # is not set, the same class type (for which one container instance is # created). All tasks were neither is available are handled one-by-one buckets = {} buckets['unbound'] = [] # no container adaptor for these [tasks] buckets['bound'] = {} # dict of container adaptors [tasks] for task in self.tasks: if task._adaptor and task._adaptor._container: # the task's adaptor has a valid associated container class # which can handle the container ops - great! c = task._adaptor._container m = task._method_type if not c in buckets['bound']: buckets['bound'][c] = {} if not m in buckets['bound'][c]: buckets['bound'][c][m] = [] buckets['bound'][c][m].append(task) else: # we have no container to handle this task -- so # put it into the fallback list buckets['unbound'].append(task) return buckets
def create (cls, url_in=None, session=None, ttype=sc.SYNC) : """ This is the asynchronous class constructor, returning a :class:`saga:Task` instance. For details on the accepted parameters, please see the description of :func:`__init__`. """ return cls (url_in, session, _ttype=ttype)._init_task # -------------------------------------------------------------------------- # @rus.takes ('Manager', rus.optional (rus.one_of (c.COMPUTE, c.STORAGE, c.NETWORK)), rus.optional (rus.one_of (SYNC, ASYNC, TASK))) @rus.returns ((rus.list_of (basestring), st.Task)) def list (self, rtype=None, ttype=None) : """ list(rtype=None) List known resource instances (which can be acquired). Returns a list of IDs. :type rtype: None or enum (COMPUTE | STORAGE | NETWORK) :param rtype: filter for one or more resource types """ return self._adaptor.list (rtype, ttype=ttype) # --------------------------------------------------------------------------
class LogicalFile(nsentry.Entry, sa.Attributes): # -------------------------------------------------------------------------- # @rus.takes('LogicalFile', rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(sab.Base), rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(rus.nothing) def __init__(self, url=None, flags=c.READ, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): ''' __init__(url=None, flags=READ, session=None) url: saga.Url flags: flags enum session: saga.Session ret: obj ''' # param checks if not flags: flags = 0 url = ru.Url(url) self._nsentry = super(LogicalFile, self) self._nsentry.__init__(url, flags, session, _adaptor, _adaptor_state, _ttype=_ttype) # -------------------------------------------------------------------------- # @classmethod @rus.takes('LogicalFile', rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(st.Task) def create(cls, url=None, flags=c.READ, session=None, ttype=None): ''' url: saga.Url flags: saga.replica.flags enum session: saga.Session ttype: saga.task.type enum ret: saga.Task ''' if not flags: flags = 0 _nsentry = super(LogicalFile, cls) return _nsentry.create(url, flags, session, ttype=ttype) # ---------------------------------------------------------------- # @rus.takes('LogicalFile', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((bool, st.Task)) def is_file(self, ttype=None): ''' is_file() ttype: saga.task.type enum ret: bool / saga.Task ''' return self.is_entry(ttype=ttype) # -------------------------------------------------------------------------- # # replica methods # @rus.takes('LogicalFile', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((int, st.Task)) def get_size(self, ttype=None): ''' get_size() Return the size of the file. ttype: saga.task.type enum ret: int / saga.Task Returns the size of the physical file represented by this logical file (in bytes) Example:: # get a file handle lf = saga.replica.LogicalFile("irods://localhost/tmp/data.bin") # print the logical file's size print lf.get_size () ''' return self._adaptor.get_size_self(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('LogicalFile', rus.optional((ru.Url, basestring)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def add_location(self, name, ttype=None): ''' add_location(name) Add a physical location. name: saga.Url ttype: saga.task.type enum ret: None / saga.Task ''' return self._adaptor.add_location(name, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('LogicalFile', rus.optional((ru.Url, basestring)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def remove_location(self, name, ttype=None): ''' remove_location(name) Remove a physical location. name: saga.Url ttype: saga.task.type enum ret: None / saga.Task ''' return self._adaptor.remove_location(name, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('LogicalFile', rus.optional((ru.Url, basestring)), rus.optional((ru.Url, basestring)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def update_location(self, old, new, ttype=None): ''' update_location(old, new) Updates a physical location. old: saga.Url new: saga.Url ttype: saga.task.type enum ret: None / saga.Task ''' return self._adaptor.update_location(old, new, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('LogicalFile', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(ru.Url), st.Task)) def list_locations(self, ttype=None): ''' list_locations() List all physical locations of a logical file. ttype: saga.task.type enum ret: list [saga.Url] / saga.Task ''' return self._adaptor.list_locations(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('LogicalFile', (ru.Url, basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def replicate(self, name, flags=None, ttype=None): ''' replicate(name) Replicate a logical file. name: saga.Url flags: flags enum ttype: saga.task.type enum ret: None / saga.Task ''' if not flags: flags = 0 return self._adaptor.replicate(name, flags, ttype=ttype) # -------------------------------------------------------------------------- # non-GFD.90 # @rus.takes('LogicalFile', (ru.Url, basestring), rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def upload(self, name, tgt=None, flags=None, ttype=None): ''' upload(name, tgt=None, flags=None) Upload a physical file. name: saga.Url tgt: saga.Url flags: flags enum ttype: saga.task.type enum ret: None / saga.Task ''' if not flags: flags = 0 return self._adaptor.upload(name, tgt, flags, ttype=ttype) # -------------------------------------------------------------------------- # non-GFD.90 # @rus.takes('LogicalFile', (ru.Url, basestring), rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def download(self, name, src=None, flags=None, ttype=None): ''' download(name, src=None, flags=None) Download a physical file. name: saga.Url src: saga.Url flags: flags enum ttype: saga.task.type enum ret: None / saga.Task ''' if not flags: flags = 0 return self._adaptor.download(name, src, flags, ttype=ttype)
def create (cls, url_in=None, session=None, ttype=sc.SYNC) : """ This is the asynchronous class constructor, returning a :class:`saga:Task` instance. For details on the accepted parameters, please see the description of :func:`__init__`. """ return cls (url_in, session, _ttype=ttype)._init_task # -------------------------------------------------------------------------- # @rus.takes ('Manager', rus.optional (rus.one_of (COMPUTE, STORAGE, NETWORK)), rus.optional (rus.one_of (SYNC, ASYNC, TASK))) @rus.returns ((rus.list_of (basestring), st.Task)) def list (self, rtype=None, ttype=None) : """ list(rtype=None) List known resource instances (which can be acquired). Returns a list of IDs. :type rtype: None or enum (COMPUTE | STORAGE | NETWORK) :param rtype: filter for one or more resource types """ return self._adaptor.list (rtype, ttype=ttype) # --------------------------------------------------------------------------
class Session(saga.base.SimpleBase): """A SAGA Session object as defined in GFD.90. A SAGA session has the purpose of scoping the use of security credentials for remote operations. In other words, a session instance acts as a container for security Context instances -- SAGA objects (such as job.Service or filesystem.File) created in that session will then use exactly the security contexts from that session (and no others). That way, the session serves two purposes: (1) it helps SAGA to decide which security mechanism should be used for what interaction, and (2) it helps SAGA to find security credentials which would be difficult to pick up automatically. The use of a session is as follows: Example:: # define an ssh context c = saga.Context('ssh') c.user_cert = '$HOME/.ssh/special_id_rsa.pub' c.user_key = '$HOME/.ssh/special_id_rsa' # add it to a session s = saga.Session s.add_context(c) # create a job service in this session -- that job service can now # *only* use that ssh context. j = saga.job.Service('ssh://remote.host.net/', s) The session argument to the L{job.Service} constructor is fully optional -- if left out, SAGA will use default session, which picks up some default contexts as described above -- that will suffice for the majority of use cases. A session instance exposes a `context` property, which is a list of authentication contexts managed by this session. As the contexts and the session are stateless, it is safe to modify this list as needed. """ # FIXME: session deep copy not implemented # -------------------------------------------------------------------------- # @rus.takes('Session', rus.optional(bool)) @rus.returns(rus.nothing) def __init__(self, default=True): """ default: bool ret: None """ simple_base = super(Session, self) simple_base.__init__() self._logger = ru.get_logger('radical.saga') # if the default session is expected, we point our context list to the # shared list of the default session singleton. Otherwise, we create # a private list which is not populated. # a session also has a lease manager, for adaptors in this session to use. if default: default_session = DefaultSession() self.contexts = copy.deepcopy(default_session.contexts) self._lease_manager = default_session._lease_manager else: self.contexts = _ContextList(session=self) # FIXME: at the moment, the lease manager is owned by the session. # Howevwer, the pty layer is the main user of the lease manager, # and we thus keep the lease manager options in the pty subsection. # So here we are, in the session, evaluating the pty config options... config = self.get_config('saga.utils.pty') self._lease_manager = ru.LeaseManager( max_pool_size=config['connection_pool_size'].get_value(), max_pool_wait=config['connection_pool_wait'].get_value(), max_obj_age=config['connection_pool_ttl'].get_value()) # ---------------------------------------------------------------- # @rus.takes('Session') @rus.returns(basestring) def __str__(self): """String represenation.""" return "Registered contexts: %s" % (str(self.contexts)) # ---------------------------------------------------------------- # @rus.takes('Session', saga.context.Context) @rus.returns(rus.nothing) def add_context(self, ctx): """ ctx: saga.Context ret: None Add a security L{Context} to the session. It is encouraged to use the L{contexts} property instead. """ return self.contexts.insert(0, ctx=ctx, session=self) # ---------------------------------------------------------------- # @rus.takes('Session', saga.context.Context) @rus.returns(rus.nothing) def remove_context(self, ctx): """ ctx: saga.Context ret: None Remove a security L{Context} from the session. It is encouraged to use the L{contexts} property instead. """ if ctx in self.contexts: self.contexts.remove(ctx) # ---------------------------------------------------------------- # @rus.takes('Session') @rus.returns(rus.list_of(saga.context.Context)) def list_contexts(self): """ ret: list[saga.Context] Retrieve all L{Context} objects attached to the session. It is encouraged to use the L{contexts} property instead. """ return self.contexts # ---------------------------------------------------------------- # @rus.takes('Session') @rus.returns(dict) def get_config(self, section=""): """ ret: radical.utils.Configuration Return the session configuration (optional a specific section). """ return saga.engine.engine.Engine().get_config(section)
class LogicalDirectory(nsdir.Directory, sa.Attributes): # -------------------------------------------------------------------------- # @rus.takes('LogicalDirectory', rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(sab.Base), rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(rus.nothing) def __init__(self, url=None, flags=c.READ, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): ''' __init__(url, flags=READ, session=None) Create a new Logical Directory instance. url: saga.Url flags: flags enum session: saga.Session ret: obj ''' # param checks if not flags: flags = 0 url = ru.Url(url) self._nsdirec = super(LogicalDirectory, self) self._nsdirec.__init__(url, flags, session, _adaptor, _adaptor_state, _ttype=_ttype) # -------------------------------------------------------------------------- # @classmethod @rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(st.Task) def create(cls, url, flags=c.READ, session=None, ttype=None): ''' url: saga.Url flags: saga.replica.flags enum session: saga.Session ttype: saga.task.type enum ret: saga.Task ''' if not flags: flags = 0 _nsdirec = super(LogicalDirectory, cls) return _nsdirec.create(url, flags, session, ttype=ttype) @rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((bool, st.Task)) def is_file(self, tgt=None, ttype=None): ''' is_file(tgt=None) tgt: saga.Url / string ttype: saga.task.type enum ret: bool / saga.Task ''' if tgt: return self._adaptor.is_file(tgt, ttype=ttype) else: return self._nsdirec.is_entry_self(ttype=ttype) @rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(('LogicalFile', st.Task)) def open(self, tgt, flags=c.READ, ttype=None): ''' open(tgt, flags=READ) tgt: saga.Url flags: saga.namespace.flags enum ttype: saga.task.type enum ret: saga.namespace.Entry / saga.Task ''' if not flags: flags = 0 tgt_url = ru.Url(tgt) return self._adaptor.open(tgt_url, flags, ttype=ttype) # ---------------------------------------------------------------- # @rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(('LogicalDirectory', st.Task)) def open_dir(self, tgt, flags=c.READ, ttype=None): ''' open_dir(tgt, flags=READ) :param tgt: name/path of the directory to open :param flags: directory creation flags ttype: saga.task.type enum ret: saga.namespace.Directory / saga.Task Open and return a new directoy The call opens and returns a directory at the given location. Example:: # create a subdir 'data' in /tmp dir = saga.namespace.Directory("sftp://localhost/tmp/") data = dir.open_dir ('data/', saga.namespace.Create) ''' if not flags: flags = 0 tgt_url = ru.Url(tgt) return self._adaptor.open_dir(tgt_url, flags, ttype=ttype) # ---------------------------------------------------------------- # # replica methods # # -------------------------------------------------------------------------- # @rus.takes('LogicalDirectory', rus.one_of(ru.Url, basestring), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((int, st.Task)) def get_size(self, tgt, ttype=None): ''' get_size(tgt) tgt: logical file to get size for ttype: saga.task.type enum ret: int / saga.Task Returns the size of the physical file represented by the given logical file (in bytes) Example:: # get a logical directory handle lf = saga.replica.LogicalFile("irods://localhost/tmp/data/") # print a logical file's size print lf.get_size ('data.dat') ''' tgt_url = ru.Url(tgt) return self._adaptor.get_size(tgt_url, ttype=ttype) # ---------------------------------------------------------------- # @rus.takes('LogicalDirectory', rus.optional(basestring), rus.optional(basestring), rus.optional(int, rus.nothing), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(ru.Url), st.Task)) def find(self, name_pattern, attr_pattern=None, flags=c.RECURSIVE, ttype=None): ''' find(name_pattern, attr_pattern=None, flags=RECURSIVE) name_pattern: string attr_pattern: string flags: flags enum ttype: saga.task.type enum ret: list [saga.Url] / saga.Task ''' if not flags: flags = 0 if attr_pattern: return self._adaptor.find_replicas(name_pattern, attr_pattern, flags, ttype=ttype) else: return self._nsdirec.find(name_pattern, flags, ttype=ttype)
class File(nsentry.Entry): """ Represents a local or remote file. The saga.filesystem.File class represents, as the name indicates, a file on some (local or remote) filesystem. That class offers a number of operations on that file, such as copy, move and remove:: # get a file handle file = saga.filesystem.File("sftp://localhost/tmp/data/data.bin") # copy the file file.copy ("sftp://localhost/tmp/data/data.bak") # move the file file.move ("sftp://localhost/tmp/data/data.new") """ # -------------------------------------------------------------------------- # @rus.takes('File', rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(sab.Base), rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(rus.nothing) def __init__(self, url=None, flags=READ, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): """ __init__(url, flags=READ, session) Construct a new file object :param url: Url of the (remote) file :type url: :class:`saga.Url` :fgs: :ref:`filesystemflags` :param session: :class:`saga.Session` The specified file is expected to exist -- otherwise a DoesNotExist exception is raised. Also, the URL must point to a file (not to a directory), otherwise a BadParameter exception is raised. Example:: # get a file handle file = saga.filesystem.File("sftp://localhost/tmp/data/data.bin") # print the file's size print file.get_size () """ # param checks if not flags: flags = 0 url = ru.Url(url) if not url.schema: url.schema = 'file' if not url.host: url.host = 'localhost' self._nsentry = super(File, self) self._nsentry.__init__(url, flags, session, _adaptor, _adaptor_state, _ttype=_ttype) # -------------------------------------------------------------------------- # @classmethod @rus.takes('File', rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(st.Task) def create(cls, url=None, flags=READ, session=None, ttype=None): """ create(url, flags, session) url: saga.Url flags: saga.replica.flags enum session: saga.Session ttype: saga.task.type enum ret: saga.Task """ if not flags: flags = 0 _nsentry = super(File, cls) return _nsentry.create(url, flags, session, ttype=ttype) # ---------------------------------------------------------------- # # filesystem methods # # -------------------------------------------------------------------------- # @rus.takes('File', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((bool, st.Task)) def is_file(self, ttype=None): """ is_file() Returns `True` if instance points to a file, `False` otherwise. """ return self._adaptor.is_file_self(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((int, st.Task)) def get_size(self, ttype=None): ''' get_size() Returns the size (in bytes) of a file. Example:: # get a file handle file = saga.filesystem.File("sftp://localhost/tmp/data/data.bin") # print the file's size print file.get_size () ''' return self._adaptor.get_size_self(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', rus.optional(int), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((basestring, st.Task)) def read(self, size=None, ttype=None): ''' size : int ttype: saga.task.type enum ret: string / bytearray / saga.Task ''' return self._adaptor.read(size, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', rus.optional(bool)) @rus.returns(st.Task) def close(self, kill=True, ttype=None): ''' kill : bool ttype: saga.task.type enum ret: string / bytearray / saga.Task ''' return self._adaptor.close() # -------------------------------------------------------------------------- # @rus.takes('File', basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((int, st.Task)) def write(self, data, ttype=None): ''' data : string / bytearray ttype: saga.task.type enum ret: int / saga.Task ''' return self._adaptor.write(data, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', int, rus.optional(rus.one_of(START, CURRENT, END)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((int, st.Task)) def seek(self, offset, whence=START, ttype=None): ''' offset: int whence: seek_mode enum ttype: saga.task.type enum ret: int / saga.Task ''' return self._adaptor.seek(offset, whence, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', rus.list_of(rus.tuple_of(int)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((basestring, st.Task)) def read_v(self, iovecs, ttype=None): ''' iovecs: list [tuple (int, int)] ttype: saga.task.type enum ret: list [bytearray] / saga.Task ''' return self._adaptor.read_v(iovecs, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', rus.list_of(rus.tuple_of((int, basestring))), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(int), st.Task)) def write_v(self, data, ttype=None): ''' data: list [tuple (int, string / bytearray)] ttype: saga.task.type enum ret: list [int] / saga.Task ''' return self._adaptor.write_v(data, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((int, st.Task)) def size_p(self, pattern, ttype=None): ''' pattern: string ttype: saga.task.type enum ret: int / saga.Task ''' return self._adaptor.size_p(pattern, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((basestring, st.Task)) def read_p(self, pattern, ttype=None): ''' pattern: string ttype: saga.task.type enum ret: string / bytearray / saga.Task ''' return self._adaptor.read_p(pattern, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', basestring, basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((int, st.Task)) def write_p(self, pattern, data, ttype=None): ''' pattern: string data: string / bytearray ttype: saga.task.type enum ret: int / saga.Task ''' return self._adaptor.write_p(pattern, data, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(basestring), st.Task)) def modes_e(self, ttype=None): ''' ttype: saga.task.type enum ret: list [string] / saga.Task ''' return self._adaptor.modes_e(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', basestring, basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((int, st.Task)) def size_e(self, emode, spec, ttype=None): ''' emode: string spec: string ttype: saga.task.type enum ret: int / saga.Task ''' return self._adaptor.size_e(emode, spec, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', basestring, basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((basestring, st.Task)) def read_e(self, emode, spec, ttype=None): ''' emode: string spec: string ttype: saga.task.type enum ret: bytearray / saga.Task ''' return self._adaptor.read_e(emode, spec, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('File', basestring, basestring, basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((int, st.Task)) def write_e(self, emode, spec, data, ttype=None): ''' emode: string spec: string data: string / bytearray ttype: saga.task.type enum ret: int / saga.Task ''' return self._adaptor.read_e(emode, spec, data, ttype=ttype) size = property(get_size) # int modes_e = property(modes_e) # list [string]
class Entry(nsentry.Entry, sa.Attributes): # -------------------------------------------------------------------------- # @rus.takes('Entry', rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(sab.Base), rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(rus.nothing) def __init__(self, url=None, flags=READ, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): ''' url: saga.Url flags: flags enum session: saga.Session ret: obj ''' # param checks url = ru.Url(url) self._nsentry = super(Entry, self) self._nsentry.__init__(url, flags, session, _adaptor, _adaptor_state, _ttype=_ttype) # set attribute interface properties self._attributes_allow_private(True) self._attributes_camelcasing(True) self._attributes_extensible(True, getter=self._attribute_getter, setter=self._attribute_setter, lister=self._attribute_lister, caller=self._attribute_caller) # register properties with the attribute interface self._attributes_register(ATTRIBUTE, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register(OBJECT, None, sa.ANY, sa.SCALAR, sa.READONLY) self._attributes_register(TTL, None, sa.INT, sa.SCALAR, sa.WRITEABLE) self._attributes_set_setter(TTL, self.set_ttl) self._attributes_set_getter(TTL, self.get_ttl) self._attributes_set_setter(OBJECT, self.store_object) self._attributes_set_getter(OBJECT, self.retrieve_object) # -------------------------------------------------------------------------- # @classmethod @rus.takes('Entry', rus.optional((ru.Url, basestring)), rus.optional(int, rus.nothing), rus.optional(ss.Session), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(st.Task) def create(cls, url=None, flags=READ, session=None, ttype=None): ''' url: saga.Url flags: saga.advert.flags enum session: saga.Session ttype: saga.task.type enum ret: saga.Task ''' if not flags: flags = 0 _nsentry = super(Entry, cls) return _nsentry.create(url, flags, session, ttype=ttype) # -------------------------------------------------------------------------- # # attribute methods # # NOTE: we do not yet pass ttype, as async calls are not yet supported by # the attribute interface # @rus.takes('Entry', basestring, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.anything, st.Task)) def _attribute_getter(self, key, ttype=None): return self._adaptor.attribute_getter(key) # -------------------------------------------------------------------------- # @rus.takes('Entry', basestring, rus.anything, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def _attribute_setter(self, key, val, ttype=None): return self._adaptor.attribute_setter(key, val) # -------------------------------------------------------------------------- # @rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(rus.anything), st.Task)) def _attribute_lister(self, ttype=None): return self._adaptor.attribute_lister() # -------------------------------------------------------------------------- # @rus.takes('Entry', basestring, int, callable, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.anything, st.Task)) def _attribute_caller(self, key, id, cb, ttype=None): return self._adaptor.attribute_caller(key, id, cb) # -------------------------------------------------------------------------- # # advert methods # @rus.takes('Entry', float, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def set_ttl(self, ttl=-1.0, ttype=None): """ ttl : int ttype: saga.task.type enum ret: None / saga.Task """ return self._adaptor.set_ttl(ttl, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((float, st.Task)) def get_ttl(self, ttype=None): """ ttype: saga.task.type enum ret: int / saga.Task """ return self._adaptor.get_ttl(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Entry', object, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def store_object(self, object, ttype=None): """ object : <object type> ttype: saga.task.type enum ret: None / saga.Task """ return self._adaptor.store_object(object, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((object, st.Task)) def retrieve_object(self, ttype=None): """ ttype: saga.task.type enum ret: any / saga.Task """ return self._adaptor.retrieve_object(ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Entry', rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def delete_object(self, ttype=None): """ ttype: saga.task.type enum ret: None / saga.Task """ return self._adaptor.delete_object(ttype=ttype)
class Monitorable(sa.Attributes): # -------------------------------------------------------------------------- # def __init__(self): self._attr = super(Monitorable, self) self._attr.__init__() # -------------------------------------------------------------------------- # # since we have no means to call the saga.Base constructor explicitly (we # don't inherit from it), we have to rely that the classes which implement # the Monitorable interface are correctly calling the Base constructure -- # otherwise we won't have an self._adaptor to talk to... # # This helper method checks the existence of self._adaptor, and should be # used before each call forwarding. # def _check(self): if not hasattr(self, '_adaptor'): raise se.IncorrectState("object is not fully initialized") # -------------------------------------------------------------------------- # @rus.takes('Monitorable') @rus.returns(rus.list_of(basestring)) def list_metrics(self): self._check() return self._adaptor.list_metrics() # -------------------------------------------------------------------------- # # Metrics are not implemented in RADICAL-SAGA # # @rus.takes ('Monitorable', basestring) # @rus.returns ('Metric') # def get_metric (name) : # # self._check () # return self._adaptor.get_metric (name) # -------------------------------------------------------------------------- # @rus.takes('Monitorable', basestring, rus.one_of('saga.Callback', callable)) @rus.returns(int) def add_callback(self, name, cb): self._check() return self._adaptor.add_callback(name, cb) # -------------------------------------------------------------------------- # @rus.takes('Monitorable', int) @rus.returns(rus.nothing) def remove_callback(self, cookie): self._check() return self._adaptor.remove_callback(cookie)
class Manager(sb.Base, sasync.Async): """ In the context of RADICAL-SAGA, a *ResourceManager* is a service which asserts control over a set of resources. That manager can, on request, render control over subsets of those resources (resource slices) to an application. This :class:`Manager` class represents the contact point to such ResourceManager instances -- the application can thus acquire compute, data or network resources, according to some resource specification, for a bound or unbound amount of time. """ # -------------------------------------------------------------------------- # @rus.takes('Manager', rus.optional(str, ru.Url), rus.optional(ss.Session), rus.optional(sab.Base), rus.optional(dict), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(rus.nothing) def __init__(self, url=None, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): """ __init__(url) Create a new Manager instance. Connect to a remote resource management endpoint. :type url: :class:`saga.Url` :param url: resource management endpoint """ # param checks _url = ru.Url(url) scheme = _url.scheme.lower() if not session: session = ss.Session(default=True) self._base = super(Manager, self) self._base.__init__(scheme, _adaptor, _adaptor_state, _url, session, ttype=_ttype) # -------------------------------------------------------------------------- # @classmethod @rus.takes('Manager', rus.optional((ru.Url, str)), rus.optional(ss.Session), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns(st.Task) def create(cls, url_in=None, session=None, ttype=sc.SYNC): """ This is the asynchronous class constructor, returning a :class:`saga:Task` instance. For details on the accepted parameters, please see the description of :func:`__init__`. """ return cls(url_in, session, _ttype=ttype)._init_task # -------------------------------------------------------------------------- # @rus.takes('Manager', rus.optional(rus.one_of(c.COMPUTE, c.STORAGE, c.NETWORK)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(str), st.Task)) def list(self, rtype=None, ttype=None): """ list(rtype=None) List known resource instances (which can be acquired). Returns a list of IDs. :type rtype: None or enum (COMPUTE | STORAGE | NETWORK) :param rtype: filter for one or more resource types """ return self._adaptor.list(rtype, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Manager', rus.optional(str), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((descr.Description, st.Task)) def get_description(self, rid, ttype=None): """ get_description(rid) Get the resource :class:`Description` for the specified resource. :type rid: str :param rid: identifies the resource to be described. """ # TODO / NOTE: if rid is None, should we return a description of # the managed resources? return self._adaptor.get_description(id, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Manager', rus.optional(rus.one_of(c.COMPUTE, c.STORAGE, c.NETWORK)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(str), st.Task)) def list_templates(self, rtype=None, ttype=None): """ list_templates(rtype=None) List template names available for the specified resource type(s). Returns a list of strings. :type rtype: None or enum (COMPUTE | STORAGE | NETWORK) :param rtype: filter for one or more resource types """ return self._adaptor.list_templates(rtype, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Manager', rus.optional(str), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((descr.Description, st.Task)) def get_template(self, name, ttype=None): """ get_template(name) Get a :class:`Description` for the specified template. :type name: str :param name: specifies the name of the template The returned resource description instance may not have all attributes filled, and may in fact not sufficiently complete to allow for successful resource acquisition. The only guaranteed attribute in the returned description is `TEMPLATE`, containing the very template id specified in the call parameters. """ return self._adaptor.get_template(name, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Manager', rus.optional(rus.one_of(c.COMPUTE, c.STORAGE, c.NETWORK)), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.list_of(str), st.Task)) def list_images(self, rtype=None, ttype=None): """ list_images(rtype=None) List image names available for the specified resource type(s). Returns a list of strings. :type rtype: None or enum (COMPUTE | STORAGE | NETWORK) :param rtype: filter for one or more resource types """ return self._adaptor.list_images(rtype, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Manager', str, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((dict, st.Task)) def get_image(self, name, ttype=None): """ get_image(name) Get a description string for the specified image. :type name: str :param name: specifies the image name """ return self._adaptor.get_image(name, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Manager', (str, ru.Url, descr.Description), rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((resrc.Resource, st.Task)) def acquire(self, spec, ttype=None): """ acquire(desc) Create a new :class:`saga.resource.Resource` handle for a resource specified by the description. :type spec: :class:`Description` or Url :param spec: specifies the resource Depending on the `RTYPE` attribute in the description, the returned resource may be a :class:`saga.resource.Compute`, :class:`saga.resource.Storage` or :class:`saga.resource.Network` instance. If the `spec` parameter is """ if isinstance (spec, ru.Url) or \ isinstance (spec, str) : return self._adaptor.acquire_by_id(spec, ttype=ttype) else: # make sure at least 'executable' is defined if spec.rtype is None: raise se.BadParameter("resource type undefined in description") # spec_copy = descr.Description () # spec._attributes_deep_copy (spec_copy) return self._adaptor.acquire(spec, ttype=ttype) # -------------------------------------------------------------------------- # @rus.takes('Manager', str, rus.optional(rus.one_of(SYNC, ASYNC, TASK))) @rus.returns((rus.nothing, st.Task)) def destroy(self, rid, ttype=None): """ destroy(rid) Destroy / release a resource. :type rid : string :param rid : identifies the resource to be released """ return self._adaptor.destroy(rid, ttype=ttype)