def get_all_frames(command, tags=None, url=None, mode=None): if not url: url = os.environ.get('RADICAL_SYNAPSE_DBURL') if not url: print "warning: need dburl to retrieve profiles" return None docs = list() url = ru.Url(url) if url.scheme != 'file': raise ValueError('can only handle file:// based dburls, not %s' % url) path = url.path for p in glob.glob("%s/*/" % path): tmp_url = ru.Url(url) tmp_url.path = p tmp_docs = get_profiles(command, tags=tags, url=tmp_url, mode=mode) if tmp_docs: docs += tmp_docs return make_frames(docs)
def _get_unit_sandbox(self, unit, pilot): # If a sandbox is specified in the unit description, then interpret # relative paths as relativet to the pilot sandbox. # unit sandboxes are cached in the unit dict unit_sandbox = unit.get('unit_sandbox') if unit_sandbox: return unit_sandbox # specified in description? if not unit_sandbox: sandbox = unit['description'].get('sandbox') if sandbox: unit_sandbox = ru.Url(self._get_pilot_sandbox(pilot)) if sandbox[0] == '/': unit_sandbox.path = sandbox else: unit_sandbox.path += '/%s/' % sandbox # default if not unit_sandbox: unit_sandbox = ru.Url(self._get_pilot_sandbox(pilot)) unit_sandbox.path += "/%s/" % unit['uid'] # cache unit['unit_sandbox'] = str(unit_sandbox) return unit_sandbox
def __init__(self, pmgr, descr): # 'static' members self._descr = descr.as_dict() # sanity checks on description for check in ['resource', 'cores', 'runtime']: if not self._descr.get(check): raise ValueError("ComputePilotDescription needs '%s'" % check) # initialize state self._pmgr = pmgr self._session = self._pmgr.session self._prof = self._session._prof self._uid = ru.generate_id('pilot.%(counter)04d', ru.ID_CUSTOM) self._state = rps.NEW self._log = pmgr._log self._pilot_dict = dict() self._callbacks = dict() self._cache = dict() # cache of SAGA dir handles self._cb_lock = threading.RLock() self._exit_on_error = self._descr.get('exit_on_error') for m in rpt.PMGR_METRICS: self._callbacks[m] = dict() # we always invoke the default state cb self._callbacks[rpt.PILOT_STATE][self._default_state_cb.__name__] = { 'cb': self._default_state_cb, 'cb_data': None } # `as_dict()` needs `pilot_dict` and other attributes. Those should all # be available at this point (apart from the sandboxes), so we now # query for those sandboxes. self._pilot_jsurl = ru.Url() self._pilot_jshop = ru.Url() self._resource_sandbox = ru.Url() self._pilot_sandbox = ru.Url() self._client_sandbox = ru.Url() self._log.debug(' ===== 1: %s [%s]', self._pilot_sandbox, type(self._pilot_sandbox)) pilot = self.as_dict() self._log.debug(' ===== 2: %s [%s]', pilot['pilot_sandbox'], type(pilot['pilot_sandbox'])) self._pilot_jsurl, self._pilot_jshop \ = self._session._get_jsurl (pilot) self._resource_sandbox = self._session._get_resource_sandbox(pilot) self._pilot_sandbox = self._session._get_pilot_sandbox(pilot) self._client_sandbox = self._session._get_client_sandbox() self._log.debug(' ===== 3: %s [%s]', self._pilot_sandbox, type(self._pilot_sandbox))
def __init__(self, url=None, flags=c.READ, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): ''' __init__(url, flags=READ, session=None) Create a new Logical Directory instance. url: saga.Url flags: flags enum session: saga.Session ret: obj ''' # param checks if not flags: flags = 0 url = ru.Url(url) self._nsdirec = super(LogicalDirectory, self) self._nsdirec.__init__(url, flags, session, _adaptor, _adaptor_state, _ttype=_ttype)
def test_url_issue_61 () : """ Test url issue #61 """ url = ru.Url ("advert://localhost/?dbtype=sqlite3") assert url.query == "dbtype=sqlite3"
def __init__(self, url=None, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): """ __init__(url) Create a new Manager instance. Connect to a remote resource management endpoint. :type url: :class:`saga.Url` :param url: resource management endpoint """ # param checks _url = ru.Url(url) scheme = _url.scheme.lower() if not session: session = ss.Session(default=True) self._base = super(Manager, self) self._base.__init__(scheme, _adaptor, _adaptor_state, _url, session, ttype=_ttype)
def __init__(self, db_url, db_name="radicalpilot"): """ Le constructeur. Should not be called directrly, but rather via the static methods new() or reconnect(). """ url = ru.Url(db_url) if db_name: url.path = db_name mongo, db, dbname, pname, cname = ru.mongodb_connect(url) self._client = mongo self._db = db self._dburl = str(url) self._dbname = dbname if url.username and url.password: self._dbauth = "%s:%s" % (url.username, url.password) else: self._dbauth = None self._session_id = None self._s = None self._w = None self._um = None self._p = None self._pm = None
def __init__(self, url=None, flags=c.READ, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): ''' __init__(url=None, flags=READ, session=None) url: saga.Url flags: flags enum session: saga.Session ret: obj ''' # param checks if not flags: flags = 0 url = ru.Url(url) self._nsentry = super(LogicalFile, self) self._nsentry.__init__(url, flags, session, _adaptor, _adaptor_state, _ttype=_ttype)
def _assign_pilot(self, unit, pilot): ''' assign a unit to a pilot. This is also a good opportunity to determine the unit sandbox(es). ''' pid = pilot['uid'] uid = unit['uid'] self._log.debug('assign %s to %s', uid, pid) unit['pilot'] = pid unit['client_sandbox'] = str(self._session._get_client_sandbox()) unit['resource_sandbox'] = str( self._session._get_resource_sandbox(pilot)) unit['pilot_sandbox'] = str(self._session._get_pilot_sandbox(pilot)) unit['unit_sandbox'] = str(self._session._get_unit_sandbox( unit, pilot)) unit['unit_sandbox_path'] = ru.Url(unit['unit_sandbox']).path with self._units_lock: if pid not in self._units: self._units[pid] = list() self._units[pid].append(uid)
def __init__ (self, url=None, flags=READ, session=None, _adaptor=None, _adaptor_state={}, _ttype=None) : ''' url: saga.Url flags: flags enum session: saga.Session ret: obj ''' # param checks if not flags : flags = 0 url = ru.Url (url) self._nsdirec = super (Directory, self) self._nsdirec.__init__ (url, flags, session, _adaptor, _adaptor_state, _ttype=_ttype) # set attribute interface properties self._attributes_allow_private (True) self._attributes_camelcasing (True) self._attributes_extensible (True, getter=self._attribute_getter, setter=self._attribute_setter, lister=self._attribute_lister, caller=self._attribute_caller) # register properties with the attribute interface self._attributes_register (ATTRIBUTE, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register (CHANGE, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register (NEW, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register (DELETE, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register (TTL, None, sa.INT, sa.SCALAR, sa.WRITEABLE) self._attributes_set_setter (TTL, self.set_ttl) self._attributes_set_getter (TTL, self.get_ttl)
def __init__(self, descr: dict, executor: jpsi.JobExecutor, url: str) -> None: jpsi.ExecutorAdaptorBase.__init__(self, descr, executor, url) self._url = ru.Url(url) if self._url.schema != 'rp': raise ValueError('handle only rp:// URLs, not %s', self._url) try: self._jobs = dict() # {job.uid : [JPSI_JOB, RP_TASK] self._lock = mt.Lock() self._session = rp.Session() self._pmgr = rp.PilotManager(session=self._session) self._tmgr = rp.TaskManager(session=self._session) self._pmgr.register_callback(self._pilot_state_cb) self._tmgr.register_callback(self._task_state_cb) # this is layer 0, so we just create a dummy pilot pd = rp.PilotDescription({ 'resource': 'local.localhost', 'cores': 16, 'runtime': 60 }) self._pilot = self._pmgr.submit_pilots(pd) self._tmgr.add_pilots(self._pilot) except Exception: self._log.exception('init failed') raise
def test_url_issue_rs_305(): # This compensates # # >>> import os # >>> os.path.normpath('//path//to//dir//') # '//path/to/dir' # # to a normalization resulting in # # '/path/to/dir' # # as required by the SAGA spec url1 = ru.Url("advert://localhost/path/to/file") url2 = ru.Url("advert://localhost//path/to/file") assert url1.path == url2.path
def init_instance(self, adaptor_state, rm_url, session): """ service instance constructor """ self.rm = rm_url self.session = session self.ppn = None self.is_cray = "" self.queue = None self.shell = None self.jobs = dict() self.gres = None # the monitoring thread - one per service instance self.mt = _job_state_monitor(job_service=self) self.mt.start() rm_scheme = rm_url.scheme pty_url = ru.Url(rm_url) # this adaptor supports options that can be passed via the # 'query' component of the job service URL. if rm_url.query: for key, val in parse_qs(rm_url.query).items(): if key == 'queue': self.queue = val[0] elif key == 'craytype': self.is_cray = val[0] elif key == 'ppn': self.ppn = int(val[0]) elif key == 'gres': self.gres = val[0] # we need to extract the scheme for PTYShell. That's basically the # job.Service Url without the pbs+ part. We use the PTYShell to execute # pbs commands either locally or via gsissh or ssh. if rm_scheme == "pbs": pty_url.scheme = "fork" elif rm_scheme == "pbs+ssh": pty_url.scheme = "ssh" elif rm_scheme == "pbs+gsissh": pty_url.scheme = "gsissh" # these are the commands that we need in order to interact with PBS. # the adaptor will try to find them during initialize(self) and bail # out in case they are note available. self._commands = { 'pbsnodes': None, 'qstat': None, 'qsub': None, 'qdel': None } self.shell = rsups.PTYShell(pty_url, self.session) # self.shell.set_initialize_hook(self.initialize) # self.shell.set_finalize_hook(self.finalize) self.initialize() return self.get_api()
def init_instance(self, adaptor_state, rm_url, session): """ service instance constructor """ self.rm = rm_url self.session = session self.pe_list = list() self.jobs = dict() self.queue = None self.memreqs = None self.shell = None self.mandatory_memreqs = list() self.accounting = False self.temp_path = self._adaptor.base_workdir rm_scheme = rm_url.scheme pty_url = ru.Url(rm_url) # this adaptor supports options that can be passed via the # 'query' component of the job service URL. if rm_url.query is not None: for key, val in parse_qs(rm_url.query).items(): if key == 'queue': self.queue = val[0] elif key == 'memreqs': self.memreqs = val[0] # we need to extrac the scheme for PTYShell. That's basically the # job.Serivce Url withou the sge+ part. We use the PTYShell to execute # pbs commands either locally or via gsissh or ssh. if rm_scheme == "sge": pty_url.scheme = "fork" elif rm_scheme == "sge+ssh": pty_url.scheme = "ssh" elif rm_scheme == "sge+gsissh": pty_url.scheme = "gsissh" # these are the commands that we need in order to interact with SGE. # the adaptor will try to find them during initialize(self) and bail # out in case they are not available. self._commands = { 'qstat': None, 'qsub': None, 'qdel': None, 'qconf': None, 'qacct': None } self.shell = rsups.PTYShell(pty_url, self.session) # self.shell.set_initialize_hook(self.initialize) # self.shell.set_finalize_hook(self.finalize) self.initialize() return self.get_api()
def open(self, name, flags=None, ttype=None): ''' name: saga.Url flags: saga.namespace.flags enum ttype: saga.task.type enum ret: saga.namespace.Entry / saga.Task ''' if not flags: flags = 0 url = ru.Url(name) return self._adaptor.open(url, flags, ttype=ttype)
def test_url_issue_49(): # ensure correct str serialization after setting elements url = ru.Url("scheme://*****:*****@host:123/dir/file?query#fragment") url.set_host('remote.host.net') url.set_scheme('sftp') url.set_path('/tmp/data') assert str(url) == "sftp://*****:*****@remote.host.net:123/tmp/data"
def _handle_unit_stdio(self, unit): sandbox = ru.Url(unit['unit_sandbox']).path uid = unit['uid'] self._prof.prof('staging_stdout_start', uid=uid) # TODO: disable this at scale? if os.path.isfile(unit['stdout_file']): with open(unit['stdout_file'], 'r') as stdout_f: try: txt = unicode(stdout_f.read(), "utf-8") except UnicodeDecodeError: txt = "unit stdout is binary -- use file staging" unit['stdout'] += rpu.tail(txt) self._prof.prof('staging_stdout_stop', uid=uid) self._prof.prof('staging_stderr_start', uid=uid) # TODO: disable this at scale? if os.path.isfile(unit['stderr_file']): with open(unit['stderr_file'], 'r') as stderr_f: try: txt = unicode(stderr_f.read(), "utf-8") except UnicodeDecodeError: txt = "unit stderr is binary -- use file staging" unit['stderr'] += rpu.tail(txt) self._prof.prof('staging_stderr_stop', uid=uid) self._prof.prof('staging_uprof_start', uid=uid) unit_prof = "%s/%s.prof" % (sandbox, uid) if os.path.isfile(unit_prof): try: with open(unit_prof, 'r') as prof_f: txt = prof_f.read() for line in txt.split("\n"): if line: ts, event, comp, tid, _uid, state, msg = line.split( ',') self._prof.prof(timestamp=float(ts), event=event, comp=comp, tid=tid, uid=_uid, state=state, msg=msg) except Exception as e: self._log.error("Pre/Post profile read failed: `%s`" % e) self._prof.prof('staging_uprof_stop', uid=uid)
def test_url_scheme_issue(): # test basic functionality for invalid schemas u1 = ru.Url("unknownscheme://*****:*****@hostname.domain:9999/path") assert u1.scheme == "unknownscheme" assert u1.username == "user" assert u1.password == "pwd" assert u1.host == "hostname.domain" assert u1.port == int(9999)
def create_task_from_cu(cu, prof=None): """ Purpose: Create a Task based on the Compute Unit. Details: Currently, only the uid, parent_stage and parent_pipeline are retrieved. The exact initial Task (that was converted to a CUD) cannot be recovered as the RP API does not provide the same attributes for a CU as for a CUD. Also, this is not required for the most part. TODO: Add exit code, stdout, stderr and path attributes to a Task. These can be extracted from a CU :arguments: :cu: RP Compute Unit :return: Task """ try: logger.debug('Create Task from CU %s' % cu.name) if prof: prof.prof('task from cu - create', uid=cu.name.split(',')[0].strip()) task = Task() task.uid = cu.name.split(',')[0].strip() task.name = cu.name.split(',')[1].strip() task.parent_stage['uid'] = cu.name.split(',')[2].strip() task.parent_stage['name'] = cu.name.split(',')[3].strip() task.parent_pipeline['uid'] = cu.name.split(',')[4].strip() task.parent_pipeline['name'] = cu.name.split(',')[5].strip() task.rts_uid = cu.uid if cu.exit_code is not None: task.exit_code = cu.exit_code else: if cu.state == rp.DONE: task.exit_code = 0 else: task.exit_code = 1 task.path = ru.Url(cu.sandbox).path if prof: prof.prof('task from cu - done', uid=cu.name.split(',')[0].strip()) logger.debug('Task %s created from CU %s' % (task.uid, cu.name)) return task except Exception, ex: logger.error('Task creation from CU failed, error: %s' % ex) raise
def test_url_api(): # test basic functionality for valid schemas u1 = ru.Url("ssh://*****:*****@hostname.domain:9999/path") assert u1.scheme == "ssh" assert u1.username == "user" assert u1.password == "pwd" assert u1.host == "hostname.domain" assert u1.port == int(9999)
def test_url_properties () : """ Test url properties """ url = ru.Url("") assert str(url) == "" url.scheme = "scheme" assert str(url) == "scheme://" assert url.get_scheme() == "scheme" url.set_scheme("tscheme") assert url.get_scheme() == "tscheme" url.scheme = "scheme" url.host = "host" assert str(url) == "scheme://host" assert url.get_host() == "host" url.set_host("thost") assert url.get_host() == "thost" url.host = "host" url.port = 42 assert str(url) == "scheme://*****:*****@host:42" assert url.get_username() == "username" url.set_username("tusername") assert url.get_username() == "tusername" url.username = "******" url.password = "******" assert str(url) == "scheme://*****:*****@host:42" assert url.get_password() == "password" url.set_password("tpassword") assert url.get_password() == "tpassword" url.password = "******" url.path = "/path/" assert str(url) == "scheme://*****:*****@host:42/path/" assert url.get_path() == "/path/" url.set_path("tpath") assert url.get_path() == "/tpath"
def test_url_issue_49 () : """ Test url issue #49 """ url = ru.Url ("scheme://*****:*****@host:123/dir/file?query#fragment") url.set_host ('remote.host.net') url.set_scheme ('sftp') url.set_path ('/tmp/data') assert str(url) == "sftp://*****:*****@remote.host.net:123/tmp/data"
def test_url_compatibility () : """ Test basic url compatibility """ u1 = ru.Url("ssh://*****:*****@hostname.domain:9999/path") assert u1.scheme == "ssh" assert u1.username == "user" assert u1.password == "pwd" assert u1.host == "hostname.domain" assert u1.port == int(9999)
def __init__(self, url, log=None, rep=None, prof=None): if log: self._log = log else: self._log = ru.Logger('radical.nge') if rep: self._rep = log else: self._rep = ru.Reporter('radical.nge') if prof: self._prof = prof else: self._prof = ru.Profiler('radical.nge') self._cookies = list() self._url = ru.Url(url) self._qbase = ru.Url(url) # self._qbase.username = None # self._qbase.password = None self._qbase = str(self._qbase).rstrip('/') if self._url.username and self._url.password: self.login(self._url.username, self._url.password)
def open(self, tgt, flags=c.READ, ttype=None): ''' open(tgt, flags=READ) tgt: saga.Url flags: saga.namespace.flags enum ttype: saga.task.type enum ret: saga.namespace.Entry / saga.Task ''' if not flags: flags = 0 tgt_url = ru.Url(tgt) return self._adaptor.open(tgt_url, flags, ttype=ttype)
def test_url_scheme_issue () : """ Test url schema issues """ u1 = ru.Url("unknownscheme://*****:*****@hostname.domain:9999/path") assert u1.scheme == "unknownscheme" assert u1.username == "user" assert u1.password == "pwd" assert u1.host == "hostname.domain" assert u1.port == int(9999)
def init_instance(self, adaptor_state, rm_url, session): """ service instance constructor """ self.rm = rm_url self.session = session self.ppn = 1 self.queue = None self.shell = None self.jobs = dict() # the monitoring thread - one per service instance self.mt = _job_state_monitor(job_service=self) self.mt.start() rm_scheme = rm_url.scheme pty_url = ru.Url(rm_url) # this adaptor supports options that can be passed via the # 'query' component of the job service URL. if rm_url.query: for key, val in parse_qs(rm_url.query).items(): if key == 'queue': self.queue = val[0] else: raise rse.BadParameter('unsupported url query %s' % key) # we need to extrac the scheme for PTYShell. That's basically the # job.Serivce Url withou the lsf+ part. We use the PTYShell to execute # lsf commands either locally or via gsissh or ssh. elems = rm_scheme.split('+') if 'gsissh' in elems: pty_url.scheme = "gsissh" elif 'ssh' in elems: pty_url.scheme = "ssh" else: pty_url.scheme = "fork" # these are the commands that we need in order to interact with LSF. # the adaptor will try to find them during initialize(self) and bail # out in case they are note avaialbe. self._commands = { 'bqueues': dict(), 'bjobs': dict(), 'bsub': dict(), 'bkill': dict() } self.shell = rsups.PTYShell(pty_url, self.session) # self.shell.set_initialize_hook(self.initialize) # self.shell.set_finalize_hook(self.finalize) self.initialize() return self.get_api()
def __init__(self, url=None, flags=READ, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): """ __init__(url, flags=READ, session) Construct a new directory object :param url: Url of the (remote) directory :type url: :class:`saga.Url` :param flags: :ref:`filesystemflags` :param session: :class:`saga.Session` The specified directory is expected to exist -- otherwise a DoesNotExist exception is raised. Also, the URL must point to a directory (not to a file), otherwise a BadParameter exception is raised. Example:: # open some directory dir = saga.filesystem.Directory("sftp://localhost/tmp/") # and list its contents files = dir.list () """ # param checks if not flags: flags = 0 url = ru.Url(url) if not url.schema: url.schema = 'file' if not url.host: url.host = 'localhost' self._nsdirec = super(Directory, self) self._nsdirec.__init__(url, flags, session, _adaptor, _adaptor_state, _ttype=_ttype)
def __init__(self, url=None, flags=None, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): ''' :param url: Url of the (remote) entry :type url: :class:`saga.Url` flags: flags enum session: saga.Session ret: obj Construct a new entry object The specified entry is expected to exist -- otherwise a DoesNotExist exception is raised. Also, the URL must point to an entry (not to a directory), otherwise a BadParameter exception is raised. Example:: # get an entry handle entry = saga.namespace.Entry("sftp://localhost/tmp/data/data.bin") # print the entry's url print(entry.get_url ()) ''' self._session = session self._is_recursive = False # recursion guard (FIXME: NOT THREAD SAFE) # param checks if not session: session = ss.Session(default=True) if not flags: flags = 0 url = ru.Url(url) scheme = url.scheme.lower() self._base = super(Entry, self) self._base.__init__(scheme, _adaptor, _adaptor_state, url, flags, session, ttype=_ttype)
def __init__(self, sid, name, dburl): """ Creates a new session A session is a distinct collection with three sub-collections in MongoDB: radical.pilot.<sid> | Base collection. Holds some metadata. | self._s radical.pilot.<sid>.cu | Collection holding all compute units. | self._w radical.pilot.<sid>.um | Collection holding all unit managers. | self._um radical.pilot.<sid>.p | Collection holding all pilots. | self._p radical.pilot.<sid>.pm | Collection holding all pilot managers. | self._pm All collections are created with a new session. Since MongoDB uses lazy-create, they only appear in the database after the first insert. That's ok. """ # mpongodb_connect wants a string at the moment mongo, db, _, _, _ = ru.mongodb_connect(str(dburl)) if not mongo or not db: raise RuntimeError("Could not connect to database at %s" % dburl) self._client = mongo self._db = db self._dburl = ru.Url(dburl) self._session_id = sid self._created = timestamp() self._connected = self._created self._closed = None # make sure session doesn't exist already if self._db[sid].count() != 0: raise RuntimeError("Session '%s' already exists." % sid) # create the db entry self._s = self._db["%s" % sid] self._s.insert({ "_id": sid, "name": name, "created": self._created, "connected": self._created }) # Create the collection shortcut: self._w = self._db["%s.cu" % sid] self._um = self._db["%s.um" % sid] self._p = self._db["%s.p" % sid] self._pm = self._db["%s.pm" % sid]