def get_js_shell(self, resource, schema): if resource not in self._cache['js_shells']: self._cache['js_shells'][resource] = dict() if schema not in self._cache['js_shells'][resource]: rcfg = self.get_resource_config(resource, schema) js_url = rcfg['job_manager_endpoint'] js_url = rcfg.get('job_manager_hop', js_url) js_url = rs.Url(js_url) elems = js_url.schema.split('+') if 'ssh' in elems: js_url.schema = 'ssh' elif 'gsissh' in elems: js_url.schema = 'gsissh' elif 'fork' in elems: js_url.schema = 'fork' elif len(elems) == 1 : js_url.schema = 'fork' else: raise Exception("invalid schema: %s" % js_url.schema) if js_url.schema == 'fork': js_url.hostname = 'localhost' self._log.debug("rsup.PTYShell('%s')", js_url) shell = rsup.PTYShell(js_url, self) self._cache['js_shells'][resource][schema] = shell return self._cache['js_shells'][resource][schema]
def test_ptyshell_nok(): """ Test pty_shell which runs command unsuccessfully """ conf = config() shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session) txt = "______1______2_____3_____" ret, out, _ = shell.run_sync("printf \"%s\" ; false" % txt) assert (ret == 1), "%s" % (repr(ret)) assert (out == txt), "%s == %s" % (repr(out), repr(txt)) assert (shell.alive()) shell.finalize(True) assert (not shell.alive())
def test_ptyshell_file_stage(): """ Test pty_shell file staging """ conf = config() shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session) txt = "______1______2_____3_____" shell.write_to_remote(txt, "/tmp/saga-test-staging") out = shell.read_from_remote("/tmp/saga-test-staging") assert (txt == out), "%s == %s" % (repr(out), repr(txt)) ret, out, _ = shell.run_sync("rm /tmp/saga-test-staging") assert (ret == 0), "%s" % (repr(ret)) assert (out == ""), "%s == ''" % (repr(out))
def test_ptyshell_async(): """ Test pty_shell which runs command successfully """ conf = config() shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session) txt = "______1______2_____3_____\n" shell.run_async("cat <<EOT") shell.send(txt) shell.send('EOT\n') ret, out = shell.find_prompt() assert (ret == 0), "%s" % (repr(ret)) assert (out == txt), "%s == %s" % (repr(out), repr(txt)) assert (shell.alive()) shell.finalize(True) assert (not shell.alive())
def test_ptyshell_prompt(): """ Test pty_shell with prompt change """ conf = config() shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session) txt = "______1______2_____3_____" ret, out, _ = shell.run_sync("printf \"%s\"" % txt) assert (ret == 0), "%s" % (repr(ret)) assert (out == txt), "%s == %s" % (repr(out), repr(txt)) shell.run_sync('export PS1="HALLO-(\\$?)-PROMPT>"', new_prompt='HALLO-\((\d)\)-PROMPT>') txt = "______1______2_____3_____" ret, out, _ = shell.run_sync("printf \"%s\"" % txt) assert (ret == 0), "%s" % (repr(ret)) assert (out == txt), "%s == %s" % (repr(out), repr(txt)) assert (shell.alive()) shell.finalize(True) assert (not shell.alive())
def initialize(self): from .... import pilot as rp self._pwd = os.getcwd() self.register_input(rps.EXECUTING_PENDING, rpc.AGENT_EXECUTING_QUEUE, self.work) self.register_output(rps.AGENT_STAGING_OUTPUT_PENDING, rpc.AGENT_STAGING_OUTPUT_QUEUE) self.register_publisher(rpc.AGENT_UNSCHEDULE_PUBSUB) self.register_subscriber(rpc.CONTROL_PUBSUB, self.command_cb) # Mimic what virtualenv's "deactivate" would do self._deactivate = "\n# deactivate pilot virtualenv\n" old_path = os.environ.get('_OLD_VIRTUAL_PATH', None) old_ppath = os.environ.get('_OLD_VIRTUAL_PYTHONPATH', None) old_home = os.environ.get('_OLD_VIRTUAL_PYTHONHOME', None) old_ps1 = os.environ.get('_OLD_VIRTUAL_PS1', None) if old_ppath: self._deactivate += 'export PATH="%s"\n' % old_ppath if old_path: self._deactivate += 'export PYTHONPATH="%s"\n' % old_path if old_home: self._deactivate += 'export PYTHON_HOME="%s"\n' % old_home if old_ps1: self._deactivate += 'export PS1="%s"\n' % old_ps1 self._deactivate += 'unset VIRTUAL_ENV\n\n' # FIXME: we should not alter the environment of the running agent, but # only make sure that the CU finds a pristine env. That also # holds for the unsetting below -- AM if old_path: os.environ['PATH'] = old_path if old_ppath: os.environ['PYTHONPATH'] = old_ppath if old_home: os.environ['PYTHON_HOME'] = old_home if old_ps1: os.environ['PS1'] = old_ps1 if 'VIRTUAL_ENV' in os.environ: del (os.environ['VIRTUAL_ENV']) # simplify shell startup / prompt detection os.environ['PS1'] = '$ ' self._task_launcher = rp.agent.LaunchMethod.create( name=self._cfg['task_launch_method'], cfg=self._cfg, session=self._session) self._mpi_launcher = rp.agent.LaunchMethod.create( name=self._cfg['mpi_launch_method'], cfg=self._cfg, session=self._session) # TODO: test that this actually works # Remove the configured set of environment variables from the # environment that we pass to Popen. for e in list(os.environ.keys()): env_removables = list() if self._mpi_launcher: env_removables += self._mpi_launcher.env_removables if self._task_launcher: env_removables += self._task_launcher.env_removables for r in env_removables: if e.startswith(r): os.environ.pop(e, None) # if we need to transplant any original env into the CU, we dig the # respective keys from the dump made by bootstrap_0.sh self._env_cu_export = dict() if self._cfg.get('export_to_cu'): with open('env.orig', 'r') as f: for line in f.readlines(): if '=' in line: k, v = line.split('=', 1) key = k.strip() val = v.strip() if key in self._cfg['export_to_cu']: self._env_cu_export[key] = val # the registry keeps track of units to watch, indexed by their shell # spawner process ID. As the registry is shared between the spawner and # watcher thread, we use a lock while accessing it. self._registry = dict() self._registry_lock = ru.RLock() self._cus_to_cancel = list() self._cancel_lock = ru.RLock() self._cached_events = list() # keep monitoring events for pid's which # are not yet known # get some threads going -- those will do all the work. import radical.saga.utils.pty_shell as sups self.launcher_shell = sups.PTYShell("fork://localhost/") self.monitor_shell = sups.PTYShell("fork://localhost/") # run the spawner on the shells # tmp = tempfile.gettempdir() # Moving back to shared file system again, until it reaches maturity, # as this breaks launch methods with a hop, e.g. ssh. # FIXME: see #658 self._pid = self._cfg['pid'] self._spawner_tmp = "/%s/%s-%s" % (self._pwd, self._pid, self.uid) ret, out, _ = self.launcher_shell.run_sync \ ("/bin/sh %s/agent/executing/shell_spawner.sh %s" % (os.path.dirname (rp.__file__), self._spawner_tmp)) if ret != 0: raise RuntimeError("launcher bootstrap failed: (%s)(%s)", ret, out) ret, out, _ = self.monitor_shell.run_sync \ ("/bin/sh %s/agent/executing/shell_spawner.sh %s" % (os.path.dirname (rp.__file__), self._spawner_tmp)) if ret != 0: raise RuntimeError("monitor bootstrap failed: (%s)(%s)", ret, out) # run watcher thread self._terminate = threading.Event() self._watcher = threading.Thread(target=self._watch, name="Watcher") self._watcher.daemon = True self._watcher.start() self.gtod = "%s/gtod" % self._pwd