예제 #1
0
    def get_js_shell(self, resource, schema):

        if resource not in self._cache['js_shells']:
            self._cache['js_shells'][resource] = dict()

        if schema not in self._cache['js_shells'][resource]:

            rcfg   = self.get_resource_config(resource, schema)

            js_url = rcfg['job_manager_endpoint']
            js_url = rcfg.get('job_manager_hop', js_url)
            js_url = rs.Url(js_url)

            elems  = js_url.schema.split('+')

            if   'ssh'    in elems: js_url.schema = 'ssh'
            elif 'gsissh' in elems: js_url.schema = 'gsissh'
            elif 'fork'   in elems: js_url.schema = 'fork'
            elif len(elems) == 1  : js_url.schema = 'fork'
            else: raise Exception("invalid schema: %s" % js_url.schema)

            if js_url.schema == 'fork':
                js_url.hostname = 'localhost'

            self._log.debug("rsup.PTYShell('%s')", js_url)
            shell = rsup.PTYShell(js_url, self)
            self._cache['js_shells'][resource][schema] = shell

        return self._cache['js_shells'][resource][schema]
예제 #2
0
def test_ptyshell_nok():
    """ Test pty_shell which runs command unsuccessfully """
    conf = config()
    shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session)

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync("printf \"%s\" ; false" % txt)
    assert (ret == 1), "%s" % (repr(ret))
    assert (out == txt), "%s == %s" % (repr(out), repr(txt))

    assert (shell.alive())
    shell.finalize(True)
    assert (not shell.alive())
예제 #3
0
def test_ptyshell_file_stage():
    """ Test pty_shell file staging """
    conf = config()
    shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session)

    txt = "______1______2_____3_____"
    shell.write_to_remote(txt, "/tmp/saga-test-staging")
    out = shell.read_from_remote("/tmp/saga-test-staging")

    assert (txt == out), "%s == %s" % (repr(out), repr(txt))

    ret, out, _ = shell.run_sync("rm /tmp/saga-test-staging")
    assert (ret == 0), "%s" % (repr(ret))
    assert (out == ""), "%s == ''" % (repr(out))
예제 #4
0
def test_ptyshell_async():
    """ Test pty_shell which runs command successfully """
    conf = config()
    shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session)

    txt = "______1______2_____3_____\n"
    shell.run_async("cat <<EOT")

    shell.send(txt)
    shell.send('EOT\n')

    ret, out = shell.find_prompt()

    assert (ret == 0), "%s" % (repr(ret))
    assert (out == txt), "%s == %s" % (repr(out), repr(txt))

    assert (shell.alive())
    shell.finalize(True)
    assert (not shell.alive())
예제 #5
0
def test_ptyshell_prompt():
    """ Test pty_shell with prompt change """
    conf = config()
    shell = sups.PTYShell(saga.Url(conf.job_service_url), conf.session)

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync("printf \"%s\"" % txt)
    assert (ret == 0), "%s" % (repr(ret))
    assert (out == txt), "%s == %s" % (repr(out), repr(txt))

    shell.run_sync('export PS1="HALLO-(\\$?)-PROMPT>"',
                   new_prompt='HALLO-\((\d)\)-PROMPT>')

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync("printf \"%s\"" % txt)
    assert (ret == 0), "%s" % (repr(ret))
    assert (out == txt), "%s == %s" % (repr(out), repr(txt))

    assert (shell.alive())
    shell.finalize(True)
    assert (not shell.alive())
예제 #6
0
    def initialize(self):

        from .... import pilot as rp

        self._pwd = os.getcwd()

        self.register_input(rps.EXECUTING_PENDING, rpc.AGENT_EXECUTING_QUEUE,
                            self.work)

        self.register_output(rps.AGENT_STAGING_OUTPUT_PENDING,
                             rpc.AGENT_STAGING_OUTPUT_QUEUE)

        self.register_publisher(rpc.AGENT_UNSCHEDULE_PUBSUB)
        self.register_subscriber(rpc.CONTROL_PUBSUB, self.command_cb)

        # Mimic what virtualenv's "deactivate" would do
        self._deactivate = "\n# deactivate pilot virtualenv\n"

        old_path = os.environ.get('_OLD_VIRTUAL_PATH', None)
        old_ppath = os.environ.get('_OLD_VIRTUAL_PYTHONPATH', None)
        old_home = os.environ.get('_OLD_VIRTUAL_PYTHONHOME', None)
        old_ps1 = os.environ.get('_OLD_VIRTUAL_PS1', None)

        if old_ppath: self._deactivate += 'export PATH="%s"\n' % old_ppath
        if old_path: self._deactivate += 'export PYTHONPATH="%s"\n' % old_path
        if old_home: self._deactivate += 'export PYTHON_HOME="%s"\n' % old_home
        if old_ps1: self._deactivate += 'export PS1="%s"\n' % old_ps1

        self._deactivate += 'unset VIRTUAL_ENV\n\n'

        # FIXME: we should not alter the environment of the running agent, but
        #        only make sure that the CU finds a pristine env.  That also
        #        holds for the unsetting below -- AM
        if old_path: os.environ['PATH'] = old_path
        if old_ppath: os.environ['PYTHONPATH'] = old_ppath
        if old_home: os.environ['PYTHON_HOME'] = old_home
        if old_ps1: os.environ['PS1'] = old_ps1

        if 'VIRTUAL_ENV' in os.environ:
            del (os.environ['VIRTUAL_ENV'])

        # simplify shell startup / prompt detection
        os.environ['PS1'] = '$ '

        self._task_launcher = rp.agent.LaunchMethod.create(
            name=self._cfg['task_launch_method'],
            cfg=self._cfg,
            session=self._session)

        self._mpi_launcher = rp.agent.LaunchMethod.create(
            name=self._cfg['mpi_launch_method'],
            cfg=self._cfg,
            session=self._session)

        # TODO: test that this actually works
        # Remove the configured set of environment variables from the
        # environment that we pass to Popen.
        for e in list(os.environ.keys()):

            env_removables = list()

            if self._mpi_launcher:
                env_removables += self._mpi_launcher.env_removables

            if self._task_launcher:
                env_removables += self._task_launcher.env_removables

            for r in env_removables:
                if e.startswith(r):
                    os.environ.pop(e, None)

        # if we need to transplant any original env into the CU, we dig the
        # respective keys from the dump made by bootstrap_0.sh
        self._env_cu_export = dict()
        if self._cfg.get('export_to_cu'):
            with open('env.orig', 'r') as f:
                for line in f.readlines():
                    if '=' in line:
                        k, v = line.split('=', 1)
                        key = k.strip()
                        val = v.strip()
                        if key in self._cfg['export_to_cu']:
                            self._env_cu_export[key] = val

        # the registry keeps track of units to watch, indexed by their shell
        # spawner process ID.  As the registry is shared between the spawner and
        # watcher thread, we use a lock while accessing it.
        self._registry = dict()
        self._registry_lock = ru.RLock()

        self._cus_to_cancel = list()
        self._cancel_lock = ru.RLock()

        self._cached_events = list()  # keep monitoring events for pid's which
        # are not yet known

        # get some threads going -- those will do all the work.
        import radical.saga.utils.pty_shell as sups

        self.launcher_shell = sups.PTYShell("fork://localhost/")
        self.monitor_shell = sups.PTYShell("fork://localhost/")

        # run the spawner on the shells
        # tmp = tempfile.gettempdir()
        # Moving back to shared file system again, until it reaches maturity,
        # as this breaks launch methods with a hop, e.g. ssh.
        # FIXME: see #658
        self._pid = self._cfg['pid']
        self._spawner_tmp = "/%s/%s-%s" % (self._pwd, self._pid, self.uid)

        ret, out, _  = self.launcher_shell.run_sync \
                           ("/bin/sh %s/agent/executing/shell_spawner.sh %s"
                           % (os.path.dirname (rp.__file__), self._spawner_tmp))
        if ret != 0:
            raise RuntimeError("launcher bootstrap failed: (%s)(%s)", ret, out)

        ret, out, _  = self.monitor_shell.run_sync \
                           ("/bin/sh %s/agent/executing/shell_spawner.sh %s"
                           % (os.path.dirname (rp.__file__), self._spawner_tmp))
        if ret != 0:
            raise RuntimeError("monitor bootstrap failed: (%s)(%s)", ret, out)

        # run watcher thread
        self._terminate = threading.Event()
        self._watcher = threading.Thread(target=self._watch, name="Watcher")
        self._watcher.daemon = True
        self._watcher.start()

        self.gtod = "%s/gtod" % self._pwd