Exemplo n.º 1
0
        def heartbeat_callback(heartbeat, headers):

            eeagent_id = heartbeat['eeagent_id']
            agent_client = SimpleResourceAgentClient(eeagent_id, name=eeagent_id, process=FakeProcess())
            ee_client = ExecutionEngineAgentClient(agent_client, timeout=2)

            try:
                ee_client.dump_state()
            except:
                log.exception("Heartbeat Failed!")
                beat_died[0] = True
Exemplo n.º 2
0
        def heartbeat_callback(heartbeat, headers):

            eeagent_id = heartbeat["eeagent_id"]
            agent_client = SimpleResourceAgentClient(eeagent_id, name=eeagent_id, process=FakeProcess())
            ee_client = ExecutionEngineAgentClient(agent_client, timeout=2)

            try:
                ee_client.dump_state()
            except:
                log.exception("Heartbeat Failed!")
                beat_died[0] = True
Exemplo n.º 3
0
class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):
    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url("res/deploy/r2cei.yml")

        self.resource_id = "eeagent_1234"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            "eeagent": {
                "heartbeat": 0,
                "slots": 100,
                "name": "pyon_eeagent",
                "launch_type": {
                    "name": "supd",
                    "pyon_directory": os.getcwd(),
                    "supd_directory": self.supd_directory,
                    "supdexe": "bin/supervisord",
                },
            },
            "agent": {"resource_id": self.resource_id},
            "logging": {
                "loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
                "root": {"handlers": ["console"]},
            },
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent",
            config=self.agent_config,
        )
        log.info("Agent pid=%s.", str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info("Got eea client %s.", str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.supd_directory)

    def wait_for_state(self, upid, desired_state, timeout=5):
        attempts = 0
        while timeout > attempts:
            state = self.eea_client.dump_state().result
            proc = get_proc_for_upid(state, upid)
            if proc.get("state") == desired_state:
                return
            gevent.sleep(1)
            attempts += 1

        assert False, "Process %s took too long to get to %s" % (upid, desired_state)

    @needs_eeagent
    def test_basics(self):
        true_u_pid = "test0"
        round = 0
        run_type = "supd"
        true_parameters = {"exec": "true", "argv": []}
        self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)

        self.wait_for_state(true_u_pid, [800, "EXITED"])

        cat_u_pid = "test1"
        round = 0
        run_type = "supd"
        cat_parameters = {"exec": "cat", "argv": []}
        self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
        self.wait_for_state(cat_u_pid, [500, "RUNNING"])

        self.eea_client.terminate_process(cat_u_pid, round)
        self.wait_for_state(cat_u_pid, [700, "TERMINATED"])
Exemplo n.º 4
0
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):

    _webserver = None

    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url("res/deploy/r2cei.yml")

        self.resource_id = "eeagent_123456789"
        self._eea_name = "eeagent"

        self.persistence_directory = tempfile.mkdtemp()

        self.agent_config = {
            "eeagent": {
                "heartbeat": 1,
                "slots": 100,
                "name": "pyon_eeagent",
                "launch_type": {"name": "pyon", "persistence_directory": self.persistence_directory},
            },
            "agent": {"resource_id": self.resource_id},
            "logging": {
                "loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
                "root": {"handlers": ["console"]},
            },
        }

        self._start_eeagent()

    def _start_eeagent(self):
        self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
        self.container = self.container_client._get_container_instance()

        # Start eeagent.
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent",
            config=self.agent_config,
        )
        log.info("Agent pid=%s.", str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info("Got eea client %s.", str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self._stop_webserver()
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.persistence_directory)

    def _start_webserver(self, directory_to_serve, port=None):
        """ Start a webserver for testing code download
        Note: tries really hard to get a port, and if it can't use
        the suggested port, randomly picks another, and returns it
        """

        def log_message(self, format, *args):
            # swallow log massages
            pass

        class Server(HTTPServer):

            requests = 0

            def serve_forever(self):
                self._serving = 1
                while self._serving:
                    self.handle_request()
                    self.requests += 1

            def stop(self):
                self._serving = 0

        if port is None:
            port = 8008
        Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
        Handler.log_message = log_message

        for i in range(0, 100):
            try:
                self._webserver = Server(("localhost", port), Handler)
            except socket.error:
                print "port %s is in use, picking another" % port
                port = randint(8000, 10000)
                continue
            else:
                break

        self._web_glet = gevent.spawn(self._webserver.serve_forever)
        return port

    def _stop_webserver(self):
        if self._webserver is not None:
            self._web_glet.kill()

    def _enable_code_download(self, whitelist=None):

        if whitelist is None:
            whitelist = []

        self.container.terminate_process(self._eea_pid)
        self.agent_config["eeagent"]["code_download"] = {"enabled": True, "whitelist": whitelist}
        self._start_eeagent()

    def wait_for_state(self, upid, desired_state, timeout=30):
        attempts = 0
        last_state = None
        while timeout > attempts:
            try:
                state = self.eea_client.dump_state().result
            except Timeout:
                log.warn("Timeout calling EEAgent dump_state. retrying.")
                continue
            proc = get_proc_for_upid(state, upid)
            last_state = proc.get("state")
            if last_state == desired_state:
                return
            gevent.sleep(1)
            attempts += 1

        assert False, "Process %s took too long to get to %s, had %s" % (upid, desired_state, last_state)

    @needs_eeagent
    def test_basics(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = "test_x"
        module = "ion.agents.cei.test.test_eeagent"
        cls = "TestProcess"
        parameters = {"name": proc_name, "module": module, "cls": cls}

        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, "RUNNING"])

        state = self.eea_client.dump_state().result
        assert len(state["processes"]) == 1

        self.eea_client.terminate_process(u_pid, round)
        self.wait_for_state(u_pid, [700, "TERMINATED"])

        state = self.eea_client.dump_state().result
        assert len(state["processes"]) == 1

        self.eea_client.cleanup_process(u_pid, round)
        state = self.eea_client.dump_state().result
        assert len(state["processes"]) == 0

    @needs_eeagent
    def test_restart(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = "test_x"
        module = "ion.agents.cei.test.test_eeagent"
        cls = "TestProcess"
        parameters = {"name": proc_name, "module": module, "cls": cls}

        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, "RUNNING"])

        state = self.eea_client.dump_state().result
        assert len(state["processes"]) == 1

        # Start again with incremented round. eeagent should restart the process
        round += 1

        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, "RUNNING"])

        state = self.eea_client.dump_state().result
        ee_round = state["processes"][0]["round"]
        assert round == int(ee_round)

        # TODO: this test is disabled, as the restart op is disabled
        # Run restart with incremented round. eeagent should restart the process
        # round += 1

        # self.eea_client.restart_process(u_pid, round)
        # self.wait_for_state(u_pid, [500, 'RUNNING'])

        # state = self.eea_client.dump_state().result
        # ee_round = state['processes'][0]['round']
        # assert round == int(ee_round)

        self.eea_client.terminate_process(u_pid, round)
        self.wait_for_state(u_pid, [700, "TERMINATED"])

        state = self.eea_client.dump_state().result
        assert len(state["processes"]) == 1

        self.eea_client.cleanup_process(u_pid, round)
        state = self.eea_client.dump_state().result
        assert len(state["processes"]) == 0

    @needs_eeagent
    def test_failing_process(self):
        u_pid = "testfail"
        round = 0
        run_type = "pyon"
        proc_name = "test_x"
        module = "ion.agents.cei.test.test_eeagent"
        cls = "TestProcessFail"
        parameters = {"name": proc_name, "module": module, "cls": cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [850, "FAILED"])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_slow_to_start(self):
        upids = map(lambda i: str(uuid.uuid4().hex), range(0, 10))
        round = 0
        run_type = "pyon"
        proc_name = "test_x"
        module = "ion.agents.cei.test.test_eeagent"
        cls = "TestProcessSlowStart"
        parameters = {"name": proc_name, "module": module, "cls": cls}
        for upid in upids:
            self.eea_client.launch_process(upid, round, run_type, parameters)

        for upid in upids:
            self.wait_for_state(upid, [500, "RUNNING"], timeout=60)

    @needs_eeagent
    def test_start_cancel(self):
        upid = str(uuid.uuid4().hex)
        round = 0
        run_type = "pyon"
        proc_name = "test_x"
        module = "ion.agents.cei.test.test_eeagent"
        cls = "TestProcessSlowStart"
        parameters = {"name": proc_name, "module": module, "cls": cls}
        self.eea_client.launch_process(upid, round, run_type, parameters)
        self.wait_for_state(upid, [400, "PENDING"])
        self.eea_client.terminate_process(upid, round)
        self.wait_for_state(upid, [700, "TERMINATED"])

    @needs_eeagent
    def test_kill_and_revive(self):
        """test_kill_and_revive
        Ensure that when an eeagent dies, it pulls the processes it owned from
        persistence, and marks them as failed, so the PD can figure out what to
        do with them
        """
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = "test_transform"
        module = "ion.agents.cei.test.test_eeagent"
        cls = "TestProcess"
        parameters = {"name": proc_name, "module": module, "cls": cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, "RUNNING"])

        # Kill and restart eeagent. Also, kill proc started by eea to simulate
        # a killed container
        old_eea_pid = str(self._eea_pid)
        self.container.terminate_process(self._eea_pid)
        proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
        self.assertIsNotNone(proc_to_kill)
        self.container.terminate_process(proc_to_kill.id)

        self._start_eeagent()

        self.assertNotEqual(old_eea_pid, self._eea_pid)

        self.wait_for_state(u_pid, [850, "FAILED"])

    @needs_eeagent
    def test_run_out_of_slots(self):
        """test_run_out_of_slots
        """
        old_eea_pid = str(self._eea_pid)
        self.container.terminate_process(self._eea_pid)
        self.agent_config["eeagent"]["slots"] = 1
        self._start_eeagent()
        self.assertNotEqual(old_eea_pid, self._eea_pid)

        u_pid_0, u_pid_1 = "test0", "test1"
        round = 0
        run_type = "pyon"
        proc_name = "test_transform"
        module = "ion.agents.cei.test.test_eeagent"
        cls = "TestProcess"
        parameters = {"name": proc_name, "module": module, "cls": cls}

        self.eea_client.launch_process(u_pid_0, round, run_type, parameters)
        self.wait_for_state(u_pid_0, [500, "RUNNING"])

        self.eea_client.launch_process(u_pid_1, round, run_type, parameters)
        self.wait_for_state(u_pid_1, [900, "REJECTED"])

        old_eea_pid = str(self._eea_pid)
        self.container.terminate_process(self._eea_pid)
        self.agent_config["eeagent"]["slots"] = 1
        self._start_eeagent()
        self.assertNotEqual(old_eea_pid, self._eea_pid)

        self.wait_for_state(u_pid_0, [850, "FAILED"])
        self.wait_for_state(u_pid_1, [900, "REJECTED"])

    @needs_eeagent
    def test_download_code(self):

        self._enable_code_download(whitelist=["*"])

        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = "test_transform"
        module = "ion.my.module.to.download"
        module_uri = "file://%s/downloads/module_to_download.py" % get_this_directory()
        bad_module_uri = "file:///tmp/notreal/module_to_download.py"

        cls = "TestDownloadProcess"

        parameters = {"name": proc_name, "module": module, "module_uri": bad_module_uri, "cls": cls}
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        print response
        assert response.status == 404
        assert "Unable to download" in response.result

        parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, "RUNNING"])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_whitelist(self):

        downloads_directory = os.path.join(get_this_directory(), "downloads")
        http_port = 8910
        http_port = self._start_webserver(downloads_directory, port=http_port)

        while self._webserver is None:
            print "Waiting for webserver to come up"
            gevent.sleep(1)

        assert self._webserver.requests == 0

        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = "test_transform"
        module = "ion.my.module"
        module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
        cls = "TestDownloadProcess"
        parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        assert response.status == 401
        assert "Code download not enabled" in response.result

        # Test no whitelist
        self._enable_code_download()

        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        print response
        assert response.status == 401
        assert "not in code_download whitelist" in response.result

        # Test not matching
        self._enable_code_download(whitelist=["blork"])

        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        assert response.status == 401
        assert "not in code_download whitelist" in response.result

        # Test exact matching
        self._enable_code_download(whitelist=["localhost"])

        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, "RUNNING"])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

        # Test wildcard
        self._enable_code_download(whitelist=["*"])

        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, "RUNNING"])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_caching(self):

        downloads_directory = os.path.join(get_this_directory(), "downloads")
        http_port = 8910
        http_port = self._start_webserver(downloads_directory, port=http_port)

        while self._webserver is None:
            print "Waiting for webserver to come up"
            gevent.sleep(1)

        self._enable_code_download(["*"])
        assert self._webserver.requests == 0

        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = "test_transform"
        module = "ion.my.module"
        module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
        cls = "TestDownloadProcess"
        parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}

        # Launch a process, check that webserver is hit
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, "RUNNING"])
        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

        assert self._webserver.requests == 1

        # Launch another process, check that webserver is still only hit once
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, "RUNNING"])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

        assert self._webserver.requests == 1

        u_pid = "test5"
        round = 0
        run_type = "pyon"
        proc_name = "test_transformx"
        module = "ion.agents.cei.test.test_eeagent"
        module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
        cls = "TestProcess"
        parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}

        # Test that a module that is already available in tarball won't trigger a download
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, "RUNNING"])
        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

        assert self._webserver.requests == 1

        u_pid = "test9"
        round = 0
        run_type = "pyon"
        proc_name = "test_transformx"
        module = "ion.agents.cei.test.test_eeagent"
        module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
        cls = "TestProcessNotReal"
        parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}

        # Test behaviour of a non existant class with no download
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [850, "FAILED"])
        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)
Exemplo n.º 5
0
class ExecutionEngineAgentPyonSingleIntTest(IonIntegrationTestCase):
    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url("res/deploy/r2cei.yml")

        self.resource_id = "eeagent_123456"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            "eeagent": {
                "heartbeat": 0,
                "slots": 100,
                "name": "pyon_eeagent",
                "launch_type": {
                    "name": "pyon_single",
                    "pyon_directory": os.getcwd(),
                    "supd_directory": self.supd_directory,
                    "supdexe": "bin/supervisord",
                },
            },
            "agent": {"resource_id": self.resource_id},
            "logging": {
                "loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
                "root": {"handlers": ["console"]},
            },
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent",
            config=self.agent_config,
        )
        log.info("Agent pid=%s.", str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info("Got eea client %s.", str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.supd_directory)

    @needs_eeagent
    def test_basics(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon_single"
        proc_name = "test_transform"
        module = "ion.agents.cei.test.test_eeagent"
        cls = "TestProcess"
        parameters = {"name": proc_name, "module": module, "cls": cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertEqual(proc.get("state"), [500, "RUNNING"])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)
        self.assertEqual(proc.get("state"), [700, "TERMINATED"])
Exemplo n.º 6
0
class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):

    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_1234"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
              'heartbeat': 0,
              'slots': 100,
              'name': 'pyon_eeagent',
              'launch_type': {
                'name': 'supd',
                'pyon_directory': os.getcwd(),
                'supd_directory': self.supd_directory,
                'supdexe': 'bin/supervisord'
              },
            },
            'agent': {'resource_id': self.resource_id},
            'logging': {
            'loggers': {
              'eeagent': {
                'level': 'DEBUG',
                'handlers': ['console']
              }
            },
            'root': {
              'handlers': ['console']
            },
          }
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = ResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.supd_directory)

    @needs_eeagent
    def test_basics(self):
        true_u_pid = "test0"
        round = 0
        run_type = "supd"
        true_parameters = {'exec': 'true', 'argv': []}
        self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, true_u_pid)

        self.assertEqual(proc.get('state'), [800, 'EXITED'])

        cat_u_pid = "test1"
        round = 0
        run_type = "supd"
        cat_parameters = {'exec': 'cat', 'argv': []}
        self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, cat_u_pid)
        self.assertEqual(proc.get('state'), [500, 'RUNNING'])

        self.eea_client.terminate_process(cat_u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, cat_u_pid)
        self.assertEqual(proc.get('state'), [700, 'TERMINATED'])
Exemplo n.º 7
0
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
    from ion.agents.cei.execution_engine_agent import ExecutionEngineAgentClient

    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_123456789"
        self._eea_name = "eeagent"

        self.persistence_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
              'heartbeat': 0,
              'slots': 100,
              'name': 'pyon_eeagent',
              'launch_type': {
                'name': 'pyon',
                'persistence_directory': self.persistence_directory,
              },
            },
            'agent': {'resource_id': self.resource_id},
            'logging': {
            'loggers': {
              'eeagent': {
                'level': 'DEBUG',
                'handlers': ['console']
              }
            },
            'root': {
              'handlers': ['console']
            },
          }
        }

        self._start_eeagent()

    def _start_eeagent(self):
        self.container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        self.container = self.container_client._get_container_instance()

        # Start eeagent.
        self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = ResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.persistence_directory)

    @needs_eeagent
    def test_basics(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
        self.assertEqual(proc.get('state'), [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_kill_and_revive(self):
        """test_kill_and_revive
        Ensure that when an eeagent dies, it pulls the processes it owned from
        persistence, and marks them as failed, so the PD can figure out what to
        do with them
        """
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
        self.assertEqual(proc.get('state'), [500, 'RUNNING'])

        # Kill and restart eeagent. Also, kill proc started by eea to simulate
        # a killed container
        old_eea_pid = str(self._eea_pid)
        self.container.terminate_process(self._eea_pid)
        proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
        self.assertIsNotNone(proc_to_kill)
        self.container.terminate_process(proc_to_kill.id)

        self._start_eeagent()

        self.assertNotEqual(old_eea_pid, self._eea_pid)

        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
        self.assertEqual(proc.get('state'), [850, 'FAILED'])
Exemplo n.º 8
0
class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):

    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_1234"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
                'heartbeat': 0,
                'slots': 100,
                'name': 'pyon_eeagent',
                'launch_type': {
                    'name': 'supd',
                    'pyon_directory': os.getcwd(),
                    'supd_directory': self.supd_directory,
                    'supdexe': 'bin/supervisord'
                },
            },
            'agent': {'resource_id': self.resource_id},
            'logging': {
                'loggers': {
                    'eeagent': {
                        'level': 'DEBUG',
                        'handlers': ['console']
                    }
                },
                'root': {
                    'handlers': ['console']
                }
            }
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(
            node=self.container.node, name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.supd_directory)

    def wait_for_state(self, upid, desired_state, timeout=5):
        attempts = 0
        while timeout > attempts:
            state = self.eea_client.dump_state().result
            proc = get_proc_for_upid(state, upid)
            if proc.get('state') == desired_state:
                return
            gevent.sleep(1)
            attempts += 1

        assert False, "Process %s took too long to get to %s" % (upid, desired_state)

    @needs_eeagent
    def test_basics(self):
        true_u_pid = "test0"
        round = 0
        run_type = "supd"
        true_parameters = {'exec': 'true', 'argv': []}
        self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)

        self.wait_for_state(true_u_pid, [800, 'EXITED'])

        cat_u_pid = "test1"
        round = 0
        run_type = "supd"
        cat_parameters = {'exec': 'cat', 'argv': []}
        self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
        self.wait_for_state(cat_u_pid, [500, 'RUNNING'])

        self.eea_client.terminate_process(cat_u_pid, round)
        self.wait_for_state(cat_u_pid, [700, 'TERMINATED'])
Exemplo n.º 9
0
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):

    _webserver = None

    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_123456789"
        self._eea_name = "eeagent"

        self.persistence_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
                'heartbeat': 1,
                'slots': 100,
                'name': 'pyon_eeagent',
                'launch_type': {
                    'name': 'pyon',
                    'persistence_directory': self.persistence_directory,
                }
            },
            'agent': {'resource_id': self.resource_id},
            'logging': {
                'loggers': {
                    'eeagent': {
                        'level': 'DEBUG',
                        'handlers': ['console']
                    }
                },
                'root': {
                    'handlers': ['console']
                },
            }
        }

        self._start_eeagent()

    def _start_eeagent(self):
        self.container_client = ContainerAgentClient(
            node=self.container.node, name=self.container.name)
        self.container = self.container_client._get_container_instance()

        # Start eeagent.
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self._stop_webserver()
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.persistence_directory)

    def _start_webserver(self, directory_to_serve, port=None):
        """ Start a webserver for testing code download
        Note: tries really hard to get a port, and if it can't use
        the suggested port, randomly picks another, and returns it
        """
        def log_message(self, format, *args):
            #swallow log massages
            pass

        class Server(HTTPServer):

            requests = 0

            def serve_forever(self):
                self._serving = 1
                while self._serving:
                    self.handle_request()
                    self.requests += 1

            def stop(self):
                self._serving = 0

        if port is None:
            port = 8008
        Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
        Handler.log_message = log_message

        for i in range(0, 100):
            try:
                self._webserver = Server(("localhost", port), Handler)
            except socket.error:
                print "port %s is in use, picking another" % port
                port = randint(8000, 10000)
                continue
            else:
                break

        self._web_glet = gevent.spawn(self._webserver.serve_forever)
        return port

    def _stop_webserver(self):
        if self._webserver is not None:
            self._web_glet.kill()

    def _enable_code_download(self, whitelist=None):

        if whitelist is None:
            whitelist = []

        self.container.terminate_process(self._eea_pid)
        self.agent_config['eeagent']['code_download'] = {
            'enabled': True,
            'whitelist': whitelist
        }
        self._start_eeagent()

    def wait_for_state(self, upid, desired_state, timeout=30):
        attempts = 0
        last_state = None
        while timeout > attempts:
            try:
                state = self.eea_client.dump_state().result
            except Timeout:
                log.warn("Timeout calling EEAgent dump_state. retrying.")
                continue
            proc = get_proc_for_upid(state, upid)
            last_state = proc.get('state')
            if last_state == desired_state:
                return
            gevent.sleep(1)
            attempts += 1

        assert False, "Process %s took too long to get to %s, had %s" % (upid, desired_state, last_state)

    @needs_eeagent
    def test_basics(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}

        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, 'RUNNING'])

        state = self.eea_client.dump_state().result
        assert len(state['processes']) == 1

        self.eea_client.terminate_process(u_pid, round)
        self.wait_for_state(u_pid, [700, 'TERMINATED'])

        state = self.eea_client.dump_state().result
        assert len(state['processes']) == 1

        self.eea_client.cleanup_process(u_pid, round)
        state = self.eea_client.dump_state().result
        assert len(state['processes']) == 0

    @needs_eeagent
    def test_duplicate(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}

        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, 'RUNNING'])

        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, 'RUNNING'])

        state = self.eea_client.dump_state().result
        assert len(state['processes']) == 1

        self.eea_client.terminate_process(u_pid, round)
        self.wait_for_state(u_pid, [700, 'TERMINATED'])

        state = self.eea_client.dump_state().result
        assert len(state['processes']) == 1

        self.eea_client.cleanup_process(u_pid, round)
        state = self.eea_client.dump_state().result
        assert len(state['processes']) == 0

    @needs_eeagent
    def test_restart(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}

        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, 'RUNNING'])

        state = self.eea_client.dump_state().result
        assert len(state['processes']) == 1

        # Start again with incremented round. eeagent should restart the process
        round += 1

        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, 'RUNNING'])

        state = self.eea_client.dump_state().result
        ee_round = state['processes'][0]['round']
        assert round == int(ee_round)

        # TODO: this test is disabled, as the restart op is disabled
        # Run restart with incremented round. eeagent should restart the process
        #round += 1

        #self.eea_client.restart_process(u_pid, round)
        #self.wait_for_state(u_pid, [500, 'RUNNING'])

        #state = self.eea_client.dump_state().result
        #ee_round = state['processes'][0]['round']
        #assert round == int(ee_round)

        self.eea_client.terminate_process(u_pid, round)
        self.wait_for_state(u_pid, [700, 'TERMINATED'])

        state = self.eea_client.dump_state().result
        assert len(state['processes']) == 1

        self.eea_client.cleanup_process(u_pid, round)
        state = self.eea_client.dump_state().result
        assert len(state['processes']) == 0

    @needs_eeagent
    def test_failing_process(self):
        u_pid = "testfail"
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcessFail'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [850, 'FAILED'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_slow_to_start(self):
        upids = map(lambda i: str(uuid.uuid4().hex), range(0, 10))
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcessSlowStart'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        for upid in upids:
            self.eea_client.launch_process(upid, round, run_type, parameters)

        for upid in upids:
            self.wait_for_state(upid, [500, 'RUNNING'], timeout=60)

    @needs_eeagent
    def test_start_cancel(self):
        upid = str(uuid.uuid4().hex)
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcessSlowStart'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(upid, round, run_type, parameters)
        self.wait_for_state(upid, [400, 'PENDING'])
        self.eea_client.terminate_process(upid, round)
        self.wait_for_state(upid, [700, 'TERMINATED'])

    @needs_eeagent
    def test_kill_and_revive(self):
        """test_kill_and_revive
        Ensure that when an eeagent dies, it pulls the processes it owned from
        persistence, and marks them as failed, so the PD can figure out what to
        do with them
        """
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, 'RUNNING'])

        # Kill and restart eeagent. Also, kill proc started by eea to simulate
        # a killed container
        old_eea_pid = str(self._eea_pid)
        self.container.terminate_process(self._eea_pid)
        proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
        self.assertIsNotNone(proc_to_kill)
        self.container.terminate_process(proc_to_kill.id)

        self._start_eeagent()

        self.assertNotEqual(old_eea_pid, self._eea_pid)

        self.wait_for_state(u_pid, [850, 'FAILED'])

    @needs_eeagent
    def test_run_out_of_slots(self):
        """test_run_out_of_slots
        """
        old_eea_pid = str(self._eea_pid)
        self.container.terminate_process(self._eea_pid)
        self.agent_config['eeagent']['slots'] = 1
        self._start_eeagent()
        self.assertNotEqual(old_eea_pid, self._eea_pid)

        u_pid_0, u_pid_1 = "test0", "test1"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}

        self.eea_client.launch_process(u_pid_0, round, run_type, parameters)
        self.wait_for_state(u_pid_0, [500, 'RUNNING'])

        self.eea_client.launch_process(u_pid_1, round, run_type, parameters)
        self.wait_for_state(u_pid_1, [900, 'REJECTED'])

        old_eea_pid = str(self._eea_pid)
        self.container.terminate_process(self._eea_pid)
        self.agent_config['eeagent']['slots'] = 1
        self._start_eeagent()
        self.assertNotEqual(old_eea_pid, self._eea_pid)

        self.wait_for_state(u_pid_0, [850, 'FAILED'])
        self.wait_for_state(u_pid_1, [900, 'REJECTED'])

    @needs_eeagent
    def test_download_code(self):

        self._enable_code_download(whitelist=['*'])

        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = "ion.my.module.to.download"
        module_uri = 'file://%s/downloads/module_to_download.py' % get_this_directory()
        bad_module_uri = 'file:///tmp/notreal/module_to_download.py'

        cls = 'TestDownloadProcess'

        parameters = {'name': proc_name, 'module': module, 'module_uri': bad_module_uri, 'cls': cls}
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        print response
        assert response.status == 404
        assert "Unable to download" in response.result

        parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
        round += 1
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_whitelist(self):

        downloads_directory = os.path.join(get_this_directory(), "downloads")
        http_port = 8910
        http_port = self._start_webserver(downloads_directory, port=http_port)

        while self._webserver is None:
            print "Waiting for webserver to come up"
            gevent.sleep(1)

        assert self._webserver.requests == 0

        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = "ion.my.module"
        module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
        cls = 'TestDownloadProcess'
        parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        assert response.status == 401
        assert "Code download not enabled" in response.result

        # Test no whitelist
        self._enable_code_download()

        round += 1
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        print response
        assert response.status == 401
        assert "not in code_download whitelist" in response.result

        # Test not matching
        self._enable_code_download(whitelist=['blork'])

        round += 1
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        assert response.status == 401
        assert "not in code_download whitelist" in response.result

        # Test exact matching
        self._enable_code_download(whitelist=['localhost'])

        round += 1
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

        # Test wildcard
        self._enable_code_download(whitelist=['*'])

        round += 1
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_caching(self):

        downloads_directory = os.path.join(get_this_directory(), "downloads")
        http_port = 8910
        http_port = self._start_webserver(downloads_directory, port=http_port)

        while self._webserver is None:
            print "Waiting for webserver to come up"
            gevent.sleep(1)

        self._enable_code_download(['*'])
        assert self._webserver.requests == 0

        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = "ion.my.module"
        module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
        cls = 'TestDownloadProcess'
        parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}

        # Launch a process, check that webserver is hit
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, 'RUNNING'])
        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

        assert self._webserver.requests == 1

        # Launch another process, check that webserver is still only hit once
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

        assert self._webserver.requests == 1

        u_pid = "test5"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transformx'
        module = "ion.agents.cei.test.test_eeagent"
        module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}

        # Test that a module that is already available in tarball won't trigger a download
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, 'RUNNING'])
        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)

        assert self._webserver.requests == 1

        u_pid = "test9"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transformx'
        module = "ion.agents.cei.test.test_eeagent"
        module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
        cls = 'TestProcessNotReal'
        parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}

        # Test behaviour of a non existant class with no download
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [850, 'FAILED'])
        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        get_proc_for_upid(state, u_pid)
Exemplo n.º 10
0
class ExecutionEngineAgentPyonSingleIntTest(IonIntegrationTestCase):

    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_123456"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
                'heartbeat': 0,
                'slots': 100,
                'name': 'pyon_eeagent',
                'launch_type': {
                    'name': 'pyon_single',
                    'pyon_directory': os.getcwd(),
                    'supd_directory': self.supd_directory,
                    'supdexe': 'bin/supervisord'
                },
            },
            'agent': {'resource_id': self.resource_id},
            'logging': {
                'loggers': {
                    'eeagent': {
                        'level': 'DEBUG',
                        'handlers': ['console']
                    }
                },
                'root': {
                    'handlers': ['console']
                },
            }
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(
            node=self.container.node, name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.supd_directory)

    @needs_eeagent
    def test_basics(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon_single"
        proc_name = 'test_transform'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertEqual(proc.get('state'), [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)
        self.assertEqual(proc.get('state'), [700, 'TERMINATED'])
Exemplo n.º 11
0
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):

    _webserver = None

    @needs_eeagent
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_123456789"
        self._eea_name = "eeagent"

        self.persistence_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
              'heartbeat': 0,
              'slots': 100,
              'name': 'pyon_eeagent',
              'launch_type': {
                'name': 'pyon',
                'persistence_directory': self.persistence_directory,
              }
            },
            'agent': {'resource_id': self.resource_id},
            'logging': {
            'loggers': {
              'eeagent': {
                'level': 'DEBUG',
                'handlers': ['console']
              }
            },
            'root': {
              'handlers': ['console']
            },
          }
        }

        self._start_eeagent()

    def _start_eeagent(self):
        self.container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        self.container = self.container_client._get_container_instance()

        # Start eeagent.
        self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent", config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self._stop_webserver()
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.persistence_directory)

    def _start_webserver(self, directory_to_serve, port=None):
        """ Start a webserver for testing code download
        Note: tries really hard to get a port, and if it can't use
        the suggested port, randomly picks another, and returns it
        """
        def log_message(self, format, *args):
            #swallow log massages
            pass

        class Server(HTTPServer):

            requests = 0

            def serve_forever(self):
                self._serving = 1
                while self._serving:
                    self.handle_request()
                    self.requests += 1

            def stop(self):
                self._serving = 0

        if port is None:
            port = 8008
        self.old_cwd = os.getcwd()
        os.chdir(directory_to_serve)
        Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
        Handler.log_message = log_message

        for i in range(0, 100):
            try:
                self._webserver = Server(("localhost", port), Handler)
            except socket.error:
                print "port %s is in use, picking another" % port
                port = randint(8000, 10000)
                continue
            else:
                break

        self._web_glet = gevent.spawn(self._webserver.serve_forever)
        return port

    def _stop_webserver(self):
        if self._webserver is not None:
            self._web_glet.kill()
            os.chdir(self.old_cwd)

    def _enable_code_download(self, whitelist=None):

        if whitelist is None:
            whitelist = []

        self.container.terminate_process(self._eea_pid)
        self.agent_config['eeagent']['code_download'] = {
            'enabled': True,
            'whitelist': whitelist
        }
        self._start_eeagent()

    def wait_for_state(self, upid, desired_state, timeout=30):
        attempts = 0
        last_state = None
        while timeout > attempts:
            state = self.eea_client.dump_state().result
            proc = get_proc_for_upid(state, upid)
            last_state = proc.get('state')
            if last_state == desired_state:
                return
            gevent.sleep(1)
            attempts += 1

        assert False, "Process %s took too long to get to %s, had %s" % (upid, desired_state, last_state)

    @needs_eeagent
    def test_basics(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}

        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        self.wait_for_state(u_pid, [700, 'TERMINATED'])

    @needs_eeagent
    def test_failing_process(self):
        u_pid = "testfail"
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcessFail'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [850, 'FAILED'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_slow_to_start(self):
        upids = map(lambda i: str(uuid.uuid4().hex), range(0, 10))
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcessSlowStart'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        for upid in upids:
            self.eea_client.launch_process(upid, round, run_type, parameters)

        for upid in upids:
            self.wait_for_state(upid, [500, 'RUNNING'], timeout=60)

    @needs_eeagent
    def test_start_cancel(self):
        upid = str(uuid.uuid4().hex)
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcessSlowStart'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(upid, round, run_type, parameters)
        self.wait_for_state(upid, [400, 'PENDING'])
        self.eea_client.terminate_process(upid, round)
        self.wait_for_state(upid, [700, 'TERMINATED'])


    @needs_eeagent
    def test_kill_and_revive(self):
        """test_kill_and_revive
        Ensure that when an eeagent dies, it pulls the processes it owned from
        persistence, and marks them as failed, so the PD can figure out what to
        do with them
        """
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, 'RUNNING'])

        # Kill and restart eeagent. Also, kill proc started by eea to simulate
        # a killed container
        old_eea_pid = str(self._eea_pid)
        self.container.terminate_process(self._eea_pid)
        proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
        self.assertIsNotNone(proc_to_kill)
        self.container.terminate_process(proc_to_kill.id)

        self._start_eeagent()

        self.assertNotEqual(old_eea_pid, self._eea_pid)

        self.wait_for_state(u_pid, [850, 'FAILED'])

    @needs_eeagent
    def test_download_code(self):

        self._enable_code_download(whitelist=['*'])

        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = "ion.my.module.to.download"
        module_uri = 'file://%s/downloads/module_to_download.py' % get_this_directory()
        bad_module_uri = 'file:///tmp/notreal/module_to_download.py'

        cls = 'TestDownloadProcess'

        parameters = {'name': proc_name, 'module': module, 'module_uri': bad_module_uri, 'cls': cls}
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        print response
        assert response.status == 404
        assert "Unable to download" in response.result

        parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_whitelist(self):

        downloads_directory = os.path.join(get_this_directory(), "downloads")
        http_port = 8910
        http_port = self._start_webserver(downloads_directory, port=http_port)

        while self._webserver is None:
            print "Waiting for webserver to come up"
            gevent.sleep(1)

        assert self._webserver.requests == 0

        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = "ion.my.module"
        module_uri = "http://localhost:%s/module_to_download.py" % http_port
        cls = 'TestDownloadProcess'
        parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        assert response.status == 401
        assert "Code download not enabled" in response.result

        # Test no whitelist
        self._enable_code_download()

        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        print response
        assert response.status == 401
        assert "not in code_download whitelist" in response.result

        # Test not matching
        self._enable_code_download(whitelist=['blork'])

        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        assert response.status == 401
        assert "not in code_download whitelist" in response.result

        # Test exact matching
        self._enable_code_download(whitelist=['localhost'])

        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        # Test wildcard
        self._enable_code_download(whitelist=['*'])

        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_caching(self):

        downloads_directory = os.path.join(get_this_directory(), "downloads")
        http_port = 8910
        http_port = self._start_webserver(downloads_directory, port=http_port)

        while self._webserver is None:
            print "Waiting for webserver to come up"
            gevent.sleep(1)

        self._enable_code_download(['*'])
        assert self._webserver.requests == 0

        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = "ion.my.module"
        module_uri = "http://localhost:%s/module_to_download.py" % http_port
        cls = 'TestDownloadProcess'
        parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}

        # Launch a process, check that webserver is hit
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, 'RUNNING'])
        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        assert self._webserver.requests == 1


        # Launch another process, check that webserver is still only hit once
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)

        self.wait_for_state(u_pid, [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        assert self._webserver.requests == 1

        u_pid = "test5"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transformx'
        module = "ion.agents.cei.test.test_eeagent"
        module_uri = "http://localhost:%s/module_to_download.py" % http_port
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}

        # Test that a module that is already available in tarball won't trigger a download
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [500, 'RUNNING'])
        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        assert self._webserver.requests == 1

        u_pid = "test9"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transformx'
        module = "ion.agents.cei.test.test_eeagent"
        module_uri = "http://localhost:%s/module_to_download.py" % http_port
        cls = 'TestProcessNotReal'
        parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}

        # Test behaviour of a non existant class with no download
        response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
        self.wait_for_state(u_pid, [850, 'FAILED'])
        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)
Exemplo n.º 12
0
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
    from ion.agents.cei.execution_engine_agent import ExecutionEngineAgentClient

    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_123456789"
        self._eea_name = "eeagent"

        self.persistence_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
                'heartbeat': 0,
                'slots': 100,
                'name': 'pyon_eeagent',
                'launch_type': {
                    'name': 'pyon',
                    'persistence_directory': self.persistence_directory,
                },
            },
            'agent': {
                'resource_id': self.resource_id
            },
            'logging': {
                'loggers': {
                    'eeagent': {
                        'level': 'DEBUG',
                        'handlers': ['console']
                    }
                },
                'root': {
                    'handlers': ['console']
                },
            }
        }

        self._start_eeagent()

    def _start_eeagent(self):
        self.container_client = ContainerAgentClient(node=self.container.node,
                                                     name=self.container.name)
        self.container = self.container_client._get_container_instance()

        # Start eeagent.
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent",
            config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = ResourceAgentClient(self.resource_id,
                                                    process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)

    def tearDown(self):
        self.container.terminate_process(self._eea_pid)
        shutil.rmtree(self.persistence_directory)

    @needs_eeagent
    def test_basics(self):
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_x'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
        self.assertEqual(proc.get('state'), [500, 'RUNNING'])

        self.eea_client.terminate_process(u_pid, round)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

    @needs_eeagent
    def test_kill_and_revive(self):
        """test_kill_and_revive
        Ensure that when an eeagent dies, it pulls the processes it owned from
        persistence, and marks them as failed, so the PD can figure out what to
        do with them
        """
        u_pid = "test0"
        round = 0
        run_type = "pyon"
        proc_name = 'test_transform'
        module = 'ion.agents.cei.test.test_eeagent'
        cls = 'TestProcess'
        parameters = {'name': proc_name, 'module': module, 'cls': cls}
        self.eea_client.launch_process(u_pid, round, run_type, parameters)
        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
        self.assertEqual(proc.get('state'), [500, 'RUNNING'])

        # Kill and restart eeagent. Also, kill proc started by eea to simulate
        # a killed container
        old_eea_pid = str(self._eea_pid)
        self.container.terminate_process(self._eea_pid)
        proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
        self.assertIsNotNone(proc_to_kill)
        self.container.terminate_process(proc_to_kill.id)

        self._start_eeagent()

        self.assertNotEqual(old_eea_pid, self._eea_pid)

        state = self.eea_client.dump_state().result
        proc = get_proc_for_upid(state, u_pid)

        self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
        self.assertEqual(proc.get('state'), [850, 'FAILED'])