def heartbeat_callback(heartbeat, headers):
eeagent_id = heartbeat['eeagent_id']
agent_client = SimpleResourceAgentClient(eeagent_id, name=eeagent_id, process=FakeProcess())
ee_client = ExecutionEngineAgentClient(agent_client, timeout=2)
try:
ee_client.dump_state()
except:
log.exception("Heartbeat Failed!")
beat_died[0] = True
def heartbeat_callback(heartbeat, headers):
eeagent_id = heartbeat["eeagent_id"]
agent_client = SimpleResourceAgentClient(eeagent_id, name=eeagent_id, process=FakeProcess())
ee_client = ExecutionEngineAgentClient(agent_client, timeout=2)
try:
ee_client.dump_state()
except:
log.exception("Heartbeat Failed!")
beat_died[0] = True
class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2cei.yml")
self.resource_id = "eeagent_1234"
self._eea_name = "eeagent"
self.supd_directory = tempfile.mkdtemp()
self.agent_config = {
"eeagent": {
"heartbeat": 0,
"slots": 100,
"name": "pyon_eeagent",
"launch_type": {
"name": "supd",
"pyon_directory": os.getcwd(),
"supd_directory": self.supd_directory,
"supdexe": "bin/supervisord",
},
},
"agent": {"resource_id": self.resource_id},
"logging": {
"loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
"root": {"handlers": ["console"]},
},
}
# Start eeagent.
self._eea_pid = None
self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=self.agent_config,
)
log.info("Agent pid=%s.", str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info("Got eea client %s.", str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.supd_directory)
def wait_for_state(self, upid, desired_state, timeout=5):
attempts = 0
while timeout > attempts:
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, upid)
if proc.get("state") == desired_state:
return
gevent.sleep(1)
attempts += 1
assert False, "Process %s took too long to get to %s" % (upid, desired_state)
@needs_eeagent
def test_basics(self):
true_u_pid = "test0"
round = 0
run_type = "supd"
true_parameters = {"exec": "true", "argv": []}
self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)
self.wait_for_state(true_u_pid, [800, "EXITED"])
cat_u_pid = "test1"
round = 0
run_type = "supd"
cat_parameters = {"exec": "cat", "argv": []}
self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
self.wait_for_state(cat_u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(cat_u_pid, round)
self.wait_for_state(cat_u_pid, [700, "TERMINATED"])
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
_webserver = None
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2cei.yml")
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
"eeagent": {
"heartbeat": 1,
"slots": 100,
"name": "pyon_eeagent",
"launch_type": {"name": "pyon", "persistence_directory": self.persistence_directory},
},
"agent": {"resource_id": self.resource_id},
"logging": {
"loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
"root": {"handlers": ["console"]},
},
}
self._start_eeagent()
def _start_eeagent(self):
self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
self.container = self.container_client._get_container_instance()
# Start eeagent.
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=self.agent_config,
)
log.info("Agent pid=%s.", str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info("Got eea client %s.", str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self._stop_webserver()
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
def _start_webserver(self, directory_to_serve, port=None):
""" Start a webserver for testing code download
Note: tries really hard to get a port, and if it can't use
the suggested port, randomly picks another, and returns it
"""
def log_message(self, format, *args):
# swallow log massages
pass
class Server(HTTPServer):
requests = 0
def serve_forever(self):
self._serving = 1
while self._serving:
self.handle_request()
self.requests += 1
def stop(self):
self._serving = 0
if port is None:
port = 8008
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
Handler.log_message = log_message
for i in range(0, 100):
try:
self._webserver = Server(("localhost", port), Handler)
except socket.error:
print "port %s is in use, picking another" % port
port = randint(8000, 10000)
continue
else:
break
self._web_glet = gevent.spawn(self._webserver.serve_forever)
return port
def _stop_webserver(self):
if self._webserver is not None:
self._web_glet.kill()
def _enable_code_download(self, whitelist=None):
if whitelist is None:
whitelist = []
self.container.terminate_process(self._eea_pid)
self.agent_config["eeagent"]["code_download"] = {"enabled": True, "whitelist": whitelist}
self._start_eeagent()
def wait_for_state(self, upid, desired_state, timeout=30):
attempts = 0
last_state = None
while timeout > attempts:
try:
state = self.eea_client.dump_state().result
except Timeout:
log.warn("Timeout calling EEAgent dump_state. retrying.")
continue
proc = get_proc_for_upid(state, upid)
last_state = proc.get("state")
if last_state == desired_state:
return
gevent.sleep(1)
attempts += 1
assert False, "Process %s took too long to get to %s, had %s" % (upid, desired_state, last_state)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_x"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 1
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, "TERMINATED"])
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 1
self.eea_client.cleanup_process(u_pid, round)
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 0
@needs_eeagent
def test_restart(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_x"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 1
# Start again with incremented round. eeagent should restart the process
round += 1
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
state = self.eea_client.dump_state().result
ee_round = state["processes"][0]["round"]
assert round == int(ee_round)
# TODO: this test is disabled, as the restart op is disabled
# Run restart with incremented round. eeagent should restart the process
# round += 1
# self.eea_client.restart_process(u_pid, round)
# self.wait_for_state(u_pid, [500, 'RUNNING'])
# state = self.eea_client.dump_state().result
# ee_round = state['processes'][0]['round']
# assert round == int(ee_round)
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, "TERMINATED"])
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 1
self.eea_client.cleanup_process(u_pid, round)
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 0
@needs_eeagent
def test_failing_process(self):
u_pid = "testfail"
round = 0
run_type = "pyon"
proc_name = "test_x"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcessFail"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [850, "FAILED"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_slow_to_start(self):
upids = map(lambda i: str(uuid.uuid4().hex), range(0, 10))
round = 0
run_type = "pyon"
proc_name = "test_x"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcessSlowStart"
parameters = {"name": proc_name, "module": module, "cls": cls}
for upid in upids:
self.eea_client.launch_process(upid, round, run_type, parameters)
for upid in upids:
self.wait_for_state(upid, [500, "RUNNING"], timeout=60)
@needs_eeagent
def test_start_cancel(self):
upid = str(uuid.uuid4().hex)
round = 0
run_type = "pyon"
proc_name = "test_x"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcessSlowStart"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(upid, round, run_type, parameters)
self.wait_for_state(upid, [400, "PENDING"])
self.eea_client.terminate_process(upid, round)
self.wait_for_state(upid, [700, "TERMINATED"])
@needs_eeagent
def test_kill_and_revive(self):
"""test_kill_and_revive
Ensure that when an eeagent dies, it pulls the processes it owned from
persistence, and marks them as failed, so the PD can figure out what to
do with them
"""
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_transform"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
# Kill and restart eeagent. Also, kill proc started by eea to simulate
# a killed container
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
self.assertIsNotNone(proc_to_kill)
self.container.terminate_process(proc_to_kill.id)
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
self.wait_for_state(u_pid, [850, "FAILED"])
@needs_eeagent
def test_run_out_of_slots(self):
"""test_run_out_of_slots
"""
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
self.agent_config["eeagent"]["slots"] = 1
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
u_pid_0, u_pid_1 = "test0", "test1"
round = 0
run_type = "pyon"
proc_name = "test_transform"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid_0, round, run_type, parameters)
self.wait_for_state(u_pid_0, [500, "RUNNING"])
self.eea_client.launch_process(u_pid_1, round, run_type, parameters)
self.wait_for_state(u_pid_1, [900, "REJECTED"])
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
self.agent_config["eeagent"]["slots"] = 1
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
self.wait_for_state(u_pid_0, [850, "FAILED"])
self.wait_for_state(u_pid_1, [900, "REJECTED"])
@needs_eeagent
def test_download_code(self):
self._enable_code_download(whitelist=["*"])
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_transform"
module = "ion.my.module.to.download"
module_uri = "file://%s/downloads/module_to_download.py" % get_this_directory()
bad_module_uri = "file:///tmp/notreal/module_to_download.py"
cls = "TestDownloadProcess"
parameters = {"name": proc_name, "module": module, "module_uri": bad_module_uri, "cls": cls}
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
print response
assert response.status == 404
assert "Unable to download" in response.result
parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_whitelist(self):
downloads_directory = os.path.join(get_this_directory(), "downloads")
http_port = 8910
http_port = self._start_webserver(downloads_directory, port=http_port)
while self._webserver is None:
print "Waiting for webserver to come up"
gevent.sleep(1)
assert self._webserver.requests == 0
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_transform"
module = "ion.my.module"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = "TestDownloadProcess"
parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
assert response.status == 401
assert "Code download not enabled" in response.result
# Test no whitelist
self._enable_code_download()
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
print response
assert response.status == 401
assert "not in code_download whitelist" in response.result
# Test not matching
self._enable_code_download(whitelist=["blork"])
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
assert response.status == 401
assert "not in code_download whitelist" in response.result
# Test exact matching
self._enable_code_download(whitelist=["localhost"])
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
# Test wildcard
self._enable_code_download(whitelist=["*"])
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_caching(self):
downloads_directory = os.path.join(get_this_directory(), "downloads")
http_port = 8910
http_port = self._start_webserver(downloads_directory, port=http_port)
while self._webserver is None:
print "Waiting for webserver to come up"
gevent.sleep(1)
self._enable_code_download(["*"])
assert self._webserver.requests == 0
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_transform"
module = "ion.my.module"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = "TestDownloadProcess"
parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
# Launch a process, check that webserver is hit
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
# Launch another process, check that webserver is still only hit once
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
u_pid = "test5"
round = 0
run_type = "pyon"
proc_name = "test_transformx"
module = "ion.agents.cei.test.test_eeagent"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
# Test that a module that is already available in tarball won't trigger a download
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
u_pid = "test9"
round = 0
run_type = "pyon"
proc_name = "test_transformx"
module = "ion.agents.cei.test.test_eeagent"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = "TestProcessNotReal"
parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
# Test behaviour of a non existant class with no download
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [850, "FAILED"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
class ExecutionEngineAgentPyonSingleIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2cei.yml")
self.resource_id = "eeagent_123456"
self._eea_name = "eeagent"
self.supd_directory = tempfile.mkdtemp()
self.agent_config = {
"eeagent": {
"heartbeat": 0,
"slots": 100,
"name": "pyon_eeagent",
"launch_type": {
"name": "pyon_single",
"pyon_directory": os.getcwd(),
"supd_directory": self.supd_directory,
"supdexe": "bin/supervisord",
},
},
"agent": {"resource_id": self.resource_id},
"logging": {
"loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
"root": {"handlers": ["console"]},
},
}
# Start eeagent.
self._eea_pid = None
self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=self.agent_config,
)
log.info("Agent pid=%s.", str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info("Got eea client %s.", str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.supd_directory)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon_single"
proc_name = "test_transform"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertEqual(proc.get("state"), [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertEqual(proc.get("state"), [700, "TERMINATED"])
class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_1234"
self._eea_name = "eeagent"
self.supd_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'supd',
'pyon_directory': os.getcwd(),
'supd_directory': self.supd_directory,
'supdexe': 'bin/supervisord'
},
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
# Start eeagent.
self._eea_pid = None
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = ResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.supd_directory)
@needs_eeagent
def test_basics(self):
true_u_pid = "test0"
round = 0
run_type = "supd"
true_parameters = {'exec': 'true', 'argv': []}
self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, true_u_pid)
self.assertEqual(proc.get('state'), [800, 'EXITED'])
cat_u_pid = "test1"
round = 0
run_type = "supd"
cat_parameters = {'exec': 'cat', 'argv': []}
self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, cat_u_pid)
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
self.eea_client.terminate_process(cat_u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, cat_u_pid)
self.assertEqual(proc.get('state'), [700, 'TERMINATED'])
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
from ion.agents.cei.execution_engine_agent import ExecutionEngineAgentClient
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
},
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
self._start_eeagent()
def _start_eeagent(self):
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
# Start eeagent.
self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = ResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_kill_and_revive(self):
"""test_kill_and_revive
Ensure that when an eeagent dies, it pulls the processes it owned from
persistence, and marks them as failed, so the PD can figure out what to
do with them
"""
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
# Kill and restart eeagent. Also, kill proc started by eea to simulate
# a killed container
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
self.assertIsNotNone(proc_to_kill)
self.container.terminate_process(proc_to_kill.id)
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [850, 'FAILED'])
class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_1234"
self._eea_name = "eeagent"
self.supd_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'supd',
'pyon_directory': os.getcwd(),
'supd_directory': self.supd_directory,
'supdexe': 'bin/supervisord'
},
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
}
}
}
# Start eeagent.
self._eea_pid = None
self.container_client = ContainerAgentClient(
node=self.container.node, name=self.container.name)
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.supd_directory)
def wait_for_state(self, upid, desired_state, timeout=5):
attempts = 0
while timeout > attempts:
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, upid)
if proc.get('state') == desired_state:
return
gevent.sleep(1)
attempts += 1
assert False, "Process %s took too long to get to %s" % (upid, desired_state)
@needs_eeagent
def test_basics(self):
true_u_pid = "test0"
round = 0
run_type = "supd"
true_parameters = {'exec': 'true', 'argv': []}
self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)
self.wait_for_state(true_u_pid, [800, 'EXITED'])
cat_u_pid = "test1"
round = 0
run_type = "supd"
cat_parameters = {'exec': 'cat', 'argv': []}
self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
self.wait_for_state(cat_u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(cat_u_pid, round)
self.wait_for_state(cat_u_pid, [700, 'TERMINATED'])
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
_webserver = None
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 1,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
}
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
self._start_eeagent()
def _start_eeagent(self):
self.container_client = ContainerAgentClient(
node=self.container.node, name=self.container.name)
self.container = self.container_client._get_container_instance()
# Start eeagent.
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self._stop_webserver()
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
def _start_webserver(self, directory_to_serve, port=None):
""" Start a webserver for testing code download
Note: tries really hard to get a port, and if it can't use
the suggested port, randomly picks another, and returns it
"""
def log_message(self, format, *args):
#swallow log massages
pass
class Server(HTTPServer):
requests = 0
def serve_forever(self):
self._serving = 1
while self._serving:
self.handle_request()
self.requests += 1
def stop(self):
self._serving = 0
if port is None:
port = 8008
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
Handler.log_message = log_message
for i in range(0, 100):
try:
self._webserver = Server(("localhost", port), Handler)
except socket.error:
print "port %s is in use, picking another" % port
port = randint(8000, 10000)
continue
else:
break
self._web_glet = gevent.spawn(self._webserver.serve_forever)
return port
def _stop_webserver(self):
if self._webserver is not None:
self._web_glet.kill()
def _enable_code_download(self, whitelist=None):
if whitelist is None:
whitelist = []
self.container.terminate_process(self._eea_pid)
self.agent_config['eeagent']['code_download'] = {
'enabled': True,
'whitelist': whitelist
}
self._start_eeagent()
def wait_for_state(self, upid, desired_state, timeout=30):
attempts = 0
last_state = None
while timeout > attempts:
try:
state = self.eea_client.dump_state().result
except Timeout:
log.warn("Timeout calling EEAgent dump_state. retrying.")
continue
proc = get_proc_for_upid(state, upid)
last_state = proc.get('state')
if last_state == desired_state:
return
gevent.sleep(1)
attempts += 1
assert False, "Process %s took too long to get to %s, had %s" % (upid, desired_state, last_state)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, 'TERMINATED'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
self.eea_client.cleanup_process(u_pid, round)
state = self.eea_client.dump_state().result
assert len(state['processes']) == 0
@needs_eeagent
def test_duplicate(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, 'TERMINATED'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
self.eea_client.cleanup_process(u_pid, round)
state = self.eea_client.dump_state().result
assert len(state['processes']) == 0
@needs_eeagent
def test_restart(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
# Start again with incremented round. eeagent should restart the process
round += 1
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
state = self.eea_client.dump_state().result
ee_round = state['processes'][0]['round']
assert round == int(ee_round)
# TODO: this test is disabled, as the restart op is disabled
# Run restart with incremented round. eeagent should restart the process
#round += 1
#self.eea_client.restart_process(u_pid, round)
#self.wait_for_state(u_pid, [500, 'RUNNING'])
#state = self.eea_client.dump_state().result
#ee_round = state['processes'][0]['round']
#assert round == int(ee_round)
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, 'TERMINATED'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
self.eea_client.cleanup_process(u_pid, round)
state = self.eea_client.dump_state().result
assert len(state['processes']) == 0
@needs_eeagent
def test_failing_process(self):
u_pid = "testfail"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcessFail'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [850, 'FAILED'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_slow_to_start(self):
upids = map(lambda i: str(uuid.uuid4().hex), range(0, 10))
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcessSlowStart'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
for upid in upids:
self.eea_client.launch_process(upid, round, run_type, parameters)
for upid in upids:
self.wait_for_state(upid, [500, 'RUNNING'], timeout=60)
@needs_eeagent
def test_start_cancel(self):
upid = str(uuid.uuid4().hex)
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcessSlowStart'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(upid, round, run_type, parameters)
self.wait_for_state(upid, [400, 'PENDING'])
self.eea_client.terminate_process(upid, round)
self.wait_for_state(upid, [700, 'TERMINATED'])
@needs_eeagent
def test_kill_and_revive(self):
"""test_kill_and_revive
Ensure that when an eeagent dies, it pulls the processes it owned from
persistence, and marks them as failed, so the PD can figure out what to
do with them
"""
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
# Kill and restart eeagent. Also, kill proc started by eea to simulate
# a killed container
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
self.assertIsNotNone(proc_to_kill)
self.container.terminate_process(proc_to_kill.id)
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
self.wait_for_state(u_pid, [850, 'FAILED'])
@needs_eeagent
def test_run_out_of_slots(self):
"""test_run_out_of_slots
"""
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
self.agent_config['eeagent']['slots'] = 1
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
u_pid_0, u_pid_1 = "test0", "test1"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid_0, round, run_type, parameters)
self.wait_for_state(u_pid_0, [500, 'RUNNING'])
self.eea_client.launch_process(u_pid_1, round, run_type, parameters)
self.wait_for_state(u_pid_1, [900, 'REJECTED'])
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
self.agent_config['eeagent']['slots'] = 1
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
self.wait_for_state(u_pid_0, [850, 'FAILED'])
self.wait_for_state(u_pid_1, [900, 'REJECTED'])
@needs_eeagent
def test_download_code(self):
self._enable_code_download(whitelist=['*'])
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = "ion.my.module.to.download"
module_uri = 'file://%s/downloads/module_to_download.py' % get_this_directory()
bad_module_uri = 'file:///tmp/notreal/module_to_download.py'
cls = 'TestDownloadProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': bad_module_uri, 'cls': cls}
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
print response
assert response.status == 404
assert "Unable to download" in response.result
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
round += 1
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_whitelist(self):
downloads_directory = os.path.join(get_this_directory(), "downloads")
http_port = 8910
http_port = self._start_webserver(downloads_directory, port=http_port)
while self._webserver is None:
print "Waiting for webserver to come up"
gevent.sleep(1)
assert self._webserver.requests == 0
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = "ion.my.module"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = 'TestDownloadProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
assert response.status == 401
assert "Code download not enabled" in response.result
# Test no whitelist
self._enable_code_download()
round += 1
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
print response
assert response.status == 401
assert "not in code_download whitelist" in response.result
# Test not matching
self._enable_code_download(whitelist=['blork'])
round += 1
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
assert response.status == 401
assert "not in code_download whitelist" in response.result
# Test exact matching
self._enable_code_download(whitelist=['localhost'])
round += 1
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
# Test wildcard
self._enable_code_download(whitelist=['*'])
round += 1
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_caching(self):
downloads_directory = os.path.join(get_this_directory(), "downloads")
http_port = 8910
http_port = self._start_webserver(downloads_directory, port=http_port)
while self._webserver is None:
print "Waiting for webserver to come up"
gevent.sleep(1)
self._enable_code_download(['*'])
assert self._webserver.requests == 0
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = "ion.my.module"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = 'TestDownloadProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
# Launch a process, check that webserver is hit
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
# Launch another process, check that webserver is still only hit once
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
u_pid = "test5"
round = 0
run_type = "pyon"
proc_name = 'test_transformx'
module = "ion.agents.cei.test.test_eeagent"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
# Test that a module that is already available in tarball won't trigger a download
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
u_pid = "test9"
round = 0
run_type = "pyon"
proc_name = 'test_transformx'
module = "ion.agents.cei.test.test_eeagent"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = 'TestProcessNotReal'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
# Test behaviour of a non existant class with no download
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [850, 'FAILED'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
class ExecutionEngineAgentPyonSingleIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_123456"
self._eea_name = "eeagent"
self.supd_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon_single',
'pyon_directory': os.getcwd(),
'supd_directory': self.supd_directory,
'supdexe': 'bin/supervisord'
},
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
# Start eeagent.
self._eea_pid = None
self.container_client = ContainerAgentClient(
node=self.container.node, name=self.container.name)
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.supd_directory)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon_single"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertEqual(proc.get('state'), [700, 'TERMINATED'])
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
_webserver = None
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
}
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
self._start_eeagent()
def _start_eeagent(self):
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
# Start eeagent.
self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self._stop_webserver()
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
def _start_webserver(self, directory_to_serve, port=None):
""" Start a webserver for testing code download
Note: tries really hard to get a port, and if it can't use
the suggested port, randomly picks another, and returns it
"""
def log_message(self, format, *args):
#swallow log massages
pass
class Server(HTTPServer):
requests = 0
def serve_forever(self):
self._serving = 1
while self._serving:
self.handle_request()
self.requests += 1
def stop(self):
self._serving = 0
if port is None:
port = 8008
self.old_cwd = os.getcwd()
os.chdir(directory_to_serve)
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
Handler.log_message = log_message
for i in range(0, 100):
try:
self._webserver = Server(("localhost", port), Handler)
except socket.error:
print "port %s is in use, picking another" % port
port = randint(8000, 10000)
continue
else:
break
self._web_glet = gevent.spawn(self._webserver.serve_forever)
return port
def _stop_webserver(self):
if self._webserver is not None:
self._web_glet.kill()
os.chdir(self.old_cwd)
def _enable_code_download(self, whitelist=None):
if whitelist is None:
whitelist = []
self.container.terminate_process(self._eea_pid)
self.agent_config['eeagent']['code_download'] = {
'enabled': True,
'whitelist': whitelist
}
self._start_eeagent()
def wait_for_state(self, upid, desired_state, timeout=30):
attempts = 0
last_state = None
while timeout > attempts:
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, upid)
last_state = proc.get('state')
if last_state == desired_state:
return
gevent.sleep(1)
attempts += 1
assert False, "Process %s took too long to get to %s, had %s" % (upid, desired_state, last_state)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, 'TERMINATED'])
@needs_eeagent
def test_failing_process(self):
u_pid = "testfail"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcessFail'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [850, 'FAILED'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_slow_to_start(self):
upids = map(lambda i: str(uuid.uuid4().hex), range(0, 10))
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcessSlowStart'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
for upid in upids:
self.eea_client.launch_process(upid, round, run_type, parameters)
for upid in upids:
self.wait_for_state(upid, [500, 'RUNNING'], timeout=60)
@needs_eeagent
def test_start_cancel(self):
upid = str(uuid.uuid4().hex)
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcessSlowStart'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(upid, round, run_type, parameters)
self.wait_for_state(upid, [400, 'PENDING'])
self.eea_client.terminate_process(upid, round)
self.wait_for_state(upid, [700, 'TERMINATED'])
@needs_eeagent
def test_kill_and_revive(self):
"""test_kill_and_revive
Ensure that when an eeagent dies, it pulls the processes it owned from
persistence, and marks them as failed, so the PD can figure out what to
do with them
"""
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
# Kill and restart eeagent. Also, kill proc started by eea to simulate
# a killed container
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
self.assertIsNotNone(proc_to_kill)
self.container.terminate_process(proc_to_kill.id)
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
self.wait_for_state(u_pid, [850, 'FAILED'])
@needs_eeagent
def test_download_code(self):
self._enable_code_download(whitelist=['*'])
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = "ion.my.module.to.download"
module_uri = 'file://%s/downloads/module_to_download.py' % get_this_directory()
bad_module_uri = 'file:///tmp/notreal/module_to_download.py'
cls = 'TestDownloadProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': bad_module_uri, 'cls': cls}
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
print response
assert response.status == 404
assert "Unable to download" in response.result
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_whitelist(self):
downloads_directory = os.path.join(get_this_directory(), "downloads")
http_port = 8910
http_port = self._start_webserver(downloads_directory, port=http_port)
while self._webserver is None:
print "Waiting for webserver to come up"
gevent.sleep(1)
assert self._webserver.requests == 0
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = "ion.my.module"
module_uri = "http://localhost:%s/module_to_download.py" % http_port
cls = 'TestDownloadProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
assert response.status == 401
assert "Code download not enabled" in response.result
# Test no whitelist
self._enable_code_download()
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
print response
assert response.status == 401
assert "not in code_download whitelist" in response.result
# Test not matching
self._enable_code_download(whitelist=['blork'])
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
assert response.status == 401
assert "not in code_download whitelist" in response.result
# Test exact matching
self._enable_code_download(whitelist=['localhost'])
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
# Test wildcard
self._enable_code_download(whitelist=['*'])
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_caching(self):
downloads_directory = os.path.join(get_this_directory(), "downloads")
http_port = 8910
http_port = self._start_webserver(downloads_directory, port=http_port)
while self._webserver is None:
print "Waiting for webserver to come up"
gevent.sleep(1)
self._enable_code_download(['*'])
assert self._webserver.requests == 0
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = "ion.my.module"
module_uri = "http://localhost:%s/module_to_download.py" % http_port
cls = 'TestDownloadProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
# Launch a process, check that webserver is hit
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
# Launch another process, check that webserver is still only hit once
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
u_pid = "test5"
round = 0
run_type = "pyon"
proc_name = 'test_transformx'
module = "ion.agents.cei.test.test_eeagent"
module_uri = "http://localhost:%s/module_to_download.py" % http_port
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
# Test that a module that is already available in tarball won't trigger a download
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
u_pid = "test9"
round = 0
run_type = "pyon"
proc_name = 'test_transformx'
module = "ion.agents.cei.test.test_eeagent"
module_uri = "http://localhost:%s/module_to_download.py" % http_port
cls = 'TestProcessNotReal'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
# Test behaviour of a non existant class with no download
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [850, 'FAILED'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
from ion.agents.cei.execution_engine_agent import ExecutionEngineAgentClient
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
},
},
'agent': {
'resource_id': self.resource_id
},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
self._start_eeagent()
def _start_eeagent(self):
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
# Start eeagent.
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = ResourceAgentClient(self.resource_id,
process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_kill_and_revive(self):
"""test_kill_and_revive
Ensure that when an eeagent dies, it pulls the processes it owned from
persistence, and marks them as failed, so the PD can figure out what to
do with them
"""
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
# Kill and restart eeagent. Also, kill proc started by eea to simulate
# a killed container
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
self.assertIsNotNone(proc_to_kill)
self.container.terminate_process(proc_to_kill.id)
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [850, 'FAILED'])