def start_instrument_agent_process(container, stream_config={}, resource_id=IA_RESOURCE_ID, resource_name=IA_NAME, org_name=None, message_headers=None):
log.info("foobar")
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : stream_config,
'agent' : {'resource_id': resource_id},
'test_mode' : True
}
if org_name is not None:
agent_config['org_name'] = org_name
# Start instrument agent.
log.info("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=container.node,
name=container.name)
log.info("Agent setup")
ia_pid = container_client.spawn_process(name=resource_name,
module=IA_MOD,
cls=IA_CLS,
config=agent_config, headers=message_headers)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
ia_client = ResourceAgentClient(resource_id, process=FakeProcess())
log.info('Got ia client %s.', str(ia_client))
return ia_client
def start_fake_instrument_agent(container, stream_config={}, message_headers=None):
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=container.node,
name=container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config, headers=message_headers)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(ia_client))
return ia_client
def setUp(self):
"""
Initialize test members.
Start port agent.
Start container and client.
Start streams and subscribers.
Start agent, client.
"""
TrhphTestCase.setUp(self)
# Start port agent, add stop to cleanup.
self._pagent = None
self._start_pagent()
self.addCleanup(self._stop_pagent)
# Start container.
self._start_container()
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url("res/deploy/r2dm.yml")
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._no_samples = None
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# Start event subscribers, add stop to cleanup.
self._no_events = None
self._async_event_result = AsyncResult()
self._events_received = []
self._event_subscribers = []
self._start_event_subscribers()
self.addCleanup(self._stop_event_subscribers)
# Create agent config.
agent_config = {
"driver_config": DVR_CONFIG,
"stream_config": self._stream_config,
"agent": {"resource_id": IA_RESOURCE_ID},
"test_mode": True,
}
# Start instrument agent.
self._ia_pid = None
log.debug("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config)
log.info("Agent pid=%s.", str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info("Got ia client %s.", str(self._ia_client))
def start_client(self, name, module, cls, config, resource_id, deploy_file = DEFAULT_DEPLOY, message_headers=None, bootmode=None):
"""
@brief Start up the instrument agent client
"""
self.start_container(deploy_file=deploy_file)
# Start instrument agent.
log.debug("Starting Agent Client")
container_client = ContainerAgentClient(node=self.container.node,
to_name=self.container.name)
agent_config = deepcopy(config)
agent_config['bootmode'] = bootmode
log.debug("Agent config: %s", agent_config)
instrument_agent_pid = container_client.spawn_process(
name=name,
module=module,
cls=cls,
config=agent_config,
headers=message_headers)
log.info('Agent pid=%s.', instrument_agent_pid)
ia_client = ResourceAgentClient(resource_id, process=FakeProcess())
log.info('Got ia client %s.', str(ia_client))
self.instrument_agent_client = ia_client
def start_client(self, name, module, cls, config, resource_id, deploy_file = DEFAULT_DEPLOY, bootmode=None):
"""
@brief Start up the instrument agent client
"""
self.start_container(deploy_file=deploy_file)
# Start instrument agent.
log.debug("Starting Agent Client")
container_client = ContainerAgentClient(node=self.container.node,
to_name=self.container.name)
agent_config = deepcopy(config)
agent_config['bootmode'] = bootmode
log.debug("Agent pid: %s", self.instrument_agent_pid)
log.debug("Bootmode: %s", bootmode)
log.debug("Agent config: %s", agent_config)
self.instrument_agent_pid = container_client.spawn_process(
name=name,
module=module,
cls=cls,
config=agent_config,
process_id=str(self.instrument_agent_pid))
log.info('Agent pid=%s.', self.instrument_agent_pid)
ia_client = ResourceAgentClient(resource_id, process=FakeProcess())
log.info('Got ia client %s.', str(ia_client))
self.instrument_agent_client = ia_client
def test_start_hello(self):
# start a service over messaging
self._start_container()
cc_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
p = cc_client.spawn_process('hello', 'examples.service.hello_service',
'HelloService')
def start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _action_spawn_process(self, action_kwargs):
cc_agent_name = action_kwargs["cc_agent"]
proc_name = action_kwargs["proc_name"]
module = action_kwargs["module"]
cls = action_kwargs["cls"]
config = action_kwargs["config"]
target_cc_agent = ContainerAgentClient(to_name=cc_agent_name)
proc_id = target_cc_agent.spawn_process(proc_name, module, cls, config)
return proc_id
def _action_spawn_process(self, action_kwargs):
cc_agent_name = action_kwargs["cc_agent"]
proc_name = action_kwargs["proc_name"]
module = action_kwargs["module"]
cls = action_kwargs["cls"]
config = action_kwargs["config"]
target_cc_agent = ContainerAgentClient(to_name=cc_agent_name)
proc_id = target_cc_agent.spawn_process(proc_name, module, cls, config)
return proc_id
def _start_agent(self):
"""
"""
container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
self._ia_pid = container_client.spawn_process(
name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=self._agent_config
)
log.info("Started instrument agent pid=%s.", str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info("Got instrument agent client %s.", str(self._ia_client))
def _start_agent(self):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(
name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=self._agent_config)
log.info('Started instrument agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _start_agent(self, bootmode=None):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = bootmode
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config,
process_id=self._ia_pid)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _start_agent(self, bootmode=None):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = bootmode
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config,
process_id=self._ia_pid)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _start_agent(self):
"""
Start an instrument agent and client.
"""
log.info("Creating driver integration test support:")
log.info("driver module: %s", DRV_MOD)
log.info("driver class: %s", DRV_CLS)
log.info("device address: %s", DEV_ADDR)
log.info("device port: %s", DEV_PORT)
log.info("log delimiter: %s", DELIM)
log.info("work dir: %s", WORK_DIR)
self._support = DriverIntegrationTestSupport(
DRV_MOD, DRV_CLS, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR
)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info("Port agent started at port %i", port)
# Configure driver to use port agent port number.
DVR_CONFIG["comms_config"] = {"addr": "localhost", "port": port}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
"driver_config": DVR_CONFIG,
"stream_config": {},
"agent": {"resource_id": IA_RESOURCE_ID},
"test_mode": True,
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config)
log.info("Agent pid=%s.", str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info("Got ia client %s.", str(self._ia_client))
def start_client(self, name, module, cls, config, resource_id, deploy_file = DEFAULT_DEPLOY):
"""
@brief Start up the instrument agent client
"""
self.start_container(deploy_file=deploy_file)
# Start instrument agent.
log.debug("Starting Instrument Agent Client")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
instrument_agent_pid = container_client.spawn_process(
name=name,
module=module,
cls=cls,
config=config)
log.info('Agent pid=%s.', instrument_agent_pid)
ia_client = ResourceAgentClient(resource_id, process=FakeProcess())
log.info('Got ia client %s.', str(ia_client))
self.instrument_agent_client = ia_client
class HeartbeaterIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
"eeagent": {
"heartbeat": "0.01",
"slots": 100,
"name": "pyon_eeagent",
"launch_type": {"name": "pyon", "persistence_directory": self.persistence_directory},
},
"agent": {"resource_id": self.resource_id},
"logging": {
"loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
"root": {"handlers": ["console"]},
},
}
def _start_eeagent(self):
self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
self.container = self.container_client._get_container_instance()
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=self.agent_config,
)
log.info("Agent pid=%s.", str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info("Got eea client %s.", str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
@needs_eeagent
@unittest.skipIf(os.getenv("CEI_LAUNCH_TEST", False), "Skip test while in CEI LAUNCH mode")
def test_heartbeater(self):
"""test_heartbeater
Test whether the eeagent waits until the eeagent listener is ready before sending
a heartbeat to the PD
"""
# beat_died is a list because of a hack to get around a limitation in python 2.7
# See: http://stackoverflow.com/questions/8934772/local-var-referenced-before-assignment
beat_died = [False]
def heartbeat_callback(heartbeat, headers):
eeagent_id = heartbeat["eeagent_id"]
agent_client = SimpleResourceAgentClient(eeagent_id, name=eeagent_id, process=FakeProcess())
ee_client = ExecutionEngineAgentClient(agent_client, timeout=2)
try:
ee_client.dump_state()
except:
log.exception("Heartbeat Failed!")
beat_died[0] = True
self.beat_subscriber = HeartbeatSubscriber(
"heartbeat_queue", callback=heartbeat_callback, node=self.container.node
)
self.beat_subscriber.start()
try:
self._start_eeagent()
for i in range(0, 5):
if beat_died[0] is True:
assert False, "A Hearbeat callback wasn't able to contact the eeagent"
gevent.sleep(0.5)
finally:
self.beat_subscriber.stop()
class ExecutionEngineAgentPyonSingleIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2cei.yml")
self.resource_id = "eeagent_123456"
self._eea_name = "eeagent"
self.supd_directory = tempfile.mkdtemp()
self.agent_config = {
"eeagent": {
"heartbeat": 0,
"slots": 100,
"name": "pyon_eeagent",
"launch_type": {
"name": "pyon_single",
"pyon_directory": os.getcwd(),
"supd_directory": self.supd_directory,
"supdexe": "bin/supervisord",
},
},
"agent": {"resource_id": self.resource_id},
"logging": {
"loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
"root": {"handlers": ["console"]},
},
}
# Start eeagent.
self._eea_pid = None
self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=self.agent_config,
)
log.info("Agent pid=%s.", str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info("Got eea client %s.", str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.supd_directory)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon_single"
proc_name = "test_transform"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertEqual(proc.get("state"), [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertEqual(proc.get("state"), [700, "TERMINATED"])
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
print '#####################'
print 'IN SETUP'
self._ia_client = None
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
#log.info('driver uri: %s', DRV_URI)
#log.info('device address: %s', DEV_ADDR)
#log.info('device port: %s', DEV_PORT)
#log.info('work dir: %s', WORK_DIR)
# Create agent config.
agent_config = {
'driver_config': DVR_CONFIG,
'stream_config': self._stream_config,
'agent': {
'resource_id': IA_RESOURCE_ID
},
'test_mode': True,
'forget_past': True,
'enable_persistence': False
}
#if org_governance_name is not None:
# agent_config['org_governance_name'] = org_governance_name
# Start instrument agent.
log.info("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
log.info("Agent setup")
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
self.addCleanup(self._verify_agent_reset)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
log.info('test setup complete')
class ProcessDispatcherEEAgentIntTest(ProcessDispatcherServiceIntTest):
"""Run the basic int tests again, with a different environment
"""
def setUp(self):
self.dashi = None
self._start_container()
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
app = dict(name="process_dispatcher", processapp=("process_dispatcher",
"ion.services.cei.process_dispatcher_service",
"ProcessDispatcherService"))
self.container.start_app(app, config=pd_config)
self.rr_cli = self.container.resource_registry
self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.process_definition = ProcessDefinition(name='test_process')
self.process_definition.executable = {'module': 'ion.services.cei.test.test_process_dispatcher',
'class': 'TestProcess'}
self.process_definition_id = self.pd_cli.create_process_definition(self.process_definition)
self._eea_pids = []
self._tmpdirs = []
self.dashi = get_dashi(uuid.uuid4().hex,
pd_config['processdispatcher']['dashi_uri'],
pd_config['processdispatcher']['dashi_exchange'])
#send a fake node_state message to PD's dashi binding.
self.node1_id = uuid.uuid4().hex
self._send_node_state("engine1", self.node1_id)
self._start_eeagent(self.node1_id)
self.waiter = ProcessStateWaiter()
def _send_node_state(self, engine_id, node_id=None):
node_id = node_id or uuid.uuid4().hex
node_state = dict(node_id=node_id, state=InstanceState.RUNNING,
domain_id=domain_id_from_engine(engine_id))
self.dashi.fire(get_pd_dashi_name(), "node_state", args=node_state)
def _start_eeagent(self, node_id):
persistence_dir = tempfile.mkdtemp()
self._tmpdirs.append(persistence_dir)
agent_config = _get_eeagent_config(node_id, persistence_dir)
pid = self.container_client.spawn_process(name="eeagent",
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=agent_config)
log.info('Agent pid=%s.', str(pid))
self._eea_pids.append(pid)
def tearDown(self):
for pid in self._eea_pids:
self.container.terminate_process(pid)
for d in self._tmpdirs:
shutil.rmtree(d)
self.waiter.stop()
if self.dashi:
self.dashi.cancel()
def test_requested_ee(self):
# request non-default engine
process_target = ProcessTarget(execution_engine_id="engine2")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
process_schedule.target = process_target
pid = self.pd_cli.create_process(self.process_definition_id)
self.waiter.start()
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid)
self.waiter.await_state_event(pid, ProcessStateEnum.WAITING)
# request unknown engine, with NEVER queuing mode. The request
# should be rejected.
# verifies L4-CI-CEI-RQ52
process_target = ProcessTarget(execution_engine_id="not-a-real-ee")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.NEVER
process_schedule.target = process_target
rejected_pid = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=rejected_pid)
self.waiter.await_state_event(rejected_pid, ProcessStateEnum.REJECTED)
# now add a node and eeagent for engine2. original process should leave
# queue and start running
node2_id = uuid.uuid4().hex
self._send_node_state("engine2", node2_id)
self._start_eeagent(node2_id)
self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING)
# spawn another process. it should start immediately.
process_target = ProcessTarget(execution_engine_id="engine2")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.NEVER
process_schedule.target = process_target
pid2 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.RUNNING)
# one more with node exclusive
process_target = ProcessTarget(execution_engine_id="engine2",
node_exclusive="hats")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.NEVER
process_schedule.target = process_target
pid3 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.RUNNING)
# kill the processes for good
self.pd_cli.cancel_process(pid)
self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.TERMINATED)
def test_node_exclusive(self):
# the node_exclusive constraint is used to ensure multiple processes
# of the same "kind" each get a VM exclusive of each other. Other
# processes may run on these VMs, just not processes with the same
# node_exclusive tag. Since we cannot directly query the contents
# of each node in this test, we prove the capability by scheduling
# processes one by one and checking their state.
# verifies L4-CI-CEI-RQ121
# verifies L4-CI-CEI-RQ57
# first off, setUp() created a single node and eeagent.
# We schedule two processes with the same "abc" node_exclusive
# tag. Since there is only one node, the first process should run
# and the second should be queued.
process_target = ProcessTarget(execution_engine_id="engine1")
process_target.node_exclusive = "abc"
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
process_schedule.target = process_target
pid1 = self.pd_cli.create_process(self.process_definition_id)
self.waiter.start()
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid1)
self.waiter.await_state_event(pid1, ProcessStateEnum.RUNNING)
pid2 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.WAITING)
# now demonstrate that the node itself is not full by launching
# a third process without a node_exclusive tag -- it should start
# immediately
process_target.node_exclusive = None
pid3 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.RUNNING)
# finally, add a second node to the engine. pid2 should be started
# since there is an exclusive "abc" node free.
node2_id = uuid.uuid4().hex
self._send_node_state("engine1", node2_id)
self._start_eeagent(node2_id)
self.waiter.await_state_event(pid2, ProcessStateEnum.RUNNING)
# kill the processes for good
self.pd_cli.cancel_process(pid1)
self.waiter.await_state_event(pid1, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.TERMINATED)
def test_code_download(self):
# create a process definition that has no URL; only module and class.
process_definition_no_url = ProcessDefinition(name='test_process_nodownload')
process_definition_no_url.executable = {'module': 'ion.my.test.process',
'class': 'TestProcess'}
process_definition_id_no_url = self.pd_cli.create_process_definition(process_definition_no_url)
# create another that has a URL of the python file (this very file)
# verifies L4-CI-CEI-RQ114
url = "file://%s" % os.path.join(os.path.dirname(__file__), 'test_process_dispatcher.py')
process_definition = ProcessDefinition(name='test_process_download')
process_definition.executable = {'module': 'ion.my.test.process',
'class': 'TestProcess', 'url': url}
process_definition_id = self.pd_cli.create_process_definition(process_definition)
process_target = ProcessTarget()
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
process_schedule.target = process_target
self.waiter.start()
# Test a module with no download fails
pid_no_url = self.pd_cli.create_process(process_definition_id_no_url)
self.pd_cli.schedule_process(process_definition_id_no_url,
process_schedule, process_id=pid_no_url)
self.waiter.await_state_event(pid_no_url, ProcessStateEnum.FAILED)
# Test a module with a URL runs
pid = self.pd_cli.create_process(process_definition_id)
self.pd_cli.schedule_process(process_definition_id,
process_schedule, process_id=pid)
self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING)
def _initialize(self):
"""
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
try:
"""
Get the information for the driver. This can be read from the yml
files; the user can run switch_driver to change the current driver.
"""
self.fetch_metadata()
self.fetch_driver_class()
self.fetch_comm_config()
if not exists(PIPE_PATH):
mkfifo(PIPE_PATH)
if not exists(self.metadata.driver_dir()):
raise DriverDoesNotExist( "%s/%s/$%s" % (self.metadata.driver_make,
self.metadata.driver_model,
self.driver_name))
driver_module = DRIVER_MODULE_ROOT + self.metadata.driver_make + '.' + self.metadata.driver_model + '.' + self.metadata.driver_name + DRIVER_MODULE_LEAF
log.info('driver module: %s', driver_module)
log.info('driver class: %s', self.driver_class)
log.info('device address: %s', self.ip_address)
log.info('device data port: %s', self.data_port)
log.info('device command port: %s', self.command_port)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
DVR_CONFIG.update({'dvr_mod' : driver_module, 'dvr_cls' : self.driver_class})
"""
self._support = DriverIntegrationTestSupport(driver_module,
self.driver_class,
self.ip_address,
self.data_port,
DELIM,
WORK_DIR)
"""
# Start port agent, add stop to cleanup (not sure if that's
# necessary yet).
print( "------------------>>>> Starting Port Agent <<<<------------------" )
self.start_pagent()
# Start a monitor window if specified.
if self.monitor_window:
self.monitor_file = self._pagent.port_agent.logfname
strXterm = "xterm -T InstrumentMonitor -sb -rightbar"
#pOpenString = "xterm -T InstrumentMonitor -e tail -f " + self.monitor_file
pOpenString = strXterm + " -e tail -f " + self.monitor_file
x = subprocess.Popen(pOpenString, shell=True)
"""
DHE: Added self._cleanups to make base classes happy
"""
self.addCleanup(self.stop_pagent)
# Start container.
print( "------------------>>>> Starting Capability Container <<<<------------------" )
self._start_container()
# Bring up services in a deploy file (no need to message)
print( "------------------>>>> Starting Deploy Services <<<<------------------" )
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Setup stream config.
self._build_stream_config()
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
print( "------------------>>>> Starting Instrument Agent <<<<------------------" )
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
if self.subcriber_window:
self._start_data_subscribers(6)
#self.addCleanup(self._stop_data_subscribers)
except:
log.error("initialize(): Exception occurred; shutting down.", exc_info=True)
return False
else:
return True
class ProcessDispatcherEEAgentIntTest(ProcessDispatcherServiceIntTest):
"""Run the basic int tests again, with a different environment
"""
def setUp(self):
self._start_container()
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
app = dict(processapp=("process_dispatcher",
"ion.services.cei.process_dispatcher_service",
"ProcessDispatcherService"))
self.container.start_app(app, config=pd_config)
self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.process_definition = ProcessDefinition(name='test_process')
self.process_definition.executable = {'module': 'ion.services.cei.test.test_process_dispatcher',
'class': 'TestProcess'}
self.process_definition_id = self.pd_cli.create_process_definition(self.process_definition)
self.event_queue = queue.Queue()
self.event_sub = None
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 1,
'heartbeat_queue': 'hbeatq',
'slots': 100,
'name': 'pyon_eeagent',
'node_id': 'somenodeid',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
},
},
'agent': {'resource_id': self.resource_id},
}
#send a fake dt_state message to PD's dashi binding.
dashi = get_dashi(uuid.uuid4().hex,
pd_config['processdispatcher']['dashi_uri'],
pd_config['processdispatcher']['dashi_exchange'])
dt_state = dict(node_id="somenodeid", state=InstanceState.RUNNING,
deployable_type="eeagent_pyon")
dashi.fire(get_pd_dashi_name(), "dt_state", args=dt_state)
self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
def _start_eeagent(self):
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
# Start eeagent.
self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
if self.event_sub:
self.event_sub.stop()
self._stop_container()
class TestTerrestrialEndpoint(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
Setup fake remote components.
Start remote server.
Set internal configuration and test variables.
Start container.
Start services.
Spawn endpoint.
Create and start subscribers.
"""
# Create fake remote client and server.
# Add clean up to shut down properly.
# Start remote server on a random port.
self._remote_server = R3PCServer(self.consume_req,
self.remote_server_close)
self._remote_client = R3PCClient(self.consume_ack,
self.remote_client_close)
self.addCleanup(self._remote_server.stop)
self.addCleanup(self._remote_client.stop)
self._other_port = self._remote_server.start('*', 0)
log.debug('Remote server binding to *:%i', self._other_port)
# Set internal variables.
self._other_host = 'localhost'
self._xs_name = 'remote1'
self._svc_name = 'terrestrial_endpoint'
self._listen_name = self._svc_name + self._xs_name
self._platform_resource_id = 'abc123'
self._resource_id = 'fake_id'
self._rmt_svc_name = 'fake_svc'
self._no_requests = 10
self._requests_sent = {}
self._results_recv = {}
self._workers = []
self._done_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._telem_evts = []
self._no_telem_evts = 0
self._no_queue_mod_evts = 0
self._no_cmd_tx_evts = 0
self._done_queue_mod_evts = AsyncResult()
self._done_telem_evts = AsyncResult()
self._done_cmd_tx_evts = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# The following spawn config creates the process with the remote
# name tagged to the service name.
"""
listen_name = terrestrial_endpointremote1
2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1)
2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1)
2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,Edwards-MacBook-Pro_local_2624.33,B: Edwards-MacBook-Pro_local_2624.33)
"""
# Create agent config.
endpoint_config = {
'other_host': self._other_host,
'other_port': self._other_port,
'this_port': 0,
'xs_name': self._xs_name,
'platform_resource_id': self._platform_resource_id,
'process': {
'listen_name': self._listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._te_pid = self._container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(self._te_pid))
# Create an endpoint client.
# The to_name may be either the process pid or
# the listen_name, which for this remote bridge
# is svc_name + remote_name as above.
self.te_client = TerrestrialEndpointClient(process=FakeProcess(),
to_name=self._listen_name)
log.debug('Got te client %s.', str(self.te_client))
# Remember the terrestrial port.
self._this_port = self.te_client.get_port()
# Start the event publisher.
self._event_publisher = EventPublisher()
# Start the event subscriber.
self._event_subscriber = EventSubscriber(event_type='PlatformEvent',
callback=self.consume_event,
origin=self._xs_name)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(
timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._event_subscriber.stop)
# Start the result subscriber.
self._result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=self._resource_id,
callback=self.consume_event)
self._result_subscriber.start()
self._result_subscriber._ready_event.wait(
timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._result_subscriber.stop)
def consume_event(self, evt, *args, **kwargs):
"""
Test callback for events.
"""
log.debug('Got event: %s, args: %s, kwargs: %s', str(evt), str(args),
str(kwargs))
if evt.type_ == 'PublicPlatformTelemetryEvent':
self._telem_evts.append(evt)
if self._no_telem_evts > 0 and self._no_telem_evts == len(
self._telem_evts):
self._done_telem_evts.set()
elif evt.type_ == 'RemoteQueueModifiedEvent':
self._queue_mod_evts.append(evt)
if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(
self._queue_mod_evts):
self._done_queue_mod_evts.set()
elif evt.type_ == 'RemoteCommandTransmittedEvent':
self._cmd_tx_evts.append(evt)
if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(
self._cmd_tx_evts):
self._done_cmd_tx_evts.set()
elif evt.type_ == 'RemoteCommandResult':
cmd = evt.command
self._results_recv[cmd.command_id] = cmd
if len(self._results_recv) == self._no_requests:
self._done_evt.set()
def on_link_up(self):
"""
Called by a test to simulate turning the link on.
"""
log.debug('Remote client connecting to localhost:%i.', self._this_port)
self._remote_client.start('localhost', self._this_port)
# Publish a link up event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status=TelemetryStatusType.AVAILABLE)
def on_link_down(self):
"""
Called by a test to simulate turning the link off.
"""
self._remote_client.stop()
# Publish a link down event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status=TelemetryStatusType.UNAVAILABLE)
def consume_req(self, request):
"""
Remote request callback.
Fire a greenlet to do some fake work before returning via
the remote client to terrestrial endpoint.
"""
# Spawn a greenlet to sleep briefly with each request and
# then respond with a result through the remote client.
log.debug('Remote endpoint got request: %s', str(request))
greenlet = gevent.spawn(self.process_remote_request, request)
self._workers.append(greenlet)
def consume_ack(self, request):
"""
Remote ack callback.
"""
log.debug('Remote endpoint got ack: %s', str(request))
def process_remote_request(self, request):
"""
Process remote request.
Do random amount of fake work and enqueue result for return to
terrestrial endpoint.
"""
worktime = random.uniform(.1, 3)
gevent.sleep(worktime)
result = {'command_id': request.command_id, 'result': 'fake_result'}
log.debug('Finished processing request: %s', str(request))
self._remote_client.enqueue(result)
def remote_server_close(self):
"""
Remote server closed callback.
"""
log.debug('The remote server closed.')
def remote_client_close(self):
"""
Remoe client closed callback.
"""
log.debug('The remote client closed.')
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id=self._resource_id,
command=cmdstr,
args=['arg1', 23],
kwargs={'kwargs1': 'someval'})
return cmd
def make_fake_svc_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
svc_name=self._rmt_svc_name,
command=cmdstr,
args=['arg1', 23],
kwargs={'kwargs1': 'someval'})
return cmd
def test_process_queued(self):
"""
test_process_queued
Test forwarding of queued commands upon link up.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_process_online(self):
"""
test_process_online
Test forwarding commands when the link is up.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
self.on_link_up()
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
gevent.sleep(.2)
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_remote_late(self):
"""
test_remote_late
Test simulates behavior when the remote side is initially unavailable.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
self.on_link_up()
gevent.sleep(2)
self._remote_server.stop()
self._remote_client.stop()
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
gevent.sleep(3)
self._remote_client.start('localhost', self._this_port)
self._remote_server.start('*', self._other_port)
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_get_clear_queue(self):
"""
test_get_clear_queue
Test endpoint queue get and clear manipulators.
"""
# Set up for events expected.
self._no_queue_mod_evts = self._no_requests * 2
# Queue commands.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Queue commands.
for i in range(self._no_requests):
cmd = self.make_fake_svc_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests * 2)
# Confirm get queue with id.
queue = self.te_client.get_queue(resource_id=self._resource_id)
self.assertEqual(len(queue), self._no_requests)
# Confirm get queue with svc name.
queue = self.te_client.get_queue(svc_name=self._rmt_svc_name)
self.assertEqual(len(queue), self._no_requests)
# Confirm get queue with bogus id.
queue = self.te_client.get_queue(resource_id='bogus_id')
self.assertEqual(len(queue), 0)
# Confirm get queue with bogus id.
queue = self.te_client.get_queue(svc_name='bogus_svc')
self.assertEqual(len(queue), 0)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with no id.
poped = self.te_client.clear_queue()
# Confirm queue mod event and mods.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests * 2)
self.assertEqual(len(queue), 0)
# Queue new commands and confirm event.
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests * 2
self._done_queue_mod_evts = AsyncResult()
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
for i in range(self._no_requests):
cmd = self.make_fake_svc_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with id.
poped = self.te_client.clear_queue(resource_id=self._resource_id)
# Confirm mods and mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests)
self.assertEqual(len(queue), self._no_requests)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with id.
poped = self.te_client.clear_queue(svc_name=self._rmt_svc_name)
# Confirm mods and mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests)
self.assertEqual(len(queue), 0)
# Queue new commands and confirm events.
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evts = AsyncResult()
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Clear queue with bogus id.
poped = self.te_client.clear_queue(resource_id='bogus id')
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 0)
self.assertEqual(len(queue), self._no_requests)
# Clear queue with bogus svc name.
poped = self.te_client.clear_queue(svc_name='bogus id')
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 0)
self.assertEqual(len(queue), self._no_requests)
# Clear queue and confirm empty.
self.te_client.clear_queue()
queue = self.te_client.get_queue()
self.assertEqual(len(queue), 0)
# Turn on link and wait a few seconds.
# Confirm no data or tx events arrive.
self.on_link_up()
gevent.sleep(2)
self.assertEqual(len(self._cmd_tx_evts), 0)
self.assertEqual(len(self._results_recv), 0)
self._no_telem_evts = 2
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
def test_pop_pending_queue(self):
"""
test_pop_pending_queue
Test endpoint queue pop manipulators.
"""
# Set up for events expected.
self._no_queue_mod_evts = self._no_requests
# Queue commands.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Pop a few commands from the queue, confirm events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 3
self._done_queue_mod_evts = AsyncResult()
cmd_ids = self._requests_sent.keys()[:3]
poped = []
for x in cmd_ids:
poped.append(self.te_client.pop_queue(x))
self._requests_sent.pop(x)
# Try poping with illegal args. This should have no effect
poped.append(self.te_client.pop_queue())
poped.append(self.te_client.pop_queue('bogus id'))
poped = [x for x in poped if x != None]
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 3)
self.assertEqual(len(queue), self._no_requests - 3)
# Turn on the link and verify that only the remaining commands
# get processed.
self._no_telem_evts = 2
self._no_requests = self._no_requests - 3
self._no_cmd_tx_evts = self._no_requests
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
def test_repeated_clear_pop(self):
"""
test_repeated_clear_pop
Test endpoint queue pop manipulators.
"""
# Set up for events expected.
self._no_queue_mod_evts = self._no_requests
for i in range(3):
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evts = AsyncResult()
# Queue commands.
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with no id.
poped = self.te_client.clear_queue()
# Confirm queue mod event and mods.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests)
self.assertEqual(len(queue), 0)
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evts = AsyncResult()
# Queue commands.
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Pop a few commands from the queue, confirm events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 3
self._done_queue_mod_evts = AsyncResult()
cmd_ids = self._requests_sent.keys()[:3]
poped = []
for x in cmd_ids:
poped.append(self.te_client.pop_queue(x))
self._requests_sent.pop(x)
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 3)
self.assertEqual(len(queue), self._no_requests - 3)
self._no_telem_evts = 2
self._no_requests = self._no_requests - 3
self._no_cmd_tx_evts = self._no_requests
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
def test_get_pending(self):
"""
test_process_queued
Test forwarding of queued commands upon link up.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self.on_link_up()
self._no_requests = 3
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending(resource_id=self._resource_id)
for x in pending:
self.assertIn(x.command_id, self._requests_sent.keys())
self._no_requests = 10
self._done_evt = AsyncResult()
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
#@unittest.skip('Wait for verification of resource registry use.')
def test_persistent_queue(self):
"""
test_persistent_queue
Test ability of endpoint to restore a persistent queue, survive
reboot, etc.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Stop and restart the endpoint process.
# Verify the queue is restored.
self._container_client.terminate_process(self._te_pid)
# Create agent config.
endpoint_config = {
'other_host': self._other_host,
'other_port': self._other_port,
'this_port': 0,
'xs_name': self._xs_name,
'platform_resource_id': self._platform_resource_id,
'process': {
'listen_name': self._listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._te_pid = self._container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(self._te_pid))
# Create an endpoint client.
# The to_name may be either the process pid or
# the listen_name, which for this remote bridge
# is svc_name + remote_name as above.
self.te_client = TerrestrialEndpointClient(process=FakeProcess(),
to_name=self._listen_name)
log.debug('Got te client %s.', str(self.te_client))
# Remember the terrestrial port.
self._this_port = self.te_client.get_port()
# Confirm we restored the queue with the previous commands.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
_webserver = None
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 1,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
}
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
self._start_eeagent()
def _start_eeagent(self):
self.container_client = ContainerAgentClient(
node=self.container.node, name=self.container.name)
self.container = self.container_client._get_container_instance()
# Start eeagent.
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self._stop_webserver()
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
def _start_webserver(self, directory_to_serve, port=None):
""" Start a webserver for testing code download
Note: tries really hard to get a port, and if it can't use
the suggested port, randomly picks another, and returns it
"""
def log_message(self, format, *args):
#swallow log massages
pass
class Server(HTTPServer):
requests = 0
def serve_forever(self):
self._serving = 1
while self._serving:
self.handle_request()
self.requests += 1
def stop(self):
self._serving = 0
if port is None:
port = 8008
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
Handler.log_message = log_message
for i in range(0, 100):
try:
self._webserver = Server(("localhost", port), Handler)
except socket.error:
print "port %s is in use, picking another" % port
port = randint(8000, 10000)
continue
else:
break
self._web_glet = gevent.spawn(self._webserver.serve_forever)
return port
def _stop_webserver(self):
if self._webserver is not None:
self._web_glet.kill()
def _enable_code_download(self, whitelist=None):
if whitelist is None:
whitelist = []
self.container.terminate_process(self._eea_pid)
self.agent_config['eeagent']['code_download'] = {
'enabled': True,
'whitelist': whitelist
}
self._start_eeagent()
def wait_for_state(self, upid, desired_state, timeout=30):
attempts = 0
last_state = None
while timeout > attempts:
try:
state = self.eea_client.dump_state().result
except Timeout:
log.warn("Timeout calling EEAgent dump_state. retrying.")
continue
proc = get_proc_for_upid(state, upid)
last_state = proc.get('state')
if last_state == desired_state:
return
gevent.sleep(1)
attempts += 1
assert False, "Process %s took too long to get to %s, had %s" % (upid, desired_state, last_state)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, 'TERMINATED'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
self.eea_client.cleanup_process(u_pid, round)
state = self.eea_client.dump_state().result
assert len(state['processes']) == 0
@needs_eeagent
def test_duplicate(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, 'TERMINATED'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
self.eea_client.cleanup_process(u_pid, round)
state = self.eea_client.dump_state().result
assert len(state['processes']) == 0
@needs_eeagent
def test_restart(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
# Start again with incremented round. eeagent should restart the process
round += 1
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
state = self.eea_client.dump_state().result
ee_round = state['processes'][0]['round']
assert round == int(ee_round)
# TODO: this test is disabled, as the restart op is disabled
# Run restart with incremented round. eeagent should restart the process
#round += 1
#self.eea_client.restart_process(u_pid, round)
#self.wait_for_state(u_pid, [500, 'RUNNING'])
#state = self.eea_client.dump_state().result
#ee_round = state['processes'][0]['round']
#assert round == int(ee_round)
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, 'TERMINATED'])
state = self.eea_client.dump_state().result
assert len(state['processes']) == 1
self.eea_client.cleanup_process(u_pid, round)
state = self.eea_client.dump_state().result
assert len(state['processes']) == 0
@needs_eeagent
def test_failing_process(self):
u_pid = "testfail"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcessFail'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [850, 'FAILED'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_slow_to_start(self):
upids = map(lambda i: str(uuid.uuid4().hex), range(0, 10))
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcessSlowStart'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
for upid in upids:
self.eea_client.launch_process(upid, round, run_type, parameters)
for upid in upids:
self.wait_for_state(upid, [500, 'RUNNING'], timeout=60)
@needs_eeagent
def test_start_cancel(self):
upid = str(uuid.uuid4().hex)
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcessSlowStart'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(upid, round, run_type, parameters)
self.wait_for_state(upid, [400, 'PENDING'])
self.eea_client.terminate_process(upid, round)
self.wait_for_state(upid, [700, 'TERMINATED'])
@needs_eeagent
def test_kill_and_revive(self):
"""test_kill_and_revive
Ensure that when an eeagent dies, it pulls the processes it owned from
persistence, and marks them as failed, so the PD can figure out what to
do with them
"""
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
# Kill and restart eeagent. Also, kill proc started by eea to simulate
# a killed container
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
self.assertIsNotNone(proc_to_kill)
self.container.terminate_process(proc_to_kill.id)
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
self.wait_for_state(u_pid, [850, 'FAILED'])
@needs_eeagent
def test_run_out_of_slots(self):
"""test_run_out_of_slots
"""
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
self.agent_config['eeagent']['slots'] = 1
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
u_pid_0, u_pid_1 = "test0", "test1"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid_0, round, run_type, parameters)
self.wait_for_state(u_pid_0, [500, 'RUNNING'])
self.eea_client.launch_process(u_pid_1, round, run_type, parameters)
self.wait_for_state(u_pid_1, [900, 'REJECTED'])
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
self.agent_config['eeagent']['slots'] = 1
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
self.wait_for_state(u_pid_0, [850, 'FAILED'])
self.wait_for_state(u_pid_1, [900, 'REJECTED'])
@needs_eeagent
def test_download_code(self):
self._enable_code_download(whitelist=['*'])
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = "ion.my.module.to.download"
module_uri = 'file://%s/downloads/module_to_download.py' % get_this_directory()
bad_module_uri = 'file:///tmp/notreal/module_to_download.py'
cls = 'TestDownloadProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': bad_module_uri, 'cls': cls}
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
print response
assert response.status == 404
assert "Unable to download" in response.result
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
round += 1
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_whitelist(self):
downloads_directory = os.path.join(get_this_directory(), "downloads")
http_port = 8910
http_port = self._start_webserver(downloads_directory, port=http_port)
while self._webserver is None:
print "Waiting for webserver to come up"
gevent.sleep(1)
assert self._webserver.requests == 0
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = "ion.my.module"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = 'TestDownloadProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
assert response.status == 401
assert "Code download not enabled" in response.result
# Test no whitelist
self._enable_code_download()
round += 1
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
print response
assert response.status == 401
assert "not in code_download whitelist" in response.result
# Test not matching
self._enable_code_download(whitelist=['blork'])
round += 1
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
assert response.status == 401
assert "not in code_download whitelist" in response.result
# Test exact matching
self._enable_code_download(whitelist=['localhost'])
round += 1
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
# Test wildcard
self._enable_code_download(whitelist=['*'])
round += 1
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_caching(self):
downloads_directory = os.path.join(get_this_directory(), "downloads")
http_port = 8910
http_port = self._start_webserver(downloads_directory, port=http_port)
while self._webserver is None:
print "Waiting for webserver to come up"
gevent.sleep(1)
self._enable_code_download(['*'])
assert self._webserver.requests == 0
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = "ion.my.module"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = 'TestDownloadProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
# Launch a process, check that webserver is hit
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
# Launch another process, check that webserver is still only hit once
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
u_pid = "test5"
round = 0
run_type = "pyon"
proc_name = 'test_transformx'
module = "ion.agents.cei.test.test_eeagent"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
# Test that a module that is already available in tarball won't trigger a download
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
u_pid = "test9"
round = 0
run_type = "pyon"
proc_name = 'test_transformx'
module = "ion.agents.cei.test.test_eeagent"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = 'TestProcessNotReal'
parameters = {'name': proc_name, 'module': module, 'module_uri': module_uri, 'cls': cls}
# Test behaviour of a non existant class with no download
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [850, 'FAILED'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
class ExecutionEngineAgentPyonSingleIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_123456"
self._eea_name = "eeagent"
self.supd_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon_single',
'pyon_directory': os.getcwd(),
'supd_directory': self.supd_directory,
'supdexe': 'bin/supervisord'
},
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
# Start eeagent.
self._eea_pid = None
self.container_client = ContainerAgentClient(
node=self.container.node, name=self.container.name)
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.supd_directory)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon_single"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertEqual(proc.get('state'), [700, 'TERMINATED'])
class ExternalDatasetAgentTestBase(object):
# Agent parameters.
EDA_RESOURCE_ID = '123xyz'
EDA_NAME = 'ExampleEDA'
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Initialize test members.
"""
# log.warn('Starting the container')
# Start container.
self._start_container()
# Bring up services in a deploy file
# log.warn('Starting the rel')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a pubsub client to create streams.
# log.warn('Init a pubsub client')
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# log.warn('Init a ContainerAgentClient')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
# TODO: Finish dealing with the resources and whatnot
# TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources()
self._setup_resources()
#TG: Setup/configure the granule logger to log granules as they're published
# Create agent config.
agent_config = {
'driver_config': self.DVR_CONFIG,
'stream_config': {},
'agent': {
'resource_id': self.EDA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
log.debug('TestInstrumentAgent.setup(): starting EDA.')
self._ia_pid = self._container_client.spawn_process(
name=self.EDA_NAME,
module=self.EDA_MOD,
cls=self.EDA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
########################################
# Private "setup" functions
########################################
def _setup_resources(self):
raise NotImplementedError(
'_setup_resources must be implemented in the subclass')
def create_stream_and_logger(self, name, stream_id=''):
if not stream_id or stream_id is '':
stream_id = self._pubsub_client.create_stream(name=name,
encoding='ION R2')
pid = self._container_client.spawn_process(
name=name + '_logger',
module='ion.processes.data.stream_granule_logger',
cls='StreamGranuleLogger',
config={'process': {
'stream_id': stream_id
}})
log.info(
'Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'.
format(pid, stream_id))
return stream_id
def _start_finished_event_subscriber(self):
def consume_event(*args, **kwargs):
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(
self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(
len(self._finished_events_received))
log.debug(
'Called self._async_finished_result.set({0})'.format(
len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(
event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.activate()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.deactivate()
self._finished_event_subscriber = None
########################################
# Custom assertion functions
########################################
def assertListsEqual(self, lst1, lst2):
lst1.sort()
lst2.sort()
return lst1 == lst2
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
#{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
self.assertTrue(isinstance(val, dict))
self.assertTrue(val.has_key('c'))
self.assertTrue(val.has_key('t'))
self.assertTrue(val.has_key('p'))
self.assertTrue(val.has_key('time'))
c = val['c'][0]
t = val['t'][0]
p = val['p'][0]
time = val['time'][0]
self.assertTrue(isinstance(c, float))
self.assertTrue(isinstance(t, float))
self.assertTrue(isinstance(p, float))
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(PARAMS.has_key(key))
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val * .01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
########################################
# Test functions
########################################
def test_acquire_data(self):
cmd = AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
self._finished_count = 3
log.info('Send an unconstrained request for data (\'new data\')')
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
log.info(
'Send a second unconstrained request for data (\'new data\'), should be rejected'
)
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
config_mods = {}
log.info(
'Send a constrained request for data: constraints = HIST_CONSTRAINTS_1'
)
config_mods['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_1')
config_mods['constraints'] = self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
log.info(
'Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2'
)
config_mods['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_2')
config_mods['constraints'] = self.HIST_CONSTRAINTS_2
# config={'stream_id':'second_historical','TESTING':True, 'constraints':self.HIST_CONSTRAINTS_2}
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
finished = self._async_finished_result.get(timeout=10)
self.assertEqual(finished, self._finished_count)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_acquire_data_while_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {'POLLING_INTERVAL': 5}
self._ia_client.set_param(params)
self._finished_count = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
config = get_safe(self.DVR_CONFIG, 'dh_cfg', {})
log.info(
'Send a constrained request for data: constraints = HIST_CONSTRAINTS_1'
)
config['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_1')
config['constraints'] = self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config])
reply = self._ia_client.execute(cmd)
self.assertNotEqual(reply.status, 660)
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {'POLLING_INTERVAL': 5}
self._ia_client.set_param(params)
self._finished_count = 3
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
# Wait for some samples to roll in.
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_observatory(self):
# Test instrument driver get and set interface.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Retrieve all resource parameters.
reply = self._ia_client.get_param('DRIVER_PARAMETER_ALL')
self.assertParamDict(reply, True)
orig_config = reply
## Retrieve a subset of resource parameters.
params = ['POLLING_INTERVAL']
reply = self._ia_client.get_param(params)
self.assertParamDict(reply)
orig_params = reply
# Set a subset of resource parameters.
new_params = {
'POLLING_INTERVAL': (orig_params['POLLING_INTERVAL'] * 2),
}
self._ia_client.set_param(new_params)
check_new_params = self._ia_client.get_param(params)
self.assertParamVals(check_new_params, new_params)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_get_set_param(self):
cmd = AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
# Get a couple parameters
retval = self._ia_client.get_param(
['POLLING_INTERVAL', 'PATCHABLE_CONFIG_KEYS'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(type(retval['POLLING_INTERVAL']), int)
self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']), list)
# Attempt to get a parameter that doesn't exist
log.debug('Try getting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.get_param,
['BAD_PARAM'])
# Set the polling_interval to a new value, then get it to make sure it set properly
self._ia_client.set_param({'POLLING_INTERVAL': 10})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(retval['POLLING_INTERVAL'], 10)
# Attempt to set a parameter that doesn't exist
log.debug('Try setting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.set_param,
{'BAD_PARAM': 'bad_val'})
# Attempt to set one parameter that does exist, and one that doesn't
self.assertRaises(InstParameterError, self._ia_client.set_param, {
'POLLING_INTERVAL': 20,
'BAD_PARAM': 'bad_val'
})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(retval['POLLING_INTERVAL'], 20)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_initialize(self):
# Test agent initialize command. This causes creation of driver process and transition to inactive.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_states(self):
# Test agent state transitions.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._finished_count = 1
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
gevent.sleep(5)
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._async_finished_result.get(timeout=5)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_capabilities(self):
# Test the ability to retrieve agent and resource parameter and command capabilities.
acmds = self._ia_client.get_capabilities(['AGT_CMD'])
log.debug('Agent Commands: {0}'.format(acmds))
acmds = [item[1] for item in acmds]
self.assertListsEqual(acmds, AGT_CMDS.keys())
apars = self._ia_client.get_capabilities(['AGT_PAR'])
log.debug('Agent Parameters: {0}'.format(apars))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
rcmds = self._ia_client.get_capabilities(['RES_CMD'])
log.debug('Resource Commands: {0}'.format(rcmds))
rcmds = [item[1] for item in rcmds]
self.assertListsEqual(rcmds, CMDS.keys())
rpars = self._ia_client.get_capabilities(['RES_PAR'])
log.debug('Resource Parameters: {0}'.format(rpars))
rpars = [item[1] for item in rpars]
self.assertListsEqual(rpars, PARAMS.keys())
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_errors(self):
# Test illegal behavior and replies.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
# Can't go active in unitialized state.
# Status 660 is state error.
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
log.info('GO ACTIVE CMD %s', str(retval))
self.assertEquals(retval.status, 660)
# Can't command driver in this state.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertEqual(reply.status, 660)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# 404 unknown agent command.
cmd = AgentCommand(command='kiss_edward')
retval = self._ia_client.execute_agent(cmd)
self.assertEquals(retval.status, 404)
# 670 unknown driver command.
cmd = AgentCommand(command='acquire_sample_please')
retval = self._ia_client.execute(cmd)
self.assertEqual(retval.status, 670)
# 630 Parameter error.
self.assertRaises(InstParameterError, self._ia_client.get_param,
'bogus bogus')
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
class ProcessDispatcherEEAgentIntTest(ProcessDispatcherServiceIntTest):
"""Run the basic int tests again, with a different environment
"""
def setUp(self):
self.dashi = None
self._start_container()
from pyon.public import CFG
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
app = dict(name="process_dispatcher",
processapp=("process_dispatcher",
"ion.services.cei.process_dispatcher_service",
"ProcessDispatcherService"))
self.container.start_app(app, config=pd_config)
self.rr_cli = self.container.resource_registry
self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.process_definition = ProcessDefinition(name='test_process')
self.process_definition.executable = {
'module': 'ion.services.cei.test.test_process_dispatcher',
'class': 'TestProcess'
}
self.process_definition_id = self.pd_cli.create_process_definition(
self.process_definition)
self._eea_pids = []
self._eea_pid_to_resource_id = {}
self._eea_pid_to_persistence_dir = {}
self._tmpdirs = []
self.dashi = get_dashi(
uuid.uuid4().hex,
pd_config['processdispatcher']['dashi_uri'],
pd_config['processdispatcher']['dashi_exchange'],
sysname=CFG.get_safe("dashi.sysname"))
#send a fake node_state message to PD's dashi binding.
self.node1_id = uuid.uuid4().hex
self._send_node_state("engine1", self.node1_id)
self._initial_eea_pid = self._start_eeagent(self.node1_id)
self.waiter = ProcessStateWaiter()
def _send_node_state(self, engine_id, node_id=None):
node_id = node_id or uuid.uuid4().hex
node_state = dict(node_id=node_id,
state=InstanceState.RUNNING,
domain_id=domain_id_from_engine(engine_id))
self.dashi.fire(get_pd_dashi_name(), "node_state", args=node_state)
def _start_eeagent(self, node_id, resource_id=None, persistence_dir=None):
if not persistence_dir:
persistence_dir = tempfile.mkdtemp()
self._tmpdirs.append(persistence_dir)
resource_id = resource_id or uuid.uuid4().hex
agent_config = _get_eeagent_config(node_id,
persistence_dir,
resource_id=resource_id)
pid = self.container_client.spawn_process(
name="eeagent",
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=agent_config)
log.info('Agent pid=%s.', str(pid))
self._eea_pids.append(pid)
self._eea_pid_to_resource_id[pid] = resource_id
self._eea_pid_to_persistence_dir[pid] = persistence_dir
return pid
def _kill_eeagent(self, pid):
self.assertTrue(pid in self._eea_pids)
self.container.terminate_process(pid)
self._eea_pids.remove(pid)
del self._eea_pid_to_resource_id[pid]
del self._eea_pid_to_persistence_dir[pid]
def tearDown(self):
for pid in list(self._eea_pids):
self._kill_eeagent(pid)
for d in self._tmpdirs:
shutil.rmtree(d)
self.waiter.stop()
if self.dashi:
self.dashi.cancel()
def test_requested_ee(self):
# request non-default engine
process_target = ProcessTarget(execution_engine_id="engine2")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
process_schedule.target = process_target
pid = self.pd_cli.create_process(self.process_definition_id)
self.waiter.start()
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
process_id=pid)
self.waiter.await_state_event(pid, ProcessStateEnum.WAITING)
# request unknown engine, with NEVER queuing mode. The request
# should be rejected.
# verifies L4-CI-CEI-RQ52
process_target = ProcessTarget(execution_engine_id="not-a-real-ee")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.NEVER
process_schedule.target = process_target
rejected_pid = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
process_id=rejected_pid)
self.waiter.await_state_event(rejected_pid, ProcessStateEnum.REJECTED)
# now add a node and eeagent for engine2. original process should leave
# queue and start running
node2_id = uuid.uuid4().hex
self._send_node_state("engine2", node2_id)
self._start_eeagent(node2_id)
self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING)
# spawn another process. it should start immediately.
process_target = ProcessTarget(execution_engine_id="engine2")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.NEVER
process_schedule.target = process_target
pid2 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
process_id=pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.RUNNING)
# one more with node exclusive
process_target = ProcessTarget(execution_engine_id="engine2",
node_exclusive="hats")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.NEVER
process_schedule.target = process_target
pid3 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
process_id=pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.RUNNING)
# kill the processes for good
self.pd_cli.cancel_process(pid)
self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.TERMINATED)
def test_node_exclusive(self):
# the node_exclusive constraint is used to ensure multiple processes
# of the same "kind" each get a VM exclusive of each other. Other
# processes may run on these VMs, just not processes with the same
# node_exclusive tag. Since we cannot directly query the contents
# of each node in this test, we prove the capability by scheduling
# processes one by one and checking their state.
# verifies L4-CI-CEI-RQ121
# verifies L4-CI-CEI-RQ57
# first off, setUp() created a single node and eeagent.
# We schedule two processes with the same "abc" node_exclusive
# tag. Since there is only one node, the first process should run
# and the second should be queued.
process_target = ProcessTarget(execution_engine_id="engine1")
process_target.node_exclusive = "abc"
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
process_schedule.target = process_target
pid1 = self.pd_cli.create_process(self.process_definition_id)
self.waiter.start()
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
process_id=pid1)
self.waiter.await_state_event(pid1, ProcessStateEnum.RUNNING)
pid2 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
process_id=pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.WAITING)
# now demonstrate that the node itself is not full by launching
# a third process without a node_exclusive tag -- it should start
# immediately
process_target.node_exclusive = None
pid3 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
process_id=pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.RUNNING)
# finally, add a second node to the engine. pid2 should be started
# since there is an exclusive "abc" node free.
node2_id = uuid.uuid4().hex
self._send_node_state("engine1", node2_id)
self._start_eeagent(node2_id)
self.waiter.await_state_event(pid2, ProcessStateEnum.RUNNING)
# kill the processes for good
self.pd_cli.cancel_process(pid1)
self.waiter.await_state_event(pid1, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.TERMINATED)
def test_code_download(self):
# create a process definition that has no URL; only module and class.
process_definition_no_url = ProcessDefinition(
name='test_process_nodownload')
process_definition_no_url.executable = {
'module': 'ion.my.test.process',
'class': 'TestProcess'
}
process_definition_id_no_url = self.pd_cli.create_process_definition(
process_definition_no_url)
# create another that has a URL of the python file (this very file)
# verifies L4-CI-CEI-RQ114
url = "file://%s" % os.path.join(os.path.dirname(__file__),
'test_process_dispatcher.py')
process_definition = ProcessDefinition(name='test_process_download')
process_definition.executable = {
'module': 'ion.my.test.process',
'class': 'TestProcess',
'url': url
}
process_definition_id = self.pd_cli.create_process_definition(
process_definition)
process_target = ProcessTarget()
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
process_schedule.target = process_target
self.waiter.start()
# Test a module with no download fails
pid_no_url = self.pd_cli.create_process(process_definition_id_no_url)
self.pd_cli.schedule_process(process_definition_id_no_url,
process_schedule,
process_id=pid_no_url)
self.waiter.await_state_event(pid_no_url, ProcessStateEnum.FAILED)
# Test a module with a URL runs
pid = self.pd_cli.create_process(process_definition_id)
self.pd_cli.schedule_process(process_definition_id,
process_schedule,
process_id=pid)
self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING)
def _add_test_process(self, restart_mode=None):
process_schedule = ProcessSchedule()
if restart_mode is not None:
process_schedule.restart_mode = restart_mode
pid = self.pd_cli.create_process(self.process_definition_id)
pid_listen_name = "PDtestproc_%s" % uuid.uuid4().hex
config = {'process': {'listen_name': pid_listen_name}}
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
process_id=pid,
configuration=config)
client = TestClient(to_name=pid_listen_name)
return pid, client
def test_restart(self):
self.waiter.start()
restartable_pids = []
nonrestartable_pids = []
clients = {}
# start 10 processes with RestartMode.ALWAYS
for _ in range(10):
pid, client = self._add_test_process(ProcessRestartMode.ALWAYS)
restartable_pids.append(pid)
clients[pid] = client
# and 10 processes with RestartMode.ABNORMAL
for _ in range(10):
pid, client = self._add_test_process(ProcessRestartMode.ABNORMAL)
restartable_pids.append(pid)
clients[pid] = client
# and 10 with RestartMode.NEVER
for _ in range(10):
pid, client = self._add_test_process(ProcessRestartMode.NEVER)
nonrestartable_pids.append(pid)
clients[pid] = client
all_pids = restartable_pids + nonrestartable_pids
self.waiter.await_many_state_events(all_pids, ProcessStateEnum.RUNNING)
for pid in all_pids:
client = clients[pid]
self.assertFalse(client.is_restart())
self.assertEqual(client.count(), 1)
# now kill the whole eeagent and restart it. processes should
# show up as FAILED in the next heartbeat.
resource_id = self._eea_pid_to_resource_id[self._initial_eea_pid]
persistence_dir = self._eea_pid_to_persistence_dir[
self._initial_eea_pid]
log.debug("Restarting eeagent %s", self._initial_eea_pid)
self._kill_eeagent(self._initial_eea_pid)
# manually kill the processes to simulate a real container failure
for pid in all_pids:
self.container.terminate_process(pid)
self._start_eeagent(self.node1_id,
resource_id=resource_id,
persistence_dir=persistence_dir)
# wait for restartables to restart
self.waiter.await_many_state_events(restartable_pids,
ProcessStateEnum.RUNNING)
# query the processes again. it should have restart mode config
for pid in restartable_pids:
client = clients[pid]
self.assertTrue(client.is_restart())
self.assertEqual(client.count(), 1)
# meanwhile some procs should not have restarted
for pid in nonrestartable_pids:
proc = self.pd_cli.read_process(pid)
self.assertEqual(proc.process_state, ProcessStateEnum.FAILED)
# guard against extraneous events we were receiving as part of a bug:
# processes restarting again after they were already restarted
self.waiter.await_nothing(timeout=5)
def test_idempotency(self):
# ensure every operation can be safely retried
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
proc_name = 'myreallygoodname'
pid = self.pd_cli.create_process(self.process_definition_id)
self.waiter.start(pid)
# note: if we import UNSCHEDULED state into ProcessStateEnum,
# this assertion will need to change.
proc = self.pd_cli.read_process(pid)
self.assertEqual(proc.process_id, pid)
self.assertEqual(proc.process_state, ProcessStateEnum.REQUESTED)
pid2 = self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
configuration={},
process_id=pid,
name=proc_name)
self.assertEqual(pid, pid2)
self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING)
# repeating schedule is harmless
pid2 = self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
configuration={},
process_id=pid,
name=proc_name)
self.assertEqual(pid, pid2)
proc = self.pd_cli.read_process(pid)
self.assertEqual(proc.process_id, pid)
self.assertEqual(proc.process_configuration, {})
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)
self.pd_cli.cancel_process(pid)
self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED)
# repeating cancel is harmless
self.pd_cli.cancel_process(pid)
proc = self.pd_cli.read_process(pid)
self.assertEqual(proc.process_id, pid)
self.assertEqual(proc.process_configuration, {})
self.assertEqual(proc.process_state, ProcessStateEnum.TERMINATED)
def setUp(self):
"""
Initialize test members.
Start port agent.
Start container and client.
Start streams and subscribers.
Start agent, client.
"""
TrhphTestCase.setUp(self)
self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS,
self.device_address,
self.device_port, DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._pagent = None
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
self._start_container()
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._no_samples = None
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# Start event subscribers, add stop to cleanup.
self._no_events = None
self._async_event_result = AsyncResult()
self._events_received = []
self._event_subscribers = []
self._start_event_subscribers()
self.addCleanup(self._stop_event_subscribers)
# Create agent config.
agent_config = {
'driver_config': DVR_CONFIG,
'stream_config': self._stream_config,
'agent': {
'resource_id': IA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
log.debug("TestInstrumentAgentWithTrhph.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
# make sure the driver is stopped
self.addCleanup(self._reset)
class TestInstrumentAgent(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def customCleanUp(self):
log.info('CUSTOM CLEAN UP ******************************************************************************')
def setUp(self):
"""
Setup the test environment to exersice use of instrumet agent, including:
* define driver_config parameters.
* create container with required services and container client.
* create publication stream ids for each driver data stream.
* create stream_config parameters.
* create and activate subscriptions for agent data streams.
* spawn instrument agent process and create agent client.
* add cleanup functions to cause subscribers to get stopped.
"""
# params = { ('CTD', 'TA2'): -1.9434316e-05,
# ('CTD', 'PTCA1'): 1.3206866,
# ('CTD', 'TCALDATE'): [8, 11, 2006] }
# for tup in params:
# print tup
self.addCleanup(self.customCleanUp)
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.services.mi.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
'''
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.services.mi.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': '137.110.112.119',
'device_port': 4001,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
'''
# Start container.
self._start_container()
# Establish endpoint with container (used in tests below)
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
# Create a pubsub client to create streams.
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
container=self.container)
self.subs = []
# Create streams for each stream named in driver.
self.stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = self._pubsub_client.create_stream_definition(
container=stream_def)
stream_id = self._pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self.stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = self._pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name)
self._pubsub_client.activate_subscription(sub_id)
self.subs.append(sub)
# Add cleanup function to stop subscribers.
def stop_subscriber(sub_list):
for sub in sub_list:
sub.stop()
self.addCleanup(stop_subscriber, self.subs)
# Create agent config.
self.agent_resource_id = '123xyz'
self.agent_config = {
'driver_config' : self.driver_config,
'stream_config' : self.stream_config,
'agent' : {'resource_id': self.agent_resource_id}
}
# Launch an instrument agent process.
self._ia_name = 'agent007'
self._ia_mod = 'ion.services.mi.instrument_agent'
self._ia_class = 'InstrumentAgent'
self._ia_pid = self._container_client.spawn_process(name=self._ia_name,
module=self._ia_mod, cls=self._ia_class,
config=self.agent_config)
log.info('got pid=%s', str(self._ia_pid))
self._ia_client = None
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(self.agent_resource_id, process=FakeProcess())
log.info('got ia client %s', str(self._ia_client))
def test_initialize(self):
"""
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
log.info(reply)
time.sleep(2)
caps = gw_agent_get_capabilities(self.agent_resource_id)
log.info('Capabilities: %s',str(caps))
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
log.info(reply)
def test_direct_access(self):
"""
Test agent direct_access command. This causes creation of
driver process and transition to direct access.
"""
print("test initing")
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_active")
cmd = AgentCommand(command='go_active')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test run")
cmd = AgentCommand(command='run')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_da")
cmd = AgentCommand(command='go_direct_access')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
print("reply=" + str(reply))
time.sleep(2)
print("test go_ob")
cmd = AgentCommand(command='go_observatory')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_inactive")
cmd = AgentCommand(command='go_inactive')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test reset")
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
def test_go_active(self):
"""
Test agent go_active command. This causes a driver process to
launch a connection broker, connect to device hardware, determine
entry state of driver and initialize driver parameters.
"""
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='go_inactive')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
class BaseHighAvailabilityAgentTest(IonIntegrationTestCase):
@needs_epu
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
#self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.pd_cli = ProcessDispatcherServiceClient(to_name="process_dispatcher")
self.process_definition_id = uuid4().hex
self.process_definition_name = 'test_haagent_%s' % self.process_definition_id
self.process_definition = ProcessDefinition(name=self.process_definition_name, executable={
'module': 'ion.agents.cei.test.test_haagent',
'class': 'TestProcess'
})
self.pd_cli.create_process_definition(self.process_definition, self.process_definition_id)
service_definition = SERVICE_DEFINITION_TMPL % self.process_definition_name
sd = IonObject(RT.ServiceDefinition, {"name": self.process_definition_name,
"definition": service_definition})
self.service_def_id, _ = self.container.resource_registry.create(sd)
self.resource_id = "haagent_1234"
self._haa_name = "high_availability_agent"
self._haa_dashi_name = "dashi_haa_" + uuid4().hex
self._haa_dashi_uri = get_dashi_uri_from_cfg()
self._haa_dashi_exchange = "hatests"
self._haa_config = self._get_haagent_config()
self._base_services, _ = self.container.resource_registry.find_resources(
restype="Service", name=self.process_definition_name)
self._base_procs = self.pd_cli.list_processes()
self.waiter = ProcessStateWaiter()
self.waiter.start()
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._spawn_haagent()
self._setup_haa_client()
def _get_haagent_config(self):
return {
'highavailability': {
'policy': {
'interval': 1,
'name': 'npreserving',
'parameters': {
'preserve_n': 0
}
},
'process_definition_id': self.process_definition_id,
'dashi_messaging': True,
'dashi_exchange': self._haa_dashi_exchange,
'dashi_name': self._haa_dashi_name
},
'agent': {'resource_id': self.resource_id},
}
def _setup_haa_client(self):
# Start a resource agent client to talk with the instrument agent.
self._haa_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
self.haa_client = HighAvailabilityAgentClient(self._haa_pyon_client)
def _spawn_haagent(self, policy_parameters=None):
config = deepcopy(self._haa_config)
if policy_parameters is not None:
config['highavailability']['policy']['parameters'] = policy_parameters
self._haa_pid = self.container_client.spawn_process(name=self._haa_name,
module="ion.agents.cei.high_availability_agent",
cls="HighAvailabilityAgent", config=config)
def _kill_haagent(self):
self.container.terminate_process(self._haa_pid)
def tearDown(self):
new_policy = {'preserve_n': 0}
self.haa_client.reconfigure_policy(new_policy)
self.assertEqual(len(self.get_running_procs()), 0)
self.await_ha_state('STEADY')
self.waiter.stop()
try:
self._kill_haagent()
except BadRequest:
log.warning("Couldn't terminate HA Agent in teardown (May have been terminated by a test)")
self.container.resource_registry.delete(self.service_def_id, del_associations=True)
self._stop_container()
def get_running_procs(self):
"""returns a normalized set of running procs (removes the ones that
were there at setup time)
"""
base = self._base_procs
base_pids = [proc.process_id for proc in base]
current = self.pd_cli.list_processes()
current_pids = [proc.process_id for proc in current]
print "filtering base procs %s from %s" % (base_pids, current_pids)
normal = [cproc for cproc in current if cproc.process_id not in base_pids and cproc.process_state == ProcessStateEnum.RUNNING]
return normal
def get_new_services(self):
base = self._base_services
base_names = [i.name for i in base]
services_registered, _ = self.container.resource_registry.find_resources(
restype="Service", name=self.process_definition_name)
normal = [cserv for cserv in services_registered if cserv.name not in base_names]
return normal
def await_ha_state(self, want_state, timeout=20):
for i in range(0, timeout):
try:
status = self.haa_client.status().result
if status == want_state:
return
else:
procs = self.get_running_procs()
num_procs = len(procs)
log.debug("assert wants state %s, got state %s, with %s procs" % (want_state,status, num_procs))
except Exception:
log.exception("Problem getting HA status, trying again...")
gevent.sleep(1)
raise Exception("Took more than %s to get to ha state %s" % (timeout, want_state))
def await_pyon_ha_state(self, want_state, timeout=20):
for i in range(0, timeout):
try:
result = self.haa_client.dump().result
service_id = result.get('service_id')
service = self.container.resource_registry.read(service_id)
if service.state == want_state:
return
else:
log.debug("want state %s, got state %s") % (want_state, service.state)
except Exception:
log.exception("Problem getting HA status, trying again...")
gevent.sleep(1)
raise Exception("Took more than %s to get to pyon ha state %s" % (timeout, want_state))
class HeartbeaterIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': "0.01",
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
}
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
def _start_eeagent(self):
self.container_client = ContainerAgentClient(
node=self.container.node, name=self.container.name)
self.container = self.container_client._get_container_instance()
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
@needs_eeagent
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_heartbeater(self):
"""test_heartbeater
Test whether the eeagent waits until the eeagent listener is ready before sending
a heartbeat to the PD
"""
# beat_died is a list because of a hack to get around a limitation in python 2.7
# See: http://stackoverflow.com/questions/8934772/local-var-referenced-before-assignment
beat_died = [False]
def heartbeat_callback(heartbeat, headers):
eeagent_id = heartbeat['eeagent_id']
agent_client = SimpleResourceAgentClient(eeagent_id, name=eeagent_id, process=FakeProcess())
ee_client = ExecutionEngineAgentClient(agent_client, timeout=2)
try:
ee_client.dump_state()
except:
log.exception("Heartbeat Failed!")
beat_died[0] = True
self.beat_subscriber = HeartbeatSubscriber("heartbeat_queue",
callback=heartbeat_callback, node=self.container.node)
self.beat_subscriber.start()
try:
self._start_eeagent()
for i in range(0, 5):
if beat_died[0] is True:
assert False, "A Hearbeat callback wasn't able to contact the eeagent"
gevent.sleep(0.5)
finally:
self.beat_subscriber.stop()
def _initialize(self):
"""
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
try:
"""
Get the information for the driver. This can be read from the yml
files; the user can run switch_driver to change the current driver.
"""
self.fetch_metadata()
self.fetch_driver_class()
self.fetch_comm_config()
if not exists(PIPE_PATH):
mkfifo(PIPE_PATH)
if not exists(self.metadata.driver_dir()):
raise DriverDoesNotExist(
"%s/%s/$%s" %
(self.metadata.driver_make, self.metadata.driver_model,
self.driver_name))
driver_module = DRIVER_MODULE_ROOT + self.metadata.driver_make + '.' + self.metadata.driver_model + '.' + self.metadata.driver_name + DRIVER_MODULE_LEAF
log.info('driver module: %s', driver_module)
log.info('driver class: %s', self.driver_class)
log.info('device address: %s', self.ip_address)
log.info('device data port: %s', self.data_port)
log.info('device command port: %s', self.command_port)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
DVR_CONFIG.update({
'dvr_mod': driver_module,
'dvr_cls': self.driver_class
})
"""
self._support = DriverIntegrationTestSupport(driver_module,
self.driver_class,
self.ip_address,
self.data_port,
DELIM,
WORK_DIR)
"""
# Start port agent, add stop to cleanup (not sure if that's
# necessary yet).
print(
"------------------>>>> Starting Port Agent <<<<------------------"
)
self.start_pagent()
# Start a monitor window if specified.
if self.monitor_window:
self.monitor_file = self._pagent.port_agent.logfname
strXterm = "xterm -T InstrumentMonitor -sb -rightbar"
#pOpenString = "xterm -T InstrumentMonitor -e tail -f " + self.monitor_file
pOpenString = strXterm + " -e tail -f " + self.monitor_file
x = subprocess.Popen(pOpenString, shell=True)
"""
DHE: Added self._cleanups to make base classes happy
"""
self.addCleanup(self.stop_pagent)
# Start container.
print(
"------------------>>>> Starting Capability Container <<<<------------------"
)
self._start_container()
# Bring up services in a deploy file (no need to message)
print(
"------------------>>>> Starting Deploy Services <<<<------------------"
)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Setup stream config.
self._build_stream_config()
# Create agent config.
agent_config = {
'driver_config': DVR_CONFIG,
'stream_config': self.stream_config,
'agent': {
'resource_id': IA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
print(
"------------------>>>> Starting Instrument Agent <<<<------------------"
)
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
if self.subcriber_window:
self._start_data_subscribers(6)
#self.addCleanup(self._stop_data_subscribers)
except:
log.error("initialize(): Exception occurred; shutting down.",
exc_info=True)
return False
else:
return True
class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_1234"
self._eea_name = "eeagent"
self.supd_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'supd',
'pyon_directory': os.getcwd(),
'supd_directory': self.supd_directory,
'supdexe': 'bin/supervisord'
},
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
}
}
}
# Start eeagent.
self._eea_pid = None
self.container_client = ContainerAgentClient(
node=self.container.node, name=self.container.name)
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.supd_directory)
def wait_for_state(self, upid, desired_state, timeout=5):
attempts = 0
while timeout > attempts:
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, upid)
if proc.get('state') == desired_state:
return
gevent.sleep(1)
attempts += 1
assert False, "Process %s took too long to get to %s" % (upid, desired_state)
@needs_eeagent
def test_basics(self):
true_u_pid = "test0"
round = 0
run_type = "supd"
true_parameters = {'exec': 'true', 'argv': []}
self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)
self.wait_for_state(true_u_pid, [800, 'EXITED'])
cat_u_pid = "test1"
round = 0
run_type = "supd"
cat_parameters = {'exec': 'cat', 'argv': []}
self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
self.wait_for_state(cat_u_pid, [500, 'RUNNING'])
self.eea_client.terminate_process(cat_u_pid, round)
self.wait_for_state(cat_u_pid, [700, 'TERMINATED'])
class Test2CAA(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
"""
###################################################################
# Internal parameters and container.
###################################################################
# Internal parameters.
self._terrestrial_platform_id = 'terrestrial_id'
self._remote_platform_id = 'remote_id'
self._resource_id = 'fake_id'
self._xs_name = 'remote1'
self._terrestrial_svc_name = 'terrestrial_endpoint'
self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
self._remote_svc_name = 'remote_endpoint'
self._remote_listen_name = self._remote_svc_name + self._xs_name
self._remote_port = 0
self._terrestrial_port = 0
self._te_client = None
self._re_client = None
self._remote_pid = None
self._terrestrial_pid = None
# Async test results.
self._no_requests = 10
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._telem_evts = []
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._results_recv = {}
self._requests_sent = {}
self._done_telem_evt = AsyncResult()
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
###################################################################
# Start endpoints, agent.
###################################################################
self._start_terrestrial()
self._start_remote()
self._start_agent()
###################################################################
# Assign client ports.
# This is primarily for test purposes as the IP config in
# deployment will be fixed in advance.
###################################################################
self.te_client.set_client_port(self._remote_port)
check_port = self.te_client.get_client_port()
log.debug('Terrestrial client port is: %i', check_port)
self.re_client.set_client_port(self._terrestrial_port)
check_port = self.re_client.get_client_port()
log.debug('Remote client port is: %i', check_port)
###################################################################
# Start the event publisher and subscribers.
# Used to send fake agent telemetry publications to the endpoints,
# and to receive endpoint publications.
###################################################################
self._event_publisher = EventPublisher()
# Start the event subscriber for remote namespace platform events.
# This event could be changed to RemoteNamespaceEvent.
self._event_subscriber = EventSubscriber(
event_type='PlatformEvent',
callback=self.consume_event,
origin=self._xs_name)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._event_subscriber.stop)
# Start the result subscriber for remote resource events.
self._resource_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=IA_RESOURCE_ID,
callback=self.consume_event)
self._resource_result_subscriber.start()
self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._resource_result_subscriber.stop)
# Start the result subscriber for remote service events.
self._service_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin='resource_registry' + 'remote1',
callback=self.consume_event)
self._service_result_subscriber.start()
self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._service_result_subscriber.stop)
# Start the result subscriber for fake resource results.
self._fake_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=self._resource_id,
callback=self.consume_event)
self._fake_result_subscriber.start()
self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._fake_result_subscriber.stop)
###################################################################
# Start/stop helpers.
###################################################################
def _start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _start_terrestrial(self):
"""
"""
# Create terrestrial config.
terrestrial_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._remote_port,
'this_port' : self._terrestrial_port,
'platform_resource_id' : self._terrestrial_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._terrestrial_listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._terrestrial_pid = self._container_client.spawn_process(
name=self._terrestrial_listen_name,
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=terrestrial_endpoint_config)
log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))
# Create a terrestrial client.
self.te_client = TerrestrialEndpointClient(
process=FakeProcess(),
to_name=self._terrestrial_listen_name)
log.debug('Got te client %s.', str(self.te_client))
self._terrestrial_port = self.te_client.get_port()
log.debug('Terrestrial port is: %i', self._terrestrial_port)
def _start_remote(self):
"""
"""
# Create agent config.
remote_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._terrestrial_port,
'this_port' : self._remote_port,
'platform_resource_id' : self._remote_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._remote_listen_name
}
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
self._remote_pid = self._container_client.spawn_process(
name=self._remote_listen_name,
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=remote_endpoint_config)
log.debug('Remote endpoint pid=%s.', str(self._remote_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=self._remote_listen_name)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._remote_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._remote_port)
###################################################################
# Telemetry publications to start/top endpoint.
# (Normally be published by appropriate platform agents.)
###################################################################
def terrestrial_link_up(self):
"""
Publish telemetry available to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.AVAILABLE)
def terrestrial_link_down(self):
"""
Publish telemetry unavailable to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def remote_link_up(self):
"""
Publish telemetry available to the remote endpoint.
"""
# Publish a link up event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.AVAILABLE)
def remote_link_down(self):
"""
Publish telemetry unavailable to the remote endpoint.
"""
# Publish a link down event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_event(self, evt, *args, **kwargs):
"""
Test callback for events.
"""
log.debug('Test got event: %s, args: %s, kwargs: %s',
str(evt), str(args), str(kwargs))
if evt.type_ == 'PublicPlatformTelemetryEvent':
self._telem_evts.append(evt)
if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts):
self._done_telem_evt.set()
elif evt.type_ == 'RemoteQueueModifiedEvent':
self._queue_mod_evts.append(evt)
if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts):
self._done_queue_mod_evt.set()
elif evt.type_ == 'RemoteCommandTransmittedEvent':
self._cmd_tx_evts.append(evt)
if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts):
self._done_cmd_tx_evt.set()
elif evt.type_ == 'RemoteCommandResult':
cmd = evt.command
self._results_recv[cmd.command_id] = cmd
if len(self._results_recv) == self._no_requests:
self._done_cmd_evt.set()
###################################################################
# Misc helpers.
###################################################################
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id='fake_id',
command=cmdstr,
args=['arg1', 23],
kwargs={'worktime':3})
return cmd
###################################################################
# Tests.
###################################################################
def test_queued_fake(self):
"""
test_queued_fake
Test fake resource commands queued prior to linkup.
"""
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
# The following error occurs when queue persistence is enabled.
# Need to verify correct solution to enable persistent queues.
"""
2012-11-06 14:27:13,517 INFO pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run
result = self._run(*self.args, **self.kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop
self._callback(cmd_result)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete
self._client.enqueue(result)
AttributeError: 'NoneType' object has no attribute 'enqueue'
<Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError
2012-11-06 14:27:13,586 INFO pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7
2012-11-06 14:27:13,665 INFO pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6
2012-11-06 14:27:13,739 INFO pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5
2012-11-06 14:27:13,807 INFO pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4
2012-11-06 14:27:13,874 INFO pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3
2012-11-06 14:27:13,941 INFO pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1
2012-11-06 14:27:13,945 INFO pyon.event.event EventSubscriber stopped. Event pattern=#
2012-11-06 14:27:14,124 INFO pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984
2012-11-06 14:27:14,399 INFO pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB
======================================================================
ERROR: test_process_online
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner
return f(*args, **kw)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online
cmd = self.te_client.enqueue_command(cmd)
File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command
return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request
return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request
retval, headers = e.send(msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send
result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send
return self._invite_and_send(to_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send
return self._send(to_role, to_role_name, msg, header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send
return self._end_point_unit._message_send(msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send
return ProcessRPCRequestEndpointUnit.send(self, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send
return self._send(_msg, _header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send
raise ex
Conflict: 409 - Object not based on most current version
"""
def test_process_online(self):
"""
test_process_online
Test fake resource commands queued while link is up.
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mods, command transmissions and results.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_remote_late(self):
"""
test_remote_late
Test fake resource commands queued prior to linkup.
Delay remote side linkup substantially to test terrestrial
behavior when remote server not initially available.
"""
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Actually stop the remote process, since the server is
# available even if it hasn't linked up yet and the test is running
# in the same container. The test will remember the appropriate
# remote port numbers.
self._container_client.terminate_process(self._remote_pid)
self.terrestrial_link_up()
gevent.sleep(10)
# Restart remote side and publish remote link up.
self._start_remote()
self.remote_link_up()
# Block for transmission and result events.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_resource_commands(self):
"""
test_resource_commands
"""
# Set up to verify the two commands queued.
self._no_requests = 2
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
# Use IA client to verify IA.
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Create and enqueue commands.
state_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={})
state_cmd = self.te_client.enqueue_command(state_cmd)
self._requests_sent[state_cmd.command_id] = state_cmd
ping_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='ping_agent',
args=[],
kwargs={})
ping_cmd = self.te_client.enqueue_command(ping_cmd)
self._requests_sent[ping_cmd.command_id] = ping_cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
self.assertEqual(self._results_recv[state_cmd.command_id].result,
ResourceAgentState.UNINITIALIZED)
self.assertIn('ping from InstrumentAgent',
self._results_recv[ping_cmd.command_id].result)
def test_service_command_sequence(self):
"""
test_service_commands
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='create',
args=[obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a tuple of strings.
#{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
#'1-00a308c45fff459c7cda1db9a7314de6'],
#'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a user info object with the name set.
#{'lcstate': 'DEPLOYED_AVAILABLE',
#'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
#'description': '',
#'ts_updated': '1349213207638',
#'type_': 'UserInfo',
#'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
#'_id': '27832d93f4cd4535a75ac75c06e00a7e',
#'ts_created': '1349213207638',
#'variables': [{'name': '', 'value': ''}],
#'name': 'some_name'}
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
read_obj.name = 'some_other_name'
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='update',
args=[read_obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='delete',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
class HeartbeaterIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 300,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
}
},
'agent': {
'resource_id': self.resource_id
},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
def _start_eeagent(self):
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(
self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
@needs_eeagent
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_heartbeater(self):
"""test_heartbeater
Test whether the eeagent waits until the eeagent listener is ready before sending
a heartbeat to the PD
"""
beat_died = threading.Event()
beat_succeeded = threading.Event()
def heartbeat_callback(heartbeat, headers):
eeagent_id = heartbeat['eeagent_id']
agent_client = SimpleResourceAgentClient(eeagent_id,
name=eeagent_id,
process=FakeProcess())
ee_client = ExecutionEngineAgentClient(agent_client, timeout=10)
try:
ee_client.dump_state()
beat_succeeded.set()
except:
log.exception("Heartbeat Failed!")
beat_died.set()
self.beat_subscriber = HeartbeatSubscriber("heartbeat_queue",
callback=heartbeat_callback,
node=self.container.node)
self.beat_subscriber.start()
self._start_eeagent()
success = beat_succeeded.wait(20)
if success is False:
died = beat_died.wait(20)
assert died is False, "A Hearbeat callback wasn't able to contact the eeagent"
class TestInstrumentAgent(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def customCleanUp(self):
log.info(
'CUSTOM CLEAN UP ******************************************************************************'
)
def setUp(self):
"""
Setup the test environment to exersice use of instrumet agent, including:
* define driver_config parameters.
* create container with required services and container client.
* create publication stream ids for each driver data stream.
* create stream_config parameters.
* create and activate subscriptions for agent data streams.
* spawn instrument agent process and create agent client.
* add cleanup functions to cause subscribers to get stopped.
"""
# params = { ('CTD', 'TA2'): -1.9434316e-05,
# ('CTD', 'PTCA1'): 1.3206866,
# ('CTD', 'TCALDATE'): [8, 11, 2006] }
# for tup in params:
# print tup
self.addCleanup(self.customCleanUp)
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method': 'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
'''
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': '137.110.112.119',
'device_port': 4001,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
'''
# Start container.
self._start_container()
# Establish endpoint with container (used in tests below)
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
# Create a pubsub client to create streams.
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(
process=self.container, node=self.container.node)
self.subs = []
# Create streams for each stream named in driver.
self.stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = self._pubsub_client.create_stream_definition(
container=stream_def)
stream_id = self._pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self.stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(
exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = self._pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name)
self._pubsub_client.activate_subscription(sub_id)
self.subs.append(sub)
# Add cleanup function to stop subscribers.
def stop_subscriber(sub_list):
for sub in sub_list:
sub.stop()
self.addCleanup(stop_subscriber, self.subs)
# Create agent config.
self.agent_resource_id = '123xyz'
self.agent_config = {
'driver_config': self.driver_config,
'stream_config': self.stream_config,
'agent': {
'resource_id': self.agent_resource_id
}
}
# Launch an instrument agent process.
self._ia_name = 'agent007'
self._ia_mod = 'ion.agents.instrument.instrument_agent'
self._ia_class = 'InstrumentAgent'
self._ia_pid = self._container_client.spawn_process(
name=self._ia_name,
module=self._ia_mod,
cls=self._ia_class,
config=self.agent_config)
log.info('got pid=%s', str(self._ia_pid))
self._ia_client = None
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(self.agent_resource_id,
process=FakeProcess())
log.info('got ia client %s', str(self._ia_client))
def test_initialize(self):
"""
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
log.info(reply)
time.sleep(2)
caps = gw_agent_get_capabilities(self.agent_resource_id)
log.info('Capabilities: %s', str(caps))
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
log.info(reply)
def test_direct_access(self):
"""
Test agent direct_access command. This causes creation of
driver process and transition to direct access.
"""
print("test initing")
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_active")
cmd = AgentCommand(command='go_active')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test run")
cmd = AgentCommand(command='run')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_da")
cmd = AgentCommand(command='go_direct_access')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
print("reply=" + str(reply))
time.sleep(2)
print("test go_ob")
cmd = AgentCommand(command='go_observatory')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_inactive")
cmd = AgentCommand(command='go_inactive')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test reset")
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
def test_go_active(self):
"""
Test agent go_active command. This causes a driver process to
launch a connection broker, connect to device hardware, determine
entry state of driver and initialize driver parameters.
"""
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='go_inactive')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
def test_start_hello(self):
# start a service over messaging
self._start_container()
cc_client = ContainerAgentClient(name=self.container.name)
p = cc_client.spawn_process('hello', 'ion.services.examples.hello_service', 'HelloService')
def setUp(self):
"""
Initialize test members.
Start port agent.
Start container and client.
Start streams and subscribers.
Start agent, client.
"""
TrhphTestCase.setUp(self)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
self.device_address,
self.device_port,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._pagent = None
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
self._start_container()
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._no_samples = None
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# Start event subscribers, add stop to cleanup.
self._no_events = None
self._async_event_result = AsyncResult()
self._events_received = []
self._event_subscribers = []
self._start_event_subscribers()
self.addCleanup(self._stop_event_subscribers)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
log.debug("TestInstrumentAgentWithTrhph.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
# make sure the driver is stopped
self.addCleanup(self._reset)
class TestRemoteClient(IonIntegrationTestCase):
"""
Test cases for 2CAA remote clients.
"""
def setUp(self):
"""
Setup the test parameters.
Start the container and retrieve client.
Start the endpoints and resource agent.
Start publisher and subscribers.
"""
###################################################################
# Internal parameters and container.
###################################################################
# Internal parameters.
self._terrestrial_platform_id = 'terrestrial_id'
self._remote_platform_id = 'remote_id'
self._resource_id = 'fake_id'
self._xs_name = 'remote1'
self._terrestrial_svc_name = 'terrestrial_endpoint'
self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
self._remote_svc_name = 'remote_endpoint'
self._remote_listen_name = self._remote_svc_name + self._xs_name
self._remote_port = 0
self._terrestrial_port = 0
self._te_client = None
self._re_client = None
self._remote_pid = None
self._terrestrial_pid = None
# Async test results.
self._no_requests = 10
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._telem_evts = []
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._results_recv = {}
self._requests_sent = {}
self._done_telem_evt = AsyncResult()
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
###################################################################
# Start endpoints, agent.
###################################################################
self._start_terrestrial()
self._start_remote()
self._start_agent()
###################################################################
# Assign client ports.
# This is primarily for test purposes as the IP config in
# deployment will be fixed in advance.
###################################################################
self.te_client.set_client_port(self._remote_port)
check_port = self.te_client.get_client_port()
log.debug('Terrestrial client port is: %i', check_port)
self.re_client.set_client_port(self._terrestrial_port)
check_port = self.re_client.get_client_port()
log.debug('Remote client port is: %i', check_port)
###################################################################
# Start the event publisher and subscribers.
# Used to send fake agent telemetry publications to the endpoints,
# and to receive endpoint publications.
###################################################################
self._event_publisher = EventPublisher()
# Start the event subscriber for remote namespace platform events.
# This event could be changed to RemoteNamespaceEvent.
self._event_subscriber = EventSubscriber(event_type='PlatformEvent',
callback=self.consume_event,
origin=self._xs_name)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(
timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._event_subscriber.stop)
# Start the result subscriber for remote resource events.
self._resource_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=IA_RESOURCE_ID,
callback=self.consume_event)
self._resource_result_subscriber.start()
self._resource_result_subscriber._ready_event.wait(
timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._resource_result_subscriber.stop)
# Start the result subscriber for remote service events.
self._service_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin='resource_registry' + 'remote1',
callback=self.consume_event)
self._service_result_subscriber.start()
self._service_result_subscriber._ready_event.wait(
timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._service_result_subscriber.stop)
# Start the result subscriber for fake resource results.
self._fake_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=self._resource_id,
callback=self.consume_event)
self._fake_result_subscriber.start()
self._fake_result_subscriber._ready_event.wait(
timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._fake_result_subscriber.stop)
###################################################################
# Start/stop helpers.
###################################################################
def _start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS,
DEV_ADDR, DEV_PORT,
DATA_PORT, CMD_PORT,
PA_BINARY, DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i', port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {'addr': 'localhost', 'port': port}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config': DVR_CONFIG,
'stream_config': {},
'agent': {
'resource_id': IA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _start_terrestrial(self):
"""
Start up the terrestrial endpoint.
"""
# Create terrestrial config.
terrestrial_endpoint_config = {
'other_host': 'localhost',
'other_port': self._remote_port,
'this_port': self._terrestrial_port,
'platform_resource_id': self._terrestrial_platform_id,
'xs_name': self._xs_name,
'process': {
'listen_name': self._terrestrial_listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._terrestrial_pid = self._container_client.spawn_process(
name=self._terrestrial_listen_name,
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=terrestrial_endpoint_config)
log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))
# Create a terrestrial client.
self.te_client = TerrestrialEndpointClient(
process=FakeProcess(), to_name=self._terrestrial_listen_name)
log.debug('Got te client %s.', str(self.te_client))
self._terrestrial_port = self.te_client.get_port()
log.debug('Terrestrial port is: %i', self._terrestrial_port)
def _start_remote(self):
"""
Start up the remote endpoint.
"""
# Create agent config.
remote_endpoint_config = {
'other_host': 'localhost',
'other_port': self._terrestrial_port,
'this_port': self._remote_port,
'platform_resource_id': self._remote_platform_id,
'xs_name': self._xs_name,
'process': {
'listen_name': self._remote_listen_name
}
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
self._remote_pid = self._container_client.spawn_process(
name=self._remote_listen_name,
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=remote_endpoint_config)
log.debug('Remote endpoint pid=%s.', str(self._remote_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(process=FakeProcess(),
to_name=self._remote_listen_name)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._remote_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._remote_port)
###################################################################
# Telemetry publications to start/top endpoint.
# (Normally be published by appropriate platform agents.)
###################################################################
def terrestrial_link_up(self):
"""
Publish telemetry available to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status=TelemetryStatusType.AVAILABLE)
def terrestrial_link_down(self):
"""
Publish telemetry unavailable to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status=TelemetryStatusType.UNAVAILABLE)
def remote_link_up(self):
"""
Publish telemetry available to the remote endpoint.
"""
# Publish a link up event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status=TelemetryStatusType.AVAILABLE)
def remote_link_down(self):
"""
Publish telemetry unavailable to the remote endpoint.
"""
# Publish a link down event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status=TelemetryStatusType.UNAVAILABLE)
def consume_event(self, evt, *args, **kwargs):
"""
Test callback for events.
"""
log.debug('Test got event: %s, args: %s, kwargs: %s', str(evt),
str(args), str(kwargs))
if evt.type_ == 'PublicPlatformTelemetryEvent':
self._telem_evts.append(evt)
if self._no_telem_evts > 0 and self._no_telem_evts == len(
self._telem_evts):
self._done_telem_evt.set()
elif evt.type_ == 'RemoteQueueModifiedEvent':
self._queue_mod_evts.append(evt)
if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(
self._queue_mod_evts):
self._done_queue_mod_evt.set()
elif evt.type_ == 'RemoteCommandTransmittedEvent':
self._cmd_tx_evts.append(evt)
if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(
self._cmd_tx_evts):
self._done_cmd_tx_evt.set()
elif evt.type_ == 'RemoteCommandResult':
cmd = evt.command
self._results_recv[cmd.command_id] = cmd
if len(self._results_recv) == self._no_requests:
self._done_cmd_evt.set()
###################################################################
# Misc helpers.
###################################################################
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id='fake_id',
command=cmdstr,
args=['arg1', 23],
kwargs={'kwargs1': 'someval'})
return cmd
###################################################################
# Tests.
###################################################################
def test_resource_client_online(self):
"""
test_recourse_client_online
Test the client transparently forwards commands to the remote
resource while link is up.
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
remote_client = RemoteClient(iface=IResourceAgent,
xs_name=self._xs_name,
resource_id='fake_id',
process=FakeProcess())
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = remote_client.ping_agent()
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
#@unittest.skip('For some reason this bastard wont run on the builder.')
def test_resource_client_blocking(self):
"""
test_resource_client_blocking
Test the client can block on remote resource command results.
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
remote_client = RemoteClient(iface=IResourceAgent,
xs_name=self._xs_name,
resource_id=IA_RESOURCE_ID,
process=FakeProcess())
# Queue up a series of fake commands to be handled by the remote side.
"""
{'time_completed': 1350421095.804607, 'resource_id': '123xyz',
'time_queued': 1350421095.623531, 'args': [], 'type_': 'RemoteCommand',
'command': 'ping_agent', 'result': 'ping from InstrumentAgent
(name=Agent007,id=Edwards-MacBook-Pro_local_10126.35,type=agent),
time: 1350421095757', 'kwargs': {}, 'svc_name': '',
'command_id': '76be11b4-a22c-49de-89cd-4e019463d7c9'}
"""
for i in range(self._no_requests):
cmd = remote_client.ping_agent(
remote_timeout=CFG.endpoint.receive.timeout)
self.assertEqual(cmd.resource_id, IA_RESOURCE_ID)
self.assertEqual(cmd.command, 'ping_agent')
self.assertIn('ping from InstrumentAgent', cmd.result)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
#@unittest.skip('For some reason this bastard wont run on the builder.')
def test_service_client_blocking(self):
"""
test_service_client_blocking
Test the client can command remote services and block on their
results.
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Create remote client for the resource registry.
remote_client = RemoteClient(iface=IResourceRegistryService,
xs_name=self._xs_name,
svc_name='resource_registry',
process=FakeProcess())
# Create user object.
obj = IonObject("UserInfo", name="some_name")
result = remote_client.create(
obj, remote_timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = result.result
# Confirm the results are valid.
#Result is a tuple of strings.
#{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
#'1-00a308c45fff459c7cda1db9a7314de6'],
#'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Read user object.
result = remote_client.read(
obj_id, remote_timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = result.result
# Confirm the results are valid.
#Result is a user info object with the name set.
#{'lcstate': 'DEPLOYED_AVAILABLE',
#'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
#'description': '',
#'ts_updated': '1349213207638',
#'type_': 'UserInfo',
#'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
#'_id': '27832d93f4cd4535a75ac75c06e00a7e',
#'ts_created': '1349213207638',
#'variables': [{'name': '', 'value': ''}],
#'name': 'some_name'}
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Update user object.
read_obj.name = 'some_other_name'
result = remote_client.update(
read_obj, remote_timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Read user object.
result = remote_client.read(
obj_id, remote_timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = result.result
# Confirm results are valid.
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Delete user object.
result = remote_client.delete(
obj_id, remote_timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
def test_queue_manipulators(self):
"""
test_queue_manipulators
Test ability to instpect and manipulate the command queue corresponding
to this resource or service.
"""
remote_client = RemoteClient(iface=IResourceAgent,
xs_name=self._xs_name,
resource_id='fake_id',
process=FakeProcess())
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = remote_client.ping_agent(link=False)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
queue = remote_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
popped = remote_client.clear_queue()
self.assertEqual(len(popped), self._no_requests)
self._requests_sent = {}
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = remote_client.ping_agent(link=False)
self._requests_sent[cmd.command_id] = cmd
# Pop the last three commands.
cmd_ids = self._requests_sent.keys()[:3]
poped = []
for x in cmd_ids:
poped.append(remote_client.pop_queue(x))
self._requests_sent.pop(x)
queue = remote_client.get_queue()
self.assertEqual(len(queue), self._no_requests - 3)
self.assertEqual(len(poped), 3)
self._no_requests = self._no_requests - 3
self._no_cmd_tx_evts = self._no_requests
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = remote_client.get_pending()
for x in pending:
self.assertIn(x.command_id, self._requests_sent.keys())
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_errors(self):
"""
test_errors
Test various error conditions.
"""
# Constructed without a xs name.
with self.assertRaises(ConfigNotFound):
remote_client = RemoteClient(iface=IResourceAgent,
resource_id=IA_RESOURCE_ID,
process=FakeProcess())
# Constructed without an interface.
with self.assertRaises(ConfigNotFound):
remote_client = RemoteClient(xs_name=self._xs_name,
resource_id=IA_RESOURCE_ID,
process=FakeProcess())
# Construct with invalid interface.
with self.assertRaises(ConfigNotFound):
remote_client = RemoteClient('Bogus_interface',
xs_name=self._xs_name,
resource_id=IA_RESOURCE_ID,
process=FakeProcess())
# Construct with no resource or service specified.
with self.assertRaises(ConfigNotFound):
remote_client = RemoteClient(iface=IResourceAgent,
xs_name=self._xs_name,
process=FakeProcess())
# Construct with both resource and service specified.
with self.assertRaises(ConfigNotFound):
remote_client = RemoteClient(iface=IResourceAgent,
xs_name=self._xs_name,
resource_id=IA_RESOURCE_ID,
svc_name='resource_registry',
process=FakeProcess())
# Create a valid resource client.
remote_client = RemoteClient(iface=IResourceAgent,
xs_name=self._xs_name,
resource_id=IA_RESOURCE_ID,
process=FakeProcess())
# Send a command while link is down.
with self.assertRaises(Conflict):
result = remote_client.ping_agent()
# Test port manipulators refused by remote proxy client.
with self.assertRaises(BadRequest):
remote_client.get_port()
with self.assertRaises(BadRequest):
remote_client.set_client_port()
with self.assertRaises(BadRequest):
remote_client.get_client_port()
def test_interfaces(self):
"""
test_interfaces
Test that the client declare the correct interfaces.
"""
remote_client = RemoteClient(iface=IResourceAgent,
xs_name=self._xs_name,
resource_id='fake_id',
process=FakeProcess())
interfaces = providedBy(remote_client)
self.assertIn(IResourceAgent, interfaces)
self.assertIn(ITerrestrialEndpoint, interfaces)
class Test2CAA(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
"""
###################################################################
# Internal parameters and container.
###################################################################
# Internal parameters.
self._terrestrial_platform_id = 'terrestrial_id'
self._remote_platform_id = 'remote_id'
self._resource_id = 'fake_id'
self._xs_name = 'remote1'
self._terrestrial_svc_name = 'terrestrial_endpoint'
self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
self._remote_svc_name = 'remote_endpoint'
self._remote_listen_name = self._remote_svc_name + self._xs_name
self._remote_port = 0
self._terrestrial_port = 0
self._te_client = None
self._re_client = None
self._remote_pid = None
self._terrestrial_pid = None
# Async test results.
self._no_requests = 10
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._telem_evts = []
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._results_recv = {}
self._requests_sent = {}
self._done_telem_evt = AsyncResult()
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
###################################################################
# Start endpoints, agent.
###################################################################
self._start_terrestrial()
self._start_remote()
self._start_agent()
###################################################################
# Assign client ports.
# This is primarily for test purposes as the IP config in
# deployment will be fixed in advance.
###################################################################
self.te_client.set_client_port(self._remote_port)
check_port = self.te_client.get_client_port()
log.debug('Terrestrial client port is: %i', check_port)
self.re_client.set_client_port(self._terrestrial_port)
check_port = self.re_client.get_client_port()
log.debug('Remote client port is: %i', check_port)
###################################################################
# Start the event publisher and subscribers.
# Used to send fake agent telemetry publications to the endpoints,
# and to receive endpoint publications.
###################################################################
self._event_publisher = EventPublisher()
# Start the event subscriber for remote namespace platform events.
# This event could be changed to RemoteNamespaceEvent.
self._event_subscriber = EventSubscriber(
event_type='PlatformEvent',
callback=self.consume_event,
origin=self._xs_name)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._event_subscriber.stop)
# Start the result subscriber for remote resource events.
self._resource_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=IA_RESOURCE_ID,
callback=self.consume_event)
self._resource_result_subscriber.start()
self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._resource_result_subscriber.stop)
# Start the result subscriber for remote service events.
self._service_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin='resource_registry' + 'remote1',
callback=self.consume_event)
self._service_result_subscriber.start()
self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._service_result_subscriber.stop)
# Start the result subscriber for fake resource results.
self._fake_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=self._resource_id,
callback=self.consume_event)
self._fake_result_subscriber.start()
self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._fake_result_subscriber.stop)
###################################################################
# Start/stop helpers.
###################################################################
def _start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _start_terrestrial(self):
"""
"""
# Create terrestrial config.
terrestrial_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._remote_port,
'this_port' : self._terrestrial_port,
'platform_resource_id' : self._terrestrial_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._terrestrial_listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._terrestrial_pid = self._container_client.spawn_process(
name=self._terrestrial_listen_name,
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=terrestrial_endpoint_config)
log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))
# Create a terrestrial client.
self.te_client = TerrestrialEndpointClient(
process=FakeProcess(),
to_name=self._terrestrial_listen_name)
log.debug('Got te client %s.', str(self.te_client))
self._terrestrial_port = self.te_client.get_port()
log.debug('Terrestrial port is: %i', self._terrestrial_port)
def _start_remote(self):
"""
"""
# Create agent config.
remote_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._terrestrial_port,
'this_port' : self._remote_port,
'platform_resource_id' : self._remote_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._remote_listen_name
}
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
self._remote_pid = self._container_client.spawn_process(
name=self._remote_listen_name,
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=remote_endpoint_config)
log.debug('Remote endpoint pid=%s.', str(self._remote_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=self._remote_listen_name)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._remote_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._remote_port)
###################################################################
# Telemetry publications to start/top endpoint.
# (Normally be published by appropriate platform agents.)
###################################################################
def terrestrial_link_up(self):
"""
Publish telemetry available to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.AVAILABLE)
def terrestrial_link_down(self):
"""
Publish telemetry unavailable to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def remote_link_up(self):
"""
Publish telemetry available to the remote endpoint.
"""
# Publish a link up event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.AVAILABLE)
def remote_link_down(self):
"""
Publish telemetry unavailable to the remote endpoint.
"""
# Publish a link down event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_event(self, evt, *args, **kwargs):
"""
Test callback for events.
"""
log.debug('Test got event: %s, args: %s, kwargs: %s',
str(evt), str(args), str(kwargs))
if evt.type_ == 'PublicPlatformTelemetryEvent':
self._telem_evts.append(evt)
if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts):
self._done_telem_evt.set()
elif evt.type_ == 'RemoteQueueModifiedEvent':
self._queue_mod_evts.append(evt)
if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts):
self._done_queue_mod_evt.set()
elif evt.type_ == 'RemoteCommandTransmittedEvent':
self._cmd_tx_evts.append(evt)
if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts):
self._done_cmd_tx_evt.set()
elif evt.type_ == 'RemoteCommandResult':
cmd = evt.command
self._results_recv[cmd.command_id] = cmd
if len(self._results_recv) == self._no_requests:
self._done_cmd_evt.set()
###################################################################
# Misc helpers.
###################################################################
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id='fake_id',
command=cmdstr,
args=['arg1', 23],
kwargs={'kwargs1':'someval'})
return cmd
###################################################################
# Tests.
###################################################################
def test_queued_fake(self):
"""
test_queued_fake
Test fake resource commands queued prior to linkup.
"""
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
# The following error occurs when queue persistence is enabled.
# Need to verify correct solution to enable persistent queues.
"""
2012-11-06 14:27:13,517 INFO pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run
result = self._run(*self.args, **self.kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop
self._callback(cmd_result)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete
self._client.enqueue(result)
AttributeError: 'NoneType' object has no attribute 'enqueue'
<Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError
2012-11-06 14:27:13,586 INFO pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7
2012-11-06 14:27:13,665 INFO pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6
2012-11-06 14:27:13,739 INFO pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5
2012-11-06 14:27:13,807 INFO pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4
2012-11-06 14:27:13,874 INFO pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3
2012-11-06 14:27:13,941 INFO pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1
2012-11-06 14:27:13,945 INFO pyon.event.event EventSubscriber stopped. Event pattern=#
2012-11-06 14:27:14,124 INFO pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984
2012-11-06 14:27:14,399 INFO pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB
======================================================================
ERROR: test_process_online
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner
return f(*args, **kw)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online
cmd = self.te_client.enqueue_command(cmd)
File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command
return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request
return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request
retval, headers = e.send(msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send
result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send
return self._invite_and_send(to_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send
return self._send(to_role, to_role_name, msg, header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send
return self._end_point_unit._message_send(msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send
return ProcessRPCRequestEndpointUnit.send(self, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send
return self._send(_msg, _header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send
raise ex
Conflict: 409 - Object not based on most current version
"""
def test_process_online(self):
"""
test_process_online
Test fake resource commands queued while link is up.
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mods, command transmissions and results.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_remote_late(self):
"""
test_remote_late
Test fake resource commands queued prior to linkup.
Delay remote side linkup substantially to test terrestrial
behavior when remote server not initially available.
"""
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Actually stop the remote process, since the server is
# available even if it hasn't linked up yet and the test is running
# in the same container. The test will remember the appropriate
# remote port numbers.
self._container_client.terminate_process(self._remote_pid)
self.terrestrial_link_up()
gevent.sleep(10)
# Restart remote side and publish remote link up.
self._start_remote()
self.remote_link_up()
# Block for transmission and result events.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_resource_commands(self):
"""
test_resource_commands
"""
# Set up to verify the two commands queued.
self._no_requests = 2
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
# Use IA client to verify IA.
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Create and enqueue commands.
state_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={})
state_cmd = self.te_client.enqueue_command(state_cmd)
self._requests_sent[state_cmd.command_id] = state_cmd
ping_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='ping_agent',
args=[],
kwargs={})
ping_cmd = self.te_client.enqueue_command(ping_cmd)
self._requests_sent[ping_cmd.command_id] = ping_cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
self.assertEqual(self._results_recv[state_cmd.command_id].result,
ResourceAgentState.UNINITIALIZED)
self.assertIn('ping from InstrumentAgent',
self._results_recv[ping_cmd.command_id].result)
def test_service_command_sequence(self):
"""
test_service_commands
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='create',
args=[obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a tuple of strings.
#{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
#'1-00a308c45fff459c7cda1db9a7314de6'],
#'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a user info object with the name set.
#{'lcstate': 'DEPLOYED_AVAILABLE',
#'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
#'description': '',
#'ts_updated': '1349213207638',
#'type_': 'UserInfo',
#'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
#'_id': '27832d93f4cd4535a75ac75c06e00a7e',
#'ts_created': '1349213207638',
#'variables': [{'name': '', 'value': ''}],
#'name': 'some_name'}
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
read_obj.name = 'some_other_name'
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='update',
args=[read_obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='delete',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
class BaseHighAvailabilityAgentTest(IonIntegrationTestCase):
@needs_epu
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
#self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.pd_cli = ProcessDispatcherServiceClient(
to_name="process_dispatcher")
self.process_definition_id = uuid4().hex
self.process_definition_name = 'test_haagent_%s' % self.process_definition_id
self.process_definition = ProcessDefinition(
name=self.process_definition_name,
executable={
'module': 'ion.agents.cei.test.test_haagent',
'class': 'TestProcess'
})
self.pd_cli.create_process_definition(self.process_definition,
self.process_definition_id)
service_definition = SERVICE_DEFINITION_TMPL % self.process_definition_name
sd = IonObject(RT.ServiceDefinition, {
"name": self.process_definition_name,
"definition": service_definition
})
self.service_def_id, _ = self.container.resource_registry.create(sd)
self.resource_id = "haagent_1234"
self._haa_name = "high_availability_agent"
self._haa_dashi_name = "dashi_haa_" + uuid4().hex
self._haa_dashi_uri = get_dashi_uri_from_cfg()
self._haa_dashi_exchange = "hatests"
self._haa_config = self._get_haagent_config()
self._base_services, _ = self.container.resource_registry.find_resources(
restype="Service", name=self.process_definition_name)
self._base_procs = self.pd_cli.list_processes()
self.waiter = ProcessStateWaiter()
self.waiter.start()
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._spawn_haagent()
self._setup_haa_client()
def _get_haagent_config(self):
return {
'highavailability': {
'policy': {
'interval': 1,
'name': 'npreserving',
'parameters': {
'preserve_n': 0
}
},
'process_definition_id': self.process_definition_id,
'dashi_messaging': True,
'dashi_exchange': self._haa_dashi_exchange,
'dashi_name': self._haa_dashi_name
},
'agent': {
'resource_id': self.resource_id
},
}
def _setup_haa_client(self):
# Start a resource agent client to talk with the instrument agent.
self._haa_pyon_client = SimpleResourceAgentClient(
self.resource_id, process=FakeProcess())
self.haa_client = HighAvailabilityAgentClient(self._haa_pyon_client)
def _spawn_haagent(self, policy_parameters=None):
config = deepcopy(self._haa_config)
if policy_parameters is not None:
config['highavailability']['policy'][
'parameters'] = policy_parameters
self._haa_pid = self.container_client.spawn_process(
name=self._haa_name,
module="ion.agents.cei.high_availability_agent",
cls="HighAvailabilityAgent",
config=config)
def _kill_haagent(self):
self.container.terminate_process(self._haa_pid)
def tearDown(self):
new_policy = {'preserve_n': 0}
self.haa_client.reconfigure_policy(new_policy)
self.assertEqual(len(self.get_running_procs()), 0)
self.await_ha_state('STEADY')
self.waiter.stop()
try:
self._kill_haagent()
except BadRequest:
log.warning(
"Couldn't terminate HA Agent in teardown (May have been terminated by a test)"
)
self.container.resource_registry.delete(self.service_def_id,
del_associations=True)
self._stop_container()
def get_running_procs(self):
"""returns a normalized set of running procs (removes the ones that
were there at setup time)
"""
base = self._base_procs
base_pids = [proc.process_id for proc in base]
current = self.pd_cli.list_processes()
current_pids = [proc.process_id for proc in current]
print "filtering base procs %s from %s" % (base_pids, current_pids)
normal = [
cproc for cproc in current if cproc.process_id not in base_pids
and cproc.process_state == ProcessStateEnum.RUNNING
]
return normal
def get_new_services(self):
base = self._base_services
base_names = [i.name for i in base]
services_registered, _ = self.container.resource_registry.find_resources(
restype="Service", name=self.process_definition_name)
normal = [
cserv for cserv in services_registered
if cserv.name not in base_names
]
return normal
def await_ha_state(self, want_state, timeout=20):
for i in range(0, timeout):
try:
status = self.haa_client.status().result
if status == want_state:
return
else:
procs = self.get_running_procs()
num_procs = len(procs)
log.debug(
"assert wants state %s, got state %s, with %s procs" %
(want_state, status, num_procs))
except Exception:
log.exception("Problem getting HA status, trying again...")
gevent.sleep(1)
raise Exception("Took more than %s to get to ha state %s" %
(timeout, want_state))
def await_pyon_ha_state(self, want_state, timeout=20):
for i in range(0, timeout):
try:
result = self.haa_client.dump().result
service_id = result.get('service_id')
service = self.container.resource_registry.read(service_id)
if service.state == want_state:
return
else:
log.debug("want state %s, got state %s") % (want_state,
service.state)
except Exception:
log.exception("Problem getting HA status, trying again...")
gevent.sleep(1)
raise Exception("Took more than %s to get to pyon ha state %s" %
(timeout, want_state))
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
_webserver = None
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2cei.yml")
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
"eeagent": {
"heartbeat": 1,
"slots": 100,
"name": "pyon_eeagent",
"launch_type": {"name": "pyon", "persistence_directory": self.persistence_directory},
},
"agent": {"resource_id": self.resource_id},
"logging": {
"loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
"root": {"handlers": ["console"]},
},
}
self._start_eeagent()
def _start_eeagent(self):
self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
self.container = self.container_client._get_container_instance()
# Start eeagent.
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=self.agent_config,
)
log.info("Agent pid=%s.", str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info("Got eea client %s.", str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self._stop_webserver()
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
def _start_webserver(self, directory_to_serve, port=None):
""" Start a webserver for testing code download
Note: tries really hard to get a port, and if it can't use
the suggested port, randomly picks another, and returns it
"""
def log_message(self, format, *args):
# swallow log massages
pass
class Server(HTTPServer):
requests = 0
def serve_forever(self):
self._serving = 1
while self._serving:
self.handle_request()
self.requests += 1
def stop(self):
self._serving = 0
if port is None:
port = 8008
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
Handler.log_message = log_message
for i in range(0, 100):
try:
self._webserver = Server(("localhost", port), Handler)
except socket.error:
print "port %s is in use, picking another" % port
port = randint(8000, 10000)
continue
else:
break
self._web_glet = gevent.spawn(self._webserver.serve_forever)
return port
def _stop_webserver(self):
if self._webserver is not None:
self._web_glet.kill()
def _enable_code_download(self, whitelist=None):
if whitelist is None:
whitelist = []
self.container.terminate_process(self._eea_pid)
self.agent_config["eeagent"]["code_download"] = {"enabled": True, "whitelist": whitelist}
self._start_eeagent()
def wait_for_state(self, upid, desired_state, timeout=30):
attempts = 0
last_state = None
while timeout > attempts:
try:
state = self.eea_client.dump_state().result
except Timeout:
log.warn("Timeout calling EEAgent dump_state. retrying.")
continue
proc = get_proc_for_upid(state, upid)
last_state = proc.get("state")
if last_state == desired_state:
return
gevent.sleep(1)
attempts += 1
assert False, "Process %s took too long to get to %s, had %s" % (upid, desired_state, last_state)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_x"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 1
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, "TERMINATED"])
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 1
self.eea_client.cleanup_process(u_pid, round)
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 0
@needs_eeagent
def test_restart(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_x"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 1
# Start again with incremented round. eeagent should restart the process
round += 1
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
state = self.eea_client.dump_state().result
ee_round = state["processes"][0]["round"]
assert round == int(ee_round)
# TODO: this test is disabled, as the restart op is disabled
# Run restart with incremented round. eeagent should restart the process
# round += 1
# self.eea_client.restart_process(u_pid, round)
# self.wait_for_state(u_pid, [500, 'RUNNING'])
# state = self.eea_client.dump_state().result
# ee_round = state['processes'][0]['round']
# assert round == int(ee_round)
self.eea_client.terminate_process(u_pid, round)
self.wait_for_state(u_pid, [700, "TERMINATED"])
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 1
self.eea_client.cleanup_process(u_pid, round)
state = self.eea_client.dump_state().result
assert len(state["processes"]) == 0
@needs_eeagent
def test_failing_process(self):
u_pid = "testfail"
round = 0
run_type = "pyon"
proc_name = "test_x"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcessFail"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [850, "FAILED"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_slow_to_start(self):
upids = map(lambda i: str(uuid.uuid4().hex), range(0, 10))
round = 0
run_type = "pyon"
proc_name = "test_x"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcessSlowStart"
parameters = {"name": proc_name, "module": module, "cls": cls}
for upid in upids:
self.eea_client.launch_process(upid, round, run_type, parameters)
for upid in upids:
self.wait_for_state(upid, [500, "RUNNING"], timeout=60)
@needs_eeagent
def test_start_cancel(self):
upid = str(uuid.uuid4().hex)
round = 0
run_type = "pyon"
proc_name = "test_x"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcessSlowStart"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(upid, round, run_type, parameters)
self.wait_for_state(upid, [400, "PENDING"])
self.eea_client.terminate_process(upid, round)
self.wait_for_state(upid, [700, "TERMINATED"])
@needs_eeagent
def test_kill_and_revive(self):
"""test_kill_and_revive
Ensure that when an eeagent dies, it pulls the processes it owned from
persistence, and marks them as failed, so the PD can figure out what to
do with them
"""
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_transform"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
# Kill and restart eeagent. Also, kill proc started by eea to simulate
# a killed container
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
self.assertIsNotNone(proc_to_kill)
self.container.terminate_process(proc_to_kill.id)
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
self.wait_for_state(u_pid, [850, "FAILED"])
@needs_eeagent
def test_run_out_of_slots(self):
"""test_run_out_of_slots
"""
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
self.agent_config["eeagent"]["slots"] = 1
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
u_pid_0, u_pid_1 = "test0", "test1"
round = 0
run_type = "pyon"
proc_name = "test_transform"
module = "ion.agents.cei.test.test_eeagent"
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "cls": cls}
self.eea_client.launch_process(u_pid_0, round, run_type, parameters)
self.wait_for_state(u_pid_0, [500, "RUNNING"])
self.eea_client.launch_process(u_pid_1, round, run_type, parameters)
self.wait_for_state(u_pid_1, [900, "REJECTED"])
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
self.agent_config["eeagent"]["slots"] = 1
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
self.wait_for_state(u_pid_0, [850, "FAILED"])
self.wait_for_state(u_pid_1, [900, "REJECTED"])
@needs_eeagent
def test_download_code(self):
self._enable_code_download(whitelist=["*"])
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_transform"
module = "ion.my.module.to.download"
module_uri = "file://%s/downloads/module_to_download.py" % get_this_directory()
bad_module_uri = "file:///tmp/notreal/module_to_download.py"
cls = "TestDownloadProcess"
parameters = {"name": proc_name, "module": module, "module_uri": bad_module_uri, "cls": cls}
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
print response
assert response.status == 404
assert "Unable to download" in response.result
parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_whitelist(self):
downloads_directory = os.path.join(get_this_directory(), "downloads")
http_port = 8910
http_port = self._start_webserver(downloads_directory, port=http_port)
while self._webserver is None:
print "Waiting for webserver to come up"
gevent.sleep(1)
assert self._webserver.requests == 0
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_transform"
module = "ion.my.module"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = "TestDownloadProcess"
parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
assert response.status == 401
assert "Code download not enabled" in response.result
# Test no whitelist
self._enable_code_download()
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
print response
assert response.status == 401
assert "not in code_download whitelist" in response.result
# Test not matching
self._enable_code_download(whitelist=["blork"])
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
assert response.status == 401
assert "not in code_download whitelist" in response.result
# Test exact matching
self._enable_code_download(whitelist=["localhost"])
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
# Test wildcard
self._enable_code_download(whitelist=["*"])
response = self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_caching(self):
downloads_directory = os.path.join(get_this_directory(), "downloads")
http_port = 8910
http_port = self._start_webserver(downloads_directory, port=http_port)
while self._webserver is None:
print "Waiting for webserver to come up"
gevent.sleep(1)
self._enable_code_download(["*"])
assert self._webserver.requests == 0
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = "test_transform"
module = "ion.my.module"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = "TestDownloadProcess"
parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
# Launch a process, check that webserver is hit
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
# Launch another process, check that webserver is still only hit once
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
u_pid = "test5"
round = 0
run_type = "pyon"
proc_name = "test_transformx"
module = "ion.agents.cei.test.test_eeagent"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = "TestProcess"
parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
# Test that a module that is already available in tarball won't trigger a download
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
assert self._webserver.requests == 1
u_pid = "test9"
round = 0
run_type = "pyon"
proc_name = "test_transformx"
module = "ion.agents.cei.test.test_eeagent"
module_uri = "http://localhost:%s/ion/agents/cei/test/downloads/module_to_download.py" % http_port
cls = "TestProcessNotReal"
parameters = {"name": proc_name, "module": module, "module_uri": module_uri, "cls": cls}
# Test behaviour of a non existant class with no download
self.eea_client.launch_process(u_pid, round, run_type, parameters)
self.wait_for_state(u_pid, [850, "FAILED"])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
get_proc_for_upid(state, u_pid)
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
print '#####################'
print 'IN SETUP'
self._ia_client = None
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
#log.info('driver uri: %s', DRV_URI)
#log.info('device address: %s', DEV_ADDR)
#log.info('device port: %s', DEV_PORT)
#log.info('work dir: %s', WORK_DIR)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : True,
'enable_persistence' : False
}
#if org_governance_name is not None:
# agent_config['org_governance_name'] = org_governance_name
# Start instrument agent.
log.info("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
log.info("Agent setup")
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
#self.addCleanup(self._verify_agent_reset)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
log.info('test setup complete')
class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2cei.yml")
self.resource_id = "eeagent_1234"
self._eea_name = "eeagent"
self.supd_directory = tempfile.mkdtemp()
self.agent_config = {
"eeagent": {
"heartbeat": 0,
"slots": 100,
"name": "pyon_eeagent",
"launch_type": {
"name": "supd",
"pyon_directory": os.getcwd(),
"supd_directory": self.supd_directory,
"supdexe": "bin/supervisord",
},
},
"agent": {"resource_id": self.resource_id},
"logging": {
"loggers": {"eeagent": {"level": "DEBUG", "handlers": ["console"]}},
"root": {"handlers": ["console"]},
},
}
# Start eeagent.
self._eea_pid = None
self.container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=self.agent_config,
)
log.info("Agent pid=%s.", str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info("Got eea client %s.", str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.supd_directory)
def wait_for_state(self, upid, desired_state, timeout=5):
attempts = 0
while timeout > attempts:
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, upid)
if proc.get("state") == desired_state:
return
gevent.sleep(1)
attempts += 1
assert False, "Process %s took too long to get to %s" % (upid, desired_state)
@needs_eeagent
def test_basics(self):
true_u_pid = "test0"
round = 0
run_type = "supd"
true_parameters = {"exec": "true", "argv": []}
self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)
self.wait_for_state(true_u_pid, [800, "EXITED"])
cat_u_pid = "test1"
round = 0
run_type = "supd"
cat_parameters = {"exec": "cat", "argv": []}
self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
self.wait_for_state(cat_u_pid, [500, "RUNNING"])
self.eea_client.terminate_process(cat_u_pid, round)
self.wait_for_state(cat_u_pid, [700, "TERMINATED"])
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
from ion.agents.cei.execution_engine_agent import ExecutionEngineAgentClient
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
},
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
self._start_eeagent()
def _start_eeagent(self):
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
# Start eeagent.
self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = ResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_kill_and_revive(self):
"""test_kill_and_revive
Ensure that when an eeagent dies, it pulls the processes it owned from
persistence, and marks them as failed, so the PD can figure out what to
do with them
"""
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
# Kill and restart eeagent. Also, kill proc started by eea to simulate
# a killed container
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
self.assertIsNotNone(proc_to_kill)
self.container.terminate_process(proc_to_kill.id)
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [850, 'FAILED'])
class HighAvailabilityAgentSensorPolicyTest(IonIntegrationTestCase):
def _start_webserver(self, port=None):
""" Start a webserver for testing code download
Note: tries really hard to get a port, and if it can't use
the suggested port, randomly picks another, and returns it
"""
def log_message(self, format, *args):
#swallow log massages
pass
class TestRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
server_version = 'test_server'
extensions_map = ''
def do_GET(self):
self.send_response(200)
self.send_header("Content-type", "text/plain")
self.send_header("Content-Length", len(self.server.response))
self.end_headers()
self.wfile.write(self.server.response)
class Server(HTTPServer):
response = ''
def serve_forever(self):
self._serving = 1
while self._serving:
self.handle_request()
def stop(self):
self._serving = 0
if port is None:
port = 8008
Handler = TestRequestHandler
Handler.log_message = log_message
for i in range(0, 100):
try:
self._webserver = Server(("localhost", port), Handler)
except socket.error:
print "port %s is in use, picking another" % port
port = randint(8000, 10000)
continue
else:
break
self._web_glet = gevent.spawn(self._webserver.serve_forever)
return port
def _stop_webserver(self):
if self._webserver is not None:
self._webserver.stop()
gevent.sleep(2)
self._web_glet.kill()
def await_ha_state(self, want_state, timeout=20):
for i in range(0, timeout):
try:
status = self.haa_client.status().result
if status == want_state:
return
else:
procs = self.get_running_procs()
num_procs = len(procs)
log.debug("assert wants state %s, got state %s, with %s procs" % (want_state,status, num_procs))
except Exception:
log.exception("Problem getting HA status, trying again...")
gevent.sleep(1)
raise Exception("Took more than %s to get to ha state %s" % (timeout, want_state))
@needs_epu
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.pd_cli = ProcessDispatcherServiceClient(to_name="process_dispatcher")
self.process_definition_id = uuid4().hex
self.process_definition = ProcessDefinition(name='test', executable={
'module': 'ion.agents.cei.test.test_haagent',
'class': 'TestProcess'
})
self.pd_cli.create_process_definition(self.process_definition,
self.process_definition_id)
http_port = 8919
http_port = self._start_webserver(port=http_port)
self.resource_id = "haagent_4567"
self._haa_name = "high_availability_agent"
self._haa_config = {
'server': {
'trafficsentinel': {
'host': 'localhost',
'port': http_port,
'protocol': 'http',
'username': '******',
'password': '******'
}
},
'highavailability': {
'policy': {
'interval': 1,
'name': 'sensor',
'parameters': {
'metric': 'app_attributes:ml',
'sample_period': 600,
'sample_function': 'Average',
'cooldown_period': 5,
'scale_up_threshold': 2.0,
'scale_up_n_processes': 1,
'scale_down_threshold': 1.0,
'scale_down_n_processes': 1,
'maximum_processes': 5,
'minimum_processes': 1,
}
},
'process_definition_id': self.process_definition_id,
"process_dispatchers": [
'process_dispatcher'
]
},
'agent': {'resource_id': self.resource_id},
}
self._base_procs = self.pd_cli.list_processes()
self.waiter = ProcessStateWaiter()
self.waiter.start()
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._haa_pid = self.container_client.spawn_process(name=self._haa_name,
module="ion.agents.cei.high_availability_agent",
cls="HighAvailabilityAgent", config=self._haa_config)
# Start a resource agent client to talk with the instrument agent.
self._haa_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got haa client %s.', str(self._haa_pyon_client))
self.haa_client = HighAvailabilityAgentClient(self._haa_pyon_client)
def tearDown(self):
new_policy = { 'metric': 'app_attributes:ml',
'sample_period': 600,
'sample_function': 'Average',
'cooldown_period': 0,
'scale_up_threshold': 2.0,
'scale_up_n_processes': 1,
'scale_down_threshold': 1.0,
'scale_down_n_processes': 1,
'maximum_processes': 0,
'minimum_processes': 0,
}
self.haa_client.reconfigure_policy(new_policy)
self.waiter.await_state_event(state=ProcessStateEnum.TERMINATED)
self.assertEqual(len(self.get_running_procs()), 0)
self.waiter.stop()
self.container.terminate_process(self._haa_pid)
self._stop_webserver()
self._stop_container()
def get_running_procs(self):
"""returns a normalized set of running procs (removes the ones that
were there at setup time)
"""
base = self._base_procs
base_pids = [proc.process_id for proc in base]
current = self.pd_cli.list_processes()
current_pids = [proc.process_id for proc in current]
print "filtering base procs %s from %s" % (base_pids, current_pids)
normal = [cproc for cproc in current if cproc.process_id not in base_pids and cproc.process_state == ProcessStateEnum.RUNNING]
return normal
def _get_managed_upids(self):
result = self.haa_client.dump().result
upids = result['managed_upids']
return upids
def _set_response(self, response):
self._webserver.response = response
def test_sensor_policy(self):
status = self.haa_client.status().result
# Ensure HA hasn't already failed
assert status in ('PENDING', 'READY', 'STEADY')
self.waiter.await_state_event(state=ProcessStateEnum.RUNNING)
self.assertEqual(len(self.get_running_procs()), 1)
self.await_ha_state('STEADY')
# Set ml for each proc such that we scale up
upids = self._get_managed_upids()
response = ""
for upid in upids:
response += "pid=%s&ml=5\n" % upid
self._set_response(response)
self.waiter.await_state_event(state=ProcessStateEnum.RUNNING)
self.assertEqual(len(self.get_running_procs()), 2)
# Set ml so we stay steady
upids = self._get_managed_upids()
response = ""
for upid in upids:
response += "pid=%s&ml=1.5\n" % upid
self._set_response(response)
self.assertEqual(len(self.get_running_procs()), 2)
self.await_ha_state('STEADY')
# Set ml so we scale down
upids = self._get_managed_upids()
response = ""
for upid in upids:
response += "pid=%s&ml=0.5\n" % upid
self._set_response(response)
self.waiter.await_state_event(state=ProcessStateEnum.TERMINATED)
self.assertEqual(len(self.get_running_procs()), 1)
self.await_ha_state('STEADY')
class ExecutionEngineAgentSupdIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_1234"
self._eea_name = "eeagent"
self.supd_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'supd',
'pyon_directory': os.getcwd(),
'supd_directory': self.supd_directory,
'supdexe': 'bin/supervisord'
},
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
# Start eeagent.
self._eea_pid = None
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._eea_pid = self.container_client.spawn_process(name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = ResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.supd_directory)
@needs_eeagent
def test_basics(self):
true_u_pid = "test0"
round = 0
run_type = "supd"
true_parameters = {'exec': 'true', 'argv': []}
self.eea_client.launch_process(true_u_pid, round, run_type, true_parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, true_u_pid)
self.assertEqual(proc.get('state'), [800, 'EXITED'])
cat_u_pid = "test1"
round = 0
run_type = "supd"
cat_parameters = {'exec': 'cat', 'argv': []}
self.eea_client.launch_process(cat_u_pid, round, run_type, cat_parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, cat_u_pid)
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
self.eea_client.terminate_process(cat_u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, cat_u_pid)
self.assertEqual(proc.get('state'), [700, 'TERMINATED'])
class HighAvailabilityAgentSensorPolicyTest(IonIntegrationTestCase):
def _start_webserver(self, port=None):
""" Start a webserver for testing code download
Note: tries really hard to get a port, and if it can't use
the suggested port, randomly picks another, and returns it
"""
def log_message(self, format, *args):
#swallow log massages
pass
class TestRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
server_version = 'test_server'
extensions_map = ''
def do_GET(self):
self.send_response(200)
self.send_header("Content-type", "text/plain")
self.send_header("Content-Length", len(self.server.response))
self.end_headers()
self.wfile.write(self.server.response)
class Server(HTTPServer):
response = ''
def serve_forever(self):
self._serving = 1
while self._serving:
self.handle_request()
def stop(self):
self._serving = 0
if port is None:
port = 8008
Handler = TestRequestHandler
Handler.log_message = log_message
for i in range(0, 100):
try:
self._webserver = Server(("localhost", port), Handler)
except socket.error:
print "port %s is in use, picking another" % port
port = randint(8000, 10000)
continue
else:
break
self._web_glet = gevent.spawn(self._webserver.serve_forever)
return port
def _stop_webserver(self):
if self._webserver is not None:
self._webserver.stop()
gevent.sleep(2)
self._web_glet.kill()
@needs_epu
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.pd_cli = ProcessDispatcherServiceClient(to_name="process_dispatcher")
self.process_definition_id = uuid4().hex
self.process_definition = ProcessDefinition(name='test', executable={
'module': 'ion.agents.cei.test.test_haagent',
'class': 'TestProcess'
})
self.pd_cli.create_process_definition(self.process_definition,
self.process_definition_id)
http_port = 8919
http_port = self._start_webserver(port=http_port)
self.resource_id = "haagent_4567"
self._haa_name = "high_availability_agent"
self._haa_config = {
'highavailability': {
'policy': {
'interval': 1,
'name': 'sensor',
'parameters': {
'metric': 'app_attributes:ml',
'sample_period': 600,
'sample_function': 'Average',
'cooldown_period': 20,
'scale_up_threshold': 2.0,
'scale_up_n_processes': 1,
'scale_down_threshold': 1.0,
'scale_down_n_processes': 1,
'maximum_processes': 5,
'minimum_processes': 1,
}
},
'aggregator': {
'type': 'trafficsentinel',
'host': 'localhost',
'port': http_port,
'protocol': 'http',
'username': '******',
'password': '******'
},
'process_definition_id': self.process_definition_id,
"process_dispatchers": [
'process_dispatcher'
]
},
'agent': {'resource_id': self.resource_id},
}
self._base_procs = self.pd_cli.list_processes()
self.waiter = ProcessStateWaiter()
self.waiter.start()
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._haa_pid = self.container_client.spawn_process(name=self._haa_name,
module="ion.agents.cei.high_availability_agent",
cls="HighAvailabilityAgent", config=self._haa_config)
# Start a resource agent client to talk with the instrument agent.
self._haa_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got haa client %s.', str(self._haa_pyon_client))
self.haa_client = HighAvailabilityAgentClient(self._haa_pyon_client)
def tearDown(self):
self.waiter.stop()
self.container.terminate_process(self._haa_pid)
self._stop_webserver()
self._stop_container()
def get_running_procs(self):
"""returns a normalized set of running procs (removes the ones that
were there at setup time)
"""
base = self._base_procs
base_pids = [proc.process_id for proc in base]
current = self.pd_cli.list_processes()
current_pids = [proc.process_id for proc in current]
print "filtering base procs %s from %s" % (base_pids, current_pids)
normal = [cproc for cproc in current if cproc.process_id not in base_pids and cproc.process_state == ProcessStateEnum.RUNNING]
return normal
def _get_managed_upids(self):
result = self.haa_client.dump().result
upids = result['managed_upids']
return upids
def _set_response(self, response):
self._webserver.response = response
def test_sensor_policy(self):
status = self.haa_client.status().result
# Ensure HA hasn't already failed
assert status in ('PENDING', 'READY', 'STEADY')
self.waiter.await_state_event(state=ProcessStateEnum.RUNNING)
self.assertEqual(len(self.get_running_procs()), 1)
for i in range(0, 5):
status = self.haa_client.status().result
try:
self.assertEqual(status, 'STEADY')
break
except:
gevent.sleep(1)
else:
assert False, "HA Service took too long to get to state STEADY"
# Set ml for each proc such that we scale up
upids = self._get_managed_upids()
response = ""
for upid in upids:
response += "%s,ml=5\n"
self._set_response(response)
self.waiter.await_state_event(state=ProcessStateEnum.RUNNING)
self.assertEqual(len(self.get_running_procs()), 2)
# Set ml so we stay steady
upids = self._get_managed_upids()
response = ""
for upid in upids:
response += "%s,ml=1.5\n"
self._set_response(response)
self.assertEqual(len(self.get_running_procs()), 2)
for i in range(0, 5):
status = self.haa_client.status().result
try:
self.assertEqual(status, 'STEADY')
break
except:
gevent.sleep(1)
else:
assert False, "HA Service took too long to get to state STEADY"
# Set ml so we scale down
upids = self._get_managed_upids()
response = ""
for upid in upids:
response += "%s,ml=0.5\n"
self._set_response(response)
self.waiter.await_state_event(state=ProcessStateEnum.TERMINATED)
self.assertEqual(len(self.get_running_procs()), 1)
for i in range(0, 5):
status = self.haa_client.status().result
try:
self.assertEqual(status, 'STEADY')
break
except:
gevent.sleep(1)
else:
assert False, "HA Service took too long to get to state STEADY"
def instrument_test_driver(container):
sa_user_header = container.governance_controller.get_system_actor_header()
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'mi.instrument.seabird.sbe37smb.ooicore.driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
_container_client = ContainerAgentClient(node=container.node,
name=container.name)
# Create a pubsub client to create streams.
_pubsub_client = PubsubManagementServiceClient(node=container.node)
# A callback for processing subscribed-to data.
def consume(message, *args, **kwargs):
log.info('Subscriber received message: %s', str(message))
subs = []
# Create streams for each stream named in driver.
stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
#@TODO: Figure out what goes inside this stream def, do we have a pdict?
stream_def_id = _pubsub_client.create_stream_definition(
name='instrument stream def')
stream_id, route = _pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
exchange_point='science_data')
stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = StandaloneStreamSubscriber(exchange_name=exchange_name, callback=consume)
sub.start()
sub_id = _pubsub_client.create_subscription(\
name=exchange_name,
stream_ids=[stream_id])
_pubsub_client.activate_subscription(sub_id)
subs.append(sub)
# Create agent config.
agent_resource_id = '123xyz'
agent_config = {
'driver_config' : driver_config,
'stream_config' : stream_config,
'agent' : {'resource_id': agent_resource_id}
}
# Launch an instrument agent process.
_ia_name = 'agent007'
_ia_mod = 'ion.agents.instrument.instrument_agent'
_ia_class = 'InstrumentAgent'
_ia_pid = _container_client.spawn_process(name=_ia_name,
module=_ia_mod, cls=_ia_class,
config=agent_config)
log.info('got pid=%s for resource_id=%s' % (str(_ia_pid), str(agent_resource_id)))
class HighAvailabilityAgentTest(IonIntegrationTestCase):
@needs_epu
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
#self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.pd_cli = ProcessDispatcherServiceClient(to_name="process_dispatcher")
self.process_definition_id = uuid4().hex
self.process_definition_name = 'test'
self.process_definition = ProcessDefinition(name=self.process_definition_name, executable={
'module': 'ion.agents.cei.test.test_haagent',
'class': 'TestProcess'
})
self.pd_cli.create_process_definition(self.process_definition, self.process_definition_id)
self.resource_id = "haagent_1234"
self._haa_name = "high_availability_agent"
self._haa_dashi_name = "dashi_haa_" + uuid4().hex
self._haa_dashi_uri = get_dashi_uri_from_cfg()
self._haa_dashi_exchange = "%s.hatests" % bootstrap.get_sys_name()
self._haa_config = {
'highavailability': {
'policy': {
'interval': 1,
'name': 'npreserving',
'parameters': {
'preserve_n': 0
}
},
'process_definition_id': self.process_definition_id,
'dashi_messaging' : True,
'dashi_exchange' : self._haa_dashi_exchange,
'dashi_name': self._haa_dashi_name
},
'agent': {'resource_id': self.resource_id},
}
self._base_services, _ = self.container.resource_registry.find_resources(
restype="Service", name=self.process_definition_name)
self._base_procs = self.pd_cli.list_processes()
self.waiter = ProcessStateWaiter()
self.waiter.start()
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._haa_pid = self.container_client.spawn_process(name=self._haa_name,
module="ion.agents.cei.high_availability_agent",
cls="HighAvailabilityAgent", config=self._haa_config)
# Start a resource agent client to talk with the instrument agent.
self._haa_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got haa client %s.', str(self._haa_pyon_client))
self.haa_client = HighAvailabilityAgentClient(self._haa_pyon_client)
def tearDown(self):
self.waiter.stop()
try:
self.container.terminate_process(self._haa_pid)
except BadRequest:
log.warning("Couldn't terminate HA Agent in teardown (May have been terminated by a test)")
self._stop_container()
def get_running_procs(self):
"""returns a normalized set of running procs (removes the ones that
were there at setup time)
"""
base = self._base_procs
base_pids = [proc.process_id for proc in base]
current = self.pd_cli.list_processes()
current_pids = [proc.process_id for proc in current]
print "filtering base procs %s from %s" % (base_pids, current_pids)
normal = [cproc for cproc in current if cproc.process_id not in base_pids and cproc.process_state == ProcessStateEnum.RUNNING]
return normal
def get_new_services(self):
base = self._base_services
base_names = [i.name for i in base]
services_registered, _ = self.container.resource_registry.find_resources(
restype="Service", name=self.process_definition_name)
current_names = [i.name for i in services_registered]
normal = [cserv for cserv in services_registered if cserv.name not in base_names]
return normal
def await_ha_state(self, want_state, timeout=10):
for i in range(0, timeout):
status = self.haa_client.status().result
if status == want_state:
return
gevent.sleep(1)
raise Exception("Took more than %s to get to ha state %s" % (timeout, want_state))
def test_features(self):
status = self.haa_client.status().result
# Ensure HA hasn't already failed
assert status in ('PENDING', 'READY', 'STEADY')
# verifies L4-CI-CEI-RQ44
# Note: the HA agent is started in the setUp() method, with config
# pointing to the test "service". The initial config is set to preserve
# 0 service processes. With this reconfigure step below, we change that
# to launch 1.
new_policy = {'preserve_n': 1}
self.haa_client.reconfigure_policy(new_policy)
result = self.haa_client.dump().result
self.assertEqual(result['policy'], new_policy)
self.waiter.await_state_event(state=ProcessStateEnum.RUNNING)
self.assertEqual(len(self.get_running_procs()), 1)
for i in range(0, 5):
status = self.haa_client.status().result
try:
self.assertEqual(status, 'STEADY')
break
except:
gevent.sleep(1)
else:
assert False, "HA Service took too long to get to state STEADY"
# verifies L4-CI-CEI-RQ122 and L4-CI-CEI-RQ124
new_policy = {'preserve_n': 2}
self.haa_client.reconfigure_policy(new_policy)
self.waiter.await_state_event(state=ProcessStateEnum.RUNNING)
self.assertEqual(len(self.get_running_procs()), 2)
new_policy = {'preserve_n': 1}
self.haa_client.reconfigure_policy(new_policy)
self.waiter.await_state_event(state=ProcessStateEnum.TERMINATED)
self.assertEqual(len(self.get_running_procs()), 1)
new_policy = {'preserve_n': 0}
self.haa_client.reconfigure_policy(new_policy)
self.waiter.await_state_event(state=ProcessStateEnum.TERMINATED)
self.assertEqual(len(self.get_running_procs()), 0)
def test_associations(self):
# Ensure that once the HA Agent starts, there is a Service object in
# the registry
result = self.haa_client.dump().result
service_id = result.get('service_id')
self.assertIsNotNone(service_id)
service = self.container.resource_registry.read(service_id)
self.assertIsNotNone(service)
# Ensure that once a process is started, there is an association between
# it and the service
new_policy = {'preserve_n': 1}
self.haa_client.reconfigure_policy(new_policy)
self.waiter.await_state_event(state=ProcessStateEnum.RUNNING)
self.assertEqual(len(self.get_running_procs()), 1)
self.await_ha_state('STEADY')
proc = self.get_running_procs()[0]
processes_associated, _ = self.container.resource_registry.find_resources(
restype="Process", name=proc.process_id)
self.assertEqual(len(processes_associated), 1)
has_processes = self.container.resource_registry.find_associations(
service, "hasProcess")
self.assertEqual(len(has_processes), 1)
self.await_ha_state('STEADY')
# Ensure that once we terminate that process, there are no associations
new_policy = {'preserve_n': 0}
self.haa_client.reconfigure_policy(new_policy)
self.waiter.await_state_event(state=ProcessStateEnum.TERMINATED)
self.assertEqual(len(self.get_running_procs()), 0)
processes_associated, _ = self.container.resource_registry.find_resources(
restype="Process", name=proc.process_id)
self.assertEqual(len(processes_associated), 0)
has_processes = self.container.resource_registry.find_associations(
service, "hasProcess")
self.assertEqual(len(has_processes), 0)
# Ensure that once we terminate that HA Agent, the Service object is
# cleaned up
self.container.terminate_process(self._haa_pid)
with self.assertRaises(NotFound):
service = self.container.resource_registry.read(service_id)
def test_dashi(self):
import dashi
dashi_conn = dashi.DashiConnection("something", self._haa_dashi_uri,
self._haa_dashi_exchange)
status = dashi_conn.call(self._haa_dashi_name, "status")
assert status in ('PENDING', 'READY', 'STEADY')
new_policy = {'preserve_n': 0}
dashi_conn.call(self._haa_dashi_name, "reconfigure_policy",
new_policy=new_policy)
class HeartbeaterIntTest(IonIntegrationTestCase):
@needs_eeagent
def setUp(self):
self._start_container()
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 300,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
}
},
'agent': {'resource_id': self.resource_id},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
def _start_eeagent(self):
self.container_client = ContainerAgentClient(
node=self.container.node, name=self.container.name)
self.container = self.container_client._get_container_instance()
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
@needs_eeagent
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_heartbeater(self):
"""test_heartbeater
Test whether the eeagent waits until the eeagent listener is ready before sending
a heartbeat to the PD
"""
beat_died = threading.Event()
beat_succeeded = threading.Event()
def heartbeat_callback(heartbeat, headers):
eeagent_id = heartbeat['eeagent_id']
agent_client = SimpleResourceAgentClient(eeagent_id, name=eeagent_id, process=FakeProcess())
ee_client = ExecutionEngineAgentClient(agent_client, timeout=10)
try:
ee_client.dump_state()
beat_succeeded.set()
except:
log.exception("Heartbeat Failed!")
beat_died.set()
self.beat_subscriber = HeartbeatSubscriber("heartbeat_queue",
callback=heartbeat_callback, node=self.container.node)
self.beat_subscriber.start()
self._start_eeagent()
success = beat_succeeded.wait(20)
if success is False:
died = beat_died.wait(20)
assert died is False, "A Hearbeat callback wasn't able to contact the eeagent"
def setUp(self):
"""
Start fake terrestrial components and add cleanup.
Start terrestrial server and retrieve port.
Set internal variables.
Start container.
Start deployment.
Start container agent.
Spawn remote endpoint process.
Create remote endpoint client and retrieve remote server port.
Create event publisher.
"""
self._terrestrial_server = R3PCServer(self.consume_req, self.terrestrial_server_close)
self._terrestrial_client = R3PCClient(self.consume_ack, self.terrestrial_client_close)
self.addCleanup(self._terrestrial_server.stop)
self.addCleanup(self._terrestrial_client.stop)
self._other_port = self._terrestrial_server.start('*', 0)
log.debug('Terrestrial server binding to *:%i', self._other_port)
self._other_host = 'localhost'
self._platform_resource_id = 'abc123'
self._resource_id = 'fake_id'
self._no_requests = 10
self._requests_sent = {}
self._results_recv = {}
self._no_telem_events = 0
self._done_evt = AsyncResult()
self._done_telem_evts = AsyncResult()
self._cmd_tx_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Create agent config.
endpoint_config = {
'other_host' : self._other_host,
'other_port' : self._other_port,
'this_port' : 0,
'platform_resource_id' : self._platform_resource_id
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
re_pid = container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(re_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=re_pid)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._this_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._this_port)
# Start the event publisher.
self._event_publisher = EventPublisher()
class ProcessDispatcherEEAgentIntTest(ProcessDispatcherServiceIntTest):
"""Run the basic int tests again, with a different environment
"""
def setUp(self):
self.dashi = None
self._start_container()
from pyon.public import CFG
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
app = dict(name="process_dispatcher", processapp=("process_dispatcher",
"ion.services.cei.process_dispatcher_service",
"ProcessDispatcherService"))
self.container.start_app(app, config=pd_config)
self.rr_cli = self.container.resource_registry
self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.process_definition = ProcessDefinition(name='test_process')
self.process_definition.executable = {'module': 'ion.services.cei.test.test_process_dispatcher',
'class': 'TestProcess'}
self.process_definition_id = self.pd_cli.create_process_definition(self.process_definition)
self._eea_pids = []
self._eea_pid_to_resource_id = {}
self._eea_pid_to_persistence_dir = {}
self._tmpdirs = []
self.dashi = get_dashi(uuid.uuid4().hex,
pd_config['processdispatcher']['dashi_uri'],
pd_config['processdispatcher']['dashi_exchange'],
sysname=CFG.get_safe("dashi.sysname")
)
#send a fake node_state message to PD's dashi binding.
self.node1_id = uuid.uuid4().hex
self._send_node_state("engine1", self.node1_id)
self._initial_eea_pid = self._start_eeagent(self.node1_id)
self.waiter = ProcessStateWaiter()
def _send_node_state(self, engine_id, node_id=None):
node_id = node_id or uuid.uuid4().hex
node_state = dict(node_id=node_id, state=InstanceState.RUNNING,
domain_id=domain_id_from_engine(engine_id))
self.dashi.fire(get_pd_dashi_name(), "node_state", args=node_state)
def _start_eeagent(self, node_id, resource_id=None, persistence_dir=None):
if not persistence_dir:
persistence_dir = tempfile.mkdtemp()
self._tmpdirs.append(persistence_dir)
resource_id = resource_id or uuid.uuid4().hex
agent_config = _get_eeagent_config(node_id, persistence_dir,
resource_id=resource_id)
pid = self.container_client.spawn_process(name="eeagent",
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent", config=agent_config)
log.info('Agent pid=%s.', str(pid))
self._eea_pids.append(pid)
self._eea_pid_to_resource_id[pid] = resource_id
self._eea_pid_to_persistence_dir[pid] = persistence_dir
return pid
def _kill_eeagent(self, pid):
self.assertTrue(pid in self._eea_pids)
self.container.terminate_process(pid)
self._eea_pids.remove(pid)
del self._eea_pid_to_resource_id[pid]
del self._eea_pid_to_persistence_dir[pid]
def tearDown(self):
for pid in list(self._eea_pids):
self._kill_eeagent(pid)
for d in self._tmpdirs:
shutil.rmtree(d)
self.waiter.stop()
if self.dashi:
self.dashi.cancel()
def test_requested_ee(self):
# request non-default engine
process_target = ProcessTarget(execution_engine_id="engine2")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
process_schedule.target = process_target
pid = self.pd_cli.create_process(self.process_definition_id)
self.waiter.start()
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid)
self.waiter.await_state_event(pid, ProcessStateEnum.WAITING)
# request unknown engine, with NEVER queuing mode. The request
# should be rejected.
# verifies L4-CI-CEI-RQ52
process_target = ProcessTarget(execution_engine_id="not-a-real-ee")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.NEVER
process_schedule.target = process_target
rejected_pid = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=rejected_pid)
self.waiter.await_state_event(rejected_pid, ProcessStateEnum.REJECTED)
# now add a node and eeagent for engine2. original process should leave
# queue and start running
node2_id = uuid.uuid4().hex
self._send_node_state("engine2", node2_id)
self._start_eeagent(node2_id)
self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING)
# spawn another process. it should start immediately.
process_target = ProcessTarget(execution_engine_id="engine2")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.NEVER
process_schedule.target = process_target
pid2 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.RUNNING)
# one more with node exclusive
process_target = ProcessTarget(execution_engine_id="engine2",
node_exclusive="hats")
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.NEVER
process_schedule.target = process_target
pid3 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.RUNNING)
# kill the processes for good
self.pd_cli.cancel_process(pid)
self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.TERMINATED)
def test_node_exclusive(self):
# the node_exclusive constraint is used to ensure multiple processes
# of the same "kind" each get a VM exclusive of each other. Other
# processes may run on these VMs, just not processes with the same
# node_exclusive tag. Since we cannot directly query the contents
# of each node in this test, we prove the capability by scheduling
# processes one by one and checking their state.
# verifies L4-CI-CEI-RQ121
# verifies L4-CI-CEI-RQ57
# first off, setUp() created a single node and eeagent.
# We schedule two processes with the same "abc" node_exclusive
# tag. Since there is only one node, the first process should run
# and the second should be queued.
process_target = ProcessTarget(execution_engine_id="engine1")
process_target.node_exclusive = "abc"
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
process_schedule.target = process_target
pid1 = self.pd_cli.create_process(self.process_definition_id)
self.waiter.start()
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid1)
self.waiter.await_state_event(pid1, ProcessStateEnum.RUNNING)
pid2 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.WAITING)
# now demonstrate that the node itself is not full by launching
# a third process without a node_exclusive tag -- it should start
# immediately
process_target.node_exclusive = None
pid3 = self.pd_cli.create_process(self.process_definition_id)
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.RUNNING)
# finally, add a second node to the engine. pid2 should be started
# since there is an exclusive "abc" node free.
node2_id = uuid.uuid4().hex
self._send_node_state("engine1", node2_id)
self._start_eeagent(node2_id)
self.waiter.await_state_event(pid2, ProcessStateEnum.RUNNING)
# kill the processes for good
self.pd_cli.cancel_process(pid1)
self.waiter.await_state_event(pid1, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid2)
self.waiter.await_state_event(pid2, ProcessStateEnum.TERMINATED)
self.pd_cli.cancel_process(pid3)
self.waiter.await_state_event(pid3, ProcessStateEnum.TERMINATED)
def test_code_download(self):
# create a process definition that has no URL; only module and class.
process_definition_no_url = ProcessDefinition(name='test_process_nodownload')
process_definition_no_url.executable = {'module': 'ion.my.test.process',
'class': 'TestProcess'}
process_definition_id_no_url = self.pd_cli.create_process_definition(process_definition_no_url)
# create another that has a URL of the python file (this very file)
# verifies L4-CI-CEI-RQ114
url = "file://%s" % os.path.join(os.path.dirname(__file__), 'test_process_dispatcher.py')
process_definition = ProcessDefinition(name='test_process_download')
process_definition.executable = {'module': 'ion.my.test.process',
'class': 'TestProcess', 'url': url}
process_definition_id = self.pd_cli.create_process_definition(process_definition)
process_target = ProcessTarget()
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
process_schedule.target = process_target
self.waiter.start()
# Test a module with no download fails
pid_no_url = self.pd_cli.create_process(process_definition_id_no_url)
self.pd_cli.schedule_process(process_definition_id_no_url,
process_schedule, process_id=pid_no_url)
self.waiter.await_state_event(pid_no_url, ProcessStateEnum.FAILED)
# Test a module with a URL runs
pid = self.pd_cli.create_process(process_definition_id)
self.pd_cli.schedule_process(process_definition_id,
process_schedule, process_id=pid)
self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING)
def _add_test_process(self, restart_mode=None):
process_schedule = ProcessSchedule()
if restart_mode is not None:
process_schedule.restart_mode = restart_mode
pid = self.pd_cli.create_process(self.process_definition_id)
pid_listen_name = "PDtestproc_%s" % uuid.uuid4().hex
config = {'process': {'listen_name': pid_listen_name}}
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, process_id=pid, configuration=config)
client = TestClient(to_name=pid_listen_name)
return pid, client
def test_restart(self):
self.waiter.start()
restartable_pids = []
nonrestartable_pids = []
clients = {}
# start 10 processes with RestartMode.ALWAYS
for _ in range(10):
pid, client = self._add_test_process(ProcessRestartMode.ALWAYS)
restartable_pids.append(pid)
clients[pid] = client
# and 10 processes with RestartMode.ABNORMAL
for _ in range(10):
pid, client = self._add_test_process(ProcessRestartMode.ABNORMAL)
restartable_pids.append(pid)
clients[pid] = client
# and 10 with RestartMode.NEVER
for _ in range(10):
pid, client = self._add_test_process(ProcessRestartMode.NEVER)
nonrestartable_pids.append(pid)
clients[pid] = client
all_pids = restartable_pids + nonrestartable_pids
self.waiter.await_many_state_events(all_pids, ProcessStateEnum.RUNNING)
for pid in all_pids:
client = clients[pid]
self.assertFalse(client.is_restart())
self.assertEqual(client.count(), 1)
# now kill the whole eeagent and restart it. processes should
# show up as FAILED in the next heartbeat.
resource_id = self._eea_pid_to_resource_id[self._initial_eea_pid]
persistence_dir = self._eea_pid_to_persistence_dir[self._initial_eea_pid]
log.debug("Restarting eeagent %s", self._initial_eea_pid)
self._kill_eeagent(self._initial_eea_pid)
# manually kill the processes to simulate a real container failure
for pid in all_pids:
self.container.terminate_process(pid)
self._start_eeagent(self.node1_id, resource_id=resource_id,
persistence_dir=persistence_dir)
# wait for restartables to restart
self.waiter.await_many_state_events(restartable_pids, ProcessStateEnum.RUNNING)
# query the processes again. it should have restart mode config
for pid in restartable_pids:
client = clients[pid]
self.assertTrue(client.is_restart())
self.assertEqual(client.count(), 1)
# meanwhile some procs should not have restarted
for pid in nonrestartable_pids:
proc = self.pd_cli.read_process(pid)
self.assertEqual(proc.process_state, ProcessStateEnum.FAILED)
# guard against extraneous events we were receiving as part of a bug:
# processes restarting again after they were already restarted
self.waiter.await_nothing(timeout=5)
def test_idempotency(self):
# ensure every operation can be safely retried
process_schedule = ProcessSchedule()
process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS
proc_name = 'myreallygoodname'
pid = self.pd_cli.create_process(self.process_definition_id)
self.waiter.start(pid)
# note: if we import UNSCHEDULED state into ProcessStateEnum,
# this assertion will need to change.
proc = self.pd_cli.read_process(pid)
self.assertEqual(proc.process_id, pid)
self.assertEqual(proc.process_state, ProcessStateEnum.REQUESTED)
pid2 = self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, configuration={}, process_id=pid, name=proc_name)
self.assertEqual(pid, pid2)
self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING)
# repeating schedule is harmless
pid2 = self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, configuration={}, process_id=pid, name=proc_name)
self.assertEqual(pid, pid2)
proc = self.pd_cli.read_process(pid)
self.assertEqual(proc.process_id, pid)
self.assertEqual(proc.process_configuration, {})
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)
self.pd_cli.cancel_process(pid)
self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED)
# repeating cancel is harmless
self.pd_cli.cancel_process(pid)
proc = self.pd_cli.read_process(pid)
self.assertEqual(proc.process_id, pid)
self.assertEqual(proc.process_configuration, {})
self.assertEqual(proc.process_state, ProcessStateEnum.TERMINATED)
def instrument_test_driver(container):
org_client = OrgManagementServiceClient(node=container.node)
id_client = IdentityManagementServiceClient(node=container.node)
system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor)
log.info('system actor:' + system_actor._id)
sa_header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id))
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
_container_client = ContainerAgentClient(node=container.node,
name=container.name)
# Create a pubsub client to create streams.
_pubsub_client = PubsubManagementServiceClient(node=container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=container,
node=container.node)
subs = []
# Create streams for each stream named in driver.
stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = _pubsub_client.create_stream_definition(
container=stream_def)
stream_id = _pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2', headers={'ion-actor-id': system_actor._id, 'ion-actor-roles': sa_header_roles })
stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = _pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name )
_pubsub_client.activate_subscription(sub_id)
subs.append(sub)
# Create agent config.
agent_resource_id = '123xyz'
agent_config = {
'driver_config' : driver_config,
'stream_config' : stream_config,
'agent' : {'resource_id': agent_resource_id}
}
# Launch an instrument agent process.
_ia_name = 'agent007'
_ia_mod = 'ion.agents.instrument.instrument_agent'
_ia_class = 'InstrumentAgent'
_ia_pid = _container_client.spawn_process(name=_ia_name,
module=_ia_mod, cls=_ia_class,
config=agent_config)
log.info('got pid=%s for resource_id=%s' % (str(_ia_pid), str(agent_resource_id)))
class ExecutionEngineAgentPyonIntTest(IonIntegrationTestCase):
from ion.agents.cei.execution_engine_agent import ExecutionEngineAgentClient
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.resource_id = "eeagent_123456789"
self._eea_name = "eeagent"
self.persistence_directory = tempfile.mkdtemp()
self.agent_config = {
'eeagent': {
'heartbeat': 0,
'slots': 100,
'name': 'pyon_eeagent',
'launch_type': {
'name': 'pyon',
'persistence_directory': self.persistence_directory,
},
},
'agent': {
'resource_id': self.resource_id
},
'logging': {
'loggers': {
'eeagent': {
'level': 'DEBUG',
'handlers': ['console']
}
},
'root': {
'handlers': ['console']
},
}
}
self._start_eeagent()
def _start_eeagent(self):
self.container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self.container = self.container_client._get_container_instance()
# Start eeagent.
self._eea_pid = self.container_client.spawn_process(
name=self._eea_name,
module="ion.agents.cei.execution_engine_agent",
cls="ExecutionEngineAgent",
config=self.agent_config)
log.info('Agent pid=%s.', str(self._eea_pid))
# Start a resource agent client to talk with the instrument agent.
self._eea_pyon_client = ResourceAgentClient(self.resource_id,
process=FakeProcess())
log.info('Got eea client %s.', str(self._eea_pyon_client))
self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
def tearDown(self):
self.container.terminate_process(self._eea_pid)
shutil.rmtree(self.persistence_directory)
@needs_eeagent
def test_basics(self):
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_x'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
self.eea_client.terminate_process(u_pid, round)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
@needs_eeagent
def test_kill_and_revive(self):
"""test_kill_and_revive
Ensure that when an eeagent dies, it pulls the processes it owned from
persistence, and marks them as failed, so the PD can figure out what to
do with them
"""
u_pid = "test0"
round = 0
run_type = "pyon"
proc_name = 'test_transform'
module = 'ion.agents.cei.test.test_eeagent'
cls = 'TestProcess'
parameters = {'name': proc_name, 'module': module, 'cls': cls}
self.eea_client.launch_process(u_pid, round, run_type, parameters)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [500, 'RUNNING'])
# Kill and restart eeagent. Also, kill proc started by eea to simulate
# a killed container
old_eea_pid = str(self._eea_pid)
self.container.terminate_process(self._eea_pid)
proc_to_kill = self.container.proc_manager.procs_by_name.get(proc_name)
self.assertIsNotNone(proc_to_kill)
self.container.terminate_process(proc_to_kill.id)
self._start_eeagent()
self.assertNotEqual(old_eea_pid, self._eea_pid)
state = self.eea_client.dump_state().result
proc = get_proc_for_upid(state, u_pid)
self.assertIsNotNone(proc, "There is no state retrieved from eeagent")
self.assertEqual(proc.get('state'), [850, 'FAILED'])
