class TestAgentPersistence(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : False,
'enable_persistence' : True,
'aparam_pubrate_config' :
{
'raw' : 2,
'parsed' : 2
}
}
self._ia_client = None
self._ia_pid = '1234'
self.addCleanup(self._verify_agent_reset)
self.addCleanup(self.container.state_repository.put_state,
self._ia_pid, {})
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self, bootmode=None):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = bootmode
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config,
process_id=self._ia_pid)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_agent_config_persistence(self):
"""
test_agent_config_persistence
Test that agent parameter configuration is persisted between running
instances.
"""
# Start the agent.
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the default agent parameters.
#{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
retval = self._ia_client.get_agent(['streams'])['streams']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
#{'pubrate': {'raw': 0, 'parsed': 0}}
retval = self._ia_client.get_agent(['pubrate'])['pubrate']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
self.assertEqual(retval['raw'], 2)
self.assertEqual(retval['parsed'], 2)
#{'alerts': []}
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertEqual(retval, [])
# Define a few new parameters and set them.
# Confirm they are set.
alert_def_1 = {
'name' : 'current_warning_interval',
'stream_name' : 'parsed',
'description' : 'Current is below normal range.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 10.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def_2 = {
'name' : 'temp_alarm_interval',
'stream_name' : 'parsed',
'description' : 'Temperatoure is critical.',
'alert_type' : StreamAlertType.ALARM,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 20.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_alerts = [alert_def_1,alert_def_2, alert_def3]
pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : orig_alerts,
'pubrate' : pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the persisted parameters.
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
def test_agent_state_persistence(self):
"""
test_agent_state_persistence
Verify that agents can be restored to their prior running state.
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : [alert_def3],
'pubrate' : orig_pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
print '## in state: ' + state
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.STOPPED:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to STOPPED.")
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_agent_rparam_persistence(self):
"""
test_agent_rparam_persistence
Verify ability to restore device configuration.
### Original values:
{'TA0': -0.0002572242, 'OUTPUTSV': False, 'NAVG': 0}
### Values after set:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
### Restore config:
{'PTCA1': 0.6603433, 'WBOTC': 1.2024e-05, 'PCALDATE': [12, 8, 2005],
'STORETIME': False, 'CPCOR': 9.57e-08, 'PTCA2': 0.00575649,
'OUTPUTSV': True, 'SAMPLENUM': 0, 'TCALDATE': [8, 11, 2005],
'OUTPUTSAL': False, 'TA2': -9.717158e-06, 'POFFSET': 0.0,
'INTERVAL': 19733, 'SYNCWAIT': 0, 'CJ': 3.339261e-05,
'CI': 0.0001334915, 'CH': 0.1417895, 'TA0': -0.0005144484,
'TA1': 0.0003138936, 'NAVG': 1, 'TA3': 2.138735e-07, '
RCALDATE': [8, 11, 2005], 'CG': -0.987093, 'CTCOR': 3.25e-06, '
PTCB0': 24.6145, 'PTCB1': -0.0009, 'PTCB2': 0.0,
'CCALDATE': [8, 11, 2005], 'PA0': 5.916199, 'PA1': 0.4851819,
'PA2': 4.596432e-07, 'SYNCMODE': False, 'PTCA0': 276.2492,
'TXREALTIME': True, 'RTCA2': -3.022745e-08, 'RTCA1': 1.686132e-06,
'RTCA0': 0.9999862}
### Of which we have:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
params = [
SBE37Parameter.OUTPUTSV,
SBE37Parameter.NAVG,
SBE37Parameter.TA0
]
retval = self._ia_client.get_resource(params)
orig_params = retval
new_params = {
SBE37Parameter.OUTPUTSV : not orig_params[SBE37Parameter.OUTPUTSV],
SBE37Parameter.NAVG : orig_params[SBE37Parameter.NAVG] + 1,
SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 2
}
#print '########### orig params'
#print str(orig_params)
self._ia_client.set_resource(new_params)
retval = self._ia_client.get_resource(params)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
#print '########### new params'
#print str(retval)
# Now stop and restart the agent.
self._stop_agent()
self._support.stop_pagent()
gevent.sleep(10)
self._start_pagent()
gevent.sleep(10)
self._start_agent('restart')
timeout = gevent.Timeout(600)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(3)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
# Verify the parameters have been restored as needed.
retval = self._ia_client.get_resource(params)
#print '########### restored params'
#print str(retval)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
@unittest.skip('Making CEI friendly.')
def test_cei_launch_mode(self):
pdc = ProcessDispatcherServiceClient(node=self.container.node)
p_def = ProcessDefinition(name='Agent007')
p_def.executable = {
'module' : 'ion.agents.instrument.instrument_agent',
'class' : 'InstrumentAgent'
}
p_def_id = pdc.create_process_definition(p_def)
pid = pdc.create_process(p_def_id)
def event_callback(event, *args, **kwargs):
print '######### proc %s in state %s' % (event.origin, ProcessStateEnum._str_map[event.state])
sub = EventSubscriber(event_type='ProcessLifecycleEvent',
callback=event_callback,
origin=pid,
origin_type='DispatchedProcess')
sub.start()
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = 'restart'
pdc.schedule_process(p_def_id, process_id=pid,
configuration=agent_config)
gevent.sleep(5)
pdc.cancel_process(pid)
gevent.sleep(15)
sub.stop()
class TestAgentPersistence(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : False,
'enable_persistence' : True,
'aparam_pubrate_config' :
{
'raw' : 2,
'parsed' : 2
}
}
self._ia_client = None
self._ia_pid = '1234'
self.addCleanup(self._verify_agent_reset)
self.addCleanup(self.container.state_repository.put_state,
self._ia_pid, {})
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self, bootmode=None):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = bootmode
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config,
process_id=self._ia_pid)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_agent_config_persistence(self):
"""
test_agent_config_persistence
Test that agent parameter configuration is persisted between running
instances.
"""
# Start the agent.
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the default agent parameters.
#{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
retval = self._ia_client.get_agent(['streams'])['streams']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
#{'pubrate': {'raw': 0, 'parsed': 0}}
retval = self._ia_client.get_agent(['pubrate'])['pubrate']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
self.assertEqual(retval['raw'], 2)
self.assertEqual(retval['parsed'], 2)
#{'alerts': []}
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertEqual(retval, [])
# Define a few new parameters and set them.
# Confirm they are set.
alert_def_1 = {
'name' : 'current_warning_interval',
'stream_name' : 'parsed',
'description' : 'Current is below normal range.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 10.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def_2 = {
'name' : 'temp_alarm_interval',
'stream_name' : 'parsed',
'description' : 'Temperatoure is critical.',
'alert_type' : StreamAlertType.ALARM,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 20.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_alerts = [alert_def_1,alert_def_2, alert_def3]
pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : orig_alerts,
'pubrate' : pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the persisted parameters.
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
def test_agent_state_persistence(self):
"""
test_agent_state_persistence
Verify that agents can be restored to their prior running state.
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : [alert_def3],
'pubrate' : orig_pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
print '## in state: ' + state
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.STOPPED:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to STOPPED.")
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_agent_rparam_persistence(self):
"""
test_agent_rparam_persistence
Verify ability to restore device configuration.
### Original values:
{'TA0': -0.0002572242, 'OUTPUTSV': False, 'NAVG': 0}
### Values after set:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
### Restore config:
{'PTCA1': 0.6603433, 'WBOTC': 1.2024e-05, 'PCALDATE': [12, 8, 2005],
'STORETIME': False, 'CPCOR': 9.57e-08, 'PTCA2': 0.00575649,
'OUTPUTSV': True, 'SAMPLENUM': 0, 'TCALDATE': [8, 11, 2005],
'OUTPUTSAL': False, 'TA2': -9.717158e-06, 'POFFSET': 0.0,
'INTERVAL': 19733, 'SYNCWAIT': 0, 'CJ': 3.339261e-05,
'CI': 0.0001334915, 'CH': 0.1417895, 'TA0': -0.0005144484,
'TA1': 0.0003138936, 'NAVG': 1, 'TA3': 2.138735e-07, '
RCALDATE': [8, 11, 2005], 'CG': -0.987093, 'CTCOR': 3.25e-06, '
PTCB0': 24.6145, 'PTCB1': -0.0009, 'PTCB2': 0.0,
'CCALDATE': [8, 11, 2005], 'PA0': 5.916199, 'PA1': 0.4851819,
'PA2': 4.596432e-07, 'SYNCMODE': False, 'PTCA0': 276.2492,
'TXREALTIME': True, 'RTCA2': -3.022745e-08, 'RTCA1': 1.686132e-06,
'RTCA0': 0.9999862}
### Of which we have:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
params = [
SBE37Parameter.OUTPUTSV,
SBE37Parameter.NAVG,
SBE37Parameter.TA0
]
retval = self._ia_client.get_resource(params)
orig_params = retval
new_params = {
SBE37Parameter.OUTPUTSV : not orig_params[SBE37Parameter.OUTPUTSV],
SBE37Parameter.NAVG : orig_params[SBE37Parameter.NAVG] + 1,
SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 2
}
#print '########### orig params'
#print str(orig_params)
self._ia_client.set_resource(new_params)
retval = self._ia_client.get_resource(params)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
#print '########### new params'
#print str(retval)
# Now stop and restart the agent.
self._stop_agent()
self._support.stop_pagent()
gevent.sleep(10)
self._start_pagent()
gevent.sleep(10)
self._start_agent('restart')
timeout = gevent.Timeout(600)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(3)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
# Verify the parameters have been restored as needed.
retval = self._ia_client.get_resource(params)
#print '########### restored params'
#print str(retval)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
@unittest.skip('Making CEI friendly.')
def test_cei_launch_mode(self):
pdc = ProcessDispatcherServiceClient(node=self.container.node)
p_def = ProcessDefinition(name='Agent007')
p_def.executable = {
'module' : 'ion.agents.instrument.instrument_agent',
'class' : 'InstrumentAgent'
}
p_def_id = pdc.create_process_definition(p_def)
pid = pdc.create_process(p_def_id)
def event_callback(event, *args, **kwargs):
print '######### proc %s in state %s' % (event.origin, ProcessStateEnum._str_map[event.state])
sub = EventSubscriber(event_type='ProcessLifecycleEvent',
callback=event_callback,
origin=pid,
origin_type='DispatchedProcess')
sub.start()
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = 'restart'
pdc.schedule_process(p_def_id, process_id=pid,
configuration=agent_config)
gevent.sleep(5)
pdc.cancel_process(pid)
gevent.sleep(15)
sub.stop()
class TestAgentConnectionFailures(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
self._ia_client = None
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Start a resource agent client to talk with the instrument agent.
log.info('starting IA process')
self._ia_client = start_instrument_agent_process(self.container, self._stream_config)
self.addCleanup(self._verify_agent_reset)
log.info('test setup complete')
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state(timeout=120.1)
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd,timeout=300)
###############################################################################
# Event helpers.
###############################################################################
def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
"""
Start a subscriber to the instrument agent events.
@param type The type of event to catch.
@count Trigger the async event result when events received reaches this.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
# Event array and async event result.
self._event_count = count
self._events_received = []
self._async_event_result = AsyncResult()
self._event_subscriber = EventSubscriber(
event_type=type, callback=consume_event,
origin=IA_RESOURCE_ID)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=5)
def _stop_event_subscriber(self):
"""
Stop event subscribers on cleanup.
"""
self._event_subscriber.stop()
self._event_subscriber = None
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
def _start_data_subscribers(self, count, raw_count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._raw_samples_received = []
self._async_sample_result = AsyncResult()
self._async_raw_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self._samples_received.append(message)
if len(self._samples_received) == count:
self._async_sample_result.set()
def recv_raw_data(message, stream_route, stream_id):
log.info('Received raw data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self._raw_samples_received.append(message)
if len(self._raw_samples_received) == raw_count:
self._async_raw_sample_result.set()
from pyon.util.containers import create_unique_identifier
stream_name = 'parsed'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" %
stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id],timeout=120.2)
pubsub_client.activate_subscription(sub_id,timeout=120.3)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
stream_name = 'raw'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" %
stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id],timeout=120.4)
pubsub_client.activate_subscription(sub_id,timeout=120.5)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber,'subscription_id'):
try:
pubsub_client.deactivate_subscription(subscriber.subscription_id,timeout=120.6)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id,timeout=120.7)
subscriber.stop()
###############################################################################
# Socket listen.
###############################################################################
def _socket_listen(self, s, prompt, timeout):
buf = ''
starttime = time.time()
while True:
try:
buf += s.recv(1024)
print '##### Listening, got: %s' % buf
if prompt and buf.find(prompt) != -1:
break
except:
gevent.sleep(1)
finally:
delta = time.time() - starttime
if delta > timeout:
break
return buf
###############################################################################
# Assert helpers.
###############################################################################
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
# AgentCommandResult.result['parsed']
"""
{'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data',
'pkt_version': 1, 'values':
[{'value_id': 'temp', 'value': 21.4894},
{'value_id': 'conductivity', 'value': 13.22157},
{'value_id': 'pressure', 'value': 146.186}],
'driver_timestamp': 3556901018.170206}
"""
self.assertIsInstance(val, dict)
self.assertTrue(val.has_key('values'))
values_list = val['values']
self.assertTrue(isinstance(values_list, list))
self.assertTrue(len(values_list)==3)
ids = ['temp', 'conductivity', 'pressure']
ids_found = []
for x in values_list:
self.assertTrue(x.has_key('value_id'))
self.assertTrue(x.has_key('value'))
ids_found.append(x['value_id'])
self.assertTrue(isinstance(x['value'], float))
self.assertItemsEqual(ids, ids_found)
self.assertTrue(val.has_key('driver_timestamp'))
time = val['driver_timestamp']
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(PARAMS.has_key(key))
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val*.01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
###############################################################################
# Tests.
###############################################################################
def test_lost_connection(self):
"""
test_lost_connection
"""
# Set up a subscriber to collect command events.
self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent', 1)
self.addCleanup(self._stop_event_subscriber)
# Start in uninitialized.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Initialize the agent.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Activate.
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
# Go into command mode.
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Start streaming.
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Wait for a while, collect some samples.
gevent.sleep(10)
# Blow the port agent out from under the agent.
self._support.stop_pagent()
# Loop until we resyncronize to LOST_CONNECTION/DISCONNECTED.
# Test will timeout if this dosn't occur.
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.LOST_CONNECTION:
break
else:
gevent.sleep(1)
# Verify the driver has transitioned to disconnected
while True:
state = self._ia_client.get_resource_state()
if state == DriverConnectionState.DISCONNECTED:
break
else:
gevent.sleep(1)
# Make sure the lost connection error event arrives.
self._async_event_result.get(timeout=CFG.endpoint.receive.timeout)
self.assertEqual(len(self._events_received), 1)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
#@unittest.skip('Fails on buildbot for some god unknown reason.')
def test_autoreconnect(self):
"""
test_autoreconnect
"""
# Set up a subscriber to collect command events.
self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent', 1)
self.addCleanup(self._stop_event_subscriber)
# Start in uninitialized.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Initialize the agent.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Activate.
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
# Go into command mode.
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
def poll_func(test):
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
while True:
try:
gevent.sleep(.5)
test._ia_client.execute_resource(cmd)
except IonException as ex:
# This exception could be ResourceException (broken pipe)
# Timeout or Conflict
log.info('#### pre shutdown exception: %s, %s', str(type(ex)), str(ex))
break
while True:
try:
gevent.sleep(.5)
test._ia_client.execute_resource(cmd)
log.info('#### post shutdown got new sample.')
break
except IonException as ex:
# This should be conflict.
log.info('#### post shutdown exception: %s, %s', str(type(ex)), str(ex))
timeout = gevent.Timeout(600)
timeout.start()
try:
# Start the command greenlet and let poll for a bit.
gl = gevent.spawn(poll_func, self)
gevent.sleep(20)
# Blow the port agent out from under the agent.
self._support.stop_pagent()
# Wait for a while, the supervisor is restarting the port agent.
gevent.sleep(10)
self._support.start_pagent()
# Wait for the device to connect and start sampling again.
gl.join()
gl = None
timeout.cancel()
except (Exception, gevent.Timeout) as ex:
if gl:
gl.kill()
gl = None
self.fail(('Could not reconnect to device: %s, %s',
str(type(ex)), str(ex)))
def test_connect_failed(self):
"""
test_connect_failed
"""
# Stop the port agent.
self._support.stop_pagent()
# Sleep a bit.
gevent.sleep(3)
# Start in uninitialized.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Initialize the agent.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Activate. This should fail because there is no port agent to connect to.
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
with self.assertRaises(ResourceError):
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
def test_get_set_alerts(self):
"""
test_get_set_alerts
Test specific of get/set alerts, including using result of get to
set later.
"""
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertItemsEqual(retval, [])
alert_def1 = {
'name' : 'temp_warning_interval',
'stream_name' : 'parsed',
'description' : 'Temperature is above normal range.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 10.5,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def2 = {
'name' : 'temp_alarm_interval',
'stream_name' : 'parsed',
'description' : 'Temperature is way above normal range.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 15.5,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
"""
Interval alerts are returned from get like this:
(value and status fields describe state of the alert)
{
'name': 'temp_warning_interval',
'stream_name': 'parsed',
'description': 'Temperature is above normal range.',
'alert_type': 1,
'aggregate_type': 2,
'value_id': 'temp',
'lower_bound': None,
'lower_rel_op': None,
'upper_bound': 10.5,
'upper_rel_op': '<',
'alert_class': 'IntervalAlert',
'status': None,
'value': None
}
"""
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
"""
Late data alerts are returned from get like this:
(value and status fields describe state of the alert)
{
'name': 'late_data_warning',
'stream_name': 'parsed',
'description': 'Expected data has not arrived.',
'alert_type': 1,
'aggregate_type': 1,
'value_id': None,
'time_delta': 180,
'alert_class': 'LateDataAlert',
'status': None,
'value': None
}
"""
"""
[
{'status': None,
'alert_type': 1,
'name': 'temp_warning_interval',
'upper_bound': 10.5,
'lower_bound': None,
'aggregate_type': 2,
'alert_class': 'IntervalAlert',
'value': None,
'value_id': 'temp',
'lower_rel_op': None,
'upper_rel_op': '<',
'description': 'Temperature is above normal range.'},
{'status': None,
'alert_type': 1,
'name': 'temp_alarm_interval',
'upper_bound': 15.5,
'lower_bound': None,
'aggregate_type': 2,
'alert_class': 'IntervalAlert',
'value': None,
'value_id': 'temp',
'lower_rel_op': None,
'upper_rel_op': '<',
'description': 'Temperature is way above normal range.'},
{'status': None,
'stream_name': 'parsed',
'alert_type': 1,
'name': 'late_data_warning',
'aggregate_type': 1,
'alert_class': 'LateDataAlert',
'value': None,
'time_delta': 180,
'description': 'Expected data has not arrived.'}
]
"""
orig_alerts = [alert_def1, alert_def2, alert_def3]
self._ia_client.set_agent({'alerts' : orig_alerts})
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertTrue(len(retval)==3)
alerts = retval
self._ia_client.set_agent({'alerts' : ['clear']})
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertItemsEqual(retval, [])
self._ia_client.set_agent({'alerts' : alerts})
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertTrue(len(retval)==3)
count = 0
for x in retval:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEquals(count, 3)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestAgentCommsAlerts(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self._event_count = 0
self._events_received = []
self._async_event_result = AsyncResult()
def consume_event(*args, **kwargs):
log.debug('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
self._event_subscriber = EventSubscriber(
event_type='DeviceStatusAlertEvent', callback=consume_event,
origin=IA_RESOURCE_ID)
self._event_subscriber.start()
def stop_subscriber():
self._event_subscriber.stop()
self._event_subscriber = None
self.addCleanup(stop_subscriber)
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'aparam_alerts_config' : [state_alert_def, command_alert_def]
}
self._ia_client = None
self._ia_pid = None
self.addCleanup(self._verify_agent_reset)
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=self._agent_config)
log.info('Started instrument agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
self._ia_pid = None
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_lost_connection_alert(self):
"""
test_lost_connection_alert
Verify that agents detect lost connection state and issue alert.
"""
self._event_count = 3
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Confirm the persisted parameters.
retval = self._ia_client.get_agent(['alerts'])['alerts']
"""
{'origin': '123xyz', 'status': 1, '_id': 'da03b90d2e064b25bf51ed90b729e82e',
'description': 'The alert is cleared.', 'time_stamps': [],
'type_': 'DeviceStatusAlertEvent', 'valid_values': [],
'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '',
'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'],
'stream_name': '', 'ts_created': '1366749987069', 'sub_type': 3,
'origin_type': 'InstrumentDevice', 'name': 'comms_warning'}
"""
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Blow the port agent out from under the agent.
self._support.stop_pagent()
# Wait for a while, the supervisor is restarting the port agent.
gevent.sleep(5)
self._support.start_pagent()
timeout = gevent.Timeout(120)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(2)
except Timeout as t:
self.fail('Could not reconnect to device.')
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
self._async_event_result.get(timeout=30)
"""
{'origin': '123xyz', 'status': 1, '_id': '9d7db919a0414741a2aa97f3dc310647', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029709', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
{'origin': '123xyz', 'status': 1, '_id': 'f746cec5e486445e856ef29af8d8d49a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029717', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
{'origin': '123xyz', 'status': 1, '_id': '126b1e20f54f4bd2b84a93584831ea8d', 'description': 'Detected comms failure.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_LOST_CONNECTION'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008062061', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
{'origin': '123xyz', 'status': 1, '_id': '90f5762112dd446b939c6675d34dd61d', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_COMMAND'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008098639', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
"""
def test_connect_failed_alert(self):
"""
test_connect_failed_alert
Verify that agents detect failed connections and issue alert.
"""
self._event_count = 4
# Remove the port agent.
self._support.stop_pagent()
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
with self.assertRaises(Exception):
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
# Now start up the port agent.
self._support.start_pagent()
gevent.sleep(5)
# This time it will work.
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
self._async_event_result.get(timeout=30)
"""
{'origin': '123xyz', 'status': 1, '_id': 'ac1ae3ec24e74f65bde24362d689346a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751167', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
{'origin': '123xyz', 'status': 1, '_id': '6af6a6156172481ebc44baad2708ec5c', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751175', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
{'origin': '123xyz', 'status': 1, '_id': 'b493698b46fd4fc1b4c66c53b70ed043', 'description': 'Detected comms failure while connecting.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007756771', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
{'origin': '123xyz', 'status': 1, '_id': 'c01e911ecb1a47c5a04cbbcbfb348a42', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007792905', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
"""
class TestAgentConnectionFailures(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
self._ia_client = None
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR,
DEV_PORT, DATA_PORT,
CMD_PORT, PA_BINARY,
DELIM, WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Start a resource agent client to talk with the instrument agent.
log.info('starting IA process')
self._ia_client = start_instrument_agent_process(
self.container, self._stream_config)
self.addCleanup(self._verify_agent_reset)
log.info('test setup complete')
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i', port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr': 'localhost',
'port': port,
'cmd_port': CMD_PORT
}
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state(timeout=120.1)
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd, timeout=300)
###############################################################################
# Event helpers.
###############################################################################
def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
"""
Start a subscriber to the instrument agent events.
@param type The type of event to catch.
@count Trigger the async event result when events received reaches this.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
# Event array and async event result.
self._event_count = count
self._events_received = []
self._async_event_result = AsyncResult()
self._event_subscriber = EventSubscriber(event_type=type,
callback=consume_event,
origin=IA_RESOURCE_ID)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=5)
def _stop_event_subscriber(self):
"""
Stop event subscribers on cleanup.
"""
self._event_subscriber.stop()
self._event_subscriber = None
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
def _start_data_subscribers(self, count, raw_count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._raw_samples_received = []
self._async_sample_result = AsyncResult()
self._async_raw_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received parsed data on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self._samples_received.append(message)
if len(self._samples_received) == count:
self._async_sample_result.set()
def recv_raw_data(message, stream_route, stream_id):
log.info('Received raw data on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self._raw_samples_received.append(message)
if len(self._raw_samples_received) == raw_count:
self._async_raw_sample_result.set()
from pyon.util.containers import create_unique_identifier
stream_name = 'parsed'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" % stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id],
timeout=120.2)
pubsub_client.activate_subscription(sub_id, timeout=120.3)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
stream_name = 'raw'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" % stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id],
timeout=120.4)
pubsub_client.activate_subscription(sub_id, timeout=120.5)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber, 'subscription_id'):
try:
pubsub_client.deactivate_subscription(
subscriber.subscription_id, timeout=120.6)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id,
timeout=120.7)
subscriber.stop()
###############################################################################
# Socket listen.
###############################################################################
def _socket_listen(self, s, prompt, timeout):
buf = ''
starttime = time.time()
while True:
try:
buf += s.recv(1024)
print '##### Listening, got: %s' % buf
if prompt and buf.find(prompt) != -1:
break
except:
gevent.sleep(1)
finally:
delta = time.time() - starttime
if delta > timeout:
break
return buf
###############################################################################
# Assert helpers.
###############################################################################
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
# AgentCommandResult.result['parsed']
"""
{'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data',
'pkt_version': 1, 'values':
[{'value_id': 'temp', 'value': 21.4894},
{'value_id': 'conductivity', 'value': 13.22157},
{'value_id': 'pressure', 'value': 146.186}],
'driver_timestamp': 3556901018.170206}
"""
self.assertIsInstance(val, dict)
self.assertTrue(val.has_key('values'))
values_list = val['values']
self.assertTrue(isinstance(values_list, list))
self.assertTrue(len(values_list) == 3)
ids = ['temp', 'conductivity', 'pressure']
ids_found = []
for x in values_list:
self.assertTrue(x.has_key('value_id'))
self.assertTrue(x.has_key('value'))
ids_found.append(x['value_id'])
self.assertTrue(isinstance(x['value'], float))
self.assertItemsEqual(ids, ids_found)
self.assertTrue(val.has_key('driver_timestamp'))
time = val['driver_timestamp']
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(PARAMS.has_key(key))
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val * .01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
###############################################################################
# Tests.
###############################################################################
def test_lost_connection(self):
"""
test_lost_connection
"""
# Set up a subscriber to collect command events.
self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent',
1)
self.addCleanup(self._stop_event_subscriber)
# Start in uninitialized.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Initialize the agent.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Activate.
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
# Go into command mode.
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Start streaming.
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Wait for a while, collect some samples.
gevent.sleep(10)
# Blow the port agent out from under the agent.
self._support.stop_pagent()
# Loop until we resyncronize to LOST_CONNECTION/DISCONNECTED.
# Test will timeout if this dosn't occur.
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.LOST_CONNECTION:
break
else:
gevent.sleep(1)
# Verify the driver has transitioned to disconnected
while True:
state = self._ia_client.get_resource_state()
if state == DriverConnectionState.DISCONNECTED:
break
else:
gevent.sleep(1)
# Make sure the lost connection error event arrives.
self._async_event_result.get(timeout=CFG.endpoint.receive.timeout)
self.assertEqual(len(self._events_received), 1)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
#@unittest.skip('Fails on buildbot for some god unknown reason.')
def test_autoreconnect(self):
"""
test_autoreconnect
"""
# Set up a subscriber to collect command events.
self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent',
1)
self.addCleanup(self._stop_event_subscriber)
# Start in uninitialized.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Initialize the agent.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Activate.
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
# Go into command mode.
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
def poll_func(test):
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
while True:
try:
gevent.sleep(.5)
test._ia_client.execute_resource(cmd)
except IonException as ex:
# This exception could be ResourceException (broken pipe)
# Timeout or Conflict
log.info('#### pre shutdown exception: %s, %s',
str(type(ex)), str(ex))
break
while True:
try:
gevent.sleep(.5)
test._ia_client.execute_resource(cmd)
log.info('#### post shutdown got new sample.')
break
except IonException as ex:
# This should be conflict.
log.info('#### post shutdown exception: %s, %s',
str(type(ex)), str(ex))
timeout = gevent.Timeout(600)
timeout.start()
try:
# Start the command greenlet and let poll for a bit.
gl = gevent.spawn(poll_func, self)
gevent.sleep(20)
# Blow the port agent out from under the agent.
self._support.stop_pagent()
# Wait for a while, the supervisor is restarting the port agent.
gevent.sleep(10)
self._support.start_pagent()
# Wait for the device to connect and start sampling again.
gl.join()
gl = None
timeout.cancel()
except (Exception, gevent.Timeout) as ex:
if gl:
gl.kill()
gl = None
self.fail(('Could not reconnect to device: %s, %s', str(type(ex)),
str(ex)))
def test_connect_failed(self):
"""
test_connect_failed
"""
# Stop the port agent.
self._support.stop_pagent()
# Sleep a bit.
gevent.sleep(3)
# Start in uninitialized.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Initialize the agent.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Activate. This should fail because there is no port agent to connect to.
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
with self.assertRaises(ResourceError):
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
def test_get_set_alerts(self):
"""
test_get_set_alerts
Test specific of get/set alerts, including using result of get to
set later.
"""
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertItemsEqual(retval, [])
alert_def1 = {
'name': 'temp_warning_interval',
'stream_name': 'parsed',
'description': 'Temperature is above normal range.',
'alert_type': StreamAlertType.WARNING,
'aggregate_type': AggregateStatusType.AGGREGATE_DATA,
'value_id': 'temp',
'lower_bound': None,
'lower_rel_op': None,
'upper_bound': 10.5,
'upper_rel_op': '<',
'alert_class': 'IntervalAlert'
}
alert_def2 = {
'name': 'temp_alarm_interval',
'stream_name': 'parsed',
'description': 'Temperature is way above normal range.',
'alert_type': StreamAlertType.WARNING,
'aggregate_type': AggregateStatusType.AGGREGATE_DATA,
'value_id': 'temp',
'lower_bound': None,
'lower_rel_op': None,
'upper_bound': 15.5,
'upper_rel_op': '<',
'alert_class': 'IntervalAlert'
}
"""
Interval alerts are returned from get like this:
(value and status fields describe state of the alert)
{
'name': 'temp_warning_interval',
'stream_name': 'parsed',
'description': 'Temperature is above normal range.',
'alert_type': 1,
'aggregate_type': 2,
'value_id': 'temp',
'lower_bound': None,
'lower_rel_op': None,
'upper_bound': 10.5,
'upper_rel_op': '<',
'alert_class': 'IntervalAlert',
'status': None,
'value': None
}
"""
alert_def3 = {
'name': 'late_data_warning',
'stream_name': 'parsed',
'description': 'Expected data has not arrived.',
'alert_type': StreamAlertType.WARNING,
'aggregate_type': AggregateStatusType.AGGREGATE_COMMS,
'time_delta': 180,
'alert_class': 'LateDataAlert'
}
"""
Late data alerts are returned from get like this:
(value and status fields describe state of the alert)
{
'name': 'late_data_warning',
'stream_name': 'parsed',
'description': 'Expected data has not arrived.',
'alert_type': 1,
'aggregate_type': 1,
'value_id': None,
'time_delta': 180,
'alert_class': 'LateDataAlert',
'status': None,
'value': None
}
"""
"""
[
{'status': None,
'alert_type': 1,
'name': 'temp_warning_interval',
'upper_bound': 10.5,
'lower_bound': None,
'aggregate_type': 2,
'alert_class': 'IntervalAlert',
'value': None,
'value_id': 'temp',
'lower_rel_op': None,
'upper_rel_op': '<',
'description': 'Temperature is above normal range.'},
{'status': None,
'alert_type': 1,
'name': 'temp_alarm_interval',
'upper_bound': 15.5,
'lower_bound': None,
'aggregate_type': 2,
'alert_class': 'IntervalAlert',
'value': None,
'value_id': 'temp',
'lower_rel_op': None,
'upper_rel_op': '<',
'description': 'Temperature is way above normal range.'},
{'status': None,
'stream_name': 'parsed',
'alert_type': 1,
'name': 'late_data_warning',
'aggregate_type': 1,
'alert_class': 'LateDataAlert',
'value': None,
'time_delta': 180,
'description': 'Expected data has not arrived.'}
]
"""
orig_alerts = [alert_def1, alert_def2, alert_def3]
self._ia_client.set_agent({'alerts': orig_alerts})
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertTrue(len(retval) == 3)
alerts = retval
self._ia_client.set_agent({'alerts': ['clear']})
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertItemsEqual(retval, [])
self._ia_client.set_agent({'alerts': alerts})
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertTrue(len(retval) == 3)
count = 0
for x in retval:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEquals(count, 3)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
