def test_platform_with_instrument_streaming(self):
#
# The following is with just a single platform and the single
# instrument "SBE37_SIM_08", which corresponds to the one on port 4008.
#
#load the paramaters and the param dicts necesssary for the VEL3D
self._load_params()
#create the instrument device/agent/mode
self._create_instrument_resources()
#create the platform device, agent and instance
self._create_platform_configuration('LPJBox_CI_Ben_Hall')
self.rrclient.create_association(subject=self.platform_device, predicate=PRED.hasDevice, object=self.instrument_device)
self._start_platform()
# self.addCleanup(self._stop_platform, p_root)
# get everything in command mode:
self._ping_agent()
self._initialize()
_ia_client = ResourceAgentClient(self.instrument_device, process=FakeProcess())
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
self._go_active()
# self._run()
gevent.sleep(3)
# note that this includes the instrument also getting to the command state
# self._stream_instruments()
# get client to the instrument:
# the i_obj is a DotDict with various pieces captured during the
# set-up of the instrument, in particular instrument_device_id
#i_obj = self._get_instrument(instr_key)
# log.debug("KK creating ResourceAgentClient")
# ia_client = ResourceAgentClient(i_obj.instrument_device_id,
# process=FakeProcess())
# log.debug("KK got ResourceAgentClient: %s", ia_client)
#
# # verify the instrument is command state:
# state = ia_client.get_agent_state()
# log.debug("KK instrument state: %s", state)
# self.assertEqual(state, ResourceAgentState.COMMAND)
self._reset()
self._shutdown()
def test_platform_with_instrument_streaming(self):
#
# The following is with just a single platform and the single
# instrument "SBE37_SIM_08", which corresponds to the one on port 4008.
#
#load the paramaters and the param dicts necesssary for the VEL3D
self._load_params()
#create the instrument device/agent/mode
self._create_instrument_resources()
#create the platform device, agent and instance
self._create_platform_configuration('LPJBox_CI_Ben_Hall')
self.rrclient.create_association(subject=self.platform_device,
predicate=PRED.hasDevice,
object=self.instrument_device)
self._start_platform()
# self.addCleanup(self._stop_platform, p_root)
# get everything in command mode:
self._ping_agent()
self._initialize()
_ia_client = ResourceAgentClient(self.instrument_device,
process=FakeProcess())
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
self._go_active()
# self._run()
gevent.sleep(3)
# note that this includes the instrument also getting to the command state
# self._stream_instruments()
# get client to the instrument:
# the i_obj is a DotDict with various pieces captured during the
# set-up of the instrument, in particular instrument_device_id
#i_obj = self._get_instrument(instr_key)
# log.debug("KK creating ResourceAgentClient")
# ia_client = ResourceAgentClient(i_obj.instrument_device_id,
# process=FakeProcess())
# log.debug("KK got ResourceAgentClient: %s", ia_client)
#
# # verify the instrument is command state:
# state = ia_client.get_agent_state()
# log.debug("KK instrument state: %s", state)
# self.assertEqual(state, ResourceAgentState.COMMAND)
self._reset()
self._shutdown()
def get_agent_state(self, resource_id=''):
"""Return the current resource agent common state.
"""
res_type = self._get_resource_type(resource_id)
if self._has_agent(res_type):
rac = ResourceAgentClient(resource_id=resource_id)
return rac.get_agent_state(resource_id=resource_id)
raise BadRequest("Not implemented for resource type %s" % res_type)
def get_agent_state(self, resource_id=''):
"""Return the current resource agent common state.
"""
res_type = self._get_resource_type(resource_id)
if self._has_agent(res_type):
rac = ResourceAgentClient(resource_id=resource_id)
return rac.get_agent_state(resource_id=resource_id)
raise BadRequest("Not implemented for resource type %s" % res_type)
def get_agent_state(self, resource_id=''):
"""Return the current resource agent common state.
@param resource_id The id of the resource agennt.
@retval A str containing the current agent state.
@param resource_id str
@retval result str
"""
res_type = self._get_resource_type(resource_id)
if self._has_agent(res_type):
rac = ResourceAgentClient(resource_id=resource_id)
return rac.get_agent_state(resource_id=resource_id)
raise BadRequest("Not implemented for resource type %s" % res_type)
def get_agent_state(self, resource_id=''):
"""Return the current resource agent common state.
@param resource_id The id of the resource agennt.
@retval A str containing the current agent state.
@param resource_id str
@retval result str
"""
res_type = self._get_resource_type(resource_id)
if self._has_agent(res_type):
rac = ResourceAgentClient(resource_id=resource_id)
return rac.get_agent_state(resource_id=resource_id)
raise BadRequest("Not implemented for resource type %s" % res_type)
def test_platform_with_instrument_streaming(self):
#
# The following is with just a single platform and the single
# instrument "SBE37_SIM_08", which corresponds to the one on port 4008.
#
#load the paramaters and the param dicts necesssary for the VEL3D
log.debug(
"load params------------------------------------------------------------------------------"
)
self._load_params()
log.debug(
" _register_oms_listener------------------------------------------------------------------------------"
)
self._register_oms_listener()
#create the instrument device/agent/mode
log.debug("---------- create_instrument_resources ----------")
self._create_instrument_resources()
#create the platform device, agent and instance
log.debug("---------- create_platform_configuration ----------")
self._create_platform_configuration('LPJBox_CI_Ben_Hall')
self.rrclient.create_association(subject=self.platform_device_id,
predicate=PRED.hasDevice,
object=self.instrument_device_id)
log.debug("---------- start_platform ----------")
self._start_platform()
self.addCleanup(self._stop_platform)
# get everything in command mode:
self._ping_agent()
log.debug(" ---------- initialize ----------")
self._initialize()
_ia_client = ResourceAgentClient(self.instrument_device_id,
process=FakeProcess())
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
log.debug(" ---------- go_active ----------")
self._go_active()
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
log.debug("---------- run ----------")
self._run()
gevent.sleep(2)
log.debug(" ---------- _start_resource_monitoring ----------")
self._start_resource_monitoring()
gevent.sleep(2)
#
# # verify the instrument is command state:
# state = ia_client.get_agent_state()
# log.debug(" TestPlatformInstrument get_agent_state: %s", state)
# self.assertEqual(state, ResourceAgentState.COMMAND) _stop_resource_monitoring
log.debug(" ---------- _stop_resource_monitoring ----------")
self._stop_resource_monitoring()
gevent.sleep(2)
log.debug(" ---------- go_inactive ----------")
self._go_inactive()
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
self._reset()
self._shutdown()
class TestDriverEgg(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def get_streamConfigs(self):
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
return raw_config, parsed_config
##########################
#
# The following tests generate different agent configs and pass them to a common base test script
#
###########################
@unittest.skip("this test can't be run from coi services. it is missing dependencies")
def test_driverLaunchModuleNoURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchModuleWithURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchNoModuleOnlyURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchBogusModuleWithURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="bogus",
driver_class="Bogus",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
@unittest.skip("Launches an egg 'process' even though the egg download should produce error 404")
def test_driverLaunchNoModule404URI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_404,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj, False)
def test_driverLaunchNoModuleBadEggURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_BAD,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj, True, False)
def base_activateInstrumentSample(self, instAgent_obj, expect_launch=True, expect_command=True):
"""
This method runs a test of launching a driver with a given agent configuration
"""
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
print 'new InstrumentModel id = %s ' % instModel_id
# Create InstrumentAgent
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
print 'new InstrumentAgent id = %s' % instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',
id_only=True)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict',
id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed',
parameter_dictionary_id=parsed_pdict_id)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw',
parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test')
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new dp_id = %s' % data_product_id1
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
print 'Data product streams1 = %s' % stream_ids
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
print 'Data set for data_product_id1 = %s' % dataset_ids[0]
self.parsed_dataset = dataset_ids[0]
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
print 'new dp_id = %s' % str(data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
print 'Data product streams2 = %s' % str(stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
print 'Data set for data_product_id2 = %s' % dataset_ids[0]
self.raw_dataset = dataset_ids[0]
# add start/stop for instrument agent
gevent.joinall([gevent.spawn(lambda:
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id))])
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
agent_process_id = ResourceAgentClient._get_agent_process_id(instDevice_id)
print "Agent process id is '%s'" % str(agent_process_id)
self.assertTrue(agent_process_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
agent_process_id,
ProcessStateEnum.RUNNING)
if not expect_launch:
self.assertFalse(gate.await(30), "The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?" %
agent_process_id)
return
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
agent_process_id)
print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING"
#print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=agent_process_id,
process=FakeProcess())
print "ResourceAgentClient created: %s" % str(self._ia_client)
print "Sending command=ResourceAgentEvent.INITIALIZE"
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
if not expect_command:
self.assertRaises(ServerError, self._ia_client.execute_agent, cmd)
return
retval = self._ia_client.execute_agent(cmd)
print "Result of INITIALIZE: %s" % str(retval)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# This gevent sleep is there to test the autosample time, which will show something different from default
# only if the instrument runs for over a minute
gevent.sleep(90)
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
autosample_string = extended_instrument.computed.uptime.value
autosampling_time = int(autosample_string.split()[4])
self.assertTrue(autosampling_time > 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
print "Sending command=ResourceAgentEvent.RESET"
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
print "Result of RESET: %s" % str(reply)
class TestAgentPersistence(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : False,
'enable_persistence' : True,
'aparam_pubrate_config' :
{
'raw' : 2,
'parsed' : 2
}
}
self._ia_client = None
self._ia_pid = '1234'
self.addCleanup(self._verify_agent_reset)
self.addCleanup(self.container.state_repository.put_state,
self._ia_pid, {})
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self, bootmode=None):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = bootmode
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config,
process_id=self._ia_pid)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_agent_config_persistence(self):
"""
test_agent_config_persistence
Test that agent parameter configuration is persisted between running
instances.
"""
# Start the agent.
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the default agent parameters.
#{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
retval = self._ia_client.get_agent(['streams'])['streams']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
#{'pubrate': {'raw': 0, 'parsed': 0}}
retval = self._ia_client.get_agent(['pubrate'])['pubrate']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
self.assertEqual(retval['raw'], 2)
self.assertEqual(retval['parsed'], 2)
#{'alerts': []}
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertEqual(retval, [])
# Define a few new parameters and set them.
# Confirm they are set.
alert_def_1 = {
'name' : 'current_warning_interval',
'stream_name' : 'parsed',
'description' : 'Current is below normal range.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 10.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def_2 = {
'name' : 'temp_alarm_interval',
'stream_name' : 'parsed',
'description' : 'Temperatoure is critical.',
'alert_type' : StreamAlertType.ALARM,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 20.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_alerts = [alert_def_1,alert_def_2, alert_def3]
pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : orig_alerts,
'pubrate' : pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the persisted parameters.
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
def test_agent_state_persistence(self):
"""
test_agent_state_persistence
Verify that agents can be restored to their prior running state.
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : [alert_def3],
'pubrate' : orig_pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
print '## in state: ' + state
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.STOPPED:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to STOPPED.")
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_agent_rparam_persistence(self):
"""
test_agent_rparam_persistence
Verify ability to restore device configuration.
### Original values:
{'TA0': -0.0002572242, 'OUTPUTSV': False, 'NAVG': 0}
### Values after set:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
### Restore config:
{'PTCA1': 0.6603433, 'WBOTC': 1.2024e-05, 'PCALDATE': [12, 8, 2005],
'STORETIME': False, 'CPCOR': 9.57e-08, 'PTCA2': 0.00575649,
'OUTPUTSV': True, 'SAMPLENUM': 0, 'TCALDATE': [8, 11, 2005],
'OUTPUTSAL': False, 'TA2': -9.717158e-06, 'POFFSET': 0.0,
'INTERVAL': 19733, 'SYNCWAIT': 0, 'CJ': 3.339261e-05,
'CI': 0.0001334915, 'CH': 0.1417895, 'TA0': -0.0005144484,
'TA1': 0.0003138936, 'NAVG': 1, 'TA3': 2.138735e-07, '
RCALDATE': [8, 11, 2005], 'CG': -0.987093, 'CTCOR': 3.25e-06, '
PTCB0': 24.6145, 'PTCB1': -0.0009, 'PTCB2': 0.0,
'CCALDATE': [8, 11, 2005], 'PA0': 5.916199, 'PA1': 0.4851819,
'PA2': 4.596432e-07, 'SYNCMODE': False, 'PTCA0': 276.2492,
'TXREALTIME': True, 'RTCA2': -3.022745e-08, 'RTCA1': 1.686132e-06,
'RTCA0': 0.9999862}
### Of which we have:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
params = [
SBE37Parameter.OUTPUTSV,
SBE37Parameter.NAVG,
SBE37Parameter.TA0
]
retval = self._ia_client.get_resource(params)
orig_params = retval
new_params = {
SBE37Parameter.OUTPUTSV : not orig_params[SBE37Parameter.OUTPUTSV],
SBE37Parameter.NAVG : orig_params[SBE37Parameter.NAVG] + 1,
SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 2
}
#print '########### orig params'
#print str(orig_params)
self._ia_client.set_resource(new_params)
retval = self._ia_client.get_resource(params)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
#print '########### new params'
#print str(retval)
# Now stop and restart the agent.
self._stop_agent()
self._support.stop_pagent()
gevent.sleep(10)
self._start_pagent()
gevent.sleep(10)
self._start_agent('restart')
timeout = gevent.Timeout(600)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(3)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
# Verify the parameters have been restored as needed.
retval = self._ia_client.get_resource(params)
#print '########### restored params'
#print str(retval)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
@unittest.skip('Making CEI friendly.')
def test_cei_launch_mode(self):
pdc = ProcessDispatcherServiceClient(node=self.container.node)
p_def = ProcessDefinition(name='Agent007')
p_def.executable = {
'module' : 'ion.agents.instrument.instrument_agent',
'class' : 'InstrumentAgent'
}
p_def_id = pdc.create_process_definition(p_def)
pid = pdc.create_process(p_def_id)
def event_callback(event, *args, **kwargs):
print '######### proc %s in state %s' % (event.origin, ProcessStateEnum._str_map[event.state])
sub = EventSubscriber(event_type='ProcessLifecycleEvent',
callback=event_callback,
origin=pid,
origin_type='DispatchedProcess')
sub.start()
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = 'restart'
pdc.schedule_process(p_def_id, process_id=pid,
configuration=agent_config)
gevent.sleep(5)
pdc.cancel_process(pid)
gevent.sleep(15)
sub.stop()
class TestPrest(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
print '#####################'
print 'IN SETUP'
self._ia_client = None
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
#log.info('driver uri: %s', DRV_URI)
#log.info('device address: %s', DEV_ADDR)
#log.info('device port: %s', DEV_PORT)
#log.info('work dir: %s', WORK_DIR)
# Create agent config.
agent_config = {
'driver_config': DVR_CONFIG,
'stream_config': self._stream_config,
'agent': {
'resource_id': IA_RESOURCE_ID
},
'test_mode': True,
'forget_past': True,
'enable_persistence': False
}
#if org_governance_name is not None:
# agent_config['org_governance_name'] = org_governance_name
# Start instrument agent.
log.info("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
log.info("Agent setup")
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
self.addCleanup(self._verify_agent_reset)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
log.info('test setup complete')
###############################################################################
# Port agent helpers.
###############################################################################
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
###############################################################################
# Event helpers.
###############################################################################
def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
"""
Start a subscriber to the instrument agent events.
@param type The type of event to catch.
@count Trigger the async event result when events received reaches this.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
# Event array and async event result.
self._event_count = count
self._events_received = []
self._async_event_result = AsyncResult()
self._event_subscriber = EventSubscriber(event_type=type,
callback=consume_event,
origin=IA_RESOURCE_ID)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=5)
def _stop_event_subscriber(self):
"""
Stop event subscribers on cleanup.
"""
self._event_subscriber.stop()
self._event_subscriber = None
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
def _start_data_subscribers(self, count, raw_count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._raw_samples_received = []
self._async_sample_result = AsyncResult()
self._async_raw_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received parsed data on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self._samples_received.append(message)
if len(self._samples_received) == count:
self._async_sample_result.set()
def recv_raw_data(message, stream_route, stream_id):
log.info('Received raw data on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self._raw_samples_received.append(message)
if len(self._raw_samples_received) == raw_count:
self._async_raw_sample_result.set()
from pyon.util.containers import create_unique_identifier
stream_name = 'parsed'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" % stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
stream_name = 'raw'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" % stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber, 'subscription_id'):
try:
pubsub_client.deactivate_subscription(
subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
###############################################################################
# Tests.
###############################################################################
@unittest.skip('Test should be run manually only.')
def test_initialize(self):
"""
test_initialize
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
print '#### in test'
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
@unittest.skip('Test should be run manually only.')
def test_xx(self):
"""
"""
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
with self.assertRaises(Conflict):
res_state = self._ia_client.get_resource_state()
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print '################################# sbe54 came up in state: ' + state
if state == ResourceAgentState.IDLE:
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
elif state == ResourceAgentState.STREAMING:
cmd = AgentCommand(command='DRIVER_EVENT_STOP_AUTOSAMPLE')
retval = self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
state = self._ia_client.get_agent_state()
print '################################# sbe54 now in state: ' + state
gevent.sleep(60 * 2.25)
state = self._ia_client.get_agent_state()
print '################################# sbe54 now in state: ' + state
"""
'DRIVER_EVENT_START_AUTOSAMPLE'
self.assertEqual(state, ResourceAgentState.IDLE)
res_state = self._ia_client.get_resource_state()
self.assertEqual(res_state, DriverProtocolState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
res_state = self._ia_client.get_resource_state()
self.assertEqual(res_state, DriverProtocolState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
with self.assertRaises(Conflict):
retval = self._ia_client.execute_resource(cmd)
"""
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete, index)
IndexManagementService._es_call(es.index_delete, index)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self,
user_name='',
instrument_id='',
product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest(
name='notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest(
name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(
notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(
notification=notification_request_2, user_id=user_id)
log.debug(
"test_activateInstrumentSample: create_user_notifications user_id %s",
str(user_id))
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(
self, expected_instrument_device_id='', extended_instrument=None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version,
ComputedFloatValue)
self.assertIsInstance(
extended_instrument.computed.last_data_received_datetime,
ComputedFloatValue)
self.assertIsInstance(
extended_instrument.computed.last_calibration_datetime,
ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
self.assertIsInstance(
extended_instrument.computed.power_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.communications_status_roll_up,
ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.location_status_roll_up,
ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
self.assertEqual(
extended_instrument.computed.communications_status_roll_up.value,
StatusType.STATUS_WARNING)
self.assertEqual(
extended_instrument.computed.data_status_roll_up.value,
StatusType.STATUS_OK)
self.assertEqual(
extended_instrument.computed.power_status_roll_up.value,
StatusType.STATUS_WARNING)
# Verify the computed attribute for user notification requests
self.assertEqual(
1,
len(extended_instrument.computed.user_notification_requests.value))
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_instrument_device_id)
self.assertEqual(notification.origin_type, "instrument")
self.assertEqual(notification.event_type, 'ResourceLifecycleEvent')
def _check_computed_attributes_of_extended_product(
self, expected_data_product_id='', extended_data_product=None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s",
extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_data_product.computed.product_download_size_estimated,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.number_active_subscriptions,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events,
ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance,
ComputedDictValue)
self.assertIsInstance(
extended_data_product.computed.user_notification_requests,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.active_user_subscriptions,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.past_user_subscriptions,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.data_contents_updated,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime,
ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertTrue('ok' in extended_data_product.computed.last_granule.
value['quality_flag'])
self.assertEqual(
2, len(extended_data_product.computed.data_datetime.value))
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_data_product_id)
self.assertEqual(notification.origin_type, "data product")
self.assertEqual(notification.event_type, 'DetectionEvent')
@attr('LOCOINT')
@unittest.skipIf(not use_es, 'No ElasticSearch')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
#Create stream alarms
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
# kwargs = {
# 'name' : 'test_sim_warning',
# 'stream_name' : 'parsed',
# 'value_id' : 'temp',
# 'message' : 'Temperature is above test range of 5.0.',
# 'type' : StreamAlarmType.WARNING,
# 'upper_bound' : 5.0,
# 'upper_rel_op' : '<'
# }
kwargs = {
'name': 'temperature_warning_interval',
'stream_name': 'parsed',
'value_id': 'temp',
'message':
'Temperature is below the normal range of 50.0 and above.',
'type': StreamAlarmType.WARNING,
'lower_bound': 50.0,
'lower_rel_op': '<'
}
# Create alarm object.
alarm = {}
alarm['type'] = 'IntervalAlarmDef'
alarm['kwargs'] = kwargs
raw_config = StreamConfiguration(
stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5)
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2,
granule_publish_rate=5,
alarms=[alarm])
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=
"http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification(user_name='user_1',
instrument_id=instDevice_id,
product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification(user_name='user_2',
instrument_id=instDevice_id,
product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('Data product streams2 = %s', str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s', dataset_ids[0])
self.raw_dataset = dataset_ids[0]
#elastic search debug
es_indexes, _ = self.container.resource_registry.find_resources(
restype='ElasticSearchIndex')
log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes])
log.debug('Bootstrap %s', CFG.bootstrap.use_es)
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#setup a subscriber to alarm events from the device
self._events_received = []
self._event_count = 0
self._samples_out_of_range = 0
self._samples_complete = False
self._async_sample_result = AsyncResult()
def consume_event(*args, **kwargs):
log.debug(
'TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
self._event_count = len(self._events_received)
self._async_sample_result.set()
self._event_subscriber = EventSubscriber(
event_type=
'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent
callback=consume_event,
origin=instDevice_id)
self._event_subscriber.start()
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
def stop_subscriber():
self._event_subscriber.stop()
self._event_subscriber = None
self.addCleanup(stop_subscriber)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
inst_agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
inst_agent_instance_obj.agent_process_id)
log.debug('Instrument agent instance obj: = %s',
str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(
instDevice_id,
to_name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s",
str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
for i in xrange(10):
retval = self._ia_client.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from sample %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
self._samples_complete = True
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("test_activateInstrumentSample: RDT parsed: %s",
str(rdt.pretty_print()))
temp_vals = rdt['temp']
self.assertEquals(len(temp_vals), 10)
log.debug("test_activateInstrumentSample: all temp_vals: %s",
temp_vals)
#out_of_range_temp_vals = [i for i in temp_vals if i > 5]
out_of_range_temp_vals = [i for i in temp_vals if i < 50.0]
log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s",
out_of_range_temp_vals)
self._samples_out_of_range = len(out_of_range_temp_vals)
# if no bad values were produced, then do not wait for an event
if self._samples_out_of_range == 0:
self._async_sample_result.set()
log.debug("test_activateInstrumentSample: _events_received: %s",
self._events_received)
log.debug("test_activateInstrumentSample: _event_count: %s",
self._event_count)
self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout)
replay_data = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("RDT raw: %s", str(rdt.pretty_print()))
raw_vals = rdt['raw']
self.assertEquals(len(raw_vals), 10)
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.
value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id1,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
#put some events into the eventsdb to test - this should set the comms and data status to WARNING
#--------------------------------------------------------------------------------
t = get_ion_ts()
self.event_publisher.publish_event(ts_created=t,
event_type='DeviceStatusEvent',
origin=instDevice_id,
state=DeviceStatusType.OUT_OF_RANGE,
values=[200])
self.event_publisher.publish_event(
ts_created=t,
event_type='DeviceCommsEvent',
origin=instDevice_id,
state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION,
lapse_interval_seconds=20)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_1)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id2,
extended_data_product=extended_product)
#---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING ---------
t = get_ion_ts()
self.event_publisher.publish_event(ts_created=t,
event_type='DeviceStatusEvent',
origin=instDevice_id,
state=DeviceStatusType.OUT_OF_RANGE,
values=[200])
self.event_publisher.publish_event(
ts_created=t,
event_type='DeviceCommsEvent',
origin=instDevice_id,
state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION,
lapse_interval_seconds=20)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class TestActivateRSNVel3DInstrument(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateRSNVel3DInstrument, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
@attr('LOCOINT')
@unittest.skip('under construction')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 180}}})
def test_activate_rsn_vel3d(self):
log.info("--------------------------------------------------------------------------------------------------------")
# load_parameter_scenarios
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict(
op="load",
scenario="BETA",
path="master",
categories="ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition",
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='Vel3DMModel',
description="Vel3DMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug( 'test_activate_rsn_vel3d new InstrumentModel id = %s ', instModel_id)
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw' )
vel3d_b_sample = StreamConfiguration(stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample')
vel3d_b_engineering = StreamConfiguration(stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering')
RSN_VEL3D_01 = {
'DEV_ADDR' : "10.180.80.6",
'DEV_PORT' : 2101,
'DATA_PORT' : 1026,
'CMD_PORT' : 1025,
'PA_BINARY' : "port_agent"
}
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='Vel3DAgent',
description="Vel3DAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg",
stream_configurations = [raw_config, vel3d_b_sample, vel3d_b_engineering])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='Vel3DDevice',
description="Vel3DDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s " , instDevice_id)
port_agent_config = {
'device_addr': '10.180.80.6',
'device_port': 2101,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': 1025,
'data_port': 1026,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='Vel3DAgentInstance',
description="Vel3DAgentInstance",
port_agent_config = port_agent_config,
alerts= [])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_sample', id_only=True)
parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition(name='vel3d_b_sample', parameter_dictionary_id=parsed_sample_pdict_id)
parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_engineering', id_only=True)
parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition(name='vel3d_b_engineering', parameter_dictionary_id=parsed_eng_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_sample',
description='vel3d_b_sample',
temporal_domain = tdom,
spatial_domain = sdom)
sample_data_product_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_sample_stream_def_id)
log.debug( 'new dp_id = %s' , sample_data_product_id)
self.dpclient.activate_data_product_persistence(data_product_id=sample_data_product_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=sample_data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasStream, None, True)
log.debug('sample_data_product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for sample_data_product = %s' , dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('vel3d_b_sample', stream_ids[0] )
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_engineering',
description='vel3d_b_engineering',
temporal_domain = tdom,
spatial_domain = sdom)
eng_data_product_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id)
log.debug( 'new dp_id = %s' , eng_data_product_id)
self.dpclient.activate_data_product_persistence(data_product_id=eng_data_product_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=eng_data_product_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug('test_activate_rsn_vel3d Data product streams2 = %s' , str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s' , dataset_ids[0])
self.raw_dataset = dataset_ids[0]
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
def check_state(label, desired_state):
actual_state = self._ia_client.get_agent_state()
log.debug("%s instrument agent is in state '%s'", label, actual_state)
self.assertEqual(desired_state, actual_state)
log.debug("test_activate_rsn_vel3d: got ia client %s" , str(self._ia_client))
check_state("just-spawned", ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: initialize %s" , str(retval))
check_state("initialized", ResourceAgentState.INACTIVE)
log.debug("test_activate_rsn_vel3d Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: return value from go_active %s" , str(reply))
check_state("activated", ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("current state after sending go_active command %s" , str(state))
#
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
check_state("commanded", ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("current state after sending run command %s" , str(state))
# cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE)
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# gevent.sleep(5)
#
# cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE)
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
# retval = self._ia_client.execute_agent(cmd)
# state = retval.result
# log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state))
#
# cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.STOPPED, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.IDLE, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.RUN)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
log.debug( "test_activate_rsn_vel3d: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: return from reset %s" , str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) )
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(sample_data_product_id)
self.dpclient.delete_data_product(eng_data_product_id)
self.dpclient.delete_data_product(data_product_id2)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete, index)
IndexManagementService._es_call(es.index_delete, index)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self,
user_name='',
instrument_id='',
product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest(
name='notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest(
name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(
notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(
notification=notification_request_2, user_id=user_id)
log.debug(
"test_activateInstrumentSample: create_user_notifications user_id %s",
str(user_id))
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(
self, expected_instrument_device_id='', extended_instrument=None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_instrument.computed.last_data_received_datetime,
ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
self.assertIsInstance(
extended_instrument.computed.power_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.communications_status_roll_up,
ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.location_status_roll_up,
ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
# Verify the computed attribute for user notification requests
self.assertEqual(
1,
len(extended_instrument.computed.user_notification_requests.value))
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_instrument_device_id, notification.origin)
self.assertEqual("instrument", notification.origin_type)
self.assertEqual('ResourceLifecycleEvent', notification.event_type)
def _check_computed_attributes_of_extended_product(
self, expected_data_product_id='', extended_data_product=None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s",
extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_data_product.computed.product_download_size_estimated,
ComputedFloatValue)
self.assertIsInstance(
extended_data_product.computed.number_active_subscriptions,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events,
ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance,
ComputedDictValue)
self.assertIsInstance(
extended_data_product.computed.user_notification_requests,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.active_user_subscriptions,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.past_user_subscriptions,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.data_contents_updated,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime,
ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertIn(
'ok',
extended_data_product.computed.last_granule.value['quality_flag'])
self.assertEqual(
2, len(extended_data_product.computed.data_datetime.value))
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_data_product_id, notification.origin)
self.assertEqual("data product", notification.origin_type)
self.assertEqual('DetectionEvent', notification.event_type)
@attr('LOCOINT')
#@unittest.skip('refactoring')
@unittest.skipIf(not use_es, 'No ElasticSearch')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 90}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='raw')
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config,
alerts=[])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification(user_name='user_1',
instrument_id=instDevice_id,
product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification(user_name='user_2',
instrument_id=instDevice_id,
product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('Data product streams2 = %s', str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s', dataset_ids[0])
self.raw_dataset = dataset_ids[0]
#elastic search debug
es_indexes, _ = self.container.resource_registry.find_resources(
restype='ElasticSearchIndex')
log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes])
log.debug('Bootstrap %s', CFG.bootstrap.use_es)
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id, ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.INACTIVE, state)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s",
str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.STOPPED, state)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
for i in xrange(10):
retval = self._ia_client.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from sample %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()))
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
all_raw = "".join(raw_vals)
# look for 't' entered after a prompt -- ">t"
t_commands = all_raw.count(">t")
if 10 != t_commands:
log.error("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals):
log.error("raw val %s: %s", i, [r])
self.fail("Expected 10 't' strings in raw_vals, got %s" %
t_commands)
else:
log.debug("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals):
log.debug("raw val %s: %s", i, [r])
replay_data_parsed = self.dataretrieverclient.retrieve(
self.parsed_dataset)
self.assertIsInstance(replay_data_parsed, Granule)
rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed)
log.debug("test_activateInstrumentSample: RDT parsed: %s",
str(rdt_parsed.pretty_print()))
self.assertIn('temp', rdt_parsed)
temp_vals = rdt_parsed['temp']
pressure_vals = rdt_parsed['pressure']
if 10 != len(temp_vals):
log.error("%s temp_vals: %s", len(temp_vals), temp_vals)
self.fail("Expected 10 temp_vals, got %s" % len(temp_vals))
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.
value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id1,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_1)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id2,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
#@unittest.skip("TBD")
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
stream_configuration= {'raw': 'ctd_raw_param_dict' , 'parsed': 'ctd_parsed_param_dict' })
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
print 'new InstrumentModel id = %s ' % instModel_id
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver" )
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
print 'new InstrumentAgent id = %s' % instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
print 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' +\
'(SA Req: L4-CI-SA-RQ-241) '
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
print "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " %\
instDevice_id
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4003,
'data_port': 4000,
'log_level': 5,
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new dp_id = %s' % data_product_id1
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
print 'Data product streams1 = %s' % stream_ids
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
print 'Data set for data_product_id1 = %s' % dataset_ids[0]
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
self.get_datastore(self.parsed_dataset)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
pid = self.create_logger('ctd_parsed', stream_ids[0] )
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
print 'new dp_id = %s' % str(data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
print 'Data product streams2 = %s' % str(stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
print 'Data set for data_product_id2 = %s' % dataset_ids[0]
self.raw_dataset = dataset_ids[0]
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
print "test_activateInstrumentSample: got ia client %s" % str(self._ia_client)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
print "test_activateInstrumentSample: initialize %s" % str(retval)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
print "(L4-CI-SA-RQ-334): Sending go_active command "
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
print "test_activateInstrument: return value from go_active %s" % str(reply)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
print "(L4-CI-SA-RQ-334): current state after sending go_active command %s" % str(state)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
print "test_activateInstrumentSample: run %s" % str(reply)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
print "test_activateInstrumentSample: return from sample %s" % str(retval)
retval = self._ia_client.execute_resource(cmd)
print "test_activateInstrumentSample: return from sample %s" % str(retval)
retval = self._ia_client.execute_resource(cmd)
print "test_activateInstrumentSample: return from sample %s" % str(retval)
print "test_activateInstrumentSample: calling reset "
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
print "test_activateInstrumentSample: return from reset %s" % str(reply)
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("RDT parsed: %s", str(rdt.pretty_print()) )
temp_vals = rdt['temp']
self.assertTrue(len(temp_vals) == 3)
replay_data = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("RDT raw: %s", str(rdt.pretty_print()) )
raw_vals = rdt['raw']
self.assertTrue(len(raw_vals) == 3)
print "l4-ci-sa-rq-138"
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
# #--------------------------------------------------------------------------------
# # Get the extended data product to see if it contains the granules
# #--------------------------------------------------------------------------------
# extended_product = self.dpclient.get_data_product_extension(data_product_id1)
# self.assertEqual(data_product_id1, extended_product._id)
# log.debug( "test_activateInstrumentSample: extended_product.computed.last_granule.value %s", str(extended_product.computed.last_granule.value) )
# log.debug( "test_activateInstrumentSample: extended_product.computed.recent_granules.value %s", str(extended_product.computed.recent_granules.value) )
# log.debug("test_activateInstrumentSample: extended_product.computed.provenance_product_list.value %s", str(extended_product.computed.provenance_product_list.value) )
#-------------------------------
# Deactivate loggers
#-------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
def test_xxx(self, host='localhost', port=DATA_PORT,
resource_id=IA_RESOURCE_ID, stream_config=None):
"""
"""
if not stream_config:
stream_config = self._stream_config
log.info("create FakePuckReader")
puck_reader = FakePuckReader()
log.info("read_puck()")
puck_data = puck_reader.read_puck(DEV_ADDR, DEV_PORT, DEV_SERIAL_LINE)
driver_config = {
'dvr_mod' : puck_data['dvr_mod'],
'dvr_cls' : puck_data['dvr_cls'],
'workdir' : WORK_DIR,
'process_type' : PROCESS_TYPE,
'comms_config' : {
'addr' : host,
'port' : port
}
}
agent_config = {
'driver_config' : driver_config,
'stream_config' : stream_config,
'agent' : {'resource_id': resource_id},
'test_mode' : True
}
log.debug("Starting instrument agent.")
ia_pid = self.container_client.spawn_process(
name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
#self.addCleanup(self._verify_agent_reset)
ia_client = ResourceAgentClient(resource_id, process=FakeProcess())
log.info('Got ia client %s.', str(ia_client))
# We start in uninitialized state.
# In this state there is no driver process.
state = ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = ia_client.ping_agent()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = ia_client.execute_agent(cmd)
state = ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = ia_client.execute_agent(cmd)
state = ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = ia_client.execute_agent(cmd)
state = ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = ia_client.execute_resource(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = ia_client.execute_agent(cmd)
state = ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class ExternalDatasetAgentTestBase(object):
# Agent parameters.
EDA_RESOURCE_ID = '123xyz'
EDA_NAME = 'ExampleEDA'
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Initialize test members.
"""
#log.warn('Starting the container')
# Start container.
self._start_container()
# Bring up services in a deploy file
#log.warn('Starting the rel')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a pubsub client to create streams.
# log.warn('Init a pubsub client')
self._pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# log.warn('Init a ContainerAgentClient')
self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
# TODO: Finish dealing with the resources and whatnot
# TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources()
self._setup_resources()
#TG: Setup/configure the granule logger to log granules as they're published
# Create agent config.
agent_config = {
'driver_config': self.DVR_CONFIG,
'stream_config': {},
'agent': {'resource_id': self.EDA_RESOURCE_ID},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
log.debug('TestInstrumentAgent.setup(): starting EDA.')
self._ia_pid = self._container_client.spawn_process(
name=self.EDA_NAME,
module=self.EDA_MOD,
cls=self.EDA_CLS,
config=agent_config
)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
########################################
# Private "setup" functions
########################################
def _setup_resources(self):
raise NotImplementedError('_setup_resources must be implemented in the subclass')
def create_stream_and_logger(self, name, stream_id='', pdict=None):
stream_def_id = ''
if not stream_id or stream_id is '':
if pdict:
stream_def_id = self._pubsub_client.create_stream_definition(parameter_dictionary=pdict.dump(), stream_type='stream')
stream_id, route = self._pubsub_client.create_stream(name=name, exchange_point='science_data', stream_definition_id=stream_def_id)
else:
route = self._pubsub_client.read_stream_route(stream_id=stream_id)
stream_def = self._pubsub_client.read_stream_definition(stream_id=stream_id)
stream_def_id = stream_def._id
pid = self._container_client.spawn_process(
name=name + '_logger',
module='ion.processes.data.stream_granule_logger',
cls='StreamGranuleLogger',
config={'process': {'stream_id': stream_id}}
)
log.info('Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'.format(pid, stream_id))
return stream_id, route, stream_def_id
def _start_finished_event_subscriber(self):
def consume_event(*args, **kwargs):
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(len(self._finished_events_received))
log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.start()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.stop()
self._finished_event_subscriber = None
########################################
# Custom assertion functions
########################################
def assertListsEqual(self, lst1, lst2):
lst1.sort()
lst2.sort()
return lst1 == lst2
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
#{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
self.assertTrue(isinstance(val, dict))
self.assertTrue('c' in val)
self.assertTrue('t' in val)
self.assertTrue('p' in val)
self.assertTrue('time' in val)
c = val['c'][0]
t = val['t'][0]
p = val['p'][0]
time = val['time'][0]
self.assertTrue(isinstance(c, float))
self.assertTrue(isinstance(t, float))
self.assertTrue(isinstance(p, float))
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(key in PARAMS)
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val * .01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
########################################
# Test functions
########################################
def test_acquire_data_while_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
params = {
'POLLING_INTERVAL': 3
}
self._ia_client.set_resource(params)
self._finished_count = 1
cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE)
self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STREAMING)
config = get_safe(self.DVR_CONFIG, 'dh_cfg', {})
log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1')
config['stream_id'], config['stream_route'], _ = self.create_stream_and_logger(name='stream_id_for_historical_1')
config['constraints'] = self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE, args=[config])
self._ia_client.execute_resource(cmd)
cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE)
self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
finished = self._async_finished_result.get(timeout=120)
self.assertEqual(finished, self._finished_count)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_acquire_data(self):
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
log.warn('Send an unconstrained request for data (\'new data\')')
cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE)
self._ia_client.execute_resource(command=cmd)
state = self._ia_client.get_agent_state()
log.info(state)
self.assertEqual(state, ResourceAgentState.COMMAND)
self._finished_count = 2
config_mods = {}
log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1')
config_mods['stream_id'], config_mods['stream_route'], _ = self.create_stream_and_logger(name='stream_id_for_historical_1')
config_mods['constraints'] = self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE, args=[config_mods])
self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
log.info('Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2')
config_mods['stream_id'], config_mods['stream_route'], _ = self.create_stream_and_logger(name='stream_id_for_historical_2')
config_mods['constraints'] = self.HIST_CONSTRAINTS_2
cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE, args=[config_mods])
self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
finished = self._async_finished_result.get(timeout=120)
self.assertEqual(finished, self._finished_count)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_streaming(self):
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
params = {
'POLLING_INTERVAL': 3
}
self._ia_client.set_resource(params)
self._finished_count = 3
cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE)
self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STREAMING)
#Assert that data was received
# self._async_finished_result.get(timeout=600)
# self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE)
self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_command(self):
# Test instrument driver get and set interface.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Retrieve all resource parameters.
reply = self._ia_client.get_resource(params=['DRIVER_PARAMETER_ALL'])
self.assertParamDict(reply, True)
## Retrieve a subset of resource parameters.
params = [
'POLLING_INTERVAL'
]
reply = self._ia_client.get_resource(params=params)
self.assertParamDict(reply)
orig_params = reply
# Set a subset of resource parameters.
new_params = {
'POLLING_INTERVAL': (orig_params['POLLING_INTERVAL'] * 2),
}
self._ia_client.set_resource(params=new_params)
check_new_params = self._ia_client.get_resource(params)
self.assertParamVals(check_new_params, new_params)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_get_set_resource(self):
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
# Get a couple parameters
retval = self._ia_client.get_resource(['POLLING_INTERVAL', 'PATCHABLE_CONFIG_KEYS'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(type(retval['POLLING_INTERVAL']), int)
self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']), list)
# Attempt to get a parameter that doesn't exist
log.debug('Try getting a non-existent parameter \'BAD_PARAM\'')
with self.assertRaises(ServerError):
self._ia_client.get_resource(['BAD_PARAM'])
# Set the polling_interval to a new value, then get it to make sure it set properly
self._ia_client.set_resource({'POLLING_INTERVAL': 10})
retval = self._ia_client.get_resource(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(retval['POLLING_INTERVAL'], 10)
# Attempt to set a parameter that doesn't exist
log.debug('Try setting a non-existent parameter \'BAD_PARAM\'')
with self.assertRaises(ServerError):
self._ia_client.set_resource({'BAD_PARAM': 'bad_val'})
# Attempt to set one parameter that does exist, and one that doesn't
with self.assertRaises(ServerError):
self._ia_client.set_resource({'POLLING_INTERVAL': 20, 'BAD_PARAM': 'bad_val'})
retval = self._ia_client.get_resource(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(retval['POLLING_INTERVAL'], 20)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_initialize(self):
# Test agent initialize command. This causes creation of driver process and transition to inactive.
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_states(self):
# Test agent state transitions.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE)
self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STREAMING)
cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE)
self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_capabilities(self):
"""
Test the ability to retrieve agent and resource parameter and command
capabilities in various system states.
"""
# Test the ability to retrieve agent and resource parameter and command capabilities.
acmds = self._ia_client.get_capabilities(['AGT_CMD'])
log.debug('Agent Commands: {0}'.format(acmds))
# acmds = [item[1] for item in acmds]
self.assertListsEqual(acmds, AGT_CMDS.keys())
apars = self._ia_client.get_capabilities(['AGT_PAR'])
log.debug('Agent Parameters: {0}'.format(apars))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
rcmds = self._ia_client.get_capabilities(['RES_CMD'])
log.debug('Resource Commands: {0}'.format(rcmds))
# rcmds = [item[1] for item in rcmds]
self.assertListsEqual(rcmds, CMDS.keys())
rpars = self._ia_client.get_capabilities(['RES_PAR'])
log.debug('Resource Parameters: {0}'.format(rpars))
# rpars = [item[1] for item in rpars]
self.assertListsEqual(rpars, PARAMS.keys())
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_errors(self):
# Test illegal behavior and replies.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Can't go active in unitialized state.
# Status 660 is state error.
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
with self.assertRaises(Conflict):
self._ia_client.execute_agent(cmd)
# Can't command driver in this state.
cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE)
with self.assertRaises(Conflict):
self._ia_client.execute_resource(cmd)
#self.assertEqual(reply.status, 660)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# 404 unknown agent command.
cmd = AgentCommand(command='kiss_edward')
with self.assertRaises(BadRequest):
self._ia_client.execute_agent(cmd)
# 670 unknown driver command.
cmd = AgentCommand(command='acquire_sample_please')
with self.assertRaises(ServerError):
self._ia_client.execute_resource(cmd)
# 630 Parameter error.
#self.assertRaises(InstParameterError, self._ia_client.get_param, 'bogus bogus')
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestPlatformAgent(IonIntegrationTestCase, HelperTestMixin):
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
network_definition_ser = NetworkUtil.serialize_network_definition(
network_definition)
if log.isEnabledFor(logging.DEBUG):
log.debug("NetworkDefinition serialization:\n%s",
network_definition_ser)
cls.PLATFORM_CONFIG = {
'platform_id': cls.PLATFORM_ID,
'driver_config': DVR_CONFIG,
'network_definition': network_definition_ser
}
NetworkUtil._gen_open_diagram(
network_definition.pnodes[cls.PLATFORM_ID])
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# Start data subscribers, add stop to cleanup.
# Define stream_config.
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# start event subscriber:
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
self._agent_config = {
'agent': {
'resource_id': PA_RESOURCE_ID
},
'stream_config': self._stream_config,
# pass platform config here
'platform_config': self.PLATFORM_CONFIG
}
if log.isEnabledFor(logging.TRACE):
log.trace("launching with agent_config=%s" %
str(self._agent_config))
self._launcher = LauncherFactory.createLauncher()
self._pid = self._launcher.launch(self.PLATFORM_ID, self._agent_config)
log.debug("LAUNCHED PLATFORM_ID=%r", self.PLATFORM_ID)
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient(PA_RESOURCE_ID,
process=FakeProcess())
log.info('Got pa client %s.' % str(self._pa_client))
def tearDown(self):
try:
self._launcher.cancel_process(self._pid)
finally:
super(TestPlatformAgent, self).tearDown()
def _start_data_subscribers(self):
"""
"""
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
#
# TODO retrieve appropriate stream definitions; for the moment, using
# adhoc_get_stream_names
#
# A callback for processing subscribed-to data.
def consume_data(message, stream_route, stream_id):
log.info('Subscriber received data message: %s.' % str(message))
self._samples_received.append(message)
self._async_data_result.set()
for stream_name in adhoc_get_stream_names():
log.info('creating stream %r ...', stream_name)
# TODO use appropriate exchange_point
stream_id, stream_route = self._pubsub_client.create_stream(
name=stream_name, exchange_point='science_data')
log.info('create_stream(%r): stream_id=%r, stream_route=%s',
stream_name, stream_id, str(stream_route))
pdict = adhoc_get_parameter_dictionary(stream_name)
stream_config = dict(stream_route=stream_route.routing_key,
stream_id=stream_id,
parameter_dictionary=pdict.dump())
self._stream_config[stream_name] = stream_config
log.info('_stream_config[%r]= %r', stream_name, stream_config)
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self._pubsub_client.create_subscription(
name=exchange_name, stream_ids=[stream_id])
self._pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self._pubsub_client.deactivate_subscription(
sub.subscription_id)
except:
pass
self._pubsub_client.delete_subscription(sub.subscription_id)
sub.stop()
def _start_event_subscriber(self,
event_type="DeviceEvent",
sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(
sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
#def _execute_agent(self, cmd, timeout=TIMEOUT):
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command,
cmd.kwargs)
time_start = time.time()
#retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s",
cmd.command, elapsed_time, str(retval))
return retval
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
#self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _connect_instrument(self):
#
# TODO more realistic settings for the connection
#
port_id = self.PORT_ID
instrument_id = self.INSTRUMENT_ID
instrument_attributes = self.INSTRUMENT_ATTRIBUTES_AND_VALUES
kwargs = dict(port_id=port_id,
instrument_id=instrument_id,
attributes=instrument_attributes)
cmd = AgentCommand(command=PlatformAgentEvent.CONNECT_INSTRUMENT,
kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("CONNECT_INSTRUMENT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
returned_attrs = self._verify_valid_instrument_id(
instrument_id, result[port_id])
if isinstance(returned_attrs, dict):
for attrName in instrument_attributes:
self.assertTrue(attrName in returned_attrs)
def _get_connected_instruments(self):
port_id = self.PORT_ID
kwargs = dict(port_id=port_id, )
cmd = AgentCommand(
command=PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("GET_CONNECTED_INSTRUMENTS = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
instrument_id = self.INSTRUMENT_ID
self.assertTrue(instrument_id in result[port_id])
def _disconnect_instrument(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
instrument_id = self.INSTRUMENT_ID
kwargs = dict(port_id=port_id, instrument_id=instrument_id)
cmd = AgentCommand(command=PlatformAgentEvent.DISCONNECT_INSTRUMENT,
kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("DISCONNECT_INSTRUMENT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
self.assertTrue(instrument_id in result[port_id])
self._verify_instrument_disconnected(instrument_id,
result[port_id][instrument_id])
def _turn_on_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_ON_PORT,
kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_ON_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_ON)
def _turn_off_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_OFF_PORT,
kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_OFF_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_OFF)
def _get_resource(self):
attrNames = self.ATTR_NAMES
#
# OOIION-631: use get_ion_ts() as a basis for using system time, which is
# a string.
#
cur_time = get_ion_ts()
from_time = str(int(cur_time) - 50000) # a 50-sec time window
kwargs = dict(attr_names=attrNames, from_time=from_time)
cmd = AgentCommand(command=PlatformAgentEvent.GET_RESOURCE,
kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attr_name in attrNames:
self._verify_valid_attribute_id(attr_name, attr_values)
def _set_resource(self):
attrNames = self.ATTR_NAMES
writ_attrNames = self.WRITABLE_ATTR_NAMES
# do valid settings:
# TODO more realistic value depending on attribute's type
attrs = [(attrName, self.VALID_ATTR_VALUE) for attrName in attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE,
kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in attrNames:
if attrName in writ_attrNames:
self._verify_valid_attribute_id(attrName, attr_values)
else:
self._verify_not_writable_attribute_id(attrName, attr_values)
# try invalid settings:
# set invalid values to writable attributes:
attrs = [(attrName, self.INVALID_ATTR_VALUE)
for attrName in writ_attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE,
kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in writ_attrNames:
self._verify_attribute_value_out_of_range(attrName, attr_values)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _pause(self):
cmd = AgentCommand(command=PlatformAgentEvent.PAUSE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.STOPPED)
def _resume(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESUME)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _start_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.MONITORING)
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
# just wait for at least one -- see consume_data
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("Received samples: %s", len(self._samples_received))
def _stop_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _get_subplatform_ids(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_SUBPLATFORM_IDS)
retval = self._execute_agent(cmd)
self.assertIsInstance(retval.result, list)
self.assertTrue(x in retval.result for x in self.SUBPLATFORM_IDS)
return retval.result
def _wait_for_external_event(self):
log.info("waiting for reception of an external event...")
# just wait for at least one -- see consume_event
self._async_event_result.get(timeout=EVENT_TIMEOUT)
self.assertTrue(len(self._events_received) >= 1)
log.info("Received events: %s", len(self._events_received))
def _check_sync(self):
cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC)
retval = self._execute_agent(cmd)
log.info("CHECK_SYNC result: %s", retval.result)
self.assertTrue(retval.result is not None)
self.assertEquals(retval.result[0:3], "OK:")
return retval.result
def test_capabilities(self):
agt_cmds_all = [
PlatformAgentEvent.INITIALIZE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.PAUSE,
PlatformAgentEvent.RESUME,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.START_MONITORING,
PlatformAgentEvent.STOP_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
def sort_caps(caps):
agt_cmds = []
agt_pars = []
res_cmds = []
res_pars = []
if len(caps) > 0 and isinstance(caps[0], AgentCapability):
agt_cmds = [
x.name for x in caps
if x.cap_type == CapabilityType.AGT_CMD
]
agt_pars = [
x.name for x in caps
if x.cap_type == CapabilityType.AGT_PAR
]
res_cmds = [
x.name for x in caps
if x.cap_type == CapabilityType.RES_CMD
]
res_pars = [
x.name for x in caps
if x.cap_type == CapabilityType.RES_PAR
]
elif len(caps) > 0 and isinstance(caps[0], dict):
agt_cmds = [
x['name'] for x in caps
if x['cap_type'] == CapabilityType.AGT_CMD
]
agt_pars = [
x['name'] for x in caps
if x['cap_type'] == CapabilityType.AGT_PAR
]
res_cmds = [
x['name'] for x in caps
if x['cap_type'] == CapabilityType.RES_CMD
]
res_pars = [
x['name'] for x in caps
if x['cap_type'] == CapabilityType.RES_PAR
]
return agt_cmds, agt_pars, res_cmds, res_pars
agt_pars_all = ['example'
] # 'cause ResourceAgent defines aparam_example
res_pars_all = []
res_cmds_all = []
##################################################################
# UNINITIALIZED
##################################################################
self._assert_state(PlatformAgentState.UNINITIALIZED)
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_uninitialized = [
PlatformAgentEvent.INITIALIZE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_uninitialized)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(current_state=False)
# Validate all capabilities as read from state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# INACTIVE
##################################################################
self._initialize()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_inactive = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_inactive)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# IDLE
##################################################################
self._go_active()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_idle = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_idle)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states as read from IDLE.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# COMMAND
##################################################################
self._run()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_command = [
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.PAUSE,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.START_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_command)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
##################################################################
# STOPPED
##################################################################
self._pause()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state STOPPED
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_stopped = [
PlatformAgentEvent.RESUME,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_stopped)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
# back to COMMAND:
self._resume()
##################################################################
# MONITORING
##################################################################
self._start_resource_monitoring()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state MONITORING
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_monitoring = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.STOP_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_monitoring)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
# return to COMMAND state:
self._stop_resource_monitoring()
###################
# ALL CAPABILITIES
###################
# Get exposed capabilities in all states as read from state COMMAND.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_all)
self.assertItemsEqual(res_pars, res_pars_all)
self._go_inactive()
self._reset()
def test_some_state_transitions(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._initialize() # -> INACTIVE
self._reset() # -> UNINITIALIZED
self._initialize() # -> INACTIVE
self._go_active() # -> IDLE
self._reset() # -> UNINITIALIZED
self._initialize() # -> INACTIVE
self._go_active() # -> IDLE
self._run() # -> COMMAND
self._pause() # -> STOPPED
self._resume() # -> COMMAND
self._reset() # -> UNINITIALIZED
def test_get_set_resources(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._get_resource()
self._set_resource()
self._go_inactive()
self._reset()
def test_some_commands(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._ping_agent()
self._ping_resource()
self._get_metadata()
self._get_ports()
self._get_subplatform_ids()
self._go_inactive()
self._reset()
def test_resource_monitoring(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._start_resource_monitoring()
self._wait_for_a_data_sample()
self._stop_resource_monitoring()
self._go_inactive()
self._reset()
def test_external_event_dispatch(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._wait_for_external_event()
self._go_inactive()
self._reset()
def test_connect_disconnect_instrument(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._connect_instrument()
self._turn_on_port()
self._get_connected_instruments()
self._turn_off_port()
self._disconnect_instrument()
self._go_inactive()
self._reset()
def test_check_sync(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._check_sync()
self._connect_instrument()
self._check_sync()
self._disconnect_instrument()
self._check_sync()
self._go_inactive()
self._reset()
class BaseIntTestPlatform(IonIntegrationTestCase, HelperTestMixin):
"""
A base class with several conveniences supporting specific platform agent
integration tests, see:
- ion/agents/platform/test/test_platform_agent_with_rsn.py
- ion/services/sa/observatory/test/test_platform_launch.py
The platform IDs used here are organized as follows:
Node1D -> MJ01C -> LJ01D
where -> goes from parent platform to child platform.
This is a subset of the whole topology defined in the simulated platform
network (network.yml), which in turn is used by the RSN OMS simulator.
- 'LJ01D' is the root platform used in test_single_platform
- 'Node1D' is the root platform used in test_hierarchy
Methods are provided to construct specific platform topologies, but
subclasses decide which to use.
"""
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.org_id = self.RR2.create(any_old(RT.Org))
log.debug("Org created: %s", self.org_id)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
CIOMSClientFactory.destroy_instance(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser)
# set attributes for the platforms:
self._platform_attributes = {}
for platform_id in self._network_definition.pnodes:
pnode = self._network_definition.pnodes[platform_id]
dic = dict((attr.attr_id, attr.defn) for attr in pnode.attrs.itervalues())
self._platform_attributes[platform_id] = dic
log.trace("_platform_attributes: %s", self._platform_attributes)
# set ports for the platforms:
self._platform_ports = {}
for platform_id in self._network_definition.pnodes:
pnode = self._network_definition.pnodes[platform_id]
dic = {}
for port_id, port in pnode.ports.iteritems():
dic[port_id] = dict(port_id=port_id,
network=port.network)
self._platform_ports[platform_id] = dic
log.trace("_platform_ports: %s", self._platform_attributes)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
#################################################################
# data subscribers handling
#################################################################
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s. stream_name=%r stream_id=%r',
str(message), stream_name, stream_id)
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
#################################################################
# event subscribers handling
#################################################################
def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
#################################################################
# config supporting methods
#################################################################
def _create_platform_config_builder(self):
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
return pconfig_builder
def _generate_parent_with_child_config(self, p_parent, p_child):
log.debug("Testing parent platform + child platform as parent config")
pconfig_builder = self._create_platform_config_builder()
pconfig_builder.set_agent_instance_object(p_parent.platform_agent_instance_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
self._verify_parent_config(parent_config,
p_parent.platform_device_id,
p_child.platform_device_id,
is_platform=True)
self._debug_config(parent_config,
"platform_agent_config_%s_->_%s.txt" % (
p_parent.platform_id, p_child.platform_id))
def _generate_platform_with_instrument_config(self, p_obj, i_obj):
log.debug("Testing parent platform + child instrument as parent config")
pconfig_builder = self._create_platform_config_builder()
pconfig_builder.set_agent_instance_object(p_obj.platform_agent_instance_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
self._verify_parent_config(parent_config,
p_obj.platform_device_id,
i_obj.instrument_device_id,
is_platform=False)
self._debug_config(parent_config,
"platform_agent_config_%s_->_%s.txt" % (
p_obj.platform_id, i_obj.instrument_device_id))
def _generate_config(self, platform_agent_instance_obj, platform_id, suffix=''):
pconfig_builder = self._create_platform_config_builder()
pconfig_builder.set_agent_instance_object(platform_agent_instance_obj)
config = pconfig_builder.prepare(will_launch=False)
self._debug_config(config, "platform_agent_config_%s%s.txt" % (platform_id, suffix))
return config
def _get_platform_stream_configs(self):
"""
This method is an adaptation of get_streamConfigs in
test_driver_egg.py
"""
return [
StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
# TODO include a "raw" stream?
]
def _get_instrument_stream_configs(self):
"""
configs copied from test_activate_instrument.py
"""
return [
StreamConfiguration(stream_name='ctd_raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5),
StreamConfiguration(stream_name='ctd_parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2, granule_publish_rate=5)
]
def _create_instrument_config_builder(self):
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
return iconfig_builder
def _generate_instrument_config(self, instrument_agent_instance_obj, instrument_id, suffix=''):
pconfig_builder = self._create_instrument_config_builder()
pconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
config = pconfig_builder.prepare(will_launch=False)
self._debug_config(config, "instrument_agent_config_%s%s.txt" % (instrument_id, suffix))
return config
def _debug_config(self, config, outname):
if log.isEnabledFor(logging.DEBUG):
import pprint
outname = "logs/%s" % outname
try:
pprint.PrettyPrinter(stream=file(outname, "w")).pprint(config)
log.debug("config pretty-printed to %s", outname)
except Exception as e:
log.warn("error printing config to %s: %s", outname, e)
def _verify_child_config(self, config, device_id, is_platform):
for key in required_config_keys:
self.assertIn(key, config)
if is_platform:
self.assertEqual(RT.PlatformDevice, config['device_type'])
for key in DVR_CONFIG.iterkeys():
self.assertIn(key, config['driver_config'])
for key in ['children', 'startup_config']:
self.assertEqual({}, config[key])
else:
self.assertEqual(RT.InstrumentDevice, config['device_type'])
for key in ['children']:
self.assertEqual({}, config[key])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
def _verify_parent_config(self, config, parent_device_id,
child_device_id, is_platform):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(RT.PlatformDevice, config['device_type'])
for key in DVR_CONFIG.iterkeys():
self.assertIn(key, config['driver_config'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
self._verify_child_config(config['children'][child_device_id],
child_device_id, is_platform)
def _create_platform_configuration(self, platform_id, parent_platform_id=None):
"""
This method is an adaptation of test_agent_instance_config in
test_instrument_management_service_integration.py
@param platform_id
@param parent_platform_id
@return a DotDict with various of the constructed elements associated
to the platform.
"""
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
#
# TODO will each platform have its own param dictionary?
#
param_dict_name = 'platform_eng_parsed'
parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
param_dict_name,
id_only=True)
self.parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed',
parameter_dictionary_id=parsed_rpdict_id)
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
driver_config = copy.deepcopy(DVR_CONFIG)
driver_config['attributes'] = self._platform_attributes[platform_id]
driver_config['ports'] = self._platform_ports[platform_id]
log.debug("driver_config: %s", driver_config)
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {
'driver_config': driver_config})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
platform_agent_obj = any_old(RT.PlatformAgent, {
"stream_configurations": self._get_platform_stream_configs(),
'driver_module': DVR_MOD,
'driver_class': DVR_CLS})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, self.org_id)
#######################################
# dataset
log.debug('data product = %s', dp_id)
stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, None, True)
log.debug('Data product stream_ids = %s', stream_ids)
stream_id = stream_ids[0]
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
#######################################
return platform_agent_instance_id, platform_agent_id, platform_device_id, stream_id
log.debug("Making the structure for a platform agent")
# TODO Note: the 'platform_config' entry is a mechanism that the
# platform agent expects to know the platform_id and parent_platform_id.
# Determine how to finally indicate this info.
platform_config = {
'platform_id': platform_id,
'parent_platform_id': parent_platform_id,
}
child_agent_config = {
'platform_config': platform_config
}
platform_agent_instance_child_id, _, platform_device_child_id, stream_id = \
_make_platform_agent_structure(child_agent_config)
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
child_config = self._generate_config(platform_agent_instance_child_obj, platform_id)
self._verify_child_config(child_config, platform_device_child_id,
is_platform=True)
self.platform_device_parent_id = platform_device_child_id
p_obj = DotDict()
p_obj.platform_id = platform_id
p_obj.parent_platform_id = parent_platform_id
p_obj.agent_config = child_config
p_obj.platform_agent_instance_obj = platform_agent_instance_child_obj
p_obj.platform_device_id = platform_device_child_id
p_obj.platform_agent_instance_id = platform_agent_instance_child_id
p_obj.stream_id = stream_id
return p_obj
def _create_platform(self, platform_id, parent_platform_id=None):
"""
The main method to create a platform configuration and do other
preparations for a given platform.
"""
p_obj = self._create_platform_configuration(platform_id, parent_platform_id)
# start corresponding data subscriber:
self._start_data_subscriber(p_obj.platform_agent_instance_id,
p_obj.stream_id)
return p_obj
#################################################################
# platform child-parent linking
#################################################################
def _assign_child_to_parent(self, p_child, p_parent):
log.debug("assigning child platform %r to parent %r",
p_child.platform_id, p_parent.platform_id)
self.RR2.assign_platform_device_to_platform_device(p_child.platform_device_id,
p_parent.platform_device_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(p_parent.platform_device_id)
self.assertNotEqual(0, len(child_device_ids))
self._generate_parent_with_child_config(p_parent, p_child)
#################################################################
# instrument
#################################################################
def _set_up_pre_environment_for_instrument(self):
"""
From test_instrument_agent.py
Basically, this method launches the port agent and the completes the
instrument driver configuration used to properly set up the
instrument agent.
@return instrument_driver_config
"""
import sys
from ion.agents.instrument.driver_process import DriverProcessType
from ion.agents.instrument.driver_process import ZMQEggDriverProcess
# A seabird driver.
DRV_URI = 'http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.7-py2.7.egg'
DRV_MOD = 'mi.instrument.seabird.sbe37smb.ooicore.driver'
DRV_CLS = 'SBE37Driver'
WORK_DIR = '/tmp/'
DELIM = ['<<', '>>']
instrument_driver_config = {
'dvr_egg' : DRV_URI,
'dvr_mod' : DRV_MOD,
'dvr_cls' : DRV_CLS,
'workdir' : WORK_DIR,
'process_type' : None
}
# Launch from egg or a local MI repo.
LAUNCH_FROM_EGG=True
if LAUNCH_FROM_EGG:
# Dynamically load the egg into the test path
launcher = ZMQEggDriverProcess(instrument_driver_config)
egg = launcher._get_egg(DRV_URI)
if not egg in sys.path: sys.path.insert(0, egg)
instrument_driver_config['process_type'] = (DriverProcessType.EGG,)
else:
mi_repo = os.getcwd() + os.sep + 'extern' + os.sep + 'mi_repo'
if not mi_repo in sys.path: sys.path.insert(0, mi_repo)
instrument_driver_config['process_type'] = (DriverProcessType.PYTHON_MODULE,)
instrument_driver_config['mi_repo'] = mi_repo
DEV_ADDR = CFG.device.sbe37.host
DEV_PORT = CFG.device.sbe37.port
DATA_PORT = CFG.device.sbe37.port_agent_data_port
CMD_PORT = CFG.device.sbe37.port_agent_cmd_port
PA_BINARY = CFG.device.sbe37.port_agent_binary
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i', port)
self.addCleanup(self._support.stop_pagent)
# Configure instrument driver to use port agent port number.
instrument_driver_config['comms_config'] = {
'addr': 'localhost',
'port': port,
'cmd_port': CMD_PORT
}
return instrument_driver_config
def _make_instrument_agent_structure(self, org_obj, agent_config=None):
if None is agent_config: agent_config = {}
# from test_activate_instrument:test_activateInstrumentSample
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
# agent creation
instrument_agent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
stream_configurations=self._get_instrument_stream_configs())
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
log.debug('new InstrumentAgent id = %s', instrument_agent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instrument_agent_id)
# device creation
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instrument_device_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instrument_device_id)
log.debug("new InstrumentDevice id = %s ", instrument_device_id)
#Create stream alarms
alert_def = {
'name' : 'temperature_warning_interval',
'stream_name' : 'ctd_parsed',
'message' : 'Temperature is below the normal range of 50.0 and above.',
'alert_type' : StreamAlertType.WARNING,
'value_id' : 'temp',
'resource_id' : instrument_device_id,
'origin_type' : 'device',
'lower_bound' : 50.0,
'lower_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
instrument_driver_config = self._set_up_pre_environment_for_instrument()
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
# instance creation
instrument_agent_instance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
driver_config=instrument_driver_config,
port_agent_config=port_agent_config,
alerts=[alert_def])
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# data products
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(org_obj)
# parsed:
parsed_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(
name='ctd_parsed',
parameter_dictionary_id=parsed_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id1)
# raw:
raw_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(
name='ctd_raw',
parameter_dictionary_id=raw_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id2)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
i_obj = DotDict()
i_obj.instrument_agent_id = instrument_agent_id
i_obj.instrument_device_id = instrument_device_id
i_obj.instrument_agent_instance_id = instrument_agent_instance_id
return i_obj
def verify_instrument_config(self, config, org_obj, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQEggDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['children']:
self.assertEqual({}, config[key])
def _create_instrument(self):
"""
The main method to create an instrument configuration.
"""
iconfig_builder = self._create_instrument_config_builder()
org_obj = any_old(RT.Org)
log.debug("making the structure for an instrument agent")
i_obj = self._make_instrument_agent_structure(org_obj)
instrument_agent_instance_obj = self.RR2.read(i_obj.instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
self.verify_instrument_config(instrument_config, org_obj,
i_obj.instrument_device_id)
self._generate_instrument_config(instrument_agent_instance_obj,
i_obj.instrument_agent_instance_id)
return i_obj
#################################################################
# instrument-platform linking
#################################################################
def _assign_instrument_to_platform(self, i_obj, p_obj):
log.debug("assigning instrument %r to platform %r",
i_obj.instrument_agent_instance_id, p_obj.platform_id)
self.RR2.assign_instrument_device_to_platform_device(
i_obj.instrument_device_id,
p_obj.platform_device_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(p_obj.platform_device_id)
self.assertNotEqual(0, len(child_device_ids))
self._generate_platform_with_instrument_config(p_obj, i_obj)
#################################################################
# some platform topologies
#################################################################
def _create_single_platform(self):
"""
Creates and prepares a platform corresponding to the
platform ID 'LJ01D', which is a leaf in the simulated network.
"""
p_root = self._create_platform('LJ01D')
return p_root
def _create_small_hierarchy(self):
"""
Creates a small platform network consisting of 3 platforms as follows:
Node1D -> MJ01C -> LJ01D
where -> goes from parent to child.
"""
p_root = self._create_platform('Node1D')
p_child = self._create_platform('MJ01C', parent_platform_id='Node1D')
p_grandchild = self._create_platform('LJ01D', parent_platform_id='MJ01C')
self._assign_child_to_parent(p_child, p_root)
self._assign_child_to_parent(p_grandchild, p_child)
self._generate_config(p_root.platform_agent_instance_obj, p_root.platform_id, "_final")
return p_root
#################################################################
# start / stop platform
#################################################################
def _start_platform(self, agent_instance_id):
log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id)
pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
agent_instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient('paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
def _stop_platform(self, agent_instance_id):
self.IMS.stop_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
#################################################################
# start / stop instrument
#################################################################
def _start_instrument(self, agent_instance_id):
log.debug("about to call start_instrument_agent_instance with id=%s", agent_instance_id)
pid = self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=agent_instance_id)
log.debug("start_instrument_agent_instance returned pid=%s", pid)
#wait for start
agent_instance_obj = self.IMS.read_instrument_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The instrument agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._ia_client = ResourceAgentClient('paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got instrument agent client %s", str(self._ia_client))
def _stop_instrument(self, agent_instance_id):
self.IMS.stop_instrument_agent_instance(instrument_agent_instance_id=agent_instance_id)
#################################################################
# misc convenience methods
#################################################################
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs)
time_start = time.time()
#retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s",
cmd.command, elapsed_time, str(retval))
return retval
#################################################################
# commands that concrete tests can call
#################################################################
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
#self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _start_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.MONITORING)
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
# just wait for at least one -- see consume_data
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("Received samples: %s", len(self._samples_received))
def _wait_for_external_event(self):
log.info("waiting for reception of an external event...")
# just wait for at least one -- see consume_event
self._async_event_result.get(timeout=EVENT_TIMEOUT)
self.assertTrue(len(self._events_received) >= 1)
log.info("Received events: %s", len(self._events_received))
def _stop_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _pause(self):
cmd = AgentCommand(command=PlatformAgentEvent.PAUSE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.STOPPED)
def _resume(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESUME)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _clear(self):
cmd = AgentCommand(command=PlatformAgentEvent.CLEAR)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _check_sync(self):
cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC)
retval = self._execute_agent(cmd)
log.info("CHECK_SYNC result: %s", retval.result)
self.assertTrue(retval.result is not None)
self.assertEquals(retval.result[0:3], "OK:")
return retval.result
class TestVel3d(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
print '#####################'
print 'IN SETUP'
self._ia_client = None
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
#log.info('driver uri: %s', DRV_URI)
#log.info('device address: %s', DEV_ADDR)
#log.info('device port: %s', DEV_PORT)
#log.info('work dir: %s', WORK_DIR)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : True,
'enable_persistence' : False
}
#if org_governance_name is not None:
# agent_config['org_governance_name'] = org_governance_name
# Start instrument agent.
log.info("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
log.info("Agent setup")
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
#self.addCleanup(self._verify_agent_reset)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
log.info('test setup complete')
###############################################################################
# Port agent helpers.
###############################################################################
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
###############################################################################
# Event helpers.
###############################################################################
def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
"""
Start a subscriber to the instrument agent events.
@param type The type of event to catch.
@count Trigger the async event result when events received reaches this.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
# Event array and async event result.
self._event_count = count
self._events_received = []
self._async_event_result = AsyncResult()
self._event_subscriber = EventSubscriber(
event_type=type, callback=consume_event,
origin=IA_RESOURCE_ID)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=5)
def _stop_event_subscriber(self):
"""
Stop event subscribers on cleanup.
"""
self._event_subscriber.stop()
self._event_subscriber = None
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
def _start_data_subscribers(self, count, raw_count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._raw_samples_received = []
self._async_sample_result = AsyncResult()
self._async_raw_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self._samples_received.append(message)
if len(self._samples_received) == count:
self._async_sample_result.set()
def recv_raw_data(message, stream_route, stream_id):
log.info('Received raw data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self._raw_samples_received.append(message)
if len(self._raw_samples_received) == raw_count:
self._async_raw_sample_result.set()
from pyon.util.containers import create_unique_identifier
stream_name = 'parsed'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" %
stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
stream_name = 'raw'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" %
stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber,'subscription_id'):
try:
pubsub_client.deactivate_subscription(subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
###############################################################################
# tcp helpers.
###############################################################################
def _start_tcp_client(self, retval):
host = retval.result['ip_address']
port = retval.result['port']
tcp_client = TcpClient(host, port)
return tcp_client
###############################################################################
# Tests.
###############################################################################
@unittest.skip('Test should be run manually only.')
def test_initialize(self):
"""
test_initialize
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
print '#### in test'
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
@unittest.skip('Test should be run manually only.')
def test_xx(self):
"""
"""
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
with self.assertRaises(Conflict):
res_state = self._ia_client.get_resource_state()
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print '################################# mavs4 came up in state: ' + state
if state == ResourceAgentState.IDLE:
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
elif state == ResourceAgentState.STREAMING:
cmd = AgentCommand(command='DRIVER_EVENT_STOP_AUTOSAMPLE')
retval = self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
state = self._ia_client.get_agent_state()
print '################################# mavs4 now in state: ' + state
cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
kwargs={'session_type': DirectAccessTypes.telnet,
'session_timeout':600,
'inactivity_timeout':600})
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.DIRECT_ACCESS)
state = self._ia_client.get_agent_state()
print '################################# mavs4 now in state: ' + state
tcp_client = self._start_tcp_client(retval)
self.assertTrue(tcp_client.start_telnet(token=retval.result['token']))
self.assertTrue(tcp_client.send_data('\r\r'))
gevent.sleep(180)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.DIRECT_ACCESS)
cmd = AgentCommand(command=ResourceAgentEvent.GO_COMMAND)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
class TestDriverEgg(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def get_streamConfigs(self):
raw_config = StreamConfiguration(
stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
return raw_config, parsed_config
##########################
#
# The following tests generate different agent configs and pass them to a common base test script
#
###########################
@unittest.skip(
"this test can't be run from coi services. it is missing dependencies")
def test_driverLaunchModuleNoURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchModuleWithURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchNoModuleOnlyURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchBogusModuleWithURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="bogus",
driver_class="Bogus",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
@unittest.skip(
"Launches an egg 'process' even though the egg download should produce error 404"
)
def test_driverLaunchNoModule404URI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_404,
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj, False)
def test_driverLaunchNoModuleBadEggURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_BAD,
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj, True, False)
def base_activateInstrumentSample(self,
instAgent_obj,
expect_launch=True,
expect_command=True):
"""
This method runs a test of launching a driver with a given agent configuration
"""
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
print 'new InstrumentModel id = %s ' % instModel_id
# Create InstrumentAgent
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
print 'new InstrumentAgent id = %s' % instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new dp_id = %s' % data_product_id1
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
print 'Data product streams1 = %s' % stream_ids
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
print 'Data set for data_product_id1 = %s' % dataset_ids[0]
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
self.get_datastore(self.parsed_dataset)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
print 'new dp_id = %s' % str(data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
print 'Data product streams2 = %s' % str(stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
print 'Data set for data_product_id2 = %s' % dataset_ids[0]
self.raw_dataset = dataset_ids[0]
# add start/stop for instrument agent
gevent.joinall([
gevent.spawn(
lambda: self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id))
])
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
agent_process_id = ResourceAgentClient._get_agent_process_id(
instDevice_id)
print "Agent process id is '%s'" % str(agent_process_id)
self.assertTrue(agent_process_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
agent_process_id, ProcessStateEnum.RUNNING)
if not expect_launch:
self.assertFalse(
gate. await (30),
"The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?"
% agent_process_id)
return
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
agent_process_id)
print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING"
#print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=agent_process_id,
process=FakeProcess())
print "ResourceAgentClient created: %s" % str(self._ia_client)
print "Sending command=ResourceAgentEvent.INITIALIZE"
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
if not expect_command:
self.assertRaises(ServerError, self._ia_client.execute_agent, cmd)
return
retval = self._ia_client.execute_agent(cmd)
print "Result of INITIALIZE: %s" % str(retval)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# This gevent sleep is there to test the autosample time, which will show something different from default
# only if the instrument runs for over a minute
gevent.sleep(90)
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
autosample_string = extended_instrument.computed.uptime.value
autosampling_time = int(autosample_string.split()[4])
self.assertTrue(autosampling_time > 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
print "Sending command=ResourceAgentEvent.RESET"
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
print "Result of RESET: %s" % str(reply)
class RunInstrument(MiIntTestCase):
"""
Main class for communicating with an instrument
"""
def __init__(self, monitor=False, subscriber=False):
self.driver_make = None
self.driver_model = None
self.driver_name = None
self.driver_class = DRIVER_CLASS
self.ip_address = None
self.data_port = None
self.command_port = None
self.driver_version = None
self._pagent = None
self.monitor_window = monitor
self.subcriber_window = subscriber
self.stream_config = {}
self._cleanups = []
def _initialize(self):
"""
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
try:
"""
Get the information for the driver. This can be read from the yml
files; the user can run switch_driver to change the current driver.
"""
self.fetch_metadata()
self.fetch_driver_class()
self.fetch_comm_config()
if not exists(PIPE_PATH):
mkfifo(PIPE_PATH)
if not exists(self.metadata.driver_dir()):
raise DriverDoesNotExist(
"%s/%s/$%s" %
(self.metadata.driver_make, self.metadata.driver_model,
self.driver_name))
driver_module = DRIVER_MODULE_ROOT + self.metadata.driver_make + '.' + self.metadata.driver_model + '.' + self.metadata.driver_name + DRIVER_MODULE_LEAF
log.info('driver module: %s', driver_module)
log.info('driver class: %s', self.driver_class)
log.info('device address: %s', self.ip_address)
log.info('device data port: %s', self.data_port)
log.info('device command port: %s', self.command_port)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
DVR_CONFIG.update({
'dvr_mod': driver_module,
'dvr_cls': self.driver_class
})
"""
self._support = DriverIntegrationTestSupport(driver_module,
self.driver_class,
self.ip_address,
self.data_port,
DELIM,
WORK_DIR)
"""
# Start port agent, add stop to cleanup (not sure if that's
# necessary yet).
print(
"------------------>>>> Starting Port Agent <<<<------------------"
)
self.start_pagent()
# Start a monitor window if specified.
if self.monitor_window:
self.monitor_file = self._pagent.port_agent.logfname
strXterm = "xterm -T InstrumentMonitor -sb -rightbar"
#pOpenString = "xterm -T InstrumentMonitor -e tail -f " + self.monitor_file
pOpenString = strXterm + " -e tail -f " + self.monitor_file
x = subprocess.Popen(pOpenString, shell=True)
"""
DHE: Added self._cleanups to make base classes happy
"""
self.addCleanup(self.stop_pagent)
# Start container.
print(
"------------------>>>> Starting Capability Container <<<<------------------"
)
self._start_container()
# Bring up services in a deploy file (no need to message)
print(
"------------------>>>> Starting Deploy Services <<<<------------------"
)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Setup stream config.
self._build_stream_config()
# Create agent config.
agent_config = {
'driver_config': DVR_CONFIG,
'stream_config': self.stream_config,
'agent': {
'resource_id': IA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
print(
"------------------>>>> Starting Instrument Agent <<<<------------------"
)
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
if self.subcriber_window:
self._start_data_subscribers(6)
#self.addCleanup(self._stop_data_subscribers)
except:
log.error("initialize(): Exception occurred; shutting down.",
exc_info=True)
return False
else:
return True
###############################################################################
# Port agent helpers.
###############################################################################
def start_pagent(self):
"""
Construct and start the port agent.
@retval port Port that was used for connection to agent
"""
# Create port agent object.
comm_config = self.comm_config
config = {
'device_addr': comm_config.device_addr,
'device_port': comm_config.device_port,
'command_port': comm_config.command_port,
'data_port': comm_config.data_port,
'process_type': PortAgentProcessType.UNIX,
'log_level': 5,
}
self._pagent = PortAgentProcess.launch_process(config,
timeout=60,
test_mode=True)
pid = self._pagent.get_pid()
port = self._pagent.get_data_port()
cmd_port = self._pagent.get_command_port()
log.info('Started port agent pid %d listening at port %d', pid, port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr': 'localhost',
'port': port,
'cmd_port': cmd_port
}
return port
def stop_pagent(self):
"""
Stop the port agent.
"""
if self._pagent:
pid = self._pagent.get_pid()
if pid:
log.info('Stopping pagent pid %i', pid)
self._pagent.stop()
else:
log.info('No port agent running.')
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self.stream_config = {}
streams = {
'parsed': 'ctd_parsed_param_dict',
'raw': 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(
DEFAULT_PARAM_DICT, id_only=True)
if (not pd_id):
log.error("No pd_id found for param_dict '%s'" %
DEFAULT_PARAM_DICT)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
pd = None
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self.stream_config[stream_name] = stream_config
def _start_data_subscribers(self, count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
#self._async_data_result = AsyncResult()
strXterm = "xterm -T InstrumentScienceData -sb -rightbar "
pOpenString = strXterm + " -e tail -f " + PIPE_PATH
subprocess.Popen([
'xterm', '-T', 'InstrumentScienceData', '-e', 'tail', '-f',
PIPE_PATH
])
#subprocess.Popen(pOpenString)
#self.pipeData = open(PIPE_PATH, "w", 1)
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
print 'Received message on ' + str(stream_id) + ' (' + str(
stream_route.exchange_point) + ',' + str(
stream_route.routing_key) + ')'
log.info('Received message on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self.pipeData = open(PIPE_PATH, "w", 1)
self.pipeData.write(str(message))
self.pipeData.flush()
self.pipeData.close()
self._samples_received.append(message)
#if len(self._samples_received) == count:
#self._async_data_result.set()
for (stream_name, stream_config) in self._stream_config.iteritems():
stream_id = stream_config['stream_id']
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
print 'stream_id: %s' % stream_id
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber, 'subscription_id'):
try:
pubsub_client.deactivate_subscription(
subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
def bring_instrument_active(self):
"""
@brief Bring the agent up to COMMAND state,
"""
"""
DHE: Don't have an event subscriber yet
# Set up a subscriber to collect error events.
#self._start_event_subscriber('ResourceAgentResourceStateEvent', 6)
#self.addCleanup(self._stop_event_subscriber)
"""
try:
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
"""
If the agent is in STREAMING state, it will not accept the run
command.
"""
if state != ResourceAgentState.STREAMING:
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
except:
log.error(
"bring_instrument_active(): Exception occurred; shutting down.",
exc_info=True)
return False
else:
return True
"""
DHE: Don't have an event subscriber yet so we've received no events.
self._async_event_result.get(timeout=2)
print "Received: " + str(len(self._events_received)) + " events."
"""
###############################################################################
# RunInstrument helpers.
###############################################################################
def get_capabilities(self):
"""
@brief Get exposed capabilities in current state.
"""
retval = self._ia_client.get_capabilities()
# Validate capabilities for state UNINITIALIZED.
self.agt_cmds = [
x.name for x in retval if x.cap_type == CapabilityType.AGT_CMD
]
self.agt_pars = [
x.name for x in retval if x.cap_type == CapabilityType.AGT_PAR
]
self.res_cmds = [
x.name for x in retval if x.cap_type == CapabilityType.RES_CMD
]
self.res_pars = [
x.name for x in retval if x.cap_type == CapabilityType.RES_PAR
]
print "\n------------------>>>> Current Capabilities <<<<------------------"
print "Agent Commands: " + str(self.agt_cmds)
#print "Agent Parameters: " + str(self.agt_pars)
print "Resource Commands: " + str(self.res_cmds)
#print "Resource Parameters: " + str(self.res_pars)
def send_agent_command(self, command):
"""
@brief Send a command to the agent.
"""
DA_WAIT_PERIOD = 60
waiting = False
print "Input command: " + str(command)
if command == 'RESOURCE_AGENT_EVENT_GO_DIRECT_ACCESS':
cmd = AgentCommand(command=command,
kwargs={
'session_type': DirectAccessTypes.telnet,
'session_timeout': 600,
'inactivity_timeout': 600
})
waiting = True
else:
cmd = AgentCommand(command=command)
retval = self._ia_client.execute_agent(cmd)
print "Results of command: " + str(retval)
while waiting:
print "Waiting " + str(
DA_WAIT_PERIOD) + " seconds for you to test direct access."
gevent.sleep(DA_WAIT_PERIOD)
still_waiting = prompt.text('Still waiting? (y/n)')
if still_waiting is 'n':
waiting = False
def send_driver_command(self, command):
"""
@brief Send a command to the instrument through the instrument agent.
First determine whether it's a get or set, which are handled separately.
"""
if command == DriverEvent.GET:
self._get_param()
elif command == DriverEvent.SET:
self._set_param()
else:
print "Input command: " + str(command)
cmd = AgentCommand(command=command)
retval = self._ia_client.execute_resource(cmd)
print "Results of command: " + str(retval)
def _get_param(self):
"""
@brief Get a single parameter from the instrument (will be updated to get
multiple later).
"""
_all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL')
print "Parameters you can get are: " + str(_all_params)
_param_valid = False
while _param_valid is False:
_param = prompt.text('\nEnter a single parameter')
if _param in _all_params:
_param_valid = True
else:
print 'Invalid parameter: ' + _param
reply = self._ia_client.get_resource([_param])
print 'Reply is :' + str(reply)
def _set_param(self):
"""
@brief Set a single parameter
"""
_all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL')
print "Parameters you can set are: " + str(_all_params)
_param_valid = False
while _param_valid is False:
_param = prompt.text('\nEnter a single parameter')
if _param in _all_params:
_param_valid = True
else:
print 'Invalid parameter: ' + _param
_value = prompt.text('Enter value')
_value = _value.lower()
"""
DHE: Need to convert to native types here; can't be string; this is a
problem for the UI because we need a way to get the metadata about
each param to the UI.
"""
if _value == 'true':
_value = True
elif _value == 'false':
_value = False
param_dict = {_param: _value}
self._ia_client.set_resource(param_dict)
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
self.metadata = Metadata()
self.driver_make = self.metadata.driver_make
self.driver_model = self.metadata.driver_model
self.driver_name = self.metadata.driver_name
if not (self.driver_make and self.driver_model and self.driver_name):
self.driver_make = prompt.text('Driver Make', self.driver_make)
self.driver_model = prompt.text('Driver Model', self.driver_model)
self.driver_name = prompt.text('Driver Name', self.driver_name)
if not (self.driver_class):
self.driver_class = prompt.text('Driver Class', self.driver_class)
self.metadata = Metadata(self.driver_make, self.driver_model,
self.driver_name)
def fetch_comm_config(self):
"""
@brief collect connection information for the logger from the user
"""
config_path = "%s/%s" % (self.metadata.driver_dir(),
CommConfig.config_filename())
self.comm_config = CommConfig.get_config_from_console(config_path)
self.comm_config.display_config()
#self.comm_config.get_from_console()
self.ip_address = self.comm_config.device_addr
self.data_port = self.comm_config.data_port
self.command_port = self.comm_config.command_port
if not (self.ip_address):
self.ip_address = prompt.text('Instrument IP Address',
self.ip_address)
if not (self.data_port):
continuing = True
while continuing:
sport = prompt.text('Instrument Port', self.data_port)
try:
self.data_port = int(sport)
continuing = False
except ValueError as e:
print "Error converting port to number: " + str(e)
print "Please enter a valid port number.\n"
def fetch_driver_class(self):
self.driver_class = prompt.text('Driver Class', self.driver_class)
def get_user_command(self, text='Enter command'):
command = prompt.text(text)
return command
def run(self):
"""
@brief Run it.
"""
print(
"------------------>>>> Starting RunInstrument <<<<------------------"
)
"""
initialize; returns True if successful, else False.
"""
continuing = self._initialize()
"""
bring_instrument_active; returns True if successful, else False
"""
if (continuing):
continuing = self.bring_instrument_active()
PROMPT = 'Enter command (\'quit\' to exit)'
text = PROMPT
while continuing:
try:
"""
Get a list of the currently available capabilities
"""
self.get_capabilities()
command = self.get_user_command(text)
text = PROMPT
if command == 'quit':
continuing = False
elif command in self.agt_cmds:
self.send_agent_command(command)
elif command in self.res_cmds:
self.send_driver_command(command)
else:
text = 'Invalid Command: ' + command + '\n' + PROMPT
except:
log.error("run(): Exception occurred; shutting down.",
exc_info=True)
continuing = False
self.stop_pagent()
print(
"------------------>>>> Stopping RunInstrument <<<<------------------"
)
class TestPlatformAgent(IonIntegrationTestCase, HelperTestMixin):
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG["oms_uri"])
network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
network_definition_ser = NetworkUtil.serialize_network_definition(network_definition)
if log.isEnabledFor(logging.DEBUG):
log.debug("NetworkDefinition serialization:\n%s", network_definition_ser)
cls.PLATFORM_CONFIG = {
"platform_id": cls.PLATFORM_ID,
"driver_config": DVR_CONFIG,
"network_definition": network_definition_ser,
}
NetworkUtil._gen_open_diagram(network_definition.pnodes[cls.PLATFORM_ID])
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
self._pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Start data subscribers, add stop to cleanup.
# Define stream_config.
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# start event subscriber:
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
self._agent_config = {
"agent": {"resource_id": PA_RESOURCE_ID},
"stream_config": self._stream_config,
# pass platform config here
"platform_config": self.PLATFORM_CONFIG,
}
if log.isEnabledFor(logging.TRACE):
log.trace("launching with agent_config=%s" % str(self._agent_config))
self._launcher = LauncherFactory.createLauncher()
self._pid = self._launcher.launch(self.PLATFORM_ID, self._agent_config)
log.debug("LAUNCHED PLATFORM_ID=%r", self.PLATFORM_ID)
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient(PA_RESOURCE_ID, process=FakeProcess())
log.info("Got pa client %s." % str(self._pa_client))
def tearDown(self):
try:
self._launcher.cancel_process(self._pid)
finally:
super(TestPlatformAgent, self).tearDown()
def _start_data_subscribers(self):
"""
"""
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
#
# TODO retrieve appropriate stream definitions; for the moment, using
# adhoc_get_stream_names
#
# A callback for processing subscribed-to data.
def consume_data(message, stream_route, stream_id):
log.info("Subscriber received data message: %s." % str(message))
self._samples_received.append(message)
self._async_data_result.set()
for stream_name in adhoc_get_stream_names():
log.info("creating stream %r ...", stream_name)
# TODO use appropriate exchange_point
stream_id, stream_route = self._pubsub_client.create_stream(name=stream_name, exchange_point="science_data")
log.info("create_stream(%r): stream_id=%r, stream_route=%s", stream_name, stream_id, str(stream_route))
pdict = adhoc_get_parameter_dictionary(stream_name)
stream_config = dict(
stream_route=stream_route.routing_key, stream_id=stream_id, parameter_dictionary=pdict.dump()
)
self._stream_config[stream_name] = stream_config
log.info("_stream_config[%r]= %r", stream_name, stream_config)
# Create subscriptions for each stream.
exchange_name = "%s_queue" % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self._pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
self._pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
for sub in self._data_subscribers:
if hasattr(sub, "subscription_id"):
try:
self._pubsub_client.deactivate_subscription(sub.subscription_id)
except:
pass
self._pubsub_client.delete_subscription(sub.subscription_id)
sub.stop()
def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info("Event subscriber received evt: %s.", str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type, sub_type=sub_type, callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r", event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, "subscription_id"):
try:
self.pubsubcli.deactivate_subscription(sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
# def _execute_agent(self, cmd, timeout=TIMEOUT):
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs)
time_start = time.time()
# retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s", cmd.command, elapsed_time, str(retval))
return retval
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
# self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _connect_instrument(self):
#
# TODO more realistic settings for the connection
#
port_id = self.PORT_ID
instrument_id = self.INSTRUMENT_ID
instrument_attributes = self.INSTRUMENT_ATTRIBUTES_AND_VALUES
kwargs = dict(port_id=port_id, instrument_id=instrument_id, attributes=instrument_attributes)
cmd = AgentCommand(command=PlatformAgentEvent.CONNECT_INSTRUMENT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("CONNECT_INSTRUMENT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
returned_attrs = self._verify_valid_instrument_id(instrument_id, result[port_id])
if isinstance(returned_attrs, dict):
for attrName in instrument_attributes:
self.assertTrue(attrName in returned_attrs)
def _get_connected_instruments(self):
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("GET_CONNECTED_INSTRUMENTS = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
instrument_id = self.INSTRUMENT_ID
self.assertTrue(instrument_id in result[port_id])
def _disconnect_instrument(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
instrument_id = self.INSTRUMENT_ID
kwargs = dict(port_id=port_id, instrument_id=instrument_id)
cmd = AgentCommand(command=PlatformAgentEvent.DISCONNECT_INSTRUMENT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("DISCONNECT_INSTRUMENT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
self.assertTrue(instrument_id in result[port_id])
self._verify_instrument_disconnected(instrument_id, result[port_id][instrument_id])
def _turn_on_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_ON_PORT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_ON_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_ON)
def _turn_off_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_OFF_PORT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_OFF_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_OFF)
def _get_resource(self):
attrNames = self.ATTR_NAMES
#
# OOIION-631: use get_ion_ts() as a basis for using system time, which is
# a string.
#
cur_time = get_ion_ts()
from_time = str(int(cur_time) - 50000) # a 50-sec time window
kwargs = dict(attr_names=attrNames, from_time=from_time)
cmd = AgentCommand(command=PlatformAgentEvent.GET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attr_name in attrNames:
self._verify_valid_attribute_id(attr_name, attr_values)
def _set_resource(self):
attrNames = self.ATTR_NAMES
writ_attrNames = self.WRITABLE_ATTR_NAMES
# do valid settings:
# TODO more realistic value depending on attribute's type
attrs = [(attrName, self.VALID_ATTR_VALUE) for attrName in attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in attrNames:
if attrName in writ_attrNames:
self._verify_valid_attribute_id(attrName, attr_values)
else:
self._verify_not_writable_attribute_id(attrName, attr_values)
# try invalid settings:
# set invalid values to writable attributes:
attrs = [(attrName, self.INVALID_ATTR_VALUE) for attrName in writ_attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in writ_attrNames:
self._verify_attribute_value_out_of_range(attrName, attr_values)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _pause(self):
cmd = AgentCommand(command=PlatformAgentEvent.PAUSE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.STOPPED)
def _resume(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESUME)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _start_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.MONITORING)
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
# just wait for at least one -- see consume_data
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("Received samples: %s", len(self._samples_received))
def _stop_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _get_subplatform_ids(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_SUBPLATFORM_IDS)
retval = self._execute_agent(cmd)
self.assertIsInstance(retval.result, list)
self.assertTrue(x in retval.result for x in self.SUBPLATFORM_IDS)
return retval.result
def _wait_for_external_event(self):
log.info("waiting for reception of an external event...")
# just wait for at least one -- see consume_event
self._async_event_result.get(timeout=EVENT_TIMEOUT)
self.assertTrue(len(self._events_received) >= 1)
log.info("Received events: %s", len(self._events_received))
def _check_sync(self):
cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC)
retval = self._execute_agent(cmd)
log.info("CHECK_SYNC result: %s", retval.result)
self.assertTrue(retval.result is not None)
self.assertEquals(retval.result[0:3], "OK:")
return retval.result
def test_capabilities(self):
agt_cmds_all = [
PlatformAgentEvent.INITIALIZE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.PAUSE,
PlatformAgentEvent.RESUME,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.START_MONITORING,
PlatformAgentEvent.STOP_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
def sort_caps(caps):
agt_cmds = []
agt_pars = []
res_cmds = []
res_pars = []
if len(caps) > 0 and isinstance(caps[0], AgentCapability):
agt_cmds = [x.name for x in caps if x.cap_type == CapabilityType.AGT_CMD]
agt_pars = [x.name for x in caps if x.cap_type == CapabilityType.AGT_PAR]
res_cmds = [x.name for x in caps if x.cap_type == CapabilityType.RES_CMD]
res_pars = [x.name for x in caps if x.cap_type == CapabilityType.RES_PAR]
elif len(caps) > 0 and isinstance(caps[0], dict):
agt_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_CMD]
agt_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_PAR]
res_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_CMD]
res_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_PAR]
return agt_cmds, agt_pars, res_cmds, res_pars
agt_pars_all = ["example"] # 'cause ResourceAgent defines aparam_example
res_pars_all = []
res_cmds_all = []
##################################################################
# UNINITIALIZED
##################################################################
self._assert_state(PlatformAgentState.UNINITIALIZED)
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_uninitialized = [PlatformAgentEvent.INITIALIZE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES]
self.assertItemsEqual(agt_cmds, agt_cmds_uninitialized)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(current_state=False)
# Validate all capabilities as read from state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# INACTIVE
##################################################################
self._initialize()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_inactive = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_inactive)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# IDLE
##################################################################
self._go_active()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_idle = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_idle)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states as read from IDLE.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# COMMAND
##################################################################
self._run()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_command = [
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.PAUSE,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.START_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_command)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
##################################################################
# STOPPED
##################################################################
self._pause()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state STOPPED
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_stopped = [
PlatformAgentEvent.RESUME,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_stopped)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
# back to COMMAND:
self._resume()
##################################################################
# MONITORING
##################################################################
self._start_resource_monitoring()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state MONITORING
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_monitoring = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.STOP_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_monitoring)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
# return to COMMAND state:
self._stop_resource_monitoring()
###################
# ALL CAPABILITIES
###################
# Get exposed capabilities in all states as read from state COMMAND.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_all)
self.assertItemsEqual(res_pars, res_pars_all)
self._go_inactive()
self._reset()
def test_some_state_transitions(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._initialize() # -> INACTIVE
self._reset() # -> UNINITIALIZED
self._initialize() # -> INACTIVE
self._go_active() # -> IDLE
self._reset() # -> UNINITIALIZED
self._initialize() # -> INACTIVE
self._go_active() # -> IDLE
self._run() # -> COMMAND
self._pause() # -> STOPPED
self._resume() # -> COMMAND
self._reset() # -> UNINITIALIZED
def test_get_set_resources(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._get_resource()
self._set_resource()
self._go_inactive()
self._reset()
def test_some_commands(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._ping_agent()
self._ping_resource()
self._get_metadata()
self._get_ports()
self._get_subplatform_ids()
self._go_inactive()
self._reset()
def test_resource_monitoring(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._start_resource_monitoring()
self._wait_for_a_data_sample()
self._stop_resource_monitoring()
self._go_inactive()
self._reset()
def test_external_event_dispatch(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._wait_for_external_event()
self._go_inactive()
self._reset()
def test_connect_disconnect_instrument(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._connect_instrument()
self._turn_on_port()
self._get_connected_instruments()
self._turn_off_port()
self._disconnect_instrument()
self._go_inactive()
self._reset()
def test_check_sync(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._check_sync()
self._connect_instrument()
self._check_sync()
self._disconnect_instrument()
self._check_sync()
self._go_inactive()
self._reset()
class TestAgentCommsAlerts(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self._event_count = 0
self._events_received = []
self._async_event_result = AsyncResult()
def consume_event(*args, **kwargs):
log.debug('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
self._event_subscriber = EventSubscriber(
event_type='DeviceStatusAlertEvent', callback=consume_event,
origin=IA_RESOURCE_ID)
self._event_subscriber.start()
def stop_subscriber():
self._event_subscriber.stop()
self._event_subscriber = None
self.addCleanup(stop_subscriber)
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'aparam_alerts_config' : [state_alert_def, command_alert_def]
}
self._ia_client = None
self._ia_pid = None
self.addCleanup(self._verify_agent_reset)
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=self._agent_config)
log.info('Started instrument agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
self._ia_pid = None
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_lost_connection_alert(self):
"""
test_lost_connection_alert
Verify that agents detect lost connection state and issue alert.
"""
self._event_count = 3
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Confirm the persisted parameters.
retval = self._ia_client.get_agent(['alerts'])['alerts']
"""
{'origin': '123xyz', 'status': 1, '_id': 'da03b90d2e064b25bf51ed90b729e82e',
'description': 'The alert is cleared.', 'time_stamps': [],
'type_': 'DeviceStatusAlertEvent', 'valid_values': [],
'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '',
'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'],
'stream_name': '', 'ts_created': '1366749987069', 'sub_type': 3,
'origin_type': 'InstrumentDevice', 'name': 'comms_warning'}
"""
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Blow the port agent out from under the agent.
self._support.stop_pagent()
# Wait for a while, the supervisor is restarting the port agent.
gevent.sleep(5)
self._support.start_pagent()
timeout = gevent.Timeout(120)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(2)
except Timeout as t:
self.fail('Could not reconnect to device.')
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
self._async_event_result.get(timeout=30)
"""
{'origin': '123xyz', 'status': 1, '_id': '9d7db919a0414741a2aa97f3dc310647', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029709', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
{'origin': '123xyz', 'status': 1, '_id': 'f746cec5e486445e856ef29af8d8d49a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029717', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
{'origin': '123xyz', 'status': 1, '_id': '126b1e20f54f4bd2b84a93584831ea8d', 'description': 'Detected comms failure.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_LOST_CONNECTION'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008062061', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
{'origin': '123xyz', 'status': 1, '_id': '90f5762112dd446b939c6675d34dd61d', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_COMMAND'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008098639', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
"""
def test_connect_failed_alert(self):
"""
test_connect_failed_alert
Verify that agents detect failed connections and issue alert.
"""
self._event_count = 4
# Remove the port agent.
self._support.stop_pagent()
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
with self.assertRaises(Exception):
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
# Now start up the port agent.
self._support.start_pagent()
gevent.sleep(5)
# This time it will work.
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
self._async_event_result.get(timeout=30)
"""
{'origin': '123xyz', 'status': 1, '_id': 'ac1ae3ec24e74f65bde24362d689346a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751167', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
{'origin': '123xyz', 'status': 1, '_id': '6af6a6156172481ebc44baad2708ec5c', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751175', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
{'origin': '123xyz', 'status': 1, '_id': 'b493698b46fd4fc1b4c66c53b70ed043', 'description': 'Detected comms failure while connecting.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007756771', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
{'origin': '123xyz', 'status': 1, '_id': 'c01e911ecb1a47c5a04cbbcbfb348a42', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007792905', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
"""
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(
host=es_host,
port=es_port,
timeout=10
)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete,index)
IndexManagementService._es_call(es.index_delete,index)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self, user_name = '', instrument_id='', product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest( name= 'notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest( name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id)
log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) )
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
self.assertEqual(extended_instrument.computed.communications_status_roll_up.value, StatusType.STATUS_WARNING)
self.assertEqual(extended_instrument.computed.data_status_roll_up.value, StatusType.STATUS_OK)
self.assertEqual(extended_instrument.computed.power_status_roll_up.value, StatusType.STATUS_WARNING)
# Verify the computed attribute for user notification requests
self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_instrument_device_id)
self.assertEqual(notification.origin_type, "instrument")
self.assertEqual(notification.event_type, 'ResourceLifecycleEvent')
def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s", extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertTrue( 'ok' in extended_data_product.computed.last_granule.value['quality_flag'] )
self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) )
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_data_product_id)
self.assertEqual(notification.origin_type, "data product")
self.assertEqual(notification.event_type, 'DetectionEvent')
@attr('LOCOINT')
@unittest.skipIf(not use_es, 'No ElasticSearch')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
#Create stream alarms
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
# kwargs = {
# 'name' : 'test_sim_warning',
# 'stream_name' : 'parsed',
# 'value_id' : 'temp',
# 'message' : 'Temperature is above test range of 5.0.',
# 'type' : StreamAlarmType.WARNING,
# 'upper_bound' : 5.0,
# 'upper_rel_op' : '<'
# }
kwargs = {
'name' : 'temperature_warning_interval',
'stream_name' : 'parsed',
'value_id' : 'temp',
'message' : 'Temperature is below the normal range of 50.0 and above.',
'type' : StreamAlarmType.WARNING,
'lower_bound' : 50.0,
'lower_rel_op' : '<'
}
# Create alarm object.
alarm = {}
alarm['type'] = 'IntervalAlarmDef'
alarm['kwargs'] = kwargs
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5, alarms=[alarm] )
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
stream_configurations = [raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s' , data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s' , dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0] )
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug('Data product streams2 = %s' , str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s' , dataset_ids[0])
self.raw_dataset = dataset_ids[0]
#elastic search debug
es_indexes, _ = self.container.resource_registry.find_resources(restype='ElasticSearchIndex')
log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes])
log.debug('Bootstrap %s', CFG.bootstrap.use_es)
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#setup a subscriber to alarm events from the device
self._events_received= []
self._event_count = 0
self._samples_out_of_range = 0
self._samples_complete = False
self._async_sample_result = AsyncResult()
def consume_event(*args, **kwargs):
log.debug('TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
self._event_count = len(self._events_received)
self._async_sample_result.set()
self._event_subscriber = EventSubscriber(
event_type= 'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent
callback=consume_event,
origin=instDevice_id)
self._event_subscriber.start()
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
def stop_subscriber():
self._event_subscriber.stop()
self._event_subscriber = None
self.addCleanup(stop_subscriber)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
inst_agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
inst_agent_instance_obj.agent_process_id)
log.debug('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s" , str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s" , str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s" , str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
for i in xrange(10):
retval = self._ia_client.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from sample %s" , str(retval))
log.debug( "test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s" , str(reply))
self._samples_complete = True
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt.pretty_print()) )
temp_vals = rdt['temp']
self.assertEquals(len(temp_vals) , 10)
log.debug("test_activateInstrumentSample: all temp_vals: %s", temp_vals )
#out_of_range_temp_vals = [i for i in temp_vals if i > 5]
out_of_range_temp_vals = [i for i in temp_vals if i < 50.0]
log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s", out_of_range_temp_vals )
self._samples_out_of_range = len(out_of_range_temp_vals)
# if no bad values were produced, then do not wait for an event
if self._samples_out_of_range == 0:
self._async_sample_result.set()
log.debug("test_activateInstrumentSample: _events_received: %s", self._events_received )
log.debug("test_activateInstrumentSample: _event_count: %s", self._event_count )
self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout)
replay_data = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("RDT raw: %s", str(rdt.pretty_print()) )
raw_vals = rdt['raw']
self.assertEquals(len(raw_vals) , 10)
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product)
#--------------------------------------------------------------------------------
#put some events into the eventsdb to test - this should set the comms and data status to WARNING
#--------------------------------------------------------------------------------
t = get_ion_ts()
self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceStatusEvent',
origin = instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values = [200] )
self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceCommsEvent',
origin = instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds = 20 )
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1)
self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product)
#---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING ---------
t = get_ion_ts()
self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceStatusEvent',
origin = instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values = [200] )
self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceCommsEvent',
origin = instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds = 20 )
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class RunInstrument(MiIntTestCase):
"""
Main class for communicating with an instrument
"""
def __init__(self, monitor=False, subscriber=False):
self.driver_make = None
self.driver_model = None
self.driver_name = None
self.driver_class = DRIVER_CLASS
self.ip_address = None
self.data_port = None
self.command_port = None
self.driver_version = None
self._pagent = None
self.monitor_window = monitor
self.subcriber_window = subscriber
self.stream_config = {}
self._cleanups = []
def _initialize(self):
"""
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
try:
"""
Get the information for the driver. This can be read from the yml
files; the user can run switch_driver to change the current driver.
"""
self.fetch_metadata()
self.fetch_driver_class()
self.fetch_comm_config()
if not exists(PIPE_PATH):
mkfifo(PIPE_PATH)
if not exists(self.metadata.driver_dir()):
raise DriverDoesNotExist( "%s/%s/$%s" % (self.metadata.driver_make,
self.metadata.driver_model,
self.driver_name))
driver_module = DRIVER_MODULE_ROOT + self.metadata.driver_make + '.' + self.metadata.driver_model + '.' + self.metadata.driver_name + DRIVER_MODULE_LEAF
log.info('driver module: %s', driver_module)
log.info('driver class: %s', self.driver_class)
log.info('device address: %s', self.ip_address)
log.info('device data port: %s', self.data_port)
log.info('device command port: %s', self.command_port)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
DVR_CONFIG.update({'dvr_mod' : driver_module, 'dvr_cls' : self.driver_class})
"""
self._support = DriverIntegrationTestSupport(driver_module,
self.driver_class,
self.ip_address,
self.data_port,
DELIM,
WORK_DIR)
"""
# Start port agent, add stop to cleanup (not sure if that's
# necessary yet).
print( "------------------>>>> Starting Port Agent <<<<------------------" )
self.start_pagent()
# Start a monitor window if specified.
if self.monitor_window:
self.monitor_file = self._pagent.port_agent.logfname
strXterm = "xterm -T InstrumentMonitor -sb -rightbar"
#pOpenString = "xterm -T InstrumentMonitor -e tail -f " + self.monitor_file
pOpenString = strXterm + " -e tail -f " + self.monitor_file
x = subprocess.Popen(pOpenString, shell=True)
"""
DHE: Added self._cleanups to make base classes happy
"""
self.addCleanup(self.stop_pagent)
# Start container.
print( "------------------>>>> Starting Capability Container <<<<------------------" )
self._start_container()
# Bring up services in a deploy file (no need to message)
print( "------------------>>>> Starting Deploy Services <<<<------------------" )
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Setup stream config.
self._build_stream_config()
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
print( "------------------>>>> Starting Instrument Agent <<<<------------------" )
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
if self.subcriber_window:
self._start_data_subscribers(6)
#self.addCleanup(self._stop_data_subscribers)
except:
log.error("initialize(): Exception occurred; shutting down.", exc_info=True)
return False
else:
return True
###############################################################################
# Port agent helpers.
###############################################################################
def start_pagent(self):
"""
Construct and start the port agent.
@retval port Port that was used for connection to agent
"""
# Create port agent object.
comm_config = self.comm_config
config = {
'device_addr' : comm_config.device_addr,
'device_port' : comm_config.device_port,
'command_port': comm_config.command_port,
'data_port': comm_config.data_port,
'process_type': PortAgentProcessType.UNIX,
'log_level': 5,
}
self._pagent = PortAgentProcess.launch_process(config, timeout = 60, test_mode = True)
pid = self._pagent.get_pid()
port = self._pagent.get_data_port()
log.info('Started port agent pid %d listening at port %d', pid, port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
return port
def stop_pagent(self):
"""
Stop the port agent.
"""
if self._pagent:
pid = self._pagent.get_pid()
if pid:
log.info('Stopping pagent pid %i', pid)
self._pagent.stop()
else:
log.info('No port agent running.')
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {
'parsed' : 'ctd_parsed_param_dict',
'raw' : 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(DEFAULT_PARAM_DICT, id_only=True)
if (not pd_id):
log.error("No pd_id found for param_dict '%s'" % DEFAULT_PARAM_DICT)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name,
parameter_dictionary_id=pd_id)
pd = None
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self.stream_config[stream_name] = stream_config
def _start_data_subscribers(self, count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
#self._async_data_result = AsyncResult()
strXterm = "xterm -T InstrumentScienceData -sb -rightbar "
pOpenString = strXterm + " -e tail -f " + PIPE_PATH
subprocess.Popen(['xterm', '-T', 'InstrumentScienceData', '-e', 'tail', '-f', PIPE_PATH])
#subprocess.Popen(pOpenString)
#self.pipeData = open(PIPE_PATH, "w", 1)
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
print 'Received message on ' + str(stream_id) + ' (' + str(stream_route.exchange_point) + ',' + str(stream_route.routing_key) + ')'
log.info('Received message on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self.pipeData = open(PIPE_PATH, "w", 1)
self.pipeData.write(str(message))
self.pipeData.flush()
self.pipeData.close()
self._samples_received.append(message)
#if len(self._samples_received) == count:
#self._async_data_result.set()
for (stream_name, stream_config) in self._stream_config.iteritems():
stream_id = stream_config['stream_id']
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
print 'stream_id: %s' % stream_id
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber,'subscription_id'):
try:
pubsub_client.deactivate_subscription(subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
def bring_instrument_active(self):
"""
@brief Bring the agent up to COMMAND state,
"""
"""
DHE: Don't have an event subscriber yet
# Set up a subscriber to collect error events.
#self._start_event_subscriber('ResourceAgentResourceStateEvent', 6)
#self.addCleanup(self._stop_event_subscriber)
"""
try:
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
"""
If the agent is in STREAMING state, it will not accept the run
command.
"""
if state != ResourceAgentState.STREAMING:
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
except:
log.error("bring_instrument_active(): Exception occurred; shutting down.", exc_info=True)
return False
else:
return True
"""
DHE: Don't have an event subscriber yet so we've received no events.
self._async_event_result.get(timeout=2)
print "Received: " + str(len(self._events_received)) + " events."
"""
###############################################################################
# RunInstrument helpers.
###############################################################################
def get_capabilities(self):
"""
@brief Get exposed capabilities in current state.
"""
retval = self._ia_client.get_capabilities()
# Validate capabilities for state UNINITIALIZED.
self.agt_cmds = [x.name for x in retval if x.cap_type==CapabilityType.AGT_CMD]
self.agt_pars = [x.name for x in retval if x.cap_type==CapabilityType.AGT_PAR]
self.res_cmds = [x.name for x in retval if x.cap_type==CapabilityType.RES_CMD]
self.res_pars = [x.name for x in retval if x.cap_type==CapabilityType.RES_PAR]
print "\n------------------>>>> Current Capabilities <<<<------------------"
print "Agent Commands: " + str(self.agt_cmds)
#print "Agent Parameters: " + str(self.agt_pars)
print "Resource Commands: " + str(self.res_cmds)
#print "Resource Parameters: " + str(self.res_pars)
def send_agent_command(self, command):
"""
@brief Send a command to the agent.
"""
DA_WAIT_PERIOD = 60
waiting = False
print "Input command: " + str(command)
if command == 'RESOURCE_AGENT_EVENT_GO_DIRECT_ACCESS':
cmd = AgentCommand(command = command,
kwargs={'session_type': DirectAccessTypes.telnet,
'session_timeout':600,
'inactivity_timeout':600})
waiting = True
else:
cmd = AgentCommand(command = command)
retval = self._ia_client.execute_agent(cmd)
print "Results of command: " + str(retval)
while waiting:
print "Waiting " + str(DA_WAIT_PERIOD) + " seconds for you to test direct access."
gevent.sleep(DA_WAIT_PERIOD)
still_waiting = prompt.text('Still waiting? (y/n)')
if still_waiting is 'n':
waiting = False
def send_driver_command(self, command):
"""
@brief Send a command to the instrument through the instrument agent.
First determine whether it's a get or set, which are handled separately.
"""
if command == DriverEvent.GET:
self._get_param()
elif command == DriverEvent.SET:
self._set_param()
else:
print "Input command: " + str(command)
cmd = AgentCommand(command = command)
retval = self._ia_client.execute_resource(cmd)
print "Results of command: " + str(retval)
def _get_param(self):
"""
@brief Get a single parameter from the instrument (will be updated to get
multiple later).
"""
_all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL')
print "Parameters you can get are: " + str(_all_params)
_param_valid = False
while _param_valid is False:
_param = prompt.text('\nEnter a single parameter')
if _param in _all_params:
_param_valid = True
else:
print 'Invalid parameter: ' + _param
reply = self._ia_client.get_resource([_param])
print 'Reply is :' + str(reply)
def _set_param(self):
"""
@brief Set a single parameter
"""
_all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL')
print "Parameters you can set are: " + str(_all_params)
_param_valid = False
while _param_valid is False:
_param = prompt.text('\nEnter a single parameter')
if _param in _all_params:
_param_valid = True
else:
print 'Invalid parameter: ' + _param
_value = prompt.text('Enter value')
_value = _value.lower()
"""
DHE: Need to convert to native types here; can't be string; this is a
problem for the UI because we need a way to get the metadata about
each param to the UI.
"""
if _value == 'true':
_value = True
elif _value == 'false':
_value = False
param_dict = {_param: _value}
self._ia_client.set_resource(param_dict)
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
self.metadata = Metadata()
self.driver_make = self.metadata.driver_make
self.driver_model = self.metadata.driver_model
self.driver_name = self.metadata.driver_name
if not (self.driver_make and self.driver_model and self.driver_name):
self.driver_make = prompt.text( 'Driver Make', self.driver_make )
self.driver_model = prompt.text( 'Driver Model', self.driver_model )
self.driver_name = prompt.text( 'Driver Name', self.driver_name )
if not (self.driver_class):
self.driver_class = prompt.text( 'Driver Class', self.driver_class )
self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name)
def fetch_comm_config(self):
"""
@brief collect connection information for the logger from the user
"""
config_path = "%s/%s" % (self.metadata.driver_dir(), CommConfig.config_filename())
self.comm_config = CommConfig.get_config_from_console(config_path)
self.comm_config.display_config()
#self.comm_config.get_from_console()
self.ip_address = self.comm_config.device_addr
self.data_port = self.comm_config.data_port
self.command_port = self.comm_config.command_port
if not (self.ip_address):
self.ip_address = prompt.text( 'Instrument IP Address', self.ip_address )
if not (self.data_port):
continuing = True
while continuing:
sport = prompt.text( 'Instrument Port', self.data_port )
try:
self.data_port = int(sport)
continuing = False
except ValueError as e:
print "Error converting port to number: " + str(e)
print "Please enter a valid port number.\n"
def fetch_driver_class(self):
self.driver_class = prompt.text( 'Driver Class', self.driver_class )
def get_user_command(self, text='Enter command'):
command = prompt.text(text)
return command
def run(self):
"""
@brief Run it.
"""
print( "------------------>>>> Starting RunInstrument <<<<------------------" )
"""
initialize; returns True if successful, else False.
"""
continuing = self._initialize()
"""
bring_instrument_active; returns True if successful, else False
"""
if (continuing):
continuing = self.bring_instrument_active()
PROMPT = 'Enter command (\'quit\' to exit)'
text = PROMPT
while continuing:
try:
"""
Get a list of the currently available capabilities
"""
self.get_capabilities()
command = self.get_user_command(text)
text = PROMPT
if command == 'quit':
continuing = False
elif command in self.agt_cmds:
self.send_agent_command(command)
elif command in self.res_cmds:
self.send_driver_command(command)
else:
text = 'Invalid Command: ' + command + '\n' + PROMPT
except:
log.error("run(): Exception occurred; shutting down.", exc_info=True)
continuing = False
self.stop_pagent()
print( "------------------>>>> Stopping RunInstrument <<<<------------------" )
class TestPlatformAgent(IonIntegrationTestCase, HelperTestMixin):
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
self._pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.PLATFORM_CONFIG = {"platform_id": self.PLATFORM_ID, "driver_config": DVR_CONFIG}
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
self._agent_config = {
"agent": {"resource_id": PA_RESOURCE_ID},
"stream_config": self._stream_config,
# pass platform config here
"platform_config": self.PLATFORM_CONFIG,
}
log.debug("launching with agent_config=%s", str(self._agent_config))
self._launcher = LauncherFactory.createLauncher()
self._pid = self._launcher.launch(self.PLATFORM_ID, self._agent_config)
log.debug("LAUNCHED PLATFORM_ID=%r", self.PLATFORM_ID)
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient(PA_RESOURCE_ID, process=FakeProcess())
log.info("Got pa client %s." % str(self._pa_client))
def tearDown(self):
try:
self._launcher.cancel_process(self._pid)
finally:
super(TestPlatformAgent, self).tearDown()
def _start_data_subscribers(self):
"""
"""
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
#
# TODO retrieve appropriate stream definitions; for the moment, using
# adhoc_get_stream_names
#
# A callback for processing subscribed-to data.
def consume_data(message, stream_route, stream_id):
log.info("Subscriber received data message: %s." % str(message))
self._samples_received.append(message)
self._async_data_result.set()
for stream_name in adhoc_get_stream_names():
log.info("creating stream %r ...", stream_name)
# TODO use appropriate exchange_point
stream_id, stream_route = self._pubsub_client.create_stream(name=stream_name, exchange_point="science_data")
log.info("create_stream(%r): stream_id=%r, stream_route=%s", stream_name, stream_id, str(stream_route))
pdict = adhoc_get_parameter_dictionary(stream_name)
stream_config = dict(
stream_route=stream_route.routing_key, stream_id=stream_id, parameter_dictionary=pdict.dump()
)
self._stream_config[stream_name] = stream_config
log.info("_stream_config[%r]= %r", stream_name, stream_config)
# Create subscriptions for each stream.
exchange_name = "%s_queue" % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self._pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
self._pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
for sub in self._data_subscribers:
if hasattr(sub, "subscription_id"):
try:
self._pubsub_client.deactivate_subscription(sub.subscription_id)
except:
pass
self._pubsub_client.delete_subscription(sub.subscription_id)
sub.stop()
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
# def _execute_agent(self, cmd, timeout=TIMEOUT):
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs)
time_start = time.time()
# retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s", cmd.command, elapsed_time, str(retval))
return retval
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
# self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _set_up_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
port_attrName = self.PORT_ATTR_NAME
kwargs = dict(port_id=port_id, attributes={port_attrName: self.VALID_PORT_ATTR_VALUE})
cmd = AgentCommand(command=PlatformAgentEvent.SET_UP_PORT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("SET_UP_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertTrue(port_attrName in result[port_id])
def _turn_on_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_ON_PORT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_ON_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], bool)
def _turn_off_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_OFF_PORT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_OFF_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], bool)
def _get_resource(self):
attrNames = self.ATTR_NAMES
cur_time = get_ion_ts()
#
# OOIION-631 Note: I'm asked to only use get_ion_ts() as a basis for
# using system time. However, other associated supporting (and more
# "numeric") routines would be convenient. In particular, note the
# following operation to subtract a number of seconds for my request:
#
from_time = str(int(cur_time) - 50000) # a 50-sec time window
kwargs = dict(attr_names=attrNames, from_time=from_time)
cmd = AgentCommand(command=PlatformAgentEvent.GET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attr_name in attrNames:
self._verify_valid_attribute_id(attr_name, attr_values)
def _set_resource(self):
attrNames = self.ATTR_NAMES
writ_attrNames = self.WRITABLE_ATTR_NAMES
# do valid settings:
# TODO more realistic value depending on attribute's type
attrs = [(attrName, self.VALID_ATTR_VALUE) for attrName in attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in attrNames:
if attrName in writ_attrNames:
self._verify_valid_attribute_id(attrName, attr_values)
else:
self._verify_not_writable_attribute_id(attrName, attr_values)
# try invalid settings:
# set invalid values to writable attributes:
attrs = [(attrName, self.INVALID_ATTR_VALUE) for attrName in writ_attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in writ_attrNames:
self._verify_attribute_value_out_of_range(attrName, attr_values)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
# kwargs = dict(plat_config=self.PLATFORM_CONFIG)
# cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE, kwargs=kwargs)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _get_subplatform_ids(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_SUBPLATFORM_IDS)
retval = self._execute_agent(cmd)
self.assertIsInstance(retval.result, list)
self.assertTrue(x in retval.result for x in self.SUBPLATFORM_IDS)
return retval.result
def _start_event_dispatch(self):
kwargs = dict(params="TODO set params")
cmd = AgentCommand(command=PlatformAgentEvent.START_EVENT_DISPATCH, kwargs=kwargs)
retval = self._execute_agent(cmd)
self.assertTrue(retval.result is not None)
return retval.result
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=15)
# just wait for at least one -- see consume_data above
self.assertTrue(len(self._samples_received) >= 1)
def _stop_event_dispatch(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_EVENT_DISPATCH)
retval = self._execute_agent(cmd)
self.assertTrue(retval.result is not None)
return retval.result
def test_capabilities(self):
# log.info("test_capabilities starting. Default timeout=%s", TIMEOUT)
log.info("test_capabilities starting. Default timeout=%i", CFG.endpoint.receive.timeout)
agt_cmds_all = [
PlatformAgentEvent.INITIALIZE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.SET_UP_PORT,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.START_EVENT_DISPATCH,
PlatformAgentEvent.STOP_EVENT_DISPATCH,
]
def sort_caps(caps):
agt_cmds = []
agt_pars = []
res_cmds = []
res_pars = []
if len(caps) > 0 and isinstance(caps[0], AgentCapability):
agt_cmds = [x.name for x in caps if x.cap_type == CapabilityType.AGT_CMD]
agt_pars = [x.name for x in caps if x.cap_type == CapabilityType.AGT_PAR]
res_cmds = [x.name for x in caps if x.cap_type == CapabilityType.RES_CMD]
res_pars = [x.name for x in caps if x.cap_type == CapabilityType.RES_PAR]
elif len(caps) > 0 and isinstance(caps[0], dict):
agt_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_CMD]
agt_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_PAR]
res_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_CMD]
res_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_PAR]
return agt_cmds, agt_pars, res_cmds, res_pars
agt_pars_all = ["example"] # 'cause ResourceAgent defines aparam_example
res_pars_all = []
res_cmds_all = []
##################################################################
# UNINITIALIZED
##################################################################
self._assert_state(PlatformAgentState.UNINITIALIZED)
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_uninitialized = [PlatformAgentEvent.INITIALIZE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES]
self.assertItemsEqual(agt_cmds, agt_cmds_uninitialized)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(current_state=False)
# Validate all capabilities as read from state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
self._initialize()
##################################################################
# INACTIVE
##################################################################
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_inactive = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_inactive)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
self._go_active()
##################################################################
# IDLE
##################################################################
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_idle = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_idle)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states as read from IDLE.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
self._run()
##################################################################
# COMMAND
##################################################################
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_command = [
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.SET_UP_PORT,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.START_EVENT_DISPATCH,
PlatformAgentEvent.STOP_EVENT_DISPATCH,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_command)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
# Get exposed capabilities in all states as read from state COMMAND.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_all)
self.assertItemsEqual(res_pars, res_pars_all)
self._go_inactive()
self._reset()
def test_go_active_and_run(self):
# log.info("test_go_active_and_run starting. Default timeout=%s", TIMEOUT)
log.info("test_capabilities starting. Default timeout=%i", CFG.endpoint.receive.timeout)
self._ping_agent()
# self._ping_resource() skipping this here while the timeout issue
# on r2_light and other builds is investigated.
self._initialize()
self._go_active()
self._run()
self._ping_agent()
self._ping_resource()
self._get_metadata()
self._get_ports()
self._get_subplatform_ids()
self._get_resource()
self._set_resource()
self._set_up_port()
self._turn_on_port()
self._start_event_dispatch()
self._wait_for_a_data_sample()
self._stop_event_dispatch()
self._turn_off_port()
self._go_inactive()
self._reset()
class TestAgentPersistence(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : False,
'enable_persistence' : True,
'aparam_pubrate_config' :
{
'raw' : 2,
'parsed' : 2
}
}
self._ia_client = None
self._ia_pid = '1234'
self.addCleanup(self._verify_agent_reset)
self.addCleanup(self.container.state_repository.put_state,
self._ia_pid, {})
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self, bootmode=None):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = bootmode
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config,
process_id=self._ia_pid)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_agent_config_persistence(self):
"""
test_agent_config_persistence
Test that agent parameter configuration is persisted between running
instances.
"""
# Start the agent.
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the default agent parameters.
#{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
retval = self._ia_client.get_agent(['streams'])['streams']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
#{'pubrate': {'raw': 0, 'parsed': 0}}
retval = self._ia_client.get_agent(['pubrate'])['pubrate']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
self.assertEqual(retval['raw'], 2)
self.assertEqual(retval['parsed'], 2)
#{'alerts': []}
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertEqual(retval, [])
# Define a few new parameters and set them.
# Confirm they are set.
alert_def_1 = {
'name' : 'current_warning_interval',
'stream_name' : 'parsed',
'description' : 'Current is below normal range.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 10.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def_2 = {
'name' : 'temp_alarm_interval',
'stream_name' : 'parsed',
'description' : 'Temperatoure is critical.',
'alert_type' : StreamAlertType.ALARM,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 20.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_alerts = [alert_def_1,alert_def_2, alert_def3]
pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : orig_alerts,
'pubrate' : pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the persisted parameters.
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
def test_agent_state_persistence(self):
"""
test_agent_state_persistence
Verify that agents can be restored to their prior running state.
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : [alert_def3],
'pubrate' : orig_pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
print '## in state: ' + state
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.STOPPED:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to STOPPED.")
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_agent_rparam_persistence(self):
"""
test_agent_rparam_persistence
Verify ability to restore device configuration.
### Original values:
{'TA0': -0.0002572242, 'OUTPUTSV': False, 'NAVG': 0}
### Values after set:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
### Restore config:
{'PTCA1': 0.6603433, 'WBOTC': 1.2024e-05, 'PCALDATE': [12, 8, 2005],
'STORETIME': False, 'CPCOR': 9.57e-08, 'PTCA2': 0.00575649,
'OUTPUTSV': True, 'SAMPLENUM': 0, 'TCALDATE': [8, 11, 2005],
'OUTPUTSAL': False, 'TA2': -9.717158e-06, 'POFFSET': 0.0,
'INTERVAL': 19733, 'SYNCWAIT': 0, 'CJ': 3.339261e-05,
'CI': 0.0001334915, 'CH': 0.1417895, 'TA0': -0.0005144484,
'TA1': 0.0003138936, 'NAVG': 1, 'TA3': 2.138735e-07, '
RCALDATE': [8, 11, 2005], 'CG': -0.987093, 'CTCOR': 3.25e-06, '
PTCB0': 24.6145, 'PTCB1': -0.0009, 'PTCB2': 0.0,
'CCALDATE': [8, 11, 2005], 'PA0': 5.916199, 'PA1': 0.4851819,
'PA2': 4.596432e-07, 'SYNCMODE': False, 'PTCA0': 276.2492,
'TXREALTIME': True, 'RTCA2': -3.022745e-08, 'RTCA1': 1.686132e-06,
'RTCA0': 0.9999862}
### Of which we have:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
params = [
SBE37Parameter.OUTPUTSV,
SBE37Parameter.NAVG,
SBE37Parameter.TA0
]
retval = self._ia_client.get_resource(params)
orig_params = retval
new_params = {
SBE37Parameter.OUTPUTSV : not orig_params[SBE37Parameter.OUTPUTSV],
SBE37Parameter.NAVG : orig_params[SBE37Parameter.NAVG] + 1,
SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 2
}
#print '########### orig params'
#print str(orig_params)
self._ia_client.set_resource(new_params)
retval = self._ia_client.get_resource(params)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
#print '########### new params'
#print str(retval)
# Now stop and restart the agent.
self._stop_agent()
self._support.stop_pagent()
gevent.sleep(10)
self._start_pagent()
gevent.sleep(10)
self._start_agent('restart')
timeout = gevent.Timeout(600)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(3)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
# Verify the parameters have been restored as needed.
retval = self._ia_client.get_resource(params)
#print '########### restored params'
#print str(retval)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
@unittest.skip('Making CEI friendly.')
def test_cei_launch_mode(self):
pdc = ProcessDispatcherServiceClient(node=self.container.node)
p_def = ProcessDefinition(name='Agent007')
p_def.executable = {
'module' : 'ion.agents.instrument.instrument_agent',
'class' : 'InstrumentAgent'
}
p_def_id = pdc.create_process_definition(p_def)
pid = pdc.create_process(p_def_id)
def event_callback(event, *args, **kwargs):
print '######### proc %s in state %s' % (event.origin, ProcessStateEnum._str_map[event.state])
sub = EventSubscriber(event_type='ProcessLifecycleEvent',
callback=event_callback,
origin=pid,
origin_type='DispatchedProcess')
sub.start()
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = 'restart'
pdc.schedule_process(p_def_id, process_id=pid,
configuration=agent_config)
gevent.sleep(5)
pdc.cancel_process(pid)
gevent.sleep(15)
sub.stop()
class TestActivateRSNVel3DInstrument(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateRSNVel3DInstrument, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
@attr('LOCOINT')
@unittest.skip('under construction')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 180}}})
def test_activate_rsn_vel3d(self):
log.info(
"--------------------------------------------------------------------------------------------------------"
)
# load_parameter_scenarios
self.container.spawn_process(
"Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
scenario="BETA",
path="master",
categories=
"ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition",
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='Vel3DMModel',
description="Vel3DMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('test_activate_rsn_vel3d new InstrumentModel id = %s ',
instModel_id)
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='raw')
vel3d_b_sample = StreamConfiguration(
stream_name='vel3d_b_sample',
parameter_dictionary_name='vel3d_b_sample')
vel3d_b_engineering = StreamConfiguration(
stream_name='vel3d_b_engineering',
parameter_dictionary_name='vel3d_b_engineering')
RSN_VEL3D_01 = {
'DEV_ADDR': "10.180.80.6",
'DEV_PORT': 2101,
'DATA_PORT': 1026,
'CMD_PORT': 1025,
'PA_BINARY': "port_agent"
}
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='Vel3DAgent',
description="Vel3DAgent",
driver_uri=
"http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg",
stream_configurations=[
raw_config, vel3d_b_sample, vel3d_b_engineering
])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s',
instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='Vel3DDevice',
description="Vel3DDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s ",
instDevice_id)
port_agent_config = {
'device_addr': '10.180.80.6',
'device_port': 2101,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': 1025,
'data_port': 1026,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='Vel3DAgentInstance',
description="Vel3DAgentInstance",
port_agent_config=port_agent_config,
alerts=[])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'vel3d_b_sample', id_only=True)
parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition(
name='vel3d_b_sample',
parameter_dictionary_id=parsed_sample_pdict_id)
parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'vel3d_b_engineering', id_only=True)
parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition(
name='vel3d_b_engineering',
parameter_dictionary_id=parsed_eng_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_sample',
description='vel3d_b_sample')
sample_data_product_id = self.dpclient.create_data_product(
data_product=dp_obj,
stream_definition_id=parsed_sample_stream_def_id)
log.debug('new dp_id = %s', sample_data_product_id)
self.dpclient.activate_data_product_persistence(
data_product_id=sample_data_product_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=sample_data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(sample_data_product_id,
PRED.hasStream, None, True)
log.debug('sample_data_product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for sample_data_product = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('vel3d_b_sample', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_engineering',
description='vel3d_b_engineering')
eng_data_product_id = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id)
log.debug('new dp_id = %s', eng_data_product_id)
self.dpclient.activate_data_product_persistence(
data_product_id=eng_data_product_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=eng_data_product_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('test_activate_rsn_vel3d Data product streams2 = %s',
str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s',
dataset_ids[0])
self.raw_dataset = dataset_ids[0]
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id, ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
def check_state(label, desired_state):
actual_state = self._ia_client.get_agent_state()
log.debug("%s instrument agent is in state '%s'", label,
actual_state)
self.assertEqual(desired_state, actual_state)
log.debug("test_activate_rsn_vel3d: got ia client %s",
str(self._ia_client))
check_state("just-spawned", ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: initialize %s", str(retval))
check_state("initialized", ResourceAgentState.INACTIVE)
log.debug("test_activate_rsn_vel3d Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: return value from go_active %s",
str(reply))
check_state("activated", ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("current state after sending go_active command %s",
str(state))
#
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: run %s", str(reply))
check_state("commanded", ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("current state after sending run command %s", str(state))
# cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE)
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# gevent.sleep(5)
#
# cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE)
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
# retval = self._ia_client.execute_agent(cmd)
# state = retval.result
# log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state))
#
# cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.STOPPED, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.IDLE, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.RUN)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
log.debug("test_activate_rsn_vel3d: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: return from reset %s", str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()))
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(sample_data_product_id)
self.dpclient.delete_data_product(eng_data_product_id)
self.dpclient.delete_data_product(data_product_id2)
class Test2CAA(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
"""
###################################################################
# Internal parameters and container.
###################################################################
# Internal parameters.
self._terrestrial_platform_id = 'terrestrial_id'
self._remote_platform_id = 'remote_id'
self._resource_id = 'fake_id'
self._xs_name = 'remote1'
self._terrestrial_svc_name = 'terrestrial_endpoint'
self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
self._remote_svc_name = 'remote_endpoint'
self._remote_listen_name = self._remote_svc_name + self._xs_name
self._remote_port = 0
self._terrestrial_port = 0
self._te_client = None
self._re_client = None
self._remote_pid = None
self._terrestrial_pid = None
# Async test results.
self._no_requests = 10
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._telem_evts = []
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._results_recv = {}
self._requests_sent = {}
self._done_telem_evt = AsyncResult()
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
###################################################################
# Start endpoints, agent.
###################################################################
self._start_terrestrial()
self._start_remote()
self._start_agent()
###################################################################
# Assign client ports.
# This is primarily for test purposes as the IP config in
# deployment will be fixed in advance.
###################################################################
self.te_client.set_client_port(self._remote_port)
check_port = self.te_client.get_client_port()
log.debug('Terrestrial client port is: %i', check_port)
self.re_client.set_client_port(self._terrestrial_port)
check_port = self.re_client.get_client_port()
log.debug('Remote client port is: %i', check_port)
###################################################################
# Start the event publisher and subscribers.
# Used to send fake agent telemetry publications to the endpoints,
# and to receive endpoint publications.
###################################################################
self._event_publisher = EventPublisher()
# Start the event subscriber for remote namespace platform events.
# This event could be changed to RemoteNamespaceEvent.
self._event_subscriber = EventSubscriber(
event_type='PlatformEvent',
callback=self.consume_event,
origin=self._xs_name)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._event_subscriber.stop)
# Start the result subscriber for remote resource events.
self._resource_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=IA_RESOURCE_ID,
callback=self.consume_event)
self._resource_result_subscriber.start()
self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._resource_result_subscriber.stop)
# Start the result subscriber for remote service events.
self._service_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin='resource_registry' + 'remote1',
callback=self.consume_event)
self._service_result_subscriber.start()
self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._service_result_subscriber.stop)
# Start the result subscriber for fake resource results.
self._fake_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=self._resource_id,
callback=self.consume_event)
self._fake_result_subscriber.start()
self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._fake_result_subscriber.stop)
###################################################################
# Start/stop helpers.
###################################################################
def _start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _start_terrestrial(self):
"""
"""
# Create terrestrial config.
terrestrial_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._remote_port,
'this_port' : self._terrestrial_port,
'platform_resource_id' : self._terrestrial_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._terrestrial_listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._terrestrial_pid = self._container_client.spawn_process(
name=self._terrestrial_listen_name,
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=terrestrial_endpoint_config)
log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))
# Create a terrestrial client.
self.te_client = TerrestrialEndpointClient(
process=FakeProcess(),
to_name=self._terrestrial_listen_name)
log.debug('Got te client %s.', str(self.te_client))
self._terrestrial_port = self.te_client.get_port()
log.debug('Terrestrial port is: %i', self._terrestrial_port)
def _start_remote(self):
"""
"""
# Create agent config.
remote_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._terrestrial_port,
'this_port' : self._remote_port,
'platform_resource_id' : self._remote_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._remote_listen_name
}
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
self._remote_pid = self._container_client.spawn_process(
name=self._remote_listen_name,
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=remote_endpoint_config)
log.debug('Remote endpoint pid=%s.', str(self._remote_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=self._remote_listen_name)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._remote_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._remote_port)
###################################################################
# Telemetry publications to start/top endpoint.
# (Normally be published by appropriate platform agents.)
###################################################################
def terrestrial_link_up(self):
"""
Publish telemetry available to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.AVAILABLE)
def terrestrial_link_down(self):
"""
Publish telemetry unavailable to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def remote_link_up(self):
"""
Publish telemetry available to the remote endpoint.
"""
# Publish a link up event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.AVAILABLE)
def remote_link_down(self):
"""
Publish telemetry unavailable to the remote endpoint.
"""
# Publish a link down event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_event(self, evt, *args, **kwargs):
"""
Test callback for events.
"""
log.debug('Test got event: %s, args: %s, kwargs: %s',
str(evt), str(args), str(kwargs))
if evt.type_ == 'PublicPlatformTelemetryEvent':
self._telem_evts.append(evt)
if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts):
self._done_telem_evt.set()
elif evt.type_ == 'RemoteQueueModifiedEvent':
self._queue_mod_evts.append(evt)
if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts):
self._done_queue_mod_evt.set()
elif evt.type_ == 'RemoteCommandTransmittedEvent':
self._cmd_tx_evts.append(evt)
if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts):
self._done_cmd_tx_evt.set()
elif evt.type_ == 'RemoteCommandResult':
cmd = evt.command
self._results_recv[cmd.command_id] = cmd
if len(self._results_recv) == self._no_requests:
self._done_cmd_evt.set()
###################################################################
# Misc helpers.
###################################################################
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id='fake_id',
command=cmdstr,
args=['arg1', 23],
kwargs={'kwargs1':'someval'})
return cmd
###################################################################
# Tests.
###################################################################
def test_queued_fake(self):
"""
test_queued_fake
Test fake resource commands queued prior to linkup.
"""
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
# The following error occurs when queue persistence is enabled.
# Need to verify correct solution to enable persistent queues.
"""
2012-11-06 14:27:13,517 INFO pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run
result = self._run(*self.args, **self.kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop
self._callback(cmd_result)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete
self._client.enqueue(result)
AttributeError: 'NoneType' object has no attribute 'enqueue'
<Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError
2012-11-06 14:27:13,586 INFO pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7
2012-11-06 14:27:13,665 INFO pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6
2012-11-06 14:27:13,739 INFO pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5
2012-11-06 14:27:13,807 INFO pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4
2012-11-06 14:27:13,874 INFO pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3
2012-11-06 14:27:13,941 INFO pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1
2012-11-06 14:27:13,945 INFO pyon.event.event EventSubscriber stopped. Event pattern=#
2012-11-06 14:27:14,124 INFO pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984
2012-11-06 14:27:14,399 INFO pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB
======================================================================
ERROR: test_process_online
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner
return f(*args, **kw)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online
cmd = self.te_client.enqueue_command(cmd)
File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command
return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request
return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request
retval, headers = e.send(msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send
result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send
return self._invite_and_send(to_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send
return self._send(to_role, to_role_name, msg, header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send
return self._end_point_unit._message_send(msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send
return ProcessRPCRequestEndpointUnit.send(self, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send
return self._send(_msg, _header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send
raise ex
Conflict: 409 - Object not based on most current version
"""
def test_process_online(self):
"""
test_process_online
Test fake resource commands queued while link is up.
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mods, command transmissions and results.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_remote_late(self):
"""
test_remote_late
Test fake resource commands queued prior to linkup.
Delay remote side linkup substantially to test terrestrial
behavior when remote server not initially available.
"""
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Actually stop the remote process, since the server is
# available even if it hasn't linked up yet and the test is running
# in the same container. The test will remember the appropriate
# remote port numbers.
self._container_client.terminate_process(self._remote_pid)
self.terrestrial_link_up()
gevent.sleep(10)
# Restart remote side and publish remote link up.
self._start_remote()
self.remote_link_up()
# Block for transmission and result events.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_resource_commands(self):
"""
test_resource_commands
"""
# Set up to verify the two commands queued.
self._no_requests = 2
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
# Use IA client to verify IA.
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Create and enqueue commands.
state_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={})
state_cmd = self.te_client.enqueue_command(state_cmd)
self._requests_sent[state_cmd.command_id] = state_cmd
ping_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='ping_agent',
args=[],
kwargs={})
ping_cmd = self.te_client.enqueue_command(ping_cmd)
self._requests_sent[ping_cmd.command_id] = ping_cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
self.assertEqual(self._results_recv[state_cmd.command_id].result,
ResourceAgentState.UNINITIALIZED)
self.assertIn('ping from InstrumentAgent',
self._results_recv[ping_cmd.command_id].result)
def test_service_command_sequence(self):
"""
test_service_commands
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='create',
args=[obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a tuple of strings.
#{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
#'1-00a308c45fff459c7cda1db9a7314de6'],
#'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a user info object with the name set.
#{'lcstate': 'DEPLOYED_AVAILABLE',
#'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
#'description': '',
#'ts_updated': '1349213207638',
#'type_': 'UserInfo',
#'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
#'_id': '27832d93f4cd4535a75ac75c06e00a7e',
#'ts_created': '1349213207638',
#'variables': [{'name': '', 'value': ''}],
#'name': 'some_name'}
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
read_obj.name = 'some_other_name'
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='update',
args=[read_obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='delete',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
class TestAgentPersistence(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : False,
'enable_persistence' : True
}
self._ia_client = None
self._ia_pid = '8989'
self.addCleanup(self._verify_agent_reset)
self.addCleanup(self.container.state_repository.put_state,
self._ia_pid, {})
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=self._agent_config,
process_id=self._ia_pid)
log.info('Started instrument agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_agent_config_persistence(self):
"""
test_agent_config_persistence
Test that agent parameter configuration is persisted between running
instances.
"""
# Start the agent.
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the default agent parameters.
#{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
retval = self._ia_client.get_agent(['streams'])['streams']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
#{'pubrate': {'raw': 0, 'parsed': 0}}
retval = self._ia_client.get_agent(['pubrate'])['pubrate']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
self.assertEqual(retval['raw'], 0)
self.assertEqual(retval['parsed'], 0)
#{'alerts': []}
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertEqual(retval, [])
# Define a few new parameters and set them.
# Confirm they are set.
alert_def_1 = {
'name' : 'current_warning_interval',
'stream_name' : 'parsed',
'message' : 'Current is below normal range.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 10.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def_2 = {
'name' : 'temp_alarm_interval',
'stream_name' : 'parsed',
'message' : 'Temperatoure is critical.',
'alert_type' : StreamAlertType.ALARM,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 20.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'message' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'value_id' : None,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_alerts = [alert_def_1,alert_def_2, alert_def3]
pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : orig_alerts,
'pubrate' : pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(5)
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the persisted parameters.
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
def test_agent_state_persistence(self):
"""
test_agent_state_persistence
Verify that agents can be restored to their prior running state.
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(3)
self._start_agent()
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(3)
self._start_agent()
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self, user_name = '', instrument_id='', product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest( name= 'notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest( name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id)
log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) )
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
# Verify the computed attribute for user notification requests
self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_instrument_device_id, notification.origin)
self.assertEqual("instrument", notification.origin_type)
self.assertEqual('ResourceLifecycleEvent', notification.event_type)
def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s", extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedFloatValue)
self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) )
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_data_product_id, notification.origin)
self.assertEqual("data product", notification.origin_type)
self.assertEqual('DetectionEvent', notification.event_type)
@attr('LOCOINT')
#@unittest.skip('refactoring')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 90}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw')
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict')
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config,
alerts= [])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s' , data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s' , dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0] )
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug('Data product streams2 = %s' , str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s' , dataset_ids[0])
self.raw_dataset = dataset_ids[0]
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s" , str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.INACTIVE, state)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s" , str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s" , str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.STOPPED, state)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
for i in xrange(10):
monitor = DatasetMonitor(dataset_id=self.parsed_dataset)
self._ia_client.execute_resource(AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE))
if not monitor.wait():
raise AssertionError('Failed on the %ith granule' % i)
monitor.stop()
# cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
# for i in xrange(10):
# retval = self._ia_client.execute_resource(cmd)
# log.debug("test_activateInstrumentSample: return from sample %s" , str(retval))
log.debug( "test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s" , str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) )
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
all_raw = "".join(raw_vals)
# look for 't' entered after a prompt -- ">t"
t_commands = all_raw.count(">t")
if 10 != t_commands:
log.error("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals): log.error("raw val %s: %s", i, [r])
self.fail("Expected 10 't' strings in raw_vals, got %s" % t_commands)
else:
log.debug("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals): log.debug("raw val %s: %s", i, [r])
replay_data_parsed = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data_parsed, Granule)
rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed)
log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt_parsed.pretty_print()) )
self.assertIn('temp', rdt_parsed)
temp_vals = rdt_parsed['temp']
pressure_vals = rdt_parsed['pressure']
if 10 != len(temp_vals):
log.error("%s temp_vals: %s", len(temp_vals), temp_vals)
self.fail("Expected 10 temp_vals, got %s" % len(temp_vals))
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
def test_platform_with_instrument_streaming(self):
#
# The following is with just a single platform and the single
# instrument "SBE37_SIM_08", which corresponds to the one on port 4008.
#
#load the paramaters and the param dicts necesssary for the VEL3D
log.debug( "load params------------------------------------------------------------------------------")
self._load_params()
log.debug( " _register_oms_listener------------------------------------------------------------------------------")
self._register_oms_listener()
#create the instrument device/agent/mode
log.debug( "---------- create_instrument_resources ----------" )
self._create_instrument_resources()
#create the platform device, agent and instance
log.debug( "---------- create_platform_configuration ----------" )
self._create_platform_configuration('LPJBox_CI_Ben_Hall')
self.rrclient.create_association(subject=self.platform_device_id, predicate=PRED.hasDevice, object=self.instrument_device_id)
log.debug( "---------- start_platform ----------" )
self._start_platform()
self.addCleanup(self._stop_platform)
# get everything in command mode:
self._ping_agent()
log.debug( " ---------- initialize ----------" )
self._initialize()
_ia_client = ResourceAgentClient(self.instrument_device_id, process=FakeProcess())
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
log.debug( " ---------- go_active ----------" )
self._go_active()
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
log.debug( "---------- run ----------" )
self._run()
gevent.sleep(2)
log.debug( " ---------- _start_resource_monitoring ----------" )
self._start_resource_monitoring()
gevent.sleep(2)
#
# # verify the instrument is command state:
# state = ia_client.get_agent_state()
# log.debug(" TestPlatformInstrument get_agent_state: %s", state)
# self.assertEqual(state, ResourceAgentState.COMMAND) _stop_resource_monitoring
log.debug( " ---------- _stop_resource_monitoring ----------" )
self._stop_resource_monitoring()
gevent.sleep(2)
log.debug( " ---------- go_inactive ----------" )
self._go_inactive()
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
self._reset()
self._shutdown()
class BaseIntTestPlatform(IonIntegrationTestCase, HelperTestMixin):
"""
A base class with several conveniences supporting specific platform agent
integration tests, see:
- ion/agents/platform/test/test_platform_agent_with_rsn.py
- ion/services/sa/observatory/test/test_platform_launch.py
The platform IDs used here are organized as follows:
Node1D -> MJ01C -> LJ01D
where -> goes from parent platform to child platform.
This is a subset of the whole topology defined in the simulated platform
network (network.yml), which in turn is used by the RSN OMS simulator.
- 'LJ01D' is the root platform used in test_single_platform
- 'Node1D' is the root platform used in test_hierarchy
Methods are provided to construct specific platform topologies, but
subclasses decide which to use.
"""
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.org_id = self.RR2.create(any_old(RT.Org))
log.debug("Org created: %s", self.org_id)
# Create InstrumentModel
# TODO create multiple models as needed; for the moment assuming all
# used instruments are the same model here.
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
self.instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', self.instModel_id)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
CIOMSClientFactory.destroy_instance(rsn_oms)
if log.isEnabledFor(logging.TRACE):
# show serialized version for the network definition:
network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
log.trace("NetworkDefinition serialization:\n%s", network_definition_ser)
# set attributes for the platforms:
self._platform_attributes = {}
for platform_id in self._network_definition.pnodes:
pnode = self._network_definition.pnodes[platform_id]
dic = dict((attr.attr_id, attr.defn) for attr in pnode.attrs.itervalues())
self._platform_attributes[platform_id] = dic
log.trace("_platform_attributes: %s", self._platform_attributes)
# set ports for the platforms:
self._platform_ports = {}
for platform_id in self._network_definition.pnodes:
pnode = self._network_definition.pnodes[platform_id]
dic = {}
for port_id, port in pnode.ports.iteritems():
dic[port_id] = dict(port_id=port_id,
network=port.network)
self._platform_ports[platform_id] = dic
log.trace("_platform_ports: %s", self._platform_attributes)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber(sub_type="platform_event")
# instruments that have been set up: instr_key: i_obj
self._setup_instruments = {}
#################################################################
# data subscribers handling
#################################################################
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s. stream_name=%r stream_id=%r',
str(message), stream_name, stream_id)
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
#################################################################
# event subscribers handling
#################################################################
def _start_event_subscriber(self, event_type="DeviceEvent",
sub_type=None,
count=0):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
if count == 0:
self._async_event_result.set(evt)
elif count == len(self._events_received):
self._async_event_result.set()
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r, count=%d",
event_type, sub_type, count)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
#################################################################
# config supporting methods
#################################################################
def _get_platform_stream_configs(self):
"""
This method is an adaptation of get_streamConfigs in
test_driver_egg.py
"""
return [
StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
# TODO include a "raw" stream?
]
def _get_instrument_stream_configs(self):
"""
configs copied from test_activate_instrument.py
"""
return [
StreamConfiguration(stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5),
StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2, granule_publish_rate=5)
]
def _verify_child_config(self, config, device_id, is_platform):
for key in required_config_keys:
self.assertIn(key, config)
if is_platform:
self.assertEqual(RT.PlatformDevice, config['device_type'])
for key in DVR_CONFIG.iterkeys():
self.assertIn(key, config['driver_config'])
for key in ['startup_config']:
self.assertEqual({}, config[key])
else:
self.assertEqual(RT.InstrumentDevice, config['device_type'])
for key in ['children']:
self.assertEqual({}, config[key])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
def _verify_parent_config(self, config, parent_device_id,
child_device_id, is_platform):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(RT.PlatformDevice, config['device_type'])
for key in DVR_CONFIG.iterkeys():
self.assertIn(key, config['driver_config'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
self._verify_child_config(config['children'][child_device_id],
child_device_id, is_platform)
def _create_platform_configuration(self, platform_id, parent_platform_id=None):
"""
This method is an adaptation of test_agent_instance_config in
test_instrument_management_service_integration.py
@param platform_id
@param parent_platform_id
@return a DotDict with various of the constructed elements associated
to the platform.
"""
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
#
# TODO will each platform have its own param dictionary?
#
param_dict_name = 'platform_eng_parsed'
parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
param_dict_name,
id_only=True)
self.parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed',
parameter_dictionary_id=parsed_rpdict_id)
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
driver_config = copy.deepcopy(DVR_CONFIG)
driver_config['attributes'] = self._platform_attributes[platform_id]
driver_config['ports'] = self._platform_ports[platform_id]
log.debug("driver_config: %s", driver_config)
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {
'driver_config': driver_config})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
platform_agent_obj = any_old(RT.PlatformAgent, {
"stream_configurations": self._get_platform_stream_configs(),
'driver_module': DVR_MOD,
'driver_class': DVR_CLS})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
self.addCleanup(self.DP.delete_data_product, dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, self.org_id)
#######################################
# dataset
log.debug('data product = %s', dp_id)
stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, None, True)
log.debug('Data product stream_ids = %s', stream_ids)
stream_id = stream_ids[0]
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
#######################################
return platform_agent_instance_id, platform_agent_id, platform_device_id, stream_id
log.debug("Making the structure for a platform agent")
# TODO Note: the 'platform_config' entry is a mechanism that the
# platform agent expects to know the platform_id and parent_platform_id.
# Determine how to finally indicate this info.
platform_config = {
'platform_id': platform_id,
'parent_platform_id': parent_platform_id,
}
child_agent_config = {
'platform_config': platform_config
}
platform_agent_instance_child_id, _, platform_device_child_id, stream_id = \
_make_platform_agent_structure(child_agent_config)
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
self.platform_device_parent_id = platform_device_child_id
p_obj = DotDict()
p_obj.platform_id = platform_id
p_obj.parent_platform_id = parent_platform_id
p_obj.platform_agent_instance_obj = platform_agent_instance_child_obj
p_obj.platform_device_id = platform_device_child_id
p_obj.platform_agent_instance_id = platform_agent_instance_child_id
p_obj.stream_id = stream_id
p_obj.pid = None # known when process launched
return p_obj
def _create_platform(self, platform_id, parent_platform_id=None):
"""
The main method to create a platform configuration and do other
preparations for a given platform.
"""
p_obj = self._create_platform_configuration(platform_id, parent_platform_id)
# start corresponding data subscriber:
self._start_data_subscriber(p_obj.platform_agent_instance_id,
p_obj.stream_id)
return p_obj
#################################################################
# platform child-parent linking
#################################################################
def _assign_child_to_parent(self, p_child, p_parent):
log.debug("assigning child platform %r to parent %r",
p_child.platform_id, p_parent.platform_id)
self.RR2.assign_platform_device_to_platform_device_with_has_device(p_child.platform_device_id,
p_parent.platform_device_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device_using_has_device(p_parent.platform_device_id)
self.assertNotEqual(0, len(child_device_ids))
#################################################################
# instrument
#################################################################
def _set_up_pre_environment_for_instrument(self, instr_info):
"""
Based on test_instrument_agent.py
Basically, this method launches a port agent and then completes the
instrument driver configuration used to properly set up a particular
instrument agent.
@param instr_info A value in instruments_dict
@return instrument_driver_config
"""
import sys
from ion.agents.instrument.driver_process import DriverProcessType
from ion.agents.instrument.driver_process import ZMQEggDriverProcess
# A seabird driver.
DRV_URI = SBE37_EGG
DRV_MOD = 'mi.instrument.seabird.sbe37smb.ooicore.driver'
DRV_CLS = 'SBE37Driver'
WORK_DIR = '/tmp/'
DELIM = ['<<', '>>']
instrument_driver_config = {
'dvr_egg' : DRV_URI,
'dvr_mod' : DRV_MOD,
'dvr_cls' : DRV_CLS,
'workdir' : WORK_DIR,
'process_type' : None
}
# Launch from egg or a local MI repo.
LAUNCH_FROM_EGG=True
if LAUNCH_FROM_EGG:
# Dynamically load the egg into the test path
launcher = ZMQEggDriverProcess(instrument_driver_config)
egg = launcher._get_egg(DRV_URI)
if not egg in sys.path: sys.path.insert(0, egg)
instrument_driver_config['process_type'] = (DriverProcessType.EGG,)
else:
mi_repo = os.getcwd() + os.sep + 'extern' + os.sep + 'mi_repo'
if not mi_repo in sys.path: sys.path.insert(0, mi_repo)
instrument_driver_config['process_type'] = (DriverProcessType.PYTHON_MODULE,)
instrument_driver_config['mi_repo'] = mi_repo
DEV_ADDR = instr_info['DEV_ADDR']
DEV_PORT = instr_info['DEV_PORT']
DATA_PORT = instr_info['DATA_PORT']
CMD_PORT = instr_info['CMD_PORT']
PA_BINARY = instr_info['PA_BINARY']
support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = support.start_pagent()
log.info('Port agent started at port %i', port)
self.addCleanup(support.stop_pagent)
# Configure instrument driver to use port agent port number.
instrument_driver_config['comms_config'] = {
'addr': 'localhost',
'port': port,
'cmd_port': CMD_PORT
}
return instrument_driver_config
def _make_instrument_agent_structure(self, instr_key, org_obj, agent_config=None):
if None is agent_config: agent_config = {}
instr_info = instruments_dict[instr_key]
# initially adapted from test_activate_instrument:test_activateInstrumentSample
# agent creation
instrument_agent_obj = IonObject(RT.InstrumentAgent,
name='agent007_%s' % instr_key,
description="SBE37IMAgent_%s" % instr_key,
driver_uri=SBE37_EGG,
stream_configurations=self._get_instrument_stream_configs())
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
log.debug('new InstrumentAgent id = %s', instrument_agent_id)
self.IMS.assign_instrument_model_to_instrument_agent(self.instModel_id, instrument_agent_id)
# device creation
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice_%s' % instr_key,
description="SBE37IMDevice_%s" % instr_key,
serial_number="12345")
instrument_device_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(self.instModel_id, instrument_device_id)
log.debug("new InstrumentDevice id = %s ", instrument_device_id)
#Create stream alarms
temp_alert_def = {
'name' : 'temperature_warning_interval',
'stream_name' : 'parsed',
'message' : 'Temperature is below the normal range of 50.0 and above.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : 50.0,
'lower_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
late_data_alert_def = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'message' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'value_id' : None,
'time_delta' : 2,
'alert_class' : 'LateDataAlert'
}
instrument_driver_config = self._set_up_pre_environment_for_instrument(instr_info)
port_agent_config = {
'device_addr': instr_info['DEV_ADDR'],
'device_port': instr_info['DEV_PORT'],
'data_port': instr_info['DATA_PORT'],
'command_port': instr_info['CMD_PORT'],
'binary_path': instr_info['PA_BINARY'],
'process_type': PortAgentProcessType.UNIX,
'port_agent_addr': 'localhost',
'log_level': 5,
'type': PortAgentType.ETHERNET
}
# instance creation
instrument_agent_instance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance_%s' % instr_key,
description="SBE37IMAgentInstance_%s" % instr_key,
driver_config=instrument_driver_config,
port_agent_config=port_agent_config,
alerts=[temp_alert_def, late_data_alert_def])
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# data products
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(org_obj)
# parsed:
parsed_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(
name='ctd_parsed',
parameter_dictionary_id=parsed_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the parsed data for %s' % instr_key,
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
self.addCleanup(self.DP.delete_data_product, data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id1)
# raw:
raw_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(
name='ctd_raw',
parameter_dictionary_id=raw_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data for %s' % instr_key,
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
self.addCleanup(self.DP.delete_data_product, data_product_id2)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id2)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
i_obj = DotDict()
i_obj.instrument_agent_id = instrument_agent_id
i_obj.instrument_device_id = instrument_device_id
i_obj.instrument_agent_instance_id = instrument_agent_instance_id
i_obj.org_obj = org_obj
log.debug("KK CREATED I_obj: %s", i_obj)
return i_obj
def _create_instrument(self, instr_key):
"""
The main method to create an instrument configuration.
@param instr_key A key in instruments_dict
@return instrument_driver_config
"""
self.assertIn(instr_key, instruments_dict)
self.assertNotIn(instr_key, self._setup_instruments)
instr_info = instruments_dict[instr_key]
log.debug("_create_instrument: creating instrument %r: %s",
instr_key, instr_info)
org_obj = any_old(RT.Org)
log.debug("making the structure for an instrument agent")
i_obj = self._make_instrument_agent_structure(instr_key, org_obj)
self._setup_instruments[instr_key] = i_obj
log.debug("_create_instrument: created instrument %r", instr_key)
return i_obj
def _get_instrument(self, instr_key):
"""
Gets the i_obj constructed by _create_instrument(instr_key).
"""
self.assertIn(instr_key, self._setup_instruments)
i_obj = self._setup_instruments[instr_key]
return i_obj
#################################################################
# instrument-platform linking
#################################################################
def _assign_instrument_to_platform(self, i_obj, p_obj):
log.debug("assigning instrument %r to platform %r",
i_obj.instrument_agent_instance_id, p_obj.platform_id)
self.RR2.assign_instrument_device_to_platform_device_with_has_device(
i_obj.instrument_device_id,
p_obj.platform_device_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device_using_has_device(p_obj.platform_device_id)
self.assertNotEqual(0, len(child_device_ids))
#################################################################
# some platform topologies
#################################################################
def _create_single_platform(self):
"""
Creates and prepares a platform corresponding to the
platform ID 'LJ01D', which is a leaf in the simulated network.
"""
p_root = self._create_platform('LJ01D')
return p_root
def _create_small_hierarchy(self):
"""
Creates a small platform network consisting of 3 platforms as follows:
Node1D -> MJ01C -> LJ01D
where -> goes from parent to child.
"""
p_root = self._create_platform('Node1D')
p_child = self._create_platform('MJ01C', parent_platform_id='Node1D')
p_grandchild = self._create_platform('LJ01D', parent_platform_id='MJ01C')
self._assign_child_to_parent(p_child, p_root)
self._assign_child_to_parent(p_grandchild, p_child)
return p_root
def _create_hierarchy(self, platform_id, p_objs, parent_obj=None):
"""
Creates a hierarchy of platforms rooted at the given platform.
@param platform_id ID of the root platform at this level
@param p_objs dict to be updated with (platform_id: p_obj)
mappings
@param parent_obj platform object of the parent, if any
@return platform object for the created root.
"""
# create the object to be returned:
p_obj = self._create_platform(platform_id)
# update (platform_id: p_obj) dict:
p_objs[platform_id] = p_obj
# recursively create child platforms:
pnode = self._network_definition.pnodes[platform_id]
for sub_platform_id in pnode.subplatforms:
self._create_hierarchy(sub_platform_id, p_objs, p_obj)
if parent_obj:
self._assign_child_to_parent(p_obj, parent_obj)
return p_obj
def _set_up_single_platform_with_some_instruments(self, instr_keys):
"""
Sets up single platform with some instruments
@param instr_keys Keys of the instruments to be assigned.
Must be keys in instruments_dict in
base_test_platform_agent_with_rsn
@return p_root for subsequent termination
"""
for instr_key in instr_keys:
self.assertIn(instr_key, instruments_dict)
p_root = self._create_single_platform()
# create and assign instruments:
for instr_key in instr_keys:
i_obj = self._create_instrument(instr_key)
self._assign_instrument_to_platform(i_obj, p_root)
return p_root
def _set_up_platform_hierarchy_with_some_instruments(self, instr_keys):
"""
Sets up a multiple-level platform hierarchy with instruments associated
to some of the platforms.
The platform hierarchy corresponds to the sub-network in the
simulated topology rooted at 'Node1B', which at time of writing
looks like this:
Node1B
Node1C
Node1D
MJ01C
LJ01D
LV01C
PC01B
SC01B
SF01B
LJ01C
LV01B
LJ01B
MJ01B
In DEBUG logging level for the platform agent, files like the following
are generated under logs/:
platform_CFG_received_Node1B.txt
platform_CFG_received_MJ01C.txt
platform_CFG_received_LJ01D.txt
@param instr_keys Keys of the instruments to be assigned.
Must be keys in instruments_dict in
base_test_platform_agent_with_rsn
@return p_root for subsequent termination
"""
for instr_key in instr_keys:
self.assertIn(instr_key, instruments_dict)
#####################################
# create platform hierarchy
#####################################
log.info("will create platform hierarchy ...")
start_time = time.time()
root_platform_id = 'Node1B'
p_objs = {}
p_root = self._create_hierarchy(root_platform_id, p_objs)
log.info("platform hierarchy built. Took %.3f secs. "
"Root platform=%r, number of platforms=%d: %s",
time.time() - start_time,
root_platform_id, len(p_objs), p_objs.keys())
self.assertIn(root_platform_id, p_objs)
self.assertEquals(13, len(p_objs))
#####################################
# create the indicated instruments
#####################################
log.info("will create %d instruments: %s", len(instr_keys), instr_keys)
start_time = time.time()
i_objs = []
for instr_key in instr_keys:
i_obj = self._create_instrument(instr_key)
i_objs.append(i_obj)
log.debug("instrument created = %r (%s)",
i_obj.instrument_agent_instance_id, instr_key)
log.info("%d instruments created. Took %.3f secs.", len(instr_keys), time.time() - start_time)
#####################################
# assign the instruments
#####################################
log.info("will assign instruments ...")
start_time = time.time()
plats_to_assign_instrs = [
'LJ01D', 'SF01B', 'LJ01B', 'MJ01B', # leaves
'MJ01C', 'Node1D', 'LV01B', 'Node1C' # intermediate
]
# assign one available instrument to a platform;
# the assignments are arbitrary.
num_assigns = min(len(instr_keys), len(plats_to_assign_instrs))
for ii in range(num_assigns):
platform_id = plats_to_assign_instrs[ii]
self.assertIn(platform_id, p_objs)
p_obj = p_objs[platform_id]
i_obj = i_objs[ii]
self._assign_instrument_to_platform(i_obj, p_obj)
log.debug("instrument %r (%s) assigned to platform %r",
i_obj.instrument_agent_instance_id,
instr_keys[ii],
platform_id)
log.info("%d instruments assigned. Took %.3f secs.",
num_assigns, time.time() - start_time)
return p_root
#################################################################
# start / stop platform
#################################################################
def _start_platform(self, p_obj):
agent_instance_id = p_obj.platform_agent_instance_id
log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id)
p_obj.pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", p_obj.pid)
#wait for start
agent_instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient('paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
def _stop_platform(self, p_obj):
try:
self.IMS.stop_platform_agent_instance(p_obj.platform_agent_instance_id)
except:
if log.isEnabledFor(logging.TRACE):
log.exception(
"platform_id=%r: Exception in IMS.stop_platform_agent_instance with "
"platform_agent_instance_id = %r",
p_obj.platform_id, p_obj.platform_agent_instance_id)
else:
log.warn(
"platform_id=%r: Exception in IMS.stop_platform_agent_instance with "
"platform_agent_instance_id = %r. Perhaps already dead.",
p_obj.platform_id, p_obj.platform_agent_instance_id)
#################################################################
# misc convenience methods
#################################################################
def _create_resource_agent_client(self, resource_id):
client = ResourceAgentClient(resource_id, process=FakeProcess())
return client
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs)
time_start = time.time()
#retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s",
cmd.command, elapsed_time, str(retval))
return retval
#################################################################
# commands that concrete tests can call
#################################################################
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _start_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.MONITORING)
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
# just wait for at least one -- see consume_data
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("Received samples: %s", len(self._samples_received))
def _wait_for_external_event(self):
log.info("waiting for reception of an external event...")
# just wait for at least one -- see consume_event
self._async_event_result.get(timeout=EVENT_TIMEOUT)
self.assertTrue(len(self._events_received) >= 1)
log.info("Received events: %s", len(self._events_received))
def _stop_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _pause(self):
cmd = AgentCommand(command=PlatformAgentEvent.PAUSE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.STOPPED)
def _resume(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESUME)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _clear(self):
cmd = AgentCommand(command=PlatformAgentEvent.CLEAR)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _check_sync(self):
cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC)
retval = self._execute_agent(cmd)
log.info("CHECK_SYNC result: %s", retval.result)
self.assertTrue(retval.result is not None)
self.assertEquals(retval.result[0:3], "OK:")
return retval.result
def _stream_instruments(self):
from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37ProtocolEvent
from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37Parameter
for instrument in self._setup_instruments.itervalues():
# instruments that have been set up: instr_key: i_obj
# Start a resource agent client to talk with the instrument agent.
_ia_client = self._create_resource_agent_client(instrument.instrument_device_id)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = _ia_client.execute_resource(cmd)
log.debug('_stream_instruments retval: %s', retval)
return
def _idle_instruments(self):
from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37ProtocolEvent
from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37Parameter
for instrument in self._setup_instruments.itervalues():
# instruments that have been set up: instr_key: i_obj
# Start a resource agent client to talk with the instrument agent.
_ia_client = self._create_resource_agent_client(instrument.instrument_device_id)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
with self.assertRaises(Conflict):
retval = _ia_client.execute_resource(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = _ia_client.execute_agent(cmd)
state = _ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
return
class Test2CAA(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
"""
###################################################################
# Internal parameters and container.
###################################################################
# Internal parameters.
self._terrestrial_platform_id = 'terrestrial_id'
self._remote_platform_id = 'remote_id'
self._resource_id = 'fake_id'
self._xs_name = 'remote1'
self._terrestrial_svc_name = 'terrestrial_endpoint'
self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
self._remote_svc_name = 'remote_endpoint'
self._remote_listen_name = self._remote_svc_name + self._xs_name
self._remote_port = 0
self._terrestrial_port = 0
self._te_client = None
self._re_client = None
self._remote_pid = None
self._terrestrial_pid = None
# Async test results.
self._no_requests = 10
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._telem_evts = []
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._results_recv = {}
self._requests_sent = {}
self._done_telem_evt = AsyncResult()
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
###################################################################
# Start endpoints, agent.
###################################################################
self._start_terrestrial()
self._start_remote()
self._start_agent()
###################################################################
# Assign client ports.
# This is primarily for test purposes as the IP config in
# deployment will be fixed in advance.
###################################################################
self.te_client.set_client_port(self._remote_port)
check_port = self.te_client.get_client_port()
log.debug('Terrestrial client port is: %i', check_port)
self.re_client.set_client_port(self._terrestrial_port)
check_port = self.re_client.get_client_port()
log.debug('Remote client port is: %i', check_port)
###################################################################
# Start the event publisher and subscribers.
# Used to send fake agent telemetry publications to the endpoints,
# and to receive endpoint publications.
###################################################################
self._event_publisher = EventPublisher()
# Start the event subscriber for remote namespace platform events.
# This event could be changed to RemoteNamespaceEvent.
self._event_subscriber = EventSubscriber(
event_type='PlatformEvent',
callback=self.consume_event,
origin=self._xs_name)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._event_subscriber.stop)
# Start the result subscriber for remote resource events.
self._resource_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=IA_RESOURCE_ID,
callback=self.consume_event)
self._resource_result_subscriber.start()
self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._resource_result_subscriber.stop)
# Start the result subscriber for remote service events.
self._service_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin='resource_registry' + 'remote1',
callback=self.consume_event)
self._service_result_subscriber.start()
self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._service_result_subscriber.stop)
# Start the result subscriber for fake resource results.
self._fake_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=self._resource_id,
callback=self.consume_event)
self._fake_result_subscriber.start()
self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._fake_result_subscriber.stop)
###################################################################
# Start/stop helpers.
###################################################################
def _start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _start_terrestrial(self):
"""
"""
# Create terrestrial config.
terrestrial_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._remote_port,
'this_port' : self._terrestrial_port,
'platform_resource_id' : self._terrestrial_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._terrestrial_listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._terrestrial_pid = self._container_client.spawn_process(
name=self._terrestrial_listen_name,
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=terrestrial_endpoint_config)
log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))
# Create a terrestrial client.
self.te_client = TerrestrialEndpointClient(
process=FakeProcess(),
to_name=self._terrestrial_listen_name)
log.debug('Got te client %s.', str(self.te_client))
self._terrestrial_port = self.te_client.get_port()
log.debug('Terrestrial port is: %i', self._terrestrial_port)
def _start_remote(self):
"""
"""
# Create agent config.
remote_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._terrestrial_port,
'this_port' : self._remote_port,
'platform_resource_id' : self._remote_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._remote_listen_name
}
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
self._remote_pid = self._container_client.spawn_process(
name=self._remote_listen_name,
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=remote_endpoint_config)
log.debug('Remote endpoint pid=%s.', str(self._remote_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=self._remote_listen_name)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._remote_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._remote_port)
###################################################################
# Telemetry publications to start/top endpoint.
# (Normally be published by appropriate platform agents.)
###################################################################
def terrestrial_link_up(self):
"""
Publish telemetry available to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.AVAILABLE)
def terrestrial_link_down(self):
"""
Publish telemetry unavailable to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def remote_link_up(self):
"""
Publish telemetry available to the remote endpoint.
"""
# Publish a link up event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.AVAILABLE)
def remote_link_down(self):
"""
Publish telemetry unavailable to the remote endpoint.
"""
# Publish a link down event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_event(self, evt, *args, **kwargs):
"""
Test callback for events.
"""
log.debug('Test got event: %s, args: %s, kwargs: %s',
str(evt), str(args), str(kwargs))
if evt.type_ == 'PublicPlatformTelemetryEvent':
self._telem_evts.append(evt)
if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts):
self._done_telem_evt.set()
elif evt.type_ == 'RemoteQueueModifiedEvent':
self._queue_mod_evts.append(evt)
if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts):
self._done_queue_mod_evt.set()
elif evt.type_ == 'RemoteCommandTransmittedEvent':
self._cmd_tx_evts.append(evt)
if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts):
self._done_cmd_tx_evt.set()
elif evt.type_ == 'RemoteCommandResult':
cmd = evt.command
self._results_recv[cmd.command_id] = cmd
if len(self._results_recv) == self._no_requests:
self._done_cmd_evt.set()
###################################################################
# Misc helpers.
###################################################################
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id='fake_id',
command=cmdstr,
args=['arg1', 23],
kwargs={'worktime':3})
return cmd
###################################################################
# Tests.
###################################################################
def test_queued_fake(self):
"""
test_queued_fake
Test fake resource commands queued prior to linkup.
"""
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
# The following error occurs when queue persistence is enabled.
# Need to verify correct solution to enable persistent queues.
"""
2012-11-06 14:27:13,517 INFO pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run
result = self._run(*self.args, **self.kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop
self._callback(cmd_result)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete
self._client.enqueue(result)
AttributeError: 'NoneType' object has no attribute 'enqueue'
<Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError
2012-11-06 14:27:13,586 INFO pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7
2012-11-06 14:27:13,665 INFO pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6
2012-11-06 14:27:13,739 INFO pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5
2012-11-06 14:27:13,807 INFO pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4
2012-11-06 14:27:13,874 INFO pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3
2012-11-06 14:27:13,941 INFO pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1
2012-11-06 14:27:13,945 INFO pyon.event.event EventSubscriber stopped. Event pattern=#
2012-11-06 14:27:14,124 INFO pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984
2012-11-06 14:27:14,399 INFO pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB
======================================================================
ERROR: test_process_online
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner
return f(*args, **kw)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online
cmd = self.te_client.enqueue_command(cmd)
File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command
return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request
return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request
retval, headers = e.send(msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send
result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send
return self._invite_and_send(to_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send
return self._send(to_role, to_role_name, msg, header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send
return self._end_point_unit._message_send(msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send
return ProcessRPCRequestEndpointUnit.send(self, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send
return self._send(_msg, _header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send
raise ex
Conflict: 409 - Object not based on most current version
"""
def test_process_online(self):
"""
test_process_online
Test fake resource commands queued while link is up.
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mods, command transmissions and results.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_remote_late(self):
"""
test_remote_late
Test fake resource commands queued prior to linkup.
Delay remote side linkup substantially to test terrestrial
behavior when remote server not initially available.
"""
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Actually stop the remote process, since the server is
# available even if it hasn't linked up yet and the test is running
# in the same container. The test will remember the appropriate
# remote port numbers.
self._container_client.terminate_process(self._remote_pid)
self.terrestrial_link_up()
gevent.sleep(10)
# Restart remote side and publish remote link up.
self._start_remote()
self.remote_link_up()
# Block for transmission and result events.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_resource_commands(self):
"""
test_resource_commands
"""
# Set up to verify the two commands queued.
self._no_requests = 2
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
# Use IA client to verify IA.
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Create and enqueue commands.
state_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={})
state_cmd = self.te_client.enqueue_command(state_cmd)
self._requests_sent[state_cmd.command_id] = state_cmd
ping_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='ping_agent',
args=[],
kwargs={})
ping_cmd = self.te_client.enqueue_command(ping_cmd)
self._requests_sent[ping_cmd.command_id] = ping_cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
self.assertEqual(self._results_recv[state_cmd.command_id].result,
ResourceAgentState.UNINITIALIZED)
self.assertIn('ping from InstrumentAgent',
self._results_recv[ping_cmd.command_id].result)
def test_service_command_sequence(self):
"""
test_service_commands
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='create',
args=[obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a tuple of strings.
#{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
#'1-00a308c45fff459c7cda1db9a7314de6'],
#'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a user info object with the name set.
#{'lcstate': 'DEPLOYED_AVAILABLE',
#'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
#'description': '',
#'ts_updated': '1349213207638',
#'type_': 'UserInfo',
#'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
#'_id': '27832d93f4cd4535a75ac75c06e00a7e',
#'ts_created': '1349213207638',
#'variables': [{'name': '', 'value': ''}],
#'name': 'some_name'}
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
read_obj.name = 'some_other_name'
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='update',
args=[read_obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='delete',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
class TestRemoteEndpoint(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
Start fake terrestrial components and add cleanup.
Start terrestrial server and retrieve port.
Set internal variables.
Start container.
Start deployment.
Start container agent.
Spawn remote endpoint process.
Create remote endpoint client and retrieve remote server port.
Create event publisher.
"""
self._terrestrial_server = R3PCServer(self.consume_req, self.terrestrial_server_close)
self._terrestrial_client = R3PCClient(self.consume_ack, self.terrestrial_client_close)
self.addCleanup(self._terrestrial_server.stop)
self.addCleanup(self._terrestrial_client.stop)
self._other_port = self._terrestrial_server.start('*', 0)
log.debug('Terrestrial server binding to *:%i', self._other_port)
self._other_host = 'localhost'
self._platform_resource_id = 'abc123'
self._resource_id = 'fake_id'
self._no_requests = 10
self._requests_sent = {}
self._results_recv = {}
self._no_telem_events = 0
self._done_evt = AsyncResult()
self._done_telem_evts = AsyncResult()
self._cmd_tx_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Create agent config.
endpoint_config = {
'other_host' : self._other_host,
'other_port' : self._other_port,
'this_port' : 0,
'platform_resource_id' : self._platform_resource_id
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
re_pid = container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(re_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=re_pid)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._this_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._this_port)
# Start the event publisher.
self._event_publisher = EventPublisher()
######################################################################
# Helpers.
######################################################################
def on_link_up(self):
"""
Called by a test to simulate turning the link on.
"""
log.debug('Terrestrial client connecting to localhost:%i.',
self._this_port)
self._terrestrial_client.start('localhost', self._this_port)
# Publish a link up event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status = TelemetryStatusType.AVAILABLE)
def on_link_down(self):
"""
Called by a test to simulate turning the link off.
"""
self._terrestrial_client.stop()
# Publish a link down event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_req(self, res):
"""
Consume a terrestrial request setting async event when necessary.
"""
command_id = res['command_id']
self._results_recv[command_id] = res
if len(self._results_recv) == self._no_requests:
self._done_evt.set()
def consume_ack(self, cmd):
"""
Consume terrestrial ack setting async event when necessary.
"""
self._requests_sent[cmd.command_id] = cmd
if len(self._requests_sent) == self._no_requests:
self._cmd_tx_evt.set()
def terrestrial_server_close(self):
"""
Callback when terrestrial server closes.
"""
pass
def terrestrial_client_close(self):
"""
Callback when terrestrial client closes.
"""
pass
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id=self._resource_id,
command=cmdstr,
args=['arg1', 23],
kwargs={'worktime':3},
command_id = str(uuid.uuid4()))
return cmd
def start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
######################################################################
# Tests.
######################################################################
def test_process_queued(self):
"""
test_process_queued
Test that queued commands are forwarded to and handled by
remote endpoint when link comes up.
"""
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_process_online(self):
"""
test_process_online
Test commands are forwarded and handled while link is up.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_terrestrial_late(self):
"""
test_terrestrial_late
Test queued commands are forwarded and handled by remote endpoint
when terrestrial side is late to come up.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Manually stop the terrestrial endpoint.
# This will cause it to be unavailable when commands are queued
# to simulate stability during asynchronous wake ups.
self._terrestrial_server.stop()
self._terrestrial_client.stop()
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Remote side awaits the terrestrial waking up.
gevent.sleep(3)
# Terrestrail endpoint eventually wakes up and starts transmitting.
self._terrestrial_client.start('localhost', self._this_port)
self._terrestrial_server.start('*', self._other_port)
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_service_commands(self):
"""
test_service_commands
Test that real service commands are handled by the remote endpoint.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='create',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = self._results_recv[cmd.command_id]['result']
# Confirm the results are valid.
"""
Result is a tuple of strings.
{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
'1-00a308c45fff459c7cda1db9a7314de6'],
'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
"""
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Read user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id]['result']
# Confirm the results are valid.
"""
Result is a user info object with the name set.
{'lcstate': 'DEPLOYED_AVAILABLE',
'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
'description': '',
'ts_updated': '1349213207638',
'type_': 'UserInfo',
'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
'_id': '27832d93f4cd4535a75ac75c06e00a7e',
'ts_created': '1349213207638',
'variables': [{'name': '', 'value': ''}],
'name': 'some_name'}
"""
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Update user object.
read_obj.name = 'some_other_name'
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='update',
args=[read_obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Read user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Delete user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='delete',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_resource_commands(self):
"""
test_resource_commands
Test that real resource commands are handled by the remote endpoint.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns agent state.
state = self._results_recv[cmd.command_id]['result']
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Ping agent via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='ping_agent',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns agent state.
ping = self._results_recv[cmd.command_id]['result']
self.assertIn('ping from InstrumentAgent', ping)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_bad_service_name_resource_id(self):
"""
test_bad_service_name_resource_id
Test for proper exception behavior when a bad service name or
resource id is used in a command forwarded to the remote endpoint.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='bogus_service',
command='create',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, NotFound)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id='bogus_resource_id',
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, NotFound)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_bad_commands(self):
"""
test_bad_commands
Test for correct exception behavior if a bad command name is forwarded
to a remote service or resource.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='what_the_flunk',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns BadRequest.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, BadRequest)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='what_the_flunk',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, BadRequest)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_resource_command_sequence(self):
"""
test_resource_command_sequence
Test for successful completion of a properly ordered sequence of
resource commands queued for forwarding to the remote endpoint.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# We execute a sequence of twelve consecutive events.
self._no_requests = 12
# Get agent state.
cmd1 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd1)
# Initialize agent.
cmd2 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.INITIALIZE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd2)
# Get agent state.
cmd3 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd3)
# Go active.
cmd4 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd4)
# Get agent state.
cmd5 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd5)
# Run.
cmd6 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.RUN)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd6)
# Get agent state.
cmd7 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd7)
# Acquire sample.
cmd8 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd8)
# Acquire sample
cmd9 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd9)
# Acquire sample.
cmd10 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd10)
# Reset.
cmd11 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.RESET)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd11)
# Get agent state.
cmd12 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd12)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Check results of command sequence.
"""
0ccf1e10-eeca-400d-aefe-f9d6888ec963 {'result': 'RESOURCE_AGENT_STATE_INACTIVE', 'command_id': '0ccf1e10-eeca-400d-aefe-f9d6888ec963'}
92531bdf-c2c8-4aa8-817d-5107c7311b37 {'result': <interface.objects.AgentCommandResult object at 0x10d7f11d0>, 'command_id': '92531bdf-c2c8-4aa8-817d-5107c7311b37'}
509934a1-5038-40d8-8014-591e2d8042b6 {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '509934a1-5038-40d8-8014-591e2d8042b6'}
88bacbb7-5366-4d27-9ecf-fff2bec34b2c {'result': <interface.objects.AgentCommandResult object at 0x10d389190>, 'command_id': '88bacbb7-5366-4d27-9ecf-fff2bec34b2c'}
f8b4d3fa-a249-439b-8bd4-ac212b6100aa {'result': <interface.objects.AgentCommandResult object at 0x10d3893d0>, 'command_id': 'f8b4d3fa-a249-439b-8bd4-ac212b6100aa'}
8ae98e39-fdb3-4218-ad8f-584620397d9f {'result': <interface.objects.AgentCommandResult object at 0x10d739990>, 'command_id': '8ae98e39-fdb3-4218-ad8f-584620397d9f'}
746364a1-c4c7-400f-96d4-ee36df5dc1a4 {'result': BadRequest('Execute argument "command" not set.',), 'command_id': '746364a1-c4c7-400f-96d4-ee36df5dc1a4'}
d516d3d9-e4f9-4ea5-80e0-34639a6377b5 {'result': <interface.objects.AgentCommandResult object at 0x10d3b2350>, 'command_id': 'd516d3d9-e4f9-4ea5-80e0-34639a6377b5'}
c7da03f5-59bc-420a-9e10-0a7794266599 {'result': 'RESOURCE_AGENT_STATE_IDLE', 'command_id': 'c7da03f5-59bc-420a-9e10-0a7794266599'}
678d870a-bf18-424a-afb0-f80ecf3277e2 {'result': <interface.objects.AgentCommandResult object at 0x10d739590>, 'command_id': '678d870a-bf18-424a-afb0-f80ecf3277e2'}
750c6a30-56eb-4535-99c2-a81fefab1b1f {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '750c6a30-56eb-4535-99c2-a81fefab1b1f'}
c17bd658-3775-4aa3-8844-02df70a0e3c0 {'result': 'RESOURCE_AGENT_STATE_UNINITIALIZED', 'command_id': 'c17bd658-3775-4aa3-8844-02df70a0e3c0'}
"""
# First result is a state string.
result1 = self._results_recv[cmd1.command_id]['result']
self.assertEqual(result1, ResourceAgentState.UNINITIALIZED)
# Second result is an empty AgentCommandResult.
result2 = self._results_recv[cmd2.command_id]['result']
# Third result is a state string.
result3 = self._results_recv[cmd3.command_id]['result']
self.assertEqual(result3, ResourceAgentState.INACTIVE)
# Fourth result is an empty AgentCommandResult.
result4 = self._results_recv[cmd4.command_id]['result']
# Fifth result is a state string.
result5 = self._results_recv[cmd5.command_id]['result']
self.assertEqual(result5, ResourceAgentState.IDLE)
# Sixth result is an empty AgentCommandResult.
result6 = self._results_recv[cmd6.command_id]['result']
# Seventh result is a state string.
result7 = self._results_recv[cmd7.command_id]['result']
self.assertEqual(result7, ResourceAgentState.COMMAND)
"""
{'raw': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
'stream_name': 'raw', 'pkt_format_id': 'JSON_Data',
'pkt_version': 1, '
values': [{'binary': True, 'value_id': 'raw',
'value': 'NzkuNDM3MywxNy4yMDU2NCwgNzYxLjg4NSwgICA2LjIxOTgsIDE1MDYuMzk3LCAwMSBGZWIgMjAwMSwgMDE6MDE6MDA='}],
'driver_timestamp': 3558286748.8039923},
'parsed': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data', 'pkt_version': 1,
'values': [{'value_id': 'temp', 'value': 79.4373},
{'value_id': 'conductivity', 'value': 17.20564},
{'value_id': 'pressure', 'value': 761.885}],
'driver_timestamp': 3558286748.8039923}}
"""
# Eigth result is an AgentCommandResult containing a sample.
result8 = self._results_recv[cmd8.command_id]['result']
self.assertTrue('parsed',result8.result )
# Ninth result is an AgentCommandResult containing a sample.
result9 = self._results_recv[cmd9.command_id]['result']
self.assertTrue('parsed',result9.result )
# Tenth result is an AgentCommandResult containing a sample.
result10 = self._results_recv[cmd10.command_id]['result']
self.assertTrue('parsed',result10.result )
# Eleventh result is an empty AgentCommandResult.
result11 = self._results_recv[cmd11.command_id]['result']
# Twelth result is a state string.
result12 = self._results_recv[cmd12.command_id]['result']
self.assertEqual(result1, ResourceAgentState.UNINITIALIZED)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
class TestDriverEgg(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
self.egg_url_good = "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg"
self.egg_url_bad = "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.1a-py2.7.egg"
self.egg_url_404 = "http://sddevrepo.oceanobservatories.org/releases/completely_made_up_404.egg"
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def get_streamConfigs(self):
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2,
granule_publish_rate=5 )
return raw_config, parsed_config
##########################
#
# The following tests generate different agent configs and pass them to a common base test script
#
###########################
@unittest.skip("this test can't be run from coi services. it is missing dependencies")
def test_driverLaunchModuleNoURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchModuleWithURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
driver_uri=self.egg_url_good,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchNoModuleOnlyURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=self.egg_url_good,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchBogusModuleWithURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="bogus",
driver_class="Bogus",
driver_uri=self.egg_url_good,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
@unittest.skip("Launches an egg 'process' even though the egg download should produce error 404")
def test_driverLaunchNoModule404URI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=self.egg_url_404,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj, False)
def test_driverLaunchNoModuleBadEggURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=self.egg_url_bad,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj, True, False)
def base_activateInstrumentSample(self, instAgent_obj, expect_launch=True, expect_command=True):
"""
This method runs a test of launching a driver with a given agent configuration
"""
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
print 'new InstrumentModel id = %s ' % instModel_id
# Create InstrumentAgent
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
print 'new InstrumentAgent id = %s' % instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',
id_only=True)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict',
id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed',
parameter_dictionary_id=parsed_pdict_id)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw',
parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new dp_id = %s' % data_product_id1
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
print 'Data product streams1 = %s' % stream_ids
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
print 'Data set for data_product_id1 = %s' % dataset_ids[0]
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
self.get_datastore(self.parsed_dataset)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
print 'new dp_id = %s' % str(data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
print 'Data product streams2 = %s' % str(stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
print 'Data set for data_product_id2 = %s' % dataset_ids[0]
self.raw_dataset = dataset_ids[0]
# add start/stop for instrument agent
gevent.joinall([gevent.spawn(lambda:
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id))])
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
if not expect_launch:
self.assertFalse(gate.await(30), "The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?" %
instance_obj.agent_process_id)
return
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING"
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
#print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
print "ResourceAgentClient created: %s" % str(self._ia_client)
print "Sending command=ResourceAgentEvent.INITIALIZE"
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
if not expect_command:
self.assertRaises(ServerError, self._ia_client.execute_agent, cmd)
return
retval = self._ia_client.execute_agent(cmd)
print "Result of INITIALIZE: %s" % str(retval)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
print "Sending command=ResourceAgentEvent.RESET"
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
print "Result of RESET: %s" % str(reply)
