def ping_agent(self, resource_id=''):
"""Ping the agent.
"""
res_type = self._get_resource_type(resource_id)
if self._has_agent(res_type):
rac = ResourceAgentClient(resource_id=resource_id)
return rac.ping_agent(resource_id=resource_id)
raise BadRequest("Not implemented for resource type %s" % res_type)
def ping_agent(self, resource_id=''):
"""Ping the agent.
"""
res_type = self._get_resource_type(resource_id)
if self._has_agent(res_type):
rac = ResourceAgentClient(resource_id=resource_id)
return rac.ping_agent(resource_id=resource_id)
raise BadRequest("Not implemented for resource type %s" % res_type)
def ping_agent(self, resource_id=''):
"""Ping the agent.
@param resource_id The id of the resource agennt.
@retval A str containing a string representation of the agent
and a timestamp.
@param resource_id str
@retval result str
"""
res_type = self._get_resource_type(resource_id)
if self._has_agent(res_type):
rac = ResourceAgentClient(resource_id=resource_id)
return rac.ping_agent(resource_id=resource_id)
raise BadRequest("Not implemented for resource type %s" % res_type)
def ping_agent(self, resource_id=''):
"""Ping the agent.
@param resource_id The id of the resource agennt.
@retval A str containing a string representation of the agent
and a timestamp.
@param resource_id str
@retval result str
"""
res_type = self._get_resource_type(resource_id)
if self._has_agent(res_type):
rac = ResourceAgentClient(resource_id=resource_id)
return rac.ping_agent(resource_id=resource_id)
raise BadRequest("Not implemented for resource type %s" % res_type)
class TestVel3d(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
print '#####################'
print 'IN SETUP'
self._ia_client = None
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
#log.info('driver uri: %s', DRV_URI)
#log.info('device address: %s', DEV_ADDR)
#log.info('device port: %s', DEV_PORT)
#log.info('work dir: %s', WORK_DIR)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : True,
'enable_persistence' : False
}
#if org_governance_name is not None:
# agent_config['org_governance_name'] = org_governance_name
# Start instrument agent.
log.info("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
log.info("Agent setup")
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
#self.addCleanup(self._verify_agent_reset)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
log.info('test setup complete')
###############################################################################
# Port agent helpers.
###############################################################################
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
###############################################################################
# Event helpers.
###############################################################################
def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
"""
Start a subscriber to the instrument agent events.
@param type The type of event to catch.
@count Trigger the async event result when events received reaches this.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
# Event array and async event result.
self._event_count = count
self._events_received = []
self._async_event_result = AsyncResult()
self._event_subscriber = EventSubscriber(
event_type=type, callback=consume_event,
origin=IA_RESOURCE_ID)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=5)
def _stop_event_subscriber(self):
"""
Stop event subscribers on cleanup.
"""
self._event_subscriber.stop()
self._event_subscriber = None
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
def _start_data_subscribers(self, count, raw_count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._raw_samples_received = []
self._async_sample_result = AsyncResult()
self._async_raw_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self._samples_received.append(message)
if len(self._samples_received) == count:
self._async_sample_result.set()
def recv_raw_data(message, stream_route, stream_id):
log.info('Received raw data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self._raw_samples_received.append(message)
if len(self._raw_samples_received) == raw_count:
self._async_raw_sample_result.set()
from pyon.util.containers import create_unique_identifier
stream_name = 'parsed'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" %
stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
stream_name = 'raw'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" %
stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber,'subscription_id'):
try:
pubsub_client.deactivate_subscription(subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
###############################################################################
# tcp helpers.
###############################################################################
def _start_tcp_client(self, retval):
host = retval.result['ip_address']
port = retval.result['port']
tcp_client = TcpClient(host, port)
return tcp_client
###############################################################################
# Tests.
###############################################################################
@unittest.skip('Test should be run manually only.')
def test_initialize(self):
"""
test_initialize
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
print '#### in test'
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
@unittest.skip('Test should be run manually only.')
def test_xx(self):
"""
"""
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
with self.assertRaises(Conflict):
res_state = self._ia_client.get_resource_state()
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print '################################# mavs4 came up in state: ' + state
if state == ResourceAgentState.IDLE:
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
elif state == ResourceAgentState.STREAMING:
cmd = AgentCommand(command='DRIVER_EVENT_STOP_AUTOSAMPLE')
retval = self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
state = self._ia_client.get_agent_state()
print '################################# mavs4 now in state: ' + state
cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
kwargs={'session_type': DirectAccessTypes.telnet,
'session_timeout':600,
'inactivity_timeout':600})
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.DIRECT_ACCESS)
state = self._ia_client.get_agent_state()
print '################################# mavs4 now in state: ' + state
tcp_client = self._start_tcp_client(retval)
self.assertTrue(tcp_client.start_telnet(token=retval.result['token']))
self.assertTrue(tcp_client.send_data('\r\r'))
gevent.sleep(180)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.DIRECT_ACCESS)
cmd = AgentCommand(command=ResourceAgentEvent.GO_COMMAND)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
class BaseIntTestPlatform(IonIntegrationTestCase, HelperTestMixin):
"""
A base class with several conveniences supporting specific platform agent
integration tests, see:
- ion/agents/platform/test/test_platform_agent_with_rsn.py
- ion/services/sa/observatory/test/test_platform_launch.py
The platform IDs used here are organized as follows:
Node1D -> MJ01C -> LJ01D
where -> goes from parent platform to child platform.
This is a subset of the whole topology defined in the simulated platform
network (network.yml), which in turn is used by the RSN OMS simulator.
- 'LJ01D' is the root platform used in test_single_platform
- 'Node1D' is the root platform used in test_hierarchy
Methods are provided to construct specific platform topologies, but
subclasses decide which to use.
"""
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.org_id = self.RR2.create(any_old(RT.Org))
log.debug("Org created: %s", self.org_id)
# Create InstrumentModel
# TODO create multiple models as needed; for the moment assuming all
# used instruments are the same model here.
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
self.instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', self.instModel_id)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
CIOMSClientFactory.destroy_instance(rsn_oms)
if log.isEnabledFor(logging.TRACE):
# show serialized version for the network definition:
network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
log.trace("NetworkDefinition serialization:\n%s", network_definition_ser)
# set attributes for the platforms:
self._platform_attributes = {}
for platform_id in self._network_definition.pnodes:
pnode = self._network_definition.pnodes[platform_id]
dic = dict((attr.attr_id, attr.defn) for attr in pnode.attrs.itervalues())
self._platform_attributes[platform_id] = dic
log.trace("_platform_attributes: %s", self._platform_attributes)
# set ports for the platforms:
self._platform_ports = {}
for platform_id in self._network_definition.pnodes:
pnode = self._network_definition.pnodes[platform_id]
dic = {}
for port_id, port in pnode.ports.iteritems():
dic[port_id] = dict(port_id=port_id,
network=port.network)
self._platform_ports[platform_id] = dic
log.trace("_platform_ports: %s", self._platform_attributes)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber(sub_type="platform_event")
# instruments that have been set up: instr_key: i_obj
self._setup_instruments = {}
#################################################################
# data subscribers handling
#################################################################
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s. stream_name=%r stream_id=%r',
str(message), stream_name, stream_id)
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
#################################################################
# event subscribers handling
#################################################################
def _start_event_subscriber(self, event_type="DeviceEvent",
sub_type=None,
count=0):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
if count == 0:
self._async_event_result.set(evt)
elif count == len(self._events_received):
self._async_event_result.set()
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r, count=%d",
event_type, sub_type, count)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
#################################################################
# config supporting methods
#################################################################
def _get_platform_stream_configs(self):
"""
This method is an adaptation of get_streamConfigs in
test_driver_egg.py
"""
return [
StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
# TODO include a "raw" stream?
]
def _get_instrument_stream_configs(self):
"""
configs copied from test_activate_instrument.py
"""
return [
StreamConfiguration(stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5),
StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2, granule_publish_rate=5)
]
def _verify_child_config(self, config, device_id, is_platform):
for key in required_config_keys:
self.assertIn(key, config)
if is_platform:
self.assertEqual(RT.PlatformDevice, config['device_type'])
for key in DVR_CONFIG.iterkeys():
self.assertIn(key, config['driver_config'])
for key in ['startup_config']:
self.assertEqual({}, config[key])
else:
self.assertEqual(RT.InstrumentDevice, config['device_type'])
for key in ['children']:
self.assertEqual({}, config[key])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
def _verify_parent_config(self, config, parent_device_id,
child_device_id, is_platform):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(RT.PlatformDevice, config['device_type'])
for key in DVR_CONFIG.iterkeys():
self.assertIn(key, config['driver_config'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
self._verify_child_config(config['children'][child_device_id],
child_device_id, is_platform)
def _create_platform_configuration(self, platform_id, parent_platform_id=None):
"""
This method is an adaptation of test_agent_instance_config in
test_instrument_management_service_integration.py
@param platform_id
@param parent_platform_id
@return a DotDict with various of the constructed elements associated
to the platform.
"""
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
#
# TODO will each platform have its own param dictionary?
#
param_dict_name = 'platform_eng_parsed'
parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
param_dict_name,
id_only=True)
self.parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed',
parameter_dictionary_id=parsed_rpdict_id)
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
driver_config = copy.deepcopy(DVR_CONFIG)
driver_config['attributes'] = self._platform_attributes[platform_id]
driver_config['ports'] = self._platform_ports[platform_id]
log.debug("driver_config: %s", driver_config)
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {
'driver_config': driver_config})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
platform_agent_obj = any_old(RT.PlatformAgent, {
"stream_configurations": self._get_platform_stream_configs(),
'driver_module': DVR_MOD,
'driver_class': DVR_CLS})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
self.addCleanup(self.DP.delete_data_product, dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, self.org_id)
#######################################
# dataset
log.debug('data product = %s', dp_id)
stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, None, True)
log.debug('Data product stream_ids = %s', stream_ids)
stream_id = stream_ids[0]
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
#######################################
return platform_agent_instance_id, platform_agent_id, platform_device_id, stream_id
log.debug("Making the structure for a platform agent")
# TODO Note: the 'platform_config' entry is a mechanism that the
# platform agent expects to know the platform_id and parent_platform_id.
# Determine how to finally indicate this info.
platform_config = {
'platform_id': platform_id,
'parent_platform_id': parent_platform_id,
}
child_agent_config = {
'platform_config': platform_config
}
platform_agent_instance_child_id, _, platform_device_child_id, stream_id = \
_make_platform_agent_structure(child_agent_config)
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
self.platform_device_parent_id = platform_device_child_id
p_obj = DotDict()
p_obj.platform_id = platform_id
p_obj.parent_platform_id = parent_platform_id
p_obj.platform_agent_instance_obj = platform_agent_instance_child_obj
p_obj.platform_device_id = platform_device_child_id
p_obj.platform_agent_instance_id = platform_agent_instance_child_id
p_obj.stream_id = stream_id
p_obj.pid = None # known when process launched
return p_obj
def _create_platform(self, platform_id, parent_platform_id=None):
"""
The main method to create a platform configuration and do other
preparations for a given platform.
"""
p_obj = self._create_platform_configuration(platform_id, parent_platform_id)
# start corresponding data subscriber:
self._start_data_subscriber(p_obj.platform_agent_instance_id,
p_obj.stream_id)
return p_obj
#################################################################
# platform child-parent linking
#################################################################
def _assign_child_to_parent(self, p_child, p_parent):
log.debug("assigning child platform %r to parent %r",
p_child.platform_id, p_parent.platform_id)
self.RR2.assign_platform_device_to_platform_device_with_has_device(p_child.platform_device_id,
p_parent.platform_device_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device_using_has_device(p_parent.platform_device_id)
self.assertNotEqual(0, len(child_device_ids))
#################################################################
# instrument
#################################################################
def _set_up_pre_environment_for_instrument(self, instr_info):
"""
Based on test_instrument_agent.py
Basically, this method launches a port agent and then completes the
instrument driver configuration used to properly set up a particular
instrument agent.
@param instr_info A value in instruments_dict
@return instrument_driver_config
"""
import sys
from ion.agents.instrument.driver_process import DriverProcessType
from ion.agents.instrument.driver_process import ZMQEggDriverProcess
# A seabird driver.
DRV_URI = SBE37_EGG
DRV_MOD = 'mi.instrument.seabird.sbe37smb.ooicore.driver'
DRV_CLS = 'SBE37Driver'
WORK_DIR = '/tmp/'
DELIM = ['<<', '>>']
instrument_driver_config = {
'dvr_egg' : DRV_URI,
'dvr_mod' : DRV_MOD,
'dvr_cls' : DRV_CLS,
'workdir' : WORK_DIR,
'process_type' : None
}
# Launch from egg or a local MI repo.
LAUNCH_FROM_EGG=True
if LAUNCH_FROM_EGG:
# Dynamically load the egg into the test path
launcher = ZMQEggDriverProcess(instrument_driver_config)
egg = launcher._get_egg(DRV_URI)
if not egg in sys.path: sys.path.insert(0, egg)
instrument_driver_config['process_type'] = (DriverProcessType.EGG,)
else:
mi_repo = os.getcwd() + os.sep + 'extern' + os.sep + 'mi_repo'
if not mi_repo in sys.path: sys.path.insert(0, mi_repo)
instrument_driver_config['process_type'] = (DriverProcessType.PYTHON_MODULE,)
instrument_driver_config['mi_repo'] = mi_repo
DEV_ADDR = instr_info['DEV_ADDR']
DEV_PORT = instr_info['DEV_PORT']
DATA_PORT = instr_info['DATA_PORT']
CMD_PORT = instr_info['CMD_PORT']
PA_BINARY = instr_info['PA_BINARY']
support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = support.start_pagent()
log.info('Port agent started at port %i', port)
self.addCleanup(support.stop_pagent)
# Configure instrument driver to use port agent port number.
instrument_driver_config['comms_config'] = {
'addr': 'localhost',
'port': port,
'cmd_port': CMD_PORT
}
return instrument_driver_config
def _make_instrument_agent_structure(self, instr_key, org_obj, agent_config=None):
if None is agent_config: agent_config = {}
instr_info = instruments_dict[instr_key]
# initially adapted from test_activate_instrument:test_activateInstrumentSample
# agent creation
instrument_agent_obj = IonObject(RT.InstrumentAgent,
name='agent007_%s' % instr_key,
description="SBE37IMAgent_%s" % instr_key,
driver_uri=SBE37_EGG,
stream_configurations=self._get_instrument_stream_configs())
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
log.debug('new InstrumentAgent id = %s', instrument_agent_id)
self.IMS.assign_instrument_model_to_instrument_agent(self.instModel_id, instrument_agent_id)
# device creation
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice_%s' % instr_key,
description="SBE37IMDevice_%s" % instr_key,
serial_number="12345")
instrument_device_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(self.instModel_id, instrument_device_id)
log.debug("new InstrumentDevice id = %s ", instrument_device_id)
#Create stream alarms
temp_alert_def = {
'name' : 'temperature_warning_interval',
'stream_name' : 'parsed',
'message' : 'Temperature is below the normal range of 50.0 and above.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : 50.0,
'lower_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
late_data_alert_def = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'message' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'value_id' : None,
'time_delta' : 2,
'alert_class' : 'LateDataAlert'
}
instrument_driver_config = self._set_up_pre_environment_for_instrument(instr_info)
port_agent_config = {
'device_addr': instr_info['DEV_ADDR'],
'device_port': instr_info['DEV_PORT'],
'data_port': instr_info['DATA_PORT'],
'command_port': instr_info['CMD_PORT'],
'binary_path': instr_info['PA_BINARY'],
'process_type': PortAgentProcessType.UNIX,
'port_agent_addr': 'localhost',
'log_level': 5,
'type': PortAgentType.ETHERNET
}
# instance creation
instrument_agent_instance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance_%s' % instr_key,
description="SBE37IMAgentInstance_%s" % instr_key,
driver_config=instrument_driver_config,
port_agent_config=port_agent_config,
alerts=[temp_alert_def, late_data_alert_def])
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# data products
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(org_obj)
# parsed:
parsed_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(
name='ctd_parsed',
parameter_dictionary_id=parsed_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the parsed data for %s' % instr_key,
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
self.addCleanup(self.DP.delete_data_product, data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id1)
# raw:
raw_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(
name='ctd_raw',
parameter_dictionary_id=raw_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data for %s' % instr_key,
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
self.addCleanup(self.DP.delete_data_product, data_product_id2)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id2)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
i_obj = DotDict()
i_obj.instrument_agent_id = instrument_agent_id
i_obj.instrument_device_id = instrument_device_id
i_obj.instrument_agent_instance_id = instrument_agent_instance_id
i_obj.org_obj = org_obj
log.debug("KK CREATED I_obj: %s", i_obj)
return i_obj
def _create_instrument(self, instr_key):
"""
The main method to create an instrument configuration.
@param instr_key A key in instruments_dict
@return instrument_driver_config
"""
self.assertIn(instr_key, instruments_dict)
self.assertNotIn(instr_key, self._setup_instruments)
instr_info = instruments_dict[instr_key]
log.debug("_create_instrument: creating instrument %r: %s",
instr_key, instr_info)
org_obj = any_old(RT.Org)
log.debug("making the structure for an instrument agent")
i_obj = self._make_instrument_agent_structure(instr_key, org_obj)
self._setup_instruments[instr_key] = i_obj
log.debug("_create_instrument: created instrument %r", instr_key)
return i_obj
def _get_instrument(self, instr_key):
"""
Gets the i_obj constructed by _create_instrument(instr_key).
"""
self.assertIn(instr_key, self._setup_instruments)
i_obj = self._setup_instruments[instr_key]
return i_obj
#################################################################
# instrument-platform linking
#################################################################
def _assign_instrument_to_platform(self, i_obj, p_obj):
log.debug("assigning instrument %r to platform %r",
i_obj.instrument_agent_instance_id, p_obj.platform_id)
self.RR2.assign_instrument_device_to_platform_device_with_has_device(
i_obj.instrument_device_id,
p_obj.platform_device_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device_using_has_device(p_obj.platform_device_id)
self.assertNotEqual(0, len(child_device_ids))
#################################################################
# some platform topologies
#################################################################
def _create_single_platform(self):
"""
Creates and prepares a platform corresponding to the
platform ID 'LJ01D', which is a leaf in the simulated network.
"""
p_root = self._create_platform('LJ01D')
return p_root
def _create_small_hierarchy(self):
"""
Creates a small platform network consisting of 3 platforms as follows:
Node1D -> MJ01C -> LJ01D
where -> goes from parent to child.
"""
p_root = self._create_platform('Node1D')
p_child = self._create_platform('MJ01C', parent_platform_id='Node1D')
p_grandchild = self._create_platform('LJ01D', parent_platform_id='MJ01C')
self._assign_child_to_parent(p_child, p_root)
self._assign_child_to_parent(p_grandchild, p_child)
return p_root
def _create_hierarchy(self, platform_id, p_objs, parent_obj=None):
"""
Creates a hierarchy of platforms rooted at the given platform.
@param platform_id ID of the root platform at this level
@param p_objs dict to be updated with (platform_id: p_obj)
mappings
@param parent_obj platform object of the parent, if any
@return platform object for the created root.
"""
# create the object to be returned:
p_obj = self._create_platform(platform_id)
# update (platform_id: p_obj) dict:
p_objs[platform_id] = p_obj
# recursively create child platforms:
pnode = self._network_definition.pnodes[platform_id]
for sub_platform_id in pnode.subplatforms:
self._create_hierarchy(sub_platform_id, p_objs, p_obj)
if parent_obj:
self._assign_child_to_parent(p_obj, parent_obj)
return p_obj
def _set_up_single_platform_with_some_instruments(self, instr_keys):
"""
Sets up single platform with some instruments
@param instr_keys Keys of the instruments to be assigned.
Must be keys in instruments_dict in
base_test_platform_agent_with_rsn
@return p_root for subsequent termination
"""
for instr_key in instr_keys:
self.assertIn(instr_key, instruments_dict)
p_root = self._create_single_platform()
# create and assign instruments:
for instr_key in instr_keys:
i_obj = self._create_instrument(instr_key)
self._assign_instrument_to_platform(i_obj, p_root)
return p_root
def _set_up_platform_hierarchy_with_some_instruments(self, instr_keys):
"""
Sets up a multiple-level platform hierarchy with instruments associated
to some of the platforms.
The platform hierarchy corresponds to the sub-network in the
simulated topology rooted at 'Node1B', which at time of writing
looks like this:
Node1B
Node1C
Node1D
MJ01C
LJ01D
LV01C
PC01B
SC01B
SF01B
LJ01C
LV01B
LJ01B
MJ01B
In DEBUG logging level for the platform agent, files like the following
are generated under logs/:
platform_CFG_received_Node1B.txt
platform_CFG_received_MJ01C.txt
platform_CFG_received_LJ01D.txt
@param instr_keys Keys of the instruments to be assigned.
Must be keys in instruments_dict in
base_test_platform_agent_with_rsn
@return p_root for subsequent termination
"""
for instr_key in instr_keys:
self.assertIn(instr_key, instruments_dict)
#####################################
# create platform hierarchy
#####################################
log.info("will create platform hierarchy ...")
start_time = time.time()
root_platform_id = 'Node1B'
p_objs = {}
p_root = self._create_hierarchy(root_platform_id, p_objs)
log.info("platform hierarchy built. Took %.3f secs. "
"Root platform=%r, number of platforms=%d: %s",
time.time() - start_time,
root_platform_id, len(p_objs), p_objs.keys())
self.assertIn(root_platform_id, p_objs)
self.assertEquals(13, len(p_objs))
#####################################
# create the indicated instruments
#####################################
log.info("will create %d instruments: %s", len(instr_keys), instr_keys)
start_time = time.time()
i_objs = []
for instr_key in instr_keys:
i_obj = self._create_instrument(instr_key)
i_objs.append(i_obj)
log.debug("instrument created = %r (%s)",
i_obj.instrument_agent_instance_id, instr_key)
log.info("%d instruments created. Took %.3f secs.", len(instr_keys), time.time() - start_time)
#####################################
# assign the instruments
#####################################
log.info("will assign instruments ...")
start_time = time.time()
plats_to_assign_instrs = [
'LJ01D', 'SF01B', 'LJ01B', 'MJ01B', # leaves
'MJ01C', 'Node1D', 'LV01B', 'Node1C' # intermediate
]
# assign one available instrument to a platform;
# the assignments are arbitrary.
num_assigns = min(len(instr_keys), len(plats_to_assign_instrs))
for ii in range(num_assigns):
platform_id = plats_to_assign_instrs[ii]
self.assertIn(platform_id, p_objs)
p_obj = p_objs[platform_id]
i_obj = i_objs[ii]
self._assign_instrument_to_platform(i_obj, p_obj)
log.debug("instrument %r (%s) assigned to platform %r",
i_obj.instrument_agent_instance_id,
instr_keys[ii],
platform_id)
log.info("%d instruments assigned. Took %.3f secs.",
num_assigns, time.time() - start_time)
return p_root
#################################################################
# start / stop platform
#################################################################
def _start_platform(self, p_obj):
agent_instance_id = p_obj.platform_agent_instance_id
log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id)
p_obj.pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", p_obj.pid)
#wait for start
agent_instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient('paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
def _stop_platform(self, p_obj):
try:
self.IMS.stop_platform_agent_instance(p_obj.platform_agent_instance_id)
except:
if log.isEnabledFor(logging.TRACE):
log.exception(
"platform_id=%r: Exception in IMS.stop_platform_agent_instance with "
"platform_agent_instance_id = %r",
p_obj.platform_id, p_obj.platform_agent_instance_id)
else:
log.warn(
"platform_id=%r: Exception in IMS.stop_platform_agent_instance with "
"platform_agent_instance_id = %r. Perhaps already dead.",
p_obj.platform_id, p_obj.platform_agent_instance_id)
#################################################################
# misc convenience methods
#################################################################
def _create_resource_agent_client(self, resource_id):
client = ResourceAgentClient(resource_id, process=FakeProcess())
return client
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs)
time_start = time.time()
#retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s",
cmd.command, elapsed_time, str(retval))
return retval
#################################################################
# commands that concrete tests can call
#################################################################
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _start_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.MONITORING)
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
# just wait for at least one -- see consume_data
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("Received samples: %s", len(self._samples_received))
def _wait_for_external_event(self):
log.info("waiting for reception of an external event...")
# just wait for at least one -- see consume_event
self._async_event_result.get(timeout=EVENT_TIMEOUT)
self.assertTrue(len(self._events_received) >= 1)
log.info("Received events: %s", len(self._events_received))
def _stop_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _pause(self):
cmd = AgentCommand(command=PlatformAgentEvent.PAUSE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.STOPPED)
def _resume(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESUME)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _clear(self):
cmd = AgentCommand(command=PlatformAgentEvent.CLEAR)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _check_sync(self):
cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC)
retval = self._execute_agent(cmd)
log.info("CHECK_SYNC result: %s", retval.result)
self.assertTrue(retval.result is not None)
self.assertEquals(retval.result[0:3], "OK:")
return retval.result
def _stream_instruments(self):
from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37ProtocolEvent
from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37Parameter
for instrument in self._setup_instruments.itervalues():
# instruments that have been set up: instr_key: i_obj
# Start a resource agent client to talk with the instrument agent.
_ia_client = self._create_resource_agent_client(instrument.instrument_device_id)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = _ia_client.execute_resource(cmd)
log.debug('_stream_instruments retval: %s', retval)
return
def _idle_instruments(self):
from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37ProtocolEvent
from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37Parameter
for instrument in self._setup_instruments.itervalues():
# instruments that have been set up: instr_key: i_obj
# Start a resource agent client to talk with the instrument agent.
_ia_client = self._create_resource_agent_client(instrument.instrument_device_id)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
with self.assertRaises(Conflict):
retval = _ia_client.execute_resource(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = _ia_client.execute_agent(cmd)
state = _ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
return
class TestAgentPersistence(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : False,
'enable_persistence' : True
}
self._ia_client = None
self._ia_pid = '8989'
self.addCleanup(self._verify_agent_reset)
self.addCleanup(self.container.state_repository.put_state,
self._ia_pid, {})
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=self._agent_config,
process_id=self._ia_pid)
log.info('Started instrument agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_agent_config_persistence(self):
"""
test_agent_config_persistence
Test that agent parameter configuration is persisted between running
instances.
"""
# Start the agent.
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the default agent parameters.
#{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
retval = self._ia_client.get_agent(['streams'])['streams']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
#{'pubrate': {'raw': 0, 'parsed': 0}}
retval = self._ia_client.get_agent(['pubrate'])['pubrate']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
self.assertEqual(retval['raw'], 0)
self.assertEqual(retval['parsed'], 0)
#{'alerts': []}
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertEqual(retval, [])
# Define a few new parameters and set them.
# Confirm they are set.
alert_def_1 = {
'name' : 'current_warning_interval',
'stream_name' : 'parsed',
'message' : 'Current is below normal range.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 10.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def_2 = {
'name' : 'temp_alarm_interval',
'stream_name' : 'parsed',
'message' : 'Temperatoure is critical.',
'alert_type' : StreamAlertType.ALARM,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 20.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'message' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'value_id' : None,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_alerts = [alert_def_1,alert_def_2, alert_def3]
pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : orig_alerts,
'pubrate' : pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(5)
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the persisted parameters.
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
def test_agent_state_persistence(self):
"""
test_agent_state_persistence
Verify that agents can be restored to their prior running state.
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(3)
self._start_agent()
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(3)
self._start_agent()
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class Test2CAA(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
"""
###################################################################
# Internal parameters and container.
###################################################################
# Internal parameters.
self._terrestrial_platform_id = 'terrestrial_id'
self._remote_platform_id = 'remote_id'
self._resource_id = 'fake_id'
self._xs_name = 'remote1'
self._terrestrial_svc_name = 'terrestrial_endpoint'
self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
self._remote_svc_name = 'remote_endpoint'
self._remote_listen_name = self._remote_svc_name + self._xs_name
self._remote_port = 0
self._terrestrial_port = 0
self._te_client = None
self._re_client = None
self._remote_pid = None
self._terrestrial_pid = None
# Async test results.
self._no_requests = 10
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._telem_evts = []
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._results_recv = {}
self._requests_sent = {}
self._done_telem_evt = AsyncResult()
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
###################################################################
# Start endpoints, agent.
###################################################################
self._start_terrestrial()
self._start_remote()
self._start_agent()
###################################################################
# Assign client ports.
# This is primarily for test purposes as the IP config in
# deployment will be fixed in advance.
###################################################################
self.te_client.set_client_port(self._remote_port)
check_port = self.te_client.get_client_port()
log.debug('Terrestrial client port is: %i', check_port)
self.re_client.set_client_port(self._terrestrial_port)
check_port = self.re_client.get_client_port()
log.debug('Remote client port is: %i', check_port)
###################################################################
# Start the event publisher and subscribers.
# Used to send fake agent telemetry publications to the endpoints,
# and to receive endpoint publications.
###################################################################
self._event_publisher = EventPublisher()
# Start the event subscriber for remote namespace platform events.
# This event could be changed to RemoteNamespaceEvent.
self._event_subscriber = EventSubscriber(
event_type='PlatformEvent',
callback=self.consume_event,
origin=self._xs_name)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._event_subscriber.stop)
# Start the result subscriber for remote resource events.
self._resource_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=IA_RESOURCE_ID,
callback=self.consume_event)
self._resource_result_subscriber.start()
self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._resource_result_subscriber.stop)
# Start the result subscriber for remote service events.
self._service_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin='resource_registry' + 'remote1',
callback=self.consume_event)
self._service_result_subscriber.start()
self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._service_result_subscriber.stop)
# Start the result subscriber for fake resource results.
self._fake_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=self._resource_id,
callback=self.consume_event)
self._fake_result_subscriber.start()
self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._fake_result_subscriber.stop)
###################################################################
# Start/stop helpers.
###################################################################
def _start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _start_terrestrial(self):
"""
"""
# Create terrestrial config.
terrestrial_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._remote_port,
'this_port' : self._terrestrial_port,
'platform_resource_id' : self._terrestrial_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._terrestrial_listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._terrestrial_pid = self._container_client.spawn_process(
name=self._terrestrial_listen_name,
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=terrestrial_endpoint_config)
log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))
# Create a terrestrial client.
self.te_client = TerrestrialEndpointClient(
process=FakeProcess(),
to_name=self._terrestrial_listen_name)
log.debug('Got te client %s.', str(self.te_client))
self._terrestrial_port = self.te_client.get_port()
log.debug('Terrestrial port is: %i', self._terrestrial_port)
def _start_remote(self):
"""
"""
# Create agent config.
remote_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._terrestrial_port,
'this_port' : self._remote_port,
'platform_resource_id' : self._remote_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._remote_listen_name
}
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
self._remote_pid = self._container_client.spawn_process(
name=self._remote_listen_name,
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=remote_endpoint_config)
log.debug('Remote endpoint pid=%s.', str(self._remote_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=self._remote_listen_name)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._remote_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._remote_port)
###################################################################
# Telemetry publications to start/top endpoint.
# (Normally be published by appropriate platform agents.)
###################################################################
def terrestrial_link_up(self):
"""
Publish telemetry available to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.AVAILABLE)
def terrestrial_link_down(self):
"""
Publish telemetry unavailable to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def remote_link_up(self):
"""
Publish telemetry available to the remote endpoint.
"""
# Publish a link up event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.AVAILABLE)
def remote_link_down(self):
"""
Publish telemetry unavailable to the remote endpoint.
"""
# Publish a link down event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_event(self, evt, *args, **kwargs):
"""
Test callback for events.
"""
log.debug('Test got event: %s, args: %s, kwargs: %s',
str(evt), str(args), str(kwargs))
if evt.type_ == 'PublicPlatformTelemetryEvent':
self._telem_evts.append(evt)
if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts):
self._done_telem_evt.set()
elif evt.type_ == 'RemoteQueueModifiedEvent':
self._queue_mod_evts.append(evt)
if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts):
self._done_queue_mod_evt.set()
elif evt.type_ == 'RemoteCommandTransmittedEvent':
self._cmd_tx_evts.append(evt)
if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts):
self._done_cmd_tx_evt.set()
elif evt.type_ == 'RemoteCommandResult':
cmd = evt.command
self._results_recv[cmd.command_id] = cmd
if len(self._results_recv) == self._no_requests:
self._done_cmd_evt.set()
###################################################################
# Misc helpers.
###################################################################
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id='fake_id',
command=cmdstr,
args=['arg1', 23],
kwargs={'kwargs1':'someval'})
return cmd
###################################################################
# Tests.
###################################################################
def test_queued_fake(self):
"""
test_queued_fake
Test fake resource commands queued prior to linkup.
"""
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
# The following error occurs when queue persistence is enabled.
# Need to verify correct solution to enable persistent queues.
"""
2012-11-06 14:27:13,517 INFO pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run
result = self._run(*self.args, **self.kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop
self._callback(cmd_result)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete
self._client.enqueue(result)
AttributeError: 'NoneType' object has no attribute 'enqueue'
<Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError
2012-11-06 14:27:13,586 INFO pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7
2012-11-06 14:27:13,665 INFO pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6
2012-11-06 14:27:13,739 INFO pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5
2012-11-06 14:27:13,807 INFO pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4
2012-11-06 14:27:13,874 INFO pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3
2012-11-06 14:27:13,941 INFO pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1
2012-11-06 14:27:13,945 INFO pyon.event.event EventSubscriber stopped. Event pattern=#
2012-11-06 14:27:14,124 INFO pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984
2012-11-06 14:27:14,399 INFO pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB
======================================================================
ERROR: test_process_online
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner
return f(*args, **kw)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online
cmd = self.te_client.enqueue_command(cmd)
File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command
return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request
return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request
retval, headers = e.send(msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send
result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send
return self._invite_and_send(to_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send
return self._send(to_role, to_role_name, msg, header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send
return self._end_point_unit._message_send(msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send
return ProcessRPCRequestEndpointUnit.send(self, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send
return self._send(_msg, _header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send
raise ex
Conflict: 409 - Object not based on most current version
"""
def test_process_online(self):
"""
test_process_online
Test fake resource commands queued while link is up.
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mods, command transmissions and results.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_remote_late(self):
"""
test_remote_late
Test fake resource commands queued prior to linkup.
Delay remote side linkup substantially to test terrestrial
behavior when remote server not initially available.
"""
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Actually stop the remote process, since the server is
# available even if it hasn't linked up yet and the test is running
# in the same container. The test will remember the appropriate
# remote port numbers.
self._container_client.terminate_process(self._remote_pid)
self.terrestrial_link_up()
gevent.sleep(10)
# Restart remote side and publish remote link up.
self._start_remote()
self.remote_link_up()
# Block for transmission and result events.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_resource_commands(self):
"""
test_resource_commands
"""
# Set up to verify the two commands queued.
self._no_requests = 2
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
# Use IA client to verify IA.
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Create and enqueue commands.
state_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={})
state_cmd = self.te_client.enqueue_command(state_cmd)
self._requests_sent[state_cmd.command_id] = state_cmd
ping_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='ping_agent',
args=[],
kwargs={})
ping_cmd = self.te_client.enqueue_command(ping_cmd)
self._requests_sent[ping_cmd.command_id] = ping_cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
self.assertEqual(self._results_recv[state_cmd.command_id].result,
ResourceAgentState.UNINITIALIZED)
self.assertIn('ping from InstrumentAgent',
self._results_recv[ping_cmd.command_id].result)
def test_service_command_sequence(self):
"""
test_service_commands
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='create',
args=[obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a tuple of strings.
#{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
#'1-00a308c45fff459c7cda1db9a7314de6'],
#'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a user info object with the name set.
#{'lcstate': 'DEPLOYED_AVAILABLE',
#'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
#'description': '',
#'ts_updated': '1349213207638',
#'type_': 'UserInfo',
#'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
#'_id': '27832d93f4cd4535a75ac75c06e00a7e',
#'ts_created': '1349213207638',
#'variables': [{'name': '', 'value': ''}],
#'name': 'some_name'}
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
read_obj.name = 'some_other_name'
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='update',
args=[read_obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='delete',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
class TestAgentPersistence(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : False,
'enable_persistence' : True,
'aparam_pubrate_config' :
{
'raw' : 2,
'parsed' : 2
}
}
self._ia_client = None
self._ia_pid = '1234'
self.addCleanup(self._verify_agent_reset)
self.addCleanup(self.container.state_repository.put_state,
self._ia_pid, {})
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self, bootmode=None):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = bootmode
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config,
process_id=self._ia_pid)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_agent_config_persistence(self):
"""
test_agent_config_persistence
Test that agent parameter configuration is persisted between running
instances.
"""
# Start the agent.
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the default agent parameters.
#{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
retval = self._ia_client.get_agent(['streams'])['streams']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
#{'pubrate': {'raw': 0, 'parsed': 0}}
retval = self._ia_client.get_agent(['pubrate'])['pubrate']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
self.assertEqual(retval['raw'], 2)
self.assertEqual(retval['parsed'], 2)
#{'alerts': []}
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertEqual(retval, [])
# Define a few new parameters and set them.
# Confirm they are set.
alert_def_1 = {
'name' : 'current_warning_interval',
'stream_name' : 'parsed',
'description' : 'Current is below normal range.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 10.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def_2 = {
'name' : 'temp_alarm_interval',
'stream_name' : 'parsed',
'description' : 'Temperatoure is critical.',
'alert_type' : StreamAlertType.ALARM,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 20.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_alerts = [alert_def_1,alert_def_2, alert_def3]
pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : orig_alerts,
'pubrate' : pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the persisted parameters.
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
def test_agent_state_persistence(self):
"""
test_agent_state_persistence
Verify that agents can be restored to their prior running state.
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : [alert_def3],
'pubrate' : orig_pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
print '## in state: ' + state
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.STOPPED:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to STOPPED.")
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_agent_rparam_persistence(self):
"""
test_agent_rparam_persistence
Verify ability to restore device configuration.
### Original values:
{'TA0': -0.0002572242, 'OUTPUTSV': False, 'NAVG': 0}
### Values after set:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
### Restore config:
{'PTCA1': 0.6603433, 'WBOTC': 1.2024e-05, 'PCALDATE': [12, 8, 2005],
'STORETIME': False, 'CPCOR': 9.57e-08, 'PTCA2': 0.00575649,
'OUTPUTSV': True, 'SAMPLENUM': 0, 'TCALDATE': [8, 11, 2005],
'OUTPUTSAL': False, 'TA2': -9.717158e-06, 'POFFSET': 0.0,
'INTERVAL': 19733, 'SYNCWAIT': 0, 'CJ': 3.339261e-05,
'CI': 0.0001334915, 'CH': 0.1417895, 'TA0': -0.0005144484,
'TA1': 0.0003138936, 'NAVG': 1, 'TA3': 2.138735e-07, '
RCALDATE': [8, 11, 2005], 'CG': -0.987093, 'CTCOR': 3.25e-06, '
PTCB0': 24.6145, 'PTCB1': -0.0009, 'PTCB2': 0.0,
'CCALDATE': [8, 11, 2005], 'PA0': 5.916199, 'PA1': 0.4851819,
'PA2': 4.596432e-07, 'SYNCMODE': False, 'PTCA0': 276.2492,
'TXREALTIME': True, 'RTCA2': -3.022745e-08, 'RTCA1': 1.686132e-06,
'RTCA0': 0.9999862}
### Of which we have:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
params = [
SBE37Parameter.OUTPUTSV,
SBE37Parameter.NAVG,
SBE37Parameter.TA0
]
retval = self._ia_client.get_resource(params)
orig_params = retval
new_params = {
SBE37Parameter.OUTPUTSV : not orig_params[SBE37Parameter.OUTPUTSV],
SBE37Parameter.NAVG : orig_params[SBE37Parameter.NAVG] + 1,
SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 2
}
#print '########### orig params'
#print str(orig_params)
self._ia_client.set_resource(new_params)
retval = self._ia_client.get_resource(params)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
#print '########### new params'
#print str(retval)
# Now stop and restart the agent.
self._stop_agent()
self._support.stop_pagent()
gevent.sleep(10)
self._start_pagent()
gevent.sleep(10)
self._start_agent('restart')
timeout = gevent.Timeout(600)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(3)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
# Verify the parameters have been restored as needed.
retval = self._ia_client.get_resource(params)
#print '########### restored params'
#print str(retval)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
@unittest.skip('Making CEI friendly.')
def test_cei_launch_mode(self):
pdc = ProcessDispatcherServiceClient(node=self.container.node)
p_def = ProcessDefinition(name='Agent007')
p_def.executable = {
'module' : 'ion.agents.instrument.instrument_agent',
'class' : 'InstrumentAgent'
}
p_def_id = pdc.create_process_definition(p_def)
pid = pdc.create_process(p_def_id)
def event_callback(event, *args, **kwargs):
print '######### proc %s in state %s' % (event.origin, ProcessStateEnum._str_map[event.state])
sub = EventSubscriber(event_type='ProcessLifecycleEvent',
callback=event_callback,
origin=pid,
origin_type='DispatchedProcess')
sub.start()
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = 'restart'
pdc.schedule_process(p_def_id, process_id=pid,
configuration=agent_config)
gevent.sleep(5)
pdc.cancel_process(pid)
gevent.sleep(15)
sub.stop()
def test_xxx(self, host='localhost', port=DATA_PORT,
resource_id=IA_RESOURCE_ID, stream_config=None):
"""
"""
if not stream_config:
stream_config = self._stream_config
log.info("create FakePuckReader")
puck_reader = FakePuckReader()
log.info("read_puck()")
puck_data = puck_reader.read_puck(DEV_ADDR, DEV_PORT, DEV_SERIAL_LINE)
driver_config = {
'dvr_mod' : puck_data['dvr_mod'],
'dvr_cls' : puck_data['dvr_cls'],
'workdir' : WORK_DIR,
'process_type' : PROCESS_TYPE,
'comms_config' : {
'addr' : host,
'port' : port
}
}
agent_config = {
'driver_config' : driver_config,
'stream_config' : stream_config,
'agent' : {'resource_id': resource_id},
'test_mode' : True
}
log.debug("Starting instrument agent.")
ia_pid = self.container_client.spawn_process(
name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
#self.addCleanup(self._verify_agent_reset)
ia_client = ResourceAgentClient(resource_id, process=FakeProcess())
log.info('Got ia client %s.', str(ia_client))
# We start in uninitialized state.
# In this state there is no driver process.
state = ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = ia_client.ping_agent()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = ia_client.execute_agent(cmd)
state = ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = ia_client.execute_agent(cmd)
state = ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = ia_client.execute_agent(cmd)
state = ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = ia_client.execute_resource(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = ia_client.execute_agent(cmd)
state = ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestPlatformLaunch(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(
self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s",
self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name,
stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
def _start_event_subscriber(self,
event_type="DeviceEvent",
sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
def _create_platform_configuration(self):
"""
Verify that agent configurations are being built properly
"""
#
# This method is an adaptation of test_agent_instance_config in
# test_instrument_management_service_integration.py
#
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(any_old(RT.Org))
inst_startup_config = {'startup': 'config'}
required_config_keys = [
'org_name', 'device_type', 'agent', 'driver_config',
'stream_config', 'startup_config', 'alarm_defs', 'children'
]
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {
'process_type': ('ZMQPyClassDriverLauncher', ),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher', )},
config['driver_config'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
if None is inst_device_id:
for key in ['alarm_defs', 'children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id],
inst_device_id)
def verify_parent_config(config,
parent_device_id,
child_device_id,
inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher', )},
config['driver_config'])
self.assertEqual({'resource_id': parent_device_id},
config['agent'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id],
child_device_id, inst_device_id)
parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
'platform_eng_parsed', id_only=True)
self.parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_rpdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(
name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance)
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(
platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
platform_agent_obj = any_old(
RT.PlatformAgent, {"stream_configurations": [raw_config]})
platform_agent_id = self.IMS.create_platform_agent(
platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(
any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {
"temporal_domain": tdom,
"spatial_domain": sdom
})
# dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
dp_id = self.DP.create_data_product(
data_product=dp_obj,
stream_definition_id=self.parsed_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id,
data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(
platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(
platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(
platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(
RT.InstrumentAgentInstance,
{"startup_config": inst_startup_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(
instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(
stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5)
instrument_agent_obj = any_old(
RT.InstrumentAgent, {"stream_configurations": [raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(
instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(
any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {
"temporal_domain": tdom,
"spatial_domain": sdom
})
dp_id = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(
input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(
instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(
instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(
instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug(
"Testing that preparing a launcher without agent instance raises an error"
)
self.assertRaises(AssertionError,
pconfig_builder.prepare,
will_launch=False)
log.debug(
"Making the structure for a platform agent, which will be the child"
)
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure(
)
platform_agent_instance_child_obj = self.RR2.read(
platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug(
"Making the structure for a platform agent, which will be the parent"
)
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure(
)
platform_agent_instance_parent_obj = self.RR2.read(
platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(
platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(
platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id,
platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure(
)
instrument_agent_instance_obj = self.RR2.read(
instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(
instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(
instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(
platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id,
platform_device_child_id, instrument_device_id)
if log.isEnabledFor(logging.TRACE):
import pprint
pp = pprint.PrettyPrinter()
pp.pprint(full_config)
log.trace("full_config = %s", pp.pformat(full_config))
return full_config
def get_streamConfigs(self):
#
# This method is an adaptation of get_streamConfigs in
# test_driver_egg.py
#
return [
StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
# TODO enable something like the following when also
# incorporating "raw" data:
#,
#StreamConfiguration(stream_name='raw',
# parameter_dictionary_name='ctd_raw_param_dict',
# records_per_granule=2,
# granule_publish_rate=5)
]
@skip("Still needs alignment with new configuration structure")
def test_hierarchy(self):
# TODO re-implement.
pass
def test_single_platform(self):
full_config = self._create_platform_configuration()
platform_id = 'LJ01D'
stream_configurations = self.get_streamConfigs()
agent__obj = IonObject(RT.PlatformAgent,
name='%s_PlatformAgent' % platform_id,
description='%s_PlatformAgent platform agent' %
platform_id,
stream_configurations=stream_configurations)
agent_id = self.IMS.create_platform_agent(agent__obj)
device__obj = IonObject(
RT.PlatformDevice,
name='%s_PlatformDevice' % platform_id,
description='%s_PlatformDevice platform device' % platform_id,
# ports=port_objs,
# platform_monitor_attributes = monitor_attribute_objs
)
self.device_id = self.IMS.create_platform_device(device__obj)
#######################################
# data product (adapted from test_instrument_management_service_integration)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='DataProduct test',
processing_level_code='Parsed_Canonical',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.DP.create_data_product(
data_product=dp_obj,
stream_definition_id=self.parsed_stream_def_id)
log.debug('data_product_id1 = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=self.device_id,
data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(
data_product_id=data_product_id1)
#######################################
#######################################
# dataset
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream,
None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1,
PRED.hasDataset, RT.Dataset,
True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#######################################
full_config['platform_config'] = {
'platform_id': platform_id,
'driver_config': DVR_CONFIG,
'network_definition': self._network_definition_ser
}
agent_instance_obj = IonObject(
RT.PlatformAgentInstance,
name='%s_PlatformAgentInstance' % platform_id,
description="%s_PlatformAgentInstance" % platform_id,
agent_config=full_config)
agent_instance_id = self.IMS.create_platform_agent_instance(
platform_agent_instance=agent_instance_obj,
platform_agent_id=agent_id,
platform_device_id=self.device_id)
stream_id = stream_ids[0]
self._start_data_subscriber(agent_instance_id, stream_id)
log.debug(
"about to call imsclient.start_platform_agent_instance with id=%s",
agent_instance_id)
pid = self.IMS.start_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (90),
"The platform agent instance did not spawn in 90 seconds")
agent_instance_obj = self.IMS.read_instrument_agent_instance(
agent_instance_id)
log.debug('Platform agent instance obj')
# Start a resource agent client to talk with the instrument agent.
self._pa_client = ResourceAgentClient(
'paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
# ping_agent can be issued before INITIALIZE
retval = self._pa_client.ping_agent(timeout=TIMEOUT)
log.debug('Base Platform ping_agent = %s', str(retval))
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform INITIALIZE = %s', str(retval))
# GO_ACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform GO_ACTIVE = %s', str(retval))
# RUN:
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform RUN = %s', str(retval))
# START_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform START_MONITORING = %s', str(retval))
# wait for data sample
# just wait for at least one -- see consume_data above
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("waiting a bit more for reception of more data samples...")
sleep(15)
log.info("Got data samples: %d", len(self._samples_received))
# wait for event
# just wait for at least one event -- see consume_event above
log.info("waiting for reception of an event...")
self._async_event_result.get(timeout=EVENT_TIMEOUT)
log.info("Received events: %s", len(self._events_received))
#get the extended platfrom which wil include platform aggreate status fields
# extended_platform = self.IMS.get_platform_device_extension(self.device_id)
# log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) )
# log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) )
# log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) )
# STOP_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform STOP_MONITORING = %s', str(retval))
# GO_INACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform GO_INACTIVE = %s', str(retval))
# RESET: Resets the base platform agent, which includes termination of
# its sub-platforms processes:
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform RESET = %s', str(retval))
#-------------------------------
# Stop Base Platform AgentInstance
#-------------------------------
self.IMS.stop_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
class BaseIntTestPlatform(IonIntegrationTestCase, HelperTestMixin):
"""
A base class with several conveniences supporting specific platform agent
integration tests, see:
- ion/agents/platform/test/test_platform_agent_with_rsn.py
- ion/services/sa/observatory/test/test_platform_launch.py
The platform IDs used here are organized as follows:
Node1D -> MJ01C -> LJ01D
where -> goes from parent platform to child platform.
This is a subset of the whole topology defined in the simulated platform
network (network.yml), which in turn is used by the RSN OMS simulator.
- 'LJ01D' is the root platform used in test_single_platform
- 'Node1D' is the root platform used in test_hierarchy
Methods are provided to construct specific platform topologies, but
subclasses decide which to use.
"""
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.org_id = self.RR2.create(any_old(RT.Org))
log.debug("Org created: %s", self.org_id)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
CIOMSClientFactory.destroy_instance(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser)
# set attributes for the platforms:
self._platform_attributes = {}
for platform_id in self._network_definition.pnodes:
pnode = self._network_definition.pnodes[platform_id]
dic = dict((attr.attr_id, attr.defn) for attr in pnode.attrs.itervalues())
self._platform_attributes[platform_id] = dic
log.trace("_platform_attributes: %s", self._platform_attributes)
# set ports for the platforms:
self._platform_ports = {}
for platform_id in self._network_definition.pnodes:
pnode = self._network_definition.pnodes[platform_id]
dic = {}
for port_id, port in pnode.ports.iteritems():
dic[port_id] = dict(port_id=port_id,
network=port.network)
self._platform_ports[platform_id] = dic
log.trace("_platform_ports: %s", self._platform_attributes)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
#################################################################
# data subscribers handling
#################################################################
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s. stream_name=%r stream_id=%r',
str(message), stream_name, stream_id)
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
#################################################################
# event subscribers handling
#################################################################
def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
#################################################################
# config supporting methods
#################################################################
def _create_platform_config_builder(self):
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
return pconfig_builder
def _generate_parent_with_child_config(self, p_parent, p_child):
log.debug("Testing parent platform + child platform as parent config")
pconfig_builder = self._create_platform_config_builder()
pconfig_builder.set_agent_instance_object(p_parent.platform_agent_instance_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
self._verify_parent_config(parent_config,
p_parent.platform_device_id,
p_child.platform_device_id,
is_platform=True)
self._debug_config(parent_config,
"platform_agent_config_%s_->_%s.txt" % (
p_parent.platform_id, p_child.platform_id))
def _generate_platform_with_instrument_config(self, p_obj, i_obj):
log.debug("Testing parent platform + child instrument as parent config")
pconfig_builder = self._create_platform_config_builder()
pconfig_builder.set_agent_instance_object(p_obj.platform_agent_instance_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
self._verify_parent_config(parent_config,
p_obj.platform_device_id,
i_obj.instrument_device_id,
is_platform=False)
self._debug_config(parent_config,
"platform_agent_config_%s_->_%s.txt" % (
p_obj.platform_id, i_obj.instrument_device_id))
def _generate_config(self, platform_agent_instance_obj, platform_id, suffix=''):
pconfig_builder = self._create_platform_config_builder()
pconfig_builder.set_agent_instance_object(platform_agent_instance_obj)
config = pconfig_builder.prepare(will_launch=False)
self._debug_config(config, "platform_agent_config_%s%s.txt" % (platform_id, suffix))
return config
def _get_platform_stream_configs(self):
"""
This method is an adaptation of get_streamConfigs in
test_driver_egg.py
"""
return [
StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
# TODO include a "raw" stream?
]
def _get_instrument_stream_configs(self):
"""
configs copied from test_activate_instrument.py
"""
return [
StreamConfiguration(stream_name='ctd_raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5),
StreamConfiguration(stream_name='ctd_parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2, granule_publish_rate=5)
]
def _create_instrument_config_builder(self):
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
return iconfig_builder
def _generate_instrument_config(self, instrument_agent_instance_obj, instrument_id, suffix=''):
pconfig_builder = self._create_instrument_config_builder()
pconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
config = pconfig_builder.prepare(will_launch=False)
self._debug_config(config, "instrument_agent_config_%s%s.txt" % (instrument_id, suffix))
return config
def _debug_config(self, config, outname):
if log.isEnabledFor(logging.DEBUG):
import pprint
outname = "logs/%s" % outname
try:
pprint.PrettyPrinter(stream=file(outname, "w")).pprint(config)
log.debug("config pretty-printed to %s", outname)
except Exception as e:
log.warn("error printing config to %s: %s", outname, e)
def _verify_child_config(self, config, device_id, is_platform):
for key in required_config_keys:
self.assertIn(key, config)
if is_platform:
self.assertEqual(RT.PlatformDevice, config['device_type'])
for key in DVR_CONFIG.iterkeys():
self.assertIn(key, config['driver_config'])
for key in ['children', 'startup_config']:
self.assertEqual({}, config[key])
else:
self.assertEqual(RT.InstrumentDevice, config['device_type'])
for key in ['children']:
self.assertEqual({}, config[key])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
def _verify_parent_config(self, config, parent_device_id,
child_device_id, is_platform):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(RT.PlatformDevice, config['device_type'])
for key in DVR_CONFIG.iterkeys():
self.assertIn(key, config['driver_config'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
self._verify_child_config(config['children'][child_device_id],
child_device_id, is_platform)
def _create_platform_configuration(self, platform_id, parent_platform_id=None):
"""
This method is an adaptation of test_agent_instance_config in
test_instrument_management_service_integration.py
@param platform_id
@param parent_platform_id
@return a DotDict with various of the constructed elements associated
to the platform.
"""
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
#
# TODO will each platform have its own param dictionary?
#
param_dict_name = 'platform_eng_parsed'
parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
param_dict_name,
id_only=True)
self.parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed',
parameter_dictionary_id=parsed_rpdict_id)
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
driver_config = copy.deepcopy(DVR_CONFIG)
driver_config['attributes'] = self._platform_attributes[platform_id]
driver_config['ports'] = self._platform_ports[platform_id]
log.debug("driver_config: %s", driver_config)
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {
'driver_config': driver_config})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
platform_agent_obj = any_old(RT.PlatformAgent, {
"stream_configurations": self._get_platform_stream_configs(),
'driver_module': DVR_MOD,
'driver_class': DVR_CLS})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, self.org_id)
#######################################
# dataset
log.debug('data product = %s', dp_id)
stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, None, True)
log.debug('Data product stream_ids = %s', stream_ids)
stream_id = stream_ids[0]
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
#######################################
return platform_agent_instance_id, platform_agent_id, platform_device_id, stream_id
log.debug("Making the structure for a platform agent")
# TODO Note: the 'platform_config' entry is a mechanism that the
# platform agent expects to know the platform_id and parent_platform_id.
# Determine how to finally indicate this info.
platform_config = {
'platform_id': platform_id,
'parent_platform_id': parent_platform_id,
}
child_agent_config = {
'platform_config': platform_config
}
platform_agent_instance_child_id, _, platform_device_child_id, stream_id = \
_make_platform_agent_structure(child_agent_config)
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
child_config = self._generate_config(platform_agent_instance_child_obj, platform_id)
self._verify_child_config(child_config, platform_device_child_id,
is_platform=True)
self.platform_device_parent_id = platform_device_child_id
p_obj = DotDict()
p_obj.platform_id = platform_id
p_obj.parent_platform_id = parent_platform_id
p_obj.agent_config = child_config
p_obj.platform_agent_instance_obj = platform_agent_instance_child_obj
p_obj.platform_device_id = platform_device_child_id
p_obj.platform_agent_instance_id = platform_agent_instance_child_id
p_obj.stream_id = stream_id
return p_obj
def _create_platform(self, platform_id, parent_platform_id=None):
"""
The main method to create a platform configuration and do other
preparations for a given platform.
"""
p_obj = self._create_platform_configuration(platform_id, parent_platform_id)
# start corresponding data subscriber:
self._start_data_subscriber(p_obj.platform_agent_instance_id,
p_obj.stream_id)
return p_obj
#################################################################
# platform child-parent linking
#################################################################
def _assign_child_to_parent(self, p_child, p_parent):
log.debug("assigning child platform %r to parent %r",
p_child.platform_id, p_parent.platform_id)
self.RR2.assign_platform_device_to_platform_device(p_child.platform_device_id,
p_parent.platform_device_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(p_parent.platform_device_id)
self.assertNotEqual(0, len(child_device_ids))
self._generate_parent_with_child_config(p_parent, p_child)
#################################################################
# instrument
#################################################################
def _set_up_pre_environment_for_instrument(self):
"""
From test_instrument_agent.py
Basically, this method launches the port agent and the completes the
instrument driver configuration used to properly set up the
instrument agent.
@return instrument_driver_config
"""
import sys
from ion.agents.instrument.driver_process import DriverProcessType
from ion.agents.instrument.driver_process import ZMQEggDriverProcess
# A seabird driver.
DRV_URI = 'http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.7-py2.7.egg'
DRV_MOD = 'mi.instrument.seabird.sbe37smb.ooicore.driver'
DRV_CLS = 'SBE37Driver'
WORK_DIR = '/tmp/'
DELIM = ['<<', '>>']
instrument_driver_config = {
'dvr_egg' : DRV_URI,
'dvr_mod' : DRV_MOD,
'dvr_cls' : DRV_CLS,
'workdir' : WORK_DIR,
'process_type' : None
}
# Launch from egg or a local MI repo.
LAUNCH_FROM_EGG=True
if LAUNCH_FROM_EGG:
# Dynamically load the egg into the test path
launcher = ZMQEggDriverProcess(instrument_driver_config)
egg = launcher._get_egg(DRV_URI)
if not egg in sys.path: sys.path.insert(0, egg)
instrument_driver_config['process_type'] = (DriverProcessType.EGG,)
else:
mi_repo = os.getcwd() + os.sep + 'extern' + os.sep + 'mi_repo'
if not mi_repo in sys.path: sys.path.insert(0, mi_repo)
instrument_driver_config['process_type'] = (DriverProcessType.PYTHON_MODULE,)
instrument_driver_config['mi_repo'] = mi_repo
DEV_ADDR = CFG.device.sbe37.host
DEV_PORT = CFG.device.sbe37.port
DATA_PORT = CFG.device.sbe37.port_agent_data_port
CMD_PORT = CFG.device.sbe37.port_agent_cmd_port
PA_BINARY = CFG.device.sbe37.port_agent_binary
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i', port)
self.addCleanup(self._support.stop_pagent)
# Configure instrument driver to use port agent port number.
instrument_driver_config['comms_config'] = {
'addr': 'localhost',
'port': port,
'cmd_port': CMD_PORT
}
return instrument_driver_config
def _make_instrument_agent_structure(self, org_obj, agent_config=None):
if None is agent_config: agent_config = {}
# from test_activate_instrument:test_activateInstrumentSample
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
# agent creation
instrument_agent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
stream_configurations=self._get_instrument_stream_configs())
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
log.debug('new InstrumentAgent id = %s', instrument_agent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instrument_agent_id)
# device creation
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instrument_device_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instrument_device_id)
log.debug("new InstrumentDevice id = %s ", instrument_device_id)
#Create stream alarms
alert_def = {
'name' : 'temperature_warning_interval',
'stream_name' : 'ctd_parsed',
'message' : 'Temperature is below the normal range of 50.0 and above.',
'alert_type' : StreamAlertType.WARNING,
'value_id' : 'temp',
'resource_id' : instrument_device_id,
'origin_type' : 'device',
'lower_bound' : 50.0,
'lower_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
instrument_driver_config = self._set_up_pre_environment_for_instrument()
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
# instance creation
instrument_agent_instance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
driver_config=instrument_driver_config,
port_agent_config=port_agent_config,
alerts=[alert_def])
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# data products
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(org_obj)
# parsed:
parsed_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(
name='ctd_parsed',
parameter_dictionary_id=parsed_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id1)
# raw:
raw_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(
name='ctd_raw',
parameter_dictionary_id=raw_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id2)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
i_obj = DotDict()
i_obj.instrument_agent_id = instrument_agent_id
i_obj.instrument_device_id = instrument_device_id
i_obj.instrument_agent_instance_id = instrument_agent_instance_id
return i_obj
def verify_instrument_config(self, config, org_obj, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQEggDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['children']:
self.assertEqual({}, config[key])
def _create_instrument(self):
"""
The main method to create an instrument configuration.
"""
iconfig_builder = self._create_instrument_config_builder()
org_obj = any_old(RT.Org)
log.debug("making the structure for an instrument agent")
i_obj = self._make_instrument_agent_structure(org_obj)
instrument_agent_instance_obj = self.RR2.read(i_obj.instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
self.verify_instrument_config(instrument_config, org_obj,
i_obj.instrument_device_id)
self._generate_instrument_config(instrument_agent_instance_obj,
i_obj.instrument_agent_instance_id)
return i_obj
#################################################################
# instrument-platform linking
#################################################################
def _assign_instrument_to_platform(self, i_obj, p_obj):
log.debug("assigning instrument %r to platform %r",
i_obj.instrument_agent_instance_id, p_obj.platform_id)
self.RR2.assign_instrument_device_to_platform_device(
i_obj.instrument_device_id,
p_obj.platform_device_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(p_obj.platform_device_id)
self.assertNotEqual(0, len(child_device_ids))
self._generate_platform_with_instrument_config(p_obj, i_obj)
#################################################################
# some platform topologies
#################################################################
def _create_single_platform(self):
"""
Creates and prepares a platform corresponding to the
platform ID 'LJ01D', which is a leaf in the simulated network.
"""
p_root = self._create_platform('LJ01D')
return p_root
def _create_small_hierarchy(self):
"""
Creates a small platform network consisting of 3 platforms as follows:
Node1D -> MJ01C -> LJ01D
where -> goes from parent to child.
"""
p_root = self._create_platform('Node1D')
p_child = self._create_platform('MJ01C', parent_platform_id='Node1D')
p_grandchild = self._create_platform('LJ01D', parent_platform_id='MJ01C')
self._assign_child_to_parent(p_child, p_root)
self._assign_child_to_parent(p_grandchild, p_child)
self._generate_config(p_root.platform_agent_instance_obj, p_root.platform_id, "_final")
return p_root
#################################################################
# start / stop platform
#################################################################
def _start_platform(self, agent_instance_id):
log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id)
pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
agent_instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient('paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
def _stop_platform(self, agent_instance_id):
self.IMS.stop_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
#################################################################
# start / stop instrument
#################################################################
def _start_instrument(self, agent_instance_id):
log.debug("about to call start_instrument_agent_instance with id=%s", agent_instance_id)
pid = self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=agent_instance_id)
log.debug("start_instrument_agent_instance returned pid=%s", pid)
#wait for start
agent_instance_obj = self.IMS.read_instrument_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The instrument agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._ia_client = ResourceAgentClient('paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got instrument agent client %s", str(self._ia_client))
def _stop_instrument(self, agent_instance_id):
self.IMS.stop_instrument_agent_instance(instrument_agent_instance_id=agent_instance_id)
#################################################################
# misc convenience methods
#################################################################
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs)
time_start = time.time()
#retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s",
cmd.command, elapsed_time, str(retval))
return retval
#################################################################
# commands that concrete tests can call
#################################################################
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
#self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _start_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.MONITORING)
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
# just wait for at least one -- see consume_data
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("Received samples: %s", len(self._samples_received))
def _wait_for_external_event(self):
log.info("waiting for reception of an external event...")
# just wait for at least one -- see consume_event
self._async_event_result.get(timeout=EVENT_TIMEOUT)
self.assertTrue(len(self._events_received) >= 1)
log.info("Received events: %s", len(self._events_received))
def _stop_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _pause(self):
cmd = AgentCommand(command=PlatformAgentEvent.PAUSE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.STOPPED)
def _resume(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESUME)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _clear(self):
cmd = AgentCommand(command=PlatformAgentEvent.CLEAR)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _check_sync(self):
cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC)
retval = self._execute_agent(cmd)
log.info("CHECK_SYNC result: %s", retval.result)
self.assertTrue(retval.result is not None)
self.assertEquals(retval.result[0:3], "OK:")
return retval.result
class TestOmsLaunch(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(
self._network_definition)
if log.isEnabledFor(logging.DEBUG):
log.debug("NetworkDefinition serialization:\n%s",
self._network_definition_ser)
self.platformModel_id = None
self.all_platforms = {}
self.agent_streamconfig_map = {}
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
self._set_up_DataProduct_obj()
self._set_up_PlatformModel_obj()
def _set_up_DataProduct_obj(self):
# Create data product object to be used for each of the platform log streams
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'platform_eng_parsed', id_only=True)
self.platform_eng_stream_def_id = self.pubsubcli.create_stream_definition(
name='platform_eng', parameter_dictionary_id=self.pdict_id)
self.dp_obj = IonObject(RT.DataProduct,
name='platform_eng data',
description='platform_eng test',
temporal_domain=tdom,
spatial_domain=sdom)
def _set_up_PlatformModel_obj(self):
# Create PlatformModel
platformModel_obj = IonObject(RT.PlatformModel,
name='RSNPlatformModel',
description="RSNPlatformModel")
try:
self.platformModel_id = self.imsclient.create_platform_model(
platformModel_obj)
except BadRequest as ex:
self.fail("failed to create new PLatformModel: %s" % ex)
log.debug('new PlatformModel id = %s', self.platformModel_id)
def _traverse(self, pnode, platform_id, parent_platform_objs=None):
"""
Recursive routine that repeatedly calls _prepare_platform to build
the object dictionary for each platform.
@param pnode PlatformNode
@param platform_id ID of the platform to be visited
@param parent_platform_objs dict of objects associated to parent
platform, if any.
@retval the dict returned by _prepare_platform at this level.
"""
log.info("Starting _traverse for %r", platform_id)
plat_objs = self._prepare_platform(pnode, platform_id,
parent_platform_objs)
self.all_platforms[platform_id] = plat_objs
# now, traverse the children:
for sub_pnode in pnode.subplatforms.itervalues():
subplatform_id = sub_pnode.platform_id
self._traverse(sub_pnode, subplatform_id, plat_objs)
return plat_objs
def _prepare_platform(self, pnode, platform_id, parent_platform_objs):
"""
This routine generalizes the manual construction originally done in
test_oms_launch.py. It is called by the recursive _traverse method so
all platforms starting from a given base platform are prepared.
Note: For simplicity in this test, sites are organized in the same
hierarchical way as the platforms themselves.
@param pnode PlatformNode
@param platform_id ID of the platform to be visited
@param parent_platform_objs dict of objects associated to parent
platform, if any.
@retval a dict of associated objects similar to those in
test_oms_launch
"""
site__obj = IonObject(RT.PlatformSite,
name='%s_PlatformSite' % platform_id,
description='%s_PlatformSite platform site' %
platform_id)
site_id = self.omsclient.create_platform_site(site__obj)
if parent_platform_objs:
# establish hasSite association with the parent
self.rrclient.create_association(
subject=parent_platform_objs['site_id'],
predicate=PRED.hasSite,
object=site_id)
# prepare platform attributes and ports:
monitor_attribute_objs, monitor_attribute_dicts = self._prepare_platform_attributes(
pnode, platform_id)
port_objs, port_dicts = self._prepare_platform_ports(
pnode, platform_id)
device__obj = IonObject(
RT.PlatformDevice,
name='%s_PlatformDevice' % platform_id,
description='%s_PlatformDevice platform device' % platform_id,
# ports=port_objs,
# platform_monitor_attributes = monitor_attribute_objs
)
device__dict = dict(
ports=port_dicts,
platform_monitor_attributes=monitor_attribute_dicts)
self.device_id = self.imsclient.create_platform_device(device__obj)
self.imsclient.assign_platform_model_to_platform_device(
self.platformModel_id, self.device_id)
self.rrclient.create_association(subject=site_id,
predicate=PRED.hasDevice,
object=self.device_id)
self.damsclient.register_instrument(instrument_id=self.device_id)
if parent_platform_objs:
# establish hasDevice association with the parent
self.rrclient.create_association(
subject=parent_platform_objs['device_id'],
predicate=PRED.hasDevice,
object=self.device_id)
agent__obj = IonObject(RT.PlatformAgent,
name='%s_PlatformAgent' % platform_id,
description='%s_PlatformAgent platform agent' %
platform_id)
agent_id = self.imsclient.create_platform_agent(agent__obj)
if parent_platform_objs:
# add this platform_id to parent's children:
parent_platform_objs['children'].append(platform_id)
self.imsclient.assign_platform_model_to_platform_agent(
self.platformModel_id, agent_id)
# agent_instance_obj = IonObject(RT.PlatformAgentInstance,
# name='%s_PlatformAgentInstance' % platform_id,
# description="%s_PlatformAgentInstance" % platform_id)
#
# agent_instance_id = self.imsclient.create_platform_agent_instance(
# agent_instance_obj, agent_id, device_id)
plat_objs = {
'platform_id': platform_id,
'site__obj': site__obj,
'site_id': site_id,
'device__obj': device__obj,
'device_id': self.device_id,
'agent__obj': agent__obj,
'agent_id': agent_id,
# 'agent_instance_obj': agent_instance_obj,
# 'agent_instance_id': agent_instance_id,
'children': []
}
log.info("plat_objs for platform_id %r = %s", platform_id,
str(plat_objs))
stream_config = self._create_stream_config(plat_objs)
self.agent_streamconfig_map[platform_id] = stream_config
# self.agent_streamconfig_map[platform_id] = None
# self._start_data_subscriber(agent_instance_id, stream_config)
return plat_objs
def _prepare_platform_attributes(self, pnode, platform_id):
"""
Returns the list of PlatformMonitorAttributes objects corresponding to
the attributes associated to the given platform.
"""
# TODO complete the clean-up of this method
ret_infos = dict((n, a.defn) for (n, a) in pnode.attrs.iteritems())
monitor_attribute_objs = []
monitor_attribute_dicts = []
for attrName, attrDfn in ret_infos.iteritems():
log.debug("platform_id=%r: preparing attribute=%r", platform_id,
attrName)
monitor_rate = attrDfn['monitorCycleSeconds']
units = attrDfn['units']
plat_attr_obj = IonObject(OT.PlatformMonitorAttributes,
id=attrName,
monitor_rate=monitor_rate,
units=units)
plat_attr_dict = dict(id=attrName,
monitor_rate=monitor_rate,
units=units)
monitor_attribute_objs.append(plat_attr_obj)
monitor_attribute_dicts.append(plat_attr_dict)
return monitor_attribute_objs, monitor_attribute_dicts
def _prepare_platform_ports(self, pnode, platform_id):
"""
Returns the list of PlatformPort objects corresponding to the ports
associated to the given platform.
"""
# TODO complete the clean-up of this method
port_objs = []
port_dicts = []
for port_id, network in pnode.ports.iteritems():
log.debug("platform_id=%r: preparing port=%r network=%s",
platform_id, port_id, network)
#
# Note: the name "IP" address has been changed to "network" address
# in the CI-OMS interface spec.
#
plat_port_obj = IonObject(OT.PlatformPort,
port_id=port_id,
ip_address=network)
plat_port_dict = dict(port_id=port_id, network=network)
port_objs.append(plat_port_obj)
port_dicts.append(plat_port_dict)
return port_objs, port_dicts
def _create_stream_config(self, plat_objs):
platform_id = plat_objs['platform_id']
device_id = plat_objs['device_id']
#create the log data product
self.dp_obj.name = '%s platform_eng data' % platform_id
self.data_product_id = self.dpclient.create_data_product(
data_product=self.dp_obj,
stream_definition_id=self.platform_eng_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=self.device_id,
data_product_id=self.data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(self.data_product_id,
PRED.hasStream, None, True)
stream_config = self._build_stream_config(stream_ids[0])
return stream_config
def _build_stream_config(self, stream_id=''):
platform_eng_dictionary = DatasetManagementService.get_parameter_dictionary_by_name(
'platform_eng_parsed')
#get the streamroute object from pubsub by passing the stream_id
stream_def_ids, _ = self.rrclient.find_objects(
stream_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
stream_route = self.pubsubcli.read_stream_route(stream_id=stream_id)
stream_config = {
'routing_key': stream_route.routing_key,
'stream_id': stream_id,
'stream_definition_ref': stream_def_ids[0],
'exchange_point': stream_route.exchange_point,
'parameter_dictionary': platform_eng_dictionary.dump()
}
return stream_config
def _set_platform_agent_instances(self):
"""
Once most of the objs/defs associated with all platforms are in
place, this method creates and associates the PlatformAgentInstance
elements.
"""
self.platform_configs = {}
for platform_id, plat_objs in self.all_platforms.iteritems():
PLATFORM_CONFIG = {
'platform_id': platform_id,
'agent_streamconfig_map': None, #self.agent_streamconfig_map,
'driver_config': DVR_CONFIG,
'network_definition': self._network_definition_ser
}
self.platform_configs[platform_id] = {
'platform_id': platform_id,
'agent_streamconfig_map': self.agent_streamconfig_map,
'driver_config': DVR_CONFIG,
'network_definition': self._network_definition_ser
}
agent_config = {
'platform_config': PLATFORM_CONFIG,
}
self.stream_id = self.agent_streamconfig_map[platform_id][
'stream_id']
# import pprint
# print '============== platform id within unit test: %s ===========' % platform_id
# pprint.pprint(agent_config)
#agent_config['platform_config']['agent_streamconfig_map'] = None
agent_instance_obj = IonObject(
RT.PlatformAgentInstance,
name='%s_PlatformAgentInstance' % platform_id,
description="%s_PlatformAgentInstance" % platform_id,
agent_config=agent_config)
agent_id = plat_objs['agent_id']
device_id = plat_objs['device_id']
agent_instance_id = self.imsclient.create_platform_agent_instance(
agent_instance_obj, agent_id, self.device_id)
plat_objs['agent_instance_obj'] = agent_instance_obj
plat_objs['agent_instance_id'] = agent_instance_id
stream_config = self.agent_streamconfig_map[platform_id]
self._start_data_subscriber(agent_instance_id, stream_config)
def _start_data_subscriber(self, stream_name, stream_config):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r', stream_name)
stream_id = self.stream_id #stream_config['stream_id']
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.pubsubcli.create_subscription(name=exchange_name,
stream_ids=[stream_id])
self.pubsubcli.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(
sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
def _start_event_subscriber(self,
event_type="DeviceEvent",
sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(
sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
@skip("IMS does't net implement topology")
def test_hierarchy(self):
self._create_launch_verify(BASE_PLATFORM_ID)
@skip("Needs alignment with recent IMS changes")
def test_single_platform(self):
self._create_launch_verify('LJ01D')
def _create_launch_verify(self, base_platform_id):
# and trigger the traversal of the branch rooted at that base platform
# to create corresponding ION objects and configuration dictionaries:
pnode = self._network_definition.pnodes[base_platform_id]
base_platform_objs = self._traverse(pnode, base_platform_id)
# now that most of the topology information is there, add the
# PlatformAgentInstance elements
self._set_platform_agent_instances()
base_platform_config = self.platform_configs[base_platform_id]
log.info("base_platform_id = %r", base_platform_id)
#-------------------------------------------------------------------------------------
# Create Data Process Definition and Data Process for the eng stream monitor process
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='DemoStreamAlertTransform',
description='For testing EventTriggeredTransform_B',
module='ion.processes.data.transforms.event_alert_transform',
class_name='DemoStreamAlertTransform')
self.platform_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#THERE SHOULD BE NO STREAMDEF REQUIRED HERE.
platform_streamdef_id = self.pubsubcli.create_stream_definition(
name='platform_eng_parsed', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
platform_streamdef_id, self.platform_dprocdef_id, binding='output')
config = {
'process': {
'timer_interval': 5,
'queue_name': 'a_queue',
'variable_name': 'input_voltage',
'time_field_name': 'preferred_timestamp',
'valid_values': [-100, 100],
'timer_origin': 'Interval Timer'
}
}
platform_data_process_id = self.dataprocessclient.create_data_process(
self.platform_dprocdef_id, [self.data_product_id], {}, config)
self.dataprocessclient.activate_data_process(platform_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process,
platform_data_process_id)
#-------------------------------
# Launch Base Platform AgentInstance, connect to the resource agent client
#-------------------------------
agent_instance_id = base_platform_objs['agent_instance_id']
log.debug(
"about to call imsclient.start_platform_agent_instance with id=%s",
agent_instance_id)
pid = self.imsclient.start_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
instance_obj = self.imsclient.read_platform_agent_instance(
agent_instance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (90),
"The platform agent instance did not spawn in 90 seconds")
agent_instance_obj = self.imsclient.read_instrument_agent_instance(
agent_instance_id)
log.debug(
'test_oms_create_and_launch: Platform agent instance obj: %s',
str(agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._pa_client = ResourceAgentClient(
'paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug(" test_oms_create_and_launch:: got pa client %s",
str(self._pa_client))
log.debug("base_platform_config =\n%s", base_platform_config)
# ping_agent can be issued before INITIALIZE
retval = self._pa_client.ping_agent(timeout=TIMEOUT)
log.debug('Base Platform ping_agent = %s', str(retval))
# issue INITIALIZE command to the base platform, which will launch the
# creation of the whole platform hierarchy rooted at base_platform_config['platform_id']
# cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE, kwargs=dict(plat_config=base_platform_config))
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform INITIALIZE = %s', str(retval))
# GO_ACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform GO_ACTIVE = %s', str(retval))
# RUN:
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform RUN = %s', str(retval))
# START_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform START_MONITORING = %s', str(retval))
# wait for data sample
# just wait for at least one -- see consume_data above
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("waiting a bit more for reception of more data samples...")
sleep(15)
log.info("Got data samples: %d", len(self._samples_received))
# wait for event
# just wait for at least one event -- see consume_event above
log.info("waiting for reception of an event...")
self._async_event_result.get(timeout=EVENT_TIMEOUT)
log.info("Received events: %s", len(self._events_received))
#get the extended platfrom which wil include platform aggreate status fields
extended_platform = self.imsclient.get_platform_device_extension(
self.device_id)
# log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) )
# log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) )
# log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) )
# STOP_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform STOP_MONITORING = %s', str(retval))
# GO_INACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform GO_INACTIVE = %s', str(retval))
# RESET: Resets the base platform agent, which includes termination of
# its sub-platforms processes:
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform RESET = %s', str(retval))
#-------------------------------
# Stop Base Platform AgentInstance
#-------------------------------
self.imsclient.stop_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
class Test2CAA(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
"""
###################################################################
# Internal parameters and container.
###################################################################
# Internal parameters.
self._terrestrial_platform_id = 'terrestrial_id'
self._remote_platform_id = 'remote_id'
self._resource_id = 'fake_id'
self._xs_name = 'remote1'
self._terrestrial_svc_name = 'terrestrial_endpoint'
self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
self._remote_svc_name = 'remote_endpoint'
self._remote_listen_name = self._remote_svc_name + self._xs_name
self._remote_port = 0
self._terrestrial_port = 0
self._te_client = None
self._re_client = None
self._remote_pid = None
self._terrestrial_pid = None
# Async test results.
self._no_requests = 10
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._telem_evts = []
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._results_recv = {}
self._requests_sent = {}
self._done_telem_evt = AsyncResult()
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
###################################################################
# Start endpoints, agent.
###################################################################
self._start_terrestrial()
self._start_remote()
self._start_agent()
###################################################################
# Assign client ports.
# This is primarily for test purposes as the IP config in
# deployment will be fixed in advance.
###################################################################
self.te_client.set_client_port(self._remote_port)
check_port = self.te_client.get_client_port()
log.debug('Terrestrial client port is: %i', check_port)
self.re_client.set_client_port(self._terrestrial_port)
check_port = self.re_client.get_client_port()
log.debug('Remote client port is: %i', check_port)
###################################################################
# Start the event publisher and subscribers.
# Used to send fake agent telemetry publications to the endpoints,
# and to receive endpoint publications.
###################################################################
self._event_publisher = EventPublisher()
# Start the event subscriber for remote namespace platform events.
# This event could be changed to RemoteNamespaceEvent.
self._event_subscriber = EventSubscriber(
event_type='PlatformEvent',
callback=self.consume_event,
origin=self._xs_name)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._event_subscriber.stop)
# Start the result subscriber for remote resource events.
self._resource_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=IA_RESOURCE_ID,
callback=self.consume_event)
self._resource_result_subscriber.start()
self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._resource_result_subscriber.stop)
# Start the result subscriber for remote service events.
self._service_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin='resource_registry' + 'remote1',
callback=self.consume_event)
self._service_result_subscriber.start()
self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._service_result_subscriber.stop)
# Start the result subscriber for fake resource results.
self._fake_result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=self._resource_id,
callback=self.consume_event)
self._fake_result_subscriber.start()
self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._fake_result_subscriber.stop)
###################################################################
# Start/stop helpers.
###################################################################
def _start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _start_terrestrial(self):
"""
"""
# Create terrestrial config.
terrestrial_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._remote_port,
'this_port' : self._terrestrial_port,
'platform_resource_id' : self._terrestrial_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._terrestrial_listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._terrestrial_pid = self._container_client.spawn_process(
name=self._terrestrial_listen_name,
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=terrestrial_endpoint_config)
log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))
# Create a terrestrial client.
self.te_client = TerrestrialEndpointClient(
process=FakeProcess(),
to_name=self._terrestrial_listen_name)
log.debug('Got te client %s.', str(self.te_client))
self._terrestrial_port = self.te_client.get_port()
log.debug('Terrestrial port is: %i', self._terrestrial_port)
def _start_remote(self):
"""
"""
# Create agent config.
remote_endpoint_config = {
'other_host' : 'localhost',
'other_port' : self._terrestrial_port,
'this_port' : self._remote_port,
'platform_resource_id' : self._remote_platform_id,
'xs_name' : self._xs_name,
'process' : {
'listen_name' : self._remote_listen_name
}
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
self._remote_pid = self._container_client.spawn_process(
name=self._remote_listen_name,
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=remote_endpoint_config)
log.debug('Remote endpoint pid=%s.', str(self._remote_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=self._remote_listen_name)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._remote_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._remote_port)
###################################################################
# Telemetry publications to start/top endpoint.
# (Normally be published by appropriate platform agents.)
###################################################################
def terrestrial_link_up(self):
"""
Publish telemetry available to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.AVAILABLE)
def terrestrial_link_down(self):
"""
Publish telemetry unavailable to the terrestrial endpoint.
"""
# Publish a link up event to be caught by the terrestrial endpoint.
log.debug('Publishing terrestrial telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._terrestrial_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def remote_link_up(self):
"""
Publish telemetry available to the remote endpoint.
"""
# Publish a link up event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry available event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.AVAILABLE)
def remote_link_down(self):
"""
Publish telemetry unavailable to the remote endpoint.
"""
# Publish a link down event to be caught by the remote endpoint.
log.debug('Publishing remote telemetry unavailable event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._remote_platform_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_event(self, evt, *args, **kwargs):
"""
Test callback for events.
"""
log.debug('Test got event: %s, args: %s, kwargs: %s',
str(evt), str(args), str(kwargs))
if evt.type_ == 'PublicPlatformTelemetryEvent':
self._telem_evts.append(evt)
if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts):
self._done_telem_evt.set()
elif evt.type_ == 'RemoteQueueModifiedEvent':
self._queue_mod_evts.append(evt)
if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts):
self._done_queue_mod_evt.set()
elif evt.type_ == 'RemoteCommandTransmittedEvent':
self._cmd_tx_evts.append(evt)
if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts):
self._done_cmd_tx_evt.set()
elif evt.type_ == 'RemoteCommandResult':
cmd = evt.command
self._results_recv[cmd.command_id] = cmd
if len(self._results_recv) == self._no_requests:
self._done_cmd_evt.set()
###################################################################
# Misc helpers.
###################################################################
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id='fake_id',
command=cmdstr,
args=['arg1', 23],
kwargs={'worktime':3})
return cmd
###################################################################
# Tests.
###################################################################
def test_queued_fake(self):
"""
test_queued_fake
Test fake resource commands queued prior to linkup.
"""
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
# The following error occurs when queue persistence is enabled.
# Need to verify correct solution to enable persistent queues.
"""
2012-11-06 14:27:13,517 INFO pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run
result = self._run(*self.args, **self.kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop
self._callback(cmd_result)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete
self._client.enqueue(result)
AttributeError: 'NoneType' object has no attribute 'enqueue'
<Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError
2012-11-06 14:27:13,586 INFO pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7
2012-11-06 14:27:13,665 INFO pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6
2012-11-06 14:27:13,739 INFO pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5
2012-11-06 14:27:13,807 INFO pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4
2012-11-06 14:27:13,874 INFO pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3
2012-11-06 14:27:13,941 INFO pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1
2012-11-06 14:27:13,945 INFO pyon.event.event EventSubscriber stopped. Event pattern=#
2012-11-06 14:27:14,124 INFO pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984
2012-11-06 14:27:14,399 INFO pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB
======================================================================
ERROR: test_process_online
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner
return f(*args, **kw)
File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online
cmd = self.te_client.enqueue_command(cmd)
File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command
return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request
return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request
retval, headers = e.send(msg, headers=headers, timeout=timeout)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send
result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send
return self._invite_and_send(to_role, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send
return self._send(to_role, to_role_name, msg, header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send
return self._end_point_unit._message_send(msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send
return ProcessRPCRequestEndpointUnit.send(self, msg, headers, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send
return self._send(_msg, _header, **kwargs)
File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send
raise ex
Conflict: 409 - Object not based on most current version
"""
def test_process_online(self):
"""
test_process_online
Test fake resource commands queued while link is up.
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Queue up a series of fake commands to be handled by the remote side.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mods, command transmissions and results.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_remote_late(self):
"""
test_remote_late
Test fake resource commands queued prior to linkup.
Delay remote side linkup substantially to test terrestrial
behavior when remote server not initially available.
"""
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Actually stop the remote process, since the server is
# available even if it hasn't linked up yet and the test is running
# in the same container. The test will remember the appropriate
# remote port numbers.
self._container_client.terminate_process(self._remote_pid)
self.terrestrial_link_up()
gevent.sleep(10)
# Restart remote side and publish remote link up.
self._start_remote()
self.remote_link_up()
# Block for transmission and result events.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_resource_commands(self):
"""
test_resource_commands
"""
# Set up to verify the two commands queued.
self._no_requests = 2
self._no_telem_evts = 2
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
# Use IA client to verify IA.
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Create and enqueue commands.
state_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={})
state_cmd = self.te_client.enqueue_command(state_cmd)
self._requests_sent[state_cmd.command_id] = state_cmd
ping_cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='ping_agent',
args=[],
kwargs={})
ping_cmd = self.te_client.enqueue_command(ping_cmd)
self._requests_sent[ping_cmd.command_id] = ping_cmd
# Block on queue mod events.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Block on command transmissions and results.
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
# Block on terrestrial public telemetry events.
self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
self.assertEqual(self._results_recv[state_cmd.command_id].result,
ResourceAgentState.UNINITIALIZED)
self.assertIn('ping from InstrumentAgent',
self._results_recv[ping_cmd.command_id].result)
def test_service_command_sequence(self):
"""
test_service_commands
"""
# Publish link up events.
self.terrestrial_link_up()
self.remote_link_up()
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='create',
args=[obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a tuple of strings.
#{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
#'1-00a308c45fff459c7cda1db9a7314de6'],
#'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
# Confirm the results are valid.
#Result is a user info object with the name set.
#{'lcstate': 'DEPLOYED_AVAILABLE',
#'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
#'description': '',
#'ts_updated': '1349213207638',
#'type_': 'UserInfo',
#'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
#'_id': '27832d93f4cd4535a75ac75c06e00a7e',
#'ts_created': '1349213207638',
#'variables': [{'name': '', 'value': ''}],
#'name': 'some_name'}
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
read_obj.name = 'some_other_name'
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='update',
args=[read_obj],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id].result
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evt = AsyncResult()
self._done_cmd_tx_evt = AsyncResult()
self._done_cmd_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._requests_sent = {}
self._results_recv = {}
# Create user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='delete',
args=[obj_id],
kwargs='')
cmd = self.te_client.enqueue_command(cmd)
# Block for the queue to be modified, the command to be transmitted,
# and the result to be received.
self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish link down events.
self.terrestrial_link_down()
self.remote_link_down()
class TestPlatformAgent(IonIntegrationTestCase, HelperTestMixin):
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
network_definition_ser = NetworkUtil.serialize_network_definition(
network_definition)
if log.isEnabledFor(logging.DEBUG):
log.debug("NetworkDefinition serialization:\n%s",
network_definition_ser)
cls.PLATFORM_CONFIG = {
'platform_id': cls.PLATFORM_ID,
'driver_config': DVR_CONFIG,
'network_definition': network_definition_ser
}
NetworkUtil._gen_open_diagram(
network_definition.pnodes[cls.PLATFORM_ID])
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# Start data subscribers, add stop to cleanup.
# Define stream_config.
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# start event subscriber:
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
self._agent_config = {
'agent': {
'resource_id': PA_RESOURCE_ID
},
'stream_config': self._stream_config,
# pass platform config here
'platform_config': self.PLATFORM_CONFIG
}
if log.isEnabledFor(logging.TRACE):
log.trace("launching with agent_config=%s" %
str(self._agent_config))
self._launcher = LauncherFactory.createLauncher()
self._pid = self._launcher.launch(self.PLATFORM_ID, self._agent_config)
log.debug("LAUNCHED PLATFORM_ID=%r", self.PLATFORM_ID)
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient(PA_RESOURCE_ID,
process=FakeProcess())
log.info('Got pa client %s.' % str(self._pa_client))
def tearDown(self):
try:
self._launcher.cancel_process(self._pid)
finally:
super(TestPlatformAgent, self).tearDown()
def _start_data_subscribers(self):
"""
"""
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
#
# TODO retrieve appropriate stream definitions; for the moment, using
# adhoc_get_stream_names
#
# A callback for processing subscribed-to data.
def consume_data(message, stream_route, stream_id):
log.info('Subscriber received data message: %s.' % str(message))
self._samples_received.append(message)
self._async_data_result.set()
for stream_name in adhoc_get_stream_names():
log.info('creating stream %r ...', stream_name)
# TODO use appropriate exchange_point
stream_id, stream_route = self._pubsub_client.create_stream(
name=stream_name, exchange_point='science_data')
log.info('create_stream(%r): stream_id=%r, stream_route=%s',
stream_name, stream_id, str(stream_route))
pdict = adhoc_get_parameter_dictionary(stream_name)
stream_config = dict(stream_route=stream_route.routing_key,
stream_id=stream_id,
parameter_dictionary=pdict.dump())
self._stream_config[stream_name] = stream_config
log.info('_stream_config[%r]= %r', stream_name, stream_config)
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self._pubsub_client.create_subscription(
name=exchange_name, stream_ids=[stream_id])
self._pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self._pubsub_client.deactivate_subscription(
sub.subscription_id)
except:
pass
self._pubsub_client.delete_subscription(sub.subscription_id)
sub.stop()
def _start_event_subscriber(self,
event_type="DeviceEvent",
sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(
sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
#def _execute_agent(self, cmd, timeout=TIMEOUT):
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command,
cmd.kwargs)
time_start = time.time()
#retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s",
cmd.command, elapsed_time, str(retval))
return retval
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
#self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _connect_instrument(self):
#
# TODO more realistic settings for the connection
#
port_id = self.PORT_ID
instrument_id = self.INSTRUMENT_ID
instrument_attributes = self.INSTRUMENT_ATTRIBUTES_AND_VALUES
kwargs = dict(port_id=port_id,
instrument_id=instrument_id,
attributes=instrument_attributes)
cmd = AgentCommand(command=PlatformAgentEvent.CONNECT_INSTRUMENT,
kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("CONNECT_INSTRUMENT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
returned_attrs = self._verify_valid_instrument_id(
instrument_id, result[port_id])
if isinstance(returned_attrs, dict):
for attrName in instrument_attributes:
self.assertTrue(attrName in returned_attrs)
def _get_connected_instruments(self):
port_id = self.PORT_ID
kwargs = dict(port_id=port_id, )
cmd = AgentCommand(
command=PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("GET_CONNECTED_INSTRUMENTS = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
instrument_id = self.INSTRUMENT_ID
self.assertTrue(instrument_id in result[port_id])
def _disconnect_instrument(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
instrument_id = self.INSTRUMENT_ID
kwargs = dict(port_id=port_id, instrument_id=instrument_id)
cmd = AgentCommand(command=PlatformAgentEvent.DISCONNECT_INSTRUMENT,
kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("DISCONNECT_INSTRUMENT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
self.assertTrue(instrument_id in result[port_id])
self._verify_instrument_disconnected(instrument_id,
result[port_id][instrument_id])
def _turn_on_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_ON_PORT,
kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_ON_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_ON)
def _turn_off_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_OFF_PORT,
kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_OFF_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_OFF)
def _get_resource(self):
attrNames = self.ATTR_NAMES
#
# OOIION-631: use get_ion_ts() as a basis for using system time, which is
# a string.
#
cur_time = get_ion_ts()
from_time = str(int(cur_time) - 50000) # a 50-sec time window
kwargs = dict(attr_names=attrNames, from_time=from_time)
cmd = AgentCommand(command=PlatformAgentEvent.GET_RESOURCE,
kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attr_name in attrNames:
self._verify_valid_attribute_id(attr_name, attr_values)
def _set_resource(self):
attrNames = self.ATTR_NAMES
writ_attrNames = self.WRITABLE_ATTR_NAMES
# do valid settings:
# TODO more realistic value depending on attribute's type
attrs = [(attrName, self.VALID_ATTR_VALUE) for attrName in attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE,
kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in attrNames:
if attrName in writ_attrNames:
self._verify_valid_attribute_id(attrName, attr_values)
else:
self._verify_not_writable_attribute_id(attrName, attr_values)
# try invalid settings:
# set invalid values to writable attributes:
attrs = [(attrName, self.INVALID_ATTR_VALUE)
for attrName in writ_attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE,
kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in writ_attrNames:
self._verify_attribute_value_out_of_range(attrName, attr_values)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _pause(self):
cmd = AgentCommand(command=PlatformAgentEvent.PAUSE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.STOPPED)
def _resume(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESUME)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _start_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.MONITORING)
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
# just wait for at least one -- see consume_data
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("Received samples: %s", len(self._samples_received))
def _stop_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _get_subplatform_ids(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_SUBPLATFORM_IDS)
retval = self._execute_agent(cmd)
self.assertIsInstance(retval.result, list)
self.assertTrue(x in retval.result for x in self.SUBPLATFORM_IDS)
return retval.result
def _wait_for_external_event(self):
log.info("waiting for reception of an external event...")
# just wait for at least one -- see consume_event
self._async_event_result.get(timeout=EVENT_TIMEOUT)
self.assertTrue(len(self._events_received) >= 1)
log.info("Received events: %s", len(self._events_received))
def _check_sync(self):
cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC)
retval = self._execute_agent(cmd)
log.info("CHECK_SYNC result: %s", retval.result)
self.assertTrue(retval.result is not None)
self.assertEquals(retval.result[0:3], "OK:")
return retval.result
def test_capabilities(self):
agt_cmds_all = [
PlatformAgentEvent.INITIALIZE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.PAUSE,
PlatformAgentEvent.RESUME,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.START_MONITORING,
PlatformAgentEvent.STOP_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
def sort_caps(caps):
agt_cmds = []
agt_pars = []
res_cmds = []
res_pars = []
if len(caps) > 0 and isinstance(caps[0], AgentCapability):
agt_cmds = [
x.name for x in caps
if x.cap_type == CapabilityType.AGT_CMD
]
agt_pars = [
x.name for x in caps
if x.cap_type == CapabilityType.AGT_PAR
]
res_cmds = [
x.name for x in caps
if x.cap_type == CapabilityType.RES_CMD
]
res_pars = [
x.name for x in caps
if x.cap_type == CapabilityType.RES_PAR
]
elif len(caps) > 0 and isinstance(caps[0], dict):
agt_cmds = [
x['name'] for x in caps
if x['cap_type'] == CapabilityType.AGT_CMD
]
agt_pars = [
x['name'] for x in caps
if x['cap_type'] == CapabilityType.AGT_PAR
]
res_cmds = [
x['name'] for x in caps
if x['cap_type'] == CapabilityType.RES_CMD
]
res_pars = [
x['name'] for x in caps
if x['cap_type'] == CapabilityType.RES_PAR
]
return agt_cmds, agt_pars, res_cmds, res_pars
agt_pars_all = ['example'
] # 'cause ResourceAgent defines aparam_example
res_pars_all = []
res_cmds_all = []
##################################################################
# UNINITIALIZED
##################################################################
self._assert_state(PlatformAgentState.UNINITIALIZED)
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_uninitialized = [
PlatformAgentEvent.INITIALIZE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_uninitialized)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(current_state=False)
# Validate all capabilities as read from state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# INACTIVE
##################################################################
self._initialize()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_inactive = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_inactive)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# IDLE
##################################################################
self._go_active()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_idle = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_idle)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states as read from IDLE.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# COMMAND
##################################################################
self._run()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_command = [
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.PAUSE,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.START_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_command)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
##################################################################
# STOPPED
##################################################################
self._pause()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state STOPPED
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_stopped = [
PlatformAgentEvent.RESUME,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_stopped)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
# back to COMMAND:
self._resume()
##################################################################
# MONITORING
##################################################################
self._start_resource_monitoring()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state MONITORING
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_monitoring = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.STOP_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_monitoring)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
# return to COMMAND state:
self._stop_resource_monitoring()
###################
# ALL CAPABILITIES
###################
# Get exposed capabilities in all states as read from state COMMAND.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_all)
self.assertItemsEqual(res_pars, res_pars_all)
self._go_inactive()
self._reset()
def test_some_state_transitions(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._initialize() # -> INACTIVE
self._reset() # -> UNINITIALIZED
self._initialize() # -> INACTIVE
self._go_active() # -> IDLE
self._reset() # -> UNINITIALIZED
self._initialize() # -> INACTIVE
self._go_active() # -> IDLE
self._run() # -> COMMAND
self._pause() # -> STOPPED
self._resume() # -> COMMAND
self._reset() # -> UNINITIALIZED
def test_get_set_resources(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._get_resource()
self._set_resource()
self._go_inactive()
self._reset()
def test_some_commands(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._ping_agent()
self._ping_resource()
self._get_metadata()
self._get_ports()
self._get_subplatform_ids()
self._go_inactive()
self._reset()
def test_resource_monitoring(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._start_resource_monitoring()
self._wait_for_a_data_sample()
self._stop_resource_monitoring()
self._go_inactive()
self._reset()
def test_external_event_dispatch(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._wait_for_external_event()
self._go_inactive()
self._reset()
def test_connect_disconnect_instrument(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._connect_instrument()
self._turn_on_port()
self._get_connected_instruments()
self._turn_off_port()
self._disconnect_instrument()
self._go_inactive()
self._reset()
def test_check_sync(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._check_sync()
self._connect_instrument()
self._check_sync()
self._disconnect_instrument()
self._check_sync()
self._go_inactive()
self._reset()
class TestAgentCommsAlerts(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self._event_count = 0
self._events_received = []
self._async_event_result = AsyncResult()
def consume_event(*args, **kwargs):
log.debug('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
self._event_subscriber = EventSubscriber(
event_type='DeviceStatusAlertEvent', callback=consume_event,
origin=IA_RESOURCE_ID)
self._event_subscriber.start()
def stop_subscriber():
self._event_subscriber.stop()
self._event_subscriber = None
self.addCleanup(stop_subscriber)
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'aparam_alerts_config' : [state_alert_def, command_alert_def]
}
self._ia_client = None
self._ia_pid = None
self.addCleanup(self._verify_agent_reset)
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=self._agent_config)
log.info('Started instrument agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
self._ia_pid = None
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_lost_connection_alert(self):
"""
test_lost_connection_alert
Verify that agents detect lost connection state and issue alert.
"""
self._event_count = 3
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Confirm the persisted parameters.
retval = self._ia_client.get_agent(['alerts'])['alerts']
"""
{'origin': '123xyz', 'status': 1, '_id': 'da03b90d2e064b25bf51ed90b729e82e',
'description': 'The alert is cleared.', 'time_stamps': [],
'type_': 'DeviceStatusAlertEvent', 'valid_values': [],
'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '',
'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'],
'stream_name': '', 'ts_created': '1366749987069', 'sub_type': 3,
'origin_type': 'InstrumentDevice', 'name': 'comms_warning'}
"""
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Blow the port agent out from under the agent.
self._support.stop_pagent()
# Wait for a while, the supervisor is restarting the port agent.
gevent.sleep(5)
self._support.start_pagent()
timeout = gevent.Timeout(120)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(2)
except Timeout as t:
self.fail('Could not reconnect to device.')
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
self._async_event_result.get(timeout=30)
"""
{'origin': '123xyz', 'status': 1, '_id': '9d7db919a0414741a2aa97f3dc310647', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029709', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
{'origin': '123xyz', 'status': 1, '_id': 'f746cec5e486445e856ef29af8d8d49a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029717', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
{'origin': '123xyz', 'status': 1, '_id': '126b1e20f54f4bd2b84a93584831ea8d', 'description': 'Detected comms failure.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_LOST_CONNECTION'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008062061', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
{'origin': '123xyz', 'status': 1, '_id': '90f5762112dd446b939c6675d34dd61d', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_COMMAND'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008098639', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
"""
def test_connect_failed_alert(self):
"""
test_connect_failed_alert
Verify that agents detect failed connections and issue alert.
"""
self._event_count = 4
# Remove the port agent.
self._support.stop_pagent()
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
with self.assertRaises(Exception):
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
# Now start up the port agent.
self._support.start_pagent()
gevent.sleep(5)
# This time it will work.
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
self._async_event_result.get(timeout=30)
"""
{'origin': '123xyz', 'status': 1, '_id': 'ac1ae3ec24e74f65bde24362d689346a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751167', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
{'origin': '123xyz', 'status': 1, '_id': '6af6a6156172481ebc44baad2708ec5c', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751175', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
{'origin': '123xyz', 'status': 1, '_id': 'b493698b46fd4fc1b4c66c53b70ed043', 'description': 'Detected comms failure while connecting.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007756771', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
{'origin': '123xyz', 'status': 1, '_id': 'c01e911ecb1a47c5a04cbbcbfb348a42', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007792905', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
"""
class TestOmsLaunch(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.platformModel_id = None
# rsn_oms: to retrieve network structure and information from RSN-OMS:
# Note that OmsClientFactory will create an "embedded" RSN OMS
# simulator object by default.
self.rsn_oms = OmsClientFactory.create_instance()
self.all_platforms = {}
self.topology = {}
self.agent_device_map = {}
self.agent_streamconfig_map = {}
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
self._set_up_DataProduct_obj()
self._set_up_PlatformModel_obj()
def _set_up_DataProduct_obj(self):
# Create data product object to be used for each of the platform log streams
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('platform_eng_parsed', id_only=True)
self.platform_eng_stream_def_id = self.pubsubcli.create_stream_definition(
name='platform_eng', parameter_dictionary_id=pdict_id)
self.dp_obj = IonObject(RT.DataProduct,
name='platform_eng data',
description='platform_eng test',
temporal_domain = tdom,
spatial_domain = sdom)
def _set_up_PlatformModel_obj(self):
# Create PlatformModel
platformModel_obj = IonObject(RT.PlatformModel,
name='RSNPlatformModel',
description="RSNPlatformModel")
try:
self.platformModel_id = self.imsclient.create_platform_model(platformModel_obj)
except BadRequest as ex:
self.fail("failed to create new PLatformModel: %s" %ex)
log.debug( 'new PlatformModel id = %s', self.platformModel_id)
def _traverse(self, platform_id, parent_platform_objs=None):
"""
Recursive routine that repeatedly calls _prepare_platform to build
the object dictionary for each platform.
@param platform_id ID of the platform to be visited
@param parent_platform_objs dict of objects associated to parent
platform, if any.
@retval the dict returned by _prepare_platform at this level.
"""
log.info("Starting _traverse for %r", platform_id)
plat_objs = self._prepare_platform(platform_id, parent_platform_objs)
self.all_platforms[platform_id] = plat_objs
# now, traverse the children:
retval = self.rsn_oms.getSubplatformIDs(platform_id)
subplatform_ids = retval[platform_id]
for subplatform_id in subplatform_ids:
self._traverse(subplatform_id, plat_objs)
# note, topology indexed by platform_id
self.topology[platform_id] = plat_objs['children']
return plat_objs
def _prepare_platform(self, platform_id, parent_platform_objs):
"""
This routine generalizes the manual construction currently done in
test_oms_launch.py. It is called by the recursive _traverse method so
all platforms starting from a given base platform are prepared.
Note: For simplicity in this test, sites are organized in the same
hierarchical way as the platforms themselves.
@param platform_id ID of the platform to be visited
@param parent_platform_objs dict of objects associated to parent
platform, if any.
@retval a dict of associated objects similar to those in
test_oms_launch
"""
site__obj = IonObject(RT.PlatformSite,
name='%s_PlatformSite' % platform_id,
description='%s_PlatformSite platform site' % platform_id)
site_id = self.omsclient.create_platform_site(site__obj)
if parent_platform_objs:
# establish hasSite association with the parent
self.rrclient.create_association(
subject=parent_platform_objs['site_id'],
predicate=PRED.hasSite,
object=site_id)
# prepare platform attributes and ports:
monitor_attributes = self._prepare_platform_attributes(platform_id)
ports = self._prepare_platform_ports(platform_id)
device__obj = IonObject(RT.PlatformDevice,
name='%s_PlatformDevice' % platform_id,
description='%s_PlatformDevice platform device' % platform_id,
ports=ports,
platform_monitor_attributes = monitor_attributes)
device_id = self.imsclient.create_platform_device(device__obj)
self.imsclient.assign_platform_model_to_platform_device(self.platformModel_id, device_id)
self.rrclient.create_association(subject=site_id, predicate=PRED.hasDevice, object=device_id)
self.damsclient.register_instrument(instrument_id=device_id)
if parent_platform_objs:
# establish hasDevice association with the parent
self.rrclient.create_association(
subject=parent_platform_objs['device_id'],
predicate=PRED.hasDevice,
object=device_id)
agent__obj = IonObject(RT.PlatformAgent,
name='%s_PlatformAgent' % platform_id,
description='%s_PlatformAgent platform agent' % platform_id)
agent_id = self.imsclient.create_platform_agent(agent__obj)
if parent_platform_objs:
# add this platform_id to parent's children:
parent_platform_objs['children'].append(platform_id)
self.imsclient.assign_platform_model_to_platform_agent(self.platformModel_id, agent_id)
# agent_instance_obj = IonObject(RT.PlatformAgentInstance,
# name='%s_PlatformAgentInstance' % platform_id,
# description="%s_PlatformAgentInstance" % platform_id)
#
# agent_instance_id = self.imsclient.create_platform_agent_instance(
# agent_instance_obj, agent_id, device_id)
plat_objs = {
'platform_id': platform_id,
'site__obj': site__obj,
'site_id': site_id,
'device__obj': device__obj,
'device_id': device_id,
'agent__obj': agent__obj,
'agent_id': agent_id,
# 'agent_instance_obj': agent_instance_obj,
# 'agent_instance_id': agent_instance_id,
'children': []
}
log.info("plat_objs for platform_id %r = %s", platform_id, str(plat_objs))
self.agent_device_map[platform_id] = device__obj
stream_config = self._create_stream_config(plat_objs)
self.agent_streamconfig_map[platform_id] = stream_config
# self._start_data_subscriber(agent_instance_id, stream_config)
return plat_objs
def _prepare_platform_attributes(self, platform_id):
"""
Returns the list of PlatformMonitorAttributes objects corresponding to
the attributes associated to the given platform.
"""
result = self.rsn_oms.getPlatformAttributes(platform_id)
self.assertTrue(platform_id in result)
ret_infos = result[platform_id]
monitor_attributes = []
for attrName, attrDfn in ret_infos.iteritems():
log.debug("platform_id=%r: preparing attribute=%r", platform_id, attrName)
monitor_rate = attrDfn['monitorCycleSeconds']
units = attrDfn['units']
plat_attr_obj = IonObject(OT.PlatformMonitorAttributes,
id=attrName,
monitor_rate=monitor_rate,
units=units)
monitor_attributes.append(plat_attr_obj)
return monitor_attributes
def _prepare_platform_ports(self, platform_id):
"""
Returns the list of PlatformPort objects corresponding to the ports
associated to the given platform.
"""
result = self.rsn_oms.getPlatformPorts(platform_id)
self.assertTrue(platform_id in result)
port_dict = result[platform_id]
ports = []
for port_id, port in port_dict.iteritems():
log.debug("platform_id=%r: preparing port=%r", platform_id, port_id)
ip_address = port['comms']['ip']
plat_port_obj = IonObject(OT.PlatformPort,
port_id=port_id,
ip_address=ip_address)
ports.append(plat_port_obj)
return ports
def _create_stream_config(self, plat_objs):
platform_id = plat_objs['platform_id']
device_id = plat_objs['device_id']
#create the log data product
self.dp_obj.name = '%s platform_eng data' % platform_id
data_product_id = self.dpclient.create_data_product(data_product=self.dp_obj, stream_definition_id=self.platform_eng_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id, PRED.hasStream, None, True)
stream_config = self._build_stream_config(stream_ids[0])
return stream_config
def _build_stream_config(self, stream_id=''):
platform_eng_dictionary = DatasetManagementService.get_parameter_dictionary_by_name('platform_eng_parsed')
#get the streamroute object from pubsub by passing the stream_id
stream_def_ids, _ = self.rrclient.find_objects(stream_id,
PRED.hasStreamDefinition,
RT.StreamDefinition,
True)
stream_route = self.pubsubcli.read_stream_route(stream_id=stream_id)
stream_config = {'routing_key' : stream_route.routing_key,
'stream_id' : stream_id,
'stream_definition_ref' : stream_def_ids[0],
'exchange_point' : stream_route.exchange_point,
'parameter_dictionary':platform_eng_dictionary.dump()}
return stream_config
def _set_platform_agent_instances(self):
"""
Once most of the objs/defs associated with all platforms are in
place, this method creates and associates the PlatformAgentInstance
elements.
"""
self.platform_configs = {}
for platform_id, plat_objs in self.all_platforms.iteritems():
PLATFORM_CONFIG = self.platform_configs[platform_id] = {
'platform_id': platform_id,
'platform_topology': self.topology,
# 'agent_device_map': self.agent_device_map,
'agent_streamconfig_map': self.agent_streamconfig_map,
'driver_config': DVR_CONFIG,
}
agent_config = {
'platform_config': PLATFORM_CONFIG,
}
agent_instance_obj = IonObject(RT.PlatformAgentInstance,
name='%s_PlatformAgentInstance' % platform_id,
description="%s_PlatformAgentInstance" % platform_id,
agent_config=agent_config)
agent_id = plat_objs['agent_id']
device_id = plat_objs['device_id']
agent_instance_id = self.imsclient.create_platform_agent_instance(
agent_instance_obj, agent_id, device_id)
plat_objs['agent_instance_obj'] = agent_instance_obj
plat_objs['agent_instance_id'] = agent_instance_id
stream_config = self.agent_streamconfig_map[platform_id]
self._start_data_subscriber(agent_instance_id, stream_config)
def _start_data_subscriber(self, stream_name, stream_config):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r', stream_name)
stream_id = stream_config['stream_id']
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.pubsubcli.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.pubsubcli.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
def _start_event_subscriber(self, event_type="PlatformAlarmEvent", sub_type="power"):
"""
Starts event subscriber for events of given event_type ("PlatformAlarmEvent"
by default) and given sub_type ("power" by default).
"""
# TODO note: ion-definitions still using 'PlatformAlarmEvent' but we
# should probably define 'PlatformExternalEvent' or something like that.
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
def test_oms_create_and_launch(self):
# pick a base platform:
base_platform_id = BASE_PLATFORM_ID
# and trigger the traversal of the branch rooted at that base platform
# to create corresponding ION objects and configuration dictionaries:
base_platform_objs = self._traverse(base_platform_id)
# now that most of the topology information is there, add the
# PlatformAgentInstance elements
self._set_platform_agent_instances()
base_platform_config = self.platform_configs[base_platform_id]
log.info("base_platform_id = %r", base_platform_id)
log.info("topology = %s", str(self.topology))
#-------------------------------
# Launch Base Platform AgentInstance, connect to the resource agent client
#-------------------------------
agent_instance_id = base_platform_objs['agent_instance_id']
pid = self.imsclient.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
instance_obj = self.imsclient.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
agent_instance_obj= self.imsclient.read_instrument_agent_instance(agent_instance_id)
log.debug('test_oms_create_and_launch: Platform agent instance obj: %s', str(agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._pa_client = ResourceAgentClient('paclient', name=agent_instance_obj.agent_process_id, process=FakeProcess())
log.debug(" test_oms_create_and_launch:: got pa client %s", str(self._pa_client))
log.debug("base_platform_config =\n%s", base_platform_config)
# ping_agent can be issued before INITIALIZE
retval = self._pa_client.ping_agent(timeout=TIMEOUT)
log.debug( 'Base Platform ping_agent = %s', str(retval) )
# issue INITIALIZE command to the base platform, which will launch the
# creation of the whole platform hierarchy rooted at base_platform_config['platform_id']
# cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE, kwargs=dict(plat_config=base_platform_config))
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform INITIALIZE = %s', str(retval) )
# GO_ACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform GO_ACTIVE = %s', str(retval) )
# RUN: this command includes the launch of the resource monitoring greenlets
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform RUN = %s', str(retval) )
# START_EVENT_DISPATCH
kwargs = dict(params="TODO set params")
cmd = AgentCommand(command=PlatformAgentEvent.START_EVENT_DISPATCH, kwargs=kwargs)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self.assertTrue(retval.result is not None)
# wait for data sample
# just wait for at least one -- see consume_data above
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("waiting a bit more for reception of more data samples...")
sleep(10)
log.info("Got data samples: %d", len(self._samples_received))
# wait for event
# just wait for at least one event -- see consume_event above
log.info("waiting for reception of an event...")
self._async_event_result.get(timeout=EVENT_TIMEOUT)
log.info("Received events: %s", len(self._events_received))
# STOP_EVENT_DISPATCH
cmd = AgentCommand(command=PlatformAgentEvent.STOP_EVENT_DISPATCH)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self.assertTrue(retval.result is not None)
# GO_INACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform GO_INACTIVE = %s', str(retval) )
# RESET: Resets the base platform agent, which includes termination of
# its sub-platforms processes:
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform RESET = %s', str(retval) )
#-------------------------------
# Stop Base Platform AgentInstance
#-------------------------------
self.imsclient.stop_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
class TestPlatformLaunch(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
def _create_platform_configuration(self):
"""
Verify that agent configurations are being built properly
"""
#
# This method is an adaptation of test_agent_instance_config in
# test_instrument_management_service_integration.py
#
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(any_old(RT.Org))
inst_startup_config = {'startup': 'config'}
required_config_keys = [
'org_name',
'device_type',
'agent',
'driver_config',
'stream_config',
'startup_config',
'alarm_defs',
'children']
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
if None is inst_device_id:
for key in ['alarm_defs', 'children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id], inst_device_id)
def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id)
parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
'platform_eng_parsed',
id_only=True)
self.parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed',
parameter_dictionary_id=parsed_rpdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance)
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
# dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
log.debug("Making the structure for a platform agent, which will be the child")
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure()
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug("Making the structure for a platform agent, which will be the parent")
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure()
platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure()
instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
if log.isEnabledFor(logging.TRACE):
import pprint
pp = pprint.PrettyPrinter()
pp.pprint(full_config)
log.trace("full_config = %s", pp.pformat(full_config))
return full_config
def get_streamConfigs(self):
#
# This method is an adaptation of get_streamConfigs in
# test_driver_egg.py
#
return [
StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
# TODO enable something like the following when also
# incorporating "raw" data:
#,
#StreamConfiguration(stream_name='raw',
# parameter_dictionary_name='ctd_raw_param_dict',
# records_per_granule=2,
# granule_publish_rate=5)
]
@skip("Still needs alignment with new configuration structure")
def test_hierarchy(self):
# TODO re-implement.
pass
def test_single_platform(self):
full_config = self._create_platform_configuration()
platform_id = 'LJ01D'
stream_configurations = self.get_streamConfigs()
agent__obj = IonObject(RT.PlatformAgent,
name='%s_PlatformAgent' % platform_id,
description='%s_PlatformAgent platform agent' % platform_id,
stream_configurations=stream_configurations)
agent_id = self.IMS.create_platform_agent(agent__obj)
device__obj = IonObject(RT.PlatformDevice,
name='%s_PlatformDevice' % platform_id,
description='%s_PlatformDevice platform device' % platform_id,
# ports=port_objs,
# platform_monitor_attributes = monitor_attribute_objs
)
self.device_id = self.IMS.create_platform_device(device__obj)
#######################################
# data product (adapted from test_instrument_management_service_integration)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='DataProduct test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
log.debug('data_product_id1 = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=self.device_id, data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
#######################################
#######################################
# dataset
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#######################################
full_config['platform_config'] = {
'platform_id': platform_id,
'driver_config': DVR_CONFIG,
'network_definition' : self._network_definition_ser
}
agent_instance_obj = IonObject(RT.PlatformAgentInstance,
name='%s_PlatformAgentInstance' % platform_id,
description="%s_PlatformAgentInstance" % platform_id,
agent_config=full_config)
agent_instance_id = self.IMS.create_platform_agent_instance(
platform_agent_instance = agent_instance_obj,
platform_agent_id = agent_id,
platform_device_id = self.device_id)
stream_id = stream_ids[0]
self._start_data_subscriber(agent_instance_id, stream_id)
log.debug("about to call imsclient.start_platform_agent_instance with id=%s", agent_instance_id)
pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
agent_instance_obj= self.IMS.read_instrument_agent_instance(agent_instance_id)
log.debug('Platform agent instance obj')
# Start a resource agent client to talk with the instrument agent.
self._pa_client = ResourceAgentClient('paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
# ping_agent can be issued before INITIALIZE
retval = self._pa_client.ping_agent(timeout=TIMEOUT)
log.debug('Base Platform ping_agent = %s', str(retval) )
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform INITIALIZE = %s', str(retval) )
# GO_ACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform GO_ACTIVE = %s', str(retval) )
# RUN:
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform RUN = %s', str(retval) )
# START_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform START_MONITORING = %s', str(retval) )
# wait for data sample
# just wait for at least one -- see consume_data above
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("waiting a bit more for reception of more data samples...")
sleep(15)
log.info("Got data samples: %d", len(self._samples_received))
# wait for event
# just wait for at least one event -- see consume_event above
log.info("waiting for reception of an event...")
self._async_event_result.get(timeout=EVENT_TIMEOUT)
log.info("Received events: %s", len(self._events_received))
#get the extended platfrom which wil include platform aggreate status fields
# extended_platform = self.IMS.get_platform_device_extension(self.device_id)
# log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) )
# log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) )
# log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) )
# STOP_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform STOP_MONITORING = %s', str(retval) )
# GO_INACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform GO_INACTIVE = %s', str(retval) )
# RESET: Resets the base platform agent, which includes termination of
# its sub-platforms processes:
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform RESET = %s', str(retval) )
#-------------------------------
# Stop Base Platform AgentInstance
#-------------------------------
self.IMS.stop_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
class TestPrest(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
print '#####################'
print 'IN SETUP'
self._ia_client = None
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
#log.info('driver uri: %s', DRV_URI)
#log.info('device address: %s', DEV_ADDR)
#log.info('device port: %s', DEV_PORT)
#log.info('work dir: %s', WORK_DIR)
# Create agent config.
agent_config = {
'driver_config': DVR_CONFIG,
'stream_config': self._stream_config,
'agent': {
'resource_id': IA_RESOURCE_ID
},
'test_mode': True,
'forget_past': True,
'enable_persistence': False
}
#if org_governance_name is not None:
# agent_config['org_governance_name'] = org_governance_name
# Start instrument agent.
log.info("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
log.info("Agent setup")
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
self.addCleanup(self._verify_agent_reset)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
log.info('test setup complete')
###############################################################################
# Port agent helpers.
###############################################################################
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
###############################################################################
# Event helpers.
###############################################################################
def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
"""
Start a subscriber to the instrument agent events.
@param type The type of event to catch.
@count Trigger the async event result when events received reaches this.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
# Event array and async event result.
self._event_count = count
self._events_received = []
self._async_event_result = AsyncResult()
self._event_subscriber = EventSubscriber(event_type=type,
callback=consume_event,
origin=IA_RESOURCE_ID)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=5)
def _stop_event_subscriber(self):
"""
Stop event subscribers on cleanup.
"""
self._event_subscriber.stop()
self._event_subscriber = None
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
def _start_data_subscribers(self, count, raw_count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._raw_samples_received = []
self._async_sample_result = AsyncResult()
self._async_raw_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received parsed data on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self._samples_received.append(message)
if len(self._samples_received) == count:
self._async_sample_result.set()
def recv_raw_data(message, stream_route, stream_id):
log.info('Received raw data on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self._raw_samples_received.append(message)
if len(self._raw_samples_received) == raw_count:
self._async_raw_sample_result.set()
from pyon.util.containers import create_unique_identifier
stream_name = 'parsed'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" % stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
stream_name = 'raw'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" % stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber, 'subscription_id'):
try:
pubsub_client.deactivate_subscription(
subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
###############################################################################
# Tests.
###############################################################################
@unittest.skip('Test should be run manually only.')
def test_initialize(self):
"""
test_initialize
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
print '#### in test'
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
@unittest.skip('Test should be run manually only.')
def test_xx(self):
"""
"""
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
with self.assertRaises(Conflict):
res_state = self._ia_client.get_resource_state()
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print '################################# sbe54 came up in state: ' + state
if state == ResourceAgentState.IDLE:
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
elif state == ResourceAgentState.STREAMING:
cmd = AgentCommand(command='DRIVER_EVENT_STOP_AUTOSAMPLE')
retval = self._ia_client.execute_resource(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
state = self._ia_client.get_agent_state()
print '################################# sbe54 now in state: ' + state
gevent.sleep(60 * 2.25)
state = self._ia_client.get_agent_state()
print '################################# sbe54 now in state: ' + state
"""
'DRIVER_EVENT_START_AUTOSAMPLE'
self.assertEqual(state, ResourceAgentState.IDLE)
res_state = self._ia_client.get_resource_state()
self.assertEqual(res_state, DriverProtocolState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
res_state = self._ia_client.get_resource_state()
self.assertEqual(res_state, DriverProtocolState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
with self.assertRaises(Conflict):
retval = self._ia_client.execute_resource(cmd)
"""
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestPlatformAgent(IonIntegrationTestCase, HelperTestMixin):
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG["oms_uri"])
network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
network_definition_ser = NetworkUtil.serialize_network_definition(network_definition)
if log.isEnabledFor(logging.DEBUG):
log.debug("NetworkDefinition serialization:\n%s", network_definition_ser)
cls.PLATFORM_CONFIG = {
"platform_id": cls.PLATFORM_ID,
"driver_config": DVR_CONFIG,
"network_definition": network_definition_ser,
}
NetworkUtil._gen_open_diagram(network_definition.pnodes[cls.PLATFORM_ID])
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
self._pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Start data subscribers, add stop to cleanup.
# Define stream_config.
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# start event subscriber:
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
self._agent_config = {
"agent": {"resource_id": PA_RESOURCE_ID},
"stream_config": self._stream_config,
# pass platform config here
"platform_config": self.PLATFORM_CONFIG,
}
if log.isEnabledFor(logging.TRACE):
log.trace("launching with agent_config=%s" % str(self._agent_config))
self._launcher = LauncherFactory.createLauncher()
self._pid = self._launcher.launch(self.PLATFORM_ID, self._agent_config)
log.debug("LAUNCHED PLATFORM_ID=%r", self.PLATFORM_ID)
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient(PA_RESOURCE_ID, process=FakeProcess())
log.info("Got pa client %s." % str(self._pa_client))
def tearDown(self):
try:
self._launcher.cancel_process(self._pid)
finally:
super(TestPlatformAgent, self).tearDown()
def _start_data_subscribers(self):
"""
"""
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
#
# TODO retrieve appropriate stream definitions; for the moment, using
# adhoc_get_stream_names
#
# A callback for processing subscribed-to data.
def consume_data(message, stream_route, stream_id):
log.info("Subscriber received data message: %s." % str(message))
self._samples_received.append(message)
self._async_data_result.set()
for stream_name in adhoc_get_stream_names():
log.info("creating stream %r ...", stream_name)
# TODO use appropriate exchange_point
stream_id, stream_route = self._pubsub_client.create_stream(name=stream_name, exchange_point="science_data")
log.info("create_stream(%r): stream_id=%r, stream_route=%s", stream_name, stream_id, str(stream_route))
pdict = adhoc_get_parameter_dictionary(stream_name)
stream_config = dict(
stream_route=stream_route.routing_key, stream_id=stream_id, parameter_dictionary=pdict.dump()
)
self._stream_config[stream_name] = stream_config
log.info("_stream_config[%r]= %r", stream_name, stream_config)
# Create subscriptions for each stream.
exchange_name = "%s_queue" % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self._pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
self._pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
for sub in self._data_subscribers:
if hasattr(sub, "subscription_id"):
try:
self._pubsub_client.deactivate_subscription(sub.subscription_id)
except:
pass
self._pubsub_client.delete_subscription(sub.subscription_id)
sub.stop()
def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info("Event subscriber received evt: %s.", str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type, sub_type=sub_type, callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r", event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, "subscription_id"):
try:
self.pubsubcli.deactivate_subscription(sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
# def _execute_agent(self, cmd, timeout=TIMEOUT):
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs)
time_start = time.time()
# retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s", cmd.command, elapsed_time, str(retval))
return retval
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
# self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _connect_instrument(self):
#
# TODO more realistic settings for the connection
#
port_id = self.PORT_ID
instrument_id = self.INSTRUMENT_ID
instrument_attributes = self.INSTRUMENT_ATTRIBUTES_AND_VALUES
kwargs = dict(port_id=port_id, instrument_id=instrument_id, attributes=instrument_attributes)
cmd = AgentCommand(command=PlatformAgentEvent.CONNECT_INSTRUMENT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("CONNECT_INSTRUMENT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
returned_attrs = self._verify_valid_instrument_id(instrument_id, result[port_id])
if isinstance(returned_attrs, dict):
for attrName in instrument_attributes:
self.assertTrue(attrName in returned_attrs)
def _get_connected_instruments(self):
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("GET_CONNECTED_INSTRUMENTS = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
instrument_id = self.INSTRUMENT_ID
self.assertTrue(instrument_id in result[port_id])
def _disconnect_instrument(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
instrument_id = self.INSTRUMENT_ID
kwargs = dict(port_id=port_id, instrument_id=instrument_id)
cmd = AgentCommand(command=PlatformAgentEvent.DISCONNECT_INSTRUMENT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("DISCONNECT_INSTRUMENT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], dict)
self.assertTrue(instrument_id in result[port_id])
self._verify_instrument_disconnected(instrument_id, result[port_id][instrument_id])
def _turn_on_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_ON_PORT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_ON_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_ON)
def _turn_off_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_OFF_PORT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_OFF_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_OFF)
def _get_resource(self):
attrNames = self.ATTR_NAMES
#
# OOIION-631: use get_ion_ts() as a basis for using system time, which is
# a string.
#
cur_time = get_ion_ts()
from_time = str(int(cur_time) - 50000) # a 50-sec time window
kwargs = dict(attr_names=attrNames, from_time=from_time)
cmd = AgentCommand(command=PlatformAgentEvent.GET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attr_name in attrNames:
self._verify_valid_attribute_id(attr_name, attr_values)
def _set_resource(self):
attrNames = self.ATTR_NAMES
writ_attrNames = self.WRITABLE_ATTR_NAMES
# do valid settings:
# TODO more realistic value depending on attribute's type
attrs = [(attrName, self.VALID_ATTR_VALUE) for attrName in attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in attrNames:
if attrName in writ_attrNames:
self._verify_valid_attribute_id(attrName, attr_values)
else:
self._verify_not_writable_attribute_id(attrName, attr_values)
# try invalid settings:
# set invalid values to writable attributes:
attrs = [(attrName, self.INVALID_ATTR_VALUE) for attrName in writ_attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in writ_attrNames:
self._verify_attribute_value_out_of_range(attrName, attr_values)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _pause(self):
cmd = AgentCommand(command=PlatformAgentEvent.PAUSE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.STOPPED)
def _resume(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESUME)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _start_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.MONITORING)
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
# just wait for at least one -- see consume_data
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("Received samples: %s", len(self._samples_received))
def _stop_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _get_subplatform_ids(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_SUBPLATFORM_IDS)
retval = self._execute_agent(cmd)
self.assertIsInstance(retval.result, list)
self.assertTrue(x in retval.result for x in self.SUBPLATFORM_IDS)
return retval.result
def _wait_for_external_event(self):
log.info("waiting for reception of an external event...")
# just wait for at least one -- see consume_event
self._async_event_result.get(timeout=EVENT_TIMEOUT)
self.assertTrue(len(self._events_received) >= 1)
log.info("Received events: %s", len(self._events_received))
def _check_sync(self):
cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC)
retval = self._execute_agent(cmd)
log.info("CHECK_SYNC result: %s", retval.result)
self.assertTrue(retval.result is not None)
self.assertEquals(retval.result[0:3], "OK:")
return retval.result
def test_capabilities(self):
agt_cmds_all = [
PlatformAgentEvent.INITIALIZE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.PAUSE,
PlatformAgentEvent.RESUME,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.START_MONITORING,
PlatformAgentEvent.STOP_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
def sort_caps(caps):
agt_cmds = []
agt_pars = []
res_cmds = []
res_pars = []
if len(caps) > 0 and isinstance(caps[0], AgentCapability):
agt_cmds = [x.name for x in caps if x.cap_type == CapabilityType.AGT_CMD]
agt_pars = [x.name for x in caps if x.cap_type == CapabilityType.AGT_PAR]
res_cmds = [x.name for x in caps if x.cap_type == CapabilityType.RES_CMD]
res_pars = [x.name for x in caps if x.cap_type == CapabilityType.RES_PAR]
elif len(caps) > 0 and isinstance(caps[0], dict):
agt_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_CMD]
agt_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_PAR]
res_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_CMD]
res_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_PAR]
return agt_cmds, agt_pars, res_cmds, res_pars
agt_pars_all = ["example"] # 'cause ResourceAgent defines aparam_example
res_pars_all = []
res_cmds_all = []
##################################################################
# UNINITIALIZED
##################################################################
self._assert_state(PlatformAgentState.UNINITIALIZED)
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_uninitialized = [PlatformAgentEvent.INITIALIZE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES]
self.assertItemsEqual(agt_cmds, agt_cmds_uninitialized)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(current_state=False)
# Validate all capabilities as read from state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# INACTIVE
##################################################################
self._initialize()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_inactive = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_inactive)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# IDLE
##################################################################
self._go_active()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_idle = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_idle)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states as read from IDLE.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
##################################################################
# COMMAND
##################################################################
self._run()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_command = [
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.PAUSE,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.START_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_command)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
##################################################################
# STOPPED
##################################################################
self._pause()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state STOPPED
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_stopped = [
PlatformAgentEvent.RESUME,
PlatformAgentEvent.CLEAR,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_stopped)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
# back to COMMAND:
self._resume()
##################################################################
# MONITORING
##################################################################
self._start_resource_monitoring()
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state MONITORING
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_monitoring = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.CONNECT_INSTRUMENT,
PlatformAgentEvent.DISCONNECT_INSTRUMENT,
PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.STOP_MONITORING,
PlatformAgentEvent.CHECK_SYNC,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_monitoring)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
# return to COMMAND state:
self._stop_resource_monitoring()
###################
# ALL CAPABILITIES
###################
# Get exposed capabilities in all states as read from state COMMAND.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_all)
self.assertItemsEqual(res_pars, res_pars_all)
self._go_inactive()
self._reset()
def test_some_state_transitions(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._initialize() # -> INACTIVE
self._reset() # -> UNINITIALIZED
self._initialize() # -> INACTIVE
self._go_active() # -> IDLE
self._reset() # -> UNINITIALIZED
self._initialize() # -> INACTIVE
self._go_active() # -> IDLE
self._run() # -> COMMAND
self._pause() # -> STOPPED
self._resume() # -> COMMAND
self._reset() # -> UNINITIALIZED
def test_get_set_resources(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._get_resource()
self._set_resource()
self._go_inactive()
self._reset()
def test_some_commands(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._ping_agent()
self._ping_resource()
self._get_metadata()
self._get_ports()
self._get_subplatform_ids()
self._go_inactive()
self._reset()
def test_resource_monitoring(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._start_resource_monitoring()
self._wait_for_a_data_sample()
self._stop_resource_monitoring()
self._go_inactive()
self._reset()
def test_external_event_dispatch(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._wait_for_external_event()
self._go_inactive()
self._reset()
def test_connect_disconnect_instrument(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._connect_instrument()
self._turn_on_port()
self._get_connected_instruments()
self._turn_off_port()
self._disconnect_instrument()
self._go_inactive()
self._reset()
def test_check_sync(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
self._ping_agent()
self._initialize()
self._go_active()
self._run()
self._check_sync()
self._connect_instrument()
self._check_sync()
self._disconnect_instrument()
self._check_sync()
self._go_inactive()
self._reset()
class TestAgentPersistence(IonIntegrationTestCase):
"""
"""
############################################################################
# Setup, teardown.
############################################################################
def setUp(self):
"""
Set up driver integration support.
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config.
"""
self._ia_client = None
log.info('Creating driver integration test support:')
log.info('driver uri: %s', DRV_URI)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.info('building stream configuration')
# Setup stream config.
self._build_stream_config()
# Create agent config.
self._agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True,
'forget_past' : False,
'enable_persistence' : True,
'aparam_pubrate_config' :
{
'raw' : 2,
'parsed' : 2
}
}
self._ia_client = None
self._ia_pid = '1234'
self.addCleanup(self._verify_agent_reset)
self.addCleanup(self.container.state_repository.put_state,
self._ia_pid, {})
###############################################################################
# Port agent helpers.
###############################################################################
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
###############################################################################
# Agent start stop helpers.
###############################################################################
def _start_agent(self, bootmode=None):
"""
"""
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = bootmode
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config,
process_id=self._ia_pid)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got instrument agent client %s.', str(self._ia_client))
def _stop_agent(self):
"""
"""
if self._ia_pid:
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
container_client.terminate_process(self._ia_pid)
if self._ia_client:
self._ia_client = None
def _verify_agent_reset(self):
"""
Check agent state and reset if necessary.
This called if a test fails and reset hasn't occurred.
"""
if self._ia_client is None:
return
state = self._ia_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
self._ia_client = None
###############################################################################
# Tests.
###############################################################################
def test_agent_config_persistence(self):
"""
test_agent_config_persistence
Test that agent parameter configuration is persisted between running
instances.
"""
# Start the agent.
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the default agent parameters.
#{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
retval = self._ia_client.get_agent(['streams'])['streams']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
#{'pubrate': {'raw': 0, 'parsed': 0}}
retval = self._ia_client.get_agent(['pubrate'])['pubrate']
self.assertIn('raw', retval.keys())
self.assertIn('parsed', retval.keys())
self.assertEqual(retval['raw'], 2)
self.assertEqual(retval['parsed'], 2)
#{'alerts': []}
retval = self._ia_client.get_agent(['alerts'])['alerts']
self.assertEqual(retval, [])
# Define a few new parameters and set them.
# Confirm they are set.
alert_def_1 = {
'name' : 'current_warning_interval',
'stream_name' : 'parsed',
'description' : 'Current is below normal range.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 10.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def_2 = {
'name' : 'temp_alarm_interval',
'stream_name' : 'parsed',
'description' : 'Temperatoure is critical.',
'alert_type' : StreamAlertType.ALARM,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'lower_bound' : None,
'lower_rel_op' : None,
'upper_bound' : 20.0,
'upper_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_alerts = [alert_def_1,alert_def_2, alert_def3]
pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : orig_alerts,
'pubrate' : pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Confirm the persisted parameters.
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
pubrate = retval['pubrate']
alerts = retval['alerts']
self.assertIn('raw', pubrate.keys())
self.assertIn('parsed', pubrate.keys())
self.assertEqual(pubrate['parsed'], 10)
self.assertEqual(pubrate['raw'], 20)
count = 0
for x in alerts:
x.pop('status')
x.pop('value')
for y in orig_alerts:
if x['name'] == y['name']:
count += 1
self.assertItemsEqual(x.keys(), y.keys())
self.assertEqual(count, 3)
def test_agent_state_persistence(self):
"""
test_agent_state_persistence
Verify that agents can be restored to their prior running state.
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
alert_def3 = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'time_delta' : 180,
'alert_class' : 'LateDataAlert'
}
orig_pubrate = {
'parsed' : 10,
'raw' : 20
}
params = {
'alerts' : [alert_def3],
'pubrate' : orig_pubrate
}
# Set the new agent params and confirm.
self._ia_client.set_agent(params)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
# Acquire sample returns a string, not a particle. The particle
# is created by the data handler though.
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
retval = self._ia_client.execute_resource(cmd)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
print '## in state: ' + state
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
params = [
'alerts',
'pubrate'
]
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
# Now stop and restart the agent.
self._stop_agent()
gevent.sleep(15)
self._start_agent('restart')
timeout = gevent.Timeout(240)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.STOPPED:
timeout.cancel()
break
else:
gevent.sleep(1)
except gevent.Timeout:
self.fail("Could not restore agent state to STOPPED.")
retval = self._ia_client.get_agent(params)
alerts = retval['alerts']
pubrate = retval['pubrate']
self.assertEqual(len(alerts), 1)
self.assertEqual(alert_def3['name'], alerts[0]['name'])
self.assertEqual(pubrate['raw'], 20)
self.assertEqual(pubrate['parsed'], 10)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def test_agent_rparam_persistence(self):
"""
test_agent_rparam_persistence
Verify ability to restore device configuration.
### Original values:
{'TA0': -0.0002572242, 'OUTPUTSV': False, 'NAVG': 0}
### Values after set:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
### Restore config:
{'PTCA1': 0.6603433, 'WBOTC': 1.2024e-05, 'PCALDATE': [12, 8, 2005],
'STORETIME': False, 'CPCOR': 9.57e-08, 'PTCA2': 0.00575649,
'OUTPUTSV': True, 'SAMPLENUM': 0, 'TCALDATE': [8, 11, 2005],
'OUTPUTSAL': False, 'TA2': -9.717158e-06, 'POFFSET': 0.0,
'INTERVAL': 19733, 'SYNCWAIT': 0, 'CJ': 3.339261e-05,
'CI': 0.0001334915, 'CH': 0.1417895, 'TA0': -0.0005144484,
'TA1': 0.0003138936, 'NAVG': 1, 'TA3': 2.138735e-07, '
RCALDATE': [8, 11, 2005], 'CG': -0.987093, 'CTCOR': 3.25e-06, '
PTCB0': 24.6145, 'PTCB1': -0.0009, 'PTCB2': 0.0,
'CCALDATE': [8, 11, 2005], 'PA0': 5.916199, 'PA1': 0.4851819,
'PA2': 4.596432e-07, 'SYNCMODE': False, 'PTCA0': 276.2492,
'TXREALTIME': True, 'RTCA2': -3.022745e-08, 'RTCA1': 1.686132e-06,
'RTCA0': 0.9999862}
### Of which we have:
{'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}
"""
self._start_agent()
# We start in uninitialized state.
# In this state there is no driver process.
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Ping the agent.
retval = self._ia_client.ping_agent()
log.info(retval)
# Initialize the agent.
# The agent is spawned with a driver config, but you can pass one in
# optinally with the initialize command. This validates the driver
# config, launches a driver process and connects to it via messaging.
# If successful, we switch to the inactive state.
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
# Ping the driver proc.
retval = self._ia_client.ping_resource()
log.info(retval)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
params = [
SBE37Parameter.OUTPUTSV,
SBE37Parameter.NAVG,
SBE37Parameter.TA0
]
retval = self._ia_client.get_resource(params)
orig_params = retval
new_params = {
SBE37Parameter.OUTPUTSV : not orig_params[SBE37Parameter.OUTPUTSV],
SBE37Parameter.NAVG : orig_params[SBE37Parameter.NAVG] + 1,
SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 2
}
#print '########### orig params'
#print str(orig_params)
self._ia_client.set_resource(new_params)
retval = self._ia_client.get_resource(params)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
#print '########### new params'
#print str(retval)
# Now stop and restart the agent.
self._stop_agent()
self._support.stop_pagent()
gevent.sleep(10)
self._start_pagent()
gevent.sleep(10)
self._start_agent('restart')
timeout = gevent.Timeout(600)
timeout.start()
try:
while True:
state = self._ia_client.get_agent_state()
if state == ResourceAgentState.COMMAND:
timeout.cancel()
break
else:
gevent.sleep(3)
except gevent.Timeout:
self.fail("Could not restore agent state to COMMAND.")
# Verify the parameters have been restored as needed.
retval = self._ia_client.get_resource(params)
#print '########### restored params'
#print str(retval)
self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
new_params[SBE37Parameter.OUTPUTSV])
self.assertEqual(retval[SBE37Parameter.NAVG],
new_params[SBE37Parameter.NAVG])
delta = max(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0])*.01
self.assertAlmostEqual(retval[SBE37Parameter.TA0],
new_params[SBE37Parameter.TA0], delta=delta)
# Reset the agent. This causes the driver messaging to be stopped,
# the driver process to end and switches us back to uninitialized.
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
@unittest.skip('Making CEI friendly.')
def test_cei_launch_mode(self):
pdc = ProcessDispatcherServiceClient(node=self.container.node)
p_def = ProcessDefinition(name='Agent007')
p_def.executable = {
'module' : 'ion.agents.instrument.instrument_agent',
'class' : 'InstrumentAgent'
}
p_def_id = pdc.create_process_definition(p_def)
pid = pdc.create_process(p_def_id)
def event_callback(event, *args, **kwargs):
print '######### proc %s in state %s' % (event.origin, ProcessStateEnum._str_map[event.state])
sub = EventSubscriber(event_type='ProcessLifecycleEvent',
callback=event_callback,
origin=pid,
origin_type='DispatchedProcess')
sub.start()
agent_config = deepcopy(self._agent_config)
agent_config['bootmode'] = 'restart'
pdc.schedule_process(p_def_id, process_id=pid,
configuration=agent_config)
gevent.sleep(5)
pdc.cancel_process(pid)
gevent.sleep(15)
sub.stop()
class TestPlatformAgent(IonIntegrationTestCase, HelperTestMixin):
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
self._pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.PLATFORM_CONFIG = {"platform_id": self.PLATFORM_ID, "driver_config": DVR_CONFIG}
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
self._agent_config = {
"agent": {"resource_id": PA_RESOURCE_ID},
"stream_config": self._stream_config,
# pass platform config here
"platform_config": self.PLATFORM_CONFIG,
}
log.debug("launching with agent_config=%s", str(self._agent_config))
self._launcher = LauncherFactory.createLauncher()
self._pid = self._launcher.launch(self.PLATFORM_ID, self._agent_config)
log.debug("LAUNCHED PLATFORM_ID=%r", self.PLATFORM_ID)
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient(PA_RESOURCE_ID, process=FakeProcess())
log.info("Got pa client %s." % str(self._pa_client))
def tearDown(self):
try:
self._launcher.cancel_process(self._pid)
finally:
super(TestPlatformAgent, self).tearDown()
def _start_data_subscribers(self):
"""
"""
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
#
# TODO retrieve appropriate stream definitions; for the moment, using
# adhoc_get_stream_names
#
# A callback for processing subscribed-to data.
def consume_data(message, stream_route, stream_id):
log.info("Subscriber received data message: %s." % str(message))
self._samples_received.append(message)
self._async_data_result.set()
for stream_name in adhoc_get_stream_names():
log.info("creating stream %r ...", stream_name)
# TODO use appropriate exchange_point
stream_id, stream_route = self._pubsub_client.create_stream(name=stream_name, exchange_point="science_data")
log.info("create_stream(%r): stream_id=%r, stream_route=%s", stream_name, stream_id, str(stream_route))
pdict = adhoc_get_parameter_dictionary(stream_name)
stream_config = dict(
stream_route=stream_route.routing_key, stream_id=stream_id, parameter_dictionary=pdict.dump()
)
self._stream_config[stream_name] = stream_config
log.info("_stream_config[%r]= %r", stream_name, stream_config)
# Create subscriptions for each stream.
exchange_name = "%s_queue" % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self._pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
self._pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
for sub in self._data_subscribers:
if hasattr(sub, "subscription_id"):
try:
self._pubsub_client.deactivate_subscription(sub.subscription_id)
except:
pass
self._pubsub_client.delete_subscription(sub.subscription_id)
sub.stop()
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
# def _execute_agent(self, cmd, timeout=TIMEOUT):
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs)
time_start = time.time()
# retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s", cmd.command, elapsed_time, str(retval))
return retval
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
# self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _set_up_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
port_attrName = self.PORT_ATTR_NAME
kwargs = dict(port_id=port_id, attributes={port_attrName: self.VALID_PORT_ATTR_VALUE})
cmd = AgentCommand(command=PlatformAgentEvent.SET_UP_PORT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("SET_UP_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertTrue(port_attrName in result[port_id])
def _turn_on_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_ON_PORT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_ON_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], bool)
def _turn_off_port(self):
# TODO real settings and corresp verification
port_id = self.PORT_ID
kwargs = dict(port_id=port_id)
cmd = AgentCommand(command=PlatformAgentEvent.TURN_OFF_PORT, kwargs=kwargs)
retval = self._execute_agent(cmd)
result = retval.result
log.info("TURN_OFF_PORT = %s", result)
self.assertIsInstance(result, dict)
self.assertTrue(port_id in result)
self.assertIsInstance(result[port_id], bool)
def _get_resource(self):
attrNames = self.ATTR_NAMES
cur_time = get_ion_ts()
#
# OOIION-631 Note: I'm asked to only use get_ion_ts() as a basis for
# using system time. However, other associated supporting (and more
# "numeric") routines would be convenient. In particular, note the
# following operation to subtract a number of seconds for my request:
#
from_time = str(int(cur_time) - 50000) # a 50-sec time window
kwargs = dict(attr_names=attrNames, from_time=from_time)
cmd = AgentCommand(command=PlatformAgentEvent.GET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attr_name in attrNames:
self._verify_valid_attribute_id(attr_name, attr_values)
def _set_resource(self):
attrNames = self.ATTR_NAMES
writ_attrNames = self.WRITABLE_ATTR_NAMES
# do valid settings:
# TODO more realistic value depending on attribute's type
attrs = [(attrName, self.VALID_ATTR_VALUE) for attrName in attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in attrNames:
if attrName in writ_attrNames:
self._verify_valid_attribute_id(attrName, attr_values)
else:
self._verify_not_writable_attribute_id(attrName, attr_values)
# try invalid settings:
# set invalid values to writable attributes:
attrs = [(attrName, self.INVALID_ATTR_VALUE) for attrName in writ_attrNames]
log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
kwargs = dict(attrs=attrs)
cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs)
retval = self._execute_agent(cmd)
attr_values = retval.result
self.assertIsInstance(attr_values, dict)
for attrName in writ_attrNames:
self._verify_attribute_value_out_of_range(attrName, attr_values)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
# kwargs = dict(plat_config=self.PLATFORM_CONFIG)
# cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE, kwargs=kwargs)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _get_subplatform_ids(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_SUBPLATFORM_IDS)
retval = self._execute_agent(cmd)
self.assertIsInstance(retval.result, list)
self.assertTrue(x in retval.result for x in self.SUBPLATFORM_IDS)
return retval.result
def _start_event_dispatch(self):
kwargs = dict(params="TODO set params")
cmd = AgentCommand(command=PlatformAgentEvent.START_EVENT_DISPATCH, kwargs=kwargs)
retval = self._execute_agent(cmd)
self.assertTrue(retval.result is not None)
return retval.result
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=15)
# just wait for at least one -- see consume_data above
self.assertTrue(len(self._samples_received) >= 1)
def _stop_event_dispatch(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_EVENT_DISPATCH)
retval = self._execute_agent(cmd)
self.assertTrue(retval.result is not None)
return retval.result
def test_capabilities(self):
# log.info("test_capabilities starting. Default timeout=%s", TIMEOUT)
log.info("test_capabilities starting. Default timeout=%i", CFG.endpoint.receive.timeout)
agt_cmds_all = [
PlatformAgentEvent.INITIALIZE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.SET_UP_PORT,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.START_EVENT_DISPATCH,
PlatformAgentEvent.STOP_EVENT_DISPATCH,
]
def sort_caps(caps):
agt_cmds = []
agt_pars = []
res_cmds = []
res_pars = []
if len(caps) > 0 and isinstance(caps[0], AgentCapability):
agt_cmds = [x.name for x in caps if x.cap_type == CapabilityType.AGT_CMD]
agt_pars = [x.name for x in caps if x.cap_type == CapabilityType.AGT_PAR]
res_cmds = [x.name for x in caps if x.cap_type == CapabilityType.RES_CMD]
res_pars = [x.name for x in caps if x.cap_type == CapabilityType.RES_PAR]
elif len(caps) > 0 and isinstance(caps[0], dict):
agt_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_CMD]
agt_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_PAR]
res_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_CMD]
res_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_PAR]
return agt_cmds, agt_pars, res_cmds, res_pars
agt_pars_all = ["example"] # 'cause ResourceAgent defines aparam_example
res_pars_all = []
res_cmds_all = []
##################################################################
# UNINITIALIZED
##################################################################
self._assert_state(PlatformAgentState.UNINITIALIZED)
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_uninitialized = [PlatformAgentEvent.INITIALIZE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES]
self.assertItemsEqual(agt_cmds, agt_cmds_uninitialized)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(current_state=False)
# Validate all capabilities as read from state UNINITIALIZED.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
self._initialize()
##################################################################
# INACTIVE
##################################################################
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_inactive = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GO_ACTIVE,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_inactive)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state INACTIVE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
self._go_active()
##################################################################
# IDLE
##################################################################
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities for state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_idle = [
PlatformAgentEvent.RESET,
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RUN,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
]
self.assertItemsEqual(agt_cmds, agt_cmds_idle)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
# Get exposed capabilities in all states as read from IDLE.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state IDLE.
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, [])
self.assertItemsEqual(res_pars, [])
self._run()
##################################################################
# COMMAND
##################################################################
# Get exposed capabilities in current state.
retval = self._pa_client.get_capabilities()
# Validate capabilities of state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
agt_cmds_command = [
PlatformAgentEvent.GO_INACTIVE,
PlatformAgentEvent.RESET,
PlatformAgentEvent.GET_METADATA,
PlatformAgentEvent.GET_PORTS,
PlatformAgentEvent.SET_UP_PORT,
PlatformAgentEvent.TURN_ON_PORT,
PlatformAgentEvent.TURN_OFF_PORT,
PlatformAgentEvent.GET_SUBPLATFORM_IDS,
PlatformAgentEvent.GET_RESOURCE_CAPABILITIES,
PlatformAgentEvent.PING_RESOURCE,
PlatformAgentEvent.GET_RESOURCE,
PlatformAgentEvent.SET_RESOURCE,
PlatformAgentEvent.START_EVENT_DISPATCH,
PlatformAgentEvent.STOP_EVENT_DISPATCH,
]
res_cmds_command = []
self.assertItemsEqual(agt_cmds, agt_cmds_command)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_command)
self.assertItemsEqual(res_pars, res_pars_all)
# Get exposed capabilities in all states as read from state COMMAND.
retval = self._pa_client.get_capabilities(False)
# Validate all capabilities as read from state COMMAND
agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval)
self.assertItemsEqual(agt_cmds, agt_cmds_all)
self.assertItemsEqual(agt_pars, agt_pars_all)
self.assertItemsEqual(res_cmds, res_cmds_all)
self.assertItemsEqual(res_pars, res_pars_all)
self._go_inactive()
self._reset()
def test_go_active_and_run(self):
# log.info("test_go_active_and_run starting. Default timeout=%s", TIMEOUT)
log.info("test_capabilities starting. Default timeout=%i", CFG.endpoint.receive.timeout)
self._ping_agent()
# self._ping_resource() skipping this here while the timeout issue
# on r2_light and other builds is investigated.
self._initialize()
self._go_active()
self._run()
self._ping_agent()
self._ping_resource()
self._get_metadata()
self._get_ports()
self._get_subplatform_ids()
self._get_resource()
self._set_resource()
self._set_up_port()
self._turn_on_port()
self._start_event_dispatch()
self._wait_for_a_data_sample()
self._stop_event_dispatch()
self._turn_off_port()
self._go_inactive()
self._reset()
class TestRemoteEndpoint(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
Start fake terrestrial components and add cleanup.
Start terrestrial server and retrieve port.
Set internal variables.
Start container.
Start deployment.
Start container agent.
Spawn remote endpoint process.
Create remote endpoint client and retrieve remote server port.
Create event publisher.
"""
self._terrestrial_server = R3PCServer(self.consume_req, self.terrestrial_server_close)
self._terrestrial_client = R3PCClient(self.consume_ack, self.terrestrial_client_close)
self.addCleanup(self._terrestrial_server.stop)
self.addCleanup(self._terrestrial_client.stop)
self._other_port = self._terrestrial_server.start('*', 0)
log.debug('Terrestrial server binding to *:%i', self._other_port)
self._other_host = 'localhost'
self._platform_resource_id = 'abc123'
self._resource_id = 'fake_id'
self._no_requests = 10
self._requests_sent = {}
self._results_recv = {}
self._no_telem_events = 0
self._done_evt = AsyncResult()
self._done_telem_evts = AsyncResult()
self._cmd_tx_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Create agent config.
endpoint_config = {
'other_host' : self._other_host,
'other_port' : self._other_port,
'this_port' : 0,
'platform_resource_id' : self._platform_resource_id
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
re_pid = container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(re_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=re_pid)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._this_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._this_port)
# Start the event publisher.
self._event_publisher = EventPublisher()
######################################################################
# Helpers.
######################################################################
def on_link_up(self):
"""
Called by a test to simulate turning the link on.
"""
log.debug('Terrestrial client connecting to localhost:%i.',
self._this_port)
self._terrestrial_client.start('localhost', self._this_port)
# Publish a link up event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status = TelemetryStatusType.AVAILABLE)
def on_link_down(self):
"""
Called by a test to simulate turning the link off.
"""
self._terrestrial_client.stop()
# Publish a link down event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_req(self, res):
"""
Consume a terrestrial request setting async event when necessary.
"""
command_id = res['command_id']
self._results_recv[command_id] = res
if len(self._results_recv) == self._no_requests:
self._done_evt.set()
def consume_ack(self, cmd):
"""
Consume terrestrial ack setting async event when necessary.
"""
self._requests_sent[cmd.command_id] = cmd
if len(self._requests_sent) == self._no_requests:
self._cmd_tx_evt.set()
def terrestrial_server_close(self):
"""
Callback when terrestrial server closes.
"""
pass
def terrestrial_client_close(self):
"""
Callback when terrestrial client closes.
"""
pass
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id=self._resource_id,
command=cmdstr,
args=['arg1', 23],
kwargs={'worktime':3},
command_id = str(uuid.uuid4()))
return cmd
def start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
######################################################################
# Tests.
######################################################################
def test_process_queued(self):
"""
test_process_queued
Test that queued commands are forwarded to and handled by
remote endpoint when link comes up.
"""
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_process_online(self):
"""
test_process_online
Test commands are forwarded and handled while link is up.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_terrestrial_late(self):
"""
test_terrestrial_late
Test queued commands are forwarded and handled by remote endpoint
when terrestrial side is late to come up.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Manually stop the terrestrial endpoint.
# This will cause it to be unavailable when commands are queued
# to simulate stability during asynchronous wake ups.
self._terrestrial_server.stop()
self._terrestrial_client.stop()
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Remote side awaits the terrestrial waking up.
gevent.sleep(3)
# Terrestrail endpoint eventually wakes up and starts transmitting.
self._terrestrial_client.start('localhost', self._this_port)
self._terrestrial_server.start('*', self._other_port)
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_service_commands(self):
"""
test_service_commands
Test that real service commands are handled by the remote endpoint.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='create',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = self._results_recv[cmd.command_id]['result']
# Confirm the results are valid.
"""
Result is a tuple of strings.
{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
'1-00a308c45fff459c7cda1db9a7314de6'],
'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
"""
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Read user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id]['result']
# Confirm the results are valid.
"""
Result is a user info object with the name set.
{'lcstate': 'DEPLOYED_AVAILABLE',
'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
'description': '',
'ts_updated': '1349213207638',
'type_': 'UserInfo',
'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
'_id': '27832d93f4cd4535a75ac75c06e00a7e',
'ts_created': '1349213207638',
'variables': [{'name': '', 'value': ''}],
'name': 'some_name'}
"""
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Update user object.
read_obj.name = 'some_other_name'
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='update',
args=[read_obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Read user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Delete user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='delete',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_resource_commands(self):
"""
test_resource_commands
Test that real resource commands are handled by the remote endpoint.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns agent state.
state = self._results_recv[cmd.command_id]['result']
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Ping agent via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='ping_agent',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns agent state.
ping = self._results_recv[cmd.command_id]['result']
self.assertIn('ping from InstrumentAgent', ping)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_bad_service_name_resource_id(self):
"""
test_bad_service_name_resource_id
Test for proper exception behavior when a bad service name or
resource id is used in a command forwarded to the remote endpoint.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='bogus_service',
command='create',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, NotFound)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id='bogus_resource_id',
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, NotFound)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_bad_commands(self):
"""
test_bad_commands
Test for correct exception behavior if a bad command name is forwarded
to a remote service or resource.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='what_the_flunk',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns BadRequest.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, BadRequest)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='what_the_flunk',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, BadRequest)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_resource_command_sequence(self):
"""
test_resource_command_sequence
Test for successful completion of a properly ordered sequence of
resource commands queued for forwarding to the remote endpoint.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# We execute a sequence of twelve consecutive events.
self._no_requests = 12
# Get agent state.
cmd1 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd1)
# Initialize agent.
cmd2 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.INITIALIZE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd2)
# Get agent state.
cmd3 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd3)
# Go active.
cmd4 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd4)
# Get agent state.
cmd5 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd5)
# Run.
cmd6 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.RUN)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd6)
# Get agent state.
cmd7 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd7)
# Acquire sample.
cmd8 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd8)
# Acquire sample
cmd9 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd9)
# Acquire sample.
cmd10 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd10)
# Reset.
cmd11 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.RESET)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd11)
# Get agent state.
cmd12 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd12)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Check results of command sequence.
"""
0ccf1e10-eeca-400d-aefe-f9d6888ec963 {'result': 'RESOURCE_AGENT_STATE_INACTIVE', 'command_id': '0ccf1e10-eeca-400d-aefe-f9d6888ec963'}
92531bdf-c2c8-4aa8-817d-5107c7311b37 {'result': <interface.objects.AgentCommandResult object at 0x10d7f11d0>, 'command_id': '92531bdf-c2c8-4aa8-817d-5107c7311b37'}
509934a1-5038-40d8-8014-591e2d8042b6 {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '509934a1-5038-40d8-8014-591e2d8042b6'}
88bacbb7-5366-4d27-9ecf-fff2bec34b2c {'result': <interface.objects.AgentCommandResult object at 0x10d389190>, 'command_id': '88bacbb7-5366-4d27-9ecf-fff2bec34b2c'}
f8b4d3fa-a249-439b-8bd4-ac212b6100aa {'result': <interface.objects.AgentCommandResult object at 0x10d3893d0>, 'command_id': 'f8b4d3fa-a249-439b-8bd4-ac212b6100aa'}
8ae98e39-fdb3-4218-ad8f-584620397d9f {'result': <interface.objects.AgentCommandResult object at 0x10d739990>, 'command_id': '8ae98e39-fdb3-4218-ad8f-584620397d9f'}
746364a1-c4c7-400f-96d4-ee36df5dc1a4 {'result': BadRequest('Execute argument "command" not set.',), 'command_id': '746364a1-c4c7-400f-96d4-ee36df5dc1a4'}
d516d3d9-e4f9-4ea5-80e0-34639a6377b5 {'result': <interface.objects.AgentCommandResult object at 0x10d3b2350>, 'command_id': 'd516d3d9-e4f9-4ea5-80e0-34639a6377b5'}
c7da03f5-59bc-420a-9e10-0a7794266599 {'result': 'RESOURCE_AGENT_STATE_IDLE', 'command_id': 'c7da03f5-59bc-420a-9e10-0a7794266599'}
678d870a-bf18-424a-afb0-f80ecf3277e2 {'result': <interface.objects.AgentCommandResult object at 0x10d739590>, 'command_id': '678d870a-bf18-424a-afb0-f80ecf3277e2'}
750c6a30-56eb-4535-99c2-a81fefab1b1f {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '750c6a30-56eb-4535-99c2-a81fefab1b1f'}
c17bd658-3775-4aa3-8844-02df70a0e3c0 {'result': 'RESOURCE_AGENT_STATE_UNINITIALIZED', 'command_id': 'c17bd658-3775-4aa3-8844-02df70a0e3c0'}
"""
# First result is a state string.
result1 = self._results_recv[cmd1.command_id]['result']
self.assertEqual(result1, ResourceAgentState.UNINITIALIZED)
# Second result is an empty AgentCommandResult.
result2 = self._results_recv[cmd2.command_id]['result']
# Third result is a state string.
result3 = self._results_recv[cmd3.command_id]['result']
self.assertEqual(result3, ResourceAgentState.INACTIVE)
# Fourth result is an empty AgentCommandResult.
result4 = self._results_recv[cmd4.command_id]['result']
# Fifth result is a state string.
result5 = self._results_recv[cmd5.command_id]['result']
self.assertEqual(result5, ResourceAgentState.IDLE)
# Sixth result is an empty AgentCommandResult.
result6 = self._results_recv[cmd6.command_id]['result']
# Seventh result is a state string.
result7 = self._results_recv[cmd7.command_id]['result']
self.assertEqual(result7, ResourceAgentState.COMMAND)
"""
{'raw': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
'stream_name': 'raw', 'pkt_format_id': 'JSON_Data',
'pkt_version': 1, '
values': [{'binary': True, 'value_id': 'raw',
'value': 'NzkuNDM3MywxNy4yMDU2NCwgNzYxLjg4NSwgICA2LjIxOTgsIDE1MDYuMzk3LCAwMSBGZWIgMjAwMSwgMDE6MDE6MDA='}],
'driver_timestamp': 3558286748.8039923},
'parsed': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data', 'pkt_version': 1,
'values': [{'value_id': 'temp', 'value': 79.4373},
{'value_id': 'conductivity', 'value': 17.20564},
{'value_id': 'pressure', 'value': 761.885}],
'driver_timestamp': 3558286748.8039923}}
"""
# Eigth result is an AgentCommandResult containing a sample.
result8 = self._results_recv[cmd8.command_id]['result']
self.assertTrue('parsed',result8.result )
# Ninth result is an AgentCommandResult containing a sample.
result9 = self._results_recv[cmd9.command_id]['result']
self.assertTrue('parsed',result9.result )
# Tenth result is an AgentCommandResult containing a sample.
result10 = self._results_recv[cmd10.command_id]['result']
self.assertTrue('parsed',result10.result )
# Eleventh result is an empty AgentCommandResult.
result11 = self._results_recv[cmd11.command_id]['result']
# Twelth result is a state string.
result12 = self._results_recv[cmd12.command_id]['result']
self.assertEqual(result1, ResourceAgentState.UNINITIALIZED)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
class TestOmsLaunch(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
if log.isEnabledFor(logging.DEBUG):
log.debug("NetworkDefinition serialization:\n%s", self._network_definition_ser)
self.platformModel_id = None
self.all_platforms = {}
self.agent_streamconfig_map = {}
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
self._set_up_DataProduct_obj()
self._set_up_PlatformModel_obj()
def _set_up_DataProduct_obj(self):
# Create data product object to be used for each of the platform log streams
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name('platform_eng_parsed', id_only=True)
self.platform_eng_stream_def_id = self.pubsubcli.create_stream_definition(
name='platform_eng', parameter_dictionary_id=self.pdict_id)
self.dp_obj = IonObject(RT.DataProduct,
name='platform_eng data',
description='platform_eng test',
temporal_domain = tdom,
spatial_domain = sdom)
def _set_up_PlatformModel_obj(self):
# Create PlatformModel
platformModel_obj = IonObject(RT.PlatformModel,
name='RSNPlatformModel',
description="RSNPlatformModel")
try:
self.platformModel_id = self.imsclient.create_platform_model(platformModel_obj)
except BadRequest as ex:
self.fail("failed to create new PLatformModel: %s" %ex)
log.debug( 'new PlatformModel id = %s', self.platformModel_id)
def _traverse(self, pnode, platform_id, parent_platform_objs=None):
"""
Recursive routine that repeatedly calls _prepare_platform to build
the object dictionary for each platform.
@param pnode PlatformNode
@param platform_id ID of the platform to be visited
@param parent_platform_objs dict of objects associated to parent
platform, if any.
@retval the dict returned by _prepare_platform at this level.
"""
log.info("Starting _traverse for %r", platform_id)
plat_objs = self._prepare_platform(pnode, platform_id, parent_platform_objs)
self.all_platforms[platform_id] = plat_objs
# now, traverse the children:
for sub_pnode in pnode.subplatforms.itervalues():
subplatform_id = sub_pnode.platform_id
self._traverse(sub_pnode, subplatform_id, plat_objs)
return plat_objs
def _prepare_platform(self, pnode, platform_id, parent_platform_objs):
"""
This routine generalizes the manual construction originally done in
test_oms_launch.py. It is called by the recursive _traverse method so
all platforms starting from a given base platform are prepared.
Note: For simplicity in this test, sites are organized in the same
hierarchical way as the platforms themselves.
@param pnode PlatformNode
@param platform_id ID of the platform to be visited
@param parent_platform_objs dict of objects associated to parent
platform, if any.
@retval a dict of associated objects similar to those in
test_oms_launch
"""
site__obj = IonObject(RT.PlatformSite,
name='%s_PlatformSite' % platform_id,
description='%s_PlatformSite platform site' % platform_id)
site_id = self.omsclient.create_platform_site(site__obj)
if parent_platform_objs:
# establish hasSite association with the parent
self.rrclient.create_association(
subject=parent_platform_objs['site_id'],
predicate=PRED.hasSite,
object=site_id)
# prepare platform attributes and ports:
monitor_attribute_objs, monitor_attribute_dicts = self._prepare_platform_attributes(pnode, platform_id)
port_objs, port_dicts = self._prepare_platform_ports(pnode, platform_id)
device__obj = IonObject(RT.PlatformDevice,
name='%s_PlatformDevice' % platform_id,
description='%s_PlatformDevice platform device' % platform_id,
# ports=port_objs,
# platform_monitor_attributes = monitor_attribute_objs
)
device__dict = dict(ports=port_dicts,
platform_monitor_attributes=monitor_attribute_dicts)
self.device_id = self.imsclient.create_platform_device(device__obj)
self.imsclient.assign_platform_model_to_platform_device(self.platformModel_id, self.device_id)
self.rrclient.create_association(subject=site_id, predicate=PRED.hasDevice, object=self.device_id)
self.damsclient.register_instrument(instrument_id=self.device_id)
if parent_platform_objs:
# establish hasDevice association with the parent
self.rrclient.create_association(
subject=parent_platform_objs['device_id'],
predicate=PRED.hasDevice,
object=self.device_id)
agent__obj = IonObject(RT.PlatformAgent,
name='%s_PlatformAgent' % platform_id,
description='%s_PlatformAgent platform agent' % platform_id)
agent_id = self.imsclient.create_platform_agent(agent__obj)
if parent_platform_objs:
# add this platform_id to parent's children:
parent_platform_objs['children'].append(platform_id)
self.imsclient.assign_platform_model_to_platform_agent(self.platformModel_id, agent_id)
# agent_instance_obj = IonObject(RT.PlatformAgentInstance,
# name='%s_PlatformAgentInstance' % platform_id,
# description="%s_PlatformAgentInstance" % platform_id)
#
# agent_instance_id = self.imsclient.create_platform_agent_instance(
# agent_instance_obj, agent_id, device_id)
plat_objs = {
'platform_id': platform_id,
'site__obj': site__obj,
'site_id': site_id,
'device__obj': device__obj,
'device_id': self.device_id,
'agent__obj': agent__obj,
'agent_id': agent_id,
# 'agent_instance_obj': agent_instance_obj,
# 'agent_instance_id': agent_instance_id,
'children': []
}
log.info("plat_objs for platform_id %r = %s", platform_id, str(plat_objs))
stream_config = self._create_stream_config(plat_objs)
self.agent_streamconfig_map[platform_id] = stream_config
# self.agent_streamconfig_map[platform_id] = None
# self._start_data_subscriber(agent_instance_id, stream_config)
return plat_objs
def _prepare_platform_attributes(self, pnode, platform_id):
"""
Returns the list of PlatformMonitorAttributes objects corresponding to
the attributes associated to the given platform.
"""
# TODO complete the clean-up of this method
ret_infos = dict((n, a.defn) for (n, a) in pnode.attrs.iteritems())
monitor_attribute_objs = []
monitor_attribute_dicts = []
for attrName, attrDfn in ret_infos.iteritems():
log.debug("platform_id=%r: preparing attribute=%r", platform_id, attrName)
monitor_rate = attrDfn['monitorCycleSeconds']
units = attrDfn['units']
plat_attr_obj = IonObject(OT.PlatformMonitorAttributes,
id=attrName,
monitor_rate=monitor_rate,
units=units)
plat_attr_dict = dict(id=attrName,
monitor_rate=monitor_rate,
units=units)
monitor_attribute_objs.append(plat_attr_obj)
monitor_attribute_dicts.append(plat_attr_dict)
return monitor_attribute_objs, monitor_attribute_dicts
def _prepare_platform_ports(self, pnode, platform_id):
"""
Returns the list of PlatformPort objects corresponding to the ports
associated to the given platform.
"""
# TODO complete the clean-up of this method
port_objs = []
port_dicts = []
for port_id, network in pnode.ports.iteritems():
log.debug("platform_id=%r: preparing port=%r network=%s",
platform_id, port_id, network)
#
# Note: the name "IP" address has been changed to "network" address
# in the CI-OMS interface spec.
#
plat_port_obj = IonObject(OT.PlatformPort,
port_id=port_id,
ip_address=network)
plat_port_dict = dict(port_id=port_id,
network=network)
port_objs.append(plat_port_obj)
port_dicts.append(plat_port_dict)
return port_objs, port_dicts
def _create_stream_config(self, plat_objs):
platform_id = plat_objs['platform_id']
device_id = plat_objs['device_id']
#create the log data product
self.dp_obj.name = '%s platform_eng data' % platform_id
self.data_product_id = self.dpclient.create_data_product(data_product=self.dp_obj, stream_definition_id=self.platform_eng_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=self.device_id, data_product_id=self.data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(self.data_product_id, PRED.hasStream, None, True)
stream_config = self._build_stream_config(stream_ids[0])
return stream_config
def _build_stream_config(self, stream_id=''):
platform_eng_dictionary = DatasetManagementService.get_parameter_dictionary_by_name('platform_eng_parsed')
#get the streamroute object from pubsub by passing the stream_id
stream_def_ids, _ = self.rrclient.find_objects(stream_id,
PRED.hasStreamDefinition,
RT.StreamDefinition,
True)
stream_route = self.pubsubcli.read_stream_route(stream_id=stream_id)
stream_config = {'routing_key' : stream_route.routing_key,
'stream_id' : stream_id,
'stream_definition_ref' : stream_def_ids[0],
'exchange_point' : stream_route.exchange_point,
'parameter_dictionary':platform_eng_dictionary.dump()}
return stream_config
def _set_platform_agent_instances(self):
"""
Once most of the objs/defs associated with all platforms are in
place, this method creates and associates the PlatformAgentInstance
elements.
"""
self.platform_configs = {}
for platform_id, plat_objs in self.all_platforms.iteritems():
PLATFORM_CONFIG = {
'platform_id': platform_id,
'agent_streamconfig_map': None, #self.agent_streamconfig_map,
'driver_config': DVR_CONFIG,
'network_definition' : self._network_definition_ser
}
self.platform_configs[platform_id] = {
'platform_id': platform_id,
'agent_streamconfig_map': self.agent_streamconfig_map,
'driver_config': DVR_CONFIG,
'network_definition' : self._network_definition_ser
}
agent_config = {
'platform_config': PLATFORM_CONFIG,
}
self.stream_id = self.agent_streamconfig_map[platform_id]['stream_id']
# import pprint
# print '============== platform id within unit test: %s ===========' % platform_id
# pprint.pprint(agent_config)
#agent_config['platform_config']['agent_streamconfig_map'] = None
agent_instance_obj = IonObject(RT.PlatformAgentInstance,
name='%s_PlatformAgentInstance' % platform_id,
description="%s_PlatformAgentInstance" % platform_id,
agent_config=agent_config)
agent_id = plat_objs['agent_id']
device_id = plat_objs['device_id']
agent_instance_id = self.imsclient.create_platform_agent_instance(
agent_instance_obj, agent_id, self.device_id)
plat_objs['agent_instance_obj'] = agent_instance_obj
plat_objs['agent_instance_id'] = agent_instance_id
stream_config = self.agent_streamconfig_map[platform_id]
self._start_data_subscriber(agent_instance_id, stream_config)
def _start_data_subscriber(self, stream_name, stream_config):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r', stream_name)
stream_id = self.stream_id #stream_config['stream_id']
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.pubsubcli.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.pubsubcli.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
@skip("IMS does't net implement topology")
def test_hierarchy(self):
self._create_launch_verify(BASE_PLATFORM_ID)
@skip("Needs alignment with recent IMS changes")
def test_single_platform(self):
self._create_launch_verify('LJ01D')
def _create_launch_verify(self, base_platform_id):
# and trigger the traversal of the branch rooted at that base platform
# to create corresponding ION objects and configuration dictionaries:
pnode = self._network_definition.pnodes[base_platform_id]
base_platform_objs = self._traverse(pnode, base_platform_id)
# now that most of the topology information is there, add the
# PlatformAgentInstance elements
self._set_platform_agent_instances()
base_platform_config = self.platform_configs[base_platform_id]
log.info("base_platform_id = %r", base_platform_id)
#-------------------------------------------------------------------------------------
# Create Data Process Definition and Data Process for the eng stream monitor process
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='DemoStreamAlertTransform',
description='For testing EventTriggeredTransform_B',
module='ion.processes.data.transforms.event_alert_transform',
class_name='DemoStreamAlertTransform')
self.platform_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#THERE SHOULD BE NO STREAMDEF REQUIRED HERE.
platform_streamdef_id = self.pubsubcli.create_stream_definition(name='platform_eng_parsed', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(platform_streamdef_id, self.platform_dprocdef_id, binding='output' )
config = {
'process':{
'timer_interval': 5,
'queue_name': 'a_queue',
'variable_name': 'input_voltage',
'time_field_name': 'preferred_timestamp',
'valid_values': [-100, 100],
'timer_origin': 'Interval Timer'
}
}
platform_data_process_id = self.dataprocessclient.create_data_process(self.platform_dprocdef_id, [self.data_product_id], {}, config)
self.dataprocessclient.activate_data_process(platform_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process, platform_data_process_id)
#-------------------------------
# Launch Base Platform AgentInstance, connect to the resource agent client
#-------------------------------
agent_instance_id = base_platform_objs['agent_instance_id']
log.debug("about to call imsclient.start_platform_agent_instance with id=%s", agent_instance_id)
pid = self.imsclient.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
instance_obj = self.imsclient.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
agent_instance_obj= self.imsclient.read_instrument_agent_instance(agent_instance_id)
log.debug('test_oms_create_and_launch: Platform agent instance obj: %s', str(agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._pa_client = ResourceAgentClient('paclient', name=agent_instance_obj.agent_process_id, process=FakeProcess())
log.debug(" test_oms_create_and_launch:: got pa client %s", str(self._pa_client))
log.debug("base_platform_config =\n%s", base_platform_config)
# ping_agent can be issued before INITIALIZE
retval = self._pa_client.ping_agent(timeout=TIMEOUT)
log.debug( 'Base Platform ping_agent = %s', str(retval) )
# issue INITIALIZE command to the base platform, which will launch the
# creation of the whole platform hierarchy rooted at base_platform_config['platform_id']
# cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE, kwargs=dict(plat_config=base_platform_config))
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform INITIALIZE = %s', str(retval) )
# GO_ACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform GO_ACTIVE = %s', str(retval) )
# RUN:
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform RUN = %s', str(retval) )
# START_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform START_MONITORING = %s', str(retval) )
# wait for data sample
# just wait for at least one -- see consume_data above
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("waiting a bit more for reception of more data samples...")
sleep(15)
log.info("Got data samples: %d", len(self._samples_received))
# wait for event
# just wait for at least one event -- see consume_event above
log.info("waiting for reception of an event...")
self._async_event_result.get(timeout=EVENT_TIMEOUT)
log.info("Received events: %s", len(self._events_received))
#get the extended platfrom which wil include platform aggreate status fields
extended_platform = self.imsclient.get_platform_device_extension(self.device_id)
# log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) )
# log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) )
# log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) )
# STOP_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform STOP_MONITORING = %s', str(retval) )
# GO_INACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform GO_INACTIVE = %s', str(retval) )
# RESET: Resets the base platform agent, which includes termination of
# its sub-platforms processes:
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform RESET = %s', str(retval) )
#-------------------------------
# Stop Base Platform AgentInstance
#-------------------------------
self.imsclient.stop_platform_agent_instance(platform_agent_instance_id=agent_instance_id)