def prepare(self, will_launch=True):
"""
Prepare (validate) an agent for launch, fetching all associated resources
@param will_launch - whether the running status should be checked -- set false if just generating config
"""
assert self.agent_instance_obj
# fetch caches just in time
if any([not self.RR2.has_cached_predicate(x) for x in self._predicates_to_cache()]):
self._update_cached_predicates()
if any([not self.RR2.has_cached_resource(x) for x in self._resources_to_cache()]):
self._update_cached_resources()
# validate the associations, then pick things up
self._collect_agent_instance_associations()
if will_launch:
# if there is an agent pid then assume that a drive is already started
agent_process_id = ResourceAgentClient._get_agent_process_id(self._get_device()._id)
if agent_process_id:
raise BadRequest("Agent Instance already running for this device pid: %s" %
str(agent_process_id))
self.will_launch = will_launch
config = self.generate_config()
return config
def prepare(self, will_launch=True):
"""
Prepare (validate) an agent for launch, fetching all associated resources
@param will_launch - whether the running status should be checked -- set false if just generating config
"""
assert self.agent_instance_obj
# fetch caches just in time
if any([
not self.RR2.has_cached_predicate(x)
for x in self._predicates_to_cache()
]):
self._update_cached_predicates()
if any([
not self.RR2.has_cached_resource(x)
for x in self._resources_to_cache()
]):
self._update_cached_resources()
# validate the associations, then pick things up
self._collect_agent_instance_associations()
if will_launch:
# if there is an agent pid then assume that a drive is already started
agent_process_id = ResourceAgentClient._get_agent_process_id(
self._get_device()._id)
if agent_process_id:
raise BadRequest(
"Agent Instance already running for this device pid: %s" %
str(agent_process_id))
self.will_launch = will_launch
return self.generate_config()
def __init__(self, read_process_fn=None, resource_id='', desired_state=None, *args, **kwargs):
agent_process_id = ResourceAgentClient._get_agent_process_id(resource_id)
super(AgentProcessStateGate, self).__init__(read_process_fn,
agent_process_id,
desired_state,
*args, **kwargs)
def test_get_agent_client_noprocess(self):
inst_device_id = self.RR2.create(any_old(RT.InstrumentDevice))
iap = ResourceAgentClient._get_agent_process_id(inst_device_id)
# should be no running agent
self.assertIsNone(iap)
# should raise NotFound
self.assertRaises(NotFound, ResourceAgentClient, inst_device_id)
def test_get_agent_client_noprocess(self):
inst_device_id = self.RR2.create(any_old(RT.InstrumentDevice))
iap = ResourceAgentClient._get_agent_process_id(inst_device_id)
# should be no running agent
self.assertIsNone(iap)
# should raise NotFound
self.assertRaises(NotFound, ResourceAgentClient, inst_device_id)
def __init__(self,
read_process_fn=None,
resource_id='',
desired_state=None,
*args,
**kwargs):
agent_process_id = ResourceAgentClient._get_agent_process_id(
resource_id)
super(AgentProcessStateGate,
self).__init__(read_process_fn, agent_process_id, desired_state,
*args, **kwargs)
def base_activateInstrumentSample(self, instAgent_obj, expect_launch=True, expect_command=True):
"""
This method runs a test of launching a driver with a given agent configuration
"""
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
print 'new InstrumentModel id = %s ' % instModel_id
# Create InstrumentAgent
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
print 'new InstrumentAgent id = %s' % instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',
id_only=True)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict',
id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed',
parameter_dictionary_id=parsed_pdict_id)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw',
parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test')
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new dp_id = %s' % data_product_id1
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
print 'Data product streams1 = %s' % stream_ids
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
print 'Data set for data_product_id1 = %s' % dataset_ids[0]
self.parsed_dataset = dataset_ids[0]
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
print 'new dp_id = %s' % str(data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
print 'Data product streams2 = %s' % str(stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
print 'Data set for data_product_id2 = %s' % dataset_ids[0]
self.raw_dataset = dataset_ids[0]
# add start/stop for instrument agent
gevent.joinall([gevent.spawn(lambda:
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id))])
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
agent_process_id = ResourceAgentClient._get_agent_process_id(instDevice_id)
print "Agent process id is '%s'" % str(agent_process_id)
self.assertTrue(agent_process_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
agent_process_id,
ProcessStateEnum.RUNNING)
if not expect_launch:
self.assertFalse(gate.await(30), "The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?" %
agent_process_id)
return
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
agent_process_id)
print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING"
#print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=agent_process_id,
process=FakeProcess())
print "ResourceAgentClient created: %s" % str(self._ia_client)
print "Sending command=ResourceAgentEvent.INITIALIZE"
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
if not expect_command:
self.assertRaises(ServerError, self._ia_client.execute_agent, cmd)
return
retval = self._ia_client.execute_agent(cmd)
print "Result of INITIALIZE: %s" % str(retval)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# This gevent sleep is there to test the autosample time, which will show something different from default
# only if the instrument runs for over a minute
gevent.sleep(90)
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
autosample_string = extended_instrument.computed.uptime.value
autosampling_time = int(autosample_string.split()[4])
self.assertTrue(autosampling_time > 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
print "Sending command=ResourceAgentEvent.RESET"
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
print "Result of RESET: %s" % str(reply)
def _start_subscriber_process_lifecycle_event(self, origin):
"""
@param origin Child's resource_id. The associated PID retrieved via
ResourceAgentClient._get_agent_process_id is used for
the event subscriber itself, but we still index
_event_subscribers with the given origin.
"""
def _got_process_lifecycle_event(evt, *args, **kwargs):
with self._lock:
if not self._active:
log.warn("%r: _got_process_lifecycle_event called but "
"manager has been destroyed",
self._platform_id)
return
if evt.type_ != "ProcessLifecycleEvent":
log.trace("%r: ignoring event type %r. Only handle "
"ProcessLifecycleEvent directly.",
self._platform_id, evt.type_)
return
# evt.origin is a PID
pid = evt.origin
if not pid in self._rids:
log.warn("%r: OOIION-1077 ignoring event from pid=%r. "
"Expecting one of %s",
self._platform_id, pid, self._rids.keys())
return
origin = self._rids[pid]
# # Before the _rids mapping, a preliminary mechanism to check
# # whether the event came from the expected origin relied on
# # the process ID having origin as a substring:
#
# if not origin in pid:
# log.warn("%r: OOIION-1077 ignoring event from origin %r. "
# "Expecting an origin containing %r",
# self._platform_id, pid, origin)
# return
# # BUT this was definitely weak. Although the PID for an
# # initial agent process seems to satisfy this assumption,
# # this is not anymore the case upon a re-start of that agent.
log.debug("%r: OOIION-1077 _got_process_lifecycle_event: "
"pid=%r origin=%r state=%r(%s)",
self._platform_id, pid, origin,
ProcessStateEnum._str_map[evt.state], evt.state)
if evt.state is ProcessStateEnum.TERMINATED:
self._device_terminated_event(origin, pid)
# use associated process ID for the subscription:
pid = ResourceAgentClient._get_agent_process_id(origin)
if pid is None:
log.warn("%r: OOIION-1077 ResourceAgentClient._get_agent_process_id"
" returned None for origin=%r. Subscriber not created.",
self._platform_id, origin)
return
sub = self._agent._create_event_subscriber(event_type="ProcessLifecycleEvent",
origin_type='DispatchedProcess',
origin=pid,
callback=_got_process_lifecycle_event)
with self._lock:
# but note that we use the given origin as index in _event_subscribers:
self._event_subscribers[origin] = sub
# and capture the pid -> origin mapping:
self._rids[pid] = origin
log.debug("%r: OOIION-1077 registered ProcessLifecycleEvent subscriber "
"with pid=%r (origin=%r)",
self._platform_id, pid, origin)
def base_activateInstrumentSample(self,
instAgent_obj,
expect_launch=True,
expect_command=True):
"""
This method runs a test of launching a driver with a given agent configuration
"""
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
print 'new InstrumentModel id = %s ' % instModel_id
# Create InstrumentAgent
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
print 'new InstrumentAgent id = %s' % instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new dp_id = %s' % data_product_id1
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
print 'Data product streams1 = %s' % stream_ids
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
print 'Data set for data_product_id1 = %s' % dataset_ids[0]
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
self.get_datastore(self.parsed_dataset)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
print 'new dp_id = %s' % str(data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
print 'Data product streams2 = %s' % str(stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
print 'Data set for data_product_id2 = %s' % dataset_ids[0]
self.raw_dataset = dataset_ids[0]
# add start/stop for instrument agent
gevent.joinall([
gevent.spawn(
lambda: self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id))
])
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
agent_process_id = ResourceAgentClient._get_agent_process_id(
instDevice_id)
print "Agent process id is '%s'" % str(agent_process_id)
self.assertTrue(agent_process_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
agent_process_id, ProcessStateEnum.RUNNING)
if not expect_launch:
self.assertFalse(
gate. await (30),
"The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?"
% agent_process_id)
return
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
agent_process_id)
print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING"
#print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=agent_process_id,
process=FakeProcess())
print "ResourceAgentClient created: %s" % str(self._ia_client)
print "Sending command=ResourceAgentEvent.INITIALIZE"
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
if not expect_command:
self.assertRaises(ServerError, self._ia_client.execute_agent, cmd)
return
retval = self._ia_client.execute_agent(cmd)
print "Result of INITIALIZE: %s" % str(retval)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# This gevent sleep is there to test the autosample time, which will show something different from default
# only if the instrument runs for over a minute
gevent.sleep(90)
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
autosample_string = extended_instrument.computed.uptime.value
autosampling_time = int(autosample_string.split()[4])
self.assertTrue(autosampling_time > 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
print "Sending command=ResourceAgentEvent.RESET"
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
print "Result of RESET: %s" % str(reply)
def _start_subscriber_process_lifecycle_event(self, origin):
"""
@param origin Child's resource_id. The associated PID retrieved via
ResourceAgentClient._get_agent_process_id is used for
the event subscriber itself, but we still index
_event_subscribers with the given origin.
"""
def _got_process_lifecycle_event(evt, *args, **kwargs):
with self._lock:
if not self._active:
log.warn(
"%r: _got_process_lifecycle_event called but "
"manager has been destroyed", self._platform_id)
return
if evt.type_ != "ProcessLifecycleEvent":
log.trace(
"%r: ignoring event type %r. Only handle "
"ProcessLifecycleEvent directly.", self._platform_id,
evt.type_)
return
# evt.origin is a PID
pid = evt.origin
if not pid in self._rids:
log.warn(
"%r: OOIION-1077 ignoring event from pid=%r. "
"Expecting one of %s", self._platform_id, pid,
self._rids.keys())
return
origin = self._rids[pid]
# # Before the _rids mapping, a preliminary mechanism to check
# # whether the event came from the expected origin relied on
# # the process ID having origin as a substring:
#
# if not origin in pid:
# log.warn("%r: OOIION-1077 ignoring event from origin %r. "
# "Expecting an origin containing %r",
# self._platform_id, pid, origin)
# return
# # BUT this was definitely weak. Although the PID for an
# # initial agent process seems to satisfy this assumption,
# # this is not anymore the case upon a re-start of that agent.
log.debug(
"%r: OOIION-1077 _got_process_lifecycle_event: "
"pid=%r origin=%r state=%r(%s)", self._platform_id, pid,
origin, ProcessStateEnum._str_map[evt.state], evt.state)
if evt.state is ProcessStateEnum.TERMINATED:
self._device_terminated_event(origin, pid)
# use associated process ID for the subscription:
pid = ResourceAgentClient._get_agent_process_id(origin)
if pid is None:
log.warn(
"%r: OOIION-1077 ResourceAgentClient._get_agent_process_id"
" returned None for origin=%r. Subscriber not created.",
self._platform_id, origin)
return
sub = self._agent._create_event_subscriber(
event_type="ProcessLifecycleEvent",
origin_type='DispatchedProcess',
origin=pid,
callback=_got_process_lifecycle_event)
with self._lock:
# but note that we use the given origin as index in _event_subscribers:
self._event_subscribers[origin] = sub
# and capture the pid -> origin mapping:
self._rids[pid] = origin
log.debug(
"%r: OOIION-1077 registered ProcessLifecycleEvent subscriber "
"with pid=%r (origin=%r)", self._platform_id, pid, origin)
