class AgentConfigurationBuilder(object):
def __init__(self, clients, RR2=None):
self.clients = clients
self.RR2 = RR2
if self.RR2 is None:
log.warn("Creating new RR2")
self.RR2 = EnhancedResourceRegistryClient(self.clients.resource_registry)
if not isinstance(self.RR2, EnhancedResourceRegistryClient):
raise AssertionError("Type of self.RR2 is %s not %s" %
(type(self.RR2), type(EnhancedResourceRegistryClient)))
self.agent_instance_obj = None
self.associated_objects = None
self.last_id = None
self.will_launch = False
self.generated_config = False
def _predicates_to_cache(self):
return [PRED.hasOutputProduct,
#PRED.hasStream,
#PRED.hasStreamDefinition,
PRED.hasAgentInstance,
PRED.hasAgentDefinition,
PRED.hasDataset,
PRED.hasDevice,
PRED.hasNetworkParent,
#PRED.hasParameterContext,
PRED.hasDeployment,
]
def _resources_to_cache(self):
return [#RT.StreamDefinition,
RT.ParameterDictionary,
#RT.ParameterContext,
RT.Deployment,
]
def _update_cached_predicates(self):
# cache some predicates for in-memory lookups
preds = self._predicates_to_cache()
log.debug("updating cached predicates: %s" % preds)
time_caching_start = get_ion_ts()
for pred in preds:
log.debug(" - %s", pred)
self.RR2.cache_predicate(pred)
time_caching_stop = get_ion_ts()
total_time = int(time_caching_stop) - int(time_caching_start)
log.info("Cached %s predicates in %s seconds", len(preds), total_time / 1000.0)
def _update_cached_resources(self):
# cache some resources for in-memory lookups
rsrcs = self._resources_to_cache()
log.debug("updating cached resources: %s" % rsrcs)
time_caching_start = get_ion_ts()
for r in rsrcs:
log.debug(" - %s", r)
self.RR2.cache_resources(r)
time_caching_stop = get_ion_ts()
total_time = int(time_caching_stop) - int(time_caching_start)
log.info("Cached %s resource types in %s seconds", len(rsrcs), total_time / 1000.0)
def _clear_caches(self):
log.warn("Clearing caches")
for r in self._resources_to_cache():
self.RR2.clear_cached_resource(r)
for p in self._predicates_to_cache():
self.RR2.clear_cached_predicate(p)
def _lookup_means(self):
"""
return a dict indicating how various related resources will be looked up
The dict is keyed on association type:
PRED.hasAgentInstance -> device type
PRED.hasModel -> model type
PRED.hasAgentDefinition -> agent type
"""
raise NotImplementedError("Extender of class must implement this")
def _augment_dict(self, title, basedict, newitems):
# TODO: pyon.util.containers has dict_merge for this purpose (without logs)
for k, v in newitems.iteritems():
if k in basedict:
prev_v = basedict[k]
# just warn if the new value is different
if v != prev_v:
log.warn("Overwriting %s[%s] of '%s' with '%s'", title, k, prev_v, v)
else:
log.debug("Overwriting %s[%s] with same value already assigned '%s'",
title, k, v)
basedict[k] = v
def _check_associations(self):
assert self.agent_instance_obj
assert self.associated_objects
lookup_means = self._lookup_means()
assert lookup_means
# make sure we've picked up the associations we expect
def check_keys(somekeys):
for k in somekeys:
assert k in lookup_means
assert lookup_means[k] in self.associated_objects
#check_keys([PRED.hasAgentInstance, PRED.hasModel, PRED.hasAgentDefinition])
check_keys([PRED.hasAgentInstance, PRED.hasAgentDefinition])
assert RT.ProcessDefinition in self.associated_objects
def set_agent_instance_object(self, agent_instance_obj):
"""
Set the agent instance object that we'll be interacting with
it may be necessary to set this several times, such as if external operations update the object
"""
assert agent_instance_obj._id
if self.last_id != agent_instance_obj._id:
self.associated_objects = None
self.agent_instance_obj = agent_instance_obj
self.last_id = agent_instance_obj._id
self.generated_config = False
def prepare(self, will_launch=True):
"""
Prepare (validate) an agent for launch, fetching all associated resources
@param will_launch - whether the running status should be checked -- set false if just generating config
"""
assert self.agent_instance_obj
# fetch caches just in time
if any([not self.RR2.has_cached_predicate(x) for x in self._predicates_to_cache()]):
self._update_cached_predicates()
if any([not self.RR2.has_cached_resource(x) for x in self._resources_to_cache()]):
self._update_cached_resources()
# validate the associations, then pick things up
self._collect_agent_instance_associations()
if will_launch:
# if there is an agent pid then assume that a drive is already started
agent_process_id = ResourceAgentClient._get_agent_process_id(self._get_device()._id)
if agent_process_id:
raise BadRequest("Agent Instance already running for this device pid: %s" %
str(agent_process_id))
self.will_launch = will_launch
config = self.generate_config()
return config
def _generate_org_governance_name(self):
log.debug("_generate_org_governance_name for %s", self.agent_instance_obj.name)
log.debug("retrieve the Org governance name to which this agent instance belongs")
try:
org_obj = self.RR2.find_subject(RT.Org, PRED.hasResource, self.agent_instance_obj._id, id_only=False)
return org_obj.org_governance_name
except NotFound:
return ''
except:
raise
def _generate_device_type(self):
log.debug("_generate_device_type for %s", self.agent_instance_obj.name)
return type(self._get_device()).__name__
def _generate_driver_config(self):
log.debug("_generate_driver_config for %s", self.agent_instance_obj.name)
# get default config
driver_config = self.agent_instance_obj.driver_config
agent_obj = self._get_agent()
# Create driver config.
add_driver_config = {
'workdir' : tempfile.gettempdir(),
'dvr_mod' : agent_obj.driver_module,
'dvr_cls' : agent_obj.driver_class
}
self._augment_dict("Agent driver_config", driver_config, add_driver_config)
return driver_config
def _get_param_dict_by_name(self, name):
dict_obj = self.RR2.find_resources_by_name(RT.ParameterDictionary, name)[0]
parameter_contexts = \
self.RR2.find_parameter_contexts_of_parameter_dictionary_using_has_parameter_context(dict_obj._id)
return DatasetManagementService.build_parameter_dictionary(dict_obj, parameter_contexts)
def _find_streamdef_for_dp_and_pdict(self, dp_id, pdict_id):
# Given a pdict_id and a data_product_id find the stream def in the middle
pdict_stream_defs = self.RR2.find_stream_definition_ids_by_parameter_dictionary_using_has_parameter_dictionary(pdict_id)
stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(dp_id)
result = stream_def_id if stream_def_id in pdict_stream_defs else None
return result
def _generate_stream_config(self):
log.debug("_generate_stream_config for %s", self.agent_instance_obj.name)
dsm = self.clients.dataset_management
psm = self.clients.pubsub_management
agent_obj = self._get_agent()
device_obj = self._get_device()
streams_dict = {}
for stream_cfg in agent_obj.stream_configurations:
#create a stream def for each param dict to match against the existing data products
streams_dict[stream_cfg.stream_name] = {'param_dict_name':stream_cfg.parameter_dictionary_name}
#retrieve the output products
# TODO: What about platforms? other things?
device_id = device_obj._id
data_product_objs = self.RR2.find_data_products_of_instrument_device_using_has_output_product(device_id)
stream_config = {}
for dp in data_product_objs:
stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(dp._id)
for stream_name, stream_info_dict in streams_dict.items():
# read objects from cache to be compared
pdict = self.RR2.find_resource_by_name(RT.ParameterDictionary, stream_info_dict.get('param_dict_name'))
stream_def_id = self._find_streamdef_for_dp_and_pdict(dp._id, pdict._id)
if stream_def_id:
#model_param_dict = self.RR2.find_resources_by_name(RT.ParameterDictionary,
# stream_info_dict.get('param_dict_name'))[0]
#model_param_dict = self._get_param_dict_by_name(stream_info_dict.get('param_dict_name'))
#stream_route = self.RR2.read(product_stream_id).stream_route
product_stream_id = self.RR2.find_stream_id_of_data_product_using_has_stream(dp._id)
stream_def = psm.read_stream_definition(stream_def_id)
stream_route = psm.read_stream_route(stream_id=product_stream_id)
from pyon.core.object import IonObjectSerializer
stream_def_dict = IonObjectSerializer().serialize(stream_def)
stream_def_dict.pop('type_')
if stream_name in stream_config:
log.warn("Overwriting stream_config[%s]", stream_name)
stream_config[stream_name] = { 'routing_key' : stream_route.routing_key, # TODO: Serialize stream_route together
'stream_id' : product_stream_id,
'stream_definition_ref' : stream_def_id,
'stream_def_dict' : stream_def_dict, # This is very large
'exchange_point' : stream_route.exchange_point,
# This is redundant and very large - the param dict is in the stream_def_dict
#'parameter_dictionary' : stream_def.parameter_dictionary,
}
if len(stream_config) < len(streams_dict):
log.warn("Found only %s matching streams by stream definition (%s) than %s defined in the agent (%s).",
len(stream_config), stream_config.keys(), len(streams_dict), streams_dict.keys())
log.debug("Stream config generated")
log.trace("generate_stream_config: %s", stream_config)
return stream_config
def _generate_agent_config(self):
log.debug("_generate_agent_config for %s", self.agent_instance_obj.name)
agent_config = {}
# Set the agent state vector from the prior agent run
if self.agent_instance_obj.saved_agent_state:
agent_config["prior_state"] = self.agent_instance_obj.saved_agent_state
return agent_config
def _generate_alerts_config(self):
log.debug("_generate_alerts_config for %s", self.agent_instance_obj.name)
# should override this
return self.agent_instance_obj.alerts
def _generate_startup_config(self):
log.debug("_generate_startup_config for %s", self.agent_instance_obj.name)
# should override this
return {}
def _generate_children(self):
log.debug("_generate_children for %s", self.agent_instance_obj.name)
# should override this
return {}
def _generate_skeleton_config_block(self):
log.info("Generating skeleton config block for %s", self.agent_instance_obj.name)
# merge the agent config into the default config
agent_config = dict_merge(self._get_agent().agent_default_config, self.agent_instance_obj.agent_config, True)
# Create agent_config.
agent_config['instance_id'] = self.agent_instance_obj._id
agent_config['instance_name'] = self.agent_instance_obj.name
agent_config['org_governance_name'] = self._generate_org_governance_name()
agent_config['device_type'] = self._generate_device_type()
agent_config['driver_config'] = self._generate_driver_config()
agent_config['stream_config'] = self._generate_stream_config()
agent_config['agent'] = self._generate_agent_config()
agent_config['aparam_alerts_config'] = self._generate_alerts_config()
agent_config['startup_config'] = self._generate_startup_config()
agent_config['children'] = self._generate_children()
log.info("DONE generating skeleton config block for %s", self.agent_instance_obj.name)
return agent_config
def _summarize_children(self, config_dict):
ret = dict([(v['instance_name'], self._summarize_children(v))
for k, v in config_dict["children"].iteritems()])
#agent_config['agent']['resource_id']
return ret
def generate_config(self):
"""
create the generic parts of the configuration including resource_id, egg_uri, and org
"""
if self.generated_config:
log.warn("Generating config again for the same Instance object (%s)", self.agent_instance_obj.name)
self._check_associations()
agent_config = self._generate_skeleton_config_block()
device_obj = self._get_device()
agent_obj = self._get_agent()
log.debug("complement agent_config with resource_id")
if 'agent' not in agent_config:
agent_config['agent'] = {'resource_id': device_obj._id}
elif 'resource_id' not in agent_config.get('agent'):
agent_config['agent']['resource_id'] = device_obj._id
log.debug("add egg URI if available")
if agent_obj.driver_uri:
agent_config['driver_config']['process_type'] = (DriverProcessType.EGG,)
agent_config['driver_config']['dvr_egg'] = agent_obj.driver_uri
else:
agent_config['driver_config']['process_type'] = (DriverProcessType.PYTHON_MODULE,)
if log.isEnabledFor(logging.INFO):
tree = self._summarize_children(agent_config)
log.info("Children of %s are %s", self.agent_instance_obj.name, tree)
self.generated_config = True
return agent_config
def record_launch_parameters(self, agent_config):
"""
record process id of the launch
"""
#self.RR2.update(self.agent_instance_obj)
log.debug('completed agent start')
def _collect_deployment(self, device_id=None):
deployment_objs = self.RR2.find_objects(device_id, PRED.hasDeployment, RT.Deployment)
# find current deployment using time constraints
current_time = int( calendar.timegm(time.gmtime()) )
for d in deployment_objs:
# find deployment start and end time
time_constraint = None
for constraint in d.constraint_list:
if constraint.type_ == OT.TemporalBounds:
if time_constraint:
log.warn('deployment %s has more than one time constraint (using first)', d.name)
else:
time_constraint = constraint
if time_constraint:
# a time constraint was provided, check if the current time is in this window
if int(time_constraint.start_datetime) < current_time < int(time_constraint.end_datetime) :
log.debug('_collect_deployment found current deployment start time: %s, end time: %s current time: %s deployment: %s ',
time_constraint.start_datetime, time_constraint.end_datetime, current_time, d)
return d
return None
def _validate_reference_designator(self, port_assignments):
#validate that each reference designator is valid / parseable
# otherwise the platform cannot pull out the port number for power mgmt
if not port_assignments:
return
if not isinstance(port_assignments, dict):
log.error('Deployment for device has invalid port assignments. device id: %s ', self._get_device()._id)
return
for device_id, platform_port in port_assignments.iteritems():
if platform_port.type_ != OT.PlatformPort:
log.error('Deployment for device has invalid port assignments for device. device id: %s', device_id)
ooi_rd = OOIReferenceDesignator(platform_port.reference_designator)
if ooi_rd.error:
log.error('Agent configuration includes a invalid reference designator for a device in this deployment. device id: %s reference designator: %s', device_id, platform_port.reference_designator)
return
def _serialize_port_assigments(self, port_assignments=None):
serializer = IonObjectSerializer()
serialized_port_assignments = {}
if isinstance(port_assignments, dict):
for device_id, platform_port in port_assignments.iteritems():
flatpp = serializer.serialize(platform_port)
serialized_port_assignments[device_id] = flatpp
return serialized_port_assignments
def _collect_agent_instance_associations(self):
"""
Collect related resources to this agent instance
Returns a dict of objects necessary to start this instance, keyed on the values of self._lookup_means()
PRED.hasAgentInstance -> device_obj
PRED.hasModel -> model_obj
PRED.hasAgentDefinition -> agent_obj
RT.ProcessDefinition -> process_def_obj
"""
assert self.agent_instance_obj
lookup_means = self._lookup_means()
assert lookup_means
assert PRED.hasAgentInstance in lookup_means
assert PRED.hasModel in lookup_means
assert PRED.hasAgentDefinition in lookup_means
#assert PRED.hasProcessDefinition in lookup_means
lu = lookup_means
ret = {}
log.debug("retrieve the associated device")
res_types = lu[PRED.hasAgentInstance]
if not hasattr(res_types, "__iter__"):
res_types = [res_types]
device_obj = None
for res_type in res_types:
try:
device_obj = self.RR2.find_subject(subject_type=res_type,
predicate=PRED.hasAgentInstance,
object=self.agent_instance_obj._id)
break
except NotFound:
pass
if not device_obj:
raise NotFound("Could not find a Device for AgentInstance %s" % self.agent_instance_obj._id)
ret[lu[PRED.hasAgentInstance]] = device_obj # Note: can be a tuple key
device_id = device_obj._id
log.debug("%s '%s' connected to %s '%s' (L4-CI-SA-RQ-363)",
lu[PRED.hasAgentInstance],
str(device_id),
self.agent_instance_obj.type_,
self.agent_instance_obj._id)
# log.debug("retrieve the model associated with the device")
# model_obj = self.RR2.find_object(subject=device_id,
# predicate=PRED.hasModel,
# object_type=lu[PRED.hasModel])
#
# ret[lu[PRED.hasModel]] = model_obj
# model_id = model_obj
#retrive the stream info for this model
#todo: add stream info to the platform model create
# streams_dict = platform_models_objs[0].custom_attributes['streams']
# if not streams_dict:
# raise BadRequest("Device model does not contain stream configuation used in launching the agent. Model: '%s", str(platform_models_objs[0]) )
#TODO: get the agent from the instance not from the model!!!!!!!
log.debug("retrieve the agent associated with the model")
agent_obj = self.RR2.find_object(subject=self.agent_instance_obj._id,
predicate=PRED.hasAgentDefinition,
object_type=lu[PRED.hasAgentDefinition])
ret[lu[PRED.hasAgentDefinition]] = agent_obj
agent_id = agent_obj._id
if not agent_obj.stream_configurations:
raise BadRequest("Agent '%s' does not contain stream configuration used in launching" %
str(agent_obj) )
log.debug("retrieve the process definition associated with this agent")
process_def_obj = self.RR2.find_object(subject=agent_id,
predicate=PRED.hasProcessDefinition,
object_type=RT.ProcessDefinition)
ret[RT.ProcessDefinition] = process_def_obj
#retrieve the output products
data_product_objs = self.RR2.find_objects(device_id, PRED.hasOutputProduct, RT.DataProduct, id_only=False)
ret[RT.DataProduct] = data_product_objs
if not data_product_objs:
raise NotFound("No output Data Products attached to this Device " + str(device_id))
#retrieve the streams assoc with each defined output product
for data_product_obj in data_product_objs:
product_id = data_product_obj._id
try:
self.RR2.find_stream_id_of_data_product_using_has_stream(product_id) # check one stream per product
except NotFound:
errmsg = "Device '%s' (%s) has data products %s. Data product '%s' (%s) has no stream ID." % \
(device_obj.name,
device_obj._id,
[dp._id for dp in data_product_objs],
data_product_obj.name,
product_id)
raise NotFound(errmsg)
# some products may not be persisted
try:
# check one dataset per product
self.RR2.find_dataset_id_of_data_product_using_has_dataset(product_id)
except NotFound:
log.warn("Data product '%s' of device %s ('%s') does not appear to be persisted -- no dataset",
product_id, device_obj.name, device_obj._id)
self.associated_objects = ret
def _get_device(self):
self._check_associations()
return self.associated_objects[self._lookup_means()[PRED.hasAgentInstance]]
# def _get_model(self):
# self._check_associations()
# return self.associated_objects[self._lookup_means()[PRED.hasModel]]
def _get_agent(self):
self._check_associations()
return self.associated_objects[self._lookup_means()[PRED.hasAgentDefinition]]
def _get_process_definition(self):
self._check_associations()
return self.associated_objects[RT.ProcessDefinition]
class AgentConfigurationBuilder(object):
def __init__(self, clients, RR2=None):
self.clients = clients
self.RR2 = RR2
if self.RR2 is None:
log.warn("Creating new RR2")
self.RR2 = EnhancedResourceRegistryClient(self.clients.resource_registry)
if not isinstance(self.RR2, EnhancedResourceRegistryClient):
raise AssertionError("Type of self.RR2 is %s not %s" %
(type(self.RR2), type(EnhancedResourceRegistryClient)))
self.agent_instance_obj = None
self.associated_objects = None
self.last_id = None
self.will_launch = False
self.generated_config = False
def _predicates_to_cache(self):
return [PRED.hasOutputProduct,
#PRED.hasStream,
#PRED.hasStreamDefinition,
PRED.hasAgentInstance,
PRED.hasAgentDefinition,
PRED.hasDataset,
PRED.hasDevice,
PRED.hasNetworkParent,
#PRED.hasParameterContext,
]
def _resources_to_cache(self):
return [#RT.StreamDefinition,
RT.ParameterDictionary,
#RT.ParameterContext,
]
def _update_cached_predicates(self):
# cache some predicates for in-memory lookups
preds = self._predicates_to_cache()
log.debug("updating cached predicates: %s" % preds)
time_caching_start = get_ion_ts()
for pred in preds:
log.debug(" - %s", pred)
self.RR2.cache_predicate(pred)
time_caching_stop = get_ion_ts()
total_time = int(time_caching_stop) - int(time_caching_start)
log.info("Cached %s predicates in %s seconds", len(preds), total_time / 1000.0)
def _update_cached_resources(self):
# cache some resources for in-memory lookups
rsrcs = self._resources_to_cache()
log.debug("updating cached resources: %s" % rsrcs)
time_caching_start = get_ion_ts()
for r in rsrcs:
log.debug(" - %s", r)
self.RR2.cache_resources(r)
time_caching_stop = get_ion_ts()
total_time = int(time_caching_stop) - int(time_caching_start)
log.info("Cached %s resource types in %s seconds", len(rsrcs), total_time / 1000.0)
def _clear_caches(self):
log.warn("Clearing caches")
for r in self._resources_to_cache():
self.RR2.clear_cached_resource(r)
for p in self._predicates_to_cache():
self.RR2.clear_cached_predicate(p)
def _lookup_means(self):
"""
return a dict indicating how various related resources will be looked up
The dict is keyed on association type:
PRED.hasAgentInstance -> device type
PRED.hasModel -> model type
PRED.hasAgentDefinition -> agent type
"""
raise NotImplementedError("Extender of class must implement this")
def _augment_dict(self, title, basedict, newitems):
for k, v in newitems.iteritems():
if k in basedict:
prev_v = basedict[k]
# just warn if the new value is different
if v != prev_v:
log.warn("Overwriting %s[%s] of '%s' with '%s'", title, k, prev_v, v)
else:
log.debug("Overwriting %s[%s] with same value already assigned '%s'",
title, k, v)
basedict[k] = v
def _check_associations(self):
assert self.agent_instance_obj
assert self.associated_objects
lookup_means = self._lookup_means()
assert lookup_means
# make sure we've picked up the associations we expect
def check_keys(somekeys):
for k in somekeys:
assert k in lookup_means
assert lookup_means[k] in self.associated_objects
#check_keys([PRED.hasAgentInstance, PRED.hasModel, PRED.hasAgentDefinition])
check_keys([PRED.hasAgentInstance, PRED.hasAgentDefinition])
assert RT.ProcessDefinition in self.associated_objects
def set_agent_instance_object(self, agent_instance_obj):
"""
Set the agent instance object that we'll be interacting with
it may be necessary to set this several times, such as if external operations update the object
"""
assert agent_instance_obj._id
if self.last_id != agent_instance_obj._id:
self.associated_objects = None
self.agent_instance_obj = agent_instance_obj
self.last_id = agent_instance_obj._id
self.generated_config = False
def prepare(self, will_launch=True):
"""
Prepare (validate) an agent for launch, fetching all associated resources
@param will_launch - whether the running status should be checked -- set false if just generating config
"""
assert self.agent_instance_obj
if will_launch:
#if there is an agent pid then assume that a drive is already started
if self.agent_instance_obj.agent_process_id:
raise BadRequest("Agent Instance already running for this device pid: %s" %
str(self.agent_instance_obj.agent_process_id))
# fetch caches just in time
if any([not self.RR2.has_cached_predicate(x) for x in self._predicates_to_cache()]):
self._update_cached_predicates()
if any([not self.RR2.has_cached_resource(x) for x in self._resources_to_cache()]):
self._update_cached_resources()
# validate the associations, then pick things up
self._collect_agent_instance_associations()
self.will_launch = will_launch
return self.generate_config()
def _generate_org_governance_name(self):
log.debug("_generate_org_governance_name for %s", self.agent_instance_obj.name)
log.debug("retrieve the Org governance name to which this agent instance belongs")
try:
org_obj = self.RR2.find_subject(RT.Org, PRED.hasResource, self.agent_instance_obj._id, id_only=False)
return org_obj.org_governance_name
except NotFound:
return ''
except:
raise
def _generate_device_type(self):
log.debug("_generate_device_type for %s", self.agent_instance_obj.name)
return type(self._get_device()).__name__
def _generate_driver_config(self):
log.debug("_generate_driver_config for %s", self.agent_instance_obj.name)
# get default config
driver_config = self.agent_instance_obj.driver_config
agent_obj = self._get_agent()
# Create driver config.
add_driver_config = {
'workdir' : tempfile.gettempdir(),
'dvr_mod' : agent_obj.driver_module,
'dvr_cls' : agent_obj.driver_class
}
self._augment_dict("Agent driver_config", driver_config, add_driver_config)
return driver_config
def _get_param_dict_by_name(self, name):
dict_obj = self.RR2.find_resources_by_name(RT.ParameterDictionary, name)[0]
parameter_contexts = \
self.RR2.find_parameter_contexts_of_parameter_dictionary_using_has_parameter_context(dict_obj._id)
return DatasetManagementService.build_parameter_dictionary(dict_obj, parameter_contexts)
def _meet_in_the_middle(self,dp_id, pdict_id):
# Given a pdict_id and a data_product_id find the stream def in the middle
pdict_stream_defs = self.RR2.find_stream_definition_ids_by_parameter_dictionary_using_has_parameter_dictionary(pdict_id)
stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(dp_id)
result = stream_def_id if stream_def_id in pdict_stream_defs else None
return result
def _generate_stream_config(self):
log.debug("_generate_stream_config for %s", self.agent_instance_obj.name)
dsm = self.clients.dataset_management
psm = self.clients.pubsub_management
agent_obj = self._get_agent()
device_obj = self._get_device()
streams_dict = {}
for stream_cfg in agent_obj.stream_configurations:
#create a stream def for each param dict to match against the existing data products
streams_dict[stream_cfg.stream_name] = {'param_dict_name':stream_cfg.parameter_dictionary_name,
#'stream_def_id':stream_def_id,
'records_per_granule': stream_cfg.records_per_granule,
'granule_publish_rate':stream_cfg.granule_publish_rate,
}
#retrieve the output products
device_id = device_obj._id
data_product_objs = self.RR2.find_data_products_of_instrument_device_using_has_output_product(device_id)
stream_config = {}
for d in data_product_objs:
stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(d._id)
for model_stream_name, stream_info_dict in streams_dict.items():
# read objects from cache to be compared
pdict = self.RR2.find_resource_by_name(RT.ParameterDictionary, stream_info_dict.get('param_dict_name'))
stream_def_id = self._meet_in_the_middle(d._id, pdict._id)
if stream_def_id:
#model_param_dict = self.RR2.find_resources_by_name(RT.ParameterDictionary,
# stream_info_dict.get('param_dict_name'))[0]
#model_param_dict = self._get_param_dict_by_name(stream_info_dict.get('param_dict_name'))
#stream_route = self.RR2.read(product_stream_id).stream_route
product_stream_id = self.RR2.find_stream_id_of_data_product_using_has_stream(d._id)
stream_def = psm.read_stream_definition(stream_def_id)
stream_route = psm.read_stream_route(stream_id=product_stream_id)
from pyon.core.object import IonObjectSerializer
stream_def_dict = IonObjectSerializer().serialize(stream_def)
sdtype = stream_def_dict.pop('type_')
if model_stream_name in stream_config:
log.warn("Overwiting stream_config[%s]", model_stream_name)
stream_config[model_stream_name] = {'routing_key' : stream_route.routing_key,
'stream_id' : product_stream_id,
'stream_definition_ref' : stream_def_id,
'stream_def_dict' : stream_def_dict,
'exchange_point' : stream_route.exchange_point,
'parameter_dictionary' : stream_def.parameter_dictionary,
'records_per_granule' : stream_info_dict.get('records_per_granule'),
'granule_publish_rate' : stream_info_dict.get('granule_publish_rate'),
}
log.debug("Stream config generated")
log.trace("generate_stream_config: %s", str(stream_config) )
return stream_config
def _generate_agent_config(self):
log.debug("_generate_agent_config for %s", self.agent_instance_obj.name)
# should override this
return {}
def _generate_alerts_config(self):
log.debug("_generate_alerts_config for %s", self.agent_instance_obj.name)
# should override this
return self.agent_instance_obj.alerts
def _generate_startup_config(self):
log.debug("_generate_startup_config for %s", self.agent_instance_obj.name)
# should override this
return {}
def _generate_children(self):
log.debug("_generate_children for %s", self.agent_instance_obj.name)
# should override this
return {}
def _generate_skeleton_config_block(self):
log.info("Generating skeleton config block for %s", self.agent_instance_obj.name)
# should override this
agent_config = self.agent_instance_obj.agent_config
# Create agent_ config.
agent_config['instance_name'] = self.agent_instance_obj.name
agent_config['org_governance_name'] = self._generate_org_governance_name()
agent_config['device_type'] = self._generate_device_type()
agent_config['driver_config'] = self._generate_driver_config()
agent_config['stream_config'] = self._generate_stream_config()
agent_config['agent'] = self._generate_agent_config()
agent_config['aparam_alerts_config'] = self._generate_alerts_config()
agent_config['startup_config'] = self._generate_startup_config()
agent_config['children'] = self._generate_children()
log.info("DONE generating skeleton config block for %s", self.agent_instance_obj.name)
return agent_config
def _summarize_children(self, config_dict):
ret = dict([(v['instance_name'], self._summarize_children(v))
for k, v in config_dict["children"].iteritems()])
#agent_config['agent']['resource_id']
return ret
def generate_config(self):
"""
create the generic parts of the configuration including resource_id, egg_uri, and org
"""
if self.generated_config:
log.warn("Generating config again for the same Instance object (%s)", self.agent_instance_obj.name)
self._check_associations()
agent_config = self._generate_skeleton_config_block()
device_obj = self._get_device()
agent_obj = self._get_agent()
log.debug("complement agent_config with resource_id")
if 'agent' not in agent_config:
agent_config['agent'] = {'resource_id': device_obj._id}
elif 'resource_id' not in agent_config.get('agent'):
agent_config['agent']['resource_id'] = device_obj._id
log.debug("add egg URI if available")
if agent_obj.driver_uri:
agent_config['driver_config']['process_type'] = (DriverProcessType.EGG,)
agent_config['driver_config']['dvr_egg'] = agent_obj.driver_uri
else:
agent_config['driver_config']['process_type'] = (DriverProcessType.PYTHON_MODULE,)
if log.isEnabledFor(logging.INFO):
tree = self._summarize_children(agent_config)
log.info("Children of %s are %s", self.agent_instance_obj.name, tree)
self.generated_config = True
return agent_config
def record_launch_parameters(self, agent_config, process_id):
"""
record process id of the launch
"""
log.debug("add the process id and update the resource")
self.agent_instance_obj.agent_config = agent_config
self.agent_instance_obj.agent_process_id = process_id
self.RR2.update(self.agent_instance_obj)
log.debug('completed agent start')
return process_id
def _collect_agent_instance_associations(self):
"""
Collect related resources to this agent instance
Returns a dict of objects necessary to start this instance, keyed on the values of self._lookup_means()
PRED.hasAgentInstance -> device_obj
PRED.hasModel -> model_obj
PRED.hasAgentDefinition -> agent_obj
RT.ProcessDefinition -> process_def_obj
"""
assert self.agent_instance_obj
lookup_means = self._lookup_means()
assert lookup_means
assert PRED.hasAgentInstance in lookup_means
assert PRED.hasModel in lookup_means
assert PRED.hasAgentDefinition in lookup_means
#assert PRED.hasProcessDefinition in lookup_means
lu = lookup_means
ret = {}
log.debug("retrieve the associated device")
device_obj = self.RR2.find_subject(subject_type=lu[PRED.hasAgentInstance],
predicate=PRED.hasAgentInstance,
object=self.agent_instance_obj._id)
ret[lu[PRED.hasAgentInstance]]= device_obj
device_id = device_obj._id
log.debug("%s '%s' connected to %s '%s' (L4-CI-SA-RQ-363)",
lu[PRED.hasAgentInstance],
str(device_id),
type(self.agent_instance_obj).__name__,
str(self.agent_instance_obj._id))
# log.debug("retrieve the model associated with the device")
# model_obj = self.RR2.find_object(subject=device_id,
# predicate=PRED.hasModel,
# object_type=lu[PRED.hasModel])
#
# ret[lu[PRED.hasModel]] = model_obj
# model_id = model_obj
#retrive the stream info for this model
#todo: add stream info to the platofrom model create
# streams_dict = platform_models_objs[0].custom_attributes['streams']
# if not streams_dict:
# raise BadRequest("Device model does not contain stream configuation used in launching the agent. Model: '%s", str(platform_models_objs[0]) )
#TODO: get the agent from the instance not from the model!!!!!!!
log.debug("retrieve the agent associated with the model")
agent_obj = self.RR2.find_object(subject=self.agent_instance_obj._id,
predicate=PRED.hasAgentDefinition,
object_type=lu[PRED.hasAgentDefinition])
ret[lu[PRED.hasAgentDefinition]] = agent_obj
agent_id = agent_obj._id
if not agent_obj.stream_configurations:
raise BadRequest("Agent '%s' does not contain stream configuration used in launching" %
str(agent_obj) )
log.debug("retrieve the process definition associated with this agent")
process_def_obj = self.RR2.find_object(subject=agent_id,
predicate=PRED.hasProcessDefinition,
object_type=RT.ProcessDefinition)
ret[RT.ProcessDefinition] = process_def_obj
#retrieve the output products
data_product_ids = self.RR2.find_objects(device_id, PRED.hasOutputProduct, RT.DataProduct, id_only=True)
if not data_product_ids:
raise NotFound("No output Data Products attached to this Device " + str(device_id))
#retrieve the streams assoc with each defined output product
for product_id in data_product_ids:
self.RR2.find_stream_id_of_data_product_using_has_stream(product_id) # check one stream per product
self.RR2.find_dataset_id_of_data_product_using_has_dataset(product_id) # check one dataset per product
self.associated_objects = ret
def _get_device(self):
self._check_associations()
return self.associated_objects[self._lookup_means()[PRED.hasAgentInstance]]
# def _get_model(self):
# self._check_associations()
# return self.associated_objects[self._lookup_means()[PRED.hasModel]]
def _get_agent(self):
self._check_associations()
return self.associated_objects[self._lookup_means()[PRED.hasAgentDefinition]]
def _get_process_definition(self):
self._check_associations()
return self.associated_objects[RT.ProcessDefinition]
def freeze():
if isinstance(resource_registry_client, EnhancedResourceRegistryClient):
RR2 = resource_registry_client
else:
RR2 = EnhancedResourceRegistryClient(resource_registry_client)
for p in predicate_list:
if not RR2.has_cached_prediate(p):
RR2.cache_predicate(p)
def get_related_resources_partial_fn(predicate_dictionary, resource_whitelist):
"""
This function generates a resource crawler from 2 data structures representing desired crawl behavior
The predicate dictionary is keyed on a predicate type, whose value is a 2-tuple of booleans
the first boolean is whether to crawl subject-object, the second boolean for object-subject
For example: dict([(PRED.hasModel, (False, True)]) would generate a crawler that could find
platforms or instruments with a given model
The resource whitelist is a list of resource types that will be crawled.
The return value of this function is a function that accepts a resource id and returns a list
of associations related (based on crawl behavior)
"""
log.trace("get_related_resources_partial_fn predicate_dict=%s rsrc_whitelist=%s",
predicate_dictionary,
resource_whitelist)
# assertions on data types
assert type({}) == type(predicate_dictionary)
for v in predicate_dictionary.values():
assert type((True, True)) == type(v)
assert type([]) == type(resource_whitelist)
for rt in resource_whitelist:
RR2.cache_resources(rt)
def lookup_fn(resource_id):
"""
return a dict of related resources as dictated by the pred dict and whitelist
- the key is the next resource id to crawl
- the value is the entire association
"""
retval = {}
sto_match = lambda assn: assn.s == resource_id and assn.ot in resource_whitelist
ots_match = lambda assn: assn.o == resource_id and assn.st in resource_whitelist
for p, (search_sto, search_ots) in predicate_dictionary.iteritems():
if search_sto:
for a in RR2.filter_cached_associations(p, sto_match):
log.trace("lookup_fn matched %s object", a.ot)
retval[a.o] = a
if search_ots:
for a in RR2.filter_cached_associations(p, ots_match):
log.trace("lookup_fn matched %s subject", a.st)
retval[a.s] = a
return retval
def get_related_resources_h(accum, input_resource_id, recursion_limit):
"""
This is a recursive helper function that does the work of crawling for related resources
The accum is a tuple: (set of associations that are deemed "Related", set of "seen" resources)
The input resource id is the current resource being crawled
The recursion limit decrements with each recursive call, ending at 0. So -1 for infinity.
The return value is a list of associations
"""
if 0 == recursion_limit:
return accum
if -1000 > recursion_limit:
log.warn("Terminating related resource recursion, hit -1000")
return accum
acc, seen = accum
matches = lookup_fn(input_resource_id)
log.trace("get_related_resources_h got matches %s",
[dict((k, "%s %s %s" % (a.st, a.p, a.ot)) for k, a in matches.iteritems())])
unseen = set(matches.keys()) - seen
seen.add(input_resource_id)
acc = acc | set(matches.values())
#if log.isEnabledFor(logging.TRACE):
# summary = {}
# for a in acc:
# label = "%s %s %s" % (a.st, a.p, a.ot)
# if not label in summary: summary[label] = 0
# summary[label] += 1
# log.trace("acc2 is now %s", ["%s x%d" % (k, v) for k, v in summary.iteritems()])
def looper(acc2, input_rsrc_id):
return get_related_resources_h(acc2, input_rsrc_id, recursion_limit - 1)
h_ret = reduce(looper, unseen, (acc, seen))
#h_ret = reduce(looper, unseen, (acc, seen))
#(h_ret_acc, h_ret_seen) = h_ret
#log.trace("h_ret is %s", ["%s %s %s" % (a.st, a.p, a.ot) for a in h_ret_acc])
return h_ret
def get_related_resources_fn(input_resource_id, recursion_limit=1024):
"""
This is the function that finds related resources.
input_resource_id and recursion_limit are self explanatory
The return value is a list of associations.
"""
retval, _ = get_related_resources_h((set([]), set([])), input_resource_id, recursion_limit)
log.trace("final_ret is %s", ["%s %s %s" % (a.st, a.p, a.ot) for a in retval])
return list(retval)
return get_related_resources_fn # retval of get_related_resources_partial_fn
return get_related_resources_partial_fn # retval of freeze()
class TestEnhancedResourceRegistryClient(PyonTestCase):
def setUp(self):
self.rr = Mock()
self.RR2 = EnhancedResourceRegistryClient(self.rr)
def sample_resource(self):
return any_old(RT.InstrumentDevice)
def test_init(self):
pass
def test_create(self):
"""
test resource creation in normal case
"""
# get objects
good_sample_resource = self.sample_resource()
#configure Mock
self.rr.create.return_value = ('111', 'bla')
self.rr.find_resources.return_value = ([], [])
sample_resource_id = self.RR2.create(good_sample_resource,
RT.InstrumentDevice)
self.rr.create.assert_called_once_with(good_sample_resource)
self.assertEqual(sample_resource_id, '111')
def test_create_bad_wrongtype(self):
"""
test resource creation failure for wrong type
"""
# get objects
bad_sample_resource = any_old(RT.PlatformDevice)
delattr(bad_sample_resource, "name")
#configure Mock
self.rr.create.return_value = ('111', 'bla')
self.rr.find_resources.return_value = ([], [])
self.assertRaises(BadRequest, self.RR2.create, bad_sample_resource,
RT.InstrumentDevice)
def test_create_bad_noname(self):
"""
test resource creation failure for no name
"""
# get objects
bad_sample_resource = self.sample_resource()
delattr(bad_sample_resource, "name")
#configure Mock
self.rr.create.return_value = ('111', 'bla')
self.rr.find_resources.return_value = ([], [])
self.assertRaises(BadRequest, self.RR2.create, bad_sample_resource,
RT.InstrumentDevice)
# def test_create_bad_dupname(self):
# """
# test resource creation failure for duplicate name
# """
# # get objects
#
# bad_sample_resource = self.sample_resource()
# #really, the resource doesn't matter; it's the retval from find that matters
#
# #configure Mock
# self.rr.create.return_value = ('111', 'bla')
# self.rr.find_resources.return_value = ([0], [0])
#
# self.assertRaises(BadRequest, self.RR2.create, bad_sample_resource, RT.InstrumentDevice)
#
def test_read(self):
"""
test resource read (passthru)
"""
# get objects
myret = self.sample_resource()
#configure Mock
self.rr.read.return_value = myret
response = self.RR2.read("111", RT.InstrumentDevice)
self.rr.read.assert_called_once_with("111")
self.assertEqual(response, myret)
#self.assertDictEqual(response.__dict__,
# self.sample_resource().__dict__)
def test_read_bad_wrongtype(self):
"""
test resource read (passthru)
"""
# get objects
myret = self.sample_resource()
#configure Mock
self.rr.read.return_value = myret
self.assertRaises(BadRequest, self.RR2.read, "111", RT.PlatformDevice)
self.rr.read.assert_called_once_with("111")
def test_update(self):
"""
test resource update in normal case
"""
# get objects
good_sample_resource = self.sample_resource()
setattr(good_sample_resource, "_id", "111")
#configure Mock
self.rr.update.return_value = ('111', 'bla')
self.rr.find_resources.return_value = ([], [])
self.RR2.update(good_sample_resource, RT.InstrumentDevice)
self.rr.update.assert_called_once_with(good_sample_resource)
def test_update_bad_wrongtype(self):
"""
test update failure due to duplicate name
"""
# get objects
bad_sample_resource = self.sample_resource()
self.assertRaises(BadRequest, self.RR2.update, bad_sample_resource,
RT.PlatformDevice)
#
# def test_update_bad_dupname(self):
# """
# test update failure due to duplicate name
# """
# # get objects
#
# bad_sample_resource = self.sample_resource()
# setattr(bad_sample_resource, "_id", "111")
#
# self.rr.find_resources.return_value = ([0], [0])
# self.assertRaises(BadRequest, self.RR2.update, bad_sample_resource, RT.InstrumentDevice)
def test_update_bad_noid(self):
"""
test update failure due to duplicate name
"""
# get objects
bad_sample_resource = self.sample_resource()
self.rr.find_resources.return_value = ([0], [0])
self.assertRaises(BadRequest, self.RR2.update, bad_sample_resource,
RT.InstrumentDevice)
def test_retire(self):
"""
test retire
"""
# get objects
myret = self.sample_resource()
#configure Mock
self.rr.read.return_value = myret
self.rr.delete.return_value = None
self.rr.retire.return_value = None
try:
self.RR2.retire("111", RT.InstrumentDevice)
except TypeError as te:
# for logic tests that run into mock trouble
if "'Mock' object is not iterable" != te.message:
raise te
else:
raise SkipTest("Must test this with INT test")
except Exception as e:
raise e
#self.rr.read.assert_called_with("111", "")
self.rr.retire.assert_called_once_with("111")
def test_retire_bad_wrongtype(self):
"""
test resource read (passthru)
"""
# get objects
myret = self.sample_resource()
#configure Mock
self.rr.read.return_value = myret
self.assertRaises(BadRequest, self.RR2.retire, "111",
RT.PlatformDevice)
self.rr.read.assert_called_once_with("111")
def test_pluck_delete(self):
"""
test delete
"""
# get objects
myret = self.sample_resource()
#configure Mock
self.rr.read.return_value = myret
self.rr.delete.return_value = None
self.rr.find_resources.return_value = None
self.rr.find_objects.return_value = (["2"], ["2"])
self.rr.find_subjects.return_value = (["3"], ["3"])
self.RR2.pluck_delete("111", RT.InstrumentDevice)
self.rr.delete.assert_called_once_with("111")
def test_advance_lcs(self):
"""
call RR when the transition ISN'T retire
"""
self.RR2.advance_lcs("111", LCE.PLAN)
self.rr.execute_lifecycle_transition.assert_called_once_with(
resource_id="111", transition_event=LCE.PLAN)
self.RR2.advance_lcs("222", LCE.RETIRE)
self.rr.retire.assert_called_once_with("222")
def test_delete_association(self):
self.rr.get_association.return_value = "111"
self.RR2.delete_association("a", "b", "c")
self.rr.delete_association.assert_called_once_with("111")
def test_delete_all_object_associations(self):
self.rr.find_associations.return_value = ["111"]
self.RR2.delete_object_associations("x")
self.rr.delete_association.assert_called_once_with("111")
def test_delete_all_subject_associations(self):
self.rr.find_associations.return_value = ["111"]
self.RR2.delete_subject_associations("x")
self.rr.delete_association.assert_called_once_with("111")
def test_pluck(self):
self.rr.find_subjects.return_value = (["111"], ["aaa"])
self.rr.find_objects.return_value = (["222"], ["bbb"])
self.RR2.pluck("x")
#self.rr.delete_association.assert_called_with("bbb")
self.rr.delete_association.assert_called_with("aaa")
self.assertEqual(self.rr.delete_association.call_count, 2)
def test_find_objects_using_id(self):
self.tbase_find_objects("x_id")
def test_find_objects_using_ionobj(self):
obj = any_old(RT.InstrumentDevice)
setattr(obj, "_id", "foo_id")
self.tbase_find_objects(obj)
def test_find_objects_using_junk(self):
self.tbase_find_objects(1)
def tbase_find_objects(self, sample_obj):
"""
test all 8 flavors of find objects: return IonObjects/ids, return single/multiple, use predicate/no-predicate
"""
def rst():
self.rr.find_objects.reset_mock()
self.rr.find_objects.return_value = ([], [])
self.assertEqual(0, self.rr.find_subjects.call_count)
def rst1():
self.rr.find_objects.reset_mock()
self.rr.find_objects.return_value = (["x"], ["x"])
self.assertEqual(0, self.rr.find_subjects.call_count)
def rst2():
self.rr.find_objects.reset_mock()
self.rr.find_objects.return_value = (["x", "y"], ["z", "k"])
self.assertEqual(0, self.rr.find_subjects.call_count)
x = sample_obj
xx = x
if hasattr(x, "_id"):
xx = x._id
# find none
rst()
self.RR2.find_instrument_models_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=False)
rst()
self.assertRaises(
NotFound, self.RR2.
find_instrument_model_of_instrument_device_using_has_model, x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=False)
rst()
self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(
x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=True)
rst()
self.assertRaises(
NotFound, self.RR2.
find_instrument_model_id_of_instrument_device_using_has_model, x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=True)
# find one
rst1()
self.RR2.find_instrument_models_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=False)
rst1()
self.RR2.find_instrument_model_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=False)
rst1()
self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(
x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=True)
rst1()
self.RR2.find_instrument_model_id_of_instrument_device_using_has_model(
x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=True)
# find multiples
rst2()
self.RR2.find_instrument_models_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=False)
rst2()
self.assertRaises(
Inconsistent, self.RR2.
find_instrument_model_of_instrument_device_using_has_model, x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=False)
rst2()
self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(
x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=True)
rst2()
self.assertRaises(
Inconsistent, self.RR2.
find_instrument_model_id_of_instrument_device_using_has_model, x)
self.rr.find_objects.assert_called_once_with(
subject=xx,
predicate=PRED.hasModel,
object_type=RT.InstrumentModel,
id_only=True)
# # find using
# rst2()
# self.RR2.find_instrument_models_of_instrument_device_using_has_model(x)
# self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=False)
#
# rst2()
# self.assertRaises(Inconsistent, self.RR2.find_instrument_model_of_instrument_device_using_has_model, x)
# self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=False)
#
# rst2()
# self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(x)
# self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=True)
#
# rst2()
# self.assertRaises(Inconsistent, self.RR2.find_instrument_model_id_of_instrument_device_using_has_model, x)
# self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=True)
def test_find_subjects_using_id(self):
self.tbase_find_subjects("x_id")
def test_find_subjects_using_ionobj(self):
obj = any_old(RT.InstrumentDevice)
setattr(obj, "_id", "foo_id")
self.tbase_find_subjects(obj)
def test_find_subjects_using_junk(self):
self.tbase_find_subjects(1)
def tbase_find_subjects(self, sample_obj):
"""
test all 8 flavors of find subjects: return IonObjects/ids, return single/multiple, use predicate/no-predicate
"""
def rst():
self.rr.find_subjects.reset_mock()
self.rr.find_subjects.return_value = ([], [])
self.assertEqual(0, self.rr.find_objects.call_count)
def rst1():
self.rr.find_subjects.reset_mock()
self.rr.find_subjects.return_value = (["x"], ["x"])
self.assertEqual(0, self.rr.find_objects.call_count)
def rst2():
self.rr.find_subjects.reset_mock()
self.rr.find_subjects.return_value = (["x", "y"], ["z", "k"])
self.assertEqual(0, self.rr.find_objects.call_count)
x = sample_obj
xx = x
if hasattr(x, "_id"):
xx = x._id
# find none
rst()
self.RR2.find_instrument_devices_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=False)
rst()
self.assertRaises(
NotFound, self.RR2.
find_instrument_device_by_instrument_model_using_has_model, x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=False)
rst()
self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(
x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=True)
rst()
self.assertRaises(
NotFound, self.RR2.
find_instrument_device_id_by_instrument_model_using_has_model, x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=True)
# find 1
rst1()
self.RR2.find_instrument_devices_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=False)
rst1()
self.RR2.find_instrument_device_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=False)
rst1()
self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(
x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=True)
rst1()
self.RR2.find_instrument_device_id_by_instrument_model_using_has_model(
x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=True)
# find multiple
rst2()
self.RR2.find_instrument_devices_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=False)
rst2()
self.assertRaises(
Inconsistent, self.RR2.
find_instrument_device_by_instrument_model_using_has_model, x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=False)
rst2()
self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(
x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=True)
rst2()
self.assertRaises(
Inconsistent, self.RR2.
find_instrument_device_id_by_instrument_model_using_has_model, x)
self.rr.find_subjects.assert_called_once_with(
object=xx,
predicate=PRED.hasModel,
subject_type=RT.InstrumentDevice,
id_only=True)
#
# # find using
# rst2()
# self.RR2.find_instrument_devices_by_instrument_model_using_has_model(x)
# self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=False)
#
# rst2()
# self.assertRaises(Inconsistent, self.RR2.find_instrument_device_by_instrument_model_using_has_model, x)
# self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=False)
#
# rst2()
# self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(x)
# self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=True)
#
# rst2()
# self.assertRaises(Inconsistent, self.RR2.find_instrument_device_id_by_instrument_model_using_has_model, x)
# self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=True)
def test_assign_unassign(self):
"""
test all flavors of assign and unassign: with/without predicates
"""
x = "x_id"
y = "y_id"
self.RR2.assign_instrument_model_to_instrument_device_with_has_model(
y, x)
self.rr.create_association.assert_called_once_with(x, PRED.hasModel, y)
self.rr.get_association.return_value = "zzz"
self.RR2.unassign_instrument_model_from_instrument_device_with_has_model(
y, x)
self.rr.delete_association.assert_called_once_with("zzz")
self.rr.create_association.reset_mock()
self.RR2.assign_data_product_to_data_process_with_has_output_product(
y, x)
self.rr.create_association.assert_called_once_with(
x, PRED.hasOutputProduct, y)
self.rr.delete_association.reset_mock()
self.rr.get_association.reset_mock()
self.rr.get_association.return_value = "aaa"
self.RR2.unassign_data_product_from_data_process_with_has_output_product(
y, x)
self.rr.delete_association.assert_called_once_with("aaa")
def test_assign_single_object(self):
x = "x_id"
y = "y_id"
def rst():
self.rr.find_objects.reset_mock()
self.rr.get_association.reset_mock()
rst()
self.rr.find_objects.return_value = ([], [])
self.RR2.assign_one_instrument_model_to_instrument_device_with_has_model(
y, x)
self.rr.create_association.assert_called_once_with(x, PRED.hasModel, y)
rst()
self.rr.find_objects.return_value = (["a", "b"], ["c", "d"])
self.assertRaises(
Inconsistent, self.RR2.
assign_one_instrument_model_to_instrument_device_with_has_model, y,
x)
rst()
self.rr.find_objects.return_value = (["a"], ["b"])
self.rr.get_association.return_value = "yay"
self.RR2.assign_one_instrument_model_to_instrument_device_with_has_model(
y, x)
rst()
self.rr.find_objects.return_value = (["a"], ["b"])
self.rr.get_association.side_effect = NotFound("")
self.assertRaises(
BadRequest, self.RR2.
assign_one_instrument_model_to_instrument_device_with_has_model, y,
x)
def test_assign_single_subject(self):
x = "x_id"
y = "y_id"
def rst():
self.rr.find_subjects.reset_mock()
self.rr.get_association.reset_mock()
rst()
self.rr.find_subjects.return_value = ([], [])
self.RR2.assign_instrument_device_to_one_instrument_site_with_has_device(
y, x)
self.rr.create_association.assert_called_once_with(
x, PRED.hasDevice, y)
rst()
self.rr.find_subjects.return_value = (["a", "b"], ["c", "d"])
self.assertRaises(
Inconsistent, self.RR2.
assign_instrument_device_to_one_instrument_site_with_has_device, y,
x)
rst()
self.rr.find_subjects.return_value = (["a"], ["b"])
self.rr.get_association.return_value = "yay"
self.RR2.assign_instrument_device_to_one_instrument_site_with_has_device(
y, x)
rst()
self.rr.find_subjects.return_value = (["a"], ["b"])
self.rr.get_association.side_effect = NotFound("")
self.assertRaises(
BadRequest, self.RR2.
assign_instrument_device_to_one_instrument_site_with_has_device, y,
x)
def test_bad_dynamics(self):
x = "x_id"
self.RR2.assign_foo_to_bar(x)
self.rr.assign_foo_to_bar.assert_called_once_with(x)
self.assertRaises(
BadRequest, getattr, self.RR2,
"find_instrument_model_of_instrument_device_using_has_site")
self.assertRaises(
BadRequest, getattr, self.RR2,
"find_instrument_model_of_instrument_device_using_has_banana")
#self.assertRaises(BadRequest, getattr, self.RR2, "find_data_product_of_data_process")
self.RR2.find_sensor_model_by_data_product(x)
self.rr.find_sensor_model_by_data_product.assert_called_once_with(x)
def test_cached_predicate_search(self):
d = "d_id"
m = "m_id"
x = "x_id"
good_assn = DotDict(s=d,
st=RT.InstrumentDevice,
p=PRED.hasModel,
o=m,
ot=RT.InstrumentModel)
bad_assn = DotDict(s=d,
st=RT.PlatformDevice,
p=PRED.hasModel,
o=m,
ot=RT.PlatformModel)
self.rr.find_associations.return_value = [good_assn, bad_assn]
self.RR2.cache_predicate(PRED.hasModel)
self.assertTrue(self.RR2.has_cached_predicate(PRED.hasModel))
self.rr.find_associations.assert_called_once_with(
predicate=PRED.hasModel, id_only=False)
# object searches that should return 0, 0, 1 results
results = self.RR2.find_objects(x, PRED.hasModel, RT.InstrumentModel,
True)
self.assertEqual([], results)
results = self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(
x)
self.assertEqual([], results)
results = self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(
d)
self.assertEqual([m], results)
self.assertEqual(0, self.rr.find_objects.call_count)
# subject searches that should return 0, 0, 1 results
results = self.RR2.find_subjects(RT.InstrumentDevice, PRED.hasModel, x,
True)
self.assertEqual([], results)
results = self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(
x)
self.assertEqual([], results)
results = self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(
m)
self.assertEqual([d], results)
self.assertEqual(0, self.rr.find_subjects.call_count)
class RollXBuilder(object):
"""
for rollups and rolldowns
"""
def __init__(self, process=None):
"""
the process should be the "self" of a service instance
"""
assert process
self.process = process
self.RR2 = EnhancedResourceRegistryClient(
self.process.clients.resource_registry)
def get_toplevel_platformsite(self, site_id):
if not self.RR2.has_cached_predicate(PRED.hasSite):
self.RR2.cache_predicate(PRED.hasSite)
parent_ids = self.RR2.find_platform_site_ids_by_platform_site_using_has_site(
site_id)
if 0 == len(parent_ids):
return site_id # assume this to be the top level
return self.get_toplevel_platformsite(parent_ids[0])
def get_parent_network_nodes(self, site_id):
"""
return the parent nodes of this network node, including the given node
"""
if not self.RR2.has_cached_predicate(PRED.hasNetworkParent):
self.RR2.cache_predicate(PRED.hasNetworkParent)
def get_h(acc, some_id):
acc.append(some_id)
parent_ids = self.RR2.find_platform_device_ids_of_platform_device_using_has_network_parent(
site_id)
if 0 == len(parent_ids):
return acc
return get_h(acc, parent_ids[0])
return get_h([], site_id)
def get_toplevel_network_node(self, device_id):
if not self.RR2.has_cached_predicate(PRED.hasNetworkParent):
self.RR2.cache_predicate(PRED.hasNetworkParent)
parent_ids = self.RR2.find_platform_device_ids_of_platform_device_using_has_network_parent(
device_id)
if 0 == len(parent_ids):
# it can only be the network parent if it has this association
if 0 < len(
self.RR2.
find_platform_device_ids_by_platform_device_using_has_network_parent(
device_id)):
return device_id # assume this to be top level
else:
return None
return self.get_toplevel_network_node(parent_ids[0])
def get_site_hierarchy(self, site_id, site_val_fn):
"""
return (child_sites, site_ancestors)
where child_sites is a dict mapping all child site ids to the value of site_val_fn(child_site_id)
and site_ancestors is a dict mapping all site ids to a list of their hasSite children.
"""
if not self.RR2.has_cached_predicate(PRED.hasSite):
self.RR2.cache_predicate(PRED.hasSite)
full_list = [site_id]
acc = {}
def _get_ancestors_h(s_id):
s_child_ids = self.RR2.find_objects(s_id,
PRED.hasSite,
id_only=True)
if s_child_ids:
acc[s_id] = s_child_ids
for scid in s_child_ids:
_get_ancestors_h(scid)
full_list.append(scid)
_get_ancestors_h(site_id)
return dict([(s, site_val_fn(s)) for s in full_list]), acc
def get_network_hierarchy(self, device_id, device_val_fn):
"""
return (child_devices, device_ancestors)
where child_devices is a dict mapping all child device ids to the value of device_val_fn(child_device_id)
and device_ancestors is a dict mapping all device ids to a list of their children as per
"""
if not self.RR2.has_cached_predicate(PRED.hasNetworkParent):
self.RR2.cache_predicate(PRED.hasNetworkParent)
full_list = [device_id]
acc = {}
def _get_ancestors_h(d_id):
d_child_ids = self.RR2.find_subjects(d_id,
PRED.hasNetworkParent,
id_only=True)
if d_child_ids:
acc[d_id] = d_child_ids
for dcid in d_child_ids:
_get_ancestors_h(dcid)
full_list.append(dcid)
_get_ancestors_h(device_id)
return dict([(d, device_val_fn(d)) for d in full_list]), acc
class TestEnhancedResourceRegistryClient(PyonTestCase):
def setUp(self):
self.rr = Mock()
self.RR2 = EnhancedResourceRegistryClient(self.rr)
def sample_resource(self):
return any_old(RT.InstrumentDevice)
def test_init(self):
pass
def test_create(self):
"""
test resource creation in normal case
"""
# get objects
good_sample_resource = self.sample_resource()
#configure Mock
self.rr.create.return_value = ('111', 'bla')
self.rr.find_resources.return_value = ([], [])
sample_resource_id = self.RR2.create(good_sample_resource, RT.InstrumentDevice)
self.rr.create.assert_called_once_with(good_sample_resource)
self.assertEqual(sample_resource_id, '111')
def test_create_bad_wrongtype(self):
"""
test resource creation failure for wrong type
"""
# get objects
bad_sample_resource = any_old(RT.PlatformDevice)
delattr(bad_sample_resource, "name")
#configure Mock
self.rr.create.return_value = ('111', 'bla')
self.rr.find_resources.return_value = ([], [])
self.assertRaises(BadRequest, self.RR2.create, bad_sample_resource, RT.InstrumentDevice)
def test_create_bad_noname(self):
"""
test resource creation failure for no name
"""
# get objects
bad_sample_resource = self.sample_resource()
delattr(bad_sample_resource, "name")
#configure Mock
self.rr.create.return_value = ('111', 'bla')
self.rr.find_resources.return_value = ([], [])
self.assertRaises(BadRequest, self.RR2.create, bad_sample_resource, RT.InstrumentDevice)
# def test_create_bad_dupname(self):
# """
# test resource creation failure for duplicate name
# """
# # get objects
#
# bad_sample_resource = self.sample_resource()
# #really, the resource doesn't matter; it's the retval from find that matters
#
# #configure Mock
# self.rr.create.return_value = ('111', 'bla')
# self.rr.find_resources.return_value = ([0], [0])
#
# self.assertRaises(BadRequest, self.RR2.create, bad_sample_resource, RT.InstrumentDevice)
#
def test_read(self):
"""
test resource read (passthru)
"""
# get objects
myret = self.sample_resource()
#configure Mock
self.rr.read.return_value = myret
response = self.RR2.read("111", RT.InstrumentDevice)
self.rr.read.assert_called_once_with("111")
self.assertEqual(response, myret)
#self.assertDictEqual(response.__dict__,
# self.sample_resource().__dict__)
def test_read_bad_wrongtype(self):
"""
test resource read (passthru)
"""
# get objects
myret = self.sample_resource()
#configure Mock
self.rr.read.return_value = myret
self.assertRaises(BadRequest, self.RR2.read, "111", RT.PlatformDevice)
self.rr.read.assert_called_once_with("111")
def test_update(self):
"""
test resource update in normal case
"""
# get objects
good_sample_resource = self.sample_resource()
setattr(good_sample_resource, "_id", "111")
#configure Mock
self.rr.update.return_value = ('111', 'bla')
self.rr.find_resources.return_value = ([], [])
self.RR2.update(good_sample_resource, RT.InstrumentDevice)
self.rr.update.assert_called_once_with(good_sample_resource)
def test_update_bad_wrongtype(self):
"""
test update failure due to duplicate name
"""
# get objects
bad_sample_resource = self.sample_resource()
self.assertRaises(BadRequest, self.RR2.update, bad_sample_resource, RT.PlatformDevice)
#
# def test_update_bad_dupname(self):
# """
# test update failure due to duplicate name
# """
# # get objects
#
# bad_sample_resource = self.sample_resource()
# setattr(bad_sample_resource, "_id", "111")
#
# self.rr.find_resources.return_value = ([0], [0])
# self.assertRaises(BadRequest, self.RR2.update, bad_sample_resource, RT.InstrumentDevice)
def test_update_bad_noid(self):
"""
test update failure due to duplicate name
"""
# get objects
bad_sample_resource = self.sample_resource()
self.rr.find_resources.return_value = ([0], [0])
self.assertRaises(BadRequest, self.RR2.update, bad_sample_resource, RT.InstrumentDevice)
def test_retire(self):
"""
test retire
"""
# get objects
myret = self.sample_resource()
#configure Mock
self.rr.read.return_value = myret
self.rr.delete.return_value = None
self.rr.retire.return_value = None
try:
self.RR2.retire("111", RT.InstrumentDevice)
except TypeError as te:
# for logic tests that run into mock trouble
if "'Mock' object is not iterable" != te.message:
raise te
else:
raise SkipTest("Must test this with INT test")
except Exception as e:
raise e
#self.rr.read.assert_called_with("111", "")
self.rr.retire.assert_called_once_with("111")
def test_retire_bad_wrongtype(self):
"""
test resource read (passthru)
"""
# get objects
myret = self.sample_resource()
#configure Mock
self.rr.read.return_value = myret
self.assertRaises(BadRequest, self.RR2.retire, "111", RT.PlatformDevice)
self.rr.read.assert_called_once_with("111")
def test_pluck_delete(self):
"""
test delete
"""
# get objects
myret = self.sample_resource()
#configure Mock
self.rr.read.return_value = myret
self.rr.delete.return_value = None
self.rr.find_resources.return_value = None
self.rr.find_objects.return_value = (["2"], ["2"])
self.rr.find_subjects.return_value = (["3"], ["3"])
self.RR2.pluck_delete("111", RT.InstrumentDevice)
self.rr.delete.assert_called_once_with("111")
def test_advance_lcs(self):
"""
call RR when the transition ISN'T retire
"""
self.RR2.advance_lcs("111", LCE.PLAN)
self.rr.execute_lifecycle_transition.assert_called_once_with(resource_id="111", transition_event=LCE.PLAN)
self.RR2.advance_lcs("222", LCE.RETIRE)
self.rr.retire.assert_called_once_with("222")
def test_delete_association(self):
self.rr.get_association.return_value = "111"
self.RR2.delete_association("a", "b", "c")
self.rr.delete_association.assert_called_once_with("111")
def test_delete_all_object_associations(self):
self.rr.find_associations.return_value = ["111"]
self.RR2.delete_object_associations("x")
self.rr.delete_association.assert_called_once_with("111")
def test_delete_all_subject_associations(self):
self.rr.find_associations.return_value = ["111"]
self.RR2.delete_subject_associations("x")
self.rr.delete_association.assert_called_once_with("111")
def test_pluck(self):
self.rr.find_subjects.return_value = (["111"], ["aaa"])
self.rr.find_objects.return_value = (["222"], ["bbb"])
self.RR2.pluck("x")
#self.rr.delete_association.assert_called_with("bbb")
self.rr.delete_association.assert_called_with("aaa")
self.assertEqual(self.rr.delete_association.call_count, 2)
def test_find_objects_using_id(self):
self.tbase_find_objects("x_id")
def test_find_objects_using_ionobj(self):
obj = any_old(RT.InstrumentDevice)
setattr(obj, "_id", "foo_id")
self.tbase_find_objects(obj)
def test_find_objects_using_junk(self):
self.tbase_find_objects(1)
def tbase_find_objects(self, sample_obj):
"""
test all 8 flavors of find objects: return IonObjects/ids, return single/multiple, use predicate/no-predicate
"""
def rst():
self.rr.find_objects.reset_mock()
self.rr.find_objects.return_value = ([], [])
self.assertEqual(0, self.rr.find_subjects.call_count)
def rst1():
self.rr.find_objects.reset_mock()
self.rr.find_objects.return_value = (["x"], ["x"])
self.assertEqual(0, self.rr.find_subjects.call_count)
def rst2():
self.rr.find_objects.reset_mock()
self.rr.find_objects.return_value = (["x", "y"], ["z", "k"])
self.assertEqual(0, self.rr.find_subjects.call_count)
x = sample_obj
xx = x
if hasattr(x, "_id"):
xx = x._id
# find none
rst()
self.RR2.find_instrument_models_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=False)
rst()
self.assertRaises(NotFound, self.RR2.find_instrument_model_of_instrument_device_using_has_model, x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=False)
rst()
self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=True)
rst()
self.assertRaises(NotFound, self.RR2.find_instrument_model_id_of_instrument_device_using_has_model, x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=True)
# find one
rst1()
self.RR2.find_instrument_models_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=False)
rst1()
self.RR2.find_instrument_model_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=False)
rst1()
self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=True)
rst1()
self.RR2.find_instrument_model_id_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=True)
# find multiples
rst2()
self.RR2.find_instrument_models_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=False)
rst2()
self.assertRaises(Inconsistent, self.RR2.find_instrument_model_of_instrument_device_using_has_model, x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=False)
rst2()
self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=True)
rst2()
self.assertRaises(Inconsistent, self.RR2.find_instrument_model_id_of_instrument_device_using_has_model, x)
self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=True)
# # find using
# rst2()
# self.RR2.find_instrument_models_of_instrument_device_using_has_model(x)
# self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=False)
#
# rst2()
# self.assertRaises(Inconsistent, self.RR2.find_instrument_model_of_instrument_device_using_has_model, x)
# self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=False)
#
# rst2()
# self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(x)
# self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=True)
#
# rst2()
# self.assertRaises(Inconsistent, self.RR2.find_instrument_model_id_of_instrument_device_using_has_model, x)
# self.rr.find_objects.assert_called_once_with(subject=xx, predicate=PRED.hasModel, object_type=RT.InstrumentModel, id_only=True)
def test_find_subjects_using_id(self):
self.tbase_find_subjects("x_id")
def test_find_subjects_using_ionobj(self):
obj = any_old(RT.InstrumentDevice)
setattr(obj, "_id", "foo_id")
self.tbase_find_subjects(obj)
def test_find_subjects_using_junk(self):
self.tbase_find_subjects(1)
def tbase_find_subjects(self, sample_obj):
"""
test all 8 flavors of find subjects: return IonObjects/ids, return single/multiple, use predicate/no-predicate
"""
def rst():
self.rr.find_subjects.reset_mock()
self.rr.find_subjects.return_value = ([], [])
self.assertEqual(0, self.rr.find_objects.call_count)
def rst1():
self.rr.find_subjects.reset_mock()
self.rr.find_subjects.return_value = (["x"], ["x"])
self.assertEqual(0, self.rr.find_objects.call_count)
def rst2():
self.rr.find_subjects.reset_mock()
self.rr.find_subjects.return_value = (["x", "y"], ["z", "k"])
self.assertEqual(0, self.rr.find_objects.call_count)
x = sample_obj
xx = x
if hasattr(x, "_id"):
xx = x._id
# find none
rst()
self.RR2.find_instrument_devices_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=False)
rst()
self.assertRaises(NotFound, self.RR2.find_instrument_device_by_instrument_model_using_has_model, x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=False)
rst()
self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=True)
rst()
self.assertRaises(NotFound, self.RR2.find_instrument_device_id_by_instrument_model_using_has_model, x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=True)
# find 1
rst1()
self.RR2.find_instrument_devices_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=False)
rst1()
self.RR2.find_instrument_device_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=False)
rst1()
self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=True)
rst1()
self.RR2.find_instrument_device_id_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=True)
# find multiple
rst2()
self.RR2.find_instrument_devices_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=False)
rst2()
self.assertRaises(Inconsistent, self.RR2.find_instrument_device_by_instrument_model_using_has_model, x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=False)
rst2()
self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=True)
rst2()
self.assertRaises(Inconsistent, self.RR2.find_instrument_device_id_by_instrument_model_using_has_model, x)
self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=True)
#
# # find using
# rst2()
# self.RR2.find_instrument_devices_by_instrument_model_using_has_model(x)
# self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=False)
#
# rst2()
# self.assertRaises(Inconsistent, self.RR2.find_instrument_device_by_instrument_model_using_has_model, x)
# self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=False)
#
# rst2()
# self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(x)
# self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=True)
#
# rst2()
# self.assertRaises(Inconsistent, self.RR2.find_instrument_device_id_by_instrument_model_using_has_model, x)
# self.rr.find_subjects.assert_called_once_with(object=xx, predicate=PRED.hasModel, subject_type=RT.InstrumentDevice, id_only=True)
def test_assign_unassign(self):
"""
test all flavors of assign and unassign: with/without predicates
"""
x = "x_id"
y = "y_id"
self.RR2.assign_instrument_model_to_instrument_device_with_has_model(y, x)
self.rr.create_association.assert_called_once_with(x, PRED.hasModel, y)
self.rr.get_association.return_value = "zzz"
self.RR2.unassign_instrument_model_from_instrument_device_with_has_model(y, x)
self.rr.delete_association.assert_called_once_with("zzz")
self.rr.create_association.reset_mock()
self.RR2.assign_data_product_to_data_process_with_has_output_product(y, x)
self.rr.create_association.assert_called_once_with(x, PRED.hasOutputProduct, y)
self.rr.delete_association.reset_mock()
self.rr.get_association.reset_mock()
self.rr.get_association.return_value = "aaa"
self.RR2.unassign_data_product_from_data_process_with_has_output_product(y, x)
self.rr.delete_association.assert_called_once_with("aaa")
def test_assign_single_object(self):
x = "x_id"
y = "y_id"
def rst():
self.rr.find_objects.reset_mock()
self.rr.get_association.reset_mock()
rst()
self.rr.find_objects.return_value = ([], [])
self.RR2.assign_one_instrument_model_to_instrument_device_with_has_model(y, x)
self.rr.create_association.assert_called_once_with(x, PRED.hasModel, y)
rst()
self.rr.find_objects.return_value = (["a", "b"], ["c", "d"])
self.assertRaises(Inconsistent, self.RR2.assign_one_instrument_model_to_instrument_device_with_has_model, y, x)
rst()
self.rr.find_objects.return_value = (["a"], ["b"])
self.rr.get_association.return_value = "yay"
self.RR2.assign_one_instrument_model_to_instrument_device_with_has_model(y, x)
rst()
self.rr.find_objects.return_value = (["a"], ["b"])
self.rr.get_association.side_effect = NotFound("")
self.assertRaises(BadRequest, self.RR2.assign_one_instrument_model_to_instrument_device_with_has_model, y, x)
def test_assign_single_subject(self):
x = "x_id"
y = "y_id"
def rst():
self.rr.find_subjects.reset_mock()
self.rr.get_association.reset_mock()
rst()
self.rr.find_subjects.return_value = ([], [])
self.RR2.assign_instrument_device_to_one_instrument_site_with_has_device(y, x)
self.rr.create_association.assert_called_once_with(x, PRED.hasDevice, y)
rst()
self.rr.find_subjects.return_value = (["a", "b"], ["c", "d"])
self.assertRaises(Inconsistent, self.RR2.assign_instrument_device_to_one_instrument_site_with_has_device, y, x)
rst()
self.rr.find_subjects.return_value = (["a"], ["b"])
self.rr.get_association.return_value = "yay"
self.RR2.assign_instrument_device_to_one_instrument_site_with_has_device(y, x)
rst()
self.rr.find_subjects.return_value = (["a"], ["b"])
self.rr.get_association.side_effect = NotFound("")
self.assertRaises(BadRequest, self.RR2.assign_instrument_device_to_one_instrument_site_with_has_device, y, x)
def test_bad_dynamics(self):
x = "x_id"
self.RR2.assign_foo_to_bar(x)
self.rr.assign_foo_to_bar.assert_called_once_with(x)
self.assertRaises(BadRequest, getattr, self.RR2, "find_instrument_model_of_instrument_device_using_has_site")
self.assertRaises(BadRequest, getattr, self.RR2, "find_instrument_model_of_instrument_device_using_has_banana")
#self.assertRaises(BadRequest, getattr, self.RR2, "find_data_product_of_data_process")
self.RR2.find_sensor_model_by_data_product(x)
self.rr.find_sensor_model_by_data_product.assert_called_once_with(x)
def test_cached_predicate_search(self):
d = "d_id"
m = "m_id"
x = "x_id"
good_assn = DotDict(s=d, st=RT.InstrumentDevice, p=PRED.hasModel, o=m, ot=RT.InstrumentModel)
bad_assn = DotDict(s=d, st=RT.PlatformDevice, p=PRED.hasModel, o=m, ot=RT.PlatformModel)
self.rr.find_associations.return_value = [good_assn, bad_assn]
self.RR2.cache_predicate(PRED.hasModel)
self.assertTrue(self.RR2.has_cached_predicate(PRED.hasModel))
self.rr.find_associations.assert_called_once_with(predicate=PRED.hasModel, id_only=False)
# object searches that should return 0, 0, 1 results
results = self.RR2.find_objects(x, PRED.hasModel, RT.InstrumentModel, True)
self.assertEqual([], results)
results = self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(x)
self.assertEqual([], results)
results = self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(d)
self.assertEqual([m], results)
self.assertEqual(0, self.rr.find_objects.call_count)
# subject searches that should return 0, 0, 1 results
results = self.RR2.find_subjects(RT.InstrumentDevice, PRED.hasModel, x, True)
self.assertEqual([], results)
results = self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(x)
self.assertEqual([], results)
results = self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(m)
self.assertEqual([d], results)
self.assertEqual(0, self.rr.find_subjects.call_count)
class AgentConfigurationBuilder(object):
def __init__(self, clients, RR2=None):
self.clients = clients
self.RR2 = RR2
if self.RR2 is None:
log.warn("Creating new RR2")
self.RR2 = EnhancedResourceRegistryClient(
self.clients.resource_registry)
if not isinstance(self.RR2, EnhancedResourceRegistryClient):
raise AssertionError(
"Type of self.RR2 is %s not %s" %
(type(self.RR2), type(EnhancedResourceRegistryClient)))
self.agent_instance_obj = None
self.associated_objects = None
self.last_id = None
self.will_launch = False
self.generated_config = False
def _predicates_to_cache(self):
return [
PRED.hasOutputProduct,
#PRED.hasStream,
#PRED.hasStreamDefinition,
PRED.hasAgentInstance,
PRED.hasAgentDefinition,
PRED.hasDataset,
PRED.hasDevice,
PRED.hasNetworkParent,
#PRED.hasParameterContext,
]
def _resources_to_cache(self):
return [ #RT.StreamDefinition,
RT.ParameterDictionary,
#RT.ParameterContext,
]
def _update_cached_predicates(self):
# cache some predicates for in-memory lookups
preds = self._predicates_to_cache()
log.debug("updating cached predicates: %s" % preds)
time_caching_start = get_ion_ts()
for pred in preds:
log.debug(" - %s", pred)
self.RR2.cache_predicate(pred)
time_caching_stop = get_ion_ts()
total_time = int(time_caching_stop) - int(time_caching_start)
log.info("Cached %s predicates in %s seconds", len(preds),
total_time / 1000.0)
def _update_cached_resources(self):
# cache some resources for in-memory lookups
rsrcs = self._resources_to_cache()
log.debug("updating cached resources: %s" % rsrcs)
time_caching_start = get_ion_ts()
for r in rsrcs:
log.debug(" - %s", r)
self.RR2.cache_resources(r)
time_caching_stop = get_ion_ts()
total_time = int(time_caching_stop) - int(time_caching_start)
log.info("Cached %s resource types in %s seconds", len(rsrcs),
total_time / 1000.0)
def _clear_caches(self):
log.warn("Clearing caches")
for r in self._resources_to_cache():
self.RR2.clear_cached_resource(r)
for p in self._predicates_to_cache():
self.RR2.clear_cached_predicate(p)
def _lookup_means(self):
"""
return a dict indicating how various related resources will be looked up
The dict is keyed on association type:
PRED.hasAgentInstance -> device type
PRED.hasModel -> model type
PRED.hasAgentDefinition -> agent type
"""
raise NotImplementedError("Extender of class must implement this")
def _augment_dict(self, title, basedict, newitems):
# TODO: pyon.util.containers has dict_merge for this purpose (without logs)
for k, v in newitems.iteritems():
if k in basedict:
prev_v = basedict[k]
# just warn if the new value is different
if v != prev_v:
log.warn("Overwriting %s[%s] of '%s' with '%s'", title, k,
prev_v, v)
else:
log.debug(
"Overwriting %s[%s] with same value already assigned '%s'",
title, k, v)
basedict[k] = v
def _check_associations(self):
assert self.agent_instance_obj
assert self.associated_objects
lookup_means = self._lookup_means()
assert lookup_means
# make sure we've picked up the associations we expect
def check_keys(somekeys):
for k in somekeys:
assert k in lookup_means
assert lookup_means[k] in self.associated_objects
#check_keys([PRED.hasAgentInstance, PRED.hasModel, PRED.hasAgentDefinition])
check_keys([PRED.hasAgentInstance, PRED.hasAgentDefinition])
assert RT.ProcessDefinition in self.associated_objects
def set_agent_instance_object(self, agent_instance_obj):
"""
Set the agent instance object that we'll be interacting with
it may be necessary to set this several times, such as if external operations update the object
"""
assert agent_instance_obj._id
if self.last_id != agent_instance_obj._id:
self.associated_objects = None
self.agent_instance_obj = agent_instance_obj
self.last_id = agent_instance_obj._id
self.generated_config = False
def prepare(self, will_launch=True):
"""
Prepare (validate) an agent for launch, fetching all associated resources
@param will_launch - whether the running status should be checked -- set false if just generating config
"""
assert self.agent_instance_obj
# fetch caches just in time
if any([
not self.RR2.has_cached_predicate(x)
for x in self._predicates_to_cache()
]):
self._update_cached_predicates()
if any([
not self.RR2.has_cached_resource(x)
for x in self._resources_to_cache()
]):
self._update_cached_resources()
# validate the associations, then pick things up
self._collect_agent_instance_associations()
if will_launch:
# if there is an agent pid then assume that a drive is already started
agent_process_id = ResourceAgentClient._get_agent_process_id(
self._get_device()._id)
if agent_process_id:
raise BadRequest(
"Agent Instance already running for this device pid: %s" %
str(agent_process_id))
self.will_launch = will_launch
return self.generate_config()
def _generate_org_governance_name(self):
log.debug("_generate_org_governance_name for %s",
self.agent_instance_obj.name)
log.debug(
"retrieve the Org governance name to which this agent instance belongs"
)
try:
org_obj = self.RR2.find_subject(RT.Org,
PRED.hasResource,
self.agent_instance_obj._id,
id_only=False)
return org_obj.org_governance_name
except NotFound:
return ''
except:
raise
def _generate_device_type(self):
log.debug("_generate_device_type for %s", self.agent_instance_obj.name)
return type(self._get_device()).__name__
def _generate_driver_config(self):
log.debug("_generate_driver_config for %s",
self.agent_instance_obj.name)
# get default config
driver_config = self.agent_instance_obj.driver_config
agent_obj = self._get_agent()
# Create driver config.
add_driver_config = {
'workdir': tempfile.gettempdir(),
'dvr_mod': agent_obj.driver_module,
'dvr_cls': agent_obj.driver_class
}
self._augment_dict("Agent driver_config", driver_config,
add_driver_config)
return driver_config
def _get_param_dict_by_name(self, name):
dict_obj = self.RR2.find_resources_by_name(RT.ParameterDictionary,
name)[0]
parameter_contexts = \
self.RR2.find_parameter_contexts_of_parameter_dictionary_using_has_parameter_context(dict_obj._id)
return DatasetManagementService.build_parameter_dictionary(
dict_obj, parameter_contexts)
def _find_streamdef_for_dp_and_pdict(self, dp_id, pdict_id):
# Given a pdict_id and a data_product_id find the stream def in the middle
pdict_stream_defs = self.RR2.find_stream_definition_ids_by_parameter_dictionary_using_has_parameter_dictionary(
pdict_id)
stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(
dp_id)
result = stream_def_id if stream_def_id in pdict_stream_defs else None
return result
def _generate_stream_config(self):
log.debug("_generate_stream_config for %s",
self.agent_instance_obj.name)
dsm = self.clients.dataset_management
psm = self.clients.pubsub_management
agent_obj = self._get_agent()
device_obj = self._get_device()
streams_dict = {}
for stream_cfg in agent_obj.stream_configurations:
#create a stream def for each param dict to match against the existing data products
streams_dict[stream_cfg.stream_name] = {
'param_dict_name': stream_cfg.parameter_dictionary_name
}
#retrieve the output products
# TODO: What about platforms? other things?
device_id = device_obj._id
data_product_objs = self.RR2.find_data_products_of_instrument_device_using_has_output_product(
device_id)
stream_config = {}
for d in data_product_objs:
stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(
d._id)
for stream_name, stream_info_dict in streams_dict.items():
# read objects from cache to be compared
pdict = self.RR2.find_resource_by_name(
RT.ParameterDictionary,
stream_info_dict.get('param_dict_name'))
stream_def_id = self._find_streamdef_for_dp_and_pdict(
d._id, pdict._id)
if stream_def_id:
#model_param_dict = self.RR2.find_resources_by_name(RT.ParameterDictionary,
# stream_info_dict.get('param_dict_name'))[0]
#model_param_dict = self._get_param_dict_by_name(stream_info_dict.get('param_dict_name'))
#stream_route = self.RR2.read(product_stream_id).stream_route
product_stream_id = self.RR2.find_stream_id_of_data_product_using_has_stream(
d._id)
stream_def = psm.read_stream_definition(stream_def_id)
stream_route = psm.read_stream_route(
stream_id=product_stream_id)
from pyon.core.object import IonObjectSerializer
stream_def_dict = IonObjectSerializer().serialize(
stream_def)
stream_def_dict.pop('type_')
if stream_name in stream_config:
log.warn("Overwriting stream_config[%s]", stream_name)
stream_config[stream_name] = {
'routing_key': stream_route.
routing_key, # TODO: Serialize stream_route together
'stream_id': product_stream_id,
'stream_definition_ref': stream_def_id,
'stream_def_dict': stream_def_dict,
'exchange_point': stream_route.exchange_point,
# TODO: This is redundant and very large - the param dict is in the stream_def_dict ???
'parameter_dictionary':
stream_def.parameter_dictionary,
}
log.debug("Stream config generated")
log.trace("generate_stream_config: %s", stream_config)
return stream_config
def _generate_agent_config(self):
log.debug("_generate_agent_config for %s",
self.agent_instance_obj.name)
# should override this
return {}
def _generate_alerts_config(self):
log.debug("_generate_alerts_config for %s",
self.agent_instance_obj.name)
# should override this
return self.agent_instance_obj.alerts
def _generate_startup_config(self):
log.debug("_generate_startup_config for %s",
self.agent_instance_obj.name)
# should override this
return {}
def _generate_children(self):
log.debug("_generate_children for %s", self.agent_instance_obj.name)
# should override this
return {}
def _generate_skeleton_config_block(self):
log.info("Generating skeleton config block for %s",
self.agent_instance_obj.name)
# merge the agent config into the default config
agent_config = dict_merge(self._get_agent().agent_default_config,
self.agent_instance_obj.agent_config, True)
# Create agent_config.
agent_config['instance_name'] = self.agent_instance_obj.name
agent_config[
'org_governance_name'] = self._generate_org_governance_name()
agent_config['device_type'] = self._generate_device_type()
agent_config['driver_config'] = self._generate_driver_config()
agent_config['stream_config'] = self._generate_stream_config()
agent_config['agent'] = self._generate_agent_config()
agent_config['aparam_alerts_config'] = self._generate_alerts_config()
agent_config['startup_config'] = self._generate_startup_config()
agent_config['children'] = self._generate_children()
log.info("DONE generating skeleton config block for %s",
self.agent_instance_obj.name)
return agent_config
def _summarize_children(self, config_dict):
ret = dict([(v['instance_name'], self._summarize_children(v))
for k, v in config_dict["children"].iteritems()])
#agent_config['agent']['resource_id']
return ret
def generate_config(self):
"""
create the generic parts of the configuration including resource_id, egg_uri, and org
"""
if self.generated_config:
log.warn(
"Generating config again for the same Instance object (%s)",
self.agent_instance_obj.name)
self._check_associations()
agent_config = self._generate_skeleton_config_block()
device_obj = self._get_device()
agent_obj = self._get_agent()
log.debug("complement agent_config with resource_id")
if 'agent' not in agent_config:
agent_config['agent'] = {'resource_id': device_obj._id}
elif 'resource_id' not in agent_config.get('agent'):
agent_config['agent']['resource_id'] = device_obj._id
log.debug("add egg URI if available")
if agent_obj.driver_uri:
agent_config['driver_config']['process_type'] = (
DriverProcessType.EGG, )
agent_config['driver_config']['dvr_egg'] = agent_obj.driver_uri
else:
agent_config['driver_config']['process_type'] = (
DriverProcessType.PYTHON_MODULE, )
if log.isEnabledFor(logging.INFO):
tree = self._summarize_children(agent_config)
log.info("Children of %s are %s", self.agent_instance_obj.name,
tree)
self.generated_config = True
return agent_config
def record_launch_parameters(self, agent_config):
"""
record process id of the launch
"""
log.debug(
"add the process id, the generated config, and update the resource"
)
self.agent_instance_obj.agent_config = agent_config
self.agent_instance_obj.agent_spawn_config = agent_config
self.RR2.update(self.agent_instance_obj)
log.debug('completed agent start')
def _collect_agent_instance_associations(self):
"""
Collect related resources to this agent instance
Returns a dict of objects necessary to start this instance, keyed on the values of self._lookup_means()
PRED.hasAgentInstance -> device_obj
PRED.hasModel -> model_obj
PRED.hasAgentDefinition -> agent_obj
RT.ProcessDefinition -> process_def_obj
"""
assert self.agent_instance_obj
lookup_means = self._lookup_means()
assert lookup_means
assert PRED.hasAgentInstance in lookup_means
assert PRED.hasModel in lookup_means
assert PRED.hasAgentDefinition in lookup_means
#assert PRED.hasProcessDefinition in lookup_means
lu = lookup_means
ret = {}
log.debug("retrieve the associated device")
device_obj = self.RR2.find_subject(
subject_type=lu[PRED.hasAgentInstance],
predicate=PRED.hasAgentInstance,
object=self.agent_instance_obj._id)
ret[lu[PRED.hasAgentInstance]] = device_obj
device_id = device_obj._id
log.debug("%s '%s' connected to %s '%s' (L4-CI-SA-RQ-363)",
lu[PRED.hasAgentInstance], str(device_id),
type(self.agent_instance_obj).__name__,
str(self.agent_instance_obj._id))
# log.debug("retrieve the model associated with the device")
# model_obj = self.RR2.find_object(subject=device_id,
# predicate=PRED.hasModel,
# object_type=lu[PRED.hasModel])
#
# ret[lu[PRED.hasModel]] = model_obj
# model_id = model_obj
#retrive the stream info for this model
#todo: add stream info to the platform model create
# streams_dict = platform_models_objs[0].custom_attributes['streams']
# if not streams_dict:
# raise BadRequest("Device model does not contain stream configuation used in launching the agent. Model: '%s", str(platform_models_objs[0]) )
#TODO: get the agent from the instance not from the model!!!!!!!
log.debug("retrieve the agent associated with the model")
agent_obj = self.RR2.find_object(
subject=self.agent_instance_obj._id,
predicate=PRED.hasAgentDefinition,
object_type=lu[PRED.hasAgentDefinition])
ret[lu[PRED.hasAgentDefinition]] = agent_obj
agent_id = agent_obj._id
if not agent_obj.stream_configurations:
raise BadRequest(
"Agent '%s' does not contain stream configuration used in launching"
% str(agent_obj))
log.debug("retrieve the process definition associated with this agent")
process_def_obj = self.RR2.find_object(
subject=agent_id,
predicate=PRED.hasProcessDefinition,
object_type=RT.ProcessDefinition)
ret[RT.ProcessDefinition] = process_def_obj
#retrieve the output products
data_product_objs = self.RR2.find_objects(device_id,
PRED.hasOutputProduct,
RT.DataProduct,
id_only=False)
if not data_product_objs:
raise NotFound("No output Data Products attached to this Device " +
str(device_id))
#retrieve the streams assoc with each defined output product
for data_product_obj in data_product_objs:
product_id = data_product_obj._id
try:
self.RR2.find_stream_id_of_data_product_using_has_stream(
product_id) # check one stream per product
except NotFound:
errmsg = "Device '%s' (%s) has data products %s. Data product '%s' (%s) has no stream ID." % \
(device_obj.name,
device_obj._id,
[dp._id for dp in data_product_objs],
data_product_obj.name,
product_id)
raise NotFound(errmsg)
# some products may not be persisted
try:
# check one dataset per product
self.RR2.find_dataset_id_of_data_product_using_has_dataset(
product_id)
except NotFound:
log.warn(
"Data product '%s' of device %s ('%s') does not appear to be persisted -- no dataset",
product_id, device_obj.name, device_obj._id)
self.associated_objects = ret
def _get_device(self):
self._check_associations()
return self.associated_objects[self._lookup_means()[
PRED.hasAgentInstance]]
# def _get_model(self):
# self._check_associations()
# return self.associated_objects[self._lookup_means()[PRED.hasModel]]
def _get_agent(self):
self._check_associations()
return self.associated_objects[self._lookup_means()[
PRED.hasAgentDefinition]]
def _get_process_definition(self):
self._check_associations()
return self.associated_objects[RT.ProcessDefinition]
class RollXBuilder(object):
"""
for rollups and rolldowns
"""
def __init__(self, process=None):
"""
the process should be the "self" of a service instance
"""
assert process
self.process = process
self.RR2 = EnhancedResourceRegistryClient(self.process.clients.resource_registry)
def get_toplevel_platformsite(self, site_id):
if not self.RR2.has_cached_predicate(PRED.hasSite):
self.RR2.cache_predicate(PRED.hasSite)
parent_ids = self.RR2.find_platform_site_ids_by_platform_site_using_has_site(site_id)
if 0 == len(parent_ids):
return site_id # assume this to be the top level
return self.get_toplevel_platformsite(parent_ids[0])
def get_parent_network_nodes(self, site_id):
"""
return the parent nodes of this network node, including the given node
"""
if not self.RR2.has_cached_predicate(PRED.hasNetworkParent):
self.RR2.cache_predicate(PRED.hasNetworkParent)
def get_h(acc, some_id):
acc.append(some_id)
parent_ids = self.RR2.find_platform_device_ids_of_platform_device_using_has_network_parent(site_id)
if 0 == len(parent_ids):
return acc
return get_h(acc, parent_ids[0])
return get_h([], site_id)
def get_toplevel_network_node(self, device_id):
if not self.RR2.has_cached_predicate(PRED.hasNetworkParent):
self.RR2.cache_predicate(PRED.hasNetworkParent)
parent_ids = self.RR2.find_platform_device_ids_of_platform_device_using_has_network_parent(device_id)
if 0 == len(parent_ids):
# it can only be the network parent if it has this association
if 0 < len(self.RR2.find_platform_device_ids_by_platform_device_using_has_network_parent(device_id)):
return device_id # assume this to be top level
else:
return None
return self.get_toplevel_network_node(parent_ids[0])
def get_site_hierarchy(self, site_id, site_val_fn):
"""
return (child_sites, site_ancestors)
where child_sites is a dict mapping all child site ids to the value of site_val_fn(child_site_id)
and site_ancestors is a dict mapping all site ids to a list of their hasSite children.
"""
if not self.RR2.has_cached_predicate(PRED.hasSite):
self.RR2.cache_predicate(PRED.hasSite)
full_list = [site_id]
acc = {}
def _get_ancestors_h(s_id):
s_child_ids = self.RR2.find_objects(s_id, PRED.hasSite, id_only=True)
if s_child_ids:
acc[s_id] = s_child_ids
for scid in s_child_ids:
_get_ancestors_h(scid)
full_list.append(scid)
_get_ancestors_h(site_id)
return dict([(s, site_val_fn(s)) for s in full_list]), acc
def get_network_hierarchy(self, device_id, device_val_fn):
"""
return (child_devices, device_ancestors)
where child_devices is a dict mapping all child device ids to the value of device_val_fn(child_device_id)
and device_ancestors is a dict mapping all device ids to a list of their children as per
"""
if not self.RR2.has_cached_predicate(PRED.hasNetworkParent):
self.RR2.cache_predicate(PRED.hasNetworkParent)
full_list = [device_id]
acc = {}
def _get_ancestors_h(d_id):
d_child_ids = self.RR2.find_subjects(d_id, PRED.hasNetworkParent, id_only=True)
if d_child_ids:
acc[d_id] = d_child_ids
for dcid in d_child_ids:
_get_ancestors_h(dcid)
full_list.append(dcid)
_get_ancestors_h(device_id)
return dict([(d, device_val_fn(d)) for d in full_list]), acc
def _get_site_extension(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''):
"""Returns an InstrumentDeviceExtension object containing additional related information
@param site_id str
@param ext_associations dict
@param ext_exclude list
@retval observatory ObservatoryExtension
@throws BadRequest A parameter is missing
@throws NotFound An object with the specified observatory_id does not exist
"""
if not site_id:
raise BadRequest("The site_id parameter is empty")
extended_resource_handler = ExtendedResourceContainer(self)
extended_site = extended_resource_handler.create_extended_resource_container(
extended_resource_type=OT.SiteExtension,
resource_id=site_id,
computed_resource_type=OT.SiteComputedAttributes,
ext_associations=ext_associations,
ext_exclude=ext_exclude,
user_id=user_id)
RR2 = EnhancedResourceRegistryClient(self.RR)
RR2.cache_predicate(PRED.hasModel)
# Get status of Site instruments.
a, b = self._get_instrument_states(extended_site.instrument_devices)
extended_site.instruments_operational, extended_site.instruments_not_operational = a, b
# lookup all hasModel predicates
# lookup is a 2d associative array of [subject type][subject id] -> object id
lookup = dict([(rt, {}) for rt in [RT.InstrumentDevice, RT.PlatformDevice]])
for a in RR2.filter_cached_associations(PRED.hasModel, lambda assn: assn.st in lookup):
lookup[a.st][a.s] = a.o
def retrieve_model_objs(rsrc_list, object_type):
# rsrc_list is devices that need models looked up. object_type is the resource type (a device)
# not all devices have models (represented as None), which kills read_mult. so, extract the models ids,
# look up all the model ids, then create the proper output
model_list = [lookup[object_type].get(r._id) for r in rsrc_list]
model_uniq = list(set([m for m in model_list if m is not None]))
model_objs = self.RR2.read_mult(model_uniq)
model_dict = dict(zip(model_uniq, model_objs))
return [model_dict.get(m) for m in model_list]
extended_site.instrument_models = retrieve_model_objs(extended_site.instrument_devices, RT.InstrumentDevice)
extended_site.platform_models = retrieve_model_objs(extended_site.platform_devices, RT.PlatformDevice)
# Status computation
extended_site.computed.instrument_status = [AgentStatusBuilder.get_aggregate_status_of_device(idev._id, "aggstatus")
for idev in extended_site.instrument_devices]
extended_site.computed.platform_status = [AgentStatusBuilder.get_aggregate_status_of_device(pdev._id, "aggstatus")
for pdev in extended_site.platform_devices]
# AgentStatusBuilder.add_device_aggregate_status_to_resource_extension(device_id,
# 'aggstatus',
# extended_site)
def status_unknown():
return ComputedIntValue(status=ComputedValueAvailability.PROVIDED, value=StatusType.STATUS_UNKNOWN)
extended_site.computed.communications_status_roll_up = status_unknown()
extended_site.computed.power_status_roll_up = status_unknown()
extended_site.computed.data_status_roll_up = status_unknown()
extended_site.computed.location_status_roll_up = status_unknown()
extended_site.computed.aggregated_status = status_unknown()
extended_site.computed.site_status = [StatusType.STATUS_UNKNOWN] * len(extended_site.sites)
return extended_site, RR2
def freeze():
if isinstance(resource_registry_client,
EnhancedResourceRegistryClient):
RR2 = resource_registry_client
else:
RR2 = EnhancedResourceRegistryClient(resource_registry_client)
for p in predicate_list:
if not RR2.has_cached_predicate(p):
RR2.cache_predicate(p)
def get_related_resources_partial_fn(predicate_dictionary,
resource_whitelist):
"""
This function generates a resource crawler from 2 data structures representing desired crawl behavior
The predicate dictionary is keyed on a predicate type, whose value is a 2-tuple of booleans
the first boolean is whether to crawl subject-object, the second boolean for object-subject
For example: dict([(PRED.hasModel, (False, True)]) would generate a crawler that could find
platforms or instruments with a given model
The resource whitelist is a list of resource types that will be crawled.
The return value of this function is a function that accepts a resource id and returns a list
of associations related (based on crawl behavior)
"""
log.trace(
"get_related_resources_partial_fn predicate_dict=%s rsrc_whitelist=%s",
predicate_dictionary, resource_whitelist)
# assertions on data types
assert type({}) == type(predicate_dictionary)
for v in predicate_dictionary.values():
assert type((True, True)) == type(v)
assert type([]) == type(resource_whitelist)
for rt in resource_whitelist:
RR2.cache_resources(rt)
def lookup_fn(resource_id):
"""
return a dict of related resources as dictated by the pred dict and whitelist
- the key is the next resource id to crawl
- the value is the entire association
"""
retval = {}
sto_match = lambda assn: assn.s == resource_id and assn.ot in resource_whitelist
ots_match = lambda assn: assn.o == resource_id and assn.st in resource_whitelist
for p, (search_sto,
search_ots) in predicate_dictionary.iteritems():
if search_sto:
for a in RR2.filter_cached_associations(
p, sto_match):
log.trace("lookup_fn matched %s object", a.ot)
retval[a.o] = a
if search_ots:
for a in RR2.filter_cached_associations(
p, ots_match):
log.trace("lookup_fn matched %s subject", a.st)
retval[a.s] = a
return retval
def get_related_resources_h(accum, input_resource_id,
recursion_limit):
"""
This is a recursive helper function that does the work of crawling for related resources
The accum is a tuple: (set of associations that are deemed "Related", set of "seen" resources)
The input resource id is the current resource being crawled
The recursion limit decrements with each recursive call, ending at 0. So -1 for infinity.
The return value is a list of associations
"""
if 0 == recursion_limit:
return accum
if -1000 > recursion_limit:
log.warn(
"Terminating related resource recursion, hit -1000"
)
return accum
acc, seen = accum
matches = lookup_fn(input_resource_id)
log.trace("get_related_resources_h got matches %s", [
dict((k, "%s %s %s" % (a.st, a.p, a.ot))
for k, a in matches.iteritems())
])
unseen = set(matches.keys()) - seen
seen.add(input_resource_id)
acc = acc | set(matches.values())
#if log.isEnabledFor(logging.TRACE):
# summary = {}
# for a in acc:
# label = "%s %s %s" % (a.st, a.p, a.ot)
# if not label in summary: summary[label] = 0
# summary[label] += 1
# log.trace("acc2 is now %s", ["%s x%d" % (k, v) for k, v in summary.iteritems()])
def looper(acc2, input_rsrc_id):
return get_related_resources_h(acc2, input_rsrc_id,
recursion_limit - 1)
h_ret = reduce(looper, unseen, (acc, seen))
#h_ret = reduce(looper, unseen, (acc, seen))
#(h_ret_acc, h_ret_seen) = h_ret
#log.trace("h_ret is %s", ["%s %s %s" % (a.st, a.p, a.ot) for a in h_ret_acc])
return h_ret
def get_related_resources_fn(input_resource_id,
recursion_limit=1024):
"""
This is the function that finds related resources.
input_resource_id and recursion_limit are self explanatory
The return value is a list of associations.
"""
retval, _ = get_related_resources_h(
(set([]), set([])), input_resource_id, recursion_limit)
log.trace("final_ret is %s",
["%s %s %s" % (a.st, a.p, a.ot) for a in retval])
return list(retval)
return get_related_resources_fn # retval of get_related_resources_partial_fn
return get_related_resources_partial_fn # retval of freeze()
