def _find_existing_relationship(self, site_id, device_id, site_type=None, device_type=None):
# look for an existing relationship between the site_id and another device.
# if this site/device pair already exists, we leave it alone
assert(type("") == type(site_id) == type(device_id))
log.debug("checking %s/%s pair for deployment", site_type, device_type)
#return a pair that should be REMOVED, or None
if site_type is None and site_id in self.site_resources:
site_type = self.site_resources[site_id].type_
if device_type is None and device_id in self.device_resources:
device_type = self.device_resources[device_id].type_
log.debug("checking existing %s hasDevice %s links", site_type, device_type)
ret_remove = None
ret_ignore = None
try:
found_device_id = self.enhanced_rr.find_object(site_id, PRED.hasDevice, device_type, True)
if found_device_id == device_id:
ret_ignore = (site_id, device_id)
else:
ret_remove = (site_id, found_device_id)
log.warning("%s '%s' already hasDevice %s", site_type, site_id, device_type)
except NotFound:
pass
return ret_remove, ret_ignore
def _validate_port_assignments(self, device_id, platform_port):
deployment_context_type = type(self.deployment_obj.context).__name__
self._validate_ooi_reference_designator(device_id, platform_port)
# a one-to-one deployment of a device onto an RSN platform
if OT.CabledInstrumentDeploymentContext == deployment_context_type or \
OT.CabledNodeDeploymentContext == deployment_context_type:
# validate IP address for a cabled node deployment
from socket import inet_aton
try:
inet_aton(platform_port.ip_address)
except :
log.error('IP address validation failed for device. Device id: %s', device_id)
# validate port_type based on deployment context
# a platform device deployment should have UPLINK port type
if OT.RemotePlatformDeploymentContext == deployment_context_type or \
OT.CabledNodeDeploymentContext == deployment_context_type:
if device_id in self.device_resources and self.device_resources[device_id].type_ is RT.PlatformDevice:
if platform_port.port_type != PortTypeEnum.UPLINK:
log.warning('Type of port for platform port assignment should be UPLINK. Device id: %s', device_id)
#validate that parent_id is provided
if not platform_port.parent_id:
log.warning('Id of parent device should be provided in port assignment information. Device id: %s', device_id)
def _validate_port_assignments(self, device_id, platform_port):
deployment_context_type = type(self.deployment_obj.context).__name__
self._validate_ooi_reference_designator(device_id, platform_port)
# a one-to-one deployment of a device onto an RSN platform
if OT.CabledInstrumentDeploymentContext == deployment_context_type or \
OT.CabledNodeDeploymentContext == deployment_context_type:
# validate IP address for a cabled node deployment
from socket import inet_aton
try:
inet_aton(platform_port.ip_address)
except :
log.error('IP address validation failed for device. Device id: %s', device_id)
# validate port_type based on deployment context
# a platform device deployment should have UPLINK port type
if OT.RemotePlatformDeploymentContext == deployment_context_type or \
OT.CabledNodeDeploymentContext == deployment_context_type:
if device_id in self.device_resources and self.device_resources[device_id].type_ is RT.PlatformDevice:
if platform_port.port_type != PortTypeEnum.UPLINK:
log.warning('Type of port for platform port assignment should be UPLINK. Device id: %s', device_id)
#validate that parent_id is provided
if not platform_port.parent_id:
log.warning('Id of parent device should be provided in port assignment information. Device id: %s', device_id)
def _find_existing_relationship(self, site_id, device_id, site_type=None, device_type=None):
# look for an existing relationship between the site_id and another device.
# if this site/device pair already exists, we leave it alone
assert(type("") == type(site_id) == type(device_id))
log.debug("checking %s/%s pair for deployment", site_type, device_type)
#return a pair that should be REMOVED, or None
if site_type is None and site_id in self.site_resources:
site_type = self.site_resources[site_id].type_
if device_type is None and device_id in self.device_resources:
device_type = self.device_resources[device_id].type_
log.debug("checking existing %s hasDevice %s links", site_type, device_type)
ret_remove = None
ret_ignore = None
try:
found_device_id = self.enhanced_rr.find_object(site_id, PRED.hasDevice, device_type, True)
if found_device_id == device_id:
ret_ignore = (site_id, device_id)
else:
ret_remove = (site_id, found_device_id)
log.warning("%s '%s' already hasDevice %s", site_type, site_id, device_type)
except NotFound:
pass
return ret_remove, ret_ignore
def prepare_activation(self, deployment_obj):
"""
Prepare (validate) a deployment for activation, returning lists of what associations need to be added
and which ones need to be removed.
"""
self.match_list = []
self.remove_list = []
self.unmatched_device_list = []
self.models_map = {}
self.top_device = ''
self.top_site = ''
self.deployment_obj = deployment_obj
self.site_resources = {}
self.device_resources = {}
self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr)
# retrieve the site tree information using the OUTIL functions; site info as well has site children
self.top_site, self.top_device = self._find_top_site_device(
deployment_obj._id)
# must have a site and a device to continue
if not self.top_site or not self.top_device:
return [], []
log.debug("port_assignments: %s", self.deployment_obj.port_assignments)
# retrieve all models to use in match validation
self._get_models()
self.site_resources, site_children = self.outil.get_child_sites(
parent_site_id=self.top_site._id, id_only=False)
log.debug("site_resources: %s", self.site_resources)
log.debug("site_children: %s", site_children)
site_ref_designator_map = self._get_site_ref_designator_map()
# retrieve the device tree from outil then cache the device resources
device_tree = self.outil.get_child_devices(
device_id=self.top_device._id)
self._get_device_resources(device_tree)
self._match_devices(self.top_device._id, device_tree,
site_ref_designator_map)
# check for hasDevice relations to remove and existing hasDevice relations
self._find_pairs_to_remove()
if self.unmatched_device_list:
log.warning("Devices not matched to sites: %s ",
self.unmatched_device_list)
return self.remove_list, self.match_list
def get_deployment_extension(self, deployment_id='', ext_associations=None, ext_exclude=None, user_id=''):
if not deployment_id:
raise BadRequest("The deployment_id parameter is empty")
extended_resource_handler = ExtendedResourceContainer(self)
extended_deployment = extended_resource_handler.create_extended_resource_container(
extended_resource_type=OT.DeploymentExtension,
resource_id=deployment_id,
computed_resource_type=OT.DeploymentComputedAttributes,
ext_associations=ext_associations,
ext_exclude=ext_exclude,
user_id=user_id)
devices = set()
instrument_device_ids = []
iplatform_device_ids = []
subjs, _ = self.RR.find_subjects( predicate=PRED.hasDeployment, object=deployment_id, id_only=False)
for subj in subjs:
log.debug('get_deployment_extension obj: %s', subj)
if subj.type_ == "InstrumentDevice":
extended_deployment.instrument_devices.append(subj)
devices.add((subj._id, PRED.hasModel))
elif subj.type_ == "InstrumentSite":
extended_deployment.instrument_sites.append(subj)
elif subj.type_ == "PlatformDevice":
extended_deployment.platform_devices.append(subj)
devices.add((subj._id, PRED.hasModel))
elif subj.type_ == "PlatformSite":
extended_deployment.platform_sites.append(subj)
else:
log.warning("get_deployment_extension found invalid type connected to deployment %s. Object details: %s ", deployment_id, subj)
all_models = set()
device_to_model_map = {}
model_map = {}
assocs = self.RR.find_associations(anyside=list(devices), id_only=False)
for assoc in assocs:
log.debug('get_deployment_extension assoc subj: %s pred: %s obj: %s', assoc.s, assoc.p, assoc.o)
all_models.add(assoc.o)
device_to_model_map[assoc.s] = assoc.o
model_objs = self.RR.read_mult( list(all_models) )
for model_obj in model_objs:
model_map[model_obj._id] = model_obj
for instrument in extended_deployment.instrument_devices:
model_id = device_to_model_map[instrument._id]
extended_deployment.instrument_models.append( model_map[model_id] )
for platform in extended_deployment.platform_devices:
model_id = device_to_model_map[platform._id]
extended_deployment.platform_models.append( model_map[model_id] )
return extended_deployment
def _validate_ooi_reference_designator(self, device_id, device_port):
ooi_rd = OOIReferenceDesignator(device_port.reference_designator)
if ooi_rd.error:
log.warning(
"Invalid OOIReferenceDesignator ( %s ) specified for device %s",
device_port.reference_designator, device_id)
if not ooi_rd.port:
log.warning(
"Invalid OOIReferenceDesignator ( %s ) specified for device %s, could not retrieve port",
device_port.reference_designator, device_id)
def prepare_activation(self, deployment_obj):
"""
Prepare (validate) a deployment for activation, returning lists of what associations need to be added
and which ones need to be removed.
"""
self.match_list = []
self.remove_list = []
self.unmatched_device_list = []
self.models_map = {}
self.top_device = ''
self.top_site = ''
self.deployment_obj = deployment_obj
self.site_resources = {}
self.device_resources = {}
self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr)
# retrieve the site tree information using the OUTIL functions; site info as well has site children
self.top_site, self.top_device = self._find_top_site_device(deployment_obj._id)
# must have a site and a device to continue
if not self.top_site or not self.top_device:
return [], []
log.debug("port_assignments: %s", self.deployment_obj.port_assignments )
# retrieve all models to use in match validation
self._get_models()
self.site_resources, site_children = self.outil.get_child_sites( parent_site_id=self.top_site._id, id_only=False)
log.debug("site_resources: %s", self.site_resources)
log.debug("site_children: %s", site_children)
site_ref_designator_map = self._get_site_ref_designator_map()
# retrieve the device tree from outil then cache the device resources
device_tree = self.outil.get_child_devices(device_id=self.top_device._id)
self._get_device_resources(device_tree)
self._match_devices(self.top_device._id, device_tree, site_ref_designator_map)
# check for hasDevice relations to remove and existing hasDevice relations
self. _find_pairs_to_remove()
if self.unmatched_device_list:
log.warning("Devices not matched to sites: %s ", self.unmatched_device_list)
return self.remove_list, self.match_list
def _get_calling_module(default_value=None):
try:
stack = inspect.stack()
# stack[0]: call to inspect.stack() on the line above
# stack[1]: call to _get_calling_module() below
# stack[2] # call to _SelfLogging.__init__() by subclass
frame=stack[3] # call to Timer() or Accumulator() by caller
if frame and frame[0]:
module = inspect.getmodule(frame[0])
if module:
return module.__name__
elif frame[1]:
return frame[1]
except:
log.warning('failed to inspect calling module', exc_info=True)
return default_value
def _merge_helper(acc, site_ptr, dev_ptr, unmatched_list):
"""
given 2 trees, try to match up all their children. assume roots already matched
"""
dev_id = dev_ptr["_id"]
site_id = site_ptr["_id"]
if not dev_ptr["model"] in site_ptr["models"]:
log.warning(
"Attempted to assign device '%s' to a site '%s' that doesn't support its model", dev_id, site_id
)
if dev_id in unmatched_list:
unmatched_list.remove(dev_id)
acc.append((site_id, dev_id))
log.debug("Add to matched list site_id: %s dev_id: %s", site_id, dev_id)
site_of_portref = {}
# creat a dict of reference_designator on sites so that devices can be matched
dev_site_obj = self.site_resources[site_id]
site_of_portref[dev_site_obj.reference_designator] = site_id
if site_id in self.site_children:
for child in self.site_children[site_id]:
dev_site_obj = self.site_resources[child]
site_of_portref[dev_site_obj.reference_designator] = child
for child_dev_id, child_dev_ptr in dev_ptr["children"].iteritems():
if not child_dev_id in portref_of_device:
log.warning("No platform port information specified for device %s" % child_dev_id)
dev_port = portref_of_device[child_dev_id]
# check that a PlatformPort object is provided
if dev_port.type_ != OT.PlatformPort:
log.warning("No platform port information specified for device %s" % child_dev_id)
ooi_rd = OOIReferenceDesignator(dev_port.reference_designator)
if ooi_rd.error:
log.warning(
"Invalid OOIReferenceDesignator ( %s ) specified for device %s",
dev_port.reference_designator,
child_dev_id,
)
if not ooi_rd.port:
log.warning(
"Invalid OOIReferenceDesignator ( %s ) specified for device %s, could not retrieve port",
dev_port.reference_designator,
child_dev_id,
)
if dev_port.reference_designator in site_of_portref:
child_site_id = site_of_portref[dev_port.reference_designator]
child_site_ptr = site_ptr["children"][child_site_id]
acc, unmatched_list = _merge_helper(acc[:], child_site_ptr, child_dev_ptr, unmatched_list[:])
else:
log.warning("Couldn't find a port on site %s (%s) called '%s'", site_ptr["name"], site_id, dev_port)
# this check is to match the ref_designator in the deployment object with the ref_designator in the target site
# todo add ref_designators to the Sites in preload to match intended deployments
return acc, unmatched_list
def _validate_ooi_reference_designator(self, device_id, device_port):
ooi_rd = OOIReferenceDesignator(device_port.reference_designator)
if ooi_rd.error:
log.warning("Invalid OOIReferenceDesignator ( %s ) specified for device %s", device_port.reference_designator, device_id)
if not ooi_rd.port:
log.warning("Invalid OOIReferenceDesignator ( %s ) specified for device %s, could not retrieve port", device_port.reference_designator, device_id)
def __setitem__(self, slice_, value):
# Always storing in first slot - ignore slice
bid = 'sparse_value_brick'
bD = (1,)
cD = None
brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, bid)
vals = [self.__serialize(v) for v in value]
vals = pack(vals)
set_arr = np.empty(1, dtype=object)
set_arr[0] = vals
data_type = h5py.special_dtype(vlen=str)
if self.inline_data_writes:
with HDFLockingFile(brick_file_path, 'a') as f:
f.require_dataset(bid, shape=bD, dtype=data_type, chunks=cD, fillvalue=None)
f[bid][0] = set_arr
else:
work_key = bid
work = ((0,), set_arr)
work_metrics = (brick_file_path, bD, cD, data_type, None)
# If the brick file doesn't exist, 'touch' it to make sure it's immediately available
if not os.path.exists(brick_file_path):
with HDFLockingFile(brick_file_path, 'a') as f:
# TODO: Due to usage concerns, currently locking chunking to "auto"
f.require_dataset(bid, shape=bD, dtype=data_type, chunks=cD, fillvalue=None)
if self.auto_flush:
# Immediately submit work to the dispatcher
self.brick_dispatcher.put_work(work_key, work_metrics, work)
else:
# Queue the work for later flushing
self._queue_work(work_key, work_metrics, work)
if self.parameter_manager.parameter_name in self.master_manager.param_groups:
try:
log.trace("Parameter: %s , Values: %s , Fill: %s", self.parameter_manager.parameter_name, value, self.fill_value)
min_val = min(value)
max_val = max(value)
log.trace("Value type: %s %s", type(value[0]), len(value))
if len(value) > 0 and isinstance(value[0], Span):
min_val = self.fill_value
max_val = self.fill_value
mins = []
maxes = []
for span in value:
log.trace("Span min/max %s", span)
if isinstance(span, Span):
tup = span.tuplize()
tup = [x for x in tup if x is not None and x is not self.fill_value and isinstance(x, numbers.Number)]
if len(tup) > 0:
mins.append(min(tup))
maxes.append(max(tup))
if len(mins) > 0:
min_val = min(mins)
if len(maxes) > 0:
max_val = max(maxes)
if max_val is not None and min_val is not None:
log.trace("SparsePersistedStorage saving %s min/max %s/%s", self.parameter_manager.parameter_name, min_val, max_val)
self.master_manager.track_data_written_to_brick(bid, 0, self.parameter_manager.parameter_name, min_val, max_val)
except TypeError as e:
log.debug("Don't store extents for types for which extents are meaningless: %s", type(v))
raise
except ValueError as e:
log.warning("Values stored for extents were invalid for brick=%s, slice=%s, param=%s min/max=%s",
bid, 0, self.parameter_manager.parameter_name, str( (min(value), max(value)) ))
raise
except Exception as e:
log.warning("Could not store Span extents for %s. Unexpected error %s",
str( (bid, 0, self.parameter_manager.parameter_name)), e.message )
raise
def prepare_activation(self, deployment_obj):
"""
Prepare (validate) a deployment for activation, returning lists of what associations need to be added
and which ones need to be removed.
"""
self.match_list = []
self.remove_list = []
self.unmatched_device_list = []
self.models_map = {}
self.top_device = ''
self.top_site = ''
self.deployment_obj = deployment_obj
self.site_resources = {}
self.device_resources = {}
self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr)
# retrieve the site tree information using the OUTIL functions; site info as well has site children
self.top_site, self.top_device = self._find_top_site_device(deployment_obj._id)
# must have a site and a device to continue
if not self.top_site or not self.top_device:
return [], []
log.debug("port_assignments: %s", self.deployment_obj.port_assignments )
# retrieve all models to use in match validation
self._get_models()
self.site_resources, site_children = self.outil.get_child_sites( parent_site_id=self.top_site._id, id_only=False)
log.debug("site_resources: %s", self.site_resources)
log.debug("site_children: %s", site_children)
site_ref_designator_map = self._get_site_ref_designator_map()
# retrieve the device tree from outil then cache the device resources
device_tree = self.outil.get_child_devices(device_id=self.top_device._id)
self._get_device_resources(device_tree)
def _match_devices(device_id):
# there will not be a port assignment for the top device
if device_id == self.top_device._id:
self._validate_models(self.top_site._id, self.top_device._id)
self.match_list.append((self.top_site._id, self.top_device._id))
tuple_list = device_tree[device_id]
for (pt, child_id, ct) in tuple_list:
log.debug(" tuple - pt: %s child_id: %s ct: %s", pt, child_id, ct)
# match this child device then if it has children, call _match_devices with this id
# check that this device is represented in device tree and in port assignments
if child_id in self.device_resources and child_id in self.deployment_obj.port_assignments:
platform_port = self.deployment_obj.port_assignments[child_id]
log.debug("device platform_port: %s", platform_port)
# validate PlatformPort info for this device
self._validate_port_assignments(child_id, platform_port)
if platform_port.reference_designator in site_ref_designator_map:
matched_site = site_ref_designator_map[platform_port.reference_designator]
self._validate_models(matched_site, child_id)
log.info("match_list append site: %s device: %s", matched_site, child_id)
self.match_list.append((matched_site, child_id))
#recurse on the children of this device
_match_devices(child_id)
# otherwise cant be matched to a site
else:
self.unmatched_device_list.append(child_id)
_match_devices(self.top_device._id)
# check for hasDevice relations to remove and existing hasDevice relations
self. _find_pairs_to_remove()
if self.unmatched_device_list:
log.warning("Devices not matched to sites: %s ", self.unmatched_device_list)
return self.remove_list, self.match_list
