def _find_existing_relationship(self, site_id, device_id, site_type=None, device_type=None): # look for an existing relationship between the site_id and another device. # if this site/device pair already exists, we leave it alone assert(type("") == type(site_id) == type(device_id)) log.debug("checking %s/%s pair for deployment", site_type, device_type) #return a pair that should be REMOVED, or None if site_type is None and site_id in self.site_resources: site_type = self.site_resources[site_id].type_ if device_type is None and device_id in self.device_resources: device_type = self.device_resources[device_id].type_ log.debug("checking existing %s hasDevice %s links", site_type, device_type) ret_remove = None ret_ignore = None try: found_device_id = self.enhanced_rr.find_object(site_id, PRED.hasDevice, device_type, True) if found_device_id == device_id: ret_ignore = (site_id, device_id) else: ret_remove = (site_id, found_device_id) log.warning("%s '%s' already hasDevice %s", site_type, site_id, device_type) except NotFound: pass return ret_remove, ret_ignore
def _validate_port_assignments(self, device_id, platform_port): deployment_context_type = type(self.deployment_obj.context).__name__ self._validate_ooi_reference_designator(device_id, platform_port) # a one-to-one deployment of a device onto an RSN platform if OT.CabledInstrumentDeploymentContext == deployment_context_type or \ OT.CabledNodeDeploymentContext == deployment_context_type: # validate IP address for a cabled node deployment from socket import inet_aton try: inet_aton(platform_port.ip_address) except : log.error('IP address validation failed for device. Device id: %s', device_id) # validate port_type based on deployment context # a platform device deployment should have UPLINK port type if OT.RemotePlatformDeploymentContext == deployment_context_type or \ OT.CabledNodeDeploymentContext == deployment_context_type: if device_id in self.device_resources and self.device_resources[device_id].type_ is RT.PlatformDevice: if platform_port.port_type != PortTypeEnum.UPLINK: log.warning('Type of port for platform port assignment should be UPLINK. Device id: %s', device_id) #validate that parent_id is provided if not platform_port.parent_id: log.warning('Id of parent device should be provided in port assignment information. Device id: %s', device_id)
def prepare_activation(self, deployment_obj): """ Prepare (validate) a deployment for activation, returning lists of what associations need to be added and which ones need to be removed. """ self.match_list = [] self.remove_list = [] self.unmatched_device_list = [] self.models_map = {} self.top_device = '' self.top_site = '' self.deployment_obj = deployment_obj self.site_resources = {} self.device_resources = {} self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) # retrieve the site tree information using the OUTIL functions; site info as well has site children self.top_site, self.top_device = self._find_top_site_device( deployment_obj._id) # must have a site and a device to continue if not self.top_site or not self.top_device: return [], [] log.debug("port_assignments: %s", self.deployment_obj.port_assignments) # retrieve all models to use in match validation self._get_models() self.site_resources, site_children = self.outil.get_child_sites( parent_site_id=self.top_site._id, id_only=False) log.debug("site_resources: %s", self.site_resources) log.debug("site_children: %s", site_children) site_ref_designator_map = self._get_site_ref_designator_map() # retrieve the device tree from outil then cache the device resources device_tree = self.outil.get_child_devices( device_id=self.top_device._id) self._get_device_resources(device_tree) self._match_devices(self.top_device._id, device_tree, site_ref_designator_map) # check for hasDevice relations to remove and existing hasDevice relations self._find_pairs_to_remove() if self.unmatched_device_list: log.warning("Devices not matched to sites: %s ", self.unmatched_device_list) return self.remove_list, self.match_list
def get_deployment_extension(self, deployment_id='', ext_associations=None, ext_exclude=None, user_id=''): if not deployment_id: raise BadRequest("The deployment_id parameter is empty") extended_resource_handler = ExtendedResourceContainer(self) extended_deployment = extended_resource_handler.create_extended_resource_container( extended_resource_type=OT.DeploymentExtension, resource_id=deployment_id, computed_resource_type=OT.DeploymentComputedAttributes, ext_associations=ext_associations, ext_exclude=ext_exclude, user_id=user_id) devices = set() instrument_device_ids = [] iplatform_device_ids = [] subjs, _ = self.RR.find_subjects( predicate=PRED.hasDeployment, object=deployment_id, id_only=False) for subj in subjs: log.debug('get_deployment_extension obj: %s', subj) if subj.type_ == "InstrumentDevice": extended_deployment.instrument_devices.append(subj) devices.add((subj._id, PRED.hasModel)) elif subj.type_ == "InstrumentSite": extended_deployment.instrument_sites.append(subj) elif subj.type_ == "PlatformDevice": extended_deployment.platform_devices.append(subj) devices.add((subj._id, PRED.hasModel)) elif subj.type_ == "PlatformSite": extended_deployment.platform_sites.append(subj) else: log.warning("get_deployment_extension found invalid type connected to deployment %s. Object details: %s ", deployment_id, subj) all_models = set() device_to_model_map = {} model_map = {} assocs = self.RR.find_associations(anyside=list(devices), id_only=False) for assoc in assocs: log.debug('get_deployment_extension assoc subj: %s pred: %s obj: %s', assoc.s, assoc.p, assoc.o) all_models.add(assoc.o) device_to_model_map[assoc.s] = assoc.o model_objs = self.RR.read_mult( list(all_models) ) for model_obj in model_objs: model_map[model_obj._id] = model_obj for instrument in extended_deployment.instrument_devices: model_id = device_to_model_map[instrument._id] extended_deployment.instrument_models.append( model_map[model_id] ) for platform in extended_deployment.platform_devices: model_id = device_to_model_map[platform._id] extended_deployment.platform_models.append( model_map[model_id] ) return extended_deployment
def _validate_ooi_reference_designator(self, device_id, device_port): ooi_rd = OOIReferenceDesignator(device_port.reference_designator) if ooi_rd.error: log.warning( "Invalid OOIReferenceDesignator ( %s ) specified for device %s", device_port.reference_designator, device_id) if not ooi_rd.port: log.warning( "Invalid OOIReferenceDesignator ( %s ) specified for device %s, could not retrieve port", device_port.reference_designator, device_id)
def prepare_activation(self, deployment_obj): """ Prepare (validate) a deployment for activation, returning lists of what associations need to be added and which ones need to be removed. """ self.match_list = [] self.remove_list = [] self.unmatched_device_list = [] self.models_map = {} self.top_device = '' self.top_site = '' self.deployment_obj = deployment_obj self.site_resources = {} self.device_resources = {} self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) # retrieve the site tree information using the OUTIL functions; site info as well has site children self.top_site, self.top_device = self._find_top_site_device(deployment_obj._id) # must have a site and a device to continue if not self.top_site or not self.top_device: return [], [] log.debug("port_assignments: %s", self.deployment_obj.port_assignments ) # retrieve all models to use in match validation self._get_models() self.site_resources, site_children = self.outil.get_child_sites( parent_site_id=self.top_site._id, id_only=False) log.debug("site_resources: %s", self.site_resources) log.debug("site_children: %s", site_children) site_ref_designator_map = self._get_site_ref_designator_map() # retrieve the device tree from outil then cache the device resources device_tree = self.outil.get_child_devices(device_id=self.top_device._id) self._get_device_resources(device_tree) self._match_devices(self.top_device._id, device_tree, site_ref_designator_map) # check for hasDevice relations to remove and existing hasDevice relations self. _find_pairs_to_remove() if self.unmatched_device_list: log.warning("Devices not matched to sites: %s ", self.unmatched_device_list) return self.remove_list, self.match_list
def _get_calling_module(default_value=None): try: stack = inspect.stack() # stack[0]: call to inspect.stack() on the line above # stack[1]: call to _get_calling_module() below # stack[2] # call to _SelfLogging.__init__() by subclass frame=stack[3] # call to Timer() or Accumulator() by caller if frame and frame[0]: module = inspect.getmodule(frame[0]) if module: return module.__name__ elif frame[1]: return frame[1] except: log.warning('failed to inspect calling module', exc_info=True) return default_value
def _merge_helper(acc, site_ptr, dev_ptr, unmatched_list): """ given 2 trees, try to match up all their children. assume roots already matched """ dev_id = dev_ptr["_id"] site_id = site_ptr["_id"] if not dev_ptr["model"] in site_ptr["models"]: log.warning( "Attempted to assign device '%s' to a site '%s' that doesn't support its model", dev_id, site_id ) if dev_id in unmatched_list: unmatched_list.remove(dev_id) acc.append((site_id, dev_id)) log.debug("Add to matched list site_id: %s dev_id: %s", site_id, dev_id) site_of_portref = {} # creat a dict of reference_designator on sites so that devices can be matched dev_site_obj = self.site_resources[site_id] site_of_portref[dev_site_obj.reference_designator] = site_id if site_id in self.site_children: for child in self.site_children[site_id]: dev_site_obj = self.site_resources[child] site_of_portref[dev_site_obj.reference_designator] = child for child_dev_id, child_dev_ptr in dev_ptr["children"].iteritems(): if not child_dev_id in portref_of_device: log.warning("No platform port information specified for device %s" % child_dev_id) dev_port = portref_of_device[child_dev_id] # check that a PlatformPort object is provided if dev_port.type_ != OT.PlatformPort: log.warning("No platform port information specified for device %s" % child_dev_id) ooi_rd = OOIReferenceDesignator(dev_port.reference_designator) if ooi_rd.error: log.warning( "Invalid OOIReferenceDesignator ( %s ) specified for device %s", dev_port.reference_designator, child_dev_id, ) if not ooi_rd.port: log.warning( "Invalid OOIReferenceDesignator ( %s ) specified for device %s, could not retrieve port", dev_port.reference_designator, child_dev_id, ) if dev_port.reference_designator in site_of_portref: child_site_id = site_of_portref[dev_port.reference_designator] child_site_ptr = site_ptr["children"][child_site_id] acc, unmatched_list = _merge_helper(acc[:], child_site_ptr, child_dev_ptr, unmatched_list[:]) else: log.warning("Couldn't find a port on site %s (%s) called '%s'", site_ptr["name"], site_id, dev_port) # this check is to match the ref_designator in the deployment object with the ref_designator in the target site # todo add ref_designators to the Sites in preload to match intended deployments return acc, unmatched_list
def _validate_ooi_reference_designator(self, device_id, device_port): ooi_rd = OOIReferenceDesignator(device_port.reference_designator) if ooi_rd.error: log.warning("Invalid OOIReferenceDesignator ( %s ) specified for device %s", device_port.reference_designator, device_id) if not ooi_rd.port: log.warning("Invalid OOIReferenceDesignator ( %s ) specified for device %s, could not retrieve port", device_port.reference_designator, device_id)
def __setitem__(self, slice_, value): # Always storing in first slot - ignore slice bid = 'sparse_value_brick' bD = (1,) cD = None brick_file_path = '{0}/{1}.hdf5'.format(self.brick_path, bid) vals = [self.__serialize(v) for v in value] vals = pack(vals) set_arr = np.empty(1, dtype=object) set_arr[0] = vals data_type = h5py.special_dtype(vlen=str) if self.inline_data_writes: with HDFLockingFile(brick_file_path, 'a') as f: f.require_dataset(bid, shape=bD, dtype=data_type, chunks=cD, fillvalue=None) f[bid][0] = set_arr else: work_key = bid work = ((0,), set_arr) work_metrics = (brick_file_path, bD, cD, data_type, None) # If the brick file doesn't exist, 'touch' it to make sure it's immediately available if not os.path.exists(brick_file_path): with HDFLockingFile(brick_file_path, 'a') as f: # TODO: Due to usage concerns, currently locking chunking to "auto" f.require_dataset(bid, shape=bD, dtype=data_type, chunks=cD, fillvalue=None) if self.auto_flush: # Immediately submit work to the dispatcher self.brick_dispatcher.put_work(work_key, work_metrics, work) else: # Queue the work for later flushing self._queue_work(work_key, work_metrics, work) if self.parameter_manager.parameter_name in self.master_manager.param_groups: try: log.trace("Parameter: %s , Values: %s , Fill: %s", self.parameter_manager.parameter_name, value, self.fill_value) min_val = min(value) max_val = max(value) log.trace("Value type: %s %s", type(value[0]), len(value)) if len(value) > 0 and isinstance(value[0], Span): min_val = self.fill_value max_val = self.fill_value mins = [] maxes = [] for span in value: log.trace("Span min/max %s", span) if isinstance(span, Span): tup = span.tuplize() tup = [x for x in tup if x is not None and x is not self.fill_value and isinstance(x, numbers.Number)] if len(tup) > 0: mins.append(min(tup)) maxes.append(max(tup)) if len(mins) > 0: min_val = min(mins) if len(maxes) > 0: max_val = max(maxes) if max_val is not None and min_val is not None: log.trace("SparsePersistedStorage saving %s min/max %s/%s", self.parameter_manager.parameter_name, min_val, max_val) self.master_manager.track_data_written_to_brick(bid, 0, self.parameter_manager.parameter_name, min_val, max_val) except TypeError as e: log.debug("Don't store extents for types for which extents are meaningless: %s", type(v)) raise except ValueError as e: log.warning("Values stored for extents were invalid for brick=%s, slice=%s, param=%s min/max=%s", bid, 0, self.parameter_manager.parameter_name, str( (min(value), max(value)) )) raise except Exception as e: log.warning("Could not store Span extents for %s. Unexpected error %s", str( (bid, 0, self.parameter_manager.parameter_name)), e.message ) raise
def prepare_activation(self, deployment_obj): """ Prepare (validate) a deployment for activation, returning lists of what associations need to be added and which ones need to be removed. """ self.match_list = [] self.remove_list = [] self.unmatched_device_list = [] self.models_map = {} self.top_device = '' self.top_site = '' self.deployment_obj = deployment_obj self.site_resources = {} self.device_resources = {} self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) # retrieve the site tree information using the OUTIL functions; site info as well has site children self.top_site, self.top_device = self._find_top_site_device(deployment_obj._id) # must have a site and a device to continue if not self.top_site or not self.top_device: return [], [] log.debug("port_assignments: %s", self.deployment_obj.port_assignments ) # retrieve all models to use in match validation self._get_models() self.site_resources, site_children = self.outil.get_child_sites( parent_site_id=self.top_site._id, id_only=False) log.debug("site_resources: %s", self.site_resources) log.debug("site_children: %s", site_children) site_ref_designator_map = self._get_site_ref_designator_map() # retrieve the device tree from outil then cache the device resources device_tree = self.outil.get_child_devices(device_id=self.top_device._id) self._get_device_resources(device_tree) def _match_devices(device_id): # there will not be a port assignment for the top device if device_id == self.top_device._id: self._validate_models(self.top_site._id, self.top_device._id) self.match_list.append((self.top_site._id, self.top_device._id)) tuple_list = device_tree[device_id] for (pt, child_id, ct) in tuple_list: log.debug(" tuple - pt: %s child_id: %s ct: %s", pt, child_id, ct) # match this child device then if it has children, call _match_devices with this id # check that this device is represented in device tree and in port assignments if child_id in self.device_resources and child_id in self.deployment_obj.port_assignments: platform_port = self.deployment_obj.port_assignments[child_id] log.debug("device platform_port: %s", platform_port) # validate PlatformPort info for this device self._validate_port_assignments(child_id, platform_port) if platform_port.reference_designator in site_ref_designator_map: matched_site = site_ref_designator_map[platform_port.reference_designator] self._validate_models(matched_site, child_id) log.info("match_list append site: %s device: %s", matched_site, child_id) self.match_list.append((matched_site, child_id)) #recurse on the children of this device _match_devices(child_id) # otherwise cant be matched to a site else: self.unmatched_device_list.append(child_id) _match_devices(self.top_device._id) # check for hasDevice relations to remove and existing hasDevice relations self. _find_pairs_to_remove() if self.unmatched_device_list: log.warning("Devices not matched to sites: %s ", self.unmatched_device_list) return self.remove_list, self.match_list