def __init__(self, clients=None, enhanced_rr=None): self.clients = clients self.enhanced_rr = enhanced_rr if not enhanced_rr: self.enhanced_rr = EnhancedResourceRegistryClient(self.clients.resource_registry) self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr)
def _notification_children(notification_origin, notification_type, observatory_util=None): if observatory_util is None: observatory_util = ObservatoryUtil() children = [] if notification_type == NotificationTypeEnum.PLATFORM: device_relations = observatory_util.get_child_devices( notification_origin) children = [ did for pt, did, dt in device_relations[notification_origin] ] elif type == NotificationTypeEnum.SITE: child_site_dict, ancestors = observatory_util.get_child_sites( notification_origin) children = child_site_dict.keys() elif type == NotificationTypeEnum.FACILITY: objects, _ = resource_registry.find_objects( subject=notification_origin, predicate=PRED.hasResource, id_only=False) for o in objects: if o.type_ == RT.DataProduct \ or o.type_ == RT.InstrumentSite \ or o.type_ == RT.InstrumentDevice \ or o.type_ == RT.PlatformSite \ or o.type_ == RT.PlatformDevice: children.append(o._id) if notification_origin in children: children.remove(notification_origin) return children
def test_get_device_data_products(self): self.mu.load_mock_resources(self.res_list + self.res_list1) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2 + self.assoc_list3) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) res_dict = self.obs_util.get_site_data_products( 'Obs_1', RT.Observatory) self.assertGreaterEqual(len(res_dict), 6) self.assertIsNone(res_dict['data_product_resources']) self.assertIn('ID_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products( 'PS_1', RT.PlatformSite) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('ID_1', res_dict['device_data_products']) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products('Org_1', RT.Org) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) res_dict = self.obs_util.get_site_data_products( 'PS_1', RT.PlatformSite, include_data_products=True) self.assertIsNotNone(res_dict['data_product_resources']) self.assertIn('DP_1', res_dict['data_product_resources'])
def test_get_child_sites_org(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=False, id_only=True) self.assertEquals(len(child_sites), 6) self.assertEquals(len(site_ancestors), 5) self.assertIn('Sub_1', child_sites) self.assertIn('PS_1', child_sites) self.assertIn('IS_1', child_sites) self.assertIn('Obs_1', child_sites) self.assertIn('Obs_2', child_sites) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=True, id_only=True) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=True, id_only=False) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) self.assertEquals(len([v for v in child_sites.values() if v is None]), 0) self.assertEquals(child_sites['Org_1']._get_type(), RT.Org)
def prepare_activation(self, deployment_obj): """ Prepare (validate) a deployment for activation, returning lists of what associations need to be added and which ones need to be removed. """ self.match_list = [] self.remove_list = [] self.unmatched_device_list = [] self.models_map = {} self.top_device = '' self.top_site = '' self.deployment_obj = deployment_obj self.site_resources = {} self.device_resources = {} self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) # retrieve the site tree information using the OUTIL functions; site info as well has site children self.top_site, self.top_device = self._find_top_site_device( deployment_obj._id) # must have a site and a device to continue if not self.top_site or not self.top_device: return [], [] log.debug("port_assignments: %s", self.deployment_obj.port_assignments) # retrieve all models to use in match validation self._get_models() self.site_resources, site_children = self.outil.get_child_sites( parent_site_id=self.top_site._id, id_only=False) log.debug("site_resources: %s", self.site_resources) log.debug("site_children: %s", site_children) site_ref_designator_map = self._get_site_ref_designator_map() # retrieve the device tree from outil then cache the device resources device_tree = self.outil.get_child_devices( device_id=self.top_device._id) self._get_device_resources(device_tree) self._match_devices(self.top_device._id, device_tree, site_ref_designator_map) # check for hasDevice relations to remove and existing hasDevice relations self._find_pairs_to_remove() if self.unmatched_device_list: log.warning("Devices not matched to sites: %s ", self.unmatched_device_list) return self.remove_list, self.match_list
def test_get_site_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_devices = self.obs_util.get_site_devices( ['Sub_1', 'PS_1', 'IS_1']) self.assertEquals(len(site_devices), 3) self.assertEquals(site_devices['Sub_1'], []) self.assertEquals(site_devices['IS_1'], [('InstrumentSite', 'ID_1', 'InstrumentDevice')])
def on_init(self): resource_collector_factory = DeploymentResourceCollectorFactory(self.clients, self.RR2) self.resource_collector = resource_collector_factory.create(self.deployment_obj) self._hasdevice_associations_to_delete = [] self._hasdevice_associations_to_create = [] self.outil = ObservatoryUtil(self, enhanced_rr=self.RR2)
def test_get_child_sites_org(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_sites, site_ancestors = self.obs_util.get_child_sites(org_id='Org_1', include_parents=False, id_only=True) self.assertEquals(len(child_sites), 6) self.assertEquals(len(site_ancestors), 5) self.assertIn('Sub_1', child_sites) self.assertIn('PS_1', child_sites) self.assertIn('IS_1', child_sites) self.assertIn('Obs_1', child_sites) self.assertIn('Obs_2', child_sites) child_sites, site_ancestors = self.obs_util.get_child_sites(org_id='Org_1', include_parents=True, id_only=True) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) child_sites, site_ancestors = self.obs_util.get_child_sites(org_id='Org_1', include_parents=True, id_only=False) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) self.assertEquals(len([v for v in child_sites.values() if v is None]), 0) self.assertEquals(child_sites['Org_1']._get_type(), RT.Org)
def test_get_device_data_products(self): self.mu.load_mock_resources(self.res_list + self.res_list1) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2 + self.assoc_list3) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) res_dict = self.obs_util.get_site_data_products('Obs_1', RT.Observatory) self.assertGreaterEqual(len(res_dict), 6) self.assertIsNone(res_dict['data_product_resources']) self.assertIn('ID_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products('PS_1', RT.PlatformSite) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('ID_1', res_dict['device_data_products']) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products('Org_1', RT.Org) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) res_dict = self.obs_util.get_site_data_products('PS_1', RT.PlatformSite, include_data_products=True) self.assertIsNotNone(res_dict['data_product_resources']) self.assertIn('DP_1', res_dict['data_product_resources'])
def test_get_status_roll_ups_platform_warn(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_device_statuses(self.status_by_device_4) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock, device_status_mgr=self.dsm_mock) # PD_1 power+comms warning status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) #log.warn("status %s" % status_rollups) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1')
def test_get_site_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_devices = self.obs_util.get_site_devices(['Sub_1', 'PS_1', 'IS_1']) self.assertEquals(len(site_devices), 3) self.assertEquals(site_devices['Sub_1'], []) self.assertEquals(site_devices['IS_1'], [('InstrumentSite', 'ID_1', 'InstrumentDevice')])
def get_deployment_sites_devices(self, deployment_obj): # retrieve all site and device ids related to this deployment site_ids = [] device_ids = [] self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) top_site, top_device = self._find_top_site_device(deployment_obj._id) site_resources, site_children = self.outil.get_child_sites( parent_site_id=top_site._id, id_only=False) site_ids = site_resources.keys() # get_site_devices returns a tuple that includes all devices linked to deployment sites site_devices = self.outil.get_site_devices(site_ids) for site, tuple_list in site_devices.iteritems(): for (site_type, device_id, device_type) in tuple_list: device_ids.append(device_id) return site_ids, device_ids
def test_get_child_sites(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_resources, site_children = self.spy_get_child_sites(parent_site_id='Obs_1', include_parents=False, id_only=True) self.assertEquals(len(site_resources), 3) self.assertEquals(len(site_children), 3) self.assertIn('Sub_1', site_resources) self.assertIn('PS_1', site_resources) self.assertIn('IS_1', site_resources) self.assertNotIn('Obs_1', site_resources) self.assertEquals(len([v for v in site_resources.values() if v is None]), 3) site_resources, site_children = self.spy_get_child_sites(parent_site_id='Obs_1', include_parents=False, id_only=False) self.assertEquals(len(site_resources), 3) self.assertEquals(len(site_children), 3) self.assertEquals(len([v for v in site_resources.values() if v is None]), 0) self.assertEquals(site_resources['Sub_1']._get_type(), RT.Subsite) site_resources, site_children = self.spy_get_child_sites(parent_site_id='Obs_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) self.assertIn('Obs_1', site_resources) site_resources, site_children = self.spy_get_child_sites(parent_site_id='Sub_1', include_parents=False) self.assertEquals(len(site_resources), 2) self.assertEquals(len(site_children), 2) self.assertNotIn('Sub_1', site_resources) site_resources, site_children = self.spy_get_child_sites(parent_site_id='Sub_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) self.assertIn('Sub_1', site_resources) self.assertIn('Obs_1', site_resources) site_resources, site_children = self.spy_get_child_sites(parent_site_id='PS_1', include_parents=False) self.assertEquals(len(site_resources), 1) self.assertEquals(len(site_children), 1) site_resources, site_children = self.spy_get_child_sites(parent_site_id='PS_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) site_resources, site_children = self.spy_get_child_sites(parent_site_id='IS_1', include_parents=False) self.assertEquals(len(site_resources), 0) self.assertEquals(len(site_children), 0) site_resources, site_children = self.spy_get_child_sites(parent_site_id='IS_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) site_resources, site_children = self.spy_get_child_sites(parent_site_id='XXX', include_parents=True) self.assertEquals(len(site_resources), 1) self.assertEquals(len(site_children), 0)
def test_get_child_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_devices = self.obs_util.get_child_devices('PD_1') self.assertEquals(len(child_devices), 2) self.assertEquals(child_devices['PD_1'][0][1], 'ID_1') child_devices = self.obs_util.get_child_devices('ID_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['ID_1'], []) child_devices = self.obs_util.get_child_devices('Sub_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['Sub_1'], []) child_devices = self.obs_util.get_child_devices('XXX') self.assertEquals(len(child_devices), 1)
def _notification_children(notification_origin, notification_type, observatory_util=None): if observatory_util is None: observatory_util = ObservatoryUtil() children = [] if notification_type == NotificationTypeEnum.PLATFORM: device_relations = observatory_util.get_child_devices(notification_origin) children = [did for pt,did,dt in device_relations[notification_origin]] elif type == NotificationTypeEnum.SITE: child_site_dict, ancestors = observatory_util.get_child_sites(notification_origin) children = child_site_dict.keys() elif type == NotificationTypeEnum.FACILITY: objects, _ = resource_registry.find_objects(subject=notification_origin, predicate=PRED.hasResource, id_only=False) for o in objects: if o.type_ == RT.DataProduct \ or o.type_ == RT.InstrumentSite \ or o.type_ == RT.InstrumentDevice \ or o.type_ == RT.PlatformSite \ or o.type_ == RT.PlatformDevice: children.append(o._id) if notification_origin in children: children.remove(notification_origin) return children
def prepare_activation(self, deployment_obj): """ Prepare (validate) a deployment for activation, returning lists of what associations need to be added and which ones need to be removed. """ self.match_list = [] self.remove_list = [] self.unmatched_device_list = [] self.models_map = {} self.top_device = '' self.top_site = '' self.deployment_obj = deployment_obj self.site_resources = {} self.device_resources = {} self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) # retrieve the site tree information using the OUTIL functions; site info as well has site children self.top_site, self.top_device = self._find_top_site_device(deployment_obj._id) # must have a site and a device to continue if not self.top_site or not self.top_device: return [], [] log.debug("port_assignments: %s", self.deployment_obj.port_assignments ) # retrieve all models to use in match validation self._get_models() self.site_resources, site_children = self.outil.get_child_sites( parent_site_id=self.top_site._id, id_only=False) log.debug("site_resources: %s", self.site_resources) log.debug("site_children: %s", site_children) site_ref_designator_map = self._get_site_ref_designator_map() # retrieve the device tree from outil then cache the device resources device_tree = self.outil.get_child_devices(device_id=self.top_device._id) self._get_device_resources(device_tree) self._match_devices(self.top_device._id, device_tree, site_ref_designator_map) # check for hasDevice relations to remove and existing hasDevice relations self. _find_pairs_to_remove() if self.unmatched_device_list: log.warning("Devices not matched to sites: %s ", self.unmatched_device_list) return self.remove_list, self.match_list
def on_init(self): IonObject("Resource") # suppress pyflakes error CFG, log, RT, PRED, LCS, LCE, NotFound, BadRequest, log #suppress pyflakes errors about "unused import" self.override_clients(self.clients) self.outil = ObservatoryUtil(self) self.HIERARCHY_DEPTH = {RT.InstrumentSite: 3, RT.PlatformSite: 2, RT.Subsite: 1, RT.Observatory: 0, } self.HIERARCHY_LOOKUP = [RT.Observatory, RT.Subsite, RT.PlatformSite, RT.InstrumentSite]
class TestObservatoryUtil(IonUnitTestCase): def setUp(self): self.mu = MockUtil() self.process_mock = self.mu.create_process_mock() self.container_mock = self.mu.create_container_mock() self.dsm_mock = self.mu.create_device_status_manager_mock() res_list = [ dict(rt='Org', _id='Org_1', attr={}), dict(rt='Observatory', _id='Obs_1', attr={}), dict(rt='Observatory', _id='Obs_2', attr={}), dict(rt='Subsite', _id='Sub_1', attr={}), dict(rt='Subsite', _id='Sub_2', attr={}), dict(rt='PlatformSite', _id='PS_1', attr={}), dict(rt='InstrumentSite', _id='IS_1', attr={}), dict(rt='PlatformDevice', _id='PD_1', attr={}), dict(rt='InstrumentDevice', _id='ID_1', attr={}), ] assoc_list = [ ['Obs_1', 'hasSite', 'Sub_1'], ['Sub_1', 'hasSite', 'PS_1'], ['PS_1', 'hasSite', 'IS_1'], ] assoc_list1 = [ ['Org_1', 'hasResource', 'Obs_1'], ['Org_1', 'hasResource', 'Obs_2'], ['Obs_2', 'hasSite', 'Sub_2'], ] assoc_list2 = [ ['PS_1', 'hasDevice', 'PD_1'], ['IS_1', 'hasDevice', 'ID_1'], ['PD_1', 'hasDevice', 'ID_1'], ] def spy_get_child_sites(self, parent_site_id=None, org_id=None, exclude_types=None, include_parents=True, id_only=True): child_sites, site_ancestors = self.obs_util.get_child_sites(parent_site_id=parent_site_id, org_id=org_id, exclude_types=exclude_types, include_parents=include_parents, id_only=id_only) print "child_sites of", parent_site_id, "are", child_sites print "site_ancestors of", parent_site_id, "are", site_ancestors return child_sites, site_ancestors def test_get_child_sites(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_resources, site_children = self.spy_get_child_sites(parent_site_id='Obs_1', include_parents=False, id_only=True) self.assertEquals(len(site_resources), 3) self.assertEquals(len(site_children), 3) self.assertIn('Sub_1', site_resources) self.assertIn('PS_1', site_resources) self.assertIn('IS_1', site_resources) self.assertNotIn('Obs_1', site_resources) self.assertEquals(len([v for v in site_resources.values() if v is None]), 3) site_resources, site_children = self.spy_get_child_sites(parent_site_id='Obs_1', include_parents=False, id_only=False) self.assertEquals(len(site_resources), 3) self.assertEquals(len(site_children), 3) self.assertEquals(len([v for v in site_resources.values() if v is None]), 0) self.assertEquals(site_resources['Sub_1']._get_type(), RT.Subsite) site_resources, site_children = self.spy_get_child_sites(parent_site_id='Obs_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) self.assertIn('Obs_1', site_resources) site_resources, site_children = self.spy_get_child_sites(parent_site_id='Sub_1', include_parents=False) self.assertEquals(len(site_resources), 2) self.assertEquals(len(site_children), 2) self.assertNotIn('Sub_1', site_resources) site_resources, site_children = self.spy_get_child_sites(parent_site_id='Sub_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) self.assertIn('Sub_1', site_resources) self.assertIn('Obs_1', site_resources) site_resources, site_children = self.spy_get_child_sites(parent_site_id='PS_1', include_parents=False) self.assertEquals(len(site_resources), 1) self.assertEquals(len(site_children), 1) site_resources, site_children = self.spy_get_child_sites(parent_site_id='PS_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) site_resources, site_children = self.spy_get_child_sites(parent_site_id='IS_1', include_parents=False) self.assertEquals(len(site_resources), 0) self.assertEquals(len(site_children), 0) site_resources, site_children = self.spy_get_child_sites(parent_site_id='IS_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) site_resources, site_children = self.spy_get_child_sites(parent_site_id='XXX', include_parents=True) self.assertEquals(len(site_resources), 1) self.assertEquals(len(site_children), 0) def test_get_child_sites_org(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_sites, site_ancestors = self.obs_util.get_child_sites(org_id='Org_1', include_parents=False, id_only=True) self.assertEquals(len(child_sites), 6) self.assertEquals(len(site_ancestors), 5) self.assertIn('Sub_1', child_sites) self.assertIn('PS_1', child_sites) self.assertIn('IS_1', child_sites) self.assertIn('Obs_1', child_sites) self.assertIn('Obs_2', child_sites) child_sites, site_ancestors = self.obs_util.get_child_sites(org_id='Org_1', include_parents=True, id_only=True) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) child_sites, site_ancestors = self.obs_util.get_child_sites(org_id='Org_1', include_parents=True, id_only=False) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) self.assertEquals(len([v for v in child_sites.values() if v is None]), 0) self.assertEquals(child_sites['Org_1']._get_type(), RT.Org) def test_get_site_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_devices = self.obs_util.get_site_devices(['Sub_1', 'PS_1', 'IS_1']) self.assertEquals(len(site_devices), 3) self.assertEquals(site_devices['Sub_1'], []) self.assertEquals(site_devices['IS_1'], [('InstrumentSite', 'ID_1', 'InstrumentDevice')]) def test_get_child_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_devices = self.obs_util.get_child_devices('PD_1') self.assertEquals(len(child_devices), 2) self.assertEquals(child_devices['PD_1'][0][1], 'ID_1') child_devices = self.obs_util.get_child_devices('ID_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['ID_1'], []) child_devices = self.obs_util.get_child_devices('Sub_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['Sub_1'], []) child_devices = self.obs_util.get_child_devices('XXX') self.assertEquals(len(child_devices), 1) def test_get_device_data_products(self): self.mu.load_mock_resources(self.res_list + self.res_list1) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2 + self.assoc_list3) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) res_dict = self.obs_util.get_site_data_products('Obs_1', RT.Observatory) self.assertGreaterEqual(len(res_dict), 6) self.assertIsNone(res_dict['data_product_resources']) self.assertIn('ID_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products('PS_1', RT.PlatformSite) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('ID_1', res_dict['device_data_products']) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products('Org_1', RT.Org) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) res_dict = self.obs_util.get_site_data_products('PS_1', RT.PlatformSite, include_data_products=True) self.assertIsNotNone(res_dict['data_product_resources']) self.assertIn('DP_1', res_dict['data_product_resources']) #import pprint #pprint.pprint(res_dict) status_by_device_1 = { "ID_1": _devstat("ID_1", DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), "PD_1": _devstat("PD_1", DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), } status_by_device_2 = { "ID_1": _devstat("ID_1", DST.STATUS_WARNING, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), "PD_1": _devstat("PD_1", DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), } status_by_device_3 = { "ID_1": _devstat("ID_1", DST.STATUS_WARNING, DST.STATUS_WARNING, DST.STATUS_OK, DST.STATUS_OK), "PD_1": _devstat("PD_1", DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), } status_by_device_4 = { "ID_1": _devstat("ID_1", DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), "PD_1": _devstat("PD_1", DST.STATUS_WARNING, DST.STATUS_WARNING, DST.STATUS_OK, DST.STATUS_OK), } def test_get_status_roll_ups(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_device_statuses(self.status_by_device_1) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock, device_status_mgr=self.dsm_mock) # No problems status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) self.assertIn('PD_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1') self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') # ID_1 power warning self.mu.load_mock_device_statuses(self.status_by_device_2) status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertIn('PS_1', status_rollups) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) # ID_1 power+comms warning self.mu.load_mock_device_statuses(self.status_by_device_3) status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) def test_get_status_roll_ups_platform_warn(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_device_statuses(self.status_by_device_4) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock, device_status_mgr=self.dsm_mock) # PD_1 power+comms warning status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) #log.warn("status %s" % status_rollups) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') def _assert_status(self, status_rollups, res_id=None, agg=DST.STATUS_OK, loc=DST.STATUS_OK, data=DST.STATUS_OK, comms=DST.STATUS_OK, power=DST.STATUS_OK): res_status = status_rollups[res_id] if res_id else status_rollups log.debug("_assert_status(%s) = %s", res_id, res_status) self.assertEquals(len(res_status), 5) if agg is not None: self.assertEquals(res_status['agg'], agg) if loc is not None: self.assertEquals(res_status[AggregateStatusType.AGGREGATE_LOCATION], loc) if data is not None: self.assertEquals(res_status[AggregateStatusType.AGGREGATE_DATA], data) if comms is not None: self.assertEquals(res_status[AggregateStatusType.AGGREGATE_COMMS], comms) if power is not None: self.assertEquals(res_status[AggregateStatusType.AGGREGATE_POWER], power) res_list1 = [ dict(rt='DataProduct', _id='DP_1', attr={}), dict(rt='DataProduct', _id='DP_2', attr={}), dict(rt='DataProduct', _id='DP_3', attr={}), dict(rt='DataProduct', _id='DP_4', attr={}), dict(rt='DataProduct', _id='DP_5', attr={}), ] assoc_list3 = [ ['DP_1', 'hasSource', 'ID_1'], ['DP_1', 'hasSource', 'IS_1'], ['DP_2', 'hasSource', 'ID_1'], ['DP_3', 'hasSource', 'ID_1'], ['DP_3', 'hasSource', 'PD_1'], ['DP_4', 'hasSource', 'PD_1'], ['DP_5', 'hasSource', 'PD_1'], ]
class DeploymentPlanner(object): """ A deployment activator validates that a set of devices will map to a set of sites in one unique way its primary purpose is to prepare( ) after which you'll be able to access what associations must be made (and unmade) """ def __init__(self, clients=None, enhanced_rr=None): self.clients = clients self.enhanced_rr = enhanced_rr if not enhanced_rr: self.enhanced_rr = EnhancedResourceRegistryClient(self.clients.resource_registry) self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) #self.resource_collector= DeploymentResourceCollector(self.clients, self.enhanced_rr) #self.resource_collector = resource_collector.create(self.deployment_obj) def _find_top_site_device(self, deployment_id): top_site = '' top_device = '' #retrieve the site tree information using the OUTIL functions; site info as well has site children deploy_items_objs, _ = self.clients.resource_registry.find_subjects(predicate=PRED.hasDeployment, object=deployment_id, id_only=False) log.debug("site_ids associated to this deployment: %s", deploy_items_objs) for obj in deploy_items_objs: rsrc_type = obj.type_ log.debug("resource type associated to this deployment:: %s", rsrc_type) if RT.PlatformDevice == rsrc_type or RT.InstrumentDevice == rsrc_type: top_device = obj elif RT.PlatformSite == rsrc_type or RT.InstrumentSite == rsrc_type: top_site = obj else: log.error('Deployment may only link to devices and sites. Deployment: %s', str(self.deployment_obj)) if not top_device or not top_site: log.error('Deployment must associate to both site and device. Deployment: %s', str(self.deployment_obj)) raise BadRequest('Deployment must associate to both site and device. Deployment: %s', str(self.deployment_obj)) return top_site, top_device def _find_pairs_to_remove(self): #figure out if any of the devices in the new mapping are already mapped and need to be removed pairs_to_remove = [] pairs_to_ignore = [] for (s, d) in self.match_list: rm_pair, ignore_pair = self._find_existing_relationship(s, d) if rm_pair: pairs_to_remove.append(rm_pair) if ignore_pair: pairs_to_ignore.append(ignore_pair) log.info("Pairs to ignore (will be removed from add list): %s", pairs_to_ignore) # make sure that anything being removed is not also being added self.match_list = filter(lambda x: x not in pairs_to_ignore, self.match_list) log.info("Pairs to remove: %s", pairs_to_remove) self.remove_list = pairs_to_remove def _find_existing_relationship(self, site_id, device_id, site_type=None, device_type=None): # look for an existing relationship between the site_id and another device. # if this site/device pair already exists, we leave it alone assert(type("") == type(site_id) == type(device_id)) log.debug("checking %s/%s pair for deployment", site_type, device_type) #return a pair that should be REMOVED, or None if site_type is None and site_id in self.site_resources: site_type = self.site_resources[site_id].type_ if device_type is None and device_id in self.device_resources: device_type = self.device_resources[device_id].type_ log.debug("checking existing %s hasDevice %s links", site_type, device_type) ret_remove = None ret_ignore = None try: found_device_id = self.enhanced_rr.find_object(site_id, PRED.hasDevice, device_type, True) if found_device_id == device_id: ret_ignore = (site_id, device_id) else: ret_remove = (site_id, found_device_id) log.warning("%s '%s' already hasDevice %s", site_type, site_id, device_type) except NotFound: pass return ret_remove, ret_ignore def _get_site_ref_designator_map(self): # create a map of site ids to their reference designator codes to facilitate matching site_ref_designator_map = {} for id, site_obj in self.site_resources.iteritems(): site_ref_designator_map[site_obj.reference_designator] = id log.debug("prepare_activation site_ref_designator_map: %s", site_ref_designator_map) return site_ref_designator_map def _get_device_resources(self, device_tree): # create a map of device ids to their full resource object to assit with lookup and validation device_objs = self.clients.resource_registry.read_mult(device_tree.keys()) log.debug("prepare_activation device_objectss: %s", device_objs) for device_obj in device_objs: self.device_resources[device_obj._id] = device_obj def _get_models(self): # retrieve all hasModel associations from the registry then filter models_tuples = {} assoc_list = self.outil._get_predicate_assocs(PRED.hasModel) for assoc in assoc_list: # only include these subject types in the map if assoc.st in [RT.InstrumentDevice, RT.InstrumentSite, RT.PlatformDevice, RT.PlatformSite]: if assoc.s not in models_tuples: models_tuples[assoc.s] = [] # a site may support more than one model so map to a list of models models_tuples[assoc.s].append((assoc.st, assoc.o, assoc.ot)) if assoc.s not in self.models_map: self.models_map[assoc.s] = [] self.models_map[assoc.s].append(assoc.o) log.debug("models_map: %s", self.models_map ) def _validate_models(self, site_id, device_id): # validate that the device and the site models are compatible if device_id in self.models_map: device_model_list = self.models_map[device_id] # devices should only be associated to one model if len(device_model_list) != 1: log.error("Device not associated to one distinct model. Device id: %s", device_id) elif device_model_list and device_model_list[0] not in self.models_map[site_id]: log.error("Device and Site to not share a compatible model. Device id: %s Site id: %s", site_id) else: log.error("Device not associated with a device model. Device id: %s", device_id) raise NotFound("Device not associated with a device model. Device id: %s", device_id) def _validate_port_assignments(self, device_id, platform_port): deployment_context_type = type(self.deployment_obj.context).__name__ self._validate_ooi_reference_designator(device_id, platform_port) # a one-to-one deployment of a device onto an RSN platform if OT.CabledInstrumentDeploymentContext == deployment_context_type or \ OT.CabledNodeDeploymentContext == deployment_context_type: # validate IP address for a cabled node deployment from socket import inet_aton try: inet_aton(platform_port.ip_address) except : log.error('IP address validation failed for device. Device id: %s', device_id) # validate port_type based on deployment context # a platform device deployment should have UPLINK port type if OT.RemotePlatformDeploymentContext == deployment_context_type or \ OT.CabledNodeDeploymentContext == deployment_context_type: if device_id in self.device_resources and self.device_resources[device_id].type_ is RT.PlatformDevice: if platform_port.port_type != PortTypeEnum.UPLINK: log.warning('Type of port for platform port assignment should be UPLINK. Device id: %s', device_id) #validate that parent_id is provided if not platform_port.parent_id: log.warning('Id of parent device should be provided in port assignment information. Device id: %s', device_id) def _validate_ooi_reference_designator(self, device_id, device_port): ooi_rd = OOIReferenceDesignator(device_port.reference_designator) if ooi_rd.error: log.warning("Invalid OOIReferenceDesignator ( %s ) specified for device %s", device_port.reference_designator, device_id) if not ooi_rd.port: log.warning("Invalid OOIReferenceDesignator ( %s ) specified for device %s, could not retrieve port", device_port.reference_designator, device_id) def get_deployment_sites_devices(self, deployment_obj): # retrieve all site and device ids related to this deployment site_ids = [] device_ids = [] self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) top_site, top_device = self._find_top_site_device(deployment_obj._id) site_resources, site_children = self.outil.get_child_sites( parent_site_id=top_site._id, id_only=False) site_ids = site_resources.keys() # get_site_devices returns a tuple that includes all devices linked to deployment sites site_devices = self.outil.get_site_devices(site_ids) for site, tuple_list in site_devices.iteritems(): for (site_type, device_id, device_type) in tuple_list: device_ids.append(device_id) return site_ids, device_ids def prepare_activation(self, deployment_obj): """ Prepare (validate) a deployment for activation, returning lists of what associations need to be added and which ones need to be removed. """ self.match_list = [] self.remove_list = [] self.unmatched_device_list = [] self.models_map = {} self.top_device = '' self.top_site = '' self.deployment_obj = deployment_obj self.site_resources = {} self.device_resources = {} self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) # retrieve the site tree information using the OUTIL functions; site info as well has site children self.top_site, self.top_device = self._find_top_site_device(deployment_obj._id) # must have a site and a device to continue if not self.top_site or not self.top_device: return [], [] log.debug("port_assignments: %s", self.deployment_obj.port_assignments ) # retrieve all models to use in match validation self._get_models() self.site_resources, site_children = self.outil.get_child_sites( parent_site_id=self.top_site._id, id_only=False) log.debug("site_resources: %s", self.site_resources) log.debug("site_children: %s", site_children) site_ref_designator_map = self._get_site_ref_designator_map() # retrieve the device tree from outil then cache the device resources device_tree = self.outil.get_child_devices(device_id=self.top_device._id) self._get_device_resources(device_tree) def _match_devices(device_id): # there will not be a port assignment for the top device if device_id == self.top_device._id: self._validate_models(self.top_site._id, self.top_device._id) self.match_list.append((self.top_site._id, self.top_device._id)) tuple_list = device_tree[device_id] for (pt, child_id, ct) in tuple_list: log.debug(" tuple - pt: %s child_id: %s ct: %s", pt, child_id, ct) # match this child device then if it has children, call _match_devices with this id # check that this device is represented in device tree and in port assignments if child_id in self.device_resources and child_id in self.deployment_obj.port_assignments: platform_port = self.deployment_obj.port_assignments[child_id] log.debug("device platform_port: %s", platform_port) # validate PlatformPort info for this device self._validate_port_assignments(child_id, platform_port) if platform_port.reference_designator in site_ref_designator_map: matched_site = site_ref_designator_map[platform_port.reference_designator] self._validate_models(matched_site, child_id) log.info("match_list append site: %s device: %s", matched_site, child_id) self.match_list.append((matched_site, child_id)) #recurse on the children of this device _match_devices(child_id) # otherwise cant be matched to a site else: self.unmatched_device_list.append(child_id) _match_devices(self.top_device._id) # check for hasDevice relations to remove and existing hasDevice relations self. _find_pairs_to_remove() if self.unmatched_device_list: log.warning("Devices not matched to sites: %s ", self.unmatched_device_list) return self.remove_list, self.match_list
def test_get_status_roll_ups(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_events(self.event_list1) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) # No problems status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) self.assertIn('PD_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1') self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') # ID_1 power warning self.mu.load_mock_events(self.event_list2) status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertIn('PS_1', status_rollups) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) # ID_1 power+comms warning self.mu.load_mock_events(self.event_list3) status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING)
class TestObservatoryUtil(IonUnitTestCase): def setUp(self): self.mu = MockUtil() self.process_mock = self.mu.create_process_mock() self.container_mock = self.mu.create_container_mock() res_list = [ dict(rt='Org', _id='Org_1', attr={}), dict(rt='Observatory', _id='Obs_1', attr={}), dict(rt='Observatory', _id='Obs_2', attr={}), dict(rt='Subsite', _id='Sub_1', attr={}), dict(rt='Subsite', _id='Sub_2', attr={}), dict(rt='PlatformSite', _id='PS_1', attr={}), dict(rt='InstrumentSite', _id='IS_1', attr={}), dict(rt='PlatformDevice', _id='PD_1', attr={}), dict(rt='InstrumentDevice', _id='ID_1', attr={}), ] assoc_list = [ ['Obs_1', 'hasSite', 'Sub_1'], ['Sub_1', 'hasSite', 'PS_1'], ['PS_1', 'hasSite', 'IS_1'], ] assoc_list1 = [ ['Org_1', 'hasResource', 'Obs_1'], ['Org_1', 'hasResource', 'Obs_2'], ['Obs_2', 'hasSite', 'Sub_2'], ] assoc_list2 = [ ['PS_1', 'hasDevice', 'PD_1'], ['IS_1', 'hasDevice', 'ID_1'], ['PD_1', 'hasDevice', 'ID_1'], ] def spy_get_child_sites(self, parent_site_id=None, org_id=None, exclude_types=None, include_parents=True, id_only=True): child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id=parent_site_id, org_id=org_id, exclude_types=exclude_types, include_parents=include_parents, id_only=id_only) print "child_sites of", parent_site_id, "are", child_sites print "site_ancestors of", parent_site_id, "are", site_ancestors return child_sites, site_ancestors def test_get_child_sites(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Obs_1', include_parents=False, id_only=True) self.assertEquals(len(site_resources), 3) self.assertEquals(len(site_children), 3) self.assertIn('Sub_1', site_resources) self.assertIn('PS_1', site_resources) self.assertIn('IS_1', site_resources) self.assertNotIn('Obs_1', site_resources) self.assertEquals( len([v for v in site_resources.values() if v is None]), 3) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Obs_1', include_parents=False, id_only=False) self.assertEquals(len(site_resources), 3) self.assertEquals(len(site_children), 3) self.assertEquals( len([v for v in site_resources.values() if v is None]), 0) self.assertEquals(site_resources['Sub_1']._get_type(), RT.Subsite) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Obs_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) self.assertIn('Obs_1', site_resources) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Sub_1', include_parents=False) self.assertEquals(len(site_resources), 2) self.assertEquals(len(site_children), 2) self.assertNotIn('Sub_1', site_resources) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Sub_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) self.assertIn('Sub_1', site_resources) self.assertIn('Obs_1', site_resources) site_resources, site_children = self.spy_get_child_sites( parent_site_id='PS_1', include_parents=False) self.assertEquals(len(site_resources), 1) self.assertEquals(len(site_children), 1) site_resources, site_children = self.spy_get_child_sites( parent_site_id='PS_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) site_resources, site_children = self.spy_get_child_sites( parent_site_id='IS_1', include_parents=False) self.assertEquals(len(site_resources), 0) self.assertEquals(len(site_children), 0) site_resources, site_children = self.spy_get_child_sites( parent_site_id='IS_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) site_resources, site_children = self.spy_get_child_sites( parent_site_id='XXX', include_parents=True) self.assertEquals(len(site_resources), 1) self.assertEquals(len(site_children), 0) def test_get_child_sites_org(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=False, id_only=True) self.assertEquals(len(child_sites), 6) self.assertEquals(len(site_ancestors), 5) self.assertIn('Sub_1', child_sites) self.assertIn('PS_1', child_sites) self.assertIn('IS_1', child_sites) self.assertIn('Obs_1', child_sites) self.assertIn('Obs_2', child_sites) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=True, id_only=True) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=True, id_only=False) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) self.assertEquals(len([v for v in child_sites.values() if v is None]), 0) self.assertEquals(child_sites['Org_1']._get_type(), RT.Org) def test_get_site_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_devices = self.obs_util.get_site_devices( ['Sub_1', 'PS_1', 'IS_1']) self.assertEquals(len(site_devices), 3) self.assertEquals(site_devices['Sub_1'], []) self.assertEquals(site_devices['IS_1'], [('InstrumentSite', 'ID_1', 'InstrumentDevice')]) def test_get_child_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_devices = self.obs_util.get_child_devices('PD_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['PD_1'][0][1], 'ID_1') child_devices = self.obs_util.get_child_devices('ID_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['ID_1'], []) child_devices = self.obs_util.get_child_devices('Sub_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['Sub_1'], []) child_devices = self.obs_util.get_child_devices('XXX') self.assertEquals(len(child_devices), 1) event_list1 = [ dict(et='DeviceStatusEvent', o='ID_1', attr=dict(status=DeviceStatusType.STATUS_WARNING)) ] event_list2 = [ dict(et='DeviceStatusEvent', o='ID_1', attr=dict(status=DeviceStatusType.STATUS_WARNING)) ] event_list3 = [ dict(et='DeviceCommsEvent', o='ID_1', attr=dict(state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION)) ] event_list4 = [ dict(et='DeviceStatusEvent', o='PD_1', attr=dict(status=DeviceStatusType.STATUS_WARNING)), dict(et='DeviceCommsEvent', o='PD_1', attr=dict(state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION)) ] def _assert_status(self, status_rollups, res_id=None, agg=DeviceStatusType.STATUS_OK, loc=DeviceStatusType.STATUS_OK, data=DeviceStatusType.STATUS_OK, comms=DeviceStatusType.STATUS_OK, power=DeviceStatusType.STATUS_OK): res_status = status_rollups[res_id] if res_id else status_rollups self.assertEquals(len(res_status), 5) # #self.assertEquals(res_status['agg'], agg) # self.assertEquals(res_status['loc'], loc) # self.assertEquals(res_status['data'], data) # self.assertEquals(res_status['comms'], comms) # self.assertEquals(res_status['power'], power) res_list1 = [ dict(rt='DataProduct', _id='DP_1', attr={}), dict(rt='DataProduct', _id='DP_2', attr={}), dict(rt='DataProduct', _id='DP_3', attr={}), dict(rt='DataProduct', _id='DP_4', attr={}), dict(rt='DataProduct', _id='DP_5', attr={}), ] assoc_list3 = [ ['DP_1', 'hasSource', 'ID_1'], ['DP_2', 'hasSource', 'ID_1'], ['DP_3', 'hasSource', 'ID_1'], ['DP_3', 'hasSource', 'PD_1'], ['DP_4', 'hasSource', 'PD_1'], ['DP_5', 'hasSource', 'PD_1'], ] def test_get_device_data_products(self): self.mu.load_mock_resources(self.res_list + self.res_list1) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2 + self.assoc_list3) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) res_dict = self.obs_util.get_site_data_products( 'Obs_1', RT.Observatory) self.assertGreaterEqual(len(res_dict), 6) self.assertIsNone(res_dict['data_product_resources']) self.assertIn('ID_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products( 'PS_1', RT.PlatformSite) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('ID_1', res_dict['device_data_products']) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products('Org_1', RT.Org) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) res_dict = self.obs_util.get_site_data_products( 'PS_1', RT.PlatformSite, include_data_products=True) self.assertIsNotNone(res_dict['data_product_resources']) self.assertIn('DP_1', res_dict['data_product_resources'])
class DeploymentActivator(DeploymentOperator): """ A deployment activator validates that a set of devices will map to a set of sites in one unique way its primary purpose is to prepare( ) after which you'll be able to access what associations must be made (and unmade) it makes use of the deployment resource colelctor """ def on_init(self): resource_collector_factory = DeploymentResourceCollectorFactory(self.clients, self.RR2) self.resource_collector = resource_collector_factory.create(self.deployment_obj) self._hasdevice_associations_to_delete = [] self._hasdevice_associations_to_create = [] self.outil = ObservatoryUtil(self, enhanced_rr=self.RR2) # these are the output accessors def hasdevice_associations_to_delete(self): return self._hasdevice_associations_to_delete[:] def hasdevice_associations_to_create(self): return self._hasdevice_associations_to_create[:] # for debugging purposes def _csp_solution_to_string(self, soln): ret = "%s" % type(soln).__name__ for k, s in soln.iteritems(): d = unpack_csp_var(k) log.trace("reading device %s", d) dev_obj = self.resource_collector.read_using_typecache(d) log.trace("reading site %s", s) site_obj = self.resource_collector.read_using_typecache(s) ret = "%s, %s '%s' -> %s '%s'" % (ret, dev_obj._get_type(), d, site_obj._get_type(), s) return ret def prepare(self): """ Prepare (validate) a deployment for activation, returning lists of what associations need to be added and which ones need to be removed. """ # retrieve the site tree information using the OUTIL functions; site info as well has site children site_ids = self.RR2.find_subjects( subject_type=RT.PlatformSite, predicate=PRED.hasDeployment, object=self.deployment_obj._id, id_only=True ) if not site_ids: site_ids = self.RR2.find_subjects( subject_type=RT.InstrumentSite, predicate=PRED.hasDeployment, object=self.deployment_obj._id, id_only=True, ) if site_ids: self.site_resources, self.site_children = self.outil.get_child_sites( parent_site_id=site_ids[0], id_only=False ) log.debug("about to collect deployment components") self.resource_collector.collect() if not self.deployment_obj.port_assignments: log.info("No port assignments, so using CSP") pairs_to_add = self._prepare_using_csp() else: log.info("Merging trees with port assignments") pairs_to_add = self._prepare_using_portassignment_trees() log.info("Pairs to add: %s", pairs_to_add) # figure out if any of the devices in the new mapping are already mapped and need to be removed pairs_to_remove = [] pairs_to_ignore = [] for (s, d) in pairs_to_add: rm_pair, ignore_pair = self._find_existing_relationship(s, d) if rm_pair: pairs_to_remove.append(rm_pair) if ignore_pair: pairs_to_ignore.append(ignore_pair) log.info("Pairs to ignore (will be removed from add list): %s", pairs_to_ignore) # make sure that anything being removed is not also being added pairs_to_add = filter(lambda x: x not in pairs_to_ignore, pairs_to_add) self._hasdevice_associations_to_create = pairs_to_add self._hasdevice_associations_to_delete = pairs_to_remove log.info("Pairs to remove: %s", pairs_to_remove) def _prepare_using_portassignment_trees(self): # return a list of (site, device) pairs # badrequest if not all devices get consumed site_tree = self.resource_collector.collected_site_tree() device_tree = self.resource_collector.collected_device_tree() merged_tree_pairs, leftover_devices = self._merge_trees(site_tree, device_tree) if leftover_devices: raise BadRequest("Merging site and device trees resulted in %s unassigned devices" % len(leftover_devices)) return merged_tree_pairs def _resource_ids_in_tree(self, some_tree): # get all the resource ids stored in a tree def all_children_h(acc, t): acc.append(t["_id"]) for c, ct in t["children"].iteritems(): acc = all_children_h(acc[:], ct) return acc return all_children_h([], some_tree) def _merge_trees(self, site_tree, device_tree): # return a list of (site, device) pairs and a list of unmatched devices portref_of_device = self.deployment_obj.port_assignments def _merge_helper(acc, site_ptr, dev_ptr, unmatched_list): """ given 2 trees, try to match up all their children. assume roots already matched """ dev_id = dev_ptr["_id"] site_id = site_ptr["_id"] if not dev_ptr["model"] in site_ptr["models"]: log.warning( "Attempted to assign device '%s' to a site '%s' that doesn't support its model", dev_id, site_id ) if dev_id in unmatched_list: unmatched_list.remove(dev_id) acc.append((site_id, dev_id)) log.debug("Add to matched list site_id: %s dev_id: %s", site_id, dev_id) site_of_portref = {} # creat a dict of reference_designator on sites so that devices can be matched dev_site_obj = self.site_resources[site_id] site_of_portref[dev_site_obj.reference_designator] = site_id if site_id in self.site_children: for child in self.site_children[site_id]: dev_site_obj = self.site_resources[child] site_of_portref[dev_site_obj.reference_designator] = child for child_dev_id, child_dev_ptr in dev_ptr["children"].iteritems(): if not child_dev_id in portref_of_device: log.warning("No platform port information specified for device %s" % child_dev_id) dev_port = portref_of_device[child_dev_id] # check that a PlatformPort object is provided if dev_port.type_ != OT.PlatformPort: log.warning("No platform port information specified for device %s" % child_dev_id) ooi_rd = OOIReferenceDesignator(dev_port.reference_designator) if ooi_rd.error: log.warning( "Invalid OOIReferenceDesignator ( %s ) specified for device %s", dev_port.reference_designator, child_dev_id, ) if not ooi_rd.port: log.warning( "Invalid OOIReferenceDesignator ( %s ) specified for device %s, could not retrieve port", dev_port.reference_designator, child_dev_id, ) if dev_port.reference_designator in site_of_portref: child_site_id = site_of_portref[dev_port.reference_designator] child_site_ptr = site_ptr["children"][child_site_id] acc, unmatched_list = _merge_helper(acc[:], child_site_ptr, child_dev_ptr, unmatched_list[:]) else: log.warning("Couldn't find a port on site %s (%s) called '%s'", site_ptr["name"], site_id, dev_port) # this check is to match the ref_designator in the deployment object with the ref_designator in the target site # todo add ref_designators to the Sites in preload to match intended deployments return acc, unmatched_list unmatched_devices = self._resource_ids_in_tree(device_tree) return _merge_helper([], site_tree, device_tree, unmatched_devices) def _find_existing_relationship(self, site_id, device_id, site_type=None, device_type=None): # look for an existing relationship between the site_id and another device. # if this site/device pair already exists, we leave it alone assert type("") == type(site_id) == type(device_id) log.debug("checking %s/%s pair for deployment", site_type, device_type) # return a pair that should be REMOVED, or None if site_type is None: site_type = self.resource_collector.get_resource_type(site_id) if device_type is None: device_type = self.resource_collector.get_resource_type(device_id) log.debug("checking existing %s hasDevice %s links", site_type, device_type) ret_remove = None ret_ignore = None try: found_device_id = self.RR2.find_object(site_id, PRED.hasDevice, device_type, True) if found_device_id == device_id: ret_ignore = (site_id, device_id) else: ret_remove = (site_id, found_device_id) log.info("%s '%s' already hasDevice %s", site_type, site_id, device_type) except NotFound: pass return ret_remove, ret_ignore def _prepare_using_csp(self): """ use the previously collected resoures in a CSP problem """ site_tree = self.resource_collector.collected_site_tree() device_tree = self.resource_collector.collected_device_tree() device_models = self.resource_collector.collected_models_by_device() site_models = self.resource_collector.collected_models_by_site() log.debug("Collected %s device models, %s site models", len(device_models), len(site_models)) # csp solver can't handle multiple platforms, because it doesn't understand hierarchy. # (parent-platformsite---hasmodel-a, child-platformsite---hasmodel-b) # would match (parent-platformdevice-hasmodel-b, child-platformdevice-hasmodel-a) # # we can avoid this by simply restricting the deployment to 1 platform device/site in this case # n_pdev = sum(RT.PlatformDevice == self.resource_collector.get_resource_type(d) for d in device_models.keys()) # if 1 < n_pdev: # raise BadRequest("Deployment activation without port_assignment is limited to 1 PlatformDevice, got %s" % n_pdev) # # n_psite = sum(RT.PlatformSite == self.resource_collector.get_resource_type(d) for d in site_models.keys()) # if 1 < n_psite: # raise BadRequest("Deployment activation without port_assignment is limited to 1 PlatformSite, got %s" % n_psite) solutions = self._get_deployment_csp_solutions(device_tree, site_tree, device_models, site_models) if 1 > len(solutions): raise BadRequest( "The set of devices could not be mapped to the set of sites, based on matching " + "models" ) # and streamdefs") if 1 == len(solutions): log.info("Found one possible way to map devices and sites. Best case scenario!") else: log.info("Found %d possible ways to map device and site", len(solutions)) log.trace("Here is the %s of all of them:", type(solutions).__name__) for i, s in enumerate(solutions): log.trace("Option %d: %s", i + 1, self._csp_solution_to_string(s)) uhoh = ( "The set of devices could be mapped to the set of sites in %s ways based only " + "on matching models, and no port assignments were specified." ) % len(solutions) # raise BadRequest(uhoh) log.warn(uhoh + " PICKING THE FIRST AVAILABLE OPTION.") # return list of site_id, device_id return [(solutions[0][mk_csp_var(device_id)], device_id) for device_id in device_models.keys()] def _get_deployment_csp_solutions(self, device_tree, site_tree, device_models, site_models): log.debug("creating a CSP solver to match devices and sites") problem = constraint.Problem() def safe_get_parent(child_site_id): try: return self.RR2.find_subject(RT.PlatformSite, PRED.hasSite, child_site_id, id_only=True) except NotFound: return None log.debug("adding variables to CSP - the devices to be assigned, and their range (possible sites)") for device_id in device_models.keys(): device_model = device_models[device_id] assert type(device_model) == type("") assert all([type("") == type(s) for s in site_models]) possible_sites = [s for s in site_models.keys() if device_model in site_models[s]] if not possible_sites: log.info("Device model: %s", device_model) log.info("Site models: %s", site_models) raise BadRequest("No sites in the deployment match the model of device '%s'" % device_id) device_var = mk_csp_var(device_id) problem.addVariable(device_var, possible_sites) # add parent-child constraints try: parent_device_id = self.RR2.find_subject(RT.PlatformDevice, PRED.hasDevice, device_id, id_only=True) if parent_device_id in device_models: problem.addConstraint( lambda child_site, parent_site: parent_site == safe_get_parent(child_site), [device_var, mk_csp_var(parent_device_id)], ) except NotFound: log.debug("Device '%s' has no parent", device_id) # no big deal log.debug("adding the constraint that all the variables have to pick their own site") problem.addConstraint( constraint.AllDifferentConstraint(), [mk_csp_var(device_id) for device_id in device_models.keys()] ) log.debug("performing CSP solve") # this will be a list of solutions, each a dict of var -> value return problem.getSolutions()
def prepare_activation(self, deployment_obj): """ Prepare (validate) a deployment for activation, returning lists of what associations need to be added and which ones need to be removed. """ self.match_list = [] self.remove_list = [] self.unmatched_device_list = [] self.models_map = {} self.top_device = '' self.top_site = '' self.deployment_obj = deployment_obj self.site_resources = {} self.device_resources = {} self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) # retrieve the site tree information using the OUTIL functions; site info as well has site children self.top_site, self.top_device = self._find_top_site_device(deployment_obj._id) # must have a site and a device to continue if not self.top_site or not self.top_device: return [], [] log.debug("port_assignments: %s", self.deployment_obj.port_assignments ) # retrieve all models to use in match validation self._get_models() self.site_resources, site_children = self.outil.get_child_sites( parent_site_id=self.top_site._id, id_only=False) log.debug("site_resources: %s", self.site_resources) log.debug("site_children: %s", site_children) site_ref_designator_map = self._get_site_ref_designator_map() # retrieve the device tree from outil then cache the device resources device_tree = self.outil.get_child_devices(device_id=self.top_device._id) self._get_device_resources(device_tree) def _match_devices(device_id): # there will not be a port assignment for the top device if device_id == self.top_device._id: self._validate_models(self.top_site._id, self.top_device._id) self.match_list.append((self.top_site._id, self.top_device._id)) tuple_list = device_tree[device_id] for (pt, child_id, ct) in tuple_list: log.debug(" tuple - pt: %s child_id: %s ct: %s", pt, child_id, ct) # match this child device then if it has children, call _match_devices with this id # check that this device is represented in device tree and in port assignments if child_id in self.device_resources and child_id in self.deployment_obj.port_assignments: platform_port = self.deployment_obj.port_assignments[child_id] log.debug("device platform_port: %s", platform_port) # validate PlatformPort info for this device self._validate_port_assignments(child_id, platform_port) if platform_port.reference_designator in site_ref_designator_map: matched_site = site_ref_designator_map[platform_port.reference_designator] self._validate_models(matched_site, child_id) log.info("match_list append site: %s device: %s", matched_site, child_id) self.match_list.append((matched_site, child_id)) #recurse on the children of this device _match_devices(child_id) # otherwise cant be matched to a site else: self.unmatched_device_list.append(child_id) _match_devices(self.top_device._id) # check for hasDevice relations to remove and existing hasDevice relations self. _find_pairs_to_remove() if self.unmatched_device_list: log.warning("Devices not matched to sites: %s ", self.unmatched_device_list) return self.remove_list, self.match_list
def _process_cmd_sites(resource_id, res_obj=None): from ion.services.sa.observatory.observatory_util import ObservatoryUtil outil = ObservatoryUtil(container=Container.instance) statuses = outil.get_status_roll_ups(resource_id, include_structure=True) fragments = [ "</pre><h3>Org, Site and Device Status</h3>", ] if '_system' in statuses: extra = statuses['_system'] child_sites, ancestors, devices = extra.get('sites', {}), extra.get('ancestors', {}), extra.get('devices', {}) root_id = outil.get_site_root(resource_id, ancestors=ancestors) if ancestors else resource_id fragments.append("<p><table>") fragments.append("<tr><th>Resource</th><th>Type</th><th>AGG</th><th>PWR</th><th>COMM</th><th>DATA</th><th>LOC</th></tr>") device_info = {} if devices: dev_id_list = [dev[1] for dev in devices.values() if dev is not None] if dev_id_list: dev_list = Container.instance.resource_registry.read_mult(dev_id_list) device_info = dict(zip([res._id for res in dev_list], dev_list)) elif ancestors: dev_id_list = [anc for anc_list in ancestors.values() if anc_list is not None for anc in anc_list] dev_id_list.append(resource_id) dev_list = Container.instance.resource_registry.read_mult(dev_id_list) device_info = dict(zip([res._id for res in dev_list], dev_list)) def stat(status, stype): stat = status.get(stype, 4) stat_str = ['', "<span style='color:green'>OK</span>","<span style='color:orange'>WARN</span>","<span style='color:red'>ERROR</span>",'?'] return stat_str[stat] def status_table(parent_id, level): fragments.append("<tr>") par_detail = child_sites.get(parent_id, None) or device_info.get(parent_id, None) par_status = statuses.get(parent_id, {}) entryname = " "*level + build_link(par_detail.name if par_detail else parent_id, "/view/%s" % parent_id) if parent_id == resource_id: entryname = "<b>" + entryname + "</b>" fragments.append("<td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td>" % ( entryname, par_detail._get_type() if par_detail else "?", stat(par_status, 'agg'), stat(par_status, 'power'), stat(par_status, 'comms'), stat(par_status, 'data'), stat(par_status, 'loc'))) fragments.append("</tr>") device = devices.get(parent_id, None) if device: status_table(device[1], level+1) ch_ids = ancestors.get(parent_id, None) or [] for ch_id in ch_ids: status_table(ch_id, level+1) status_table(root_id, 0) fragments.append("</table></p>") fragments.append("<pre>%s</pre>" % (pprint.pformat(statuses))), else: fragments.append("<pre>%s</pre>" % (pprint.pformat(statuses))), fragments.append("<pre>") content = "\n".join(fragments) return content
class TestObservatoryUtil(IonUnitTestCase): def setUp(self): self.mu = MockUtil() self.process_mock = self.mu.create_process_mock() self.container_mock = self.mu.create_container_mock() self.dsm_mock = self.mu.create_device_status_manager_mock() res_list = [ dict(rt='Org', _id='Org_1', attr={}), dict(rt='Observatory', _id='Obs_1', attr={}), dict(rt='Observatory', _id='Obs_2', attr={}), dict(rt='Subsite', _id='Sub_1', attr={}), dict(rt='Subsite', _id='Sub_2', attr={}), dict(rt='PlatformSite', _id='PS_1', attr={}), dict(rt='InstrumentSite', _id='IS_1', attr={}), dict(rt='PlatformDevice', _id='PD_1', attr={}), dict(rt='InstrumentDevice', _id='ID_1', attr={}), ] assoc_list = [ ['Obs_1', 'hasSite', 'Sub_1'], ['Sub_1', 'hasSite', 'PS_1'], ['PS_1', 'hasSite', 'IS_1'], ] assoc_list1 = [ ['Org_1', 'hasResource', 'Obs_1'], ['Org_1', 'hasResource', 'Obs_2'], ['Obs_2', 'hasSite', 'Sub_2'], ] assoc_list2 = [ ['PS_1', 'hasDevice', 'PD_1'], ['IS_1', 'hasDevice', 'ID_1'], ['PD_1', 'hasDevice', 'ID_1'], ] def spy_get_child_sites(self, parent_site_id=None, org_id=None, exclude_types=None, include_parents=True, id_only=True): child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id=parent_site_id, org_id=org_id, exclude_types=exclude_types, include_parents=include_parents, id_only=id_only) print "child_sites of", parent_site_id, "are", child_sites print "site_ancestors of", parent_site_id, "are", site_ancestors return child_sites, site_ancestors def test_get_child_sites(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Obs_1', include_parents=False, id_only=True) self.assertEquals(len(site_resources), 3) self.assertEquals(len(site_children), 3) self.assertIn('Sub_1', site_resources) self.assertIn('PS_1', site_resources) self.assertIn('IS_1', site_resources) self.assertNotIn('Obs_1', site_resources) self.assertEquals( len([v for v in site_resources.values() if v is None]), 3) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Obs_1', include_parents=False, id_only=False) self.assertEquals(len(site_resources), 3) self.assertEquals(len(site_children), 3) self.assertEquals( len([v for v in site_resources.values() if v is None]), 0) self.assertEquals(site_resources['Sub_1']._get_type(), RT.Subsite) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Obs_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) self.assertIn('Obs_1', site_resources) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Sub_1', include_parents=False) self.assertEquals(len(site_resources), 2) self.assertEquals(len(site_children), 2) self.assertNotIn('Sub_1', site_resources) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Sub_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) self.assertIn('Sub_1', site_resources) self.assertIn('Obs_1', site_resources) site_resources, site_children = self.spy_get_child_sites( parent_site_id='PS_1', include_parents=False) self.assertEquals(len(site_resources), 1) self.assertEquals(len(site_children), 1) site_resources, site_children = self.spy_get_child_sites( parent_site_id='PS_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) site_resources, site_children = self.spy_get_child_sites( parent_site_id='IS_1', include_parents=False) self.assertEquals(len(site_resources), 0) self.assertEquals(len(site_children), 0) site_resources, site_children = self.spy_get_child_sites( parent_site_id='IS_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) site_resources, site_children = self.spy_get_child_sites( parent_site_id='XXX', include_parents=True) self.assertEquals(len(site_resources), 1) self.assertEquals(len(site_children), 0) def test_get_child_sites_org(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=False, id_only=True) self.assertEquals(len(child_sites), 6) self.assertEquals(len(site_ancestors), 5) self.assertIn('Sub_1', child_sites) self.assertIn('PS_1', child_sites) self.assertIn('IS_1', child_sites) self.assertIn('Obs_1', child_sites) self.assertIn('Obs_2', child_sites) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=True, id_only=True) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=True, id_only=False) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) self.assertEquals(len([v for v in child_sites.values() if v is None]), 0) self.assertEquals(child_sites['Org_1']._get_type(), RT.Org) def test_get_site_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_devices = self.obs_util.get_site_devices( ['Sub_1', 'PS_1', 'IS_1']) self.assertEquals(len(site_devices), 3) self.assertEquals(site_devices['Sub_1'], []) self.assertEquals(site_devices['IS_1'], [('InstrumentSite', 'ID_1', 'InstrumentDevice')]) def test_get_child_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_devices = self.obs_util.get_child_devices('PD_1') self.assertEquals(len(child_devices), 2) self.assertEquals(child_devices['PD_1'][0][1], 'ID_1') child_devices = self.obs_util.get_child_devices('ID_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['ID_1'], []) child_devices = self.obs_util.get_child_devices('Sub_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['Sub_1'], []) child_devices = self.obs_util.get_child_devices('XXX') self.assertEquals(len(child_devices), 1) def test_get_device_data_products(self): self.mu.load_mock_resources(self.res_list + self.res_list1) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2 + self.assoc_list3) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) res_dict = self.obs_util.get_site_data_products( 'Obs_1', RT.Observatory) self.assertGreaterEqual(len(res_dict), 6) self.assertIsNone(res_dict['data_product_resources']) self.assertIn('ID_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products( 'PS_1', RT.PlatformSite) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('ID_1', res_dict['device_data_products']) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products('Org_1', RT.Org) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) res_dict = self.obs_util.get_site_data_products( 'PS_1', RT.PlatformSite, include_data_products=True) self.assertIsNotNone(res_dict['data_product_resources']) self.assertIn('DP_1', res_dict['data_product_resources']) #import pprint #pprint.pprint(res_dict) status_by_device_1 = { "ID_1": _devstat("ID_1", DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), "PD_1": _devstat("PD_1", DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), } status_by_device_2 = { "ID_1": _devstat("ID_1", DST.STATUS_WARNING, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), "PD_1": _devstat("PD_1", DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), } status_by_device_3 = { "ID_1": _devstat("ID_1", DST.STATUS_WARNING, DST.STATUS_WARNING, DST.STATUS_OK, DST.STATUS_OK), "PD_1": _devstat("PD_1", DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), } status_by_device_4 = { "ID_1": _devstat("ID_1", DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK, DST.STATUS_OK), "PD_1": _devstat("PD_1", DST.STATUS_WARNING, DST.STATUS_WARNING, DST.STATUS_OK, DST.STATUS_OK), } def test_get_status_roll_ups(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_device_statuses(self.status_by_device_1) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock, device_status_mgr=self.dsm_mock) # No problems status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) self.assertIn('PD_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1') self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') # ID_1 power warning self.mu.load_mock_device_statuses(self.status_by_device_2) status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertIn('PS_1', status_rollups) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING) # ID_1 power+comms warning self.mu.load_mock_device_statuses(self.status_by_device_3) status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) def test_get_status_roll_ups_platform_warn(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_device_statuses(self.status_by_device_4) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock, device_status_mgr=self.dsm_mock) # PD_1 power+comms warning status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) #log.warn("status %s" % status_rollups) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') def _assert_status(self, status_rollups, res_id=None, agg=DST.STATUS_OK, loc=DST.STATUS_OK, data=DST.STATUS_OK, comms=DST.STATUS_OK, power=DST.STATUS_OK): res_status = status_rollups[res_id] if res_id else status_rollups log.debug("_assert_status(%s) = %s", res_id, res_status) self.assertEquals(len(res_status), 5) if agg is not None: self.assertEquals(res_status['agg'], agg) if loc is not None: self.assertEquals( res_status[AggregateStatusType.AGGREGATE_LOCATION], loc) if data is not None: self.assertEquals(res_status[AggregateStatusType.AGGREGATE_DATA], data) if comms is not None: self.assertEquals(res_status[AggregateStatusType.AGGREGATE_COMMS], comms) if power is not None: self.assertEquals(res_status[AggregateStatusType.AGGREGATE_POWER], power) res_list1 = [ dict(rt='DataProduct', _id='DP_1', attr={}), dict(rt='DataProduct', _id='DP_2', attr={}), dict(rt='DataProduct', _id='DP_3', attr={}), dict(rt='DataProduct', _id='DP_4', attr={}), dict(rt='DataProduct', _id='DP_5', attr={}), ] assoc_list3 = [ ['DP_1', 'hasSource', 'ID_1'], ['DP_1', 'hasSource', 'IS_1'], ['DP_2', 'hasSource', 'ID_1'], ['DP_3', 'hasSource', 'ID_1'], ['DP_3', 'hasSource', 'PD_1'], ['DP_4', 'hasSource', 'PD_1'], ['DP_5', 'hasSource', 'PD_1'], ]
class ObservatoryManagementService(BaseObservatoryManagementService): def on_init(self): IonObject("Resource") # suppress pyflakes error CFG, log, RT, PRED, LCS, LCE, NotFound, BadRequest, log #suppress pyflakes errors about "unused import" self.override_clients(self.clients) self.outil = ObservatoryUtil(self) self.HIERARCHY_DEPTH = {RT.InstrumentSite: 3, RT.PlatformSite: 2, RT.Subsite: 1, RT.Observatory: 0, } self.HIERARCHY_LOOKUP = [RT.Observatory, RT.Subsite, RT.PlatformSite, RT.InstrumentSite] #todo: add lcs methods for these?? # # set up all of the policy interceptions # if self.container and self.container.governance_controller: # reg_precondition = self.container.governance_controller.register_process_operation_precondition # reg_precondition(self, 'execute_observatory_lifecycle', # self.RR2.policy_fn_lcs_precondition("observatory_id")) # reg_precondition(self, 'execute_subsite_lifecycle', # self.RR2.policy_fn_lcs_precondition("subsite_id")) # reg_precondition(self, 'execute_platform_site_lifecycle', # self.RR2.policy_fn_lcs_precondition("platform_site_id")) # reg_precondition(self, 'execute_instrument_site_lifecycle', # self.RR2.policy_fn_lcs_precondition("instrument_site_id")) def override_clients(self, new_clients): """ Replaces the service clients with a new set of them... and makes sure they go to the right places """ self.RR2 = EnhancedResourceRegistryClient(new_clients.resource_registry) #shortcut names for the import sub-services if hasattr(new_clients, "resource_registry"): self.RR = new_clients.resource_registry if hasattr(new_clients, "instrument_management"): self.IMS = new_clients.instrument_management if hasattr(new_clients, "data_process_management"): self.PRMS = new_clients.data_process_management #farm everything out to the impls self.dataproductclient = DataProductManagementServiceClient() self.dataprocessclient = DataProcessManagementServiceClient() def _calc_geospatial_point_center(self, site): siteTypes = [RT.Site, RT.Subsite, RT.Observatory, RT.PlatformSite, RT.InstrumentSite] if site and site.type_ in siteTypes: # if the geospatial_bounds is set then calculate the geospatial_point_center for constraint in site.constraint_list: if constraint.type_ == OT.GeospatialBounds: site.geospatial_point_center = GeoUtils.calc_geospatial_point_center(constraint) ########################################################################## # # CRUD OPS # ########################################################################## def create_marine_facility(self, org=None): """Create an Org (domain of authority) that realizes a marine facility. This Org will have set up roles for a marine facility. Shared resources, such as a device can only be registered in one marine facility Org, and additionally in many virtual observatory Orgs. The marine facility operators will have more extensive permissions and will supercede virtual observatory commands @param org Org @retval org_id str @throws BadRequest if object does not have _id or _rev attribute @throws NotFound object with specified id does not exist """ log.debug("ObservatoryManagementService.create_marine_facility(): %s", org) # create the org org.org_type = OrgTypeEnum.MARINE_FACILITY org_id = self.clients.org_management.create_org(org) #Instantiate initial set of User Roles for this marine facility instrument_operator_role = IonObject(RT.UserRole, governance_name=INSTRUMENT_OPERATOR_ROLE, name='Observatory Operator', #previously Instrument Operator description='Operate and post events related to Observatory Platforms and Instruments') self.clients.org_management.add_user_role(org_id, instrument_operator_role) observatory_operator_role = IonObject(RT.UserRole, governance_name=OBSERVATORY_OPERATOR_ROLE, name='Observatory Manager', # previously Observatory Operator description='Change Observatory configuration, post Site-related events') self.clients.org_management.add_user_role(org_id, observatory_operator_role) data_operator_role = IonObject(RT.UserRole, governance_name=DATA_OPERATOR_ROLE, name='Observatory Data Operator', # previously Data Operator description='Manipulate and post events related to Observatory Data products') self.clients.org_management.add_user_role(org_id, data_operator_role) return org_id def create_virtual_observatory(self, org=None): """Create an Org (domain of authority) that realizes a virtual observatory. This Org will have set up roles for a virtual observatory. Shared resources, such as a device can only be registered in one marine facility Org, and additionally in many virtual observatory Orgs. The marine facility operators will have more extensive permissions and will supercede virtual observatory commands @param org Org @retval org_id str @throws BadRequest if object does not have _id or _rev attribute @throws NotFound object with specified id does not exist """ log.debug("ObservatoryManagementService.create_virtual_observatory(): %s", org) # create the org org.org_type = OrgTypeEnum.VIRTUAL_OBSERVATORY org_id = self.clients.org_management.create_org(org) return org_id def create_observatory(self, observatory=None, org_id=""): """Create a Observatory resource. An observatory is coupled with one Org. The Org is created and associated as part of this call. @param observatory Observatory @retval observatory_id str @throws BadRequest if object does not have _id or _rev attribute @throws NotFound object with specified id does not exist """ # if the geospatial_bounds is set then calculate the geospatial_point_center self._calc_geospatial_point_center(observatory) # create the marine facility observatory_id = self.RR2.create(observatory, RT.Observatory) if org_id: self.assign_resource_to_observatory_org(observatory_id, org_id) return observatory_id def read_observatory(self, observatory_id=''): """Read a Observatory resource @param observatory_id str @retval observatory Observatory @throws NotFound object with specified id does not exist """ return self.RR2.read(observatory_id, RT.Observatory) def update_observatory(self, observatory=None): """Update a Observatory resource @param observatory Observatory @throws NotFound object with specified id does not exist """ # if the geospatial_bounds is set then calculate the geospatial_point_center self._calc_geospatial_point_center(observatory) return self.RR2.update(observatory, RT.Observatory) def delete_observatory(self, observatory_id=''): """Delete a Observatory resource @param observatory_id str @throws NotFound object with specified id does not exist """ return self.RR2.retire(observatory_id, RT.Observatory) def force_delete_observatory(self, observatory_id=''): return self.RR2.pluck_delete(observatory_id, RT.Observatory) def create_subsite(self, subsite=None, parent_id=''): """Create a Subsite resource. A subsite is a frame of reference within an observatory. Its parent is either the observatory or another subsite. @param subsite Subsite @param parent_id str @retval subsite_id str @throws BadRequest if object does not have _id or _rev attribute @throws NotFound object with specified id does not exist """ # if the geospatial_bounds is set then calculate the geospatial_point_center self._calc_geospatial_point_center(subsite) subsite_id = self.RR2.create(subsite, RT.Subsite) if parent_id: self.assign_site_to_site(subsite_id, parent_id) return subsite_id def read_subsite(self, subsite_id=''): """Read a Subsite resource @param subsite_id str @retval subsite Subsite @throws NotFound object with specified id does not exist """ return self.RR2.read(subsite_id, RT.Subsite) def update_subsite(self, subsite=None): """Update a Subsite resource @param subsite Subsite @throws NotFound object with specified id does not exist """ # if the geospatial_bounds is set then calculate the geospatial_point_center self._calc_geospatial_point_center(subsite) return self.RR2.update(subsite, RT.Subsite) def delete_subsite(self, subsite_id=''): """Delete a subsite resource, removes assocations to parents @param subsite_id str @throws NotFound object with specified id does not exist """ self.RR2.retire(subsite_id, RT.Subsite) def force_delete_subsite(self, subsite_id=''): self.RR2.pluck_delete(subsite_id, RT.Subsite) def create_platform_site(self, platform_site=None, parent_id=''): """Create a PlatformSite resource. A platform_site is a frame of reference within an observatory. Its parent is either the observatory or another platform_site. @param platform_site PlatformSite @param parent_id str @retval platform_site_id str @throws BadRequest if object does not have _id or _rev attribute @throws NotFound object with specified id does not exist """ # if the geospatial_bounds is set then calculate the geospatial_point_center self._calc_geospatial_point_center(platform_site) platform_site_id = self.RR2.create(platform_site, RT.PlatformSite) if parent_id: self.RR2.assign_site_to_one_site_with_has_site(platform_site_id, parent_id) return platform_site_id def read_platform_site(self, platform_site_id=''): """Read a PlatformSite resource @param platform_site_id str @retval platform_site PlatformSite @throws NotFound object with specified id does not exist """ return self.RR2.read(platform_site_id, RT.PlatformSite) def update_platform_site(self, platform_site=None): """Update a PlatformSite resource @param platform_site PlatformSite @throws NotFound object with specified id does not exist """ # if the geospatial_bounds is set then calculate the geospatial_point_center self._calc_geospatial_point_center(platform_site) return self.RR2.update(platform_site, RT.PlatformSite) def delete_platform_site(self, platform_site_id=''): """Delete a PlatformSite resource, removes assocations to parents @param platform_site_id str @throws NotFound object with specified id does not exist """ self.RR2.retire(platform_site_id, RT.PlatformSite) def force_delete_platform_site(self, platform_site_id=''): self.RR2.pluck_delete(platform_site_id, RT.PlatformSite) def create_instrument_site(self, instrument_site=None, parent_id=''): """Create a InstrumentSite resource. A instrument_site is a frame of reference within an observatory. Its parent is either the observatory or another instrument_site. @param instrument_site InstrumentSite @param parent_id str @retval instrument_site_id str @throws BadRequest if object does not have _id or _rev attribute @throws NotFound object with specified id does not exist """ # if the geospatial_bounds is set then calculate the geospatial_point_center self._calc_geospatial_point_center(instrument_site) instrument_site_id = self.RR2.create(instrument_site, RT.InstrumentSite) if parent_id: self.RR2.assign_site_to_one_site_with_has_site(instrument_site_id, parent_id) return instrument_site_id def read_instrument_site(self, instrument_site_id=''): """Read a InstrumentSite resource @param instrument_site_id str @retval instrument_site InstrumentSite @throws NotFound object with specified id does not exist """ return self.RR2.read(instrument_site_id, RT.InstrumentSite) def update_instrument_site(self, instrument_site=None): """Update a InstrumentSite resource @param instrument_site InstrumentSite @throws NotFound object with specified id does not exist """ # if the geospatial_bounds is set then calculate the geospatial_point_center self._calc_geospatial_point_center(instrument_site) return self.RR2.update(instrument_site, RT.InstrumentSite) def delete_instrument_site(self, instrument_site_id=''): """Delete a InstrumentSite resource, removes assocations to parents @param instrument_site_id str @throws NotFound object with specified id does not exist """ # todo: give InstrumentSite a lifecycle in COI so that we can remove the "True" argument here self.RR2.retire(instrument_site_id, RT.InstrumentSite) def force_delete_instrument_site(self, instrument_site_id=''): self.RR2.pluck_delete(instrument_site_id, RT.InstrumentSite) #todo: convert to resource_impl def create_deployment(self, deployment=None, site_id="", device_id=""): """ Create a Deployment resource. Represents a (possibly open-ended) time interval grouping one or more resources within a given context, such as an instrument deployment on a platform at an observatory site. """ deployment_id = self.RR2.create(deployment, RT.Deployment) #Verify that site and device exist, add links if they do if site_id: site_obj = self.RR2.read(site_id) if site_obj: self.RR2.assign_deployment_to_site_with_has_deployment(deployment_id, site_id) if device_id: device_obj = self.RR2.read(device_id) if device_obj: self.RR2.assign_deployment_to_device_with_has_deployment(deployment_id, device_id) return deployment_id def update_deployment(self, deployment=None): # Overwrite Deployment object self.RR2.update(deployment, RT.Deployment) def read_deployment(self, deployment_id=''): deployment_obj = self.RR2.read(deployment_id, RT.Deployment) return deployment_obj def delete_deployment(self, deployment_id=''): """ Delete a Deployment resource """ self.RR2.retire(deployment_id, RT.Deployment) def force_delete_deployment(self, deployment_id=''): self.RR2.pluck_delete(deployment_id, RT.Deployment) ############################ # # ASSOCIATIONS # ############################ def assign_site_to_site(self, child_site_id='', parent_site_id=''): """Connects a child site (any subtype) to a parent site (any subtype) @param child_site_id str @param parent_site_id str @throws NotFound object with specified id does not exist """ self.RR2.assign_site_to_site_with_has_site(child_site_id, parent_site_id) def unassign_site_from_site(self, child_site_id='', parent_site_id=''): """Disconnects a child site (any subtype) from a parent site (any subtype) @param child_site_id str @param parent_site_id str @throws NotFound object with specified id does not exist """ self.RR2.unassign_site_from_site_with_has_site(child_site_id, parent_site_id) def assign_device_to_site(self, device_id='', site_id=''): """Connects a device (any type) to a site (any subtype) @param device_id str @param site_id str @throws NotFound object with specified id does not exist """ self.RR2.assign_device_to_site_with_has_device(device_id, site_id) def unassign_device_from_site(self, device_id='', site_id=''): """Disconnects a device (any type) from a site (any subtype) @param device_id str @param site_id str @throws NotFound object with specified id does not exist """ self.RR2.unassign_device_from_site_with_has_device(device_id, site_id) def assign_device_to_network_parent(self, child_device_id='', parent_device_id=''): """Connects a device (any type) to parent in the RSN network @param child_device_id str @param parent_device_id str @throws NotFound object with specified id does not exist """ self.RR2.assign_device_to_one_device_with_has_network_parent(parent_device_id, child_device_id) def unassign_device_from_network_parent(self, child_device_id='', parent_device_id=''): """Disconnects a child device (any type) from parent in the RSN network @param child_device_id str @param parent_device_id str @throws NotFound object with specified id does not exist """ self.RR2.unassign_device_from_device_with_has_network_parent(parent_device_id, child_device_id) def assign_instrument_model_to_instrument_site(self, instrument_model_id='', instrument_site_id=''): self.RR2.assign_instrument_model_to_instrument_site_with_has_model(instrument_model_id, instrument_site_id) def unassign_instrument_model_from_instrument_site(self, instrument_model_id='', instrument_site_id=''): self.RR2.unassign_instrument_model_from_instrument_site_with_has_model(self, instrument_model_id, instrument_site_id) def assign_platform_model_to_platform_site(self, platform_model_id='', platform_site_id=''): self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id, platform_site_id) def unassign_platform_model_from_platform_site(self, platform_model_id='', platform_site_id=''): self.RR2.unassign_platform_model_from_platform_site_with_has_model(platform_model_id, platform_site_id) def assign_resource_to_observatory_org(self, resource_id='', org_id=''): if not org_id: raise BadRequest("Org id not given") if not resource_id: raise BadRequest("Resource id not given") #log.trace("assign_resource_to_observatory_org: org_id=%s, resource_id=%s ", org_id, resource_id) self.clients.org_management.share_resource(org_id, resource_id) def unassign_resource_from_observatory_org(self, resource_id='', org_id=''): if not org_id: raise BadRequest("Org id not given") if not resource_id: raise BadRequest("Resource id not given") self.clients.org_management.unshare_resource(org_id, resource_id) ########################################################################## # # DEPLOYMENTS # ########################################################################## def deploy_instrument_site(self, instrument_site_id='', deployment_id=''): self.RR2.assign_deployment_to_instrument_site_with_has_deployment(deployment_id, instrument_site_id) def undeploy_instrument_site(self, instrument_site_id='', deployment_id=''): self.RR2.unassign_deployment_from_instrument_site_with_has_deployment(deployment_id, instrument_site_id) def deploy_platform_site(self, platform_site_id='', deployment_id=''): self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id) def undeploy_platform_site(self, platform_site_id='', deployment_id=''): self.RR2.unassign_deployment_from_platform_site_with_has_deployment(deployment_id, platform_site_id) def activate_deployment(self, deployment_id='', activate_subscriptions=False): """ Make the devices on this deployment the primary devices for the sites """ #Verify that the deployment exists depl_obj = self.RR2.read(deployment_id) log.debug("Activing deployment '%s' (%s)", depl_obj.name, deployment_id) deployment_activator_factory = DeploymentActivatorFactory(self.clients) deployment_activator = deployment_activator_factory.create(depl_obj) deployment_activator.prepare() # process any removals for site_id, device_id in deployment_activator.hasdevice_associations_to_delete(): log.info("Unassigning hasDevice; device '%s' from site '%s'", device_id, site_id) self.unassign_device_from_site(device_id, site_id) # process the additions for site_id, device_id in deployment_activator.hasdevice_associations_to_create(): log.info("Setting primary device '%s' for site '%s'", device_id, site_id) self.assign_device_to_site(device_id, site_id) # self.RR.execute_lifecycle_transition(deployment_id, LCE.DEPLOY) def deactivate_deployment(self, deployment_id=''): """Remove the primary device designation for the deployed devices at the sites @param deployment_id str @throws NotFound object with specified id does not exist @throws BadRequest if devices can not be undeployed """ #Verify that the deployment exists deployment_obj = self.RR2.read(deployment_id) # if LCS.DEPLOYED != deployment_obj.lcstate: # raise BadRequest("This deploment is not active") # get all associated components collector_factory = DeploymentResourceCollectorFactory(self.clients) resource_collector = collector_factory.create(deployment_obj) resource_collector.collect() # must only remove from sites that are not deployed under a different active deployment # must only remove devices that are not deployed under a different active deployment def filter_alternate_deployments(resource_list): # return the list of ids for devices or sites not connected to an alternate lcs.deployed deployment ret = [] for r in resource_list: depls, _ = self.RR.find_objects(r, PRED.hasDeployment, RT.Deployment) keep = True for d in depls: if d._id != deployment_id and LCS.DEPLOYED == d.lcstate: keep = False if keep: ret.append(r) return ret device_ids = filter_alternate_deployments(resource_collector.collected_device_ids()) site_ids = filter_alternate_deployments(resource_collector.collected_site_ids()) # delete only associations where both site and device have passed the filter for s in site_ids: ds, _ = self.RR.find_objects(s, PRED.hasDevice, id_only=True) for d in ds: if d in device_ids: a = self.RR.get_association(s, PRED.hasDevice, d) self.RR.delete_association(a) # # # mark deployment as not deployed (developed seems appropriate) # self.RR.execute_lifecycle_transition(deployment_id, LCE.DEVELOPED) ########################################################################## # # FIND OPS # ########################################################################## def find_org_by_observatory(self, observatory_id=''): """ """ orgs,_ = self.RR.find_subjects(RT.Org, PRED.hasResource, observatory_id, id_only=False) return orgs def find_related_frames_of_reference(self, input_resource_id='', output_resource_type_list=None): # use the related resources crawler finder = RelatedResourcesCrawler() # generate the partial function (cached association list) get_assns = finder.generate_related_resources_partial(self.RR, [PRED.hasSite]) # run 2 searches allowing all site-based resource types: one down (subj-obj), one up (obj-subj) full_crawllist = [RT.InstrumentSite, RT.PlatformSite, RT.Subsite, RT.Observatory] search_down = get_assns({PRED.hasSite: (True, False)}, full_crawllist) search_up = get_assns({PRED.hasSite: (False, True)}, full_crawllist) # the searches return a list of association objects, so compile all the ids by extracting them retval_ids = set([]) # we want only those IDs that are not the input resource id for a in search_down(input_resource_id, -1) + search_up(input_resource_id, -1): if a.o not in retval_ids and a.o != input_resource_id: retval_ids.add(a.o) if a.s not in retval_ids and a.s != input_resource_id: retval_ids.add(a.s) log.trace("converting retrieved ids to objects = %s" % retval_ids) #initialize the dict retval = dict((restype, []) for restype in output_resource_type_list) #workaround for read_mult problem all_res = [] if retval_ids: all_res = self.RR.read_mult(list(retval_ids)) #all_res = self.RR.read_mult(retval_ids) # put resources in the slot based on their type for resource in all_res: typename = type(resource).__name__ if typename in output_resource_type_list: retval[typename].append(resource) # display a count of how many resources we retrieved log.debug("got these resources: %s", dict([(k, len(v)) for k, v in retval.iteritems()])) return retval def find_related_sites(self, parent_resource_id='', exclude_site_types=None, include_parents=False, id_only=False): if not parent_resource_id: raise BadRequest("Must provide a parent parent_resource_id") exclude_site_types = exclude_site_types or [] if not isinstance(exclude_site_types, list): raise BadRequest("exclude_site_types mut be a list, is: %s" % type(exclude_site_types)) parent_resource = self.RR.read(parent_resource_id) org_id, site_id = None, None if parent_resource.type_ == RT.Org: org_id = parent_resource_id elif RT.Site in parent_resource._get_extends(): site_id = parent_resource_id else: raise BadRequest("Illegal parent_resource_id type. Expected Org/Site, given:%s" % parent_resource.type_) site_resources, site_children = self.outil.get_child_sites(site_id, org_id, exclude_types=exclude_site_types, include_parents=include_parents, id_only=id_only) return site_resources, site_children def get_sites_devices_status(self, parent_resource_id='', include_devices=False, include_status=False): if not parent_resource_id: raise BadRequest("Must provide a parent parent_resource_id") parent_resource = self.RR.read(parent_resource_id) org_id, site_id = None, None if parent_resource.type_ == RT.Org: org_id = parent_resource_id elif RT.Site in parent_resource._get_extends(): site_id = parent_resource_id result_dict = {} if include_status: status_rollups = self.outil.get_status_roll_ups(parent_resource_id, parent_resource.type_, include_structure=True) struct_dict = status_rollups.pop("_system") if "_system" in status_rollups else {} result_dict["site_resources"] = struct_dict.get("sites", {}) result_dict["site_children"] = struct_dict.get("ancestors", {}) if include_devices: site_devices = struct_dict.get("devices", {}) result_dict["site_devices"] = site_devices device_ids = [tuple_list[0][1] for tuple_list in site_devices.values() if tuple_list] device_objs = self.RR.read_mult(device_ids) result_dict["device_resources"] = dict(zip(device_ids, device_objs)) result_dict["site_status"] = status_rollups else: site_resources, site_children = self.outil.get_child_sites(site_id, org_id, include_parents=True, id_only=False) result_dict["site_resources"] = site_resources result_dict["site_children"] = site_children return result_dict def find_site_data_products(self, parent_resource_id='', include_sites=False, include_devices=False, include_data_products=False): if not parent_resource_id: raise BadRequest("Must provide a parent parent_resource_id") res_dict = self.outil.get_site_data_products(parent_resource_id, include_sites=include_sites, include_devices=include_devices, include_data_products=include_data_products) return res_dict ############################ # # EXTENDED RESOURCES # ############################ def _get_site_extension(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''): """Returns an InstrumentDeviceExtension object containing additional related information @param site_id str @param ext_associations dict @param ext_exclude list @retval observatory ObservatoryExtension @throws BadRequest A parameter is missing @throws NotFound An object with the specified observatory_id does not exist """ if not site_id: raise BadRequest("The site_id parameter is empty") extended_resource_handler = ExtendedResourceContainer(self) extended_site = extended_resource_handler.create_extended_resource_container( extended_resource_type=OT.SiteExtension, resource_id=site_id, computed_resource_type=OT.SiteComputedAttributes, ext_associations=ext_associations, ext_exclude=ext_exclude, user_id=user_id) RR2 = EnhancedResourceRegistryClient(self.RR) RR2.cache_predicate(PRED.hasModel) # Get status of Site instruments. a, b = self._get_instrument_states(extended_site.instrument_devices) extended_site.instruments_operational, extended_site.instruments_not_operational = a, b # lookup all hasModel predicates # lookup is a 2d associative array of [subject type][subject id] -> object id lookup = dict([(rt, {}) for rt in [RT.InstrumentDevice, RT.PlatformDevice]]) for a in RR2.filter_cached_associations(PRED.hasModel, lambda assn: assn.st in lookup): lookup[a.st][a.s] = a.o def retrieve_model_objs(rsrc_list, object_type): # rsrc_list is devices that need models looked up. object_type is the resource type (a device) # not all devices have models (represented as None), which kills read_mult. so, extract the models ids, # look up all the model ids, then create the proper output model_list = [lookup[object_type].get(r._id) for r in rsrc_list] model_uniq = list(set([m for m in model_list if m is not None])) model_objs = self.RR2.read_mult(model_uniq) model_dict = dict(zip(model_uniq, model_objs)) return [model_dict.get(m) for m in model_list] extended_site.instrument_models = retrieve_model_objs(extended_site.instrument_devices, RT.InstrumentDevice) extended_site.platform_models = retrieve_model_objs(extended_site.platform_devices, RT.PlatformDevice) # Status computation extended_site.computed.instrument_status = [AgentStatusBuilder.get_aggregate_status_of_device(idev._id, "aggstatus") for idev in extended_site.instrument_devices] extended_site.computed.platform_status = [AgentStatusBuilder.get_aggregate_status_of_device(pdev._id, "aggstatus") for pdev in extended_site.platform_devices] # AgentStatusBuilder.add_device_aggregate_status_to_resource_extension(device_id, # 'aggstatus', # extended_site) def status_unknown(): return ComputedIntValue(status=ComputedValueAvailability.PROVIDED, value=StatusType.STATUS_UNKNOWN) extended_site.computed.communications_status_roll_up = status_unknown() extended_site.computed.power_status_roll_up = status_unknown() extended_site.computed.data_status_roll_up = status_unknown() extended_site.computed.location_status_roll_up = status_unknown() extended_site.computed.aggregated_status = status_unknown() extended_site.computed.site_status = [StatusType.STATUS_UNKNOWN] * len(extended_site.sites) return extended_site, RR2 def _get_site_extension_plus(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''): # the "plus" means "plus all sub-site objects" extended_site, RR2 = self._get_site_extension(site_id, ext_associations, ext_exclude, user_id) # use the related resources crawler finder = RelatedResourcesCrawler() get_assns = finder.generate_related_resources_partial(RR2, [PRED.hasSite]) full_crawllist = [RT.InstrumentSite, RT.PlatformSite, RT.Subsite] search_down = get_assns({PRED.hasSite: (True, False)}, full_crawllist) # the searches return a list of association objects, so compile all the ids by extracting them subsite_ids = set([]) # we want only those IDs that are not the input resource id for a in search_down(site_id, -1): if a.o != site_id: subsite_ids.add(a.o) log.trace("converting retrieved ids to objects = %s" % subsite_ids) subsite_objs = RR2.read_mult(list(subsite_ids)) # filtered subsites def fs(resource_type, filter_fn): both = lambda s: ((resource_type == s._get_type()) and filter_fn(s)) return filter(both, subsite_objs) def pfs(filter_fn): return fs(RT.PlatformSite, filter_fn) def ifs(filter_fn): return fs(RT.InstrumentSite, filter_fn) extended_site.computed.platform_station_sites = pfs(lambda s: "StationSite" == s.alt_resource_type) extended_site.computed.platform_component_sites = pfs(lambda s: "PlatformComponentSite" == s.alt_resource_type) extended_site.computed.platform_assembly_sites = pfs(lambda s: "PlatformAssemblySite" == s.alt_resource_type) extended_site.computed.instrument_sites = ifs(lambda _: True) return extended_site, RR2, subsite_objs # TODO: will remove this one def get_site_extension(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''): extended_site, _ = self._get_site_extension(site_id, ext_associations, ext_exclude, user_id) return extended_site def get_observatory_site_extension(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''): extended_site, RR2, subsite_objs = self._get_site_extension_plus(site_id, ext_associations, ext_exclude, user_id) return extended_site def get_platform_station_site_extension(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''): extended_site, RR2, subsite_objs = self._get_site_extension_plus(site_id, ext_associations, ext_exclude, user_id) return extended_site def get_platform_assembly_site_extension(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''): extended_site, RR2, subsite_objs = self._get_site_extension_plus(site_id, ext_associations, ext_exclude, user_id) return extended_site def get_platform_component_site_extension(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''): extended_site, RR2, subsite_objs = self._get_site_extension_plus(site_id, ext_associations, ext_exclude, user_id) return extended_site def get_instrument_site_extension(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''): extended_site, _ = self._get_site_extension(site_id, ext_associations, ext_exclude, user_id) # no subsites of instrument, so shortcut extended_site.computed.platform_station_sites = [] extended_site.computed.platform_component_sites = [] extended_site.computed.platform_assembly_sites = [] extended_site.computed.instrument_sites = [] return extended_site def _get_instrument_states(self, instrument_device_obj_list=None): op = [] non_op = [] if instrument_device_obj_list is None: instrument_device_list = [] #call eventsdb to check data-related events from this device. Use UNix vs NTP tiem for now, as # resource timestaps are in Unix, data is in NTP now = str(int(time.time() * 1000)) query_interval = str(int(time.time() - (AGENT_STATUS_EVENT_DELTA_DAYS * 86400) ) *1000) for device_obj in instrument_device_obj_list: # first check the instrument lifecycle state # if not ( device_obj.lcstate in [LCS.DEPLOYED_AVAILABLE, LCS.INTEGRATED_DISCOVERABLE] ): # TODO: check that this is the intended lcs behavior and maybe check availability if not ( device_obj.lcstate in [LCS.DEPLOYED, LCS.INTEGRATED] ): non_op.append(device_obj) else: # we dont have a find_events that takes a list yet so loop thru the instruments and get # recent events for each. events = self.clients.user_notification.find_events(origin=device_obj._id, type= 'ResourceAgentStateEvent', max_datetime = now, min_datetime = query_interval, limit=1) # the most recent event is first so assume that is the current state if not events: non_op.append(device_obj) else: current_instrument_state = events[0].state if current_instrument_state in [ResourceAgentState.STREAMING, ResourceAgentState.CALIBRATE, ResourceAgentState.BUSY, ResourceAgentState.DIRECT_ACCESS]: op.append(device_obj) else: op.append(device_obj) return op, non_op def get_deployment_extension(self, deployment_id='', ext_associations=None, ext_exclude=None, user_id=''): if not deployment_id: raise BadRequest("The deployment_id parameter is empty") extended_resource_handler = ExtendedResourceContainer(self) extended_deployment = extended_resource_handler.create_extended_resource_container( extended_resource_type=OT.DeploymentExtension, resource_id=deployment_id, computed_resource_type=OT.DeploymentComputedAttributes, ext_associations=ext_associations, ext_exclude=ext_exclude, user_id=user_id) devices = set() instrument_device_ids = [] iplatform_device_ids = [] subjs, _ = self.RR.find_subjects( predicate=PRED.hasDeployment, object=deployment_id, id_only=False) for subj in subjs: log.debug('get_deployment_extension obj: %s', subj) if subj.type_ == "InstrumentDevice": extended_deployment.instrument_devices.append(subj) devices.add((subj._id, PRED.hasModel)) elif subj.type_ == "InstrumentSite": extended_deployment.instrument_sites.append(subj) elif subj.type_ == "PlatformDevice": extended_deployment.platform_devices.append(subj) devices.add((subj._id, PRED.hasModel)) elif subj.type_ == "PlatformSite": extended_deployment.platform_sites.append(subj) else: log.warning("get_deployment_extension found invalid type connected to deployment %s. Object details: %s ", deployment_id, subj) all_models = set() device_to_model_map = {} model_map = {} assocs = self.RR.find_associations(anyside=list(devices), id_only=False) for assoc in assocs: log.debug('get_deployment_extension assoc subj: %s pred: %s obj: %s', assoc.s, assoc.p, assoc.o) all_models.add(assoc.o) device_to_model_map[assoc.s] = assoc.o model_objs = self.RR.read_mult( list(all_models) ) for model_obj in model_objs: model_map[model_obj._id] = model_obj for instrument in extended_deployment.instrument_devices: model_id = device_to_model_map[instrument._id] extended_deployment.instrument_models.append( model_map[model_id] ) for platform in extended_deployment.platform_devices: model_id = device_to_model_map[platform._id] extended_deployment.platform_models.append( model_map[model_id] ) return extended_deployment
class DeploymentPlanner(object): """ A deployment activator validates that a set of devices will map to a set of sites in one unique way its primary purpose is to prepare( ) after which you'll be able to access what associations must be made (and unmade) """ def __init__(self, clients=None, enhanced_rr=None): self.clients = clients self.enhanced_rr = enhanced_rr if not enhanced_rr: self.enhanced_rr = EnhancedResourceRegistryClient(self.clients.resource_registry) self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) #self.resource_collector= DeploymentResourceCollector(self.clients, self.enhanced_rr) #self.resource_collector = resource_collector.create(self.deployment_obj) def _find_top_site_device(self, deployment_id): top_site = '' top_device = '' #retrieve the site tree information using the OUTIL functions; site info as well has site children deploy_items_objs, _ = self.clients.resource_registry.find_subjects(predicate=PRED.hasDeployment, object=deployment_id, id_only=False) log.debug("site_ids associated to this deployment: %s", deploy_items_objs) for obj in deploy_items_objs: rsrc_type = obj.type_ log.debug("resource type associated to this deployment:: %s", rsrc_type) if RT.PlatformDevice == rsrc_type or RT.InstrumentDevice == rsrc_type: top_device = obj elif RT.PlatformSite == rsrc_type or RT.InstrumentSite == rsrc_type: top_site = obj else: log.error('Deployment may only link to devices and sites. Deployment: %s', str(self.deployment_obj)) if not top_device or not top_site: log.error('Deployment must associate to both site and device. Deployment: %s', str(self.deployment_obj)) raise BadRequest('Deployment must associate to both site and device. Deployment: %s', str(self.deployment_obj)) return top_site, top_device def _find_pairs_to_remove(self): #figure out if any of the devices in the new mapping are already mapped and need to be removed pairs_to_remove = [] pairs_to_ignore = [] for (s, d) in self.match_list: rm_pair, ignore_pair = self._find_existing_relationship(s, d) if rm_pair: pairs_to_remove.append(rm_pair) if ignore_pair: pairs_to_ignore.append(ignore_pair) log.info("Pairs to ignore (will be removed from add list): %s", pairs_to_ignore) # make sure that anything being removed is not also being added self.match_list = filter(lambda x: x not in pairs_to_ignore, self.match_list) log.info("Pairs to remove: %s", pairs_to_remove) self.remove_list = pairs_to_remove def _find_existing_relationship(self, site_id, device_id, site_type=None, device_type=None): # look for an existing relationship between the site_id and another device. # if this site/device pair already exists, we leave it alone assert(type("") == type(site_id) == type(device_id)) log.debug("checking %s/%s pair for deployment", site_type, device_type) #return a pair that should be REMOVED, or None if site_type is None and site_id in self.site_resources: site_type = self.site_resources[site_id].type_ if device_type is None and device_id in self.device_resources: device_type = self.device_resources[device_id].type_ log.debug("checking existing %s hasDevice %s links", site_type, device_type) ret_remove = None ret_ignore = None try: found_device_id = self.enhanced_rr.find_object(site_id, PRED.hasDevice, device_type, True) if found_device_id == device_id: ret_ignore = (site_id, device_id) else: ret_remove = (site_id, found_device_id) log.warning("%s '%s' already hasDevice %s", site_type, site_id, device_type) except NotFound: pass return ret_remove, ret_ignore def _get_site_ref_designator_map(self): # create a map of site ids to their reference designator codes to facilitate matching site_ref_designator_map = {} for id, site_obj in self.site_resources.iteritems(): site_ref_designator_map[site_obj.reference_designator] = id log.debug("prepare_activation site_ref_designator_map: %s", site_ref_designator_map) return site_ref_designator_map def _get_device_resources(self, device_tree): # create a map of device ids to their full resource object to assit with lookup and validation device_objs = self.clients.resource_registry.read_mult(device_tree.keys()) log.debug("prepare_activation device_objectss: %s", device_objs) for device_obj in device_objs: self.device_resources[device_obj._id] = device_obj def _get_models(self): # retrieve all hasModel associations from the registry then filter models_tuples = {} assoc_list = self.outil._get_predicate_assocs(PRED.hasModel) for assoc in assoc_list: # only include these subject types in the map if assoc.st in [RT.InstrumentDevice, RT.InstrumentSite, RT.PlatformDevice, RT.PlatformSite]: if assoc.s not in models_tuples: models_tuples[assoc.s] = [] # a site may support more than one model so map to a list of models models_tuples[assoc.s].append((assoc.st, assoc.o, assoc.ot)) if assoc.s not in self.models_map: self.models_map[assoc.s] = [] self.models_map[assoc.s].append(assoc.o) log.debug("models_map: %s", self.models_map ) def _validate_models(self, site_id, device_id): # validate that the device and the site models are compatible if device_id in self.models_map: device_model_list = self.models_map[device_id] # devices should only be associated to one model if len(device_model_list) != 1: log.error("Device not associated to one distinct model. Device id: %s", device_id) elif device_model_list and device_model_list[0] not in self.models_map[site_id]: log.error("Device and Site to not share a compatible model. Device id: %s Site id: %s", site_id) else: log.error("Device not associated with a device model. Device id: %s", device_id) raise NotFound("Device not associated with a device model. Device id: %s", device_id) def _validate_port_assignments(self, device_id, platform_port): deployment_context_type = type(self.deployment_obj.context).__name__ self._validate_ooi_reference_designator(device_id, platform_port) # a one-to-one deployment of a device onto an RSN platform if OT.CabledInstrumentDeploymentContext == deployment_context_type or \ OT.CabledNodeDeploymentContext == deployment_context_type: # validate IP address for a cabled node deployment from socket import inet_aton try: inet_aton(platform_port.ip_address) except : log.error('IP address validation failed for device. Device id: %s', device_id) # validate port_type based on deployment context # a platform device deployment should have UPLINK port type if OT.RemotePlatformDeploymentContext == deployment_context_type or \ OT.CabledNodeDeploymentContext == deployment_context_type: if device_id in self.device_resources and self.device_resources[device_id].type_ is RT.PlatformDevice: if platform_port.port_type != PortTypeEnum.UPLINK: log.warning('Type of port for platform port assignment should be UPLINK. Device id: %s', device_id) #validate that parent_id is provided if not platform_port.parent_id: log.warning('Id of parent device should be provided in port assignment information. Device id: %s', device_id) def _validate_ooi_reference_designator(self, device_id, device_port): ooi_rd = OOIReferenceDesignator(device_port.reference_designator) if ooi_rd.error: log.warning("Invalid OOIReferenceDesignator ( %s ) specified for device %s", device_port.reference_designator, device_id) if not ooi_rd.port: log.warning("Invalid OOIReferenceDesignator ( %s ) specified for device %s, could not retrieve port", device_port.reference_designator, device_id) def get_deployment_sites_devices(self, deployment_obj): # retrieve all site and device ids related to this deployment site_ids = [] device_ids = [] self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) top_site, top_device = self._find_top_site_device(deployment_obj._id) site_resources, site_children = self.outil.get_child_sites( parent_site_id=top_site._id, id_only=False) site_ids = site_resources.keys() # get_site_devices returns a tuple that includes all devices linked to deployment sites site_devices = self.outil.get_site_devices(site_ids) for site, tuple_list in site_devices.iteritems(): for (site_type, device_id, device_type) in tuple_list: device_ids.append(device_id) return site_ids, device_ids def prepare_activation(self, deployment_obj): """ Prepare (validate) a deployment for activation, returning lists of what associations need to be added and which ones need to be removed. """ self.match_list = [] self.remove_list = [] self.unmatched_device_list = [] self.models_map = {} self.top_device = '' self.top_site = '' self.deployment_obj = deployment_obj self.site_resources = {} self.device_resources = {} self.outil = ObservatoryUtil(self, enhanced_rr=self.enhanced_rr) # retrieve the site tree information using the OUTIL functions; site info as well has site children self.top_site, self.top_device = self._find_top_site_device(deployment_obj._id) # must have a site and a device to continue if not self.top_site or not self.top_device: return [], [] log.debug("port_assignments: %s", self.deployment_obj.port_assignments ) # retrieve all models to use in match validation self._get_models() self.site_resources, site_children = self.outil.get_child_sites( parent_site_id=self.top_site._id, id_only=False) log.debug("site_resources: %s", self.site_resources) log.debug("site_children: %s", site_children) site_ref_designator_map = self._get_site_ref_designator_map() # retrieve the device tree from outil then cache the device resources device_tree = self.outil.get_child_devices(device_id=self.top_device._id) self._get_device_resources(device_tree) self._match_devices(self.top_device._id, device_tree, site_ref_designator_map) # check for hasDevice relations to remove and existing hasDevice relations self. _find_pairs_to_remove() if self.unmatched_device_list: log.warning("Devices not matched to sites: %s ", self.unmatched_device_list) return self.remove_list, self.match_list def _match_devices(self, device_id, device_tree, site_ref_designator_map): # there will not be a port assignment for the top device if device_id == self.top_device._id: self._validate_models(self.top_site._id, self.top_device._id) self.match_list.append((self.top_site._id, self.top_device._id)) tuple_list = device_tree[device_id] for (pt, child_id, ct) in tuple_list: log.debug(" tuple - pt: %s child_id: %s ct: %s", pt, child_id, ct) # match this child device then if it has children, call _match_devices with this id # check that this device is represented in device tree and in port assignments if child_id in self.device_resources and child_id in self.deployment_obj.port_assignments: platform_port = self.deployment_obj.port_assignments[child_id] log.debug("device platform_port: %s", platform_port) # validate PlatformPort info for this device self._validate_port_assignments(child_id, platform_port) if platform_port.reference_designator in site_ref_designator_map: matched_site = site_ref_designator_map[platform_port.reference_designator] self._validate_models(matched_site, child_id) log.info("match_list append site: %s device: %s", matched_site, child_id) self.match_list.append((matched_site, child_id)) #recurse on the children of this device self._match_devices(child_id, device_tree, site_ref_designator_map) # otherwise cant be matched to a site else: self.unmatched_device_list.append(child_id)
def test_get_status_roll_ups(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_events(self.event_list1) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) # No problems status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) self.assertIn('PD_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1') self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') # ID_1 power warning self.mu.load_mock_events(self.event_list2) status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertIn('PS_1', status_rollups) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) # ID_1 power+comms warning self.mu.load_mock_events(self.event_list3) status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING)
class TestObservatoryUtil(IonUnitTestCase): def setUp(self): self.mu = MockUtil() self.process_mock = self.mu.create_process_mock() self.container_mock = self.mu.create_container_mock() res_list = [ dict(rt='Org', _id='Org_1', attr={}), dict(rt='Observatory', _id='Obs_1', attr={}), dict(rt='Observatory', _id='Obs_2', attr={}), dict(rt='Subsite', _id='Sub_1', attr={}), dict(rt='Subsite', _id='Sub_2', attr={}), dict(rt='PlatformSite', _id='PS_1', attr={}), dict(rt='InstrumentSite', _id='IS_1', attr={}), dict(rt='PlatformDevice', _id='PD_1', attr={}), dict(rt='InstrumentDevice', _id='ID_1', attr={}), ] assoc_list = [ ['Obs_1', 'hasSite', 'Sub_1'], ['Sub_1', 'hasSite', 'PS_1'], ['PS_1', 'hasSite', 'IS_1'], ] assoc_list1 = [ ['Org_1', 'hasResource', 'Obs_1'], ['Org_1', 'hasResource', 'Obs_2'], ['Obs_2', 'hasSite', 'Sub_2'], ] assoc_list2 = [ ['PS_1', 'hasDevice', 'PD_1'], ['IS_1', 'hasDevice', 'ID_1'], ['PD_1', 'hasDevice', 'ID_1'], ] def spy_get_child_sites(self, parent_site_id=None, org_id=None, exclude_types=None, include_parents=True, id_only=True): child_sites, site_ancestors = self.obs_util.get_child_sites(parent_site_id=parent_site_id, org_id=org_id, exclude_types=exclude_types, include_parents=include_parents, id_only=id_only) print "child_sites of", parent_site_id, "are", child_sites print "site_ancestors of", parent_site_id, "are", site_ancestors return child_sites, site_ancestors def test_get_child_sites(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_sites, site_ancestors = self.spy_get_child_sites(parent_site_id='Obs_1', include_parents=False, id_only=True) self.assertEquals(len(child_sites), 3) self.assertEquals(len(site_ancestors), 3) self.assertIn('Sub_1', child_sites) self.assertIn('PS_1', child_sites) self.assertIn('IS_1', child_sites) self.assertNotIn('Obs_1', child_sites) self.assertEquals(len([v for v in child_sites.values() if v is None]), 3) child_sites, site_ancestors = self.spy_get_child_sites(parent_site_id='Obs_1', include_parents=False, id_only=False) self.assertEquals(len(child_sites), 3) self.assertEquals(len(site_ancestors), 3) self.assertEquals(len([v for v in child_sites.values() if v is None]), 0) self.assertEquals(child_sites['Sub_1']._get_type(), RT.Subsite) child_sites, site_ancestors = self.spy_get_child_sites(parent_site_id='Obs_1', include_parents=True) self.assertEquals(len(child_sites), 4) self.assertEquals(len(site_ancestors), 3) self.assertIn('Obs_1', child_sites) child_sites, site_ancestors = self.spy_get_child_sites(parent_site_id='Sub_1', include_parents=False) self.assertEquals(len(child_sites), 2) self.assertEquals(len(site_ancestors), 2) self.assertNotIn('Sub_1', child_sites) child_sites, site_ancestors = self.spy_get_child_sites(parent_site_id='Sub_1', include_parents=True) self.assertEquals(len(child_sites), 4) self.assertEquals(len(site_ancestors), 3) self.assertIn('Sub_1', child_sites) self.assertIn('Obs_1', child_sites) child_sites, site_ancestors = self.spy_get_child_sites(parent_site_id='PS_1', include_parents=False) self.assertEquals(len(child_sites), 1) self.assertEquals(len(site_ancestors), 1) child_sites, site_ancestors = self.spy_get_child_sites(parent_site_id='PS_1', include_parents=True) self.assertEquals(len(child_sites), 4) self.assertEquals(len(site_ancestors), 3) child_sites, site_ancestors = self.spy_get_child_sites(parent_site_id='IS_1', include_parents=False) self.assertEquals(len(child_sites), 0) self.assertEquals(len(site_ancestors), 0) child_sites, site_ancestors = self.spy_get_child_sites(parent_site_id='IS_1', include_parents=True) self.assertEquals(len(child_sites), 4) self.assertEquals(len(site_ancestors), 3) child_sites, site_ancestors = self.spy_get_child_sites(parent_site_id='XXX', include_parents=True) self.assertEquals(len(child_sites), 1) self.assertEquals(len(site_ancestors), 0) def test_get_child_sites_org(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_sites, site_ancestors = self.obs_util.get_child_sites(org_id='Org_1', include_parents=False, id_only=True) self.assertEquals(len(child_sites), 6) self.assertEquals(len(site_ancestors), 5) self.assertIn('Sub_1', child_sites) self.assertIn('PS_1', child_sites) self.assertIn('IS_1', child_sites) self.assertIn('Obs_1', child_sites) self.assertIn('Obs_2', child_sites) child_sites, site_ancestors = self.obs_util.get_child_sites(org_id='Org_1', include_parents=True, id_only=True) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) child_sites, site_ancestors = self.obs_util.get_child_sites(org_id='Org_1', include_parents=True, id_only=False) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) self.assertEquals(len([v for v in child_sites.values() if v is None]), 0) self.assertEquals(child_sites['Org_1']._get_type(), RT.Org) def test_get_site_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_devices = self.obs_util.get_site_devices(['Sub_1', 'PS_1', 'IS_1']) self.assertEquals(len(site_devices), 3) self.assertEquals(site_devices['Sub_1'], []) self.assertEquals(site_devices['IS_1'], [('InstrumentSite', 'ID_1', 'InstrumentDevice')]) def test_get_child_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_devices = self.obs_util.get_child_devices('PD_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['PD_1'][0][1], 'ID_1') child_devices = self.obs_util.get_child_devices('ID_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['ID_1'], []) child_devices = self.obs_util.get_child_devices('Sub_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['Sub_1'], []) child_devices = self.obs_util.get_child_devices('XXX') self.assertEquals(len(child_devices), 1) event_list1 = [ dict(et='DeviceStatusEvent', o='ID_1', attr=dict(status=DeviceStatusType.STATUS_WARNING) ) ] event_list2 = [ dict(et='DeviceStatusEvent', o='ID_1', attr=dict(status=DeviceStatusType.STATUS_WARNING)) ] event_list3 = [ dict(et='DeviceCommsEvent', o='ID_1', attr=dict(state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION)) ] event_list4 = [ dict(et='DeviceStatusEvent', o='PD_1', attr=dict(status=DeviceStatusType.STATUS_WARNING)), dict(et='DeviceCommsEvent', o='PD_1', attr=dict(state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION)) ] def test_get_status_roll_ups(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_events(self.event_list1) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) # No problems status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) self.assertIn('PD_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1') self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') # ID_1 power warning self.mu.load_mock_events(self.event_list2) status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertIn('PS_1', status_rollups) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING) # ID_1 power+comms warning self.mu.load_mock_events(self.event_list3) status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) def test_get_status_roll_ups_platform_warn(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_events(self.event_list4) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) # PD_1 power+comms warning status_rollups = self.obs_util.get_status_roll_ups('ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PD_1', RT.PlatformDevice) #log.warn("status %s" % status_rollups) self._assert_status(status_rollups, 'PD_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DeviceStatusType.STATUS_WARNING, power=DeviceStatusType.STATUS_WARNING, comms=DeviceStatusType.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') def _assert_status(self, status_rollups, res_id=None, agg=DeviceStatusType.STATUS_OK, loc=DeviceStatusType.STATUS_OK, data=DeviceStatusType.STATUS_OK, comms=DeviceStatusType.STATUS_OK, power=DeviceStatusType.STATUS_OK): res_status = status_rollups[res_id] if res_id else status_rollups self.assertEquals(len(res_status), 5) # #self.assertEquals(res_status['agg'], agg) # self.assertEquals(res_status['loc'], loc) # self.assertEquals(res_status['data'], data) # self.assertEquals(res_status['comms'], comms) # self.assertEquals(res_status['power'], power) res_list1 = [ dict(rt='DataProduct', _id='DP_1', attr={}), dict(rt='DataProduct', _id='DP_2', attr={}), dict(rt='DataProduct', _id='DP_3', attr={}), dict(rt='DataProduct', _id='DP_4', attr={}), dict(rt='DataProduct', _id='DP_5', attr={}), ] assoc_list3 = [ ['DP_1', 'hasSource', 'ID_1'], ['DP_2', 'hasSource', 'ID_1'], ['DP_3', 'hasSource', 'ID_1'], ['DP_3', 'hasSource', 'PD_1'], ['DP_4', 'hasSource', 'PD_1'], ['DP_5', 'hasSource', 'PD_1'], ] def test_get_device_data_products(self): self.mu.load_mock_resources(self.res_list + self.res_list1) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2 + self.assoc_list3) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) res_dict = self.obs_util.get_site_data_products('Obs_1', RT.Observatory) self.assertGreaterEqual(len(res_dict), 6) self.assertIsNone(res_dict['data_product_resources']) self.assertIn('ID_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products('PS_1', RT.PlatformSite) self.assertEquals(len(res_dict['device_data_products']['ID_1']), 3) self.assertIn('ID_1', res_dict['device_data_products']) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) self.assertIn('PD_1', res_dict['device_data_products']) self.assertEquals(len(res_dict['device_data_products']['PD_1']), 3) res_dict = self.obs_util.get_site_data_products('Org_1', RT.Org) self.assertIn('DP_1', res_dict['device_data_products']['ID_1']) res_dict = self.obs_util.get_site_data_products('PS_1', RT.PlatformSite, include_data_products=True) self.assertIsNotNone(res_dict['data_product_resources']) self.assertIn('DP_1', res_dict['data_product_resources'])
def test_get_status_roll_ups_platform_warn(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_device_statuses(self.status_by_device_4) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock, device_status_mgr=self.dsm_mock) # PD_1 power+comms warning status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) #log.warn("status %s" % status_rollups) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=DST.STATUS_WARNING, power=DST.STATUS_WARNING, comms=DST.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1')
class TestObservatoryUtil(unittest.TestCase): def setUp(self): self.mu = MockUtil() self.process_mock = self.mu.create_process_mock() self.container_mock = self.mu.create_container_mock() res_list = [ dict(rt='Org', _id='Org_1', attr={}), dict(rt='Observatory', _id='Obs_1', attr={}), dict(rt='Observatory', _id='Obs_2', attr={}), dict(rt='Subsite', _id='Sub_1', attr={}), dict(rt='Subsite', _id='Sub_2', attr={}), dict(rt='PlatformSite', _id='PS_1', attr={}), dict(rt='InstrumentSite', _id='IS_1', attr={}), dict(rt='PlatformDevice', _id='PD_1', attr={}), dict(rt='InstrumentDevice', _id='ID_1', attr={}), ] assoc_list = [ ['Obs_1', 'hasSite', 'Sub_1'], ['Sub_1', 'hasSite', 'PS_1'], ['PS_1', 'hasSite', 'IS_1'], ] assoc_list1 = [ ['Org_1', 'hasResource', 'Obs_1'], ['Org_1', 'hasResource', 'Obs_2'], ['Obs_2', 'hasSite', 'Sub_2'], ] assoc_list2 = [ ['PS_1', 'hasDevice', 'PD_1'], ['IS_1', 'hasDevice', 'ID_1'], ['PD_1', 'hasDevice', 'ID_1'], ] def test_get_child_sites(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id='Obs_1', include_parents=False, id_only=True) self.assertEquals(len(child_sites), 3) self.assertEquals(len(site_ancestors), 3) self.assertIn('Sub_1', child_sites) self.assertIn('PS_1', child_sites) self.assertIn('IS_1', child_sites) self.assertNotIn('Obs_1', child_sites) self.assertEquals(len([v for v in child_sites.values() if v is None]), 3) child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id='Obs_1', include_parents=False, id_only=False) self.assertEquals(len(child_sites), 3) self.assertEquals(len(site_ancestors), 3) self.assertEquals(len([v for v in child_sites.values() if v is None]), 0) self.assertEquals(child_sites['Sub_1']._get_type(), RT.Subsite) child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id='Obs_1', include_parents=True) self.assertEquals(len(child_sites), 4) self.assertEquals(len(site_ancestors), 3) self.assertIn('Obs_1', child_sites) child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id='Sub_1', include_parents=False) self.assertEquals(len(child_sites), 2) self.assertEquals(len(site_ancestors), 2) self.assertNotIn('Sub_1', child_sites) child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id='Sub_1', include_parents=True) self.assertEquals(len(child_sites), 4) self.assertEquals(len(site_ancestors), 3) self.assertIn('Sub_1', child_sites) self.assertIn('Obs_1', child_sites) child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id='PS_1', include_parents=False) self.assertEquals(len(child_sites), 1) self.assertEquals(len(site_ancestors), 1) child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id='PS_1', include_parents=True) self.assertEquals(len(child_sites), 4) self.assertEquals(len(site_ancestors), 3) child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id='IS_1', include_parents=False) self.assertEquals(len(child_sites), 0) self.assertEquals(len(site_ancestors), 0) child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id='IS_1', include_parents=True) self.assertEquals(len(child_sites), 4) self.assertEquals(len(site_ancestors), 3) child_sites, site_ancestors = self.obs_util.get_child_sites( parent_site_id='XXX', include_parents=True) self.assertEquals(len(child_sites), 1) self.assertEquals(len(site_ancestors), 0) def test_get_child_sites_org(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1) self.mu.assign_mockres_find_objects(filter_predicate="hasResource") self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=False, id_only=True) self.assertEquals(len(child_sites), 6) self.assertEquals(len(site_ancestors), 5) self.assertIn('Sub_1', child_sites) self.assertIn('PS_1', child_sites) self.assertIn('IS_1', child_sites) self.assertIn('Obs_1', child_sites) self.assertIn('Obs_2', child_sites) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=True, id_only=True) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) child_sites, site_ancestors = self.obs_util.get_child_sites( org_id='Org_1', include_parents=True, id_only=False) self.assertEquals(len(child_sites), 7) self.assertEquals(len(site_ancestors), 5) self.assertEquals(len([v for v in child_sites.values() if v is None]), 0) self.assertEquals(child_sites['Org_1']._get_type(), RT.Org) def test_get_site_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_devices = self.obs_util.get_site_devices( ['Sub_1', 'PS_1', 'IS_1']) self.assertEquals(len(site_devices), 3) self.assertEquals(site_devices['Sub_1'], None) self.assertEquals(site_devices['IS_1'], ('InstrumentSite', 'ID_1', 'InstrumentDevice')) def test_get_child_devices(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list2) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) child_devices = self.obs_util.get_child_devices('PD_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['PD_1'][0][1], 'ID_1') child_devices = self.obs_util.get_child_devices('ID_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['ID_1'], []) child_devices = self.obs_util.get_child_devices('Sub_1') self.assertEquals(len(child_devices), 1) self.assertEquals(child_devices['Sub_1'], []) child_devices = self.obs_util.get_child_devices('XXX') self.assertEquals(len(child_devices), 1) event_list1 = [ dict(et='DeviceStatusEvent', o='ID_1', attr=dict(state=DeviceStatusType.OK)) ] event_list2 = [ dict(et='DeviceStatusEvent', o='ID_1', attr=dict(state=DeviceStatusType.OUT_OF_RANGE)) ] event_list3 = [ dict(et='DeviceCommsEvent', o='ID_1', attr=dict(state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION)) ] event_list4 = [ dict(et='DeviceStatusEvent', o='PD_1', attr=dict(state=DeviceStatusType.OUT_OF_RANGE)), dict(et='DeviceCommsEvent', o='PD_1', attr=dict(state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION)) ] def test_get_status_roll_ups(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_events(self.event_list1) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) # No problems status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) self.assertIn('PD_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1') self._assert_status(status_rollups, 'Sub_1') self._assert_status(status_rollups, 'PS_1') self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') # ID_1 power warning self.mu.load_mock_events(self.event_list2) status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertIn('PS_1', status_rollups) self.assertIn('IS_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING) # ID_1 power+comms warning self.mu.load_mock_events(self.event_list3) status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) self._assert_status(status_rollups, 'PD_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) status_rollups = self.obs_util.get_status_roll_ups( 'Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1') self._assert_status(status_rollups, 'IS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) def test_get_status_roll_ups_platform_warn(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list + self.assoc_list1 + self.assoc_list2) self.mu.load_mock_events(self.event_list4) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) # PD_1 power+comms warning status_rollups = self.obs_util.get_status_roll_ups( 'ID_1', RT.InstrumentDevice) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PD_1', RT.PlatformDevice) #log.warn("status %s" % status_rollups) self._assert_status(status_rollups, 'PD_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'IS_1', RT.InstrumentSite) self.assertEquals(len(status_rollups), 6) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'PS_1', RT.PlatformSite) self.assertEquals(len(status_rollups), 6) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups('Sub_1', RT.Subsite) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Sub_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') status_rollups = self.obs_util.get_status_roll_ups( 'Obs_1', RT.Observatory) self.assertIn('Obs_1', status_rollups) self.assertIn('Sub_1', status_rollups) self.assertIn('PS_1', status_rollups) self.assertIn('PD_1', status_rollups) self.assertIn('IS_1', status_rollups) self.assertIn('ID_1', status_rollups) self._assert_status(status_rollups, 'Obs_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'Sub_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PS_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'PD_1', agg=StatusType.STATUS_WARNING, power=StatusType.STATUS_WARNING, comms=StatusType.STATUS_WARNING) self._assert_status(status_rollups, 'IS_1') self._assert_status(status_rollups, 'ID_1') def _assert_status(self, status_rollups, res_id=None, agg=StatusType.STATUS_OK, loc=StatusType.STATUS_OK, data=StatusType.STATUS_OK, comms=StatusType.STATUS_OK, power=StatusType.STATUS_OK): res_status = status_rollups[res_id] if res_id else status_rollups self.assertEquals(len(res_status), 5) self.assertEquals(res_status['agg'], agg) self.assertEquals(res_status['loc'], loc) self.assertEquals(res_status['data'], data) self.assertEquals(res_status['comms'], comms) self.assertEquals(res_status['power'], power)
def test_get_child_sites(self): self.mu.load_mock_resources(self.res_list) self.mu.load_mock_associations(self.assoc_list) self.obs_util = ObservatoryUtil(self.process_mock, self.container_mock) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Obs_1', include_parents=False, id_only=True) self.assertEquals(len(site_resources), 3) self.assertEquals(len(site_children), 3) self.assertIn('Sub_1', site_resources) self.assertIn('PS_1', site_resources) self.assertIn('IS_1', site_resources) self.assertNotIn('Obs_1', site_resources) self.assertEquals( len([v for v in site_resources.values() if v is None]), 3) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Obs_1', include_parents=False, id_only=False) self.assertEquals(len(site_resources), 3) self.assertEquals(len(site_children), 3) self.assertEquals( len([v for v in site_resources.values() if v is None]), 0) self.assertEquals(site_resources['Sub_1']._get_type(), RT.Subsite) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Obs_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) self.assertIn('Obs_1', site_resources) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Sub_1', include_parents=False) self.assertEquals(len(site_resources), 2) self.assertEquals(len(site_children), 2) self.assertNotIn('Sub_1', site_resources) site_resources, site_children = self.spy_get_child_sites( parent_site_id='Sub_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) self.assertIn('Sub_1', site_resources) self.assertIn('Obs_1', site_resources) site_resources, site_children = self.spy_get_child_sites( parent_site_id='PS_1', include_parents=False) self.assertEquals(len(site_resources), 1) self.assertEquals(len(site_children), 1) site_resources, site_children = self.spy_get_child_sites( parent_site_id='PS_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) site_resources, site_children = self.spy_get_child_sites( parent_site_id='IS_1', include_parents=False) self.assertEquals(len(site_resources), 0) self.assertEquals(len(site_children), 0) site_resources, site_children = self.spy_get_child_sites( parent_site_id='IS_1', include_parents=True) self.assertEquals(len(site_resources), 4) self.assertEquals(len(site_children), 3) site_resources, site_children = self.spy_get_child_sites( parent_site_id='XXX', include_parents=True) self.assertEquals(len(site_resources), 1) self.assertEquals(len(site_children), 0)