def test_network_view_management(self): # prepare grid self._create_default_grid() # prepare members self._create_simple_members() db_members = infoblox_db.get_members(self.ctx.session) gm_member = utils.find_one_in_list('member_type', 'GM', db_members) # should be no network views db_network_views = infoblox_db.get_network_views(self.ctx.session) self.assertEqual(0, len(db_network_views)) # test network view additions netview_dict = { 'default': gm_member.member_id, 'hs-view-1': gm_member.member_id, 'hs-view-2': gm_member.member_id, 'hs-view-3': gm_member.member_id } self._create_network_views(netview_dict) db_network_views = infoblox_db.get_network_views(self.ctx.session) actual_rows = utils.get_values_from_records('network_view', db_network_views) self.assertEqual(netview_dict.keys(), actual_rows) # test network view removals # - remove 'hs-view-1', 'hs-view-2' removing_list = [netview_dict['hs-view-1'], netview_dict['hs-view-2']] infoblox_db.remove_network_views_by_names(self.ctx.session, removing_list, self.grid_id) db_network_views = infoblox_db.get_network_views(self.ctx.session) actual_rows = utils.get_values_from_records('network_view', db_network_views) actual_set = set(actual_rows) expected_set = set(netview_dict.keys()).difference(removing_list) self.assertEqual(expected_set, actual_set) # - remove 'hs-view-3' removing_netview_name = 'hs-view-3' removing_netview = utils.find_one_in_list('network_view', removing_netview_name, db_network_views) removing_netview_id = removing_netview.id infoblox_db.remove_network_views(self.ctx.session, [removing_netview_id]) actual_network_views = infoblox_db.get_network_views( self.ctx.session, network_view=removing_netview_name) self.assertEqual([], actual_network_views)
def test_network_view_management(self): # prepare grid self._create_default_grid() # prepare members self._create_simple_members() db_members = infoblox_db.get_members(self.ctx.session) gm_member = utils.find_one_in_list('member_type', 'GM', db_members) # should be no network views db_network_views = infoblox_db.get_network_views(self.ctx.session) self.assertEqual(0, len(db_network_views)) # test network view additions netview_dict = {'default': gm_member.member_id, 'hs-view-1': gm_member.member_id, 'hs-view-2': gm_member.member_id, 'hs-view-3': gm_member.member_id} self._create_network_views(netview_dict) db_network_views = infoblox_db.get_network_views(self.ctx.session) actual_rows = utils.get_values_from_records('network_view', db_network_views) self.assertEqual(netview_dict.keys(), actual_rows) # test network view removals # - remove 'hs-view-1', 'hs-view-2' removing_list = [netview_dict['hs-view-1'], netview_dict['hs-view-2']] infoblox_db.remove_network_views_by_names(self.ctx.session, removing_list, self.grid_id) db_network_views = infoblox_db.get_network_views(self.ctx.session) actual_rows = utils.get_values_from_records('network_view', db_network_views) actual_set = set(actual_rows) expected_set = set(netview_dict.keys()).difference(removing_list) self.assertEqual(expected_set, actual_set) # - remove 'hs-view-3' removing_netview_name = 'hs-view-3' removing_netview = utils.find_one_in_list('network_view', removing_netview_name, db_network_views) removing_netview_id = removing_netview.id infoblox_db.remove_network_views(self.ctx.session, [removing_netview_id]) actual_network_views = infoblox_db.get_network_views( self.ctx.session, network_view=removing_netview_name) self.assertEqual([], actual_network_views)
def _validate_network_views(self, network_view_json): db_network_views = dbi.get_network_views(self.ctx.session) self.assertEqual(len(network_view_json), len(db_network_views)) expected = [nv['name'] for nv in network_view_json] actual = utils.get_values_from_records('network_view', db_network_views) self.assertEqual(expected, actual)
def sync_grid(self): """Synchronize an active grid. Only one active grid should be kept where grid_status is set to 'ON'. """ session = self._context.session grid_connection = self._grid_config.get_grid_connection() grid_connection_json = jsonutils.dumps(grid_connection) db_grids = dbi.get_grids(session) db_grid_ids = utils.get_values_from_records('grid_id', db_grids) # update the existing grid or add new grid if self._grid_config.grid_id in db_grid_ids: dbi.update_grid(session, self._grid_config.grid_id, self._grid_config.grid_name, grid_connection_json, const.GRID_STATUS_ON) else: dbi.add_grid(session, self._grid_config.grid_id, self._grid_config.grid_name, grid_connection_json, const.GRID_STATUS_ON, utils.get_hash()) # deleting grids are delicate operation so we won't allow it # but we will set grid status to OFF to unused grids. persisted_set = set(db_grid_ids) disable_set = persisted_set.difference([self._grid_config.grid_id]) disabling_grid_ids = list(disable_set) for grid_id in disabling_grid_ids: dbi.update_grid(session, grid_id, grid_status=const.GRID_STATUS_OFF) session.flush()
def _validate_network_views(self, network_view_json): db_network_views = dbi.get_network_views(self.ctx.session) self.assertEqual(len(network_view_json), len(db_network_views)) expected = [nv['name'] for nv in network_view_json] actual = utils.get_values_from_records('network_view', db_network_views) self.assertEqual(expected, actual)
def test_get_values_from_records(self): grid_1_id = 100 grid_2_id = 200 dbi.add_grid(self.ctx.session, grid_1_id, 'test grid 1', '{}', 'ON', 'gm-id-1') dbi.add_grid(self.ctx.session, grid_2_id, 'test grid 2', '{}', 'OFF', 'gm-id-2') grids = dbi.get_grids(self.ctx.session) grid_ids = utils.get_values_from_records('grid_id', grids) self.assertEqual(grid_1_id, grid_ids[0]) self.assertEqual(grid_2_id, grid_ids[1]) grid_names = utils.get_values_from_records('grid_name', grids) self.assertEqual('test grid 1', grid_names[0]) self.assertEqual('test grid 2', grid_names[1])
def test_sync_member_with_cloud_support_with_member_licenses(self): member_mgr = member.GridMemberManager(self.test_grid_config) member_mgr.sync_grid() member_json = self.connector_fixture.get_object( base.FixtureResourceMap.FAKE_MEMBERS_WITH_CLOUD) license_json = self.connector_fixture.get_object( base.FixtureResourceMap.FAKE_MEMBER_LICENSES) self._mock_member_mgr(member_mgr, discover_members=member_json, discover_member_licenses=license_json) member_mgr.sync_members() # member_type is computed base on member licenses. To prove correctness # of member types, we need to first figure out which member has # 'CLOUD_API' license. A member with 'CLOUD_API' is CPM; otherwise # 'REGULAR'. Grid master member is 'GM' cloud_member_list = [] cloud_lecensed_members = utils.find_in_list('type', ['CLOUD_API'], license_json) member_hwids = utils.get_values_from_records('hwid', cloud_lecensed_members) for m in member_json: member_id_arg = str(self.test_grid_config.grid_id) + m['host_name'] member_id = utils.get_hash(member_id_arg) member_hwid = m['node_info'][0].get('hwid') if member_hwid in member_hwids: cloud_member_list.append(member_id) # now we know member licenses so get members in db members = dbi.get_members(self.ctx.session) self.assertEqual(len(member_json), len(members)) # verify member types for m in members: if self.test_grid_config.grid_master_host == m['member_ip']: self.assertEqual('GM', m['member_type']) else: if m['member_id'] in cloud_member_list: self.assertEqual('CPM', m['member_type']) else: self.assertEqual('REGULAR', m['member_type'])
def test_sync_member_with_cloud_support_with_member_licenses(self): member_mgr = member.GridMemberManager(self.test_grid_config) member_mgr.sync_grid() member_json = self.connector_fixture.get_object( base.FixtureResourceMap.FAKE_MEMBERS_WITH_CLOUD) license_json = self.connector_fixture.get_object( base.FixtureResourceMap.FAKE_MEMBER_LICENSES) self._mock_member_mgr(member_mgr, discover_members=member_json, discover_member_licenses=license_json) member_mgr.sync_members() # member_type is computed base on member licenses. To prove correctness # of member types, we need to first figure out which member has # 'CLOUD_API' license. A member with 'CLOUD_API' is CPM; otherwise # 'REGULAR'. Grid master member is 'GM' cloud_member_list = [] cloud_lecensed_members = utils.find_in_list('type', ['CLOUD_API'], license_json) member_hwids = utils.get_values_from_records('hwid', cloud_lecensed_members) for m in member_json: member_id_arg = str(self.test_grid_config.grid_id) + m['host_name'] member_id = utils.get_hash(member_id_arg) member_hwid = m['node_info'][0].get('hwid') if member_hwid in member_hwids: cloud_member_list.append(member_id) # now we know member licenses so get members in db members = dbi.get_members(self.ctx.session) self.assertEqual(len(member_json), len(members)) # verify member types for m in members: if self.test_grid_config.grid_master_host == m['member_ip']: self.assertEqual('GM', m['member_type']) else: if m['member_id'] in cloud_member_list: self.assertEqual('CPM', m['member_type']) else: self.assertEqual('REGULAR', m['member_type'])
def sync_grid(self): """Synchronize an active grid. Only one active grid should be kept where grid_status is set to 'ON'. """ session = self._context.session grid_connection = self._grid_config.get_grid_connection() grid_connection_json = jsonutils.dumps(grid_connection) db_grids = dbi.get_grids(session) db_grid_ids = utils.get_values_from_records('grid_id', db_grids) # update the existing grid or add new grid if self._grid_config.grid_id in db_grid_ids: dbi.update_grid(session, self._grid_config.grid_id, self._grid_config.grid_name, grid_connection_json, const.GRID_STATUS_ON) else: dbi.add_grid(session, self._grid_config.grid_id, self._grid_config.grid_name, grid_connection_json, const.GRID_STATUS_ON, utils.get_hash()) # deleting grids are delicate operation so we won't allow it # but we will set grid status to OFF to unused grids. persisted_set = set(db_grid_ids) disable_set = persisted_set.difference([self._grid_config.grid_id]) disabling_grid_ids = list(disable_set) for grid_id in disabling_grid_ids: dbi.update_grid(session, grid_id, grid_status=const.GRID_STATUS_OFF) session.flush()
def _sync_network_views(self, discovered_netviews, dns_views): """Discover network views and sync with db. The discovered network view json contains the following data: - network view - cloud_info for delegated member if cloud platform is supported - mapping conditional EAs So discovered information will be updated in tables such as infoblox_network_views and infoblox_mapping_conditions. :param discovered_netviews: discovered network view json :return: None """ session = self._context.session self._load_persisted_mappings() discovered_delegations = dict() persisted_netview_ids = utils.get_values_from_records( 'id', self.db_network_views) discovered_netview_ids = [] for netview in discovered_netviews: netview_name = netview['name'] is_default = netview[const.IS_DEFAULT] netview_id = utils.get_network_view_id(self._grid_id, netview['_ref']) cloud_adapter_id_vals = utils.get_ea_value( const.EA_CLOUD_ADAPTER_ID, netview, True) if cloud_adapter_id_vals is None: participated = False else: cloud_adapter_ids = [ gid for gid in cloud_adapter_id_vals if int(gid) == self._grid_id ] participated = True if cloud_adapter_ids else False if not participated: continue shared_val = utils.get_ea_value(const.EA_IS_SHARED, netview) is_shared = types.Boolean()(shared_val) if shared_val else False # authority member is default to GM gm_row = utils.find_one_in_list('member_type', const.MEMBER_TYPE_GRID_MASTER, self.db_members) authority_member_id = gm_row.member_id # get delegation member if cloud platform is supported delegated_member = self._get_delegated_member(netview) if delegated_member: authority_member_id = delegated_member.member_id discovered_delegations[netview_name] = ( delegated_member.member_id) dns_view = (dns_views[netview_name] if dns_views.get(netview_name) else None) # see if the network view already exists in db netview_row = utils.find_one_in_list('id', netview_id, self.db_network_views) if netview_row: dbi.update_network_view(session, netview_id, netview_name, authority_member_id, is_shared, dns_view, participated, is_default) else: internal_netview = (const.DEFAULT_NETWORK_VIEW if is_default else netview_name) internal_dnsview = (const.DEFAULT_DNS_VIEW if is_default else dns_view) dbi.add_network_view(session, netview_id, netview_name, self._grid_id, authority_member_id, is_shared, dns_view, internal_netview, internal_dnsview, participated, is_default) discovered_netview_ids.append(netview_id) # update mapping conditions for the current network view self._update_mapping_conditions(netview, netview_id, participated) # we have added new network views. now let's remove persisted # network views not found from discovery persisted_set = set(persisted_netview_ids) removable_set = persisted_set.difference(discovered_netview_ids) removable_netviews = list(removable_set) if removable_netviews: dbi.remove_network_views(session, removable_netviews) session.flush() return discovered_delegations
def sync_members(self): """Synchronizes grid members. Members in the active grid are discovered from NIOS backend and grid members are in sync in neutron db. The members who are no longer in used are set to 'OFF' status. """ session = self._context.session grid_id = self._grid_config.grid_id db_grids = dbi.get_grids(session) db_grid = utils.find_one_in_list('grid_id', grid_id, db_grids) gm_member_id = db_grid.gm_id db_members = dbi.get_members(session, grid_id=grid_id) gm_member = utils.find_one_in_list('member_id', gm_member_id, db_members) discovered_members = self._discover_members() if not discovered_members: return dns_member_settings = self._discover_dns_settings() dhcp_member_settings = self._discover_dhcp_settings() discovered_licenses = self._discover_member_licenses() discovered_member_ids = [] for member in discovered_members: member_name = member['host_name'] member_ip, member_ipv6 = self._get_lan1_ips(member) member_wapi = member_ip if member_ip else member_ipv6 member_hwid = member['node_info'][0].get('hwid') member_status = self._get_member_status( member['node_info'][0]['service_status']) member_type = self._get_member_type(discovered_licenses, member_name, member_hwid) require_db_update = False if member_type == const.MEMBER_TYPE_GRID_MASTER: if gm_member: require_db_update = True member_id = gm_member_id member_wapi = self._grid_config.grid_master_host else: # no need to process 'Is Cloud Member' flag for non GM members ea_is_cloud_member = utils.get_ea_value( const.EA_IS_CLOUD_MEMBER, member) is_cloud_member = (types.Boolean()(ea_is_cloud_member) if ea_is_cloud_member else False) if not is_cloud_member: continue db_member = utils.find_one_in_list('member_name', member_name, db_members) if db_member: require_db_update = True member_id = db_member.member_id else: member_id = utils.get_hash(str(grid_id) + member_name) member_dhcp_ip, member_dhcp_ipv6 = self._get_dhcp_ips( member, dhcp_member_settings) member_dns_ip, member_dns_ipv6 = self._get_dns_ips( member, dns_member_settings) if require_db_update: dbi.update_member(session, member_id, grid_id, member_name, member_ip, member_ipv6, member_type, member_status, member_dhcp_ip, member_dhcp_ipv6, member_dns_ip, member_dns_ipv6, member_wapi) else: dbi.add_member(session, member_id, grid_id, member_name, member_ip, member_ipv6, member_type, member_status, member_dhcp_ip, member_dhcp_ipv6, member_dns_ip, member_dns_ipv6, member_wapi) discovered_member_ids.append(member_id) # deleting members are delicate operation so we won't allow it # but we will set member status to OFF to unused members. db_member_ids = utils.get_values_from_records('member_id', db_members) persisted_set = set(db_member_ids) discovered_set = set(discovered_member_ids) disable_set = persisted_set.difference(discovered_set) disabling_member_ids = list(disable_set) for member_id in disabling_member_ids: dbi.update_member(session, member_id, grid_id, member_status=const.MEMBER_STATUS_OFF) session.flush()
def _sync_network_views(self, discovered_netviews, dns_views): """Discover network views and sync with db. The discovered network view json contains the following data: - network view - cloud_info for delegated member if cloud platform is supported - mapping conditional EAs So discovered information will be updated in tables such as infoblox_network_views and infoblox_mapping_conditions. :param discovered_netviews: discovered network view json :return: None """ session = self._context.session self._load_persisted_mappings() discovered_delegations = dict() persisted_netview_ids = utils.get_values_from_records( 'id', self.db_network_views) discovered_netview_ids = [] for netview in discovered_netviews: netview_name = netview['name'] is_default = netview[const.IS_DEFAULT] netview_id = utils.get_network_view_id(self._grid_id, netview['_ref']) cloud_adapter_id_vals = utils.get_ea_value( const.EA_CLOUD_ADAPTER_ID, netview, True) if cloud_adapter_id_vals is None: participated = False else: cloud_adapter_ids = [gid for gid in cloud_adapter_id_vals if int(gid) == self._grid_id] participated = True if cloud_adapter_ids else False if not participated: continue shared_val = utils.get_ea_value(const.EA_IS_SHARED, netview) is_shared = types.Boolean()(shared_val) if shared_val else False # authority member is default to GM gm_row = utils.find_one_in_list('member_type', const.MEMBER_TYPE_GRID_MASTER, self.db_members) authority_member_id = gm_row.member_id # get delegation member if cloud platform is supported delegated_member = self._get_delegated_member(netview) if delegated_member: authority_member_id = delegated_member.member_id discovered_delegations[netview_name] = ( delegated_member.member_id) dns_view = (dns_views[netview_name] if dns_views.get(netview_name) else None) # see if the network view already exists in db netview_row = utils.find_one_in_list('id', netview_id, self.db_network_views) if netview_row: dbi.update_network_view(session, netview_id, netview_name, authority_member_id, is_shared, dns_view, participated, is_default) else: internal_netview = (const.DEFAULT_NETWORK_VIEW if is_default else netview_name) internal_dnsview = (const.DEFAULT_DNS_VIEW if is_default else dns_view) dbi.add_network_view(session, netview_id, netview_name, self._grid_id, authority_member_id, is_shared, dns_view, internal_netview, internal_dnsview, participated, is_default) discovered_netview_ids.append(netview_id) # update mapping conditions for the current network view self._update_mapping_conditions(netview, netview_id, participated) # we have added new network views. now let's remove persisted # network views not found from discovery persisted_set = set(persisted_netview_ids) removable_set = persisted_set.difference(discovered_netview_ids) removable_netviews = list(removable_set) if removable_netviews: dbi.remove_network_views(session, removable_netviews) session.flush() return discovered_delegations
def sync_members(self): """Synchronizes grid members. Members in the active grid are discovered from NIOS backend and grid members are in sync in neutron db. The members who are no longer in used are set to 'OFF' status. """ session = self._context.session grid_id = self._grid_config.grid_id db_grids = dbi.get_grids(session) db_grid = utils.find_one_in_list('grid_id', grid_id, db_grids) gm_member_id = db_grid.gm_id db_members = dbi.get_members(session, grid_id=grid_id) gm_member = utils.find_one_in_list('member_id', gm_member_id, db_members) discovered_members = self._discover_members() if not discovered_members: return dns_member_settings = self._discover_dns_settings() dhcp_member_settings = self._discover_dhcp_settings() discovered_licenses = self._discover_member_licenses() discovered_member_ids = [] for member in discovered_members: member_name = member['host_name'] member_ip, member_ipv6 = self._get_lan1_ips(member) member_wapi = member_ip if member_ip else member_ipv6 member_hwid = member['node_info'][0].get('hwid') member_status = self._get_member_status( member['node_info'][0]['service_status']) member_type = self._get_member_type(discovered_licenses, member_name, member_hwid) require_db_update = False if member_type == const.MEMBER_TYPE_GRID_MASTER: if gm_member: require_db_update = True member_id = gm_member_id member_wapi = self._grid_config.grid_master_host else: # no need to process 'Is Cloud Member' flag for non GM members ea_is_cloud_member = utils.get_ea_value( const.EA_IS_CLOUD_MEMBER, member) is_cloud_member = (types.Boolean()(ea_is_cloud_member) if ea_is_cloud_member else False) if not is_cloud_member: continue db_member = utils.find_one_in_list('member_name', member_name, db_members) if db_member: require_db_update = True member_id = db_member.member_id else: member_id = utils.get_hash(str(grid_id) + member_name) member_dhcp_ip, member_dhcp_ipv6 = self._get_dhcp_ips( member, dhcp_member_settings) member_dns_ip, member_dns_ipv6 = self._get_dns_ips( member, dns_member_settings) if require_db_update: dbi.update_member(session, member_id, grid_id, member_name, member_ip, member_ipv6, member_type, member_status, member_dhcp_ip, member_dhcp_ipv6, member_dns_ip, member_dns_ipv6, member_wapi) else: dbi.add_member(session, member_id, grid_id, member_name, member_ip, member_ipv6, member_type, member_status, member_dhcp_ip, member_dhcp_ipv6, member_dns_ip, member_dns_ipv6, member_wapi) discovered_member_ids.append(member_id) # deleting members are delicate operation so we won't allow it # but we will set member status to OFF to unused members. db_member_ids = utils.get_values_from_records('member_id', db_members) persisted_set = set(db_member_ids) discovered_set = set(discovered_member_ids) disable_set = persisted_set.difference(discovered_set) disabling_member_ids = list(disable_set) for member_id in disabling_member_ids: dbi.update_member(session, member_id, grid_id, member_status=const.MEMBER_STATUS_OFF) session.flush()