def upgrade_release_with_nic_and_bond_attributes(): connection = op.get_bind() select_query = sa.sql.text("SELECT id, version FROM releases") update_query = sa.sql.text( "UPDATE releases SET nic_attributes = :nic_attributes, " "bond_attributes = :bond_attributes WHERE id = :id") for id, version in connection.execute(select_query): new_nic_attrs = copy.deepcopy(DEFAULT_RELEASE_NIC_ATTRIBUTES) new_bond_attrs = copy.deepcopy(DEFAULT_RELEASE_BOND_ATTRIBUTES) if is_feature_supported(version, FUEL_DPDK_HUGEPAGES_VERSION): new_nic_attrs.update(DEFAULT_RELEASE_NIC_NFV_ATTRIBUTES) new_bond_attrs.update(DEFAULT_RELEASE_BOND_NFV_ATTRIBUTES) connection.execute(update_query, id=id, nic_attributes=jsonutils.dumps(new_nic_attrs), bond_attributes=jsonutils.dumps(new_bond_attrs))
def update_bonding_availability(bonding_availability): connection = op.get_bind() select_query = sa.sql.text( "SELECT id, version, networks_metadata FROM releases " "WHERE networks_metadata IS NOT NULL") update_query = sa.sql.text( "UPDATE releases SET networks_metadata = :networks_metadata " "WHERE id = :id") for id, version, nets in connection.execute(select_query): if not is_feature_supported(version, FUEL_DPDK_HUGEPAGES_VERSION): continue nets = jsonutils.loads(nets) if 'bonding' in nets and 'availability' in nets['bonding']: nets['bonding']['availability'] = bonding_availability connection.execute(update_query, id=id, networks_metadata=jsonutils.dumps(nets))
def test_tags_migration_for_supported_releases(self): releases = self.meta.tables['releases'] query = sa.select([ releases.c.version, releases.c.roles_metadata, releases.c.tags_metadata ]) for version, roles_meta, tags_meta in db.execute(query): if not is_feature_supported(version, FUEL_TAGS_SUPPORT): continue roles_meta = jsonutils.loads(roles_meta) for role_name, role_meta in six.iteritems(NEW_ROLES_META): self.assertItemsEqual(roles_meta[role_name]['tags'], role_meta['tags']) tags_meta = jsonutils.loads(tags_meta) missing_tags = set(NEW_TAGS_LIST) - set(tags_meta) self.assertEqual(len(missing_tags), 0)
def test_tags_migration_for_not_supported_releases(self): releases = self.meta.tables['releases'] query = sa.select([releases.c.version, releases.c.roles_metadata, releases.c.tags_metadata]) for version, roles_meta, tags_meta in db.execute(query): if is_feature_supported(version, FUEL_TAGS_SUPPORT): continue roles_meta = jsonutils.loads(roles_meta) for role_name, role_meta in six.iteritems(NEW_ROLES_META): common_tags = (set(role_meta['tags']) & set(roles_meta[role_name]['tags'])) # common tag 'controller' for backward compatibility self.assertEqual(len(common_tags), 1) tags_meta = jsonutils.loads(tags_meta) wrong_tags = set(NEW_TAGS_LIST) - set(tags_meta) self.assertNotEqual(len(wrong_tags), 0)
def test_tags_migration_for_supported_releases(self): releases = self.meta.tables['releases'] query = sa.select([releases.c.version, releases.c.roles_metadata, releases.c.tags_metadata]) for version, roles_meta, tags_meta in db.execute(query): if not is_feature_supported(version, FUEL_TAGS_SUPPORT): continue roles_meta = jsonutils.loads(roles_meta) for role_name, role_meta in six.iteritems(NEW_ROLES_META): self.assertItemsEqual( roles_meta[role_name]['tags'], role_meta['tags'] ) tags_meta = jsonutils.loads(tags_meta) missing_tags = set(NEW_TAGS_LIST) - set(tags_meta) self.assertEqual(len(missing_tags), 0)
def update_bonding_availability(bonding_availability): connection = op.get_bind() select_query = sa.sql.text( "SELECT id, version, networks_metadata FROM releases " "WHERE networks_metadata IS NOT NULL") update_query = sa.sql.text( "UPDATE releases SET networks_metadata = :networks_metadata " "WHERE id = :id") for id, version, nets in connection.execute(select_query): if not is_feature_supported(version, FUEL_DPDK_HUGEPAGES_VERSION): continue nets = jsonutils.loads(nets) if 'bonding' in nets and 'availability' in nets['bonding']: nets['bonding']['availability'] = bonding_availability connection.execute( update_query, id=id, networks_metadata=jsonutils.dumps(nets))
def upgrade_release_with_nic_and_bond_attributes(): connection = op.get_bind() select_query = sa.sql.text( "SELECT id, version FROM releases") update_query = sa.sql.text( "UPDATE releases SET nic_attributes = :nic_attributes, " "bond_attributes = :bond_attributes WHERE id = :id") for id, version in connection.execute(select_query): new_nic_attrs = copy.deepcopy(DEFAULT_RELEASE_NIC_ATTRIBUTES) new_bond_attrs = copy.deepcopy(DEFAULT_RELEASE_BOND_ATTRIBUTES) if is_feature_supported(version, FUEL_DPDK_HUGEPAGES_VERSION): new_nic_attrs.update(DEFAULT_RELEASE_NIC_NFV_ATTRIBUTES) new_bond_attrs.update(DEFAULT_RELEASE_BOND_NFV_ATTRIBUTES) connection.execute( update_query, id=id, nic_attributes=jsonutils.dumps(new_nic_attrs), bond_attributes=jsonutils.dumps(new_bond_attrs) )
def test_tags_migration_for_not_supported_releases(self): releases = self.meta.tables['releases'] query = sa.select([ releases.c.version, releases.c.roles_metadata, releases.c.tags_metadata ]) for version, roles_meta, tags_meta in db.execute(query): if is_feature_supported(version, FUEL_TAGS_SUPPORT): continue roles_meta = jsonutils.loads(roles_meta) for role_name, role_meta in six.iteritems(NEW_ROLES_META): common_tags = (set(role_meta['tags']) & set(roles_meta[role_name]['tags'])) # common tag 'controller' for backward compatibility self.assertEqual(len(common_tags), 1) tags_meta = jsonutils.loads(tags_meta) wrong_tags = set(NEW_TAGS_LIST) - set(tags_meta) self.assertNotEqual(len(wrong_tags), 0)
def upgrade_tags_set(): connection = op.get_bind() q_get_role_tags_meta = sa.text( "SELECT id, version, roles_metadata, tags_metadata FROM releases") q_update_role_tags_meta = sa.text( "UPDATE releases " "SET roles_metadata = :roles_meta, tags_metadata = :tags_meta " "WHERE id = :obj_id") for obj_id, version, roles_meta, tags_meta in connection.execute( q_get_role_tags_meta): if not is_feature_supported(version, FUEL_TAGS_SUPPORT): continue roles_meta = jsonutils.loads(roles_meta or '{}') tags_meta = jsonutils.loads(tags_meta or '{}') dict_update(roles_meta, NEW_ROLES_META) dict_update(tags_meta, NEW_TAGS_META) connection.execute(q_update_role_tags_meta, roles_meta=jsonutils.dumps(roles_meta), tags_meta=jsonutils.dumps(tags_meta), obj_id=obj_id)
def upgrade_node_bond_attributes(): def _get_offloading_modes(slaves_offloading_modes_values): result = {} if slaves_offloading_modes_values: intersected_modes = six.moves.reduce( lambda x, y: x.intersection(y), slaves_offloading_modes_values[1:], set(slaves_offloading_modes_values[0]) ) for slave_offloading_modes in slaves_offloading_modes_values: for k in intersected_modes: v = result.get(k, slave_offloading_modes[k]) v = (False if False in (v, slave_offloading_modes[k]) else v and slave_offloading_modes[k]) result[k] = v return result bond_interface_query = sa.sql.text(""" SELECT node_bond_interfaces.id, node_bond_interfaces.mode, bond_properties, interface_properties, version FROM node_bond_interfaces INNER JOIN nodes ON nodes.id = node_bond_interfaces.node_id INNER JOIN clusters ON clusters.id = nodes.cluster_id INNER JOIN releases ON releases.id = clusters.release_id """) bond_slaves_offloading_modes_query = sa.sql.text( "SELECT attributes FROM node_nic_interfaces " "WHERE parent_id = :parent_id") upgrade_bond_interface_query = sa.sql.text(""" UPDATE node_bond_interfaces SET attributes = :attributes WHERE id = :id""") connection = op.get_bind() for result in connection.execute(bond_interface_query): attributes = copy.deepcopy(DEFAULT_RELEASE_BOND_ATTRIBUTES) bond_properties = jsonutils.loads(result['bond_properties']) interface_properties = jsonutils.loads(result['interface_properties']) attributes['mtu']['value']['value'] = interface_properties.get('mtu') attributes['offloading']['disable']['value'] = \ interface_properties.get('disable_offloading', False) if is_feature_supported(result['version'], FUEL_DPDK_HUGEPAGES_VERSION): attributes.update(DEFAULT_RELEASE_BOND_NFV_ATTRIBUTES) attributes['dpdk']['enabled']['value'] = \ interface_properties.get('dpdk', {}).get('enabled', False) attributes['type__']['value'] = bond_properties.get('type__') attributes['mode']['value']['value'] = \ bond_properties.get('mode') or result['mode'] or '' for k in ('lacp_rate', 'xmit_hash_policy', 'lacp'): if k in bond_properties: attributes[k]['value']['value'] = bond_properties[k] slaves_offloading_modes_flat_values = [ jsonutils.loads(r['attributes']).get('offloading', {}).get( 'modes', {}).get('value', {}) for r in connection.execute(bond_slaves_offloading_modes_query, parent_id=result['id']) ] attributes['offloading']['modes']['value'] = _get_offloading_modes( slaves_offloading_modes_flat_values) connection.execute(upgrade_bond_interface_query, attributes=jsonutils.dumps(attributes), id=result['id'])
def upgrade_node_nic_attributes(): def _create_interface_meta(interfaces_data): interface_properties = interfaces_data.get('interface_properties', {}) return { 'offloading_modes': interfaces_data.get('offloading_modes', []), 'sriov': { 'available': interface_properties.get( 'sriov', {}).get('available', False), 'pci_id': interface_properties.get( 'sriov', {}).get('pci_id', ''), 'totalvfs': interface_properties.get( 'sriov', {}).get('sriov_totalvfs', 0) }, 'dpdk': { 'available': False }, 'pci_id': interface_properties.get('pci_id', ''), 'numa_node': interface_properties.get('numa_node') } def _offloading_modes_as_flat_dict(modes): result = dict() if modes is None: return result for mode in modes: result[mode['name']] = mode['state'] if mode.get('sub'): result.update(_offloading_modes_as_flat_dict(mode['sub'])) return result def _create_nic_attributes(interface_properties, offloading_modes, support_nfv=False): nic_attributes = copy.deepcopy(DEFAULT_RELEASE_NIC_ATTRIBUTES) nic_attributes['mtu']['value']['value'] = \ interface_properties.get('mtu') if support_nfv: nic_attributes.update(DEFAULT_RELEASE_NIC_NFV_ATTRIBUTES) nic_attributes['sriov']['enabled']['value'] = \ interface_properties.get('sriov', {}).get('enabled', False) nic_attributes['sriov']['numvfs']['value'] = \ interface_properties.get('sriov', {}).get('sriov_numvfs') nic_attributes['sriov']['physnet']['value'] = \ interface_properties.get('sriov', {}).get('physnet', 'physnet2') nic_attributes['dpdk']['enabled']['value'] = \ interface_properties.get('dpdk', {}).get('enabled', False) nic_attributes['offloading']['disable']['value'] = \ interface_properties.get('disable_offloading', False) offloading_modes = _offloading_modes_as_flat_dict(offloading_modes) nic_attributes['offloading']['modes']['value'] = offloading_modes return nic_attributes nodes_query = sa.sql.text("SELECT nodes.id, meta, cluster_id, version " "FROM nodes INNER JOIN clusters " "ON clusters.id = nodes.cluster_id " "INNER JOIN releases " "ON releases.id = clusters.release_id") select_interface_query = sa.sql.text(""" SELECT id, interface_properties, offloading_modes FROM node_nic_interfaces WHERE node_id = :node_id AND mac = :mac""") upgrade_interface_query = sa.sql.text(""" UPDATE node_nic_interfaces SET attributes = :attributes, meta = :meta WHERE id = :id""") connection = op.get_bind() query_result = connection.execute(nodes_query) for node_id, node_meta, cluster_id, version in query_result: node_meta = jsonutils.loads(node_meta or "{}") for node_interface in node_meta.get('interfaces', []): for iface_id, interface_properties, offloading_modes in \ connection.execute( select_interface_query, node_id=node_id, mac=node_interface['mac']): interface_meta = _create_interface_meta(node_interface) nic_attributes = {} if cluster_id: iface_properties = jsonutils.loads(interface_properties) nic_attributes = _create_nic_attributes( iface_properties, jsonutils.loads(offloading_modes), is_feature_supported(version, FUEL_DPDK_HUGEPAGES_VERSION) ) interface_meta['dpdk']['available'] = \ iface_properties.get('dpdk', {}).get( 'available', interface_meta['dpdk']['available']) connection.execute( upgrade_interface_query, attributes=jsonutils.dumps(nic_attributes), meta=jsonutils.dumps(interface_meta), id=iface_id)
def upgrade_node_nic_attributes(): def _create_interface_meta(interfaces_data): interface_properties = interfaces_data.get('interface_properties', {}) return { 'offloading_modes': interfaces_data.get('offloading_modes', []), 'sriov': { 'available': interface_properties.get('sriov', {}).get('available', False), 'pci_id': interface_properties.get('sriov', {}).get('pci_id', ''), 'totalvfs': interface_properties.get('sriov', {}).get('sriov_totalvfs', 0) }, 'dpdk': { 'available': False }, 'pci_id': interface_properties.get('pci_id', ''), 'numa_node': interface_properties.get('numa_node') } def _offloading_modes_as_flat_dict(modes): result = dict() if modes is None: return result for mode in modes: result[mode['name']] = mode['state'] if mode.get('sub'): result.update(_offloading_modes_as_flat_dict(mode['sub'])) return result def _create_nic_attributes(interface_properties, offloading_modes, support_nfv=False): nic_attributes = copy.deepcopy(DEFAULT_RELEASE_NIC_ATTRIBUTES) nic_attributes['mtu']['value']['value'] = \ interface_properties.get('mtu') if support_nfv: nic_attributes.update(DEFAULT_RELEASE_NIC_NFV_ATTRIBUTES) nic_attributes['sriov']['enabled']['value'] = \ interface_properties.get('sriov', {}).get('enabled', False) nic_attributes['sriov']['numvfs']['value'] = \ interface_properties.get('sriov', {}).get('sriov_numvfs') nic_attributes['sriov']['physnet']['value'] = \ interface_properties.get('sriov', {}).get('physnet', 'physnet2') nic_attributes['dpdk']['enabled']['value'] = \ interface_properties.get('dpdk', {}).get('enabled', False) nic_attributes['offloading']['disable']['value'] = \ interface_properties.get('disable_offloading', False) offloading_modes = _offloading_modes_as_flat_dict(offloading_modes) nic_attributes['offloading']['modes']['value'] = offloading_modes return nic_attributes nodes_query = sa.sql.text("SELECT nodes.id, meta, cluster_id, version " "FROM nodes INNER JOIN clusters " "ON clusters.id = nodes.cluster_id " "INNER JOIN releases " "ON releases.id = clusters.release_id") select_interface_query = sa.sql.text(""" SELECT id, interface_properties, offloading_modes FROM node_nic_interfaces WHERE node_id = :node_id AND mac = :mac""") upgrade_interface_query = sa.sql.text(""" UPDATE node_nic_interfaces SET attributes = :attributes, meta = :meta WHERE id = :id""") connection = op.get_bind() query_result = connection.execute(nodes_query) for node_id, node_meta, cluster_id, version in query_result: node_meta = jsonutils.loads(node_meta or "{}") for node_interface in node_meta.get('interfaces', []): for iface_id, interface_properties, offloading_modes in \ connection.execute( select_interface_query, node_id=node_id, mac=node_interface['mac']): interface_meta = _create_interface_meta(node_interface) nic_attributes = {} if cluster_id: iface_properties = jsonutils.loads(interface_properties) nic_attributes = _create_nic_attributes( iface_properties, jsonutils.loads(offloading_modes), is_feature_supported(version, FUEL_DPDK_HUGEPAGES_VERSION)) interface_meta['dpdk']['available'] = \ iface_properties.get('dpdk', {}).get( 'available', interface_meta['dpdk']['available']) connection.execute(upgrade_interface_query, attributes=jsonutils.dumps(nic_attributes), meta=jsonutils.dumps(interface_meta), id=iface_id)
def upgrade_node_bond_attributes(): def _get_offloading_modes(slaves_offloading_modes_values): result = {} if slaves_offloading_modes_values: intersected_modes = six.moves.reduce( lambda x, y: x.intersection(y), slaves_offloading_modes_values[1:], set(slaves_offloading_modes_values[0])) for slave_offloading_modes in slaves_offloading_modes_values: for k in intersected_modes: v = result.get(k, slave_offloading_modes[k]) v = (False if False in (v, slave_offloading_modes[k]) else v and slave_offloading_modes[k]) result[k] = v return result bond_interface_query = sa.sql.text(""" SELECT node_bond_interfaces.id, node_bond_interfaces.mode, bond_properties, interface_properties, version FROM node_bond_interfaces INNER JOIN nodes ON nodes.id = node_bond_interfaces.node_id INNER JOIN clusters ON clusters.id = nodes.cluster_id INNER JOIN releases ON releases.id = clusters.release_id """) bond_slaves_offloading_modes_query = sa.sql.text( "SELECT attributes FROM node_nic_interfaces " "WHERE parent_id = :parent_id") upgrade_bond_interface_query = sa.sql.text(""" UPDATE node_bond_interfaces SET attributes = :attributes WHERE id = :id""") connection = op.get_bind() for result in connection.execute(bond_interface_query): attributes = copy.deepcopy(DEFAULT_RELEASE_BOND_ATTRIBUTES) bond_properties = jsonutils.loads(result['bond_properties']) interface_properties = jsonutils.loads(result['interface_properties']) attributes['mtu']['value']['value'] = interface_properties.get('mtu') attributes['offloading']['disable']['value'] = \ interface_properties.get('disable_offloading', False) if is_feature_supported(result['version'], FUEL_DPDK_HUGEPAGES_VERSION): attributes.update(DEFAULT_RELEASE_BOND_NFV_ATTRIBUTES) attributes['dpdk']['enabled']['value'] = \ interface_properties.get('dpdk', {}).get('enabled', False) attributes['type__']['value'] = bond_properties.get('type__') attributes['mode']['value']['value'] = \ bond_properties.get('mode') or result['mode'] or '' for k in ('lacp_rate', 'xmit_hash_policy', 'lacp'): if k in bond_properties: attributes[k]['value']['value'] = bond_properties[k] slaves_offloading_modes_flat_values = [ jsonutils.loads(r['attributes']).get('offloading', {}).get('modes', {}).get('value', {}) for r in connection.execute(bond_slaves_offloading_modes_query, parent_id=result['id']) ] attributes['offloading']['modes']['value'] = _get_offloading_modes( slaves_offloading_modes_flat_values) connection.execute(upgrade_bond_interface_query, attributes=jsonutils.dumps(attributes), id=result['id'])