def upgrade_release_with_nic_and_bond_attributes():
connection = op.get_bind()
select_query = sa.sql.text("SELECT id, version FROM releases")
update_query = sa.sql.text(
"UPDATE releases SET nic_attributes = :nic_attributes, "
"bond_attributes = :bond_attributes WHERE id = :id")
for id, version in connection.execute(select_query):
new_nic_attrs = copy.deepcopy(DEFAULT_RELEASE_NIC_ATTRIBUTES)
new_bond_attrs = copy.deepcopy(DEFAULT_RELEASE_BOND_ATTRIBUTES)
if is_feature_supported(version, FUEL_DPDK_HUGEPAGES_VERSION):
new_nic_attrs.update(DEFAULT_RELEASE_NIC_NFV_ATTRIBUTES)
new_bond_attrs.update(DEFAULT_RELEASE_BOND_NFV_ATTRIBUTES)
connection.execute(update_query,
id=id,
nic_attributes=jsonutils.dumps(new_nic_attrs),
bond_attributes=jsonutils.dumps(new_bond_attrs))
def update_bonding_availability(bonding_availability):
connection = op.get_bind()
select_query = sa.sql.text(
"SELECT id, version, networks_metadata FROM releases "
"WHERE networks_metadata IS NOT NULL")
update_query = sa.sql.text(
"UPDATE releases SET networks_metadata = :networks_metadata "
"WHERE id = :id")
for id, version, nets in connection.execute(select_query):
if not is_feature_supported(version, FUEL_DPDK_HUGEPAGES_VERSION):
continue
nets = jsonutils.loads(nets)
if 'bonding' in nets and 'availability' in nets['bonding']:
nets['bonding']['availability'] = bonding_availability
connection.execute(update_query,
id=id,
networks_metadata=jsonutils.dumps(nets))
def test_tags_migration_for_supported_releases(self):
releases = self.meta.tables['releases']
query = sa.select([
releases.c.version, releases.c.roles_metadata,
releases.c.tags_metadata
])
for version, roles_meta, tags_meta in db.execute(query):
if not is_feature_supported(version, FUEL_TAGS_SUPPORT):
continue
roles_meta = jsonutils.loads(roles_meta)
for role_name, role_meta in six.iteritems(NEW_ROLES_META):
self.assertItemsEqual(roles_meta[role_name]['tags'],
role_meta['tags'])
tags_meta = jsonutils.loads(tags_meta)
missing_tags = set(NEW_TAGS_LIST) - set(tags_meta)
self.assertEqual(len(missing_tags), 0)
def test_tags_migration_for_not_supported_releases(self):
releases = self.meta.tables['releases']
query = sa.select([releases.c.version,
releases.c.roles_metadata,
releases.c.tags_metadata])
for version, roles_meta, tags_meta in db.execute(query):
if is_feature_supported(version, FUEL_TAGS_SUPPORT):
continue
roles_meta = jsonutils.loads(roles_meta)
for role_name, role_meta in six.iteritems(NEW_ROLES_META):
common_tags = (set(role_meta['tags']) &
set(roles_meta[role_name]['tags']))
# common tag 'controller' for backward compatibility
self.assertEqual(len(common_tags), 1)
tags_meta = jsonutils.loads(tags_meta)
wrong_tags = set(NEW_TAGS_LIST) - set(tags_meta)
self.assertNotEqual(len(wrong_tags), 0)
def test_tags_migration_for_supported_releases(self):
releases = self.meta.tables['releases']
query = sa.select([releases.c.version,
releases.c.roles_metadata,
releases.c.tags_metadata])
for version, roles_meta, tags_meta in db.execute(query):
if not is_feature_supported(version, FUEL_TAGS_SUPPORT):
continue
roles_meta = jsonutils.loads(roles_meta)
for role_name, role_meta in six.iteritems(NEW_ROLES_META):
self.assertItemsEqual(
roles_meta[role_name]['tags'],
role_meta['tags']
)
tags_meta = jsonutils.loads(tags_meta)
missing_tags = set(NEW_TAGS_LIST) - set(tags_meta)
self.assertEqual(len(missing_tags), 0)
def update_bonding_availability(bonding_availability):
connection = op.get_bind()
select_query = sa.sql.text(
"SELECT id, version, networks_metadata FROM releases "
"WHERE networks_metadata IS NOT NULL")
update_query = sa.sql.text(
"UPDATE releases SET networks_metadata = :networks_metadata "
"WHERE id = :id")
for id, version, nets in connection.execute(select_query):
if not is_feature_supported(version, FUEL_DPDK_HUGEPAGES_VERSION):
continue
nets = jsonutils.loads(nets)
if 'bonding' in nets and 'availability' in nets['bonding']:
nets['bonding']['availability'] = bonding_availability
connection.execute(
update_query,
id=id,
networks_metadata=jsonutils.dumps(nets))
def upgrade_release_with_nic_and_bond_attributes():
connection = op.get_bind()
select_query = sa.sql.text(
"SELECT id, version FROM releases")
update_query = sa.sql.text(
"UPDATE releases SET nic_attributes = :nic_attributes, "
"bond_attributes = :bond_attributes WHERE id = :id")
for id, version in connection.execute(select_query):
new_nic_attrs = copy.deepcopy(DEFAULT_RELEASE_NIC_ATTRIBUTES)
new_bond_attrs = copy.deepcopy(DEFAULT_RELEASE_BOND_ATTRIBUTES)
if is_feature_supported(version, FUEL_DPDK_HUGEPAGES_VERSION):
new_nic_attrs.update(DEFAULT_RELEASE_NIC_NFV_ATTRIBUTES)
new_bond_attrs.update(DEFAULT_RELEASE_BOND_NFV_ATTRIBUTES)
connection.execute(
update_query,
id=id,
nic_attributes=jsonutils.dumps(new_nic_attrs),
bond_attributes=jsonutils.dumps(new_bond_attrs)
)
def test_tags_migration_for_not_supported_releases(self):
releases = self.meta.tables['releases']
query = sa.select([
releases.c.version, releases.c.roles_metadata,
releases.c.tags_metadata
])
for version, roles_meta, tags_meta in db.execute(query):
if is_feature_supported(version, FUEL_TAGS_SUPPORT):
continue
roles_meta = jsonutils.loads(roles_meta)
for role_name, role_meta in six.iteritems(NEW_ROLES_META):
common_tags = (set(role_meta['tags'])
& set(roles_meta[role_name]['tags']))
# common tag 'controller' for backward compatibility
self.assertEqual(len(common_tags), 1)
tags_meta = jsonutils.loads(tags_meta)
wrong_tags = set(NEW_TAGS_LIST) - set(tags_meta)
self.assertNotEqual(len(wrong_tags), 0)
def upgrade_tags_set():
connection = op.get_bind()
q_get_role_tags_meta = sa.text(
"SELECT id, version, roles_metadata, tags_metadata FROM releases")
q_update_role_tags_meta = sa.text(
"UPDATE releases "
"SET roles_metadata = :roles_meta, tags_metadata = :tags_meta "
"WHERE id = :obj_id")
for obj_id, version, roles_meta, tags_meta in connection.execute(
q_get_role_tags_meta):
if not is_feature_supported(version, FUEL_TAGS_SUPPORT):
continue
roles_meta = jsonutils.loads(roles_meta or '{}')
tags_meta = jsonutils.loads(tags_meta or '{}')
dict_update(roles_meta, NEW_ROLES_META)
dict_update(tags_meta, NEW_TAGS_META)
connection.execute(q_update_role_tags_meta,
roles_meta=jsonutils.dumps(roles_meta),
tags_meta=jsonutils.dumps(tags_meta),
obj_id=obj_id)
def upgrade_tags_set():
connection = op.get_bind()
q_get_role_tags_meta = sa.text(
"SELECT id, version, roles_metadata, tags_metadata FROM releases")
q_update_role_tags_meta = sa.text(
"UPDATE releases "
"SET roles_metadata = :roles_meta, tags_metadata = :tags_meta "
"WHERE id = :obj_id")
for obj_id, version, roles_meta, tags_meta in connection.execute(
q_get_role_tags_meta):
if not is_feature_supported(version, FUEL_TAGS_SUPPORT):
continue
roles_meta = jsonutils.loads(roles_meta or '{}')
tags_meta = jsonutils.loads(tags_meta or '{}')
dict_update(roles_meta, NEW_ROLES_META)
dict_update(tags_meta, NEW_TAGS_META)
connection.execute(q_update_role_tags_meta,
roles_meta=jsonutils.dumps(roles_meta),
tags_meta=jsonutils.dumps(tags_meta),
obj_id=obj_id)
def upgrade_node_bond_attributes():
def _get_offloading_modes(slaves_offloading_modes_values):
result = {}
if slaves_offloading_modes_values:
intersected_modes = six.moves.reduce(
lambda x, y: x.intersection(y),
slaves_offloading_modes_values[1:],
set(slaves_offloading_modes_values[0])
)
for slave_offloading_modes in slaves_offloading_modes_values:
for k in intersected_modes:
v = result.get(k, slave_offloading_modes[k])
v = (False if False in (v, slave_offloading_modes[k])
else v and slave_offloading_modes[k])
result[k] = v
return result
bond_interface_query = sa.sql.text("""
SELECT node_bond_interfaces.id, node_bond_interfaces.mode,
bond_properties, interface_properties, version
FROM node_bond_interfaces
INNER JOIN nodes ON nodes.id = node_bond_interfaces.node_id
INNER JOIN clusters ON clusters.id = nodes.cluster_id
INNER JOIN releases ON releases.id = clusters.release_id """)
bond_slaves_offloading_modes_query = sa.sql.text(
"SELECT attributes FROM node_nic_interfaces "
"WHERE parent_id = :parent_id")
upgrade_bond_interface_query = sa.sql.text("""
UPDATE node_bond_interfaces
SET attributes = :attributes
WHERE id = :id""")
connection = op.get_bind()
for result in connection.execute(bond_interface_query):
attributes = copy.deepcopy(DEFAULT_RELEASE_BOND_ATTRIBUTES)
bond_properties = jsonutils.loads(result['bond_properties'])
interface_properties = jsonutils.loads(result['interface_properties'])
attributes['mtu']['value']['value'] = interface_properties.get('mtu')
attributes['offloading']['disable']['value'] = \
interface_properties.get('disable_offloading', False)
if is_feature_supported(result['version'],
FUEL_DPDK_HUGEPAGES_VERSION):
attributes.update(DEFAULT_RELEASE_BOND_NFV_ATTRIBUTES)
attributes['dpdk']['enabled']['value'] = \
interface_properties.get('dpdk', {}).get('enabled', False)
attributes['type__']['value'] = bond_properties.get('type__')
attributes['mode']['value']['value'] = \
bond_properties.get('mode') or result['mode'] or ''
for k in ('lacp_rate', 'xmit_hash_policy', 'lacp'):
if k in bond_properties:
attributes[k]['value']['value'] = bond_properties[k]
slaves_offloading_modes_flat_values = [
jsonutils.loads(r['attributes']).get('offloading', {}).get(
'modes', {}).get('value', {})
for r in connection.execute(bond_slaves_offloading_modes_query,
parent_id=result['id'])
]
attributes['offloading']['modes']['value'] = _get_offloading_modes(
slaves_offloading_modes_flat_values)
connection.execute(upgrade_bond_interface_query,
attributes=jsonutils.dumps(attributes),
id=result['id'])
def upgrade_node_nic_attributes():
def _create_interface_meta(interfaces_data):
interface_properties = interfaces_data.get('interface_properties', {})
return {
'offloading_modes': interfaces_data.get('offloading_modes', []),
'sriov': {
'available': interface_properties.get(
'sriov', {}).get('available', False),
'pci_id': interface_properties.get(
'sriov', {}).get('pci_id', ''),
'totalvfs': interface_properties.get(
'sriov', {}).get('sriov_totalvfs', 0)
},
'dpdk': {
'available': False
},
'pci_id': interface_properties.get('pci_id', ''),
'numa_node': interface_properties.get('numa_node')
}
def _offloading_modes_as_flat_dict(modes):
result = dict()
if modes is None:
return result
for mode in modes:
result[mode['name']] = mode['state']
if mode.get('sub'):
result.update(_offloading_modes_as_flat_dict(mode['sub']))
return result
def _create_nic_attributes(interface_properties, offloading_modes,
support_nfv=False):
nic_attributes = copy.deepcopy(DEFAULT_RELEASE_NIC_ATTRIBUTES)
nic_attributes['mtu']['value']['value'] = \
interface_properties.get('mtu')
if support_nfv:
nic_attributes.update(DEFAULT_RELEASE_NIC_NFV_ATTRIBUTES)
nic_attributes['sriov']['enabled']['value'] = \
interface_properties.get('sriov', {}).get('enabled', False)
nic_attributes['sriov']['numvfs']['value'] = \
interface_properties.get('sriov', {}).get('sriov_numvfs')
nic_attributes['sriov']['physnet']['value'] = \
interface_properties.get('sriov', {}).get('physnet',
'physnet2')
nic_attributes['dpdk']['enabled']['value'] = \
interface_properties.get('dpdk', {}).get('enabled', False)
nic_attributes['offloading']['disable']['value'] = \
interface_properties.get('disable_offloading', False)
offloading_modes = _offloading_modes_as_flat_dict(offloading_modes)
nic_attributes['offloading']['modes']['value'] = offloading_modes
return nic_attributes
nodes_query = sa.sql.text("SELECT nodes.id, meta, cluster_id, version "
"FROM nodes INNER JOIN clusters "
"ON clusters.id = nodes.cluster_id "
"INNER JOIN releases "
"ON releases.id = clusters.release_id")
select_interface_query = sa.sql.text("""
SELECT id, interface_properties, offloading_modes
FROM node_nic_interfaces
WHERE node_id = :node_id AND mac = :mac""")
upgrade_interface_query = sa.sql.text("""
UPDATE node_nic_interfaces
SET attributes = :attributes, meta = :meta
WHERE id = :id""")
connection = op.get_bind()
query_result = connection.execute(nodes_query)
for node_id, node_meta, cluster_id, version in query_result:
node_meta = jsonutils.loads(node_meta or "{}")
for node_interface in node_meta.get('interfaces', []):
for iface_id, interface_properties, offloading_modes in \
connection.execute(
select_interface_query,
node_id=node_id,
mac=node_interface['mac']):
interface_meta = _create_interface_meta(node_interface)
nic_attributes = {}
if cluster_id:
iface_properties = jsonutils.loads(interface_properties)
nic_attributes = _create_nic_attributes(
iface_properties,
jsonutils.loads(offloading_modes),
is_feature_supported(version,
FUEL_DPDK_HUGEPAGES_VERSION)
)
interface_meta['dpdk']['available'] = \
iface_properties.get('dpdk', {}).get(
'available', interface_meta['dpdk']['available'])
connection.execute(
upgrade_interface_query,
attributes=jsonutils.dumps(nic_attributes),
meta=jsonutils.dumps(interface_meta),
id=iface_id)
def upgrade_node_nic_attributes():
def _create_interface_meta(interfaces_data):
interface_properties = interfaces_data.get('interface_properties', {})
return {
'offloading_modes': interfaces_data.get('offloading_modes', []),
'sriov': {
'available':
interface_properties.get('sriov', {}).get('available', False),
'pci_id':
interface_properties.get('sriov', {}).get('pci_id', ''),
'totalvfs':
interface_properties.get('sriov', {}).get('sriov_totalvfs', 0)
},
'dpdk': {
'available': False
},
'pci_id': interface_properties.get('pci_id', ''),
'numa_node': interface_properties.get('numa_node')
}
def _offloading_modes_as_flat_dict(modes):
result = dict()
if modes is None:
return result
for mode in modes:
result[mode['name']] = mode['state']
if mode.get('sub'):
result.update(_offloading_modes_as_flat_dict(mode['sub']))
return result
def _create_nic_attributes(interface_properties,
offloading_modes,
support_nfv=False):
nic_attributes = copy.deepcopy(DEFAULT_RELEASE_NIC_ATTRIBUTES)
nic_attributes['mtu']['value']['value'] = \
interface_properties.get('mtu')
if support_nfv:
nic_attributes.update(DEFAULT_RELEASE_NIC_NFV_ATTRIBUTES)
nic_attributes['sriov']['enabled']['value'] = \
interface_properties.get('sriov', {}).get('enabled', False)
nic_attributes['sriov']['numvfs']['value'] = \
interface_properties.get('sriov', {}).get('sriov_numvfs')
nic_attributes['sriov']['physnet']['value'] = \
interface_properties.get('sriov', {}).get('physnet',
'physnet2')
nic_attributes['dpdk']['enabled']['value'] = \
interface_properties.get('dpdk', {}).get('enabled', False)
nic_attributes['offloading']['disable']['value'] = \
interface_properties.get('disable_offloading', False)
offloading_modes = _offloading_modes_as_flat_dict(offloading_modes)
nic_attributes['offloading']['modes']['value'] = offloading_modes
return nic_attributes
nodes_query = sa.sql.text("SELECT nodes.id, meta, cluster_id, version "
"FROM nodes INNER JOIN clusters "
"ON clusters.id = nodes.cluster_id "
"INNER JOIN releases "
"ON releases.id = clusters.release_id")
select_interface_query = sa.sql.text("""
SELECT id, interface_properties, offloading_modes
FROM node_nic_interfaces
WHERE node_id = :node_id AND mac = :mac""")
upgrade_interface_query = sa.sql.text("""
UPDATE node_nic_interfaces
SET attributes = :attributes, meta = :meta
WHERE id = :id""")
connection = op.get_bind()
query_result = connection.execute(nodes_query)
for node_id, node_meta, cluster_id, version in query_result:
node_meta = jsonutils.loads(node_meta or "{}")
for node_interface in node_meta.get('interfaces', []):
for iface_id, interface_properties, offloading_modes in \
connection.execute(
select_interface_query,
node_id=node_id,
mac=node_interface['mac']):
interface_meta = _create_interface_meta(node_interface)
nic_attributes = {}
if cluster_id:
iface_properties = jsonutils.loads(interface_properties)
nic_attributes = _create_nic_attributes(
iface_properties, jsonutils.loads(offloading_modes),
is_feature_supported(version,
FUEL_DPDK_HUGEPAGES_VERSION))
interface_meta['dpdk']['available'] = \
iface_properties.get('dpdk', {}).get(
'available', interface_meta['dpdk']['available'])
connection.execute(upgrade_interface_query,
attributes=jsonutils.dumps(nic_attributes),
meta=jsonutils.dumps(interface_meta),
id=iface_id)
def upgrade_node_bond_attributes():
def _get_offloading_modes(slaves_offloading_modes_values):
result = {}
if slaves_offloading_modes_values:
intersected_modes = six.moves.reduce(
lambda x, y: x.intersection(y),
slaves_offloading_modes_values[1:],
set(slaves_offloading_modes_values[0]))
for slave_offloading_modes in slaves_offloading_modes_values:
for k in intersected_modes:
v = result.get(k, slave_offloading_modes[k])
v = (False if False in (v, slave_offloading_modes[k]) else
v and slave_offloading_modes[k])
result[k] = v
return result
bond_interface_query = sa.sql.text("""
SELECT node_bond_interfaces.id, node_bond_interfaces.mode,
bond_properties, interface_properties, version
FROM node_bond_interfaces
INNER JOIN nodes ON nodes.id = node_bond_interfaces.node_id
INNER JOIN clusters ON clusters.id = nodes.cluster_id
INNER JOIN releases ON releases.id = clusters.release_id """)
bond_slaves_offloading_modes_query = sa.sql.text(
"SELECT attributes FROM node_nic_interfaces "
"WHERE parent_id = :parent_id")
upgrade_bond_interface_query = sa.sql.text("""
UPDATE node_bond_interfaces
SET attributes = :attributes
WHERE id = :id""")
connection = op.get_bind()
for result in connection.execute(bond_interface_query):
attributes = copy.deepcopy(DEFAULT_RELEASE_BOND_ATTRIBUTES)
bond_properties = jsonutils.loads(result['bond_properties'])
interface_properties = jsonutils.loads(result['interface_properties'])
attributes['mtu']['value']['value'] = interface_properties.get('mtu')
attributes['offloading']['disable']['value'] = \
interface_properties.get('disable_offloading', False)
if is_feature_supported(result['version'],
FUEL_DPDK_HUGEPAGES_VERSION):
attributes.update(DEFAULT_RELEASE_BOND_NFV_ATTRIBUTES)
attributes['dpdk']['enabled']['value'] = \
interface_properties.get('dpdk', {}).get('enabled', False)
attributes['type__']['value'] = bond_properties.get('type__')
attributes['mode']['value']['value'] = \
bond_properties.get('mode') or result['mode'] or ''
for k in ('lacp_rate', 'xmit_hash_policy', 'lacp'):
if k in bond_properties:
attributes[k]['value']['value'] = bond_properties[k]
slaves_offloading_modes_flat_values = [
jsonutils.loads(r['attributes']).get('offloading',
{}).get('modes',
{}).get('value', {})
for r in connection.execute(bond_slaves_offloading_modes_query,
parent_id=result['id'])
]
attributes['offloading']['modes']['value'] = _get_offloading_modes(
slaves_offloading_modes_flat_values)
connection.execute(upgrade_bond_interface_query,
attributes=jsonutils.dumps(attributes),
id=result['id'])