def ovs_3769_validation_test():
"""
Create an albanode with an asd statistics part set to empty dictionary
Assert code does not raise
"""
an = AlbaNode()
an.password = '******'
an.node_id = 'ovs3769an'
an.port = 1234
an.ip = '127.0.0.1'
an.username = '******'
an.save()
bet = GeneralBackend.get_backendtype_by_code('alba')
be = Backend()
be.backend_type = bet
be.name = 'ovs3769be'
be.save()
abe = AlbaBackend()
abe.backend = be
abe.save()
ad = AlbaDisk()
ad.name = 'ovs3769ad'
ad.alba_node = an
ad.save()
asd = AlbaASD()
asd.alba_backend = abe
asd.asd_id = 'ovs3769asd'
asd.alba_disk = ad
asd.save()
try:
abe.statistics
except KeyError, ex:
logger.error('Regression OVS-3769 - asd statistics raises a KeyError: {0}'.format(str(ex)))
def build_service_structure(structure, previous_structure=None):
"""
Builds a service structure
Example:
structure = Helper.build_service_structure({
'alba_backends': [1],
'alba_nodes': [1]
})
"""
if previous_structure is None:
previous_structure = {}
backend_types = previous_structure.get('backend_types', {})
service_types = previous_structure.get('service_types', {})
alba_backends = previous_structure.get('alba_backends', {})
alba_nodes = previous_structure.get('alba_nodes', {})
alba_disks = previous_structure.get('alba_disks', {})
alba_osds = previous_structure.get('alba_osds', {})
if 1 not in backend_types:
backend_type = BackendType()
backend_type.code = 'alba'
backend_type.name = 'ALBA'
backend_type.save()
backend_types[1] = backend_type
if 1 not in service_types:
service_type = ServiceType()
service_type.name = 'AlbaManager'
service_type.save()
service_types[1] = service_type
for ab_id in structure.get('alba_backends', ()):
if ab_id not in alba_backends:
backend = Backend()
backend.name = 'backend_{0}'.format(ab_id)
backend.backend_type = backend_types[1]
backend.save()
alba_backend = AlbaBackend()
alba_backend.backend = backend
alba_backend.scaling = AlbaBackend.SCALINGS.LOCAL
alba_backend.save()
alba_backends[ab_id] = alba_backend
service = Service()
service.name = 'backend_{0}_abm'.format(ab_id)
service.type = service_types[1]
service.ports = []
service.save()
abm_service = ABMService()
abm_service.service = service
abm_service.alba_backend = alba_backend
abm_service.save()
for an_id in structure.get('alba_nodes', []):
if an_id not in alba_nodes:
alba_node = AlbaNode()
alba_node.ip = '10.1.0.{0}'.format(an_id)
alba_node.port = 8500
alba_node.username = str(an_id)
alba_node.password = str(an_id)
alba_node.node_id = 'node_{0}'.format(an_id)
alba_node.save()
alba_nodes[an_id] = alba_node
for ad_id, an_id in structure.get('alba_disks', ()):
if ad_id not in alba_disks:
alba_disk = AlbaDisk()
alba_disk.aliases = ['/dev/alba_disk_{0}'.format(ad_id)]
alba_disk.alba_node = alba_nodes[an_id]
alba_disk.save()
alba_disks[ad_id] = alba_disk
for ao_id, ad_id, ab_id in structure.get('alba_osds', ()):
if ao_id not in alba_osds:
osd = AlbaOSD()
osd.osd_id = 'alba_osd_{0}'.format(ao_id)
osd.osd_type = AlbaOSD.OSD_TYPES.ASD
osd.alba_backend = alba_backends[ab_id]
osd.alba_disk = alba_disks[ad_id]
osd.save()
alba_osds[ao_id] = osd
return {'backend_types': backend_types,
'service_types': service_types,
'alba_backends': alba_backends,
'alba_nodes': alba_nodes,
'alba_disks': alba_disks,
'alba_osds': alba_osds}
def test_asd_statistics(self):
"""
Validates whether the ASD statistics work as expected.
* Add keys that were not passed in
* Collapse certain keys
* Calculate correct per-second, average, total, min and max values
"""
from ovs.extensions.plugins.albacli import AlbaCLI
from ovs.extensions.plugins.asdmanager import ASDManagerClient
from ovs.dal.hybrids.albaasd import AlbaASD
from ovs.dal.hybrids.albanode import AlbaNode
from ovs.dal.hybrids.albabackend import AlbaBackend
from ovs.dal.hybrids.backend import Backend
from ovs.dal.hybrids.backendtype import BackendType
expected_0 = {'statistics': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'range': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'range_entries': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'multi_get': {'max': 10, 'n_ps': 0, 'min': 1, 'avg': 13, 'n': 5},
'apply': {'max': 5, 'n_ps': 0, 'min': 5, 'avg': 5, 'n': 1},
'timestamp': None}
expected_1 = {'statistics': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'range': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'range_entries': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'multi_get': {'max': 10, 'n_ps': 1, 'min': 1, 'avg': 12.5, 'n': 10},
'apply': {'max': 5, 'n_ps': 0, 'min': 5, 'avg': 5, 'n': 1},
'timestamp': None}
base_time = time.time()
backend_type = BackendType()
backend_type.code = 'alba'
backend_type.name = 'ALBA'
backend_type.save()
backend = Backend()
backend.name = 'foobar'
backend.backend_type = backend_type
backend.save()
alba_backend = AlbaBackend()
alba_backend.backend = backend
alba_backend.save()
alba_node = AlbaNode()
alba_node.ip = '127.0.0.1'
alba_node.port = 8500
alba_node.username = '******'
alba_node.password = '******'
alba_node.node_id = 'foobar'
alba_node.save()
asd = AlbaASD()
asd.asd_id = 'foo'
asd.alba_backend = alba_backend
asd.alba_node = alba_node
asd.save()
ASDManagerClient.results['get_disks'] = []
AlbaCLI.run_results['asd-multistatistics'] = {'foo': {'success': True,
'result': {'Apply': {'n': 1, 'avg': 5, 'min': 5, 'max': 5},
'MultiGet': {'n': 2, 'avg': 10, 'min': 5, 'max': 10},
'MultiGet2': {'n': 3, 'avg': 15, 'min': 1, 'max': 5}}}}
statistics = asd._statistics(AlbaASD._dynamics[4])
expected_0['timestamp'] = base_time
self.assertDictEqual(statistics, expected_0, 'The first statistics should be as expected: {0} vs {1}'.format(statistics, expected_0))
time.sleep(5)
ASDManagerClient.results['get_disks'] = []
AlbaCLI.run_results['asd-multistatistics'] = {'foo': {'success': True,
'result': {'Apply': {'n': 1, 'avg': 5, 'min': 5, 'max': 5},
'MultiGet': {'n': 5, 'avg': 10, 'min': 5, 'max': 10},
'MultiGet2': {'n': 5, 'avg': 15, 'min': 1, 'max': 5}}}}
statistics = asd._statistics(AlbaASD._dynamics[4])
expected_1['timestamp'] = base_time + 5
self.assertDictEqual(statistics, expected_1, 'The second statistics should be as expected: {0} vs {1}'.format(statistics, expected_1))
def test_asd_statistics(self):
"""
Validates whether the ASD statistics work as expected.
* Add keys that were not passed in
* Collapse certain keys
* Calculate correct per-second, average, total, min and max values
"""
expected_0 = {'statistics': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'range': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'range_entries': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'multi_get': {'max': 10, 'n_ps': 0, 'min': 1, 'avg': 13, 'n': 5},
'apply': {'max': 5, 'n_ps': 0, 'min': 5, 'avg': 5, 'n': 1},
'timestamp': None}
expected_1 = {'statistics': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'range': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'range_entries': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
'multi_get': {'max': 10, 'n_ps': 1, 'min': 1, 'avg': 12.5, 'n': 10},
'apply': {'max': 5, 'n_ps': 0, 'min': 5, 'avg': 5, 'n': 1},
'timestamp': None}
base_time = time.time()
backend_type = BackendType()
backend_type.code = 'alba'
backend_type.name = 'ALBA'
backend_type.save()
backend = Backend()
backend.name = 'foobar'
backend.backend_type = backend_type
backend.save()
alba_backend = AlbaBackend()
alba_backend.backend = backend
alba_backend.save()
alba_node = AlbaNode()
alba_node.ip = '127.0.0.1'
alba_node.port = 8500
alba_node.username = '******'
alba_node.password = '******'
alba_node.node_id = 'foobar'
alba_node.save()
alba_disk = AlbaDisk()
alba_disk.name = 'foo'
alba_disk.alba_node = alba_node
alba_disk.save()
asd = AlbaASD()
asd.asd_id = 'foo'
asd.alba_backend = alba_backend
asd.alba_disk = alba_disk
asd.save()
service_type = ServiceType()
service_type.name = 'AlbaManager'
service_type.save()
service = Service()
service.name = 'foobar'
service.type = service_type
service.ports = []
service.save()
abm_service = ABMService()
abm_service.service = service
abm_service.alba_backend = alba_backend
abm_service.save()
asdmanager_client = ASDManagerClient('')
asdmanager_client._results['get_disks'] = []
AlbaCLI._run_results['asd-multistatistics'] = {'foo': {'success': True,
'result': {'Apply': {'n': 1, 'avg': 5, 'min': 5, 'max': 5},
'MultiGet': {'n': 2, 'avg': 10, 'min': 5, 'max': 10},
'MultiGet2': {'n': 3, 'avg': 15, 'min': 1, 'max': 5}}}}
statistics = asd._statistics(AlbaASD._dynamics[0])
expected_0['timestamp'] = base_time
self.assertDictEqual(statistics, expected_0, 'The first statistics should be as expected: {0} vs {1}'.format(statistics, expected_0))
time.sleep(5)
asdmanager_client._results['get_disks'] = []
AlbaCLI._run_results['asd-multistatistics'] = {'foo': {'success': True,
'result': {'Apply': {'n': 1, 'avg': 5, 'min': 5, 'max': 5},
'MultiGet': {'n': 5, 'avg': 10, 'min': 5, 'max': 10},
'MultiGet2': {'n': 5, 'avg': 15, 'min': 1, 'max': 5}}}}
statistics = asd._statistics(AlbaASD._dynamics[0])
expected_1['timestamp'] = base_time + 5
self.assertDictEqual(statistics, expected_1, 'The second statistics should be as expected: {0} vs {1}'.format(statistics, expected_1))
def build_dal_structure(structure, previous_structure=None):
"""
Builds a service structure
Example:
structure = AlbaDalHelper.build_service_structure({
'alba_backends': [1],
'alba_nodes': [1]
})
"""
if previous_structure is None:
previous_structure = {}
alba_osds = previous_structure.get('alba_osds', {})
alba_nodes = previous_structure.get('alba_nodes', {})
backend_types = previous_structure.get('backend_types', {})
service_types = previous_structure.get('service_types', {})
alba_backends = previous_structure.get('alba_backends', {})
alba_abm_clusters = previous_structure.get('alba_abm_clusters', {})
alba_nsm_clusters = previous_structure.get('alba_nsm_clusters', {})
if 1 not in backend_types:
backend_type = BackendType()
backend_type.code = 'alba'
backend_type.name = 'ALBA'
backend_type.save()
backend_types[1] = backend_type
if 'AlbaManager' not in service_types:
service_type = ServiceTypeList.get_by_name('AlbaManager')
if service_type is None:
service_type = ServiceType()
service_type.name = 'AlbaManager'
service_type.save()
service_types['AlbaManager'] = service_type
if 'NamespaceManager' not in service_types:
service_type = ServiceTypeList.get_by_name('NamespaceManager')
if service_type is None:
service_type = ServiceType()
service_type.name = 'NamespaceManager'
service_type.save()
service_types['NamespaceManager'] = service_type
for ab_id, scaling in structure.get('alba_backends', ()):
if ab_id not in alba_backends:
backend = Backend()
backend.name = 'backend_{0}'.format(ab_id)
backend.backend_type = backend_types[1]
backend.save()
alba_backend = AlbaBackend()
alba_backend.backend = backend
alba_backend.scaling = getattr(AlbaBackend.SCALINGS, scaling)
alba_backend.alba_id = str(ab_id)
alba_backend.save()
alba_backends[ab_id] = alba_backend
for ab_id in structure.get('alba_abm_clusters', ()):
if ab_id not in alba_abm_clusters:
if ab_id not in alba_backends:
raise ValueError('Non-existing ALBA Backend ID provided')
alba_backend = alba_backends[ab_id]
abm_cluster = ABMCluster()
abm_cluster.name = '{0}-abm'.format(alba_backend.name)
abm_cluster.alba_backend = alba_backend
abm_cluster.config_location = '/ovs/arakoon/{0}-abm/config'.format(
alba_backend.name)
abm_cluster.save()
abm_service = Service()
abm_service.name = 'arakoon-{0}-abm'.format(alba_backend.name)
abm_service.type = service_types['AlbaManager']
abm_service.ports = []
abm_service.storagerouter = None
abm_service.save()
abm_junction_service = ABMService()
abm_junction_service.service = abm_service
abm_junction_service.abm_cluster = abm_cluster
abm_junction_service.save()
alba_abm_clusters[ab_id] = abm_cluster
for ab_id, amount in structure.get('alba_nsm_clusters', ()):
if ab_id not in alba_nsm_clusters or amount != len(
alba_nsm_clusters[ab_id]):
if ab_id not in alba_backends:
raise ValueError('Non-existing ALBA Backend ID provided')
alba_backend = alba_backends[ab_id]
alba_nsm_clusters[ab_id] = []
nsm_clusters = dict(
(nsm_cluster.number, nsm_cluster)
for nsm_cluster in alba_backend.nsm_clusters)
for number in range(amount):
if number in nsm_clusters:
alba_nsm_clusters[ab_id].append(nsm_clusters[number])
continue
nsm_cluster = NSMCluster()
nsm_cluster.name = '{0}-nsm_{1}'.format(
alba_backend.name, number)
nsm_cluster.number = number
nsm_cluster.alba_backend = alba_backend
nsm_cluster.config_location = '/ovs/arakoon/{0}-nsm_{1}/config'.format(
alba_backend.name, number)
nsm_cluster.save()
nsm_service = Service()
nsm_service.name = 'arakoon-{0}-nsm_{1}'.format(
alba_backend.name, number)
nsm_service.type = service_types['NamespaceManager']
nsm_service.ports = []
nsm_service.storagerouter = None
nsm_service.save()
nsm_junction_service = NSMService()
nsm_junction_service.service = nsm_service
nsm_junction_service.nsm_cluster = nsm_cluster
nsm_junction_service.save()
alba_nsm_clusters[ab_id].append(nsm_cluster)
for an_id in structure.get('alba_nodes', []):
if an_id not in alba_nodes:
alba_node = AlbaNode()
alba_node.ip = '10.1.0.{0}'.format(an_id)
alba_node.port = 8500
alba_node.username = str(an_id)
alba_node.password = str(an_id)
alba_node.node_id = 'node_{0}'.format(an_id)
alba_node.save()
alba_nodes[an_id] = alba_node
if alba_node in ManagerClientMockup.test_results:
ManagerClientMockup.test_results[alba_node].update(
{'get_metadata': {
'_version': 3
}})
else:
ManagerClientMockup.test_results[alba_node] = {
'get_metadata': {
'_version': 3
}
}
for ao_id, ab_id, an_id, slot_id in structure.get('alba_osds', ()):
if ao_id not in alba_osds:
osd = AlbaOSD()
osd.osd_id = 'alba_osd_{0}'.format(ao_id)
osd.osd_type = AlbaOSD.OSD_TYPES.ASD
osd.alba_backend = alba_backends[ab_id]
osd.alba_node = alba_nodes[an_id]
osd.slot_id = 'alba_slot_{0}'.format(slot_id)
osd.ips = ['127.0.0.{0}'.format(ao_id)]
osd.port = 35000 + ao_id
osd.save()
alba_osds[ao_id] = osd
return {
'alba_osds': alba_osds,
'alba_nodes': alba_nodes,
'backend_types': backend_types,
'service_types': service_types,
'alba_backends': alba_backends,
'alba_abm_clusters': alba_abm_clusters,
'alba_nsm_clusters': alba_nsm_clusters
}