def ovs_3769_validation_test():
        """
        Create an albanode with an asd statistics part set to empty dictionary
        Assert code does not raise
        """
        an = AlbaNode()
        an.password = '******'
        an.node_id = 'ovs3769an'
        an.port = 1234
        an.ip = '127.0.0.1'
        an.username = '******'
        an.save()

        bet = GeneralBackend.get_backendtype_by_code('alba')

        be = Backend()
        be.backend_type = bet
        be.name = 'ovs3769be'
        be.save()

        abe = AlbaBackend()
        abe.backend = be
        abe.save()

        ad = AlbaDisk()
        ad.name = 'ovs3769ad'
        ad.alba_node = an
        ad.save()

        asd = AlbaASD()
        asd.alba_backend = abe
        asd.asd_id = 'ovs3769asd'
        asd.alba_disk = ad
        asd.save()

        try:
            abe.statistics
        except KeyError, ex:
            logger.error('Regression OVS-3769 - asd statistics raises a KeyError: {0}'.format(str(ex)))
    def build_service_structure(structure, previous_structure=None):
        """
        Builds a service structure
        Example:
            structure = Helper.build_service_structure({
                'alba_backends': [1],
                'alba_nodes': [1]
            })
        """
        if previous_structure is None:
            previous_structure = {}
        backend_types = previous_structure.get('backend_types', {})
        service_types = previous_structure.get('service_types', {})
        alba_backends = previous_structure.get('alba_backends', {})
        alba_nodes = previous_structure.get('alba_nodes', {})
        alba_disks = previous_structure.get('alba_disks', {})
        alba_osds = previous_structure.get('alba_osds', {})

        if 1 not in backend_types:
            backend_type = BackendType()
            backend_type.code = 'alba'
            backend_type.name = 'ALBA'
            backend_type.save()
            backend_types[1] = backend_type
        if 1 not in service_types:
            service_type = ServiceType()
            service_type.name = 'AlbaManager'
            service_type.save()
            service_types[1] = service_type
        for ab_id in structure.get('alba_backends', ()):
            if ab_id not in alba_backends:
                backend = Backend()
                backend.name = 'backend_{0}'.format(ab_id)
                backend.backend_type = backend_types[1]
                backend.save()
                alba_backend = AlbaBackend()
                alba_backend.backend = backend
                alba_backend.scaling = AlbaBackend.SCALINGS.LOCAL
                alba_backend.save()
                alba_backends[ab_id] = alba_backend
                service = Service()
                service.name = 'backend_{0}_abm'.format(ab_id)
                service.type = service_types[1]
                service.ports = []
                service.save()
                abm_service = ABMService()
                abm_service.service = service
                abm_service.alba_backend = alba_backend
                abm_service.save()
        for an_id in structure.get('alba_nodes', []):
            if an_id not in alba_nodes:
                alba_node = AlbaNode()
                alba_node.ip = '10.1.0.{0}'.format(an_id)
                alba_node.port = 8500
                alba_node.username = str(an_id)
                alba_node.password = str(an_id)
                alba_node.node_id = 'node_{0}'.format(an_id)
                alba_node.save()
                alba_nodes[an_id] = alba_node
        for ad_id, an_id in structure.get('alba_disks', ()):
            if ad_id not in alba_disks:
                alba_disk = AlbaDisk()
                alba_disk.aliases = ['/dev/alba_disk_{0}'.format(ad_id)]
                alba_disk.alba_node = alba_nodes[an_id]
                alba_disk.save()
                alba_disks[ad_id] = alba_disk
        for ao_id, ad_id, ab_id in structure.get('alba_osds', ()):
            if ao_id not in alba_osds:
                osd = AlbaOSD()
                osd.osd_id = 'alba_osd_{0}'.format(ao_id)
                osd.osd_type = AlbaOSD.OSD_TYPES.ASD
                osd.alba_backend = alba_backends[ab_id]
                osd.alba_disk = alba_disks[ad_id]
                osd.save()
                alba_osds[ao_id] = osd
        return {'backend_types': backend_types,
                'service_types': service_types,
                'alba_backends': alba_backends,
                'alba_nodes': alba_nodes,
                'alba_disks': alba_disks,
                'alba_osds': alba_osds}
 def test_asd_statistics(self):
     """
     Validates whether the ASD statistics work as expected.
     * Add keys that were not passed in
     * Collapse certain keys
     * Calculate correct per-second, average, total, min and max values
     """
     from ovs.extensions.plugins.albacli import AlbaCLI
     from ovs.extensions.plugins.asdmanager import ASDManagerClient
     from ovs.dal.hybrids.albaasd import AlbaASD
     from ovs.dal.hybrids.albanode import AlbaNode
     from ovs.dal.hybrids.albabackend import AlbaBackend
     from ovs.dal.hybrids.backend import Backend
     from ovs.dal.hybrids.backendtype import BackendType
     expected_0 = {'statistics': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                   'range': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                   'range_entries': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                   'multi_get': {'max': 10, 'n_ps': 0, 'min': 1, 'avg': 13, 'n': 5},
                   'apply': {'max': 5, 'n_ps': 0, 'min': 5, 'avg': 5, 'n': 1},
                   'timestamp': None}
     expected_1 = {'statistics': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                   'range': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                   'range_entries': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                   'multi_get': {'max': 10, 'n_ps': 1, 'min': 1, 'avg': 12.5, 'n': 10},
                   'apply': {'max': 5, 'n_ps': 0, 'min': 5, 'avg': 5, 'n': 1},
                   'timestamp': None}
     base_time = time.time()
     backend_type = BackendType()
     backend_type.code = 'alba'
     backend_type.name = 'ALBA'
     backend_type.save()
     backend = Backend()
     backend.name = 'foobar'
     backend.backend_type = backend_type
     backend.save()
     alba_backend = AlbaBackend()
     alba_backend.backend = backend
     alba_backend.save()
     alba_node = AlbaNode()
     alba_node.ip = '127.0.0.1'
     alba_node.port = 8500
     alba_node.username = '******'
     alba_node.password = '******'
     alba_node.node_id = 'foobar'
     alba_node.save()
     asd = AlbaASD()
     asd.asd_id = 'foo'
     asd.alba_backend = alba_backend
     asd.alba_node = alba_node
     asd.save()
     ASDManagerClient.results['get_disks'] = []
     AlbaCLI.run_results['asd-multistatistics'] = {'foo': {'success': True,
                                                           'result': {'Apply': {'n': 1, 'avg': 5, 'min': 5, 'max': 5},
                                                                      'MultiGet': {'n': 2, 'avg': 10, 'min': 5, 'max': 10},
                                                                      'MultiGet2': {'n': 3, 'avg': 15, 'min': 1, 'max': 5}}}}
     statistics = asd._statistics(AlbaASD._dynamics[4])
     expected_0['timestamp'] = base_time
     self.assertDictEqual(statistics, expected_0, 'The first statistics should be as expected: {0} vs {1}'.format(statistics, expected_0))
     time.sleep(5)
     ASDManagerClient.results['get_disks'] = []
     AlbaCLI.run_results['asd-multistatistics'] = {'foo': {'success': True,
                                                           'result': {'Apply': {'n': 1, 'avg': 5, 'min': 5, 'max': 5},
                                                                      'MultiGet': {'n': 5, 'avg': 10, 'min': 5, 'max': 10},
                                                                      'MultiGet2': {'n': 5, 'avg': 15, 'min': 1, 'max': 5}}}}
     statistics = asd._statistics(AlbaASD._dynamics[4])
     expected_1['timestamp'] = base_time + 5
     self.assertDictEqual(statistics, expected_1, 'The second statistics should be as expected: {0} vs {1}'.format(statistics, expected_1))
    def test_asd_statistics(self):
        """
        Validates whether the ASD statistics work as expected.
        * Add keys that were not passed in
        * Collapse certain keys
        * Calculate correct per-second, average, total, min and max values
        """
        expected_0 = {'statistics': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                      'range': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                      'range_entries': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                      'multi_get': {'max': 10, 'n_ps': 0, 'min': 1, 'avg': 13, 'n': 5},
                      'apply': {'max': 5, 'n_ps': 0, 'min': 5, 'avg': 5, 'n': 1},
                      'timestamp': None}
        expected_1 = {'statistics': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                      'range': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                      'range_entries': {'max': 0, 'n_ps': 0, 'min': 0, 'avg': 0, 'n': 0},
                      'multi_get': {'max': 10, 'n_ps': 1, 'min': 1, 'avg': 12.5, 'n': 10},
                      'apply': {'max': 5, 'n_ps': 0, 'min': 5, 'avg': 5, 'n': 1},
                      'timestamp': None}
        base_time = time.time()
        backend_type = BackendType()
        backend_type.code = 'alba'
        backend_type.name = 'ALBA'
        backend_type.save()
        backend = Backend()
        backend.name = 'foobar'
        backend.backend_type = backend_type
        backend.save()
        alba_backend = AlbaBackend()
        alba_backend.backend = backend
        alba_backend.save()
        alba_node = AlbaNode()
        alba_node.ip = '127.0.0.1'
        alba_node.port = 8500
        alba_node.username = '******'
        alba_node.password = '******'
        alba_node.node_id = 'foobar'
        alba_node.save()
        alba_disk = AlbaDisk()
        alba_disk.name = 'foo'
        alba_disk.alba_node = alba_node
        alba_disk.save()
        asd = AlbaASD()
        asd.asd_id = 'foo'
        asd.alba_backend = alba_backend
        asd.alba_disk = alba_disk
        asd.save()
        service_type = ServiceType()
        service_type.name = 'AlbaManager'
        service_type.save()
        service = Service()
        service.name = 'foobar'
        service.type = service_type
        service.ports = []
        service.save()
        abm_service = ABMService()
        abm_service.service = service
        abm_service.alba_backend = alba_backend
        abm_service.save()

        asdmanager_client = ASDManagerClient('')
        asdmanager_client._results['get_disks'] = []
        AlbaCLI._run_results['asd-multistatistics'] = {'foo': {'success': True,
                                                               'result': {'Apply': {'n': 1, 'avg': 5, 'min': 5, 'max': 5},
                                                                          'MultiGet': {'n': 2, 'avg': 10, 'min': 5, 'max': 10},
                                                                          'MultiGet2': {'n': 3, 'avg': 15, 'min': 1, 'max': 5}}}}
        statistics = asd._statistics(AlbaASD._dynamics[0])
        expected_0['timestamp'] = base_time
        self.assertDictEqual(statistics, expected_0, 'The first statistics should be as expected: {0} vs {1}'.format(statistics, expected_0))
        time.sleep(5)
        asdmanager_client._results['get_disks'] = []
        AlbaCLI._run_results['asd-multistatistics'] = {'foo': {'success': True,
                                                               'result': {'Apply': {'n': 1, 'avg': 5, 'min': 5, 'max': 5},
                                                                          'MultiGet': {'n': 5, 'avg': 10, 'min': 5, 'max': 10},
                                                                          'MultiGet2': {'n': 5, 'avg': 15, 'min': 1, 'max': 5}}}}
        statistics = asd._statistics(AlbaASD._dynamics[0])
        expected_1['timestamp'] = base_time + 5
        self.assertDictEqual(statistics, expected_1, 'The second statistics should be as expected: {0} vs {1}'.format(statistics, expected_1))
    def build_dal_structure(structure, previous_structure=None):
        """
        Builds a service structure
        Example:
            structure = AlbaDalHelper.build_service_structure({
                'alba_backends': [1],
                'alba_nodes': [1]
            })
        """
        if previous_structure is None:
            previous_structure = {}
        alba_osds = previous_structure.get('alba_osds', {})
        alba_nodes = previous_structure.get('alba_nodes', {})
        backend_types = previous_structure.get('backend_types', {})
        service_types = previous_structure.get('service_types', {})
        alba_backends = previous_structure.get('alba_backends', {})
        alba_abm_clusters = previous_structure.get('alba_abm_clusters', {})
        alba_nsm_clusters = previous_structure.get('alba_nsm_clusters', {})

        if 1 not in backend_types:
            backend_type = BackendType()
            backend_type.code = 'alba'
            backend_type.name = 'ALBA'
            backend_type.save()
            backend_types[1] = backend_type

        if 'AlbaManager' not in service_types:
            service_type = ServiceTypeList.get_by_name('AlbaManager')
            if service_type is None:
                service_type = ServiceType()
                service_type.name = 'AlbaManager'
                service_type.save()
            service_types['AlbaManager'] = service_type
        if 'NamespaceManager' not in service_types:
            service_type = ServiceTypeList.get_by_name('NamespaceManager')
            if service_type is None:
                service_type = ServiceType()
                service_type.name = 'NamespaceManager'
                service_type.save()
            service_types['NamespaceManager'] = service_type
        for ab_id, scaling in structure.get('alba_backends', ()):
            if ab_id not in alba_backends:
                backend = Backend()
                backend.name = 'backend_{0}'.format(ab_id)
                backend.backend_type = backend_types[1]
                backend.save()
                alba_backend = AlbaBackend()
                alba_backend.backend = backend
                alba_backend.scaling = getattr(AlbaBackend.SCALINGS, scaling)
                alba_backend.alba_id = str(ab_id)
                alba_backend.save()
                alba_backends[ab_id] = alba_backend
        for ab_id in structure.get('alba_abm_clusters', ()):
            if ab_id not in alba_abm_clusters:
                if ab_id not in alba_backends:
                    raise ValueError('Non-existing ALBA Backend ID provided')
                alba_backend = alba_backends[ab_id]
                abm_cluster = ABMCluster()
                abm_cluster.name = '{0}-abm'.format(alba_backend.name)
                abm_cluster.alba_backend = alba_backend
                abm_cluster.config_location = '/ovs/arakoon/{0}-abm/config'.format(
                    alba_backend.name)
                abm_cluster.save()
                abm_service = Service()
                abm_service.name = 'arakoon-{0}-abm'.format(alba_backend.name)
                abm_service.type = service_types['AlbaManager']
                abm_service.ports = []
                abm_service.storagerouter = None
                abm_service.save()
                abm_junction_service = ABMService()
                abm_junction_service.service = abm_service
                abm_junction_service.abm_cluster = abm_cluster
                abm_junction_service.save()
                alba_abm_clusters[ab_id] = abm_cluster
        for ab_id, amount in structure.get('alba_nsm_clusters', ()):
            if ab_id not in alba_nsm_clusters or amount != len(
                    alba_nsm_clusters[ab_id]):
                if ab_id not in alba_backends:
                    raise ValueError('Non-existing ALBA Backend ID provided')
                alba_backend = alba_backends[ab_id]
                alba_nsm_clusters[ab_id] = []
                nsm_clusters = dict(
                    (nsm_cluster.number, nsm_cluster)
                    for nsm_cluster in alba_backend.nsm_clusters)
                for number in range(amount):
                    if number in nsm_clusters:
                        alba_nsm_clusters[ab_id].append(nsm_clusters[number])
                        continue
                    nsm_cluster = NSMCluster()
                    nsm_cluster.name = '{0}-nsm_{1}'.format(
                        alba_backend.name, number)
                    nsm_cluster.number = number
                    nsm_cluster.alba_backend = alba_backend
                    nsm_cluster.config_location = '/ovs/arakoon/{0}-nsm_{1}/config'.format(
                        alba_backend.name, number)
                    nsm_cluster.save()
                    nsm_service = Service()
                    nsm_service.name = 'arakoon-{0}-nsm_{1}'.format(
                        alba_backend.name, number)
                    nsm_service.type = service_types['NamespaceManager']
                    nsm_service.ports = []
                    nsm_service.storagerouter = None
                    nsm_service.save()
                    nsm_junction_service = NSMService()
                    nsm_junction_service.service = nsm_service
                    nsm_junction_service.nsm_cluster = nsm_cluster
                    nsm_junction_service.save()
                    alba_nsm_clusters[ab_id].append(nsm_cluster)
        for an_id in structure.get('alba_nodes', []):
            if an_id not in alba_nodes:
                alba_node = AlbaNode()
                alba_node.ip = '10.1.0.{0}'.format(an_id)
                alba_node.port = 8500
                alba_node.username = str(an_id)
                alba_node.password = str(an_id)
                alba_node.node_id = 'node_{0}'.format(an_id)
                alba_node.save()
                alba_nodes[an_id] = alba_node
                if alba_node in ManagerClientMockup.test_results:
                    ManagerClientMockup.test_results[alba_node].update(
                        {'get_metadata': {
                            '_version': 3
                        }})
                else:
                    ManagerClientMockup.test_results[alba_node] = {
                        'get_metadata': {
                            '_version': 3
                        }
                    }
        for ao_id, ab_id, an_id, slot_id in structure.get('alba_osds', ()):
            if ao_id not in alba_osds:
                osd = AlbaOSD()
                osd.osd_id = 'alba_osd_{0}'.format(ao_id)
                osd.osd_type = AlbaOSD.OSD_TYPES.ASD
                osd.alba_backend = alba_backends[ab_id]
                osd.alba_node = alba_nodes[an_id]
                osd.slot_id = 'alba_slot_{0}'.format(slot_id)
                osd.ips = ['127.0.0.{0}'.format(ao_id)]
                osd.port = 35000 + ao_id
                osd.save()
                alba_osds[ao_id] = osd
        return {
            'alba_osds': alba_osds,
            'alba_nodes': alba_nodes,
            'backend_types': backend_types,
            'service_types': service_types,
            'alba_backends': alba_backends,
            'alba_abm_clusters': alba_abm_clusters,
            'alba_nsm_clusters': alba_nsm_clusters
        }