Exemple #1
0
 def create_multiple_from_template(self, vmachine, pmachineguids, amount,
                                   start, name, description):
     """
     Creates a certain amount of vMachines based on a vTemplate
     """
     if len(pmachineguids) == 0:
         raise NotAcceptable
     try:
         for pmachienguid in pmachineguids:
             _ = PMachine(pmachienguid)
     except ObjectNotFoundException:
         raise Http404('pMachine could not be found')
     if vmachine.is_vtemplate is False:
         raise NotAcceptable('vMachine is not a vTemplate')
     if not isinstance(amount, int) or not isinstance(start, int):
         raise NotAcceptable('Fields amount and start should be numeric')
     amount = max(1, amount)
     start = max(0, start)
     return VMachineController.create_multiple_from_template.delay(
         machineguid=vmachine.guid,
         pmachineguids=pmachineguids,
         amount=amount,
         start=start,
         name=str(name),
         description=str(description))
Exemple #2
0
    def setUpClass(cls):
        """
        Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
        This makes sure the unittests can be executed without those libraries installed
        """
        # Load dummy stores
        PersistentFactory.store = DummyPersistentStore()
        VolatileFactory.store = DummyVolatileStore()
        # Replace mocked classes
        sys.modules[
            'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule
        # Import required modules/classes after mocking is done
        from ovs.dal.hybrids.vdisk import VDisk
        from ovs.dal.hybrids.service import Service
        from ovs.dal.hybrids.vpool import VPool
        from ovs.dal.hybrids.storagerouter import StorageRouter
        from ovs.dal.hybrids.pmachine import PMachine
        from ovs.dal.hybrids.servicetype import ServiceType
        from ovs.dal.hybrids.storagedriver import StorageDriver
        from ovs.dal.hybrids.backendtype import BackendType
        from ovs.dal.hybrids.j_mdsservice import MDSService
        from ovs.dal.hybrids.j_mdsservicevdisk import MDSServiceVDisk
        from ovs.extensions.generic.volatilemutex import VolatileMutex
        from ovs.lib.mdsservice import MDSServiceController
        # Globalize mocked classes
        global VDisk
        global VPool
        global Service
        global StorageRouter
        global StorageDriver
        global BackendType
        global PMachine
        global MDSService
        global ServiceType
        global MDSServiceVDisk
        global VolatileMutex
        global MDSServiceController
        _ = VDisk(), VPool(), Service(), MDSService(), MDSServiceVDisk(), ServiceType(), \
            StorageRouter(), StorageDriver(), BackendType(), PMachine(), \
            VolatileMutex('dummy'), MDSServiceController

        # Configuration
        def _get(key):
            c = PersistentFactory.get_client()
            if c.exists(key):
                return c.get(key)
            return None

        Configuration.get = staticmethod(_get)

        # Cleaning storage
        VolatileFactory.store.clean()
        PersistentFactory.store.clean()
Exemple #3
0
 def unconfigure_vpool_for_host(pmachine_guid, vpool_guid):
     pmachine = PMachine(pmachine_guid)
     mgmt_center_client = None
     try:
         mgmt_center_client = Factory.get_mgmtcenter(pmachine=pmachine)
     except Exception as ex:
         logger.error('Cannot get management center client: {0}'.format(ex))
     if mgmt_center_client is not None:
         logger.info('Unconfiguring vPool {0} on host {1}'.format(
             vpool_guid, pmachine.name))
         mgmt_center_client.unconfigure_vpool_for_host(
             vpool_guid, False, pmachine.ip)
Exemple #4
0
    def setUpClass(cls):
        """
        Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
        This makes sure the unittests can be executed without those libraries installed
        """
        # Load dummy stores
        PersistentFactory.store = DummyPersistentStore()
        VolatileFactory.store = DummyVolatileStore()
        # Replace mocked classes
        sys.modules[
            'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule
        sys.modules['ovs.extensions.hypervisor.hypervisors.kvm'] = KVMModule

        # Import required modules/classes after mocking is done
        from ovs.dal.hybrids.backendtype import BackendType
        from ovs.dal.hybrids.vdisk import VDisk
        from ovs.dal.hybrids.j_mdsservice import MDSService
        from ovs.dal.hybrids.j_mdsservicevdisk import MDSServiceVDisk
        from ovs.lib.vdisk import VDiskController
        from ovs.dal.hybrids.pmachine import PMachine
        from ovs.dal.hybrids.vmachine import VMachine
        from ovs.dal.hybrids.vpool import VPool
        from ovs.dal.hybrids.storagedriver import StorageDriver
        from ovs.dal.hybrids.storagerouter import StorageRouter
        from ovs.dal.hybrids.failuredomain import FailureDomain
        from ovs.dal.hybrids.service import Service
        from ovs.dal.hybrids.servicetype import ServiceType
        from ovs.dal.lists.vdisklist import VDiskList
        from ovs.lib.mdsservice import MDSServiceController

        # Globalize mocked classes
        global VDisk
        global VDiskController
        global PMachine
        global VMachine
        global BackendType
        global VPool
        global StorageDriver
        global StorageRouter
        global FailureDomain
        global MDSService
        global MDSServiceVDisk
        global Service
        global ServiceType
        global VDiskList
        global MDSServiceController
        _ = VDisk(), PMachine(), VMachine(), VDiskController, VPool(), BackendType(), StorageDriver(), StorageRouter(), \
            FailureDomain(), MDSService(), MDSServiceVDisk(), Service(), ServiceType(), VDiskList, MDSServiceController

        # Cleaning storage
        VolatileFactory.store.clean()
        PersistentFactory.store.clean()
Exemple #5
0
    def is_host_configured(pmachine_guid):
        pmachine = PMachine(pmachine_guid)
        mgmt_center_client = None
        try:
            mgmt_center_client = Factory.get_mgmtcenter(pmachine=pmachine)
        except Exception as ex:
            if pmachine.mgmtcenter_guid:
                MgmtCenterController._logger.error(
                    'Cannot get management center client: {0}'.format(ex))

        if mgmt_center_client is not None:
            return mgmt_center_client.is_host_configured(pmachine.ip)
        return False
Exemple #6
0
 def create_from_template(self, vmachine, pmachineguid, name, description):
     """
     Creates a vMachine based on a vTemplate
     """
     try:
         pmachine = PMachine(pmachineguid)
     except ObjectNotFoundException:
         raise Http404('pMachine could not be found')
     if vmachine.is_vtemplate is False:
         raise NotAcceptable('vMachine is not a vTemplate')
     return VMachineController.create_from_template.delay(machineguid=vmachine.guid,
                                                          pmachineguid=pmachine.guid,
                                                          name=str(name),
                                                          description=str(description))
Exemple #7
0
    def clone(diskguid,
              snapshotid,
              devicename,
              pmachineguid,
              machinename,
              machineguid=None,
              **kwargs):
        """
        Clone a disk

        @param location: location where virtual device should be created (eg: myVM)
        @param devicename: device file name for the disk (eg: mydisk-flat.vmdk)
        @param parentdiskguid: guid of the disk
        @param snapshotid: guid of the snapshot
        @param machineguid: guid of the machine to assign disk to
        """
        _ = kwargs
        pmachine = PMachine(pmachineguid)
        hypervisor = Factory.get(pmachine)
        description = '{} {}'.format(machinename, devicename)
        properties_to_clone = [
            'description', 'size', 'type', 'retentionpolicyguid',
            'snapshotpolicyguid', 'autobackup'
        ]

        new_disk = VDisk()
        disk = VDisk(diskguid)
        _log = 'Clone snapshot {} of disk {} to location {}'
        _location = hypervisor.get_backing_disk_path(machinename, devicename)
        _id = '{}'.format(disk.volume_id)
        _snap = '{}'.format(snapshotid)
        logger.info(_log.format(_snap, disk.name, _location))
        volume_id = disk.storagedriver_client.create_clone(
            _location, _id, _snap)
        new_disk.copy(disk, include=properties_to_clone)
        new_disk.parent_vdisk = disk
        new_disk.name = '{}-clone'.format(disk.name)
        new_disk.description = description
        new_disk.volume_id = volume_id
        new_disk.devicename = hypervisor.clean_backing_disk_filename(_location)
        new_disk.parentsnapshot = snapshotid
        new_disk.vmachine = VMachine(
            machineguid) if machineguid else disk.vmachine
        new_disk.vpool = disk.vpool
        new_disk.save()
        return {
            'diskguid': new_disk.guid,
            'name': new_disk.name,
            'backingdevice': _location
        }
Exemple #8
0
 def configure_host(pmachine_guid, mgmtcenter_guid, update_link):
     pmachine = PMachine(pmachine_guid)
     mgmt_center = MgmtCenter(mgmtcenter_guid)
     mgmt_center_client = None
     try:
         mgmt_center_client = Factory.get_mgmtcenter(
             mgmt_center=mgmt_center)
     except Exception as ex:
         logger.error('Cannot get management center client: {0}'.format(ex))
     if mgmt_center_client is not None:
         logger.info('Configuring host {0} on management center {1}'.format(
             pmachine.name, mgmt_center.name))
         mgmt_center_client.configure_host(pmachine.ip)
         if update_link is True:
             pmachine.mgmtcenter = mgmt_center
             pmachine.save()
Exemple #9
0
    def setUpClass(cls):
        """
        Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
        This makes sure the unittests can be executed without those libraries installed
        """
        # Load dummy stores
        PersistentFactory.store = DummyPersistentStore()
        VolatileFactory.store = DummyVolatileStore()
        # Replace mocked classes
        sys.modules[
            'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule
        # Import required modules/classes after mocking is done
        from ovs.dal.hybrids.backendtype import BackendType
        from ovs.dal.hybrids.disk import Disk
        from ovs.dal.hybrids.diskpartition import DiskPartition
        from ovs.dal.hybrids.failuredomain import FailureDomain
        from ovs.dal.hybrids.pmachine import PMachine
        from ovs.dal.hybrids.storagerouter import StorageRouter
        from ovs.dal.hybrids.vdisk import VDisk
        from ovs.dal.hybrids.vmachine import VMachine
        from ovs.dal.hybrids.vpool import VPool
        from ovs.extensions.generic.volatilemutex import VolatileMutex
        from ovs.lib.vmachine import VMachineController
        from ovs.lib.vdisk import VDiskController
        from ovs.lib.scheduledtask import ScheduledTaskController
        # Globalize mocked classes
        global Disk
        global VDisk
        global VMachine
        global PMachine
        global VPool
        global BackendType
        global DiskPartition
        global FailureDomain
        global StorageRouter
        global VolatileMutex
        global VMachineController
        global VDiskController
        global ScheduledTaskController
        _ = VDisk(), VolatileMutex('dummy'), VMachine(), PMachine(), VPool(), BackendType(), FailureDomain(), \
            VMachineController, VDiskController, ScheduledTaskController, StorageRouter(), Disk(), DiskPartition()

        # Cleaning storage
        VolatileFactory.store.clean()
        PersistentFactory.store.clean()
Exemple #10
0
    def test_happypath(self):
        """
        Validates the happy path; Hourly snapshots are taken with a few manual consistent
        every now an then. The delete policy is executed every day
        """
        # Setup
        # There are 2 machines; one with two disks, one with one disk and a stand-alone additional disk
        failure_domain = FailureDomain()
        failure_domain.name = 'Test'
        failure_domain.save()
        backend_type = BackendType()
        backend_type.name = 'BackendType'
        backend_type.code = 'BT'
        backend_type.save()
        vpool = VPool()
        vpool.name = 'vpool'
        vpool.status = 'RUNNING'
        vpool.backend_type = backend_type
        vpool.save()
        pmachine = PMachine()
        pmachine.name = 'PMachine'
        pmachine.username = '******'
        pmachine.ip = '127.0.0.1'
        pmachine.hvtype = 'VMWARE'
        pmachine.save()
        storage_router = StorageRouter()
        storage_router.name = 'storage_router'
        storage_router.ip = '127.0.0.1'
        storage_router.pmachine = pmachine
        storage_router.machine_id = System.get_my_machine_id()
        storage_router.rdma_capable = False
        storage_router.primary_failure_domain = failure_domain
        storage_router.save()
        disk = Disk()
        disk.name = 'physical_disk_1'
        disk.path = '/dev/non-existent'
        disk.size = 500 * 1024**3
        disk.state = 'OK'
        disk.is_ssd = True
        disk.storagerouter = storage_router
        disk.save()
        disk_partition = DiskPartition()
        disk_partition.id = 'disk_partition_id'
        disk_partition.disk = disk
        disk_partition.path = '/dev/disk/non-existent'
        disk_partition.size = 400 * 1024**3
        disk_partition.state = 'OK'
        disk_partition.offset = 1024
        disk_partition.roles = [DiskPartition.ROLES.SCRUB]
        disk_partition.mountpoint = '/var/tmp'
        disk_partition.save()
        vmachine_1 = VMachine()
        vmachine_1.name = 'vmachine_1'
        vmachine_1.devicename = 'dummy'
        vmachine_1.pmachine = pmachine
        vmachine_1.save()
        vdisk_1_1 = VDisk()
        vdisk_1_1.name = 'vdisk_1_1'
        vdisk_1_1.volume_id = 'vdisk_1_1'
        vdisk_1_1.vmachine = vmachine_1
        vdisk_1_1.vpool = vpool
        vdisk_1_1.devicename = 'dummy'
        vdisk_1_1.size = 0
        vdisk_1_1.save()
        vdisk_1_1.reload_client()
        vdisk_1_2 = VDisk()
        vdisk_1_2.name = 'vdisk_1_2'
        vdisk_1_2.volume_id = 'vdisk_1_2'
        vdisk_1_2.vmachine = vmachine_1
        vdisk_1_2.vpool = vpool
        vdisk_1_2.devicename = 'dummy'
        vdisk_1_2.size = 0
        vdisk_1_2.save()
        vdisk_1_2.reload_client()
        vmachine_2 = VMachine()
        vmachine_2.name = 'vmachine_2'
        vmachine_2.devicename = 'dummy'
        vmachine_2.pmachine = pmachine
        vmachine_2.save()
        vdisk_2_1 = VDisk()
        vdisk_2_1.name = 'vdisk_2_1'
        vdisk_2_1.volume_id = 'vdisk_2_1'
        vdisk_2_1.vmachine = vmachine_2
        vdisk_2_1.vpool = vpool
        vdisk_2_1.devicename = 'dummy'
        vdisk_2_1.size = 0
        vdisk_2_1.save()
        vdisk_2_1.reload_client()
        vdisk_3 = VDisk()
        vdisk_3.name = 'vdisk_3'
        vdisk_3.volume_id = 'vdisk_3'
        vdisk_3.vpool = vpool
        vdisk_3.devicename = 'dummy'
        vdisk_3.size = 0
        vdisk_3.save()
        vdisk_3.reload_client()

        for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
            [
                dynamic for dynamic in disk._dynamics
                if dynamic.name == 'snapshots'
            ][0].timeout = 0

        # Run the testing scenario
        travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
        if travis is True:
            print 'Running in Travis, reducing output.'
        debug = not travis
        amount_of_days = 50
        base = datetime.datetime.now().date()
        day = datetime.timedelta(1)
        minute = 60
        hour = minute * 60

        for d in xrange(0, amount_of_days):
            base_timestamp = self._make_timestamp(base, day * d)
            print ''
            print 'Day cycle: {0}: {1}'.format(
                d,
                datetime.datetime.fromtimestamp(base_timestamp).strftime(
                    '%Y-%m-%d'))

            # At the start of the day, delete snapshot policy runs at 00:30
            print '- Deleting snapshots'
            ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
                                                     (minute * 30))

            # Validate snapshots
            print '- Validating snapshots'
            for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
                self._validate(vdisk, d, base, amount_of_days, debug)

            # During the day, snapshots are taken
            # - Create non consistent snapshot every hour, between 2:00 and 22:00
            # - Create consistent snapshot at 6:30, 12:30, 18:30
            print '- Creating snapshots'
            for h in xrange(2, 23):
                timestamp = base_timestamp + (hour * h)
                for vm in [vmachine_1, vmachine_2]:
                    VMachineController.snapshot(machineguid=vm.guid,
                                                label='ss_i_{0}:00'.format(
                                                    str(h)),
                                                is_consistent=False,
                                                timestamp=timestamp)
                    if h in [6, 12, 18]:
                        ts = (timestamp + (minute * 30))
                        VMachineController.snapshot(machineguid=vm.guid,
                                                    label='ss_c_{0}:30'.format(
                                                        str(h)),
                                                    is_consistent=True,
                                                    timestamp=ts)

                VDiskController.create_snapshot(diskguid=vdisk_3.guid,
                                                metadata={
                                                    'label':
                                                    'ss_i_{0}:00'.format(
                                                        str(h)),
                                                    'is_consistent':
                                                    False,
                                                    'timestamp':
                                                    str(timestamp),
                                                    'machineguid':
                                                    None
                                                })
                if h in [6, 12, 18]:
                    ts = (timestamp + (minute * 30))
                    VDiskController.create_snapshot(diskguid=vdisk_3.guid,
                                                    metadata={
                                                        'label':
                                                        'ss_c_{0}:30'.format(
                                                            str(h)),
                                                        'is_consistent':
                                                        True,
                                                        'timestamp':
                                                        str(ts),
                                                        'machineguid':
                                                        None
                                                    })
Exemple #11
0
    def create_from_template(name,
                             machineguid,
                             pmachineguid,
                             description=None):
        """
        Create a new vmachine using an existing vmachine template

        @param machineguid: guid of the template vmachine
        @param name: name of new vmachine
        @param pmachineguid: guid of hypervisor to create new vmachine on
        @return: guid of the newly created vmachine | False on any failure
        """

        template_vm = VMachine(machineguid)
        if not template_vm.is_vtemplate:
            return False

        target_pm = PMachine(pmachineguid)
        target_hypervisor = Factory.get(target_pm)

        storagerouters = [
            sr for sr in StorageRouterList.get_storagerouters()
            if sr.pmachine_guid == target_pm.guid
        ]
        if len(storagerouters) == 1:
            target_storagerouter = storagerouters[0]
        else:
            raise ValueError(
                'Pmachine {} has no StorageRouter assigned to it'.format(
                    pmachineguid))
        routing_key = "sr.{0}".format(target_storagerouter.machine_id)

        vpool = None
        vpool_guids = set()
        if template_vm.vpool is not None:
            vpool = template_vm.vpool
            vpool_guids.add(vpool.guid)
        for disk in template_vm.vdisks:
            vpool = disk.vpool
            vpool_guids.add(vpool.guid)
        if len(vpool_guids) != 1:
            raise RuntimeError(
                'Only 1 vpool supported on template disk(s) - {0} found!'.
                format(len(vpool_guids)))

        if not template_vm.pmachine.hvtype == target_pm.hvtype:
            raise RuntimeError('Source and target hypervisor not identical')

        # Currently, only one vPool is supported, so we can just use whatever the `vpool` variable above
        # was set to as 'the' vPool for the code below. This obviously will have to change once vPool mixes
        # are supported.

        target_storagedriver = None
        source_storagedriver = None
        for vpool_storagedriver in vpool.storagedrivers:
            if vpool_storagedriver.storagerouter.pmachine_guid == target_pm.guid:
                target_storagedriver = vpool_storagedriver
            if vpool_storagedriver.storagerouter.pmachine_guid == template_vm.pmachine_guid:
                source_storagedriver = vpool_storagedriver
        if target_storagedriver is None:
            raise RuntimeError('Volume not served on target hypervisor')

        source_hv = Factory.get(template_vm.pmachine)
        target_hv = Factory.get(target_pm)
        if not source_hv.is_datastore_available(
                source_storagedriver.storage_ip,
                source_storagedriver.mountpoint):
            raise RuntimeError('Datastore unavailable on source hypervisor')
        if not target_hv.is_datastore_available(
                target_storagedriver.storage_ip,
                target_storagedriver.mountpoint):
            raise RuntimeError('Datastore unavailable on target hypervisor')

        source_vm = source_hv.get_vm_object(template_vm.hypervisor_id)
        if not source_vm:
            raise RuntimeError('VM with key reference {0} not found'.format(
                template_vm.hypervisor_id))

        name_duplicates = VMachineList.get_vmachine_by_name(name)
        if name_duplicates is not None and len(name_duplicates) > 0:
            raise RuntimeError(
                'A vMachine with name {0} already exists'.format(name))

        vm_path = target_hypervisor.get_vmachine_path(
            name, target_storagedriver.storagerouter.machine_id)

        new_vm = VMachine()
        new_vm.copy(template_vm)
        new_vm.hypervisor_id = ''
        new_vm.vpool = template_vm.vpool
        new_vm.pmachine = target_pm
        new_vm.name = name
        new_vm.description = description
        new_vm.is_vtemplate = False
        new_vm.devicename = target_hypervisor.clean_vmachine_filename(vm_path)
        new_vm.status = 'CREATED'
        new_vm.save()

        storagedrivers = [
            storagedriver for storagedriver in vpool.storagedrivers if
            storagedriver.storagerouter.pmachine_guid == new_vm.pmachine_guid
        ]
        if len(storagedrivers) == 0:
            raise RuntimeError(
                'Cannot find Storage Driver serving {0} on {1}'.format(
                    vpool.name, new_vm.pmachine.name))
        storagedriverguid = storagedrivers[0].guid

        disks = []
        disks_by_order = sorted(template_vm.vdisks, key=lambda x: x.order)
        try:
            for disk in disks_by_order:
                prefix = '{0}-clone'.format(disk.name)
                result = VDiskController.create_from_template(
                    diskguid=disk.guid,
                    devicename=prefix,
                    pmachineguid=target_pm.guid,
                    machinename=new_vm.name,
                    machineguid=new_vm.guid,
                    storagedriver_guid=storagedriverguid)
                disks.append(result)
                logger.debug('Disk appended: {0}'.format(result))
        except Exception as exception:
            logger.error('Creation of disk {0} failed: {1}'.format(
                disk.name, str(exception)),
                         print_msg=True)
            VMachineController.delete.s(machineguid=new_vm.guid).apply_async(
                routing_key=routing_key)
            raise

        try:
            result = target_hv.create_vm_from_template(
                name,
                source_vm,
                disks,
                target_storagedriver.storage_ip,
                target_storagedriver.mountpoint,
                wait=True)
        except Exception as exception:
            logger.error('Creation of vm {0} on hypervisor failed: {1}'.format(
                new_vm.name, str(exception)),
                         print_msg=True)
            VMachineController.delete.s(machineguid=new_vm.guid).apply_async(
                routing_key=routing_key)
            raise

        new_vm.hypervisor_id = result
        new_vm.status = 'SYNC'
        new_vm.save()
        return new_vm.guid
    def test_happypath(self):
        """
        Validates the happy path; Hourly snapshots are taken with a few manual consistents
        every now an then. The delelete policy is exectued every day
        """
        # Setup
        # There are 2 machines; one with two disks, one with one disk and an additional disk
        vpool = VPool()
        vpool.name = 'vpool'
        vpool.backend_type = BackendType()
        vpool.save()
        vmachine_1 = VMachine()
        vmachine_1.name = 'vmachine_1'
        vmachine_1.devicename = 'dummy'
        vmachine_1.pmachine = PMachine()
        vmachine_1.save()
        vdisk_1_1 = VDisk()
        vdisk_1_1.name = 'vdisk_1_1'
        vdisk_1_1.volume_id = 'vdisk_1_1'
        vdisk_1_1.vmachine = vmachine_1
        vdisk_1_1.vpool = vpool
        vdisk_1_1.devicename = 'dummy'
        vdisk_1_1.size = 0
        vdisk_1_1.save()
        vdisk_1_1.reload_client()
        vdisk_1_2 = VDisk()
        vdisk_1_2.name = 'vdisk_1_2'
        vdisk_1_2.volume_id = 'vdisk_1_2'
        vdisk_1_2.vmachine = vmachine_1
        vdisk_1_2.vpool = vpool
        vdisk_1_2.devicename = 'dummy'
        vdisk_1_2.size = 0
        vdisk_1_2.save()
        vdisk_1_2.reload_client()
        vmachine_2 = VMachine()
        vmachine_2.name = 'vmachine_2'
        vmachine_2.devicename = 'dummy'
        vmachine_2.pmachine = PMachine()
        vmachine_2.save()
        vdisk_2_1 = VDisk()
        vdisk_2_1.name = 'vdisk_2_1'
        vdisk_2_1.volume_id = 'vdisk_2_1'
        vdisk_2_1.vmachine = vmachine_2
        vdisk_2_1.vpool = vpool
        vdisk_2_1.devicename = 'dummy'
        vdisk_2_1.size = 0
        vdisk_2_1.save()
        vdisk_2_1.reload_client()
        vdisk_3 = VDisk()
        vdisk_3.name = 'vdisk_3'
        vdisk_3.volume_id = 'vdisk_3'
        vdisk_3.vpool = vpool
        vdisk_3.devicename = 'dummy'
        vdisk_3.size = 0
        vdisk_3.save()
        vdisk_3.reload_client()

        for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
            [dynamic for dynamic in disk._dynamics if dynamic.name == 'snapshots'][0].timeout = 0

        # Run the testing scenario
        debug = True
        amount_of_days = 50
        now = int(mktime(datetime.now().date().timetuple()))  # Last night
        minute = 60
        hour = minute * 60
        day = hour * 24
        for d in xrange(0, amount_of_days):
            base_timestamp = now + (day * d)
            print ''
            print 'Day cycle: {}: {}'.format(
                d,
                datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d')
            )

            # At the start of the day, delete snapshot policy runs at 00:30
            print '- Deleting snapshots'
            ScheduledTaskController.deletescrubsnapshots(timestamp=base_timestamp + (minute * 30))

            # Validate snapshots
            print '- Validating snapshots'
            for vdisk in [vdisk_3]:  # [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
                self._validate(vdisk, d, now, amount_of_days, debug)

            # During the day, snapshots are taken
            # - Create non consistent snapshot every hour, between 2:00 and 22:00
            # - Create consistent snapshot at 6:30, 12:30, 18:30
            print '- Creating snapshots'
            for h in xrange(2, 23):
                timestamp = base_timestamp + (hour * h)
                for vm in [vmachine_1, vmachine_2]:
                    VMachineController.snapshot(machineguid=vm.guid,
                                                label='ss_i_{}:00'.format(str(h)),
                                                is_consistent=False,
                                                timestamp=timestamp)
                    if h in [6, 12, 18]:
                        ts = (timestamp + (minute * 30))
                        VMachineController.snapshot(machineguid=vm.guid,
                                                    label='ss_c_{}:30'.format(str(h)),
                                                    is_consistent=True,
                                                    timestamp=ts)

                VDiskController.create_snapshot(diskguid=vdisk_3.guid,
                                                metadata={'label': 'ss_i_{}:00'.format(str(h)),
                                                          'is_consistent': False,
                                                          'timestamp': timestamp,
                                                          'machineguid': None})
                if h in [6, 12, 18]:
                    ts = (timestamp + (minute * 30))
                    VDiskController.create_snapshot(diskguid=vdisk_3.guid,
                                                    metadata={'label': 'ss_c_{}:30'.format(str(h)),
                                                              'is_consistent': True,
                                                              'timestamp': ts,
                                                              'machineguid': None})
Exemple #13
0
    def create_from_template(diskguid,
                             machinename,
                             devicename,
                             pmachineguid,
                             machineguid=None,
                             storagedriver_guid=None):
        """
        Create a disk from a template

        @param parentdiskguid: guid of the disk
        @param location: location where virtual device should be created (eg: myVM)
        @param devicename: device file name for the disk (eg: mydisk-flat.vmdk)
        @param machineguid: guid of the machine to assign disk to
        @return diskguid: guid of new disk
        """

        pmachine = PMachine(pmachineguid)
        hypervisor = Factory.get(pmachine)
        disk_path = hypervisor.get_disk_path(machinename, devicename)

        description = '{} {}'.format(machinename, devicename)
        properties_to_clone = [
            'description', 'size', 'type', 'retentionpolicyid',
            'snapshotpolicyid', 'vmachine', 'vpool'
        ]

        disk = VDisk(diskguid)
        if disk.vmachine and not disk.vmachine.is_vtemplate:
            # Disk might not be attached to a vmachine, but still be a template
            raise RuntimeError('The given disk does not belong to a template')

        if storagedriver_guid is not None:
            storagedriver_id = StorageDriver(
                storagedriver_guid).storagedriver_id
        else:
            storagedriver_id = disk.storagedriver_id

        new_disk = VDisk()
        new_disk.copy(disk, include=properties_to_clone)
        new_disk.vpool = disk.vpool
        new_disk.devicename = hypervisor.clean_backing_disk_filename(disk_path)
        new_disk.parent_vdisk = disk
        new_disk.name = '{}-clone'.format(disk.name)
        new_disk.description = description
        new_disk.vmachine = VMachine(
            machineguid) if machineguid else disk.vmachine
        new_disk.save()

        logger.info(
            'Create disk from template {} to new disk {} to location {}'.
            format(disk.name, new_disk.name, disk_path))
        try:
            volume_id = disk.storagedriver_client.create_clone_from_template(
                disk_path, str(disk.volume_id), node_id=str(storagedriver_id))
            new_disk.volume_id = volume_id
            new_disk.save()
        except Exception as ex:
            logger.error(
                'Clone disk on volumedriver level failed with exception: {0}'.
                format(str(ex)))
            new_disk.delete()
            raise

        return {
            'diskguid': new_disk.guid,
            'name': new_disk.name,
            'backingdevice': disk_path
        }
Exemple #14
0
    def _prepare(self):
        # Setup
        failure_domain = FailureDomain()
        failure_domain.name = 'Test'
        failure_domain.save()
        backend_type = BackendType()
        backend_type.name = 'BackendType'
        backend_type.code = 'BT'
        backend_type.save()
        vpool = VPool()
        vpool.name = 'vpool'
        vpool.backend_type = backend_type
        vpool.save()
        pmachine = PMachine()
        pmachine.name = 'PMachine'
        pmachine.username = '******'
        pmachine.ip = '127.0.0.1'
        pmachine.hvtype = 'KVM'
        pmachine.save()
        vmachine_1 = VMachine()
        vmachine_1.name = 'vmachine_1'
        vmachine_1.devicename = 'dummy'
        vmachine_1.pmachine = pmachine
        vmachine_1.is_vtemplate = True
        vmachine_1.save()
        vdisk_1_1 = VDisk()
        vdisk_1_1.name = 'vdisk_1_1'
        vdisk_1_1.volume_id = 'vdisk_1_1'
        vdisk_1_1.vmachine = vmachine_1
        vdisk_1_1.vpool = vpool
        vdisk_1_1.devicename = 'dummy'
        vdisk_1_1.size = 0
        vdisk_1_1.save()
        vdisk_1_1.reload_client()
        storage_router = StorageRouter()
        storage_router.name = 'storage_router'
        storage_router.ip = '127.0.0.1'
        storage_router.pmachine = pmachine
        storage_router.machine_id = System.get_my_machine_id()
        storage_router.rdma_capable = False
        storage_router.primary_failure_domain = failure_domain
        storage_router.save()
        storagedriver = StorageDriver()
        storagedriver.vpool = vpool
        storagedriver.storagerouter = storage_router
        storagedriver.name = '1'
        storagedriver.mountpoint = '/'
        storagedriver.cluster_ip = storage_router.ip
        storagedriver.storage_ip = '127.0.0.1'
        storagedriver.storagedriver_id = '1'
        storagedriver.ports = [1, 2, 3]
        storagedriver.save()
        service_type = ServiceType()
        service_type.name = 'MetadataServer'
        service_type.save()
        s_id = '{0}-{1}'.format(storagedriver.storagerouter.name, '1')
        service = Service()
        service.name = s_id
        service.storagerouter = storagedriver.storagerouter
        service.ports = [1]
        service.type = service_type
        service.save()
        mds_service = MDSService()
        mds_service.service = service
        mds_service.number = 0
        mds_service.capacity = 10
        mds_service.vpool = storagedriver.vpool
        mds_service.save()

        def ensure_safety(vdisk):
            pass

        class Dtl_Checkup():
            @staticmethod
            def delay(vpool_guid=None,
                      vdisk_guid=None,
                      storagerouters_to_exclude=None):
                pass

        MDSServiceController.ensure_safety = staticmethod(ensure_safety)
        VDiskController.dtl_checkup = Dtl_Checkup
        return vdisk_1_1, pmachine
Exemple #15
0
 def _build_service_structure(self, structure):
     """
     Builds an MDS service structure
     """
     vpools = {}
     storagerouters = {}
     storagedrivers = {}
     services = {}
     mds_services = {}
     service_type = ServiceType()
     service_type.name = 'MetadataServer'
     service_type.save()
     for vpool_id in structure['vpools']:
         vpool = VPool()
         vpool.name = str(vpool_id)
         vpool.backend_type = BackendType()
         vpool.save()
         vpools[vpool_id] = vpool
     for sr_id in structure['storagerouters']:
         storagerouter = StorageRouter()
         storagerouter.name = str(sr_id)
         storagerouter.ip = '10.0.0.{0}'.format(sr_id)
         storagerouter.pmachine = PMachine()
         storagerouter.save()
         storagerouters[sr_id] = storagerouter
     for sd_info in structure['storagedrivers']:
         sd_id, vpool_id, sr_id = sd_info
         storagedriver = StorageDriver()
         storagedriver.vpool = vpools[vpool_id]
         storagedriver.storagerouter = storagerouters[sr_id]
         storagedriver.name = str(sd_id)
         storagedriver.mountpoint_temp = '/'
         storagedriver.mountpoint_foc = '/'
         storagedriver.mountpoint_readcaches = ['/']
         storagedriver.mountpoint_writecaches = ['/']
         storagedriver.mountpoint_temp = '/'
         storagedriver.mountpoint_md = '/'
         storagedriver.mountpoint_bfs = '/'
         storagedriver.mountpoint_fragmentcache = '/'
         storagedriver.mountpoint = '/'
         storagedriver.cluster_ip = storagerouters[sr_id].ip
         storagedriver.storage_ip = '127.0.0.1'
         storagedriver.storagedriver_id = str(sd_id)
         storagedriver.ports = [1, 2, 3]
         storagedriver.save()
         storagedrivers[sd_id] = storagedriver
     for mds_info in structure['mds_services']:
         mds_id, sd_id = mds_info
         sd = storagedrivers[sd_id]
         s_id = '{0}-{1}'.format(sd.storagerouter.name, mds_id)
         service = Service()
         service.name = s_id
         service.storagerouter = sd.storagerouter
         service.ports = [mds_id]
         service.type = service_type
         service.save()
         services[s_id] = service
         mds_service = MDSService()
         mds_service.service = service
         mds_service.number = 0
         mds_service.capacity = 10
         mds_service.vpool = sd.vpool
         mds_service.save()
         mds_services[mds_id] = mds_service
     return vpools, storagerouters, storagedrivers, services, mds_services, service_type
Exemple #16
0
    def create_from_template(diskguid,
                             machinename,
                             devicename,
                             pmachineguid,
                             machineguid=None,
                             storagedriver_guid=None):
        """
        Create a disk from a template

        @param devicename: device file name for the disk (eg: mydisk-flat.vmdk)
        @param machineguid: guid of the machine to assign disk to
        @return diskguid: guid of new disk
        """

        pmachine = PMachine(pmachineguid)
        hypervisor = Factory.get(pmachine)
        disk_path = hypervisor.get_disk_path(machinename, devicename)

        description = '{} {}'.format(machinename, devicename)
        properties_to_clone = [
            'description', 'size', 'type', 'retentionpolicyid',
            'snapshotpolicyid', 'vmachine', 'vpool'
        ]

        vdisk = VDisk(diskguid)
        if vdisk.vmachine and not vdisk.vmachine.is_vtemplate:
            # Disk might not be attached to a vmachine, but still be a template
            raise RuntimeError('The given vdisk does not belong to a template')

        if storagedriver_guid is not None:
            storagedriver_id = StorageDriver(
                storagedriver_guid).storagedriver_id
        else:
            storagedriver_id = vdisk.storagedriver_id
        storagedriver = StorageDriverList.get_by_storagedriver_id(
            storagedriver_id)
        if storagedriver is None:
            raise RuntimeError(
                'Could not find StorageDriver with id {0}'.format(
                    storagedriver_id))

        new_vdisk = VDisk()
        new_vdisk.copy(vdisk, include=properties_to_clone)
        new_vdisk.vpool = vdisk.vpool
        new_vdisk.devicename = hypervisor.clean_backing_disk_filename(
            disk_path)
        new_vdisk.parent_vdisk = vdisk
        new_vdisk.name = '{}-clone'.format(vdisk.name)
        new_vdisk.description = description
        new_vdisk.vmachine = VMachine(
            machineguid) if machineguid else vdisk.vmachine
        new_vdisk.save()

        mds_service = MDSServiceController.get_preferred_mds(
            storagedriver.storagerouter, vdisk.vpool)
        if mds_service is None:
            raise RuntimeError('Could not find a MDS service')

        logger.info(
            'Create disk from template {} to new disk {} to location {}'.
            format(vdisk.name, new_vdisk.name, disk_path))
        try:
            volume_id = vdisk.storagedriver_client.create_clone_from_template(
                target_path=disk_path,
                metadata_backend_config=MDSMetaDataBackendConfig([
                    MDSNodeConfig(address=str(
                        mds_service.service.storagerouter.ip),
                                  port=mds_service.service.ports[0])
                ]),
                parent_volume_id=str(vdisk.volume_id),
                node_id=str(storagedriver_id))
            new_vdisk.volume_id = volume_id
            new_vdisk.save()
            MDSServiceController.ensure_safety(new_vdisk)

        except Exception as ex:
            logger.error(
                'Clone disk on volumedriver level failed with exception: {0}'.
                format(str(ex)))
            new_vdisk.delete()
            raise

        return {
            'diskguid': new_vdisk.guid,
            'name': new_vdisk.name,
            'backingdevice': disk_path
        }
Exemple #17
0
    def clone(diskguid,
              snapshotid,
              devicename,
              pmachineguid,
              machinename,
              machineguid=None):
        """
        Clone a disk
        """
        pmachine = PMachine(pmachineguid)
        hypervisor = Factory.get(pmachine)
        description = '{} {}'.format(machinename, devicename)
        properties_to_clone = [
            'description', 'size', 'type', 'retentionpolicyguid',
            'snapshotpolicyguid', 'autobackup'
        ]
        vdisk = VDisk(diskguid)
        location = hypervisor.get_backing_disk_path(machinename, devicename)

        new_vdisk = VDisk()
        new_vdisk.copy(vdisk, include=properties_to_clone)
        new_vdisk.parent_vdisk = vdisk
        new_vdisk.name = '{0}-clone'.format(vdisk.name)
        new_vdisk.description = description
        new_vdisk.devicename = hypervisor.clean_backing_disk_filename(location)
        new_vdisk.parentsnapshot = snapshotid
        new_vdisk.vmachine = VMachine(
            machineguid) if machineguid else vdisk.vmachine
        new_vdisk.vpool = vdisk.vpool
        new_vdisk.save()

        try:
            storagedriver = StorageDriverList.get_by_storagedriver_id(
                vdisk.storagedriver_id)
            if storagedriver is None:
                raise RuntimeError(
                    'Could not find StorageDriver with id {0}'.format(
                        vdisk.storagedriver_id))

            mds_service = MDSServiceController.get_preferred_mds(
                storagedriver.storagerouter, vdisk.vpool)
            if mds_service is None:
                raise RuntimeError('Could not find a MDS service')

            logger.info('Clone snapshot {} of disk {} to location {}'.format(
                snapshotid, vdisk.name, location))
            volume_id = vdisk.storagedriver_client.create_clone(
                target_path=location,
                metadata_backend_config=MDSMetaDataBackendConfig([
                    MDSNodeConfig(address=str(
                        mds_service.service.storagerouter.ip),
                                  port=mds_service.service.ports[0])
                ]),
                parent_volume_id=str(vdisk.volume_id),
                parent_snapshot_id=str(snapshotid),
                node_id=str(vdisk.storagedriver_id))
        except Exception as ex:
            logger.error(
                'Caught exception during clone, trying to delete the volume. {0}'
                .format(ex))
            new_vdisk.delete()
            VDiskController.delete_volume(location)
            raise

        new_vdisk.volume_id = volume_id
        new_vdisk.save()

        try:
            MDSServiceController.ensure_safety(new_vdisk)
        except Exception as ex:
            logger.error(
                'Caught exception during "ensure_safety" {0}'.format(ex))

        return {
            'diskguid': new_vdisk.guid,
            'name': new_vdisk.name,
            'backingdevice': location
        }