def __init__(self, **azure_config): """ :param ServiceManagement azure_client: an instance of the azure serivce managment api client. :param String service_name: The name of the cloud service :param names of Azure volumes to identify cluster :returns: A ``BlockDeviceVolume``. """ self._instance_id = self.compute_instance_id() creds = ServicePrincipalCredentials( client_id=azure_config['client_id'], secret=azure_config['client_secret'], tenant=azure_config['tenant_id']) self._resource_client = ResourceManagementClient( creds, azure_config['subscription_id']) self._compute_client = ComputeManagementClient( creds, azure_config['subscription_id']) self._azure_storage_client = PageBlobService( account_name=azure_config['storage_account_name'], account_key=azure_config['storage_account_key']) self._manager = DiskManager(self._resource_client, self._compute_client, self._azure_storage_client, azure_config['storage_account_container'], azure_config['group_name'], azure_config['location']) self._storage_account_name = azure_config['storage_account_name'] self._disk_container_name = azure_config['storage_account_container'] self._resource_group = azure_config['group_name']
class AzureStorageBlockDeviceAPI(object): """ An ``IBlockDeviceAsyncAPI`` which uses Azure Storage Backed Block Devices Current Support: Azure SMS API """ def __init__(self, **azure_config): """ :param ServiceManagement azure_client: an instance of the azure serivce managment api client. :param String service_name: The name of the cloud service :param names of Azure volumes to identify cluster :returns: A ``BlockDeviceVolume``. """ self._instance_id = self.compute_instance_id() creds = ServicePrincipalCredentials( client_id=azure_config['client_id'], secret=azure_config['client_secret'], tenant=azure_config['tenant_id']) self._resource_client = ResourceManagementClient( creds, azure_config['subscription_id']) self._compute_client = ComputeManagementClient( creds, azure_config['subscription_id']) self._azure_storage_client = PageBlobService( account_name=azure_config['storage_account_name'], account_key=azure_config['storage_account_key']) self._manager = DiskManager(self._resource_client, self._compute_client, self._azure_storage_client, azure_config['storage_account_container'], azure_config['group_name'], azure_config['location']) self._storage_account_name = azure_config['storage_account_name'] self._disk_container_name = azure_config['storage_account_container'] self._resource_group = azure_config['group_name'] def allocation_unit(self): """ 1GiB is the minimum allocation unit for azure disks return int: 1 GiB """ return int(GiB(1).to_Byte().value) def compute_instance_id(self): """ Azure Stored a UUID in the SDC kernel module. """ # Node host names should be unique within a vnet return unicode(socket.gethostname()) def create_volume(self, dataset_id, size): """ Create a new volume. :param UUID dataset_id: The Flocker dataset ID of the dataset on this volume. :param int size: The size of the new volume in bytes. :returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when the volume has been created. """ size_in_gb = Byte(size).to_GiB().value if size_in_gb % 1 != 0: raise UnsupportedVolumeSize(dataset_id) disk_label = self._disk_label_for_dataset_id(dataset_id) self._manager.create_disk(disk_label, size_in_gb) return BlockDeviceVolume( blockdevice_id=unicode(disk_label), size=size, attached_to=None, dataset_id=dataset_id) def destroy_volume(self, blockdevice_id): """ Destroy an existing volume. :param unicode blockdevice_id: The unique identifier for the volume to destroy. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :return: ``None`` """ log_info('Destorying block device: ' + str(blockdevice_id)) disks = self._manager.list_disks() target_disk = None for disk in disks: if disk.name == blockdevice_id: target_disk = disk break if target_disk is None: raise UnknownVolume(blockdevice_id) self._manager.destroy_disk(target_disk.name) def attach_volume(self, blockdevice_id, attach_to): """ Attach ``blockdevice_id`` to ``host``. :param unicode blockdevice_id: The unique identifier for the block device being attached. :param unicode attach_to: An identifier like the one returned by the ``compute_instance_id`` method indicating the node to which to attach the volume. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is already attached. :returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to ``host``. """ log_info('Attempting to attach ' + str(blockdevice_id) + ' to ' + str(attach_to)) _vmstate_lock.acquire() try: # Make sure disk is present. Also, need the disk size is needed. disks = self._manager.list_disks() target_disk = None for disk in disks: if disk.name == blockdevice_id: target_disk = disk break if target_disk is None: raise UnknownVolume(blockdevice_id) (disk, vmname, lun) = self._get_disk_vmname_lun(blockdevice_id) if vmname is not None: raise AlreadyAttachedVolume(blockdevice_id) self._manager.attach_disk( str(attach_to), target_disk.name, int(GiB(bytes=target_disk.properties.content_length))) finally: _vmstate_lock.release() log_info('disk attached') return self._blockdevicevolume_from_azure_volume( blockdevice_id, target_disk.properties.content_length, attach_to) def detach_volume(self, blockdevice_id): """ Detach ``blockdevice_id`` from whatever host it is attached to. :param unicode blockdevice_id: The unique identifier for the block device being detached. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to anything. :returns: ``None`` """ _vmstate_lock.acquire() try: (target_disk, vm_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) self._manager.detach_disk(vm_name, target_disk) finally: _vmstate_lock.release() def get_device_path(self, blockdevice_id): """ Return the device path that has been allocated to the block device on the host to which it is currently attached. :param unicode blockdevice_id: The unique identifier for the block device. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to a host. :returns: A ``FilePath`` for the device. """ (target_disk, vm_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) return Lun.get_device_path_for_lun(lun) def _get_details_for_disks(self, disks_in): """ Give a list of disks, returns a ''list'' of ''BlockDeviceVolume''s """ disk_info = [] disks = dict((d.name, d) for d in disks_in) # first handle disks attached to vms vms_info = {} vms = self._compute_client.virtual_machines.list(self._resource_group) for vm in vms: vm_info = self._compute_client.virtual_machines.get( self._resource_group, vm.name, expand="instanceView") vms_info[vm.name] = vm_info for data_disk in vm.storage_profile.data_disks: if 'flocker-' in data_disk.name: disk_name = data_disk.name.replace('.vhd', '') if disk_name in disks: disk_info.append( self._blockdevicevolume_from_azure_volume( disk_name, self._gibytes_to_bytes(data_disk.disk_size_gb), vm.name)) del disks[disk_name] else: # We have a data disk mounted that isn't in the known # list of blobs. log_info( "Disk attached, but not known in container: " + disk_name) for vm_info in vms_info: vm = vms_info[vm_info] if vm.instance_view is not None: for disk in vm.instance_view.disks: if 'flocker-' in disk.name: disk_name = disk.name.replace('.vhd', '') if disk_name in disks: disk_size = disks[disk_name].properties.\ content_length disk_size = disk_size - (disk_size % (1024 * 1024 * 1024)) disk_info.append( self._blockdevicevolume_from_azure_volume( disk_name, disk_size, vm.name)) del disks[disk_name] # each remaining disk should be added as not attached for disk in disks: if 'flocker-' in disk: disk_info.append(self._blockdevicevolume_from_azure_volume( disk.replace('.vhd', ''), disks[disk].properties.content_length, None)) return disk_info def list_volumes(self): """ List all the block devices available via the back end API. :returns: A ``list`` of ``BlockDeviceVolume``s. """ disks = self._manager.list_disks() disk_list = self._get_details_for_disks(disks) return disk_list def _disk_label_for_dataset_id(self, dataset_id): """ Returns a disk label for a given Dataset ID :param unicode dataset_id: The identifier of the dataset :returns string: A string representing the disk label """ label = 'flocker-' + str(dataset_id) return label def _dataset_id_for_disk_label(self, disk_label): """ Returns a UUID representing the Dataset ID for the given disk label :param string disk_label: The disk label :returns UUID: The UUID of the dataset """ return UUID(disk_label.replace('flocker-', '')) def _get_disk_vmname_lun(self, blockdevice_id): target_disk = None target_lun = None vm_name = None disk_list = self._manager.list_disks() for disk in disk_list: if 'flocker-' not in disk.name: continue if disk.name == blockdevice_id: target_disk = disk break if target_disk is None: return (None, None, None) vm_info = None vm_disk_info = None vms = self._compute_client.virtual_machines.list(self._resource_group) for vm in vms: for disk in vm.storage_profile.data_disks: if disk.name == target_disk.name: vm_disk_info = disk vm_info = vm break if vm_disk_info is not None: break if vm_info is not None: vm_name = vm_info.name target_lun = vm_disk_info.lun return (target_disk.name, vm_name, target_lun) def _gibytes_to_bytes(self, size): return int(GiB(size).to_Byte().value) def _blockdevicevolume_from_azure_volume(self, label, size, attached_to_name): # azure will report the disk size including the 512 byte footer # however flocker expects the exact value it requested for disk size # so remove the offset when reportint the size to flocker by 512 bytes if (size % self.allocation_unit()) == 512: size = size - 512 return BlockDeviceVolume( blockdevice_id=unicode(label), size=int(size), attached_to=attached_to_name, dataset_id=self._dataset_id_for_disk_label(label) ) # disk labels are formatted as flocker-<data_set_id>
class AzureStorageBlockDeviceAPI(object): """ An ``IBlockDeviceAsyncAPI`` which uses Azure Storage Backed Block Devices Current Support: Azure SMS API """ def __init__(self, **azure_config): """ :param ServiceManagement azure_client: an instance of the azure serivce managment api client. :param String service_name: The name of the cloud service :param names of Azure volumes to identify cluster :returns: A ``BlockDeviceVolume``. """ self._instance_id = self.compute_instance_id() creds = ServicePrincipalCredentials( client_id=azure_config['client_id'], secret=azure_config['client_secret'], tenant=azure_config['tenant_id']) self._resource_client = ResourceManagementClient( creds, azure_config['subscription_id']) self._compute_client = ComputeManagementClient( creds, azure_config['subscription_id']) self._azure_storage_client = PageBlobService( account_name=azure_config['storage_account_name'], account_key=azure_config['storage_account_key']) self._manager = DiskManager(self._resource_client, self._compute_client, self._azure_storage_client, azure_config['storage_account_container'], azure_config['group_name'], azure_config['location']) self._storage_account_name = azure_config['storage_account_name'] self._disk_container_name = azure_config['storage_account_container'] self._resource_group = azure_config['group_name'] def allocation_unit(self): """ 1GiB is the minimum allocation unit for azure disks return int: 1 GiB """ return int(GiB(1).to_Byte().value) def compute_instance_id(self): """ Azure Stored a UUID in the SDC kernel module. """ # Node host names should be unique within a vnet return unicode(socket.gethostname()) def create_volume(self, dataset_id, size): """ Create a new volume. :param UUID dataset_id: The Flocker dataset ID of the dataset on this volume. :param int size: The size of the new volume in bytes. :returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when the volume has been created. """ size_in_gb = Byte(size).to_GiB().value if size_in_gb % 1 != 0: raise UnsupportedVolumeSize(dataset_id) disk_label = self._disk_label_for_dataset_id(dataset_id) self._manager.create_disk(disk_label, size_in_gb) return BlockDeviceVolume( blockdevice_id=unicode(disk_label), size=size, attached_to=None, dataset_id=dataset_id) def destroy_volume(self, blockdevice_id): """ Destroy an existing volume. :param unicode blockdevice_id: The unique identifier for the volume to destroy. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :return: ``None`` """ log_info('Destorying block device: ' + str(blockdevice_id)) disks = self._manager.list_disks() target_disk = None for disk in disks: if disk.name == blockdevice_id: target_disk = disk break if target_disk is None: raise UnknownVolume(blockdevice_id) self._manager.destroy_disk(target_disk.name) def attach_volume(self, blockdevice_id, attach_to): """ Attach ``blockdevice_id`` to ``host``. :param unicode blockdevice_id: The unique identifier for the block device being attached. :param unicode attach_to: An identifier like the one returned by the ``compute_instance_id`` method indicating the node to which to attach the volume. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is already attached. :returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to ``host``. """ log_info('Attempting to attach ' + str(blockdevice_id) + ' to ' + str(attach_to)) _vmstate_lock.acquire() try: # Make sure disk is present. Also, need the disk size is needed. disks = self._manager.list_disks() target_disk = None for disk in disks: if disk.name == blockdevice_id: target_disk = disk break if target_disk is None: raise UnknownVolume(blockdevice_id) (disk, vmname, lun) = self._get_disk_vmname_lun(blockdevice_id) log_info('Disk attached on: ' + str(vmname) + '. Targeted VM: ' + str(attach_to) + '. Disk: ' + str(blockdevice_id)) if vmname is not None: if str(vmname) == str(attach_to): raise AlreadyAttachedVolume(blockdevice_id) else: self._manager.detach_disk(vmname, target_disk) self._manager.attach_disk( str(attach_to), target_disk.name, int(GiB(bytes=target_disk.properties.content_length))) finally: _vmstate_lock.release() log_info('disk attached') return self._blockdevicevolume_from_azure_volume( blockdevice_id, target_disk.properties.content_length, attach_to) def detach_volume(self, blockdevice_id): """ Detach ``blockdevice_id`` from whatever host it is attached to. :param unicode blockdevice_id: The unique identifier for the block device being detached. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to anything. :returns: ``None`` """ _vmstate_lock.acquire() try: (target_disk, vm_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) self._manager.detach_disk(vm_name, target_disk) finally: _vmstate_lock.release() def get_device_path(self, blockdevice_id): """ Return the device path that has been allocated to the block device on the host to which it is currently attached. :param unicode blockdevice_id: The unique identifier for the block device. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to a host. :returns: A ``FilePath`` for the device. """ (target_disk, vm_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) return Lun.get_device_path_for_lun(lun) def _get_details_for_disks(self, disks_in): """ Give a list of disks, returns a ''list'' of ''BlockDeviceVolume''s """ disk_info = [] disks = dict((d.name, d) for d in disks_in) # first handle disks attached to vms vms_info = {} vms = self._compute_client.virtual_machines.list(self._resource_group) for vm in vms: vm_info = self._compute_client.virtual_machines.get( self._resource_group, vm.name, expand="instanceView") vms_info[vm.name] = vm_info for data_disk in vm.storage_profile.data_disks: if 'flocker-' in data_disk.name: disk_name = data_disk.name.replace('.vhd', '') if disk_name in disks: disk_info.append( self._blockdevicevolume_from_azure_volume( disk_name, self._gibytes_to_bytes(data_disk.disk_size_gb), vm.name)) del disks[disk_name] else: # We have a data disk mounted that isn't in the known # list of blobs. log_info( "Disk attached, but not known in container: " + disk_name) for vm_info in vms_info: vm = vms_info[vm_info] if vm.instance_view is not None: for disk in vm.instance_view.disks: if 'flocker-' in disk.name: disk_name = disk.name.replace('.vhd', '') if disk_name in disks: disk_size = disks[disk_name].properties.\ content_length disk_size = disk_size - (disk_size % (1024 * 1024 * 1024)) disk_info.append( self._blockdevicevolume_from_azure_volume( disk_name, disk_size, vm.name)) del disks[disk_name] # each remaining disk should be added as not attached for disk in disks: if 'flocker-' in disk: disk_info.append(self._blockdevicevolume_from_azure_volume( disk.replace('.vhd', ''), disks[disk].properties.content_length, None)) return disk_info def list_volumes(self): """ List all the block devices available via the back end API. :returns: A ``list`` of ``BlockDeviceVolume``s. """ disks = self._manager.list_disks() disk_list = self._get_details_for_disks(disks) return disk_list def _disk_label_for_dataset_id(self, dataset_id): """ Returns a disk label for a given Dataset ID :param unicode dataset_id: The identifier of the dataset :returns string: A string representing the disk label """ label = 'flocker-' + str(dataset_id) return label def _dataset_id_for_disk_label(self, disk_label): """ Returns a UUID representing the Dataset ID for the given disk label :param string disk_label: The disk label :returns UUID: The UUID of the dataset """ return UUID(disk_label.replace('flocker-', '')) def _get_disk_vmname_lun(self, blockdevice_id): target_disk = None target_lun = None vm_name = None disk_list = self._manager.list_disks() for disk in disk_list: if 'flocker-' not in disk.name: continue if disk.name == blockdevice_id: target_disk = disk break if target_disk is None: return (None, None, None) vm_info = None vm_disk_info = None vms = self._compute_client.virtual_machines.list(self._resource_group) for vm in vms: for disk in vm.storage_profile.data_disks: if disk.name == target_disk.name: vm_disk_info = disk vm_info = vm break if vm_disk_info is not None: break if vm_info is not None: vm_name = vm_info.name target_lun = vm_disk_info.lun return (target_disk.name, vm_name, target_lun) def _gibytes_to_bytes(self, size): return int(GiB(size).to_Byte().value) def _blockdevicevolume_from_azure_volume(self, label, size, attached_to_name): # azure will report the disk size including the 512 byte footer # however flocker expects the exact value it requested for disk size # so remove the offset when reportint the size to flocker by 512 bytes if (size % self.allocation_unit()) == 512: size = size - 512 return BlockDeviceVolume( blockdevice_id=unicode(label), size=int(size), attached_to=attached_to_name, dataset_id=self._dataset_id_for_disk_label(label) ) # disk labels are formatted as flocker-<data_set_id>