class AzureServicesManager:
# Storage
container = 'vhds'
windows_blob_url = 'blob.core.windows.net'
# Linux
linux_user = '******'
linux_pass = '******'
location = 'West US'
# SSH Keys
def __init__(self, subscription_id, cert_file):
self.subscription_id = subscription_id
self.cert_file = cert_file
self.sms = ServiceManagementService(self.subscription_id, self.cert_file)
@property
def sms(self):
return self.sms
def list_locations(self):
locations = self.sms.list_locations()
for location in locations:
print location
def list_images(self):
return self.sms.list_os_images()
@utils.resource_not_found_handler
def get_hosted_service(self, service_name):
resp = self.sms.get_hosted_service_properties(service_name)
properties = resp.hosted_service_properties
return properties.__dict__
def delete_hosted_service(self, service_name):
res = self.sms.check_hosted_service_name_availability(service_name)
if not res.result:
return
self.sms.delete_hosted_service(service_name)
def create_hosted_service(self, os_user, service_name=None, random=False):
if not service_name:
service_name = self.generate_cloud_service_name(os_user, random)
available = False
while not available:
res = self.sms.check_hosted_service_name_availability(service_name)
if not res.result:
service_name = self.generate_cloud_service_name(os_user,
random)
else:
available = True
self.sms.create_hosted_service(service_name=service_name,
label=service_name,
location='West US')
return service_name
def create_virtual_machine(self, service_name, vm_name, image_name, role_size):
media_link = self._get_media_link(vm_name)
# Linux VM configuration
hostname = '-'.join((vm_name, 'host'))
linux_config = LinuxConfigurationSet(hostname,
self.linux_user,
self.linux_pass,
True)
# Hard disk for the OS
os_hd = OSVirtualHardDisk(image_name, media_link)
# Create vm
result = self.sms.create_virtual_machine_deployment(
service_name=service_name, deployment_name=vm_name,
deployment_slot='production', label=vm_name,
role_name=vm_name, system_config=linux_config,
os_virtual_hard_disk=os_hd,
role_size=role_size
)
request_id = result.request_id
return {
'request_id': request_id,
'media_link': media_link
}
def delete_virtual_machine(self, service_name, vm_name):
resp = self.sms.delete_deployment(service_name, vm_name, True)
self.sms.wait_for_operation_status(resp.request_id)
result = self.sms.delete_hosted_service(service_name)
return result
def generate_cloud_service_name(self, os_user=None, random=False):
if random:
return utils.generate_random_name(10)
return '-'.join((os_user, utils.generate_random_name(6)))
@utils.resource_not_found_handler
def get_virtual_machine_info(self, service_name, vm_name):
vm_info = {}
deploy_info = self.sms.get_deployment_by_name(service_name, vm_name)
if deploy_info and deploy_info.role_instance_list:
vm_info = deploy_info.role_instance_list[0].__dict__
return vm_info
def list_virtual_machines(self):
vm_list = []
services = self.sms.list_hosted_services()
for service in services:
deploys = service.deployments
if deploys and deploys.role_instance_list:
vm_name = deploys.role_instance_list[0].instance_name
vm_list.append(vm_name)
return vm_list
def power_on(self, service_name, vm_name):
resp = self.sms.start_role(service_name, vm_name, vm_name)
return resp.request_id
def power_off(self, service_name, vm_name):
resp = self.sms.shutdown_role(service_name, vm_name, vm_name)
return resp.request_id
def soft_reboot(self, service_name, vm_name):
resp = self.sms.restart_role(service_name, vm_name, vm_name)
return resp.request_id
def hard_reboot(self, service_name, vm_name):
resp = self.sms.reboot_role_instance(service_name, vm_name, vm_name)
return resp.request_id
def attach_volume(self, service_name, vm_name, size, lun):
disk_name = utils.generate_random_name(5, vm_name)
media_link = self._get_media_link(vm_name, disk_name)
self.sms.add_data_disk(service_name,
vm_name,
vm_name,
lun,
host_caching='ReadWrite',
media_link=media_link,
disk_name=disk_name,
logical_disk_size_in_gb=size)
def detach_volume(self, service_name, vm_name, lun):
self.sms.delete_data_disk(service_name, vm_name, vm_name, lun, True)
def get_available_lun(self, service_name, vm_name):
try:
role = self.sms.get_role(service_name, vm_name, vm_name)
except Exception:
return 0
disks = role.data_virtual_hard_disks
luns = [disk.lun for disk in disks].sort()
for i in range(1, 16):
if i not in luns:
return i
return None
def snapshot(self, service_name, vm_name, image_id, snanshot_name):
image_desc = 'Snapshot for image %s' % vm_name
image = CaptureRoleAsVMImage('Specialized', snanshot_name,
image_id, image_desc, 'english')
resp = self.sms.capture_vm_image(service_name, vm_name, vm_name, image)
self.sms.wait_for_operation_status(resp.request_id)
def _get_media_link(self, vm_name, filename=None, storage_account=None):
""" The MediaLink should be constructed as:
https://<storageAccount>.<blobLink>/<blobContainer>/<filename>.vhd
"""
if not storage_account:
storage_account = self._get_or_create_storage_account()
container = self.container
filename = vm_name if filename is None else filename
blob = vm_name + '-' + filename + '.vhd'
media_link = "http://%s.%s/%s/%s" % (storage_account,
self.windows_blob_url,
container, blob)
return media_link
def _get_or_create_storage_account(self):
account_list = self.sms.list_storage_accounts()
if account_list:
return account_list[-1].service_name
storage_account = utils.generate_random_name(10)
description = "Storage account %s description" % storage_account
label = storage_account + 'label'
self.sms.create_storage_account(storage_account,
description,
label,
location=self.location)
return storage_account
def _wait_for_operation(self, request_id, timeout=3000,
failure_callback=None,
failure_callback_kwargs=None):
try:
self.sms.wait_for_operation_status(request_id, timeout=timeout)
except Exception as ex:
if failure_callback and failure_callback_kwargs:
failure_callback(**failure_callback_kwargs)
raise ex
class AzureStorageBlockDeviceAPI(object):
"""
An ``IBlockDeviceAsyncAPI`` which uses Azure Storage Backed Block Devices
Current Support: Azure SMS API
"""
def __init__(self, **azure_config):
"""
:param ServiceManagement azure_client: an instance of the azure
serivce managment api client.
:param String service_name: The name of the cloud service
:param
names of Azure volumes to identify cluster
:returns: A ``BlockDeviceVolume``.
"""
self._instance_id = self.compute_instance_id()
self._azure_service_client = ServiceManagementService(
azure_config['subscription_id'],
azure_config['management_certificate_path'])
self._service_name = azure_config['service_name']
self._azure_storage_client = BlobService(
azure_config['storage_account_name'],
azure_config['storage_account_key'])
self._storage_account_name = azure_config['storage_account_name']
self._disk_container_name = azure_config['disk_container_name']
if azure_config['debug']:
to_file(sys.stdout)
def allocation_unit(self):
"""
1GiB is the minimum allocation unit for azure disks
return int: 1 GiB
"""
return int(GiB(1).to_Byte().value)
def compute_instance_id(self):
"""
Azure Stored a UUID in the SDC kernel module.
"""
# Node host names should be unique within a vnet
return unicode(socket.gethostname())
def create_volume(self, dataset_id, size):
"""
Create a new volume.
:param UUID dataset_id: The Flocker dataset ID of the dataset on this
volume.
:param int size: The size of the new volume in bytes.
:returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when
the volume has been created.
"""
size_in_gb = Byte(size).to_GiB().value
if size_in_gb % 1 != 0:
raise UnsupportedVolumeSize(dataset_id)
self._create_volume_blob(size, dataset_id)
label = self._disk_label_for_dataset_id(str(dataset_id))
return BlockDeviceVolume(
blockdevice_id=unicode(label),
size=size,
attached_to=None,
dataset_id=self._dataset_id_for_disk_label(label))
def destroy_volume(self, blockdevice_id):
"""
Destroy an existing volume.
:param unicode blockdevice_id: The unique identifier for the volume to
destroy.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:return: ``None``
"""
log_info('Destorying block device: ' + str(blockdevice_id))
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
request = None
if lun is not None:
request = \
self._azure_service_client.delete_data_disk(
service_name=self._service_name,
deployment_name=self._service_name,
role_name=target_disk.attached_to.role_name,
lun=lun,
delete_vhd=True)
else:
if target_disk.__class__.__name__ == 'Blob':
# unregistered disk
self._azure_storage_client.delete_blob(
self._disk_container_name, target_disk.name)
else:
request = self._azure_service_client.delete_disk(
target_disk.name, True)
if request is not None:
self._wait_for_async(request.request_id, 5000)
self._wait_for_detach(blockdevice_id)
def attach_volume(self, blockdevice_id, attach_to):
"""
Attach ``blockdevice_id`` to ``host``.
:param unicode blockdevice_id: The unique identifier for the block
device being attached.
:param unicode attach_to: An identifier like the one returned by the
``compute_instance_id`` method indicating the node to which to
attach the volume.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is
already attached.
:returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to
``host``.
"""
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
if lun is not None:
raise AlreadyAttachedVolume(blockdevice_id)
log_info('Attempting to attach ' + str(blockdevice_id)
+ ' to ' + str(attach_to))
disk_size = self._attach_disk(blockdevice_id, target_disk, attach_to)
self._wait_for_attach(blockdevice_id)
log_info('disk attached')
return self._blockdevicevolume_from_azure_volume(blockdevice_id,
disk_size,
attach_to)
def detach_volume(self, blockdevice_id):
"""
Detach ``blockdevice_id`` from whatever host it is attached to.
:param unicode blockdevice_id: The unique identifier for the block
device being detached.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises UnattachedVolume: If the supplied ``blockdevice_id`` is
not attached to anything.
:returns: ``None``
"""
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
if lun is None:
raise UnattachedVolume(blockdevice_id)
# contrary to function name it doesn't delete by default, just detachs
request = \
self._azure_service_client.delete_data_disk(
service_name=self._service_name,
deployment_name=self._service_name,
role_name=role_name, lun=lun)
self._wait_for_async(request.request_id, 5000)
self._wait_for_detach(blockdevice_id)
def get_device_path(self, blockdevice_id):
"""
Return the device path that has been allocated to the block device on
the host to which it is currently attached.
:param unicode blockdevice_id: The unique identifier for the block
device.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises UnattachedVolume: If the supplied ``blockdevice_id`` is
not attached to a host.
:returns: A ``FilePath`` for the device.
"""
(target_disk_or_blob, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk_or_blob is None:
raise UnknownVolume(blockdevice_id)
if lun is None:
raise UnattachedVolume(blockdevice_id)
return Lun.get_device_path_for_lun(lun)
def list_volumes(self):
"""
List all the block devices available via the back end API.
:returns: A ``list`` of ``BlockDeviceVolume``s.
"""
media_url_prefix = 'https://' + self._storage_account_name \
+ '.blob.core.windows.net/' + self._disk_container_name
disks = self._azure_service_client.list_disks()
disk_list = []
all_blobs = self._get_flocker_blobs()
for d in disks:
if media_url_prefix not in d.media_link or \
'flocker-' not in d.label:
continue
role_name = None
if d.attached_to is not None \
and d.attached_to.role_name is not None:
role_name = d.attached_to.role_name
disk_list.append(self._blockdevicevolume_from_azure_volume(
d.label, self._gibytes_to_bytes(d.logical_disk_size_in_gb),
role_name))
if d.label in all_blobs:
del all_blobs[d.label]
for key in all_blobs:
# include unregistered 'disk' blobs
disk_list.append(self._blockdevicevolume_from_azure_volume(
all_blobs[key].name,
all_blobs[key].properties.content_length,
None))
return disk_list
def _attach_disk(
self,
blockdevice_id,
target_disk,
attach_to):
"""
Attaches disk to specified VM
:param string blockdevice_id: The identifier of the disk
:param DataVirtualHardDisk/Blob target_disk: The Blob
or Disk to be attached
:returns int: The size of the attached disk
"""
lun = Lun.compute_next_lun(
self._azure_service_client,
self._service_name,
str(attach_to))
common_params = {
'service_name': self._service_name,
'deployment_name': self._service_name,
'role_name': attach_to,
'lun': lun
}
disk_size = None
if target_disk.__class__.__name__ == 'Blob':
# exclude 512 byte footer
disk_size = target_disk.properties.content_length
common_params['source_media_link'] = \
'https://' + self._storage_account_name \
+ '.blob.core.windows.net/' + self._disk_container_name \
+ '/' + blockdevice_id
common_params['disk_label'] = blockdevice_id
else:
disk_size = self._gibytes_to_bytes(
target_disk.logical_disk_size_in_gb)
common_params['disk_name'] = target_disk.name
request = self._azure_service_client.add_data_disk(**common_params)
self._wait_for_async(request.request_id, 5000)
return disk_size
def _create_volume_blob(self, size, dataset_id):
# Create a new page blob as a blank disk
self._azure_storage_client.put_blob(
container_name=self._disk_container_name,
blob_name=self._disk_label_for_dataset_id(dataset_id),
blob=None,
x_ms_blob_type='PageBlob',
x_ms_blob_content_type='application/octet-stream',
x_ms_blob_content_length=size)
# for disk to be a valid vhd it requires a vhd footer
# on the last 512 bytes
vhd_footer = Vhd.generate_vhd_footer(size)
self._azure_storage_client.put_page(
container_name=self._disk_container_name,
blob_name=self._disk_label_for_dataset_id(dataset_id),
page=vhd_footer,
x_ms_page_write='update',
x_ms_range='bytes=' + str((size - 512)) + '-' + str(size - 1))
def _disk_label_for_dataset_id(self, dataset_id):
"""
Returns a disk label for a given Dataset ID
:param unicode dataset_id: The identifier of the dataset
:returns string: A string representing the disk label
"""
label = 'flocker-' + str(dataset_id)
return label
def _dataset_id_for_disk_label(self, disk_label):
"""
Returns a UUID representing the Dataset ID for the given disk
label
:param string disk_label: The disk label
:returns UUID: The UUID of the dataset
"""
return UUID(disk_label.replace('flocker-', ''))
def _get_disk_vmname_lun(self, blockdevice_id):
target_disk = None
target_lun = None
role_name = None
disk_list = self._azure_service_client.list_disks()
for d in disk_list:
if 'flocker-' not in d.label:
continue
if d.label == str(blockdevice_id):
target_disk = d
break
if target_disk is None:
# check for unregisterd disk
blobs = self._get_flocker_blobs()
blob = None
if str(blockdevice_id) in blobs:
blob = blobs[str(blockdevice_id)]
return blob, None, None
vm_info = None
if hasattr(target_disk.attached_to, 'role_name'):
vm_info = self._azure_service_client.get_role(
self._service_name, self._service_name,
target_disk.attached_to.role_name)
for d in vm_info.data_virtual_hard_disks:
if d.disk_name == target_disk.name:
target_lun = d.lun
break
role_name = target_disk.attached_to.role_name
return (target_disk, role_name, target_lun)
def _get_flocker_blobs(self):
all_blobs = {}
blobs = self._azure_storage_client.list_blobs(
self._disk_container_name,
prefix='flocker-')
for b in blobs:
# todo - this could be big!
all_blobs[b.name] = b
return all_blobs
def _wait_for_detach(self, blockdevice_id):
role_name = ''
lun = -1
timeout_count = 0
log_info('waiting for azure to ' + 'report disk as detached...')
while role_name is not None or lun is not None:
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
time.sleep(1)
timeout_count += 1
if timeout_count > 5000:
raise AsynchronousTimeout()
log_info('Disk Detached')
def _wait_for_attach(self, blockdevice_id):
timeout_count = 0
lun = None
log_info('waiting for azure to report disk as attached...')
while lun is None:
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
time.sleep(.001)
timeout_count += 1
if timeout_count > 5000:
raise AsynchronousTimeout()
def _wait_for_async(self, request_id, timeout):
count = 0
result = self._azure_service_client.get_operation_status(request_id)
while result.status == 'InProgress':
count = count + 1
if count > timeout:
log_error('Timed out waiting for async operation to complete.')
raise AsynchronousTimeout()
time.sleep(.001)
log_info('.')
result = self._azure_service_client.get_operation_status(
request_id)
if result.error:
log_error(result.error.code)
log_error(str(result.error.message))
log_error(result.status + ' in ' + str(count * 5) + 's')
def _gibytes_to_bytes(self, size):
return int(GiB(size).to_Byte().value)
def _blockdevicevolume_from_azure_volume(self, label, size,
attached_to_name):
# azure will report the disk size excluding the 512 byte footer
# however flocker expects the exact value it requested for disk size
# so offset the reported size to flocker by 512 bytes
return BlockDeviceVolume(
blockdevice_id=unicode(label),
size=int(size),
attached_to=attached_to_name,
dataset_id=self._dataset_id_for_disk_label(label)
) # disk labels are formatted as flocker-<data_set_id>
class AzureServicesManager:
# Storage
container = 'vhds'
windows_blob_url = 'blob.core.windows.net'
# Linux
linux_user = '******'
linux_pass = '******'
location = 'West US'
# SSH Keys
def __init__(self, subscription_id, cert_file):
self.subscription_id = subscription_id
self.cert_file = cert_file
self.sms = ServiceManagementService(self.subscription_id,
self.cert_file)
@property
def sms(self):
return self.sms
def list_locations(self):
locations = self.sms.list_locations()
for location in locations:
print location
def list_images(self):
return self.sms.list_os_images()
@utils.resource_not_found_handler
def get_hosted_service(self, service_name):
resp = self.sms.get_hosted_service_properties(service_name)
properties = resp.hosted_service_properties
return properties.__dict__
def delete_hosted_service(self, service_name):
res = self.sms.check_hosted_service_name_availability(service_name)
if not res.result:
return
self.sms.delete_hosted_service(service_name)
def create_hosted_service(self, os_user, service_name=None, random=False):
if not service_name:
service_name = self.generate_cloud_service_name(os_user, random)
available = False
while not available:
res = self.sms.check_hosted_service_name_availability(service_name)
if not res.result:
service_name = self.generate_cloud_service_name(
os_user, random)
else:
available = True
self.sms.create_hosted_service(service_name=service_name,
label=service_name,
location='West US')
return service_name
def create_virtual_machine(self, service_name, vm_name, image_name,
role_size):
media_link = self._get_media_link(vm_name)
# Linux VM configuration
hostname = '-'.join((vm_name, 'host'))
linux_config = LinuxConfigurationSet(hostname, self.linux_user,
self.linux_pass, True)
# Hard disk for the OS
os_hd = OSVirtualHardDisk(image_name, media_link)
# Create vm
result = self.sms.create_virtual_machine_deployment(
service_name=service_name,
deployment_name=vm_name,
deployment_slot='production',
label=vm_name,
role_name=vm_name,
system_config=linux_config,
os_virtual_hard_disk=os_hd,
role_size=role_size)
request_id = result.request_id
return {'request_id': request_id, 'media_link': media_link}
def delete_virtual_machine(self, service_name, vm_name):
resp = self.sms.delete_deployment(service_name, vm_name, True)
self.sms.wait_for_operation_status(resp.request_id)
result = self.sms.delete_hosted_service(service_name)
return result
def generate_cloud_service_name(self, os_user=None, random=False):
if random:
return utils.generate_random_name(10)
return '-'.join((os_user, utils.generate_random_name(6)))
@utils.resource_not_found_handler
def get_virtual_machine_info(self, service_name, vm_name):
vm_info = {}
deploy_info = self.sms.get_deployment_by_name(service_name, vm_name)
if deploy_info and deploy_info.role_instance_list:
vm_info = deploy_info.role_instance_list[0].__dict__
return vm_info
def list_virtual_machines(self):
vm_list = []
services = self.sms.list_hosted_services()
for service in services:
deploys = service.deployments
if deploys and deploys.role_instance_list:
vm_name = deploys.role_instance_list[0].instance_name
vm_list.append(vm_name)
return vm_list
def power_on(self, service_name, vm_name):
resp = self.sms.start_role(service_name, vm_name, vm_name)
return resp.request_id
def power_off(self, service_name, vm_name):
resp = self.sms.shutdown_role(service_name, vm_name, vm_name)
return resp.request_id
def soft_reboot(self, service_name, vm_name):
resp = self.sms.restart_role(service_name, vm_name, vm_name)
return resp.request_id
def hard_reboot(self, service_name, vm_name):
resp = self.sms.reboot_role_instance(service_name, vm_name, vm_name)
return resp.request_id
def attach_volume(self, service_name, vm_name, size, lun):
disk_name = utils.generate_random_name(5, vm_name)
media_link = self._get_media_link(vm_name, disk_name)
self.sms.add_data_disk(service_name,
vm_name,
vm_name,
lun,
host_caching='ReadWrite',
media_link=media_link,
disk_name=disk_name,
logical_disk_size_in_gb=size)
def detach_volume(self, service_name, vm_name, lun):
self.sms.delete_data_disk(service_name, vm_name, vm_name, lun, True)
def get_available_lun(self, service_name, vm_name):
try:
role = self.sms.get_role(service_name, vm_name, vm_name)
except Exception:
return 0
disks = role.data_virtual_hard_disks
luns = [disk.lun for disk in disks].sort()
for i in range(1, 16):
if i not in luns:
return i
return None
def snapshot(self, service_name, vm_name, image_id, snanshot_name):
image_desc = 'Snapshot for image %s' % vm_name
image = CaptureRoleAsVMImage('Specialized', snanshot_name, image_id,
image_desc, 'english')
resp = self.sms.capture_vm_image(service_name, vm_name, vm_name, image)
self.sms.wait_for_operation_status(resp.request_id)
def _get_media_link(self, vm_name, filename=None, storage_account=None):
""" The MediaLink should be constructed as:
https://<storageAccount>.<blobLink>/<blobContainer>/<filename>.vhd
"""
if not storage_account:
storage_account = self._get_or_create_storage_account()
container = self.container
filename = vm_name if filename is None else filename
blob = vm_name + '-' + filename + '.vhd'
media_link = "http://%s.%s/%s/%s" % (
storage_account, self.windows_blob_url, container, blob)
return media_link
def _get_or_create_storage_account(self):
account_list = self.sms.list_storage_accounts()
if account_list:
return account_list[-1].service_name
storage_account = utils.generate_random_name(10)
description = "Storage account %s description" % storage_account
label = storage_account + 'label'
self.sms.create_storage_account(storage_account,
description,
label,
location=self.location)
return storage_account
def _wait_for_operation(self,
request_id,
timeout=3000,
failure_callback=None,
failure_callback_kwargs=None):
try:
self.sms.wait_for_operation_status(request_id, timeout=timeout)
except Exception as ex:
if failure_callback and failure_callback_kwargs:
failure_callback(**failure_callback_kwargs)
raise ex
class AzureStorageBlockDeviceAPI(object):
"""
An ``IBlockDeviceAsyncAPI`` which uses Azure Storage Backed Block Devices
Current Support: Azure SMS API
"""
def __init__(self, **azure_config):
"""
:param ServiceManagement azure_client: an instance of the azure
serivce managment api client.
:param String service_name: The name of the cloud service
:param
names of Azure volumes to identify cluster
:returns: A ``BlockDeviceVolume``.
"""
self._instance_id = self.compute_instance_id()
self._azure_service_client = ServiceManagementService(
azure_config['subscription_id'],
azure_config['management_certificate_path'])
self._service_name = azure_config['service_name']
self._azure_storage_client = BlobService(
azure_config['storage_account_name'],
azure_config['storage_account_key'])
self._storage_account_name = azure_config['storage_account_name']
self._disk_container_name = azure_config['disk_container_name']
if azure_config['debug']:
to_file(sys.stdout)
def allocation_unit(self):
"""
1GiB is the minimum allocation unit for azure disks
return int: 1 GiB
"""
return int(GiB(1).to_Byte().value)
def compute_instance_id(self):
"""
Azure Stored a UUID in the SDC kernel module.
"""
# Node host names should be unique within a vnet
return unicode(socket.gethostname())
def create_volume(self, dataset_id, size):
"""
Create a new volume.
:param UUID dataset_id: The Flocker dataset ID of the dataset on this
volume.
:param int size: The size of the new volume in bytes.
:returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when
the volume has been created.
"""
size_in_gb = Byte(size).to_GiB().value
if size_in_gb % 1 != 0:
raise UnsupportedVolumeSize(dataset_id)
self._create_volume_blob(size, dataset_id)
label = self._disk_label_for_dataset_id(str(dataset_id))
return BlockDeviceVolume(
blockdevice_id=unicode(label),
size=size,
attached_to=None,
dataset_id=self._dataset_id_for_disk_label(label))
def destroy_volume(self, blockdevice_id):
"""
Destroy an existing volume.
:param unicode blockdevice_id: The unique identifier for the volume to
destroy.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:return: ``None``
"""
log_info('Destorying block device: ' + str(blockdevice_id))
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
request = None
if lun is not None:
request = \
self._azure_service_client.delete_data_disk(
service_name=self._service_name,
deployment_name=self._service_name,
role_name=target_disk.attached_to.role_name,
lun=lun,
delete_vhd=True)
else:
if target_disk.__class__.__name__ == 'Blob':
# unregistered disk
self._azure_storage_client.delete_blob(
self._disk_container_name, target_disk.name)
else:
request = self._azure_service_client.delete_disk(
target_disk.name, True)
if request is not None:
self._wait_for_async(request.request_id, 5000)
self._wait_for_detach(blockdevice_id)
def attach_volume(self, blockdevice_id, attach_to):
"""
Attach ``blockdevice_id`` to ``host``.
:param unicode blockdevice_id: The unique identifier for the block
device being attached.
:param unicode attach_to: An identifier like the one returned by the
``compute_instance_id`` method indicating the node to which to
attach the volume.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is
already attached.
:returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to
``host``.
"""
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
if lun is not None:
raise AlreadyAttachedVolume(blockdevice_id)
log_info('Attempting to attach ' + str(blockdevice_id) + ' to ' +
str(attach_to))
disk_size = self._attach_disk(blockdevice_id, target_disk, attach_to)
self._wait_for_attach(blockdevice_id)
log_info('disk attached')
return self._blockdevicevolume_from_azure_volume(
blockdevice_id, disk_size, attach_to)
def detach_volume(self, blockdevice_id):
"""
Detach ``blockdevice_id`` from whatever host it is attached to.
:param unicode blockdevice_id: The unique identifier for the block
device being detached.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises UnattachedVolume: If the supplied ``blockdevice_id`` is
not attached to anything.
:returns: ``None``
"""
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
if lun is None:
raise UnattachedVolume(blockdevice_id)
# contrary to function name it doesn't delete by default, just detachs
request = \
self._azure_service_client.delete_data_disk(
service_name=self._service_name,
deployment_name=self._service_name,
role_name=role_name, lun=lun)
self._wait_for_async(request.request_id, 5000)
self._wait_for_detach(blockdevice_id)
def get_device_path(self, blockdevice_id):
"""
Return the device path that has been allocated to the block device on
the host to which it is currently attached.
:param unicode blockdevice_id: The unique identifier for the block
device.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises UnattachedVolume: If the supplied ``blockdevice_id`` is
not attached to a host.
:returns: A ``FilePath`` for the device.
"""
(target_disk_or_blob, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk_or_blob is None:
raise UnknownVolume(blockdevice_id)
if lun is None:
raise UnattachedVolume(blockdevice_id)
return Lun.get_device_path_for_lun(lun)
def list_volumes(self):
"""
List all the block devices available via the back end API.
:returns: A ``list`` of ``BlockDeviceVolume``s.
"""
media_url_prefix = 'https://' + self._storage_account_name \
+ '.blob.core.windows.net/' + self._disk_container_name
disks = self._azure_service_client.list_disks()
disk_list = []
all_blobs = self._get_flocker_blobs()
for d in disks:
if media_url_prefix not in d.media_link or \
'flocker-' not in d.label:
continue
role_name = None
if d.attached_to is not None \
and d.attached_to.role_name is not None:
role_name = d.attached_to.role_name
disk_list.append(
self._blockdevicevolume_from_azure_volume(
d.label, self._gibytes_to_bytes(d.logical_disk_size_in_gb),
role_name))
if d.label in all_blobs:
del all_blobs[d.label]
for key in all_blobs:
# include unregistered 'disk' blobs
disk_list.append(
self._blockdevicevolume_from_azure_volume(
all_blobs[key].name,
all_blobs[key].properties.content_length, None))
return disk_list
def _attach_disk(self, blockdevice_id, target_disk, attach_to):
"""
Attaches disk to specified VM
:param string blockdevice_id: The identifier of the disk
:param DataVirtualHardDisk/Blob target_disk: The Blob
or Disk to be attached
:returns int: The size of the attached disk
"""
lun = Lun.compute_next_lun(self._azure_service_client,
self._service_name, str(attach_to))
common_params = {
'service_name': self._service_name,
'deployment_name': self._service_name,
'role_name': attach_to,
'lun': lun
}
disk_size = None
if target_disk.__class__.__name__ == 'Blob':
# exclude 512 byte footer
disk_size = target_disk.properties.content_length
common_params['source_media_link'] = \
'https://' + self._storage_account_name \
+ '.blob.core.windows.net/' + self._disk_container_name \
+ '/' + blockdevice_id
common_params['disk_label'] = blockdevice_id
else:
disk_size = self._gibytes_to_bytes(
target_disk.logical_disk_size_in_gb)
common_params['disk_name'] = target_disk.name
request = self._azure_service_client.add_data_disk(**common_params)
self._wait_for_async(request.request_id, 5000)
return disk_size
def _create_volume_blob(self, size, dataset_id):
# Create a new page blob as a blank disk
self._azure_storage_client.put_blob(
container_name=self._disk_container_name,
blob_name=self._disk_label_for_dataset_id(dataset_id),
blob=None,
x_ms_blob_type='PageBlob',
x_ms_blob_content_type='application/octet-stream',
x_ms_blob_content_length=size)
# for disk to be a valid vhd it requires a vhd footer
# on the last 512 bytes
vhd_footer = Vhd.generate_vhd_footer(size)
self._azure_storage_client.put_page(
container_name=self._disk_container_name,
blob_name=self._disk_label_for_dataset_id(dataset_id),
page=vhd_footer,
x_ms_page_write='update',
x_ms_range='bytes=' + str((size - 512)) + '-' + str(size - 1))
def _disk_label_for_dataset_id(self, dataset_id):
"""
Returns a disk label for a given Dataset ID
:param unicode dataset_id: The identifier of the dataset
:returns string: A string representing the disk label
"""
label = 'flocker-' + str(dataset_id)
return label
def _dataset_id_for_disk_label(self, disk_label):
"""
Returns a UUID representing the Dataset ID for the given disk
label
:param string disk_label: The disk label
:returns UUID: The UUID of the dataset
"""
return UUID(disk_label.replace('flocker-', ''))
def _get_disk_vmname_lun(self, blockdevice_id):
target_disk = None
target_lun = None
role_name = None
disk_list = self._azure_service_client.list_disks()
for d in disk_list:
if 'flocker-' not in d.label:
continue
if d.label == str(blockdevice_id):
target_disk = d
break
if target_disk is None:
# check for unregisterd disk
blobs = self._get_flocker_blobs()
blob = None
if str(blockdevice_id) in blobs:
blob = blobs[str(blockdevice_id)]
return blob, None, None
vm_info = None
if hasattr(target_disk.attached_to, 'role_name'):
vm_info = self._azure_service_client.get_role(
self._service_name, self._service_name,
target_disk.attached_to.role_name)
for d in vm_info.data_virtual_hard_disks:
if d.disk_name == target_disk.name:
target_lun = d.lun
break
role_name = target_disk.attached_to.role_name
return (target_disk, role_name, target_lun)
def _get_flocker_blobs(self):
all_blobs = {}
blobs = self._azure_storage_client.list_blobs(
self._disk_container_name, prefix='flocker-')
for b in blobs:
# todo - this could be big!
all_blobs[b.name] = b
return all_blobs
def _wait_for_detach(self, blockdevice_id):
role_name = ''
lun = -1
timeout_count = 0
log_info('waiting for azure to ' + 'report disk as detached...')
while role_name is not None or lun is not None:
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
time.sleep(1)
timeout_count += 1
if timeout_count > 5000:
raise AsynchronousTimeout()
log_info('Disk Detached')
def _wait_for_attach(self, blockdevice_id):
timeout_count = 0
lun = None
log_info('waiting for azure to report disk as attached...')
while lun is None:
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
time.sleep(.001)
timeout_count += 1
if timeout_count > 5000:
raise AsynchronousTimeout()
def _wait_for_async(self, request_id, timeout):
count = 0
result = self._azure_service_client.get_operation_status(request_id)
while result.status == 'InProgress':
count = count + 1
if count > timeout:
log_error('Timed out waiting for async operation to complete.')
raise AsynchronousTimeout()
time.sleep(.001)
log_info('.')
result = self._azure_service_client.get_operation_status(
request_id)
if result.error:
log_error(result.error.code)
log_error(str(result.error.message))
log_error(result.status + ' in ' + str(count * 5) + 's')
def _gibytes_to_bytes(self, size):
return int(GiB(size).to_Byte().value)
def _blockdevicevolume_from_azure_volume(self, label, size,
attached_to_name):
# azure will report the disk size excluding the 512 byte footer
# however flocker expects the exact value it requested for disk size
# so offset the reported size to flocker by 512 bytes
return BlockDeviceVolume(
blockdevice_id=unicode(label),
size=int(size),
attached_to=attached_to_name,
dataset_id=self._dataset_id_for_disk_label(
label)) # disk labels are formatted as flocker-<data_set_id>