class StorageManagementServiceTest(AzureTestCase):
def setUp(self):
proxy_host = credentials.getProxyHost()
proxy_port = credentials.getProxyPort()
self.sms = ServiceManagementService(credentials.getSubscriptionId(), credentials.getManagementCertFile())
if proxy_host:
self.sms.set_proxy(proxy_host, proxy_port)
self.storage_account_name = getUniqueNameBasedOnCurrentTime('utstorage')
def tearDown(self):
try:
self.sms.delete_storage_account(self.storage_account_name)
except: pass
#--Helpers-----------------------------------------------------------------
def _wait_for_async(self, request_id):
count = 0
result = self.sms.get_operation_status(request_id)
while result.status == 'InProgress':
count = count + 1
if count > 120:
self.assertTrue(False, 'Timed out waiting for async operation to complete.')
time.sleep(5)
result = self.sms.get_operation_status(request_id)
self.assertEqual(result.status, 'Succeeded')
def _create_storage_account(self, name):
result = self.sms.create_storage_account(name, name + 'description', name + 'label', None, 'West US', False, {'ext1':'val1', 'ext2':42})
self._wait_for_async(result.request_id)
def _storage_account_exists(self, name):
try:
props = self.sms.get_storage_account_properties(name)
return props is not None
except:
return False
#--Test cases for storage accounts -----------------------------------
def test_list_storage_accounts(self):
# Arrange
self._create_storage_account(self.storage_account_name)
# Act
result = self.sms.list_storage_accounts()
# Assert
self.assertIsNotNone(result)
self.assertTrue(len(result) > 0)
storage = None
for temp in result:
if temp.service_name == self.storage_account_name:
storage = temp
break
self.assertIsNotNone(storage)
self.assertIsNotNone(storage.service_name)
self.assertIsNone(storage.storage_service_keys)
self.assertIsNotNone(storage.storage_service_properties)
self.assertIsNotNone(storage.storage_service_properties.affinity_group)
self.assertIsNotNone(storage.storage_service_properties.description)
self.assertIsNotNone(storage.storage_service_properties.geo_primary_region)
self.assertIsNotNone(storage.storage_service_properties.geo_replication_enabled)
self.assertIsNotNone(storage.storage_service_properties.geo_secondary_region)
self.assertIsNotNone(storage.storage_service_properties.label)
self.assertIsNotNone(storage.storage_service_properties.last_geo_failover_time)
self.assertIsNotNone(storage.storage_service_properties.location)
self.assertIsNotNone(storage.storage_service_properties.status)
self.assertIsNotNone(storage.storage_service_properties.status_of_primary)
self.assertIsNotNone(storage.storage_service_properties.status_of_secondary)
self.assertIsNotNone(storage.storage_service_properties.endpoints)
self.assertTrue(len(storage.storage_service_properties.endpoints) > 0)
self.assertIsNotNone(storage.extended_properties)
self.assertTrue(len(storage.extended_properties) > 0)
def test_get_storage_account_properties(self):
# Arrange
self._create_storage_account(self.storage_account_name)
# Act
result = self.sms.get_storage_account_properties(self.storage_account_name)
# Assert
self.assertIsNotNone(result)
self.assertEqual(result.service_name, self.storage_account_name)
self.assertIsNotNone(result.url)
self.assertIsNone(result.storage_service_keys)
self.assertIsNotNone(result.storage_service_properties)
self.assertIsNotNone(result.storage_service_properties.affinity_group)
self.assertIsNotNone(result.storage_service_properties.description)
self.assertIsNotNone(result.storage_service_properties.geo_primary_region)
self.assertIsNotNone(result.storage_service_properties.geo_replication_enabled)
self.assertIsNotNone(result.storage_service_properties.geo_secondary_region)
self.assertIsNotNone(result.storage_service_properties.label)
self.assertIsNotNone(result.storage_service_properties.last_geo_failover_time)
self.assertIsNotNone(result.storage_service_properties.location)
self.assertIsNotNone(result.storage_service_properties.status)
self.assertIsNotNone(result.storage_service_properties.status_of_primary)
self.assertIsNotNone(result.storage_service_properties.status_of_secondary)
self.assertIsNotNone(result.storage_service_properties.endpoints)
self.assertTrue(len(result.storage_service_properties.endpoints) > 0)
self.assertIsNotNone(result.extended_properties)
self.assertTrue(len(result.extended_properties) > 0)
self.assertIsNotNone(result.capabilities)
self.assertTrue(len(result.capabilities) > 0)
def test_get_storage_account_keys(self):
# Arrange
self._create_storage_account(self.storage_account_name)
# Act
result = self.sms.get_storage_account_keys(self.storage_account_name)
# Assert
self.assertIsNotNone(result)
self.assertIsNotNone(result.url)
self.assertIsNotNone(result.service_name)
self.assertIsNotNone(result.storage_service_keys.primary)
self.assertIsNotNone(result.storage_service_keys.secondary)
self.assertIsNone(result.storage_service_properties)
def test_regenerate_storage_account_keys(self):
# Arrange
self._create_storage_account(self.storage_account_name)
previous = self.sms.get_storage_account_keys(self.storage_account_name)
# Act
result = self.sms.regenerate_storage_account_keys(self.storage_account_name, 'Secondary')
# Assert
self.assertIsNotNone(result)
self.assertIsNotNone(result.url)
self.assertIsNotNone(result.service_name)
self.assertIsNotNone(result.storage_service_keys.primary)
self.assertIsNotNone(result.storage_service_keys.secondary)
self.assertIsNone(result.storage_service_properties)
self.assertEqual(result.storage_service_keys.primary, previous.storage_service_keys.primary)
self.assertNotEqual(result.storage_service_keys.secondary, previous.storage_service_keys.secondary)
def test_create_storage_account(self):
# Arrange
description = self.storage_account_name + 'description'
label = self.storage_account_name + 'label'
# Act
result = self.sms.create_storage_account(self.storage_account_name, description, label, None, 'West US', True, {'ext1':'val1', 'ext2':42})
self._wait_for_async(result.request_id)
# Assert
self.assertTrue(self._storage_account_exists(self.storage_account_name))
def test_update_storage_account(self):
# Arrange
self._create_storage_account(self.storage_account_name)
description = self.storage_account_name + 'descriptionupdate'
label = self.storage_account_name + 'labelupdate'
# Act
result = self.sms.update_storage_account(self.storage_account_name, description, label, False, {'ext1':'val1update', 'ext2':53, 'ext3':'brandnew'})
# Assert
self.assertIsNone(result)
props = self.sms.get_storage_account_properties(self.storage_account_name)
self.assertEqual(props.storage_service_properties.description, description)
self.assertEqual(props.storage_service_properties.label, label)
self.assertEqual(props.extended_properties['ext1'], 'val1update')
self.assertEqual(props.extended_properties['ext2'], '53')
self.assertEqual(props.extended_properties['ext3'], 'brandnew')
def test_delete_storage_account(self):
# Arrange
self._create_storage_account(self.storage_account_name)
# Act
result = self.sms.delete_storage_account(self.storage_account_name)
# Assert
self.assertIsNone(result)
self.assertFalse(self._storage_account_exists(self.storage_account_name))
def test_check_storage_account_name_availability_not_available(self):
# Arrange
self._create_storage_account(self.storage_account_name)
# Act
result = self.sms.check_storage_account_name_availability(self.storage_account_name)
# Assert
self.assertIsNotNone(result)
self.assertFalse(result.result)
def test_check_storage_account_name_availability_available(self):
# Arrange
# Act
result = self.sms.check_storage_account_name_availability(self.storage_account_name)
# Assert
self.assertIsNotNone(result)
self.assertTrue(result.result)
def test_unicode_create_storage_account_unicode_name(self):
# Arrange
self.storage_account_name = unicode(self.storage_account_name) + u'啊齄丂狛狜'
description = 'description'
label = 'label'
# Act
with self.assertRaises(WindowsAzureError):
# not supported - queue name must be alphanumeric, lowercase
result = self.sms.create_storage_account(self.storage_account_name, description, label, None, 'West US', True, {'ext1':'val1', 'ext2':42})
self._wait_for_async(result.request_id)
# Assert
def test_unicode_create_storage_account_unicode_description_label(self):
# Arrange
description = u'啊齄丂狛狜'
label = u'丂狛狜'
# Act
result = self.sms.create_storage_account(self.storage_account_name, description, label, None, 'West US', True, {'ext1':'val1', 'ext2':42})
self._wait_for_async(result.request_id)
# Assert
result = self.sms.get_storage_account_properties(self.storage_account_name)
self.assertEqual(result.storage_service_properties.description, description)
self.assertEqual(result.storage_service_properties.label, label)
def test_unicode_create_storage_account_unicode_property_value(self):
# Arrange
description = 'description'
label = 'label'
# Act
result = self.sms.create_storage_account(self.storage_account_name, description, label, None, 'West US', True, {'ext1':u'丂狛狜', 'ext2':42})
self._wait_for_async(result.request_id)
# Assert
result = self.sms.get_storage_account_properties(self.storage_account_name)
self.assertEqual(result.storage_service_properties.description, description)
self.assertEqual(result.storage_service_properties.label, label)
self.assertEqual(result.extended_properties['ext1'], u'丂狛狜')
class Deployment(object):
"""
Helper class to handle deployment of the web site.
"""
def __init__(self, config):
self.config = config
self.sms = ServiceManagementService(config.getAzureSubscriptionId(),
config.getAzureCertificatePath())
self.sbms = ServiceBusManagementService(
config.getAzureSubscriptionId(), config.getAzureCertificatePath())
@staticmethod
def _resource_exists(get_resource):
"""
Helper to check for the existence of a resource in Azure.
get_resource: Parameter-less function to invoke in order to get the resource. The resource
is assumed to exist when the call to get_resource() returns a value that is not None.
If the call to get_resource() returns None or throws a WindowsAzureMissingResourceError
exception, then it is assumed that the resource does not exist.
Returns: A boolean value which is True if the resource exists.
"""
resource = None
try:
resource = get_resource()
except WindowsAzureMissingResourceError:
pass
return resource is not None
def _wait_for_operation_success(self, request_id, timeout=600, wait=5):
"""
Waits for an asynchronous Azure operation to finish.
request_id: The ID of the request to track.
timeout: Maximum duration (in seconds) allowed for the operation to complete.
wait: Wait time (in seconds) between consecutive calls to fetch the latest operation status.
"""
result = self.sms.get_operation_status(request_id)
start_time = time.time()
max_time = start_time + timeout
now = start_time
while result.status == 'InProgress':
if now >= max_time:
raise Exception(
"Operation did not finish within the expected timeout")
logger.info(
'Waiting for operation to finish (last_status=%s wait_so_far=%s)',
result.status, round(now - start_time, 1))
time_to_wait = max(0.0, min(max_time - now, wait))
time.sleep(time_to_wait)
result = self.sms.get_operation_status(request_id)
now = time.time()
if result.status != 'Succeeded':
raise Exception("Operation terminated but it did not succeed.")
def _wait_for_role_instance_status(self,
role_instance_name,
service_name,
expected_status,
timeout=600,
wait=5):
"""
Waits for a role instance within the web site's cloud service to reach the status specified.
role_instance_name: Name of the role instance.
service_name: Name of service in which to find the role instance.
expected_status: Expected instance status.
timeout: Maximum duration (in seconds) allowed for the operation to complete.
wait: Wait time (in seconds) between consecutive calls to fetch the latest role status.
"""
start_time = time.time()
max_time = start_time + timeout
now = start_time
while True:
status = None
deployment = self.sms.get_deployment_by_name(
service_name, service_name)
for role_instance in deployment.role_instance_list:
if role_instance.instance_name == role_instance_name:
status = role_instance.instance_status
if status == expected_status:
break
if now >= max_time:
raise Exception(
"Operation did not finish within the expected timeout")
logger.info(
'Waiting for deployment status: expecting %s but got %s (wait_so_far=%s)',
expected_status, status, round(now - start_time, 1))
time_to_wait = max(0.0, min(max_time - now, wait))
time.sleep(time_to_wait)
now = time.time()
def _wait_for_disk_deletion(self, disk_name, timeout=600, wait=5):
"""
Waits for a VM disk to disappear when it is being deleted.
disk_name: Name of the VHD.
timeout: Maximum duration (in seconds) allowed for the operation to complete.
wait: Wait time (in seconds) between consecutive calls to check for the existence of the disk.
"""
start_time = time.time()
max_time = start_time + timeout
now = start_time
logger.info("Checking that disk %s has been deleted.", disk_name)
while self._resource_exists(lambda: self.sms.get_disk(disk_name)):
if now >= max_time:
raise Exception(
"Disk %s was not deleted within the expected timeout.".
format(disk_name))
logger.info("Waiting for disk %s to disappear (wait_so_far=%s).",
disk_name, round(now - start_time, 1))
time_to_wait = max(0.0, min(max_time - now, wait))
time.sleep(time_to_wait)
now = time.time()
logger.info("Disk %s has been deleted.", disk_name)
def _wait_for_namespace_active(self, name, timeout=600, wait=5):
"""
Waits for a service bus namespace to become Active.
name: Namespace name.
timeout: Maximum duration (in seconds) allowed for the operation to complete.
wait: Wait time (in seconds) between consecutive calls to check for the existence of the disk.
"""
start_time = time.time()
max_time = start_time + timeout
now = start_time
while True:
status = None
props = self.sbms.get_namespace(name)
status = props.status
if status == 'Active':
break
if now >= max_time:
raise Exception(
"Operation did not finish within the expected timeout")
logger.info(
'Waiting for namepsace status: expecting Active but got %s (wait_so_far=%s)',
status, round(now - start_time, 1))
time_to_wait = max(0.0, min(max_time - now, wait))
time.sleep(time_to_wait)
now = time.time()
def _getRoleInstances(self, service_name):
"""
Returns the role instances in the given cloud service deployment. The results are provided as
a dictionary where keys are role instance names and values are RoleInstance objects.
"""
role_instances = {}
if self._resource_exists(lambda: self.sms.get_deployment_by_name(
service_name, service_name)):
deployment = self.sms.get_deployment_by_name(
service_name, service_name)
for role_instance in deployment.role_instance_list:
role_instances[role_instance.instance_name] = role_instance
return role_instances
def _ensureAffinityGroupExists(self):
"""
Creates the affinity group if it does not exist.
"""
name = self.config.getAffinityGroupName()
location = self.config.getServiceLocation()
logger.info(
"Checking for existence of affinity group (name=%s; location=%s).",
name, location)
if self._resource_exists(
lambda: self.sms.get_affinity_group_properties(name)):
logger.warn("An affinity group named %s already exists.", name)
else:
self.sms.create_affinity_group(name, name, location)
logger.info("Created affinity group %s.", name)
def _ensureStorageAccountExists(self, name):
"""
Creates the storage account if it does not exist.
"""
logger.info("Checking for existence of storage account (name=%s).",
name)
if self._resource_exists(
lambda: self.sms.get_storage_account_properties(name)):
logger.warn("A storage account named %s already exists.", name)
else:
result = self.sms.create_storage_account(
name,
"",
name,
affinity_group=self.config.getAffinityGroupName())
self._wait_for_operation_success(
result.request_id,
timeout=self.config.getAzureOperationTimeout())
logger.info("Created storage account %s.", name)
def _getStorageAccountKey(self, account_name):
"""
Gets the storage account key (primary key) for the given storage account.
"""
storage_props = self.sms.get_storage_account_keys(account_name)
return storage_props.storage_service_keys.primary
def _ensureStorageContainersExist(self):
"""
Creates Blob storage containers required by the service.
"""
logger.info("Checking for existence of Blob containers.")
account_name = self.config.getServiceStorageAccountName()
account_key = self._getStorageAccountKey(account_name)
blob_service = BlobService(account_name, account_key)
name_and_access_list = [
(self.config.getServicePublicStorageContainer(), 'blob'),
(self.config.getServiceBundleStorageContainer(), None)
]
for name, access in name_and_access_list:
logger.info("Checking for existence of Blob container %s.", name)
blob_service.create_container(name,
x_ms_blob_public_access=access,
fail_on_exist=False)
access_info = 'private' if access is None else 'public {0}'.format(
access)
logger.info("Blob container %s is ready (access: %s).", name,
access_info)
def ensureStorageHasCorsConfiguration(self):
"""
Ensures Blob storage container for bundles is configured to allow cross-origin resource sharing.
"""
logger.info("Setting CORS rules.")
account_name = self.config.getServiceStorageAccountName()
account_key = self._getStorageAccountKey(account_name)
cors_rule = CorsRule()
cors_rule.allowed_origins = self.config.getServiceStorageCorsAllowedOrigins(
)
cors_rule.allowed_methods = 'PUT'
cors_rule.exposed_headers = '*'
cors_rule.allowed_headers = '*'
cors_rule.max_age_in_seconds = 1800
cors_rules = Cors()
cors_rules.cors_rule.append(cors_rule)
set_storage_service_cors_properties(account_name, account_key,
cors_rules)
def _ensureServiceExists(self, service_name, affinity_group_name):
"""
Creates the specified cloud service host if it does not exist.
service_name: Name of the cloud service.
affinity_group_name: Name of the affinity group (which should exists).
"""
logger.info("Checking for existence of cloud service (name=%s).",
service_name)
if self._resource_exists(
lambda: self.sms.get_hosted_service_properties(service_name)):
logger.warn("A cloud service named %s already exists.",
service_name)
else:
self.sms.create_hosted_service(service_name,
service_name,
affinity_group=affinity_group_name)
logger.info("Created cloud service %s.", service_name)
def _ensureServiceCertificateExists(self, service_name):
"""
Adds certificate to the specified cloud service.
service_name: Name of the target cloud service (which should exist).
"""
cert_format = self.config.getServiceCertificateFormat()
cert_algorithm = self.config.getServiceCertificateAlgorithm()
cert_thumbprint = self.config.getServiceCertificateThumbprint()
cert_path = self.config.getServiceCertificateFilename()
cert_password = self.config.getServiceCertificatePassword()
logger.info(
"Checking for existence of cloud service certificate for service %s.",
service_name)
get_cert = lambda: self.sms.get_service_certificate(
service_name, cert_algorithm, cert_thumbprint)
if self._resource_exists(get_cert):
logger.info("Found expected cloud service certificate.")
else:
with open(cert_path, 'rb') as f:
cert_data = base64.b64encode(f.read())
if len(cert_data) <= 0:
raise Exception("Detected invalid certificate data.")
result = self.sms.add_service_certificate(service_name, cert_data,
cert_format,
cert_password)
self._wait_for_operation_success(
result.request_id,
timeout=self.config.getAzureOperationTimeout())
logger.info("Added service certificate.")
def _assertOsImageExists(self, os_image_name):
"""
Asserts that the named OS image exists.
"""
logger.info("Checking for availability of OS image (name=%s).",
os_image_name)
if self.sms.get_os_image(os_image_name) is None:
raise Exception(
"Unable to find OS Image '{0}'.".format(os_image_name))
def _ensureVirtualMachinesExist(self):
"""
Creates the VMs for the web site.
"""
service_name = self.config.getServiceName()
cert_thumbprint = self.config.getServiceCertificateThumbprint()
vm_username = self.config.getVirtualMachineLogonUsername()
vm_password = self.config.getVirtualMachineLogonPassword()
vm_role_size = self.config.getServiceInstanceRoleSize()
vm_numbers = self.config.getServiceInstanceCount()
if vm_numbers < 1:
raise Exception(
"Detected an invalid number of instances: {0}.".format(
vm_numbers))
self._assertOsImageExists(self.config.getServiceOSImageName())
role_instances = self._getRoleInstances(service_name)
for vm_number in range(1, vm_numbers + 1):
vm_hostname = '{0}-{1}'.format(service_name, vm_number)
if vm_hostname in role_instances:
logger.warn(
"Role instance %s already exists: skipping creation.",
vm_hostname)
continue
logger.info("Role instance %s provisioning begins.", vm_hostname)
vm_diskname = '{0}.vhd'.format(vm_hostname)
vm_disk_media_link = 'http://{0}.blob.core.windows.net/vhds/{1}'.format(
self.config.getServiceStorageAccountName(), vm_diskname)
ssh_port = str(self.config.getServiceInstanceSshPort() + vm_number)
os_hd = OSVirtualHardDisk(self.config.getServiceOSImageName(),
vm_disk_media_link,
disk_name=vm_diskname,
disk_label=vm_diskname)
linux_config = LinuxConfigurationSet(vm_hostname, vm_username,
vm_password, True)
linux_config.ssh.public_keys.public_keys.append(
PublicKey(
cert_thumbprint,
u'/home/{0}/.ssh/authorized_keys'.format(vm_username)))
linux_config.ssh.key_pairs.key_pairs.append(
KeyPair(cert_thumbprint,
u'/home/{0}/.ssh/id_rsa'.format(vm_username)))
network_config = ConfigurationSet()
network_config.configuration_set_type = 'NetworkConfiguration'
ssh_endpoint = ConfigurationSetInputEndpoint(name='SSH',
protocol='TCP',
port=ssh_port,
local_port=u'22')
network_config.input_endpoints.input_endpoints.append(ssh_endpoint)
http_endpoint = ConfigurationSetInputEndpoint(
name='HTTP',
protocol='TCP',
port=u'80',
local_port=u'80',
load_balanced_endpoint_set_name=service_name)
http_endpoint.load_balancer_probe.port = '80'
http_endpoint.load_balancer_probe.protocol = 'TCP'
network_config.input_endpoints.input_endpoints.append(
http_endpoint)
if vm_number == 1:
result = self.sms.create_virtual_machine_deployment(
service_name=service_name,
deployment_name=service_name,
deployment_slot='Production',
label=vm_hostname,
role_name=vm_hostname,
system_config=linux_config,
os_virtual_hard_disk=os_hd,
network_config=network_config,
availability_set_name=service_name,
data_virtual_hard_disks=None,
role_size=vm_role_size)
self._wait_for_operation_success(
result.request_id,
timeout=self.config.getAzureOperationTimeout())
self._wait_for_role_instance_status(
vm_hostname, service_name, 'ReadyRole',
self.config.getAzureOperationTimeout())
else:
result = self.sms.add_role(service_name=service_name,
deployment_name=service_name,
role_name=vm_hostname,
system_config=linux_config,
os_virtual_hard_disk=os_hd,
network_config=network_config,
availability_set_name=service_name,
role_size=vm_role_size)
self._wait_for_operation_success(
result.request_id,
timeout=self.config.getAzureOperationTimeout())
self._wait_for_role_instance_status(
vm_hostname, service_name, 'ReadyRole',
self.config.getAzureOperationTimeout())
logger.info("Role instance %s has been created.", vm_hostname)
def _deleteVirtualMachines(self, service_name):
"""
Deletes the VMs in the given cloud service.
"""
if self._resource_exists(lambda: self.sms.get_deployment_by_name(
service_name, service_name)) == False:
logger.warn("Deployment %s not found: no VMs to delete.",
service_name)
else:
logger.info("Attempting to delete deployment %s.", service_name)
# Get set of role instances before we remove them
role_instances = self._getRoleInstances(service_name)
def update_request(request):
"""
A filter to intercept the HTTP request sent by the ServiceManagementService
so we can take advantage of a newer feature ('comp=media') in the delete deployment API
(see http://msdn.microsoft.com/en-us/library/windowsazure/ee460812.aspx)
"""
hdrs = []
for name, value in request.headers:
if 'x-ms-version' == name:
value = '2013-08-01'
hdrs.append((name, value))
request.headers = hdrs
request.path = request.path + '?comp=media'
#pylint: disable=W0212
response = self.sms._filter(request)
return response
svc = ServiceManagementService(self.sms.subscription_id,
self.sms.cert_file)
#pylint: disable=W0212
svc._filter = update_request
result = svc.delete_deployment(service_name, service_name)
logger.info(
"Deployment %s deletion in progress: waiting for delete_deployment operation.",
service_name)
self._wait_for_operation_success(result.request_id)
logger.info(
"Deployment %s deletion in progress: waiting for VM disks to be removed.",
service_name)
# Now wait for the disks to disappear
for role_instance_name in role_instances.keys():
disk_name = "{0}.vhd".format(role_instance_name)
self._wait_for_disk_deletion(disk_name)
logger.info("Deployment %s deleted.", service_name)
def _ensureBuildMachineExists(self):
"""
Creates the VM for the build server.
"""
service_name = self.config.getBuildServiceName()
service_storage_name = self.config.getStorageAccountName()
cert_thumbprint = self.config.getServiceCertificateThumbprint()
vm_username = self.config.getVirtualMachineLogonUsername()
vm_password = self.config.getVirtualMachineLogonPassword()
vm_hostname = service_name
role_instances = self._getRoleInstances(service_name)
if vm_hostname in role_instances:
logger.warn("Role instance %s already exists: skipping creation.",
vm_hostname)
else:
logger.info("Role instance %s provisioning begins.", vm_hostname)
self._assertOsImageExists(self.config.getBuildOSImageName())
vm_diskname = '{0}.vhd'.format(vm_hostname)
vm_disk_media_link = 'http://{0}.blob.core.windows.net/vhds/{1}'.format(
service_storage_name, vm_diskname)
os_hd = OSVirtualHardDisk(self.config.getBuildOSImageName(),
vm_disk_media_link,
disk_name=vm_diskname,
disk_label=vm_diskname)
linux_config = LinuxConfigurationSet(vm_hostname, vm_username,
vm_password, True)
linux_config.ssh.public_keys.public_keys.append(
PublicKey(
cert_thumbprint,
u'/home/{0}/.ssh/authorized_keys'.format(vm_username)))
linux_config.ssh.key_pairs.key_pairs.append(
KeyPair(cert_thumbprint,
u'/home/{0}/.ssh/id_rsa'.format(vm_username)))
network_config = ConfigurationSet()
network_config.configuration_set_type = 'NetworkConfiguration'
ssh_endpoint = ConfigurationSetInputEndpoint(name='SSH',
protocol='TCP',
port=u'22',
local_port=u'22')
network_config.input_endpoints.input_endpoints.append(ssh_endpoint)
result = self.sms.create_virtual_machine_deployment(
service_name=service_name,
deployment_name=service_name,
deployment_slot='Production',
label=vm_hostname,
role_name=vm_hostname,
system_config=linux_config,
os_virtual_hard_disk=os_hd,
network_config=network_config,
availability_set_name=None,
data_virtual_hard_disks=None,
role_size=self.config.getBuildInstanceRoleSize())
self._wait_for_operation_success(
result.request_id,
timeout=self.config.getAzureOperationTimeout())
self._wait_for_role_instance_status(
vm_hostname, service_name, 'ReadyRole',
self.config.getAzureOperationTimeout())
logger.info("Role instance %s has been created.", vm_hostname)
def _deleteStorageAccount(self, name):
"""
Deletes the storage account for the web site.
"""
logger.info("Attempting to delete storage account %s.", name)
if self._resource_exists(
lambda: self.sms.get_storage_account_properties(name
)) == False:
logger.warn("Storage account %s not found: nothing to delete.",
name)
else:
self.sms.delete_storage_account(name)
logger.info("Storage account %s deleted.", name)
def _deleteService(self, name):
"""
Deletes the specified cloud service.
"""
logger.info("Attempting to delete cloud service %s.", name)
if self._resource_exists(
lambda: self.sms.get_hosted_service_properties(name)) == False:
logger.warn("Cloud service %s not found: nothing to delete.", name)
else:
self.sms.delete_hosted_service(name)
logger.info("Cloud service %s deleted.", name)
def _deleteAffinityGroup(self):
"""
Deletes the affinity group for the web site.
"""
name = self.config.getAffinityGroupName()
logger.info("Attempting to delete affinity group %s.", name)
if self._resource_exists(
lambda: self.sms.get_affinity_group_properties(name)) == False:
logger.warn("Affinity group %s not found: nothing to delete.",
name)
else:
self.sms.delete_affinity_group(name)
logger.info("Affinity group %s deleted.", name)
def _ensureServiceBusNamespaceExists(self):
"""
Creates the Azure Service Bus Namespace if it does not exist.
"""
name = self.config.getServiceBusNamespace()
logger.info(
"Checking for existence of service bus namespace (name=%s).", name)
if self._resource_exists(lambda: self.sbms.get_namespace(name)):
logger.warn("A namespace named %s already exists.", name)
else:
self.sbms.create_namespace(name, self.config.getServiceLocation())
self._wait_for_namespace_active(name)
logger.info("Created namespace %s.", name)
def _ensureServiceBusQueuesExist(self):
"""
Creates Azure service bus queues required by the service.
"""
logger.info("Checking for existence of Service Bus Queues.")
namespace = self.sbms.get_namespace(
self.config.getServiceBusNamespace())
sbs = ServiceBusService(namespace.name,
namespace.default_key,
issuer='owner')
queue_names = [
'jobresponsequeue', 'windowscomputequeue', 'linuxcomputequeue'
]
for name in queue_names:
logger.info("Checking for existence of Queue %s.", name)
sbs.create_queue(name, fail_on_exist=False)
logger.info("Queue %s is ready.", name)
def _deleteServiceBusNamespace(self):
"""
Deletes the Azure Service Bus Namespace.
"""
name = self.config.getServiceBusNamespace()
logger.info("Attempting to delete service bus namespace %s.", name)
if self._resource_exists(
lambda: self.sbms.get_namespace(name)) == False:
logger.warn("Namespace %s not found: nothing to delete.", name)
else:
self.sbms.delete_namespace(name)
logger.info("Namespace %s deleted.", name)
def Deploy(self, assets):
"""
Creates a deployment.
assets: The set of assets to create. The full set is: {'build', 'web'}.
"""
if len(assets) == 0:
raise ValueError("Set of assets to deploy is not specified.")
logger.info("Starting deployment operation.")
self._ensureAffinityGroupExists()
self._ensureStorageAccountExists(self.config.getStorageAccountName())
## Build instance
if 'build' in assets:
self._ensureServiceExists(self.config.getBuildServiceName(),
self.config.getAffinityGroupName())
self._ensureServiceCertificateExists(
self.config.getBuildServiceName())
self._ensureBuildMachineExists()
# Web instances
if 'web' in assets:
self._ensureStorageAccountExists(
self.config.getServiceStorageAccountName())
self._ensureStorageContainersExist()
self.ensureStorageHasCorsConfiguration()
self._ensureServiceBusNamespaceExists()
self._ensureServiceBusQueuesExist()
self._ensureServiceExists(self.config.getServiceName(),
self.config.getAffinityGroupName())
self._ensureServiceCertificateExists(self.config.getServiceName())
self._ensureVirtualMachinesExist()
#queues
logger.info("Deployment operation is complete.")
def Teardown(self, assets):
"""
Deletes a deployment.
assets: The set of assets to delete. The full set is: {'web', 'build'}.
"""
if len(assets) == 0:
raise ValueError("Set of assets to teardown is not specified.")
logger.info("Starting teardown operation.")
if 'web' in assets:
self._deleteVirtualMachines(self.config.getServiceName())
self._deleteService(self.config.getServiceName())
self._deleteStorageAccount(
self.config.getServiceStorageAccountName())
if 'build' in assets:
self._deleteVirtualMachines(self.config.getBuildServiceName())
self._deleteService(self.config.getBuildServiceName())
self._deleteStorageAccount(self.config.getStorageAccountName())
if ('web' in assets) and ('build' in assets):
self._deleteServiceBusNamespace()
self._deleteAffinityGroup()
logger.info("Teardown operation is complete.")
def getSettingsFileContent(self):
"""
Generates the content of the local Django settings file.
"""
allowed_hosts = [
'{0}.cloudapp.net'.format(self.config.getServiceName())
]
allowed_hosts.extend(self.config.getWebHostnames())
allowed_hosts.extend(['www.codalab.org', 'codalab.org'])
ssl_allowed_hosts = self.config.getSslRewriteHosts()
if len(ssl_allowed_hosts) == 0:
ssl_allowed_hosts = allowed_hosts
storage_key = self._getStorageAccountKey(
self.config.getServiceStorageAccountName())
namespace = self.sbms.get_namespace(
self.config.getServiceBusNamespace())
if len(self.config.getSslCertificateInstalledPath()) > 0:
bundle_auth_scheme = "https"
else:
bundle_auth_scheme = "http"
if len(ssl_allowed_hosts) == 0:
bundle_auth_host = '{0}.cloudapp.net'.format(
self.config.getServiceName())
else:
bundle_auth_host = ssl_allowed_hosts[0]
bundle_auth_url = "{0}://{1}".format(bundle_auth_scheme,
bundle_auth_host)
lines = [
"from base import Base",
"from default import *",
"from configurations import Settings",
"",
"import sys",
"from os.path import dirname, abspath, join",
"from pkgutil import extend_path",
"import codalab",
"",
"class {0}(Base):".format(self.config.getDjangoConfiguration()),
"",
" DEBUG=False",
"",
" ALLOWED_HOSTS = {0}".format(allowed_hosts),
"",
" SSL_PORT = '443'",
" SSL_CERTIFICATE = '{0}'".format(
self.config.getSslCertificateInstalledPath()),
" SSL_CERTIFICATE_KEY = '{0}'".format(
self.config.getSslCertificateKeyInstalledPath()),
" SSL_ALLOWED_HOSTS = {0}".format(ssl_allowed_hosts),
"",
" DEFAULT_FILE_STORAGE = 'codalab.azure_storage.AzureStorage'",
" AZURE_ACCOUNT_NAME = '{0}'".format(
self.config.getServiceStorageAccountName()),
" AZURE_ACCOUNT_KEY = '{0}'".format(storage_key),
" AZURE_CONTAINER = '{0}'".format(
self.config.getServicePublicStorageContainer()),
" BUNDLE_AZURE_ACCOUNT_NAME = AZURE_ACCOUNT_NAME",
" BUNDLE_AZURE_ACCOUNT_KEY = AZURE_ACCOUNT_KEY",
" BUNDLE_AZURE_CONTAINER = '{0}'".format(
self.config.getServiceBundleStorageContainer()),
"",
" SBS_NAMESPACE = '{0}'".format(
self.config.getServiceBusNamespace()),
" SBS_ISSUER = 'owner'",
" SBS_ACCOUNT_KEY = '{0}'".format(namespace.default_key),
" SBS_RESPONSE_QUEUE = 'jobresponsequeue'",
" SBS_COMPUTE_QUEUE = 'windowscomputequeue'",
"",
" EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'",
" EMAIL_HOST = '{0}'".format(self.config.getEmailHost()),
" EMAIL_HOST_USER = '******'".format(self.config.getEmailUser()),
" EMAIL_HOST_PASSWORD = '******'".format(
self.config.getEmailPassword()),
" EMAIL_PORT = 587",
" EMAIL_USE_TLS = True",
" DEFAULT_FROM_EMAIL = '*****@*****.**'",
" SERVER_EMAIL = '*****@*****.**'",
"",
" # Django secret",
" SECRET_KEY = '{0}'".format(self.config.getDjangoSecretKey()),
"",
" ADMINS = (('CodaLab', '*****@*****.**'),)",
" MANAGERS = ADMINS",
"",
" DATABASES = {",
" 'default': {",
" 'ENGINE': '{0}',".format(
self.config.getDatabaseEngine()),
" 'NAME': '{0}',".format(self.config.getDatabaseName()),
" 'USER': '******',".format(self.config.getDatabaseUser()),
" 'PASSWORD': '******',".format(
self.config.getDatabasePassword()),
" 'HOST': '{0}',".format(self.config.getDatabaseHost()),
" 'PORT': '{0}', ".format(
self.config.getDatabasePort()),
" 'OPTIONS' : {",
" 'init_command': 'SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED',",
" 'read_timeout': 5",
" }",
" }",
" }",
"",
" BUNDLE_DB_NAME = '{0}'".format(
self.config.getBundleServiceDatabaseName()),
" BUNDLE_DB_USER = '******'".format(
self.config.getBundleServiceDatabaseUser()),
" BUNDLE_DB_PASSWORD = '******'".format(
self.config.getBundleServiceDatabasePassword()),
" BUNDLE_APP_ID = '{0}'".format(
self.config.getBundleServiceAppId()),
" BUNDLE_APP_KEY = '{0}'".format(
self.config.getBundleServiceAppKey()),
" BUNDLE_AUTH_URL = '{0}'".format(bundle_auth_url),
"",
" BUNDLE_SERVICE_URL = '{0}'".format(
self.config.getBundleServiceUrl()),
" BUNDLE_SERVICE_CODE_PATH = '/home/{0}/deploy/bundles'".format(
self.config.getVirtualMachineLogonUsername()),
" sys.path.append(BUNDLE_SERVICE_CODE_PATH)",
" codalab.__path__ = extend_path(codalab.__path__, codalab.__name__)",
"",
]
preview = self.config.getShowPreviewFeatures()
if preview >= 1:
if preview == 1:
lines.append(" PREVIEW_WORKSHEETS = True")
if preview > 1:
lines.append(" SHOW_BETA_FEATURES = True")
lines.append("")
return '\n'.join(lines)
class Deployment(object):
"""
Helper class to handle deployment of the web site.
"""
def __init__(self, config):
self.config = config
self.sms = ServiceManagementService(config.getAzureSubscriptionId(), config.getAzureCertificatePath())
self.sbms = ServiceBusManagementService(config.getAzureSubscriptionId(), config.getAzureCertificatePath())
@staticmethod
def _resource_exists(get_resource):
"""
Helper to check for the existence of a resource in Azure.
get_resource: Parameter-less function to invoke in order to get the resource. The resource
is assumed to exist when the call to get_resource() returns a value that is not None.
If the call to get_resource() returns None or throws a WindowsAzureMissingResourceError
exception, then it is assumed that the resource does not exist.
Returns: A boolean value which is True if the resource exists.
"""
resource = None
try:
resource = get_resource()
except WindowsAzureMissingResourceError:
pass
return resource is not None
def _wait_for_operation_success(self, request_id, timeout=600, wait=5):
"""
Waits for an asynchronous Azure operation to finish.
request_id: The ID of the request to track.
timeout: Maximum duration (in seconds) allowed for the operation to complete.
wait: Wait time (in seconds) between consecutive calls to fetch the latest operation status.
"""
result = self.sms.get_operation_status(request_id)
start_time = time.time()
max_time = start_time + timeout
now = start_time
while result.status == 'InProgress':
if now >= max_time:
raise Exception("Operation did not finish within the expected timeout")
logger.info('Waiting for operation to finish (last_status=%s wait_so_far=%s)',
result.status, round(now - start_time, 1))
time_to_wait = max(0.0, min(max_time - now, wait))
time.sleep(time_to_wait)
result = self.sms.get_operation_status(request_id)
now = time.time()
if result.status != 'Succeeded':
raise Exception("Operation terminated but it did not succeed.")
def _wait_for_role_instance_status(self, role_instance_name, service_name, expected_status, timeout=600, wait=5):
"""
Waits for a role instance within the web site's cloud service to reach the status specified.
role_instance_name: Name of the role instance.
service_name: Name of service in which to find the role instance.
expected_status: Expected instance status.
timeout: Maximum duration (in seconds) allowed for the operation to complete.
wait: Wait time (in seconds) between consecutive calls to fetch the latest role status.
"""
start_time = time.time()
max_time = start_time + timeout
now = start_time
while True:
status = None
deployment = self.sms.get_deployment_by_name(service_name, service_name)
for role_instance in deployment.role_instance_list:
if role_instance.instance_name == role_instance_name:
status = role_instance.instance_status
if status == expected_status:
break
if now >= max_time:
raise Exception("Operation did not finish within the expected timeout")
logger.info('Waiting for deployment status: expecting %s but got %s (wait_so_far=%s)',
expected_status, status, round(now - start_time, 1))
time_to_wait = max(0.0, min(max_time - now, wait))
time.sleep(time_to_wait)
now = time.time()
def _wait_for_disk_deletion(self, disk_name, timeout=600, wait=5):
"""
Waits for a VM disk to disappear when it is being deleted.
disk_name: Name of the VHD.
timeout: Maximum duration (in seconds) allowed for the operation to complete.
wait: Wait time (in seconds) between consecutive calls to check for the existence of the disk.
"""
start_time = time.time()
max_time = start_time + timeout
now = start_time
logger.info("Checking that disk %s has been deleted.", disk_name)
while self._resource_exists(lambda: self.sms.get_disk(disk_name)):
if now >= max_time:
raise Exception("Disk %s was not deleted within the expected timeout.".format(disk_name))
logger.info("Waiting for disk %s to disappear (wait_so_far=%s).", disk_name, round(now - start_time, 1))
time_to_wait = max(0.0, min(max_time - now, wait))
time.sleep(time_to_wait)
now = time.time()
logger.info("Disk %s has been deleted.", disk_name)
def _wait_for_namespace_active(self, name, timeout=600, wait=5):
"""
Waits for a service bus namespace to become Active.
name: Namespace name.
timeout: Maximum duration (in seconds) allowed for the operation to complete.
wait: Wait time (in seconds) between consecutive calls to check for the existence of the disk.
"""
start_time = time.time()
max_time = start_time + timeout
now = start_time
while True:
status = None
props = self.sbms.get_namespace(name)
status = props.status
if status == 'Active':
break
if now >= max_time:
raise Exception("Operation did not finish within the expected timeout")
logger.info('Waiting for namespace status: expecting Active but got %s (wait_so_far=%s)',
status, round(now - start_time, 1))
time_to_wait = max(0.0, min(max_time - now, wait))
time.sleep(time_to_wait)
now = time.time()
def _getRoleInstances(self, service_name):
"""
Returns the role instances in the given cloud service deployment. The results are provided as
a dictionary where keys are role instance names and values are RoleInstance objects.
"""
role_instances = {}
if self._resource_exists(lambda: self.sms.get_deployment_by_name(service_name, service_name)):
deployment = self.sms.get_deployment_by_name(service_name, service_name)
for role_instance in deployment.role_instance_list:
role_instances[role_instance.instance_name] = role_instance
return role_instances
def _ensureAffinityGroupExists(self):
"""
Creates the affinity group if it does not exist.
"""
name = self.config.getAffinityGroupName()
location = self.config.getServiceLocation()
logger.info("Checking for existence of affinity group (name=%s; location=%s).", name, location)
if self._resource_exists(lambda: self.sms.get_affinity_group_properties(name)):
logger.warn("An affinity group named %s already exists.", name)
else:
self.sms.create_affinity_group(name, name, location)
logger.info("Created affinity group %s.", name)
def _ensureStorageAccountExists(self, name):
"""
Creates the storage account if it does not exist.
"""
logger.info("Checking for existence of storage account (name=%s).", name)
if self._resource_exists(lambda: self.sms.get_storage_account_properties(name)):
logger.warn("A storage account named %s already exists.", name)
else:
result = self.sms.create_storage_account(name, "", name, affinity_group=self.config.getAffinityGroupName())
self._wait_for_operation_success(result.request_id, timeout=self.config.getAzureOperationTimeout())
logger.info("Created storage account %s.", name)
def _getStorageAccountKey(self, account_name):
"""
Gets the storage account key (primary key) for the given storage account.
"""
storage_props = self.sms.get_storage_account_keys(account_name)
return storage_props.storage_service_keys.primary
def _ensureStorageContainersExist(self):
"""
Creates Blob storage containers required by the service.
"""
logger.info("Checking for existence of Blob containers.")
account_name = self.config.getServiceStorageAccountName()
account_key = self._getStorageAccountKey(account_name)
blob_service = BlobService(account_name, account_key)
name_and_access_list = [(self.config.getServicePublicStorageContainer(), 'blob'),
(self.config.getServiceBundleStorageContainer(), None)]
for name, access in name_and_access_list:
logger.info("Checking for existence of Blob container %s.", name)
blob_service.create_container(name, x_ms_blob_public_access=access, fail_on_exist=False)
access_info = 'private' if access is None else 'public {0}'.format(access)
logger.info("Blob container %s is ready (access: %s).", name, access_info)
def ensureStorageHasCorsConfiguration(self):
"""
Ensures Blob storage container for bundles is configured to allow cross-origin resource sharing.
"""
logger.info("Setting CORS rules.")
account_name = self.config.getServiceStorageAccountName()
account_key = self._getStorageAccountKey(account_name)
cors_rule = CorsRule()
cors_rule.allowed_origins = self.config.getServiceStorageCorsAllowedOrigins()
cors_rule.allowed_methods = 'PUT'
cors_rule.exposed_headers = '*'
cors_rule.allowed_headers = '*'
cors_rule.max_age_in_seconds = 1800
cors_rules = Cors()
cors_rules.cors_rule.append(cors_rule)
set_storage_service_cors_properties(account_name, account_key, cors_rules)
def _ensureServiceExists(self, service_name, affinity_group_name):
"""
Creates the specified cloud service host if it does not exist.
service_name: Name of the cloud service.
affinity_group_name: Name of the affinity group (which should exists).
"""
logger.info("Checking for existence of cloud service (name=%s).", service_name)
if self._resource_exists(lambda: self.sms.get_hosted_service_properties(service_name)):
logger.warn("A cloud service named %s already exists.", service_name)
else:
self.sms.create_hosted_service(service_name, service_name, affinity_group=affinity_group_name)
logger.info("Created cloud service %s.", service_name)
def _ensureServiceCertificateExists(self, service_name):
"""
Adds certificate to the specified cloud service.
service_name: Name of the target cloud service (which should exist).
"""
cert_format = self.config.getServiceCertificateFormat()
cert_algorithm = self.config.getServiceCertificateAlgorithm()
cert_thumbprint = self.config.getServiceCertificateThumbprint()
cert_path = self.config.getServiceCertificateFilename()
cert_password = self.config.getServiceCertificatePassword()
logger.info("Checking for existence of cloud service certificate for service %s.", service_name)
get_cert = lambda: self.sms.get_service_certificate(service_name, cert_algorithm, cert_thumbprint)
if self._resource_exists(get_cert):
logger.info("Found expected cloud service certificate.")
else:
with open(cert_path, 'rb') as f:
cert_data = base64.b64encode(f.read())
if len(cert_data) <= 0:
raise Exception("Detected invalid certificate data.")
result = self.sms.add_service_certificate(service_name, cert_data, cert_format, cert_password)
self._wait_for_operation_success(result.request_id, timeout=self.config.getAzureOperationTimeout())
logger.info("Added service certificate.")
def _assertOsImageExists(self, os_image_name):
"""
Asserts that the named OS image exists.
"""
logger.info("Checking for availability of OS image (name=%s).", os_image_name)
if self.sms.get_os_image(os_image_name) is None:
raise Exception("Unable to find OS Image '{0}'.".format(os_image_name))
def _ensureVirtualMachinesExist(self):
"""
Creates the VMs for the web site.
"""
service_name = self.config.getServiceName()
cert_thumbprint = self.config.getServiceCertificateThumbprint()
vm_username = self.config.getVirtualMachineLogonUsername()
vm_password = self.config.getVirtualMachineLogonPassword()
vm_role_size = self.config.getServiceInstanceRoleSize()
vm_numbers = self.config.getServiceInstanceCount()
if vm_numbers < 1:
raise Exception("Detected an invalid number of instances: {0}.".format(vm_numbers))
self._assertOsImageExists(self.config.getServiceOSImageName())
role_instances = self._getRoleInstances(service_name)
for vm_number in range(1, vm_numbers+1):
vm_hostname = '{0}-{1}'.format(service_name, vm_number)
if vm_hostname in role_instances:
logger.warn("Role instance %s already exists: skipping creation.", vm_hostname)
continue
logger.info("Role instance %s provisioning begins.", vm_hostname)
vm_diskname = '{0}.vhd'.format(vm_hostname)
vm_disk_media_link = 'http://{0}.blob.core.windows.net/vhds/{1}'.format(
self.config.getServiceStorageAccountName(), vm_diskname
)
ssh_port = str(self.config.getServiceInstanceSshPort() + vm_number)
os_hd = OSVirtualHardDisk(self.config.getServiceOSImageName(),
vm_disk_media_link,
disk_name=vm_diskname,
disk_label=vm_diskname)
linux_config = LinuxConfigurationSet(vm_hostname, vm_username, vm_password, True)
linux_config.ssh.public_keys.public_keys.append(
PublicKey(cert_thumbprint, u'/home/{0}/.ssh/authorized_keys'.format(vm_username))
)
linux_config.ssh.key_pairs.key_pairs.append(
KeyPair(cert_thumbprint, u'/home/{0}/.ssh/id_rsa'.format(vm_username))
)
network_config = ConfigurationSet()
network_config.configuration_set_type = 'NetworkConfiguration'
ssh_endpoint = ConfigurationSetInputEndpoint(name='SSH',
protocol='TCP',
port=ssh_port,
local_port=u'22')
network_config.input_endpoints.input_endpoints.append(ssh_endpoint)
http_endpoint = ConfigurationSetInputEndpoint(name='HTTP',
protocol='TCP',
port=u'80',
local_port=u'80',
load_balanced_endpoint_set_name=service_name)
http_endpoint.load_balancer_probe.port = '80'
http_endpoint.load_balancer_probe.protocol = 'TCP'
network_config.input_endpoints.input_endpoints.append(http_endpoint)
if vm_number == 1:
result = self.sms.create_virtual_machine_deployment(service_name=service_name,
deployment_name=service_name,
deployment_slot='Production',
label=vm_hostname,
role_name=vm_hostname,
system_config=linux_config,
os_virtual_hard_disk=os_hd,
network_config=network_config,
availability_set_name=service_name,
data_virtual_hard_disks=None,
role_size=vm_role_size)
self._wait_for_operation_success(result.request_id,
timeout=self.config.getAzureOperationTimeout())
self._wait_for_role_instance_status(vm_hostname, service_name, 'ReadyRole',
self.config.getAzureOperationTimeout())
else:
result = self.sms.add_role(service_name=service_name,
deployment_name=service_name,
role_name=vm_hostname,
system_config=linux_config,
os_virtual_hard_disk=os_hd,
network_config=network_config,
availability_set_name=service_name,
role_size=vm_role_size)
self._wait_for_operation_success(result.request_id,
timeout=self.config.getAzureOperationTimeout())
self._wait_for_role_instance_status(vm_hostname, service_name, 'ReadyRole',
self.config.getAzureOperationTimeout())
logger.info("Role instance %s has been created.", vm_hostname)
def _deleteVirtualMachines(self, service_name):
"""
Deletes the VMs in the given cloud service.
"""
if self._resource_exists(lambda: self.sms.get_deployment_by_name(service_name, service_name)) == False:
logger.warn("Deployment %s not found: no VMs to delete.", service_name)
else:
logger.info("Attempting to delete deployment %s.", service_name)
# Get set of role instances before we remove them
role_instances = self._getRoleInstances(service_name)
def update_request(request):
"""
A filter to intercept the HTTP request sent by the ServiceManagementService
so we can take advantage of a newer feature ('comp=media') in the delete deployment API
(see http://msdn.microsoft.com/en-us/library/windowsazure/ee460812.aspx)
"""
hdrs = []
for name, value in request.headers:
if 'x-ms-version' == name:
value = '2013-08-01'
hdrs.append((name, value))
request.headers = hdrs
request.path = request.path + '?comp=media'
#pylint: disable=W0212
response = self.sms._filter(request)
return response
svc = ServiceManagementService(self.sms.subscription_id, self.sms.cert_file)
#pylint: disable=W0212
svc._filter = update_request
result = svc.delete_deployment(service_name, service_name)
logger.info("Deployment %s deletion in progress: waiting for delete_deployment operation.", service_name)
self._wait_for_operation_success(result.request_id)
logger.info("Deployment %s deletion in progress: waiting for VM disks to be removed.", service_name)
# Now wait for the disks to disappear
for role_instance_name in role_instances.keys():
disk_name = "{0}.vhd".format(role_instance_name)
self._wait_for_disk_deletion(disk_name)
logger.info("Deployment %s deleted.", service_name)
def _ensureBuildMachineExists(self):
"""
Creates the VM for the build server.
"""
service_name = self.config.getBuildServiceName()
service_storage_name = self.config.getStorageAccountName()
cert_thumbprint = self.config.getServiceCertificateThumbprint()
vm_username = self.config.getVirtualMachineLogonUsername()
vm_password = self.config.getVirtualMachineLogonPassword()
vm_hostname = service_name
role_instances = self._getRoleInstances(service_name)
if vm_hostname in role_instances:
logger.warn("Role instance %s already exists: skipping creation.", vm_hostname)
else:
logger.info("Role instance %s provisioning begins.", vm_hostname)
self._assertOsImageExists(self.config.getBuildOSImageName())
vm_diskname = '{0}.vhd'.format(vm_hostname)
vm_disk_media_link = 'http://{0}.blob.core.windows.net/vhds/{1}'.format(service_storage_name, vm_diskname)
os_hd = OSVirtualHardDisk(self.config.getBuildOSImageName(),
vm_disk_media_link,
disk_name=vm_diskname,
disk_label=vm_diskname)
linux_config = LinuxConfigurationSet(vm_hostname, vm_username, vm_password, True)
linux_config.ssh.public_keys.public_keys.append(
PublicKey(cert_thumbprint, u'/home/{0}/.ssh/authorized_keys'.format(vm_username))
)
linux_config.ssh.key_pairs.key_pairs.append(
KeyPair(cert_thumbprint, u'/home/{0}/.ssh/id_rsa'.format(vm_username))
)
network_config = ConfigurationSet()
network_config.configuration_set_type = 'NetworkConfiguration'
ssh_endpoint = ConfigurationSetInputEndpoint(name='SSH',
protocol='TCP',
port=u'22',
local_port=u'22')
network_config.input_endpoints.input_endpoints.append(ssh_endpoint)
result = self.sms.create_virtual_machine_deployment(service_name=service_name,
deployment_name=service_name,
deployment_slot='Production',
label=vm_hostname,
role_name=vm_hostname,
system_config=linux_config,
os_virtual_hard_disk=os_hd,
network_config=network_config,
availability_set_name=None,
data_virtual_hard_disks=None,
role_size=self.config.getBuildInstanceRoleSize())
self._wait_for_operation_success(result.request_id, timeout=self.config.getAzureOperationTimeout())
self._wait_for_role_instance_status(vm_hostname, service_name, 'ReadyRole',
self.config.getAzureOperationTimeout())
logger.info("Role instance %s has been created.", vm_hostname)
def _deleteStorageAccount(self, name):
"""
Deletes the storage account for the web site.
"""
logger.info("Attempting to delete storage account %s.", name)
if self._resource_exists(lambda: self.sms.get_storage_account_properties(name)) == False:
logger.warn("Storage account %s not found: nothing to delete.", name)
else:
self.sms.delete_storage_account(name)
logger.info("Storage account %s deleted.", name)
def _deleteService(self, name):
"""
Deletes the specified cloud service.
"""
logger.info("Attempting to delete cloud service %s.", name)
if self._resource_exists(lambda: self.sms.get_hosted_service_properties(name)) == False:
logger.warn("Cloud service %s not found: nothing to delete.", name)
else:
self.sms.delete_hosted_service(name)
logger.info("Cloud service %s deleted.", name)
def _deleteAffinityGroup(self):
"""
Deletes the affinity group for the web site.
"""
name = self.config.getAffinityGroupName()
logger.info("Attempting to delete affinity group %s.", name)
if self._resource_exists(lambda: self.sms.get_affinity_group_properties(name)) == False:
logger.warn("Affinity group %s not found: nothing to delete.", name)
else:
self.sms.delete_affinity_group(name)
logger.info("Affinity group %s deleted.", name)
def _ensureServiceBusNamespaceExists(self):
"""
Creates the Azure Service Bus Namespace if it does not exist.
"""
name = self.config.getServiceBusNamespace()
logger.info("Checking for existence of service bus namespace (name=%s).", name)
if self._resource_exists(lambda: self.sbms.get_namespace(name)):
logger.warn("A namespace named %s already exists.", name)
else:
self.sbms.create_namespace(name, self.config.getServiceLocation())
self._wait_for_namespace_active(name)
logger.info("Created namespace %s.", name)
def _ensureServiceBusQueuesExist(self):
"""
Creates Azure service bus queues required by the service.
"""
logger.info("Checking for existence of Service Bus Queues.")
namespace = self.sbms.get_namespace(self.config.getServiceBusNamespace())
sbs = ServiceBusService(namespace.name, namespace.default_key, issuer='owner')
queue_names = ['jobresponsequeue', 'windowscomputequeue', 'linuxcomputequeue']
for name in queue_names:
logger.info("Checking for existence of Queue %s.", name)
sbs.create_queue(name, fail_on_exist=False)
logger.info("Queue %s is ready.", name)
def _deleteServiceBusNamespace(self):
"""
Deletes the Azure Service Bus Namespace.
"""
name = self.config.getServiceBusNamespace()
logger.info("Attempting to delete service bus namespace %s.", name)
if self._resource_exists(lambda: self.sbms.get_namespace(name)) == False:
logger.warn("Namespace %s not found: nothing to delete.", name)
else:
self.sbms.delete_namespace(name)
logger.info("Namespace %s deleted.", name)
def Deploy(self, assets):
"""
Creates a deployment.
assets: The set of assets to create. The full set is: {'build', 'web'}.
"""
if len(assets) == 0:
raise ValueError("Set of assets to deploy is not specified.")
logger.info("Starting deployment operation.")
self._ensureAffinityGroupExists()
self._ensureStorageAccountExists(self.config.getStorageAccountName())
## Build instance
if 'build' in assets:
self._ensureServiceExists(self.config.getBuildServiceName(), self.config.getAffinityGroupName())
self._ensureServiceCertificateExists(self.config.getBuildServiceName())
self._ensureBuildMachineExists()
# Web instances
if 'web' in assets:
self._ensureStorageAccountExists(self.config.getServiceStorageAccountName())
self._ensureStorageContainersExist()
self.ensureStorageHasCorsConfiguration()
self._ensureServiceBusNamespaceExists()
self._ensureServiceBusQueuesExist()
self._ensureServiceExists(self.config.getServiceName(), self.config.getAffinityGroupName())
self._ensureServiceCertificateExists(self.config.getServiceName())
self._ensureVirtualMachinesExist()
#queues
logger.info("Deployment operation is complete.")
def Teardown(self, assets):
"""
Deletes a deployment.
assets: The set of assets to delete. The full set is: {'web', 'build'}.
"""
if len(assets) == 0:
raise ValueError("Set of assets to teardown is not specified.")
logger.info("Starting teardown operation.")
if 'web' in assets:
self._deleteVirtualMachines(self.config.getServiceName())
self._deleteService(self.config.getServiceName())
self._deleteStorageAccount(self.config.getServiceStorageAccountName())
if 'build' in assets:
self._deleteVirtualMachines(self.config.getBuildServiceName())
self._deleteService(self.config.getBuildServiceName())
self._deleteStorageAccount(self.config.getStorageAccountName())
if ('web' in assets) and ('build' in assets):
self._deleteServiceBusNamespace()
self._deleteAffinityGroup()
logger.info("Teardown operation is complete.")
def getSettingsFileContent(self):
"""
Generates the content of the local Django settings file.
"""
allowed_hosts = ['{0}.cloudapp.net'.format(self.config.getServiceName())]
allowed_hosts.extend(self.config.getWebHostnames())
allowed_hosts.extend(['www.codalab.org', 'codalab.org'])
ssl_allowed_hosts = self.config.getSslRewriteHosts();
if len(ssl_allowed_hosts) == 0:
ssl_allowed_hosts = allowed_hosts
storage_key = self._getStorageAccountKey(self.config.getServiceStorageAccountName())
namespace = self.sbms.get_namespace(self.config.getServiceBusNamespace())
if len(self.config.getSslCertificateInstalledPath()) > 0:
bundle_auth_scheme = "https"
else:
bundle_auth_scheme = "http"
if len(ssl_allowed_hosts) == 0:
bundle_auth_host = '{0}.cloudapp.net'.format(self.config.getServiceName())
else:
bundle_auth_host = ssl_allowed_hosts[0]
bundle_auth_url = "{0}://{1}".format(bundle_auth_scheme, bundle_auth_host)
lines = [
"from base import Base",
"from default import *",
"from configurations import Settings",
"",
"import sys",
"from os.path import dirname, abspath, join",
"from pkgutil import extend_path",
"import codalab",
"",
"class {0}(Base):".format(self.config.getDjangoConfiguration()),
"",
" DEBUG=False",
"",
" ALLOWED_HOSTS = {0}".format(allowed_hosts),
"",
" SSL_PORT = '443'",
" SSL_CERTIFICATE = '{0}'".format(self.config.getSslCertificateInstalledPath()),
" SSL_CERTIFICATE_KEY = '{0}'".format(self.config.getSslCertificateKeyInstalledPath()),
" SSL_ALLOWED_HOSTS = {0}".format(ssl_allowed_hosts),
"",
" DEFAULT_FILE_STORAGE = 'codalab.azure_storage.AzureStorage'",
" AZURE_ACCOUNT_NAME = '{0}'".format(self.config.getServiceStorageAccountName()),
" AZURE_ACCOUNT_KEY = '{0}'".format(storage_key),
" AZURE_CONTAINER = '{0}'".format(self.config.getServicePublicStorageContainer()),
" BUNDLE_AZURE_ACCOUNT_NAME = AZURE_ACCOUNT_NAME",
" BUNDLE_AZURE_ACCOUNT_KEY = AZURE_ACCOUNT_KEY",
" BUNDLE_AZURE_CONTAINER = '{0}'".format(self.config.getServiceBundleStorageContainer()),
"",
" SBS_NAMESPACE = '{0}'".format(self.config.getServiceBusNamespace()),
" SBS_ISSUER = 'owner'",
" SBS_ACCOUNT_KEY = '{0}'".format(namespace.default_key),
" SBS_RESPONSE_QUEUE = 'jobresponsequeue'",
" SBS_COMPUTE_QUEUE = 'windowscomputequeue'",
"",
" EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'",
" EMAIL_HOST = '{0}'".format(self.config.getEmailHost()),
" EMAIL_HOST_USER = '******'".format(self.config.getEmailUser()),
" EMAIL_HOST_PASSWORD = '******'".format(self.config.getEmailPassword()),
" EMAIL_PORT = 587",
" EMAIL_USE_TLS = True",
" DEFAULT_FROM_EMAIL = 'CodaLab <*****@*****.**>'",
" SERVER_EMAIL = '*****@*****.**'",
"",
" # Django secret",
" SECRET_KEY = '{0}'".format(self.config.getDjangoSecretKey()),
"",
" ADMINS = (('CodaLab', '*****@*****.**'),)",
" MANAGERS = ADMINS",
"",
" DATABASES = {",
" 'default': {",
" 'ENGINE': '{0}',".format(self.config.getDatabaseEngine()),
" 'NAME': '{0}',".format(self.config.getDatabaseName()),
" 'USER': '******',".format(self.config.getDatabaseUser()),
" 'PASSWORD': '******',".format(self.config.getDatabasePassword()),
" 'HOST': '{0}',".format(self.config.getDatabaseHost()),
" 'PORT': '{0}', ".format(self.config.getDatabasePort()),
" 'OPTIONS' : {",
" 'init_command': 'SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED',",
" 'read_timeout': 5",
" }",
" }",
" }",
"",
" BUNDLE_DB_NAME = '{0}'".format(self.config.getBundleServiceDatabaseName()),
" BUNDLE_DB_USER = '******'".format(self.config.getBundleServiceDatabaseUser()),
" BUNDLE_DB_PASSWORD = '******'".format(self.config.getBundleServiceDatabasePassword()),
" BUNDLE_APP_ID = '{0}'".format(self.config.getBundleServiceAppId()),
" BUNDLE_APP_KEY = '{0}'".format(self.config.getBundleServiceAppKey()),
" BUNDLE_AUTH_URL = '{0}'".format(bundle_auth_url),
"",
" BUNDLE_SERVICE_URL = '{0}'".format(self.config.getBundleServiceUrl()),
" BUNDLE_SERVICE_CODE_PATH = '/home/{0}/deploy/bundles'".format(self.config.getVirtualMachineLogonUsername()),
" sys.path.append(BUNDLE_SERVICE_CODE_PATH)",
" codalab.__path__ = extend_path(codalab.__path__, codalab.__name__)",
"",
]
preview = self.config.getShowPreviewFeatures()
if preview >= 1:
if preview == 1:
lines.append(" PREVIEW_WORKSHEETS = True")
if preview > 1:
lines.append(" SHOW_BETA_FEATURES = True")
lines.append("")
return '\n'.join(lines)
subscription_id = config_params["subscription_id"]
certificate_path = config_params["mgmt_cert_path"]
sms = ServiceManagementService(subscription_id, certificate_path)
# Because the name has to be unique in Their cloud :/
hosted_service_name = name_generator()
label = 'devOps test'
desc = 'Service for basic nginx server'
location = 'Central US'
# image_list = sms.list_os_images()
result = sms.create_hosted_service(hosted_service_name, label, desc, location)
operation_result = sms.get_operation_status(result.request_id)
storage_acc_name = name_generator()
label = 'mystorageaccount'
location = 'Central US'
desc = 'My storage account description.'
result = sms.create_storage_account(storage_acc_name,
desc,
label,
location=location)
operation_result = sms.get_operation_status(result.request_id)
print('Operation status: ' + operation_result.status)
print "The following services are now up:"
subscription_id = config_params["subscription_id"]
certificate_path = config_params["mgmt_cert_path"]
sms = ServiceManagementService(subscription_id, certificate_path)
# Because the name has to be unique in Their cloud :/
hosted_service_name = name_generator()
label = 'devOps test'
desc = 'Service for basic nginx server'
location = 'Central US'
# image_list = sms.list_os_images()
result = sms.create_hosted_service(hosted_service_name, label, desc, location)
operation_result = sms.get_operation_status(result.request_id)
storage_acc_name = name_generator()
label = 'mystorageaccount'
location = 'Central US'
desc = 'My storage account description.'
result = sms.create_storage_account(storage_acc_name, desc, label,
location=location)
operation_result = sms.get_operation_status(result.request_id)
print('Operation status: ' + operation_result.status)
print "The following services are now up:"
result = sms.list_hosted_services()
class AzureStorageBlockDeviceAPI(object):
"""
An ``IBlockDeviceAsyncAPI`` which uses Azure Storage Backed Block Devices
Current Support: Azure SMS API
"""
def __init__(self, **azure_config):
"""
:param ServiceManagement azure_client: an instance of the azure
serivce managment api client.
:param String service_name: The name of the cloud service
:param
names of Azure volumes to identify cluster
:returns: A ``BlockDeviceVolume``.
"""
self._instance_id = self.compute_instance_id()
self._azure_service_client = ServiceManagementService(
azure_config['subscription_id'],
azure_config['management_certificate_path'])
self._service_name = azure_config['service_name']
self._azure_storage_client = BlobService(
azure_config['storage_account_name'],
azure_config['storage_account_key'])
self._storage_account_name = azure_config['storage_account_name']
self._disk_container_name = azure_config['disk_container_name']
if azure_config['debug']:
to_file(sys.stdout)
def allocation_unit(self):
"""
1GiB is the minimum allocation unit for azure disks
return int: 1 GiB
"""
return int(GiB(1).to_Byte().value)
def compute_instance_id(self):
"""
Azure Stored a UUID in the SDC kernel module.
"""
# Node host names should be unique within a vnet
return unicode(socket.gethostname())
def create_volume(self, dataset_id, size):
"""
Create a new volume.
:param UUID dataset_id: The Flocker dataset ID of the dataset on this
volume.
:param int size: The size of the new volume in bytes.
:returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when
the volume has been created.
"""
size_in_gb = Byte(size).to_GiB().value
if size_in_gb % 1 != 0:
raise UnsupportedVolumeSize(dataset_id)
self._create_volume_blob(size, dataset_id)
label = self._disk_label_for_dataset_id(str(dataset_id))
return BlockDeviceVolume(
blockdevice_id=unicode(label),
size=size,
attached_to=None,
dataset_id=self._dataset_id_for_disk_label(label))
def destroy_volume(self, blockdevice_id):
"""
Destroy an existing volume.
:param unicode blockdevice_id: The unique identifier for the volume to
destroy.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:return: ``None``
"""
log_info('Destorying block device: ' + str(blockdevice_id))
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
request = None
if lun is not None:
request = \
self._azure_service_client.delete_data_disk(
service_name=self._service_name,
deployment_name=self._service_name,
role_name=target_disk.attached_to.role_name,
lun=lun,
delete_vhd=True)
else:
if target_disk.__class__.__name__ == 'Blob':
# unregistered disk
self._azure_storage_client.delete_blob(
self._disk_container_name, target_disk.name)
else:
request = self._azure_service_client.delete_disk(
target_disk.name, True)
if request is not None:
self._wait_for_async(request.request_id, 5000)
self._wait_for_detach(blockdevice_id)
def attach_volume(self, blockdevice_id, attach_to):
"""
Attach ``blockdevice_id`` to ``host``.
:param unicode blockdevice_id: The unique identifier for the block
device being attached.
:param unicode attach_to: An identifier like the one returned by the
``compute_instance_id`` method indicating the node to which to
attach the volume.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is
already attached.
:returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to
``host``.
"""
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
if lun is not None:
raise AlreadyAttachedVolume(blockdevice_id)
log_info('Attempting to attach ' + str(blockdevice_id)
+ ' to ' + str(attach_to))
disk_size = self._attach_disk(blockdevice_id, target_disk, attach_to)
self._wait_for_attach(blockdevice_id)
log_info('disk attached')
return self._blockdevicevolume_from_azure_volume(blockdevice_id,
disk_size,
attach_to)
def detach_volume(self, blockdevice_id):
"""
Detach ``blockdevice_id`` from whatever host it is attached to.
:param unicode blockdevice_id: The unique identifier for the block
device being detached.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises UnattachedVolume: If the supplied ``blockdevice_id`` is
not attached to anything.
:returns: ``None``
"""
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
if lun is None:
raise UnattachedVolume(blockdevice_id)
# contrary to function name it doesn't delete by default, just detachs
request = \
self._azure_service_client.delete_data_disk(
service_name=self._service_name,
deployment_name=self._service_name,
role_name=role_name, lun=lun)
self._wait_for_async(request.request_id, 5000)
self._wait_for_detach(blockdevice_id)
def get_device_path(self, blockdevice_id):
"""
Return the device path that has been allocated to the block device on
the host to which it is currently attached.
:param unicode blockdevice_id: The unique identifier for the block
device.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises UnattachedVolume: If the supplied ``blockdevice_id`` is
not attached to a host.
:returns: A ``FilePath`` for the device.
"""
(target_disk_or_blob, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk_or_blob is None:
raise UnknownVolume(blockdevice_id)
if lun is None:
raise UnattachedVolume(blockdevice_id)
return Lun.get_device_path_for_lun(lun)
def list_volumes(self):
"""
List all the block devices available via the back end API.
:returns: A ``list`` of ``BlockDeviceVolume``s.
"""
media_url_prefix = 'https://' + self._storage_account_name \
+ '.blob.core.windows.net/' + self._disk_container_name
disks = self._azure_service_client.list_disks()
disk_list = []
all_blobs = self._get_flocker_blobs()
for d in disks:
if media_url_prefix not in d.media_link or \
'flocker-' not in d.label:
continue
role_name = None
if d.attached_to is not None \
and d.attached_to.role_name is not None:
role_name = d.attached_to.role_name
disk_list.append(self._blockdevicevolume_from_azure_volume(
d.label, self._gibytes_to_bytes(d.logical_disk_size_in_gb),
role_name))
if d.label in all_blobs:
del all_blobs[d.label]
for key in all_blobs:
# include unregistered 'disk' blobs
disk_list.append(self._blockdevicevolume_from_azure_volume(
all_blobs[key].name,
all_blobs[key].properties.content_length,
None))
return disk_list
def _attach_disk(
self,
blockdevice_id,
target_disk,
attach_to):
"""
Attaches disk to specified VM
:param string blockdevice_id: The identifier of the disk
:param DataVirtualHardDisk/Blob target_disk: The Blob
or Disk to be attached
:returns int: The size of the attached disk
"""
lun = Lun.compute_next_lun(
self._azure_service_client,
self._service_name,
str(attach_to))
common_params = {
'service_name': self._service_name,
'deployment_name': self._service_name,
'role_name': attach_to,
'lun': lun
}
disk_size = None
if target_disk.__class__.__name__ == 'Blob':
# exclude 512 byte footer
disk_size = target_disk.properties.content_length
common_params['source_media_link'] = \
'https://' + self._storage_account_name \
+ '.blob.core.windows.net/' + self._disk_container_name \
+ '/' + blockdevice_id
common_params['disk_label'] = blockdevice_id
else:
disk_size = self._gibytes_to_bytes(
target_disk.logical_disk_size_in_gb)
common_params['disk_name'] = target_disk.name
request = self._azure_service_client.add_data_disk(**common_params)
self._wait_for_async(request.request_id, 5000)
return disk_size
def _create_volume_blob(self, size, dataset_id):
# Create a new page blob as a blank disk
self._azure_storage_client.put_blob(
container_name=self._disk_container_name,
blob_name=self._disk_label_for_dataset_id(dataset_id),
blob=None,
x_ms_blob_type='PageBlob',
x_ms_blob_content_type='application/octet-stream',
x_ms_blob_content_length=size)
# for disk to be a valid vhd it requires a vhd footer
# on the last 512 bytes
vhd_footer = Vhd.generate_vhd_footer(size)
self._azure_storage_client.put_page(
container_name=self._disk_container_name,
blob_name=self._disk_label_for_dataset_id(dataset_id),
page=vhd_footer,
x_ms_page_write='update',
x_ms_range='bytes=' + str((size - 512)) + '-' + str(size - 1))
def _disk_label_for_dataset_id(self, dataset_id):
"""
Returns a disk label for a given Dataset ID
:param unicode dataset_id: The identifier of the dataset
:returns string: A string representing the disk label
"""
label = 'flocker-' + str(dataset_id)
return label
def _dataset_id_for_disk_label(self, disk_label):
"""
Returns a UUID representing the Dataset ID for the given disk
label
:param string disk_label: The disk label
:returns UUID: The UUID of the dataset
"""
return UUID(disk_label.replace('flocker-', ''))
def _get_disk_vmname_lun(self, blockdevice_id):
target_disk = None
target_lun = None
role_name = None
disk_list = self._azure_service_client.list_disks()
for d in disk_list:
if 'flocker-' not in d.label:
continue
if d.label == str(blockdevice_id):
target_disk = d
break
if target_disk is None:
# check for unregisterd disk
blobs = self._get_flocker_blobs()
blob = None
if str(blockdevice_id) in blobs:
blob = blobs[str(blockdevice_id)]
return blob, None, None
vm_info = None
if hasattr(target_disk.attached_to, 'role_name'):
vm_info = self._azure_service_client.get_role(
self._service_name, self._service_name,
target_disk.attached_to.role_name)
for d in vm_info.data_virtual_hard_disks:
if d.disk_name == target_disk.name:
target_lun = d.lun
break
role_name = target_disk.attached_to.role_name
return (target_disk, role_name, target_lun)
def _get_flocker_blobs(self):
all_blobs = {}
blobs = self._azure_storage_client.list_blobs(
self._disk_container_name,
prefix='flocker-')
for b in blobs:
# todo - this could be big!
all_blobs[b.name] = b
return all_blobs
def _wait_for_detach(self, blockdevice_id):
role_name = ''
lun = -1
timeout_count = 0
log_info('waiting for azure to ' + 'report disk as detached...')
while role_name is not None or lun is not None:
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
time.sleep(1)
timeout_count += 1
if timeout_count > 5000:
raise AsynchronousTimeout()
log_info('Disk Detached')
def _wait_for_attach(self, blockdevice_id):
timeout_count = 0
lun = None
log_info('waiting for azure to report disk as attached...')
while lun is None:
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
time.sleep(.001)
timeout_count += 1
if timeout_count > 5000:
raise AsynchronousTimeout()
def _wait_for_async(self, request_id, timeout):
count = 0
result = self._azure_service_client.get_operation_status(request_id)
while result.status == 'InProgress':
count = count + 1
if count > timeout:
log_error('Timed out waiting for async operation to complete.')
raise AsynchronousTimeout()
time.sleep(.001)
log_info('.')
result = self._azure_service_client.get_operation_status(
request_id)
if result.error:
log_error(result.error.code)
log_error(str(result.error.message))
log_error(result.status + ' in ' + str(count * 5) + 's')
def _gibytes_to_bytes(self, size):
return int(GiB(size).to_Byte().value)
def _blockdevicevolume_from_azure_volume(self, label, size,
attached_to_name):
# azure will report the disk size excluding the 512 byte footer
# however flocker expects the exact value it requested for disk size
# so offset the reported size to flocker by 512 bytes
return BlockDeviceVolume(
blockdevice_id=unicode(label),
size=int(size),
attached_to=attached_to_name,
dataset_id=self._dataset_id_for_disk_label(label)
) # disk labels are formatted as flocker-<data_set_id>
class StorageManagementServiceTest(AzureTestCase):
def setUp(self):
proxy_host = credentials.getProxyHost()
proxy_port = credentials.getProxyPort()
self.sms = ServiceManagementService(
credentials.getSubscriptionId(),
credentials.getManagementCertFile())
if proxy_host:
self.sms.set_proxy(proxy_host, proxy_port)
self.storage_account_name = getUniqueNameBasedOnCurrentTime(
'utstorage')
def tearDown(self):
try:
self.sms.delete_storage_account(self.storage_account_name)
except:
pass
#--Helpers-----------------------------------------------------------------
def _wait_for_async(self, request_id):
count = 0
result = self.sms.get_operation_status(request_id)
while result.status == 'InProgress':
count = count + 1
if count > 120:
self.assertTrue(
False,
'Timed out waiting for async operation to complete.')
time.sleep(5)
result = self.sms.get_operation_status(request_id)
self.assertEqual(result.status, 'Succeeded')
def _create_storage_account(self, name):
result = self.sms.create_storage_account(name, name + 'description',
name + 'label', None,
'West US', False, {
'ext1': 'val1',
'ext2': 42
})
self._wait_for_async(result.request_id)
def _storage_account_exists(self, name):
try:
props = self.sms.get_storage_account_properties(name)
return props is not None
except:
return False
#--Test cases for storage accounts -----------------------------------
def test_list_storage_accounts(self):
# Arrange
self._create_storage_account(self.storage_account_name)
# Act
result = self.sms.list_storage_accounts()
# Assert
self.assertIsNotNone(result)
self.assertTrue(len(result) > 0)
storage = None
for temp in result:
if temp.service_name == self.storage_account_name:
storage = temp
break
self.assertIsNotNone(storage)
self.assertIsNotNone(storage.service_name)
self.assertIsNone(storage.storage_service_keys)
self.assertIsNotNone(storage.storage_service_properties)
self.assertIsNotNone(storage.storage_service_properties.affinity_group)
self.assertIsNotNone(storage.storage_service_properties.description)
self.assertIsNotNone(
storage.storage_service_properties.geo_primary_region)
self.assertIsNotNone(
storage.storage_service_properties.geo_replication_enabled)
self.assertIsNotNone(
storage.storage_service_properties.geo_secondary_region)
self.assertIsNotNone(storage.storage_service_properties.label)
self.assertIsNotNone(
storage.storage_service_properties.last_geo_failover_time)
self.assertIsNotNone(storage.storage_service_properties.location)
self.assertIsNotNone(storage.storage_service_properties.status)
self.assertIsNotNone(
storage.storage_service_properties.status_of_primary)
self.assertIsNotNone(
storage.storage_service_properties.status_of_secondary)
self.assertIsNotNone(storage.storage_service_properties.endpoints)
self.assertTrue(len(storage.storage_service_properties.endpoints) > 0)
self.assertIsNotNone(storage.extended_properties)
self.assertTrue(len(storage.extended_properties) > 0)
def test_get_storage_account_properties(self):
# Arrange
self._create_storage_account(self.storage_account_name)
# Act
result = self.sms.get_storage_account_properties(
self.storage_account_name)
# Assert
self.assertIsNotNone(result)
self.assertEqual(result.service_name, self.storage_account_name)
self.assertIsNotNone(result.url)
self.assertIsNone(result.storage_service_keys)
self.assertIsNotNone(result.storage_service_properties)
self.assertIsNotNone(result.storage_service_properties.affinity_group)
self.assertIsNotNone(result.storage_service_properties.description)
self.assertIsNotNone(
result.storage_service_properties.geo_primary_region)
self.assertIsNotNone(
result.storage_service_properties.geo_replication_enabled)
self.assertIsNotNone(
result.storage_service_properties.geo_secondary_region)
self.assertIsNotNone(result.storage_service_properties.label)
self.assertIsNotNone(
result.storage_service_properties.last_geo_failover_time)
self.assertIsNotNone(result.storage_service_properties.location)
self.assertIsNotNone(result.storage_service_properties.status)
self.assertIsNotNone(
result.storage_service_properties.status_of_primary)
self.assertIsNotNone(
result.storage_service_properties.status_of_secondary)
self.assertIsNotNone(result.storage_service_properties.endpoints)
self.assertTrue(len(result.storage_service_properties.endpoints) > 0)
self.assertIsNotNone(result.extended_properties)
self.assertTrue(len(result.extended_properties) > 0)
self.assertIsNotNone(result.capabilities)
self.assertTrue(len(result.capabilities) > 0)
def test_get_storage_account_keys(self):
# Arrange
self._create_storage_account(self.storage_account_name)
# Act
result = self.sms.get_storage_account_keys(self.storage_account_name)
# Assert
self.assertIsNotNone(result)
self.assertIsNotNone(result.url)
self.assertIsNotNone(result.service_name)
self.assertIsNotNone(result.storage_service_keys.primary)
self.assertIsNotNone(result.storage_service_keys.secondary)
self.assertIsNone(result.storage_service_properties)
def test_regenerate_storage_account_keys(self):
# Arrange
self._create_storage_account(self.storage_account_name)
previous = self.sms.get_storage_account_keys(self.storage_account_name)
# Act
result = self.sms.regenerate_storage_account_keys(
self.storage_account_name, 'Secondary')
# Assert
self.assertIsNotNone(result)
self.assertIsNotNone(result.url)
self.assertIsNotNone(result.service_name)
self.assertIsNotNone(result.storage_service_keys.primary)
self.assertIsNotNone(result.storage_service_keys.secondary)
self.assertIsNone(result.storage_service_properties)
self.assertEqual(result.storage_service_keys.primary,
previous.storage_service_keys.primary)
self.assertNotEqual(result.storage_service_keys.secondary,
previous.storage_service_keys.secondary)
def test_create_storage_account(self):
# Arrange
description = self.storage_account_name + 'description'
label = self.storage_account_name + 'label'
# Act
result = self.sms.create_storage_account(self.storage_account_name,
description, label, None,
'West US', True, {
'ext1': 'val1',
'ext2': 42
})
self._wait_for_async(result.request_id)
# Assert
self.assertTrue(self._storage_account_exists(
self.storage_account_name))
def test_update_storage_account(self):
# Arrange
self._create_storage_account(self.storage_account_name)
description = self.storage_account_name + 'descriptionupdate'
label = self.storage_account_name + 'labelupdate'
# Act
result = self.sms.update_storage_account(self.storage_account_name,
description, label, False, {
'ext1': 'val1update',
'ext2': 53,
'ext3': 'brandnew'
})
# Assert
self.assertIsNone(result)
props = self.sms.get_storage_account_properties(
self.storage_account_name)
self.assertEqual(props.storage_service_properties.description,
description)
self.assertEqual(props.storage_service_properties.label, label)
self.assertEqual(props.extended_properties['ext1'], 'val1update')
self.assertEqual(props.extended_properties['ext2'], '53')
self.assertEqual(props.extended_properties['ext3'], 'brandnew')
def test_delete_storage_account(self):
# Arrange
self._create_storage_account(self.storage_account_name)
# Act
result = self.sms.delete_storage_account(self.storage_account_name)
# Assert
self.assertIsNone(result)
self.assertFalse(
self._storage_account_exists(self.storage_account_name))
def test_check_storage_account_name_availability_not_available(self):
# Arrange
self._create_storage_account(self.storage_account_name)
# Act
result = self.sms.check_storage_account_name_availability(
self.storage_account_name)
# Assert
self.assertIsNotNone(result)
self.assertFalse(result.result)
def test_check_storage_account_name_availability_available(self):
# Arrange
# Act
result = self.sms.check_storage_account_name_availability(
self.storage_account_name)
# Assert
self.assertIsNotNone(result)
self.assertTrue(result.result)
def test_unicode_create_storage_account_unicode_name(self):
# Arrange
self.storage_account_name = unicode(
self.storage_account_name) + u'啊齄丂狛狜'
description = 'description'
label = 'label'
# Act
with self.assertRaises(WindowsAzureError):
# not supported - queue name must be alphanumeric, lowercase
result = self.sms.create_storage_account(self.storage_account_name,
description, label, None,
'West US', True, {
'ext1': 'val1',
'ext2': 42
})
self._wait_for_async(result.request_id)
# Assert
def test_unicode_create_storage_account_unicode_description_label(self):
# Arrange
description = u'啊齄丂狛狜'
label = u'丂狛狜'
# Act
result = self.sms.create_storage_account(self.storage_account_name,
description, label, None,
'West US', True, {
'ext1': 'val1',
'ext2': 42
})
self._wait_for_async(result.request_id)
# Assert
result = self.sms.get_storage_account_properties(
self.storage_account_name)
self.assertEqual(result.storage_service_properties.description,
description)
self.assertEqual(result.storage_service_properties.label, label)
def test_unicode_create_storage_account_unicode_property_value(self):
# Arrange
description = 'description'
label = 'label'
# Act
result = self.sms.create_storage_account(self.storage_account_name,
description, label, None,
'West US', True, {
'ext1': u'丂狛狜',
'ext2': 42
})
self._wait_for_async(result.request_id)
# Assert
result = self.sms.get_storage_account_properties(
self.storage_account_name)
self.assertEqual(result.storage_service_properties.description,
description)
self.assertEqual(result.storage_service_properties.label, label)
self.assertEqual(result.extended_properties['ext1'], u'丂狛狜')
def provision(instance_id):
"""
Provision an instance of this service
for the given org and space
PUT /v2/service_instances/<instance_id>:
<instance_id> is provided by the Cloud
Controller and will be used for future
requests to bind, unbind and deprovision
BODY:
{
"service_id": "<service-guid>",
"plan_id": "<plan-guid>",
"organization_guid": "<org-guid>",
"space_guid": "<space-guid>"
}
return:
JSON document with details about the
services offered through this broker
"""
if 'application/json' not in request.content_type:
abort(415, 'Unsupported Content-Type: expecting application/json, actual {0}'.format(request.content_type))
global subscription_id
global cert
global cert_file
global account_name
global account_key
if subscription_id and cert and (not account_name):
sms = ServiceManagementService(subscription_id, cert_file)
name = '{0}{1}'.format(STORAGE_ACCOUNT_NAME_PREFIX, instance_id.split('-')[0])
desc = name
label = name
location = 'West US'
result = None
try:
result = sms.create_storage_account(name, desc, label, location=location)
except WindowsAzureConflictError as e:
pass
if result:
req_id = result.request_id
operation = sms.get_operation_status(req_id)
while operation.status == 'InProgress':
time.sleep(5)
operation = sms.get_operation_status(req_id)
app.logger.info('Request ID: {0}, Operation Status: {1}'.format(req_id, operation.status))
if operation.status == 'Succeeded':
app.logger.info('Request ID: {0}, Operation Status: {1}'.format(req_id, operation.status))
account_name = name
account_key = sms.get_storage_account_keys(account_name).storage_service_keys.primary
app.logger.info('Account Name: {0}, Account key: {1}'.format(account_name, account_key))
if account_name:
blob_service = BlobService(account_name, account_key)
container_name = '{0}-{1}'.format(CONTAINER_NAME_PREFIX, instance_id)
app.logger.info('Container Name: {0}'.format(container_name))
request_body = request.get_json()
if request_body.has_key('parameters'):
parameters = request_body.pop('parameters')
container_tags = request_body
container_tags['instance_id'] = instance_id
blob_service.create_container(
container_name = container_name,
x_ms_meta_name_values = container_tags)
return jsonify({})
class AzureStorageBlockDeviceAPI(object):
"""
An ``IBlockDeviceAsyncAPI`` which uses Azure Storage Backed Block Devices
Current Support: Azure SMS API
"""
def __init__(self, **azure_config):
"""
:param ServiceManagement azure_client: an instance of the azure
serivce managment api client.
:param String service_name: The name of the cloud service
:param
names of Azure volumes to identify cluster
:returns: A ``BlockDeviceVolume``.
"""
self._instance_id = self.compute_instance_id()
self._azure_service_client = ServiceManagementService(
azure_config['subscription_id'],
azure_config['management_certificate_path'])
self._service_name = azure_config['service_name']
self._azure_storage_client = BlobService(
azure_config['storage_account_name'],
azure_config['storage_account_key'])
self._storage_account_name = azure_config['storage_account_name']
self._disk_container_name = azure_config['disk_container_name']
if azure_config['debug']:
to_file(sys.stdout)
def allocation_unit(self):
"""
1GiB is the minimum allocation unit for azure disks
return int: 1 GiB
"""
return int(GiB(1).to_Byte().value)
def compute_instance_id(self):
"""
Azure Stored a UUID in the SDC kernel module.
"""
# Node host names should be unique within a vnet
return unicode(socket.gethostname())
def create_volume(self, dataset_id, size):
"""
Create a new volume.
:param UUID dataset_id: The Flocker dataset ID of the dataset on this
volume.
:param int size: The size of the new volume in bytes.
:returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when
the volume has been created.
"""
size_in_gb = Byte(size).to_GiB().value
if size_in_gb % 1 != 0:
raise UnsupportedVolumeSize(dataset_id)
self._create_volume_blob(size, dataset_id)
label = self._disk_label_for_dataset_id(str(dataset_id))
return BlockDeviceVolume(
blockdevice_id=unicode(label),
size=size,
attached_to=None,
dataset_id=self._dataset_id_for_disk_label(label))
def destroy_volume(self, blockdevice_id):
"""
Destroy an existing volume.
:param unicode blockdevice_id: The unique identifier for the volume to
destroy.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:return: ``None``
"""
log_info('Destorying block device: ' + str(blockdevice_id))
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
request = None
if lun is not None:
request = \
self._azure_service_client.delete_data_disk(
service_name=self._service_name,
deployment_name=self._service_name,
role_name=target_disk.attached_to.role_name,
lun=lun,
delete_vhd=True)
else:
if target_disk.__class__.__name__ == 'Blob':
# unregistered disk
self._azure_storage_client.delete_blob(
self._disk_container_name, target_disk.name)
else:
request = self._azure_service_client.delete_disk(
target_disk.name, True)
if request is not None:
self._wait_for_async(request.request_id, 5000)
self._wait_for_detach(blockdevice_id)
def attach_volume(self, blockdevice_id, attach_to):
"""
Attach ``blockdevice_id`` to ``host``.
:param unicode blockdevice_id: The unique identifier for the block
device being attached.
:param unicode attach_to: An identifier like the one returned by the
``compute_instance_id`` method indicating the node to which to
attach the volume.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is
already attached.
:returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to
``host``.
"""
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
if lun is not None:
raise AlreadyAttachedVolume(blockdevice_id)
log_info('Attempting to attach ' + str(blockdevice_id) + ' to ' +
str(attach_to))
disk_size = self._attach_disk(blockdevice_id, target_disk, attach_to)
self._wait_for_attach(blockdevice_id)
log_info('disk attached')
return self._blockdevicevolume_from_azure_volume(
blockdevice_id, disk_size, attach_to)
def detach_volume(self, blockdevice_id):
"""
Detach ``blockdevice_id`` from whatever host it is attached to.
:param unicode blockdevice_id: The unique identifier for the block
device being detached.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises UnattachedVolume: If the supplied ``blockdevice_id`` is
not attached to anything.
:returns: ``None``
"""
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk is None:
raise UnknownVolume(blockdevice_id)
if lun is None:
raise UnattachedVolume(blockdevice_id)
# contrary to function name it doesn't delete by default, just detachs
request = \
self._azure_service_client.delete_data_disk(
service_name=self._service_name,
deployment_name=self._service_name,
role_name=role_name, lun=lun)
self._wait_for_async(request.request_id, 5000)
self._wait_for_detach(blockdevice_id)
def get_device_path(self, blockdevice_id):
"""
Return the device path that has been allocated to the block device on
the host to which it is currently attached.
:param unicode blockdevice_id: The unique identifier for the block
device.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises UnattachedVolume: If the supplied ``blockdevice_id`` is
not attached to a host.
:returns: A ``FilePath`` for the device.
"""
(target_disk_or_blob, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
if target_disk_or_blob is None:
raise UnknownVolume(blockdevice_id)
if lun is None:
raise UnattachedVolume(blockdevice_id)
return Lun.get_device_path_for_lun(lun)
def list_volumes(self):
"""
List all the block devices available via the back end API.
:returns: A ``list`` of ``BlockDeviceVolume``s.
"""
media_url_prefix = 'https://' + self._storage_account_name \
+ '.blob.core.windows.net/' + self._disk_container_name
disks = self._azure_service_client.list_disks()
disk_list = []
all_blobs = self._get_flocker_blobs()
for d in disks:
if media_url_prefix not in d.media_link or \
'flocker-' not in d.label:
continue
role_name = None
if d.attached_to is not None \
and d.attached_to.role_name is not None:
role_name = d.attached_to.role_name
disk_list.append(
self._blockdevicevolume_from_azure_volume(
d.label, self._gibytes_to_bytes(d.logical_disk_size_in_gb),
role_name))
if d.label in all_blobs:
del all_blobs[d.label]
for key in all_blobs:
# include unregistered 'disk' blobs
disk_list.append(
self._blockdevicevolume_from_azure_volume(
all_blobs[key].name,
all_blobs[key].properties.content_length, None))
return disk_list
def _attach_disk(self, blockdevice_id, target_disk, attach_to):
"""
Attaches disk to specified VM
:param string blockdevice_id: The identifier of the disk
:param DataVirtualHardDisk/Blob target_disk: The Blob
or Disk to be attached
:returns int: The size of the attached disk
"""
lun = Lun.compute_next_lun(self._azure_service_client,
self._service_name, str(attach_to))
common_params = {
'service_name': self._service_name,
'deployment_name': self._service_name,
'role_name': attach_to,
'lun': lun
}
disk_size = None
if target_disk.__class__.__name__ == 'Blob':
# exclude 512 byte footer
disk_size = target_disk.properties.content_length
common_params['source_media_link'] = \
'https://' + self._storage_account_name \
+ '.blob.core.windows.net/' + self._disk_container_name \
+ '/' + blockdevice_id
common_params['disk_label'] = blockdevice_id
else:
disk_size = self._gibytes_to_bytes(
target_disk.logical_disk_size_in_gb)
common_params['disk_name'] = target_disk.name
request = self._azure_service_client.add_data_disk(**common_params)
self._wait_for_async(request.request_id, 5000)
return disk_size
def _create_volume_blob(self, size, dataset_id):
# Create a new page blob as a blank disk
self._azure_storage_client.put_blob(
container_name=self._disk_container_name,
blob_name=self._disk_label_for_dataset_id(dataset_id),
blob=None,
x_ms_blob_type='PageBlob',
x_ms_blob_content_type='application/octet-stream',
x_ms_blob_content_length=size)
# for disk to be a valid vhd it requires a vhd footer
# on the last 512 bytes
vhd_footer = Vhd.generate_vhd_footer(size)
self._azure_storage_client.put_page(
container_name=self._disk_container_name,
blob_name=self._disk_label_for_dataset_id(dataset_id),
page=vhd_footer,
x_ms_page_write='update',
x_ms_range='bytes=' + str((size - 512)) + '-' + str(size - 1))
def _disk_label_for_dataset_id(self, dataset_id):
"""
Returns a disk label for a given Dataset ID
:param unicode dataset_id: The identifier of the dataset
:returns string: A string representing the disk label
"""
label = 'flocker-' + str(dataset_id)
return label
def _dataset_id_for_disk_label(self, disk_label):
"""
Returns a UUID representing the Dataset ID for the given disk
label
:param string disk_label: The disk label
:returns UUID: The UUID of the dataset
"""
return UUID(disk_label.replace('flocker-', ''))
def _get_disk_vmname_lun(self, blockdevice_id):
target_disk = None
target_lun = None
role_name = None
disk_list = self._azure_service_client.list_disks()
for d in disk_list:
if 'flocker-' not in d.label:
continue
if d.label == str(blockdevice_id):
target_disk = d
break
if target_disk is None:
# check for unregisterd disk
blobs = self._get_flocker_blobs()
blob = None
if str(blockdevice_id) in blobs:
blob = blobs[str(blockdevice_id)]
return blob, None, None
vm_info = None
if hasattr(target_disk.attached_to, 'role_name'):
vm_info = self._azure_service_client.get_role(
self._service_name, self._service_name,
target_disk.attached_to.role_name)
for d in vm_info.data_virtual_hard_disks:
if d.disk_name == target_disk.name:
target_lun = d.lun
break
role_name = target_disk.attached_to.role_name
return (target_disk, role_name, target_lun)
def _get_flocker_blobs(self):
all_blobs = {}
blobs = self._azure_storage_client.list_blobs(
self._disk_container_name, prefix='flocker-')
for b in blobs:
# todo - this could be big!
all_blobs[b.name] = b
return all_blobs
def _wait_for_detach(self, blockdevice_id):
role_name = ''
lun = -1
timeout_count = 0
log_info('waiting for azure to ' + 'report disk as detached...')
while role_name is not None or lun is not None:
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
time.sleep(1)
timeout_count += 1
if timeout_count > 5000:
raise AsynchronousTimeout()
log_info('Disk Detached')
def _wait_for_attach(self, blockdevice_id):
timeout_count = 0
lun = None
log_info('waiting for azure to report disk as attached...')
while lun is None:
(target_disk, role_name, lun) = \
self._get_disk_vmname_lun(blockdevice_id)
time.sleep(.001)
timeout_count += 1
if timeout_count > 5000:
raise AsynchronousTimeout()
def _wait_for_async(self, request_id, timeout):
count = 0
result = self._azure_service_client.get_operation_status(request_id)
while result.status == 'InProgress':
count = count + 1
if count > timeout:
log_error('Timed out waiting for async operation to complete.')
raise AsynchronousTimeout()
time.sleep(.001)
log_info('.')
result = self._azure_service_client.get_operation_status(
request_id)
if result.error:
log_error(result.error.code)
log_error(str(result.error.message))
log_error(result.status + ' in ' + str(count * 5) + 's')
def _gibytes_to_bytes(self, size):
return int(GiB(size).to_Byte().value)
def _blockdevicevolume_from_azure_volume(self, label, size,
attached_to_name):
# azure will report the disk size excluding the 512 byte footer
# however flocker expects the exact value it requested for disk size
# so offset the reported size to flocker by 512 bytes
return BlockDeviceVolume(
blockdevice_id=unicode(label),
size=int(size),
attached_to=attached_to_name,
dataset_id=self._dataset_id_for_disk_label(
label)) # disk labels are formatted as flocker-<data_set_id>
