def attach_volume(self, node, volume, device='disk',
ex_bus='virtio', ex_name=None):
"""
params: bus, name , device (disk or lun)
"""
# volume must be bound to a claim
if not volume.extra['is_bound']:
volume = self._bind_volume(volume, node.extra['namespace'])
if volume is None:
raise LibcloudError("Selected Volume (PV) could not be bound "
"(to a PVC), please select another volume",
driver=self)
claimName = volume.extra['pvc']['name']
if ex_name is None:
name = claimName
else:
name = ex_name
namespace = volume.extra['pvc']['namespace']
# check if vm is stopped
self.stop_node(node)
# check if it is the same namespace
if node.extra['namespace'] != namespace:
msg = "The PVC and the VM must be in the same namespace"
raise ValueError(msg)
vm = node.name
req = KUBEVIRT_URL + 'namespaces/' + namespace + '/virtualmachines/'\
+ vm
disk_dict = {device: {'bus': ex_bus}, 'name': name}
volumes_dict = {'persistentVolumeClaim': {'claimName': claimName},
'name': name}
# Get all the volumes of the vm
try:
result = self.connection.request(req).object
except Exception:
raise
disks = result['spec']['template']['spec']['domain'][
'devices']['disks']
volumes = result['spec']['template']['spec']['volumes']
disks.append(disk_dict)
volumes.append(volumes_dict)
# now patch the new volumes and disks lists into the resource
headers = {"Content-Type": "application/merge-patch+json"}
data = {'spec': {
'template': {
'spec': {
'volumes': volumes,
'domain': {
'devices':
{'disks': disks}
}
}
}
}
}
try:
result = self.connection.request(req, method="PATCH",
data=json.dumps(data),
headers=headers)
if 'pvcs' in node.extra:
node.extra['pvcs'].append(claimName)
else:
node.extra['pvcs'] = [claimName]
return result in VALID_RESPONSE_CODES
except Exception:
raise
def ex_iterate_multipart_uploads(self, container, prefix=None,
delimiter=None):
"""
Extension method for listing all in-progress S3 multipart uploads.
Each multipart upload which has not been committed or aborted is
considered in-progress.
:param container: The container holding the uploads
:type container: :class:`Container`
:keyword prefix: Print only uploads of objects with this prefix
:type prefix: ``str``
:keyword delimiter: The object/key names are grouped based on
being split by this delimiter
:type delimiter: ``str``
:return: A generator of S3MultipartUpload instances.
:rtype: ``generator`` of :class:`S3MultipartUpload`
"""
if not self.supports_s3_multipart_upload:
raise LibcloudError('Feature not supported', driver=self)
# Get the data for a specific container
request_path = '%s/?uploads' % (self._get_container_path(container))
params = {'max-uploads': RESPONSES_PER_REQUEST}
if prefix:
params['prefix'] = prefix
if delimiter:
params['delimiter'] = delimiter
finder = lambda node, text: node.findtext(fixxpath(xpath=text,
namespace=self.namespace))
while True:
response = self.connection.request(request_path, params=params)
if response.status != httplib.OK:
raise LibcloudError('Error fetching multipart uploads. '
'Got code: %s' % (response.status),
driver=self)
body = response.parse_body()
for node in body.findall(fixxpath(xpath='Upload',
namespace=self.namespace)):
initiator = node.find(fixxpath(xpath='Initiator',
namespace=self.namespace))
owner = node.find(fixxpath(xpath='Owner',
namespace=self.namespace))
key = finder(node, 'Key')
upload_id = finder(node, 'UploadId')
created_at = finder(node, 'Initiated')
initiator = finder(initiator, 'DisplayName')
owner = finder(owner, 'DisplayName')
yield S3MultipartUpload(key, upload_id, created_at,
initiator, owner)
# Check if this is the last entry in the listing
is_truncated = body.findtext(fixxpath(xpath='IsTruncated',
namespace=self.namespace))
if is_truncated.lower() == 'false':
break
# Provide params for the next request
upload_marker = body.findtext(fixxpath(xpath='NextUploadIdMarker',
namespace=self.namespace))
key_marker = body.findtext(fixxpath(xpath='NextKeyMarker',
namespace=self.namespace))
params['key-marker'] = key_marker
params['upload-id-marker'] = upload_marker
def _save_object(self,
response,
obj,
destination_path,
overwrite_existing=False,
delete_on_failure=True,
chunk_size=None):
"""
Save object to the provided path.
@type response: C{RawResponse}
@param response: RawResponse instance.
@type obj: C{Object}
@param obj: Object instance.
@type destination_path: C{Str}
@param destination_path: Destination directory.
@type delete_on_failure: C{bool}
@param delete_on_failure: True to delete partially downloaded object if
the download fails.
@type overwrite_existing: C{bool}
@param overwrite_existing: True to overwrite a local path if it already
exists.
@type chunk_size: C{int}
@param chunk_size: Optional chunk size (defaults to CHUNK_SIZE)
@return C{bool} True on success, False otherwise.
"""
chunk_size = chunk_size or CHUNK_SIZE
base_name = os.path.basename(destination_path)
if not base_name and not os.path.exists(destination_path):
raise LibcloudError(value='Path %s does not exist' %
(destination_path),
driver=self)
if not base_name:
file_path = pjoin(destination_path, obj.name)
else:
file_path = destination_path
if os.path.exists(file_path) and not overwrite_existing:
raise LibcloudError(value='File %s already exists, but ' %
(file_path) + 'overwrite_existing=False',
driver=self)
stream = utils.read_in_chunks(response, chunk_size)
try:
data_read = stream.next()
except StopIteration:
# Empty response?
return False
bytes_transferred = 0
with open(file_path, 'wb') as file_handle:
while len(data_read) > 0:
file_handle.write(data_read)
bytes_transferred += len(data_read)
try:
data_read = stream.next()
except StopIteration:
data_read = ''
if obj.size != bytes_transferred:
# Transfer failed, support retry?
if delete_on_failure:
try:
os.unlink(file_path)
except Exception:
pass
return False
return True
def _save_object(self,
response,
obj,
destination_path,
overwrite_existing=False,
delete_on_failure=True,
chunk_size=None):
"""
Save object to the provided path.
:param response: RawResponse instance.
:type response: :class:`RawResponse`
:param obj: Object instance.
:type obj: :class:`Object`
:param destination_path: Destination directory.
:type destination_path: ``str``
:param delete_on_failure: True to delete partially downloaded object if
the download fails.
:type delete_on_failure: ``bool``
:param overwrite_existing: True to overwrite a local path if it already
exists.
:type overwrite_existing: ``bool``
:param chunk_size: Optional chunk size
(defaults to ``libcloud.storage.base.CHUNK_SIZE``, 8kb)
:type chunk_size: ``int``
:return: ``True`` on success, ``False`` otherwise.
:rtype: ``bool``
"""
chunk_size = chunk_size or CHUNK_SIZE
base_name = os.path.basename(destination_path)
if not base_name and not os.path.exists(destination_path):
raise LibcloudError(value='Path %s does not exist' %
(destination_path),
driver=self)
if not base_name:
file_path = pjoin(destination_path, obj.name)
else:
file_path = destination_path
if os.path.exists(file_path) and not overwrite_existing:
raise LibcloudError(value='File %s already exists, but ' %
(file_path) + 'overwrite_existing=False',
driver=self)
bytes_transferred = 0
with open(file_path, 'wb') as file_handle:
for chunk in response._response.iter_content(chunk_size):
file_handle.write(b(chunk))
bytes_transferred += len(chunk)
if int(obj.size) != int(bytes_transferred):
# Transfer failed, support retry?
if delete_on_failure:
try:
os.unlink(file_path)
except Exception:
pass
return False
return True
def _put_object(self,
container,
object_name,
stream,
extra=None,
verify_hash=True,
headers=None,
blob_size=None,
file_path=None,
use_lease=False):
"""
Control function that does the real job of uploading data to a blob
"""
extra = extra or {}
content_type = extra.get('content_type', None)
meta_data = extra.get('meta_data', {})
object_path = self._get_object_path(container, object_name)
# Get a lease if required and do the operations
with AzureBlobLease(self, object_path, use_lease) as lease:
if blob_size is not None and blob_size <= AZURE_UPLOAD_CHUNK_SIZE:
result_dict = self._upload_directly(stream=stream,
object_path=object_path,
lease=lease,
blob_size=blob_size,
meta_data=meta_data,
headers=headers,
content_type=content_type,
object_name=object_name,
file_path=file_path)
else:
result_dict = self._upload_in_chunks(stream=stream,
object_path=object_path,
lease=lease,
meta_data=meta_data,
headers=headers,
content_type=content_type,
object_name=object_name,
file_path=file_path,
verify_hash=verify_hash)
response = result_dict['response']
bytes_transferred = result_dict['bytes_transferred']
data_hash = result_dict['data_hash']
headers = response.headers
if response.status != httplib.CREATED:
raise LibcloudError('Unexpected status code, status_code=%s' %
(response.status),
driver=self)
server_hash = headers.get('content-md5')
if server_hash:
server_hash = binascii.hexlify(base64.b64decode(b(server_hash)))
server_hash = server_hash.decode('utf-8')
else:
# TODO: HACK - We could poll the object for a while and get
# the hash
pass
if (verify_hash and server_hash and data_hash != server_hash):
raise ObjectHashMismatchError(
value='MD5 hash checksum does not match',
object_name=object_name,
driver=self)
return Object(name=object_name,
size=bytes_transferred,
hash=headers['etag'],
extra=None,
meta_data=meta_data,
container=container,
driver=self)
def create_node(self,
name,
size,
image,
auth,
ex_resource_group,
ex_storage_account,
ex_blob_container="vhds",
location=None,
ex_user_name="azureuser",
ex_network=None,
ex_subnet=None,
ex_nic=None,
ex_tags={},
ex_customdata=""):
"""Create a new node instance. This instance will be started
automatically.
This driver supports the ``ssh_key`` feature flag for ``created_node``
so you can upload a public key into the new instance::
>>> from libcloud.compute.drivers.azure_arm import AzureNodeDriver
>>> driver = AzureNodeDriver(...)
>>> auth = NodeAuthSSHKey('pubkey data here')
>>> node = driver.create_node("test_node", auth=auth)
This driver also supports the ``password`` feature flag for
``create_node``
so you can set a password::
>>> driver = AzureNodeDriver(...)
>>> auth = NodeAuthPassword('mysecretpassword')
>>> node = driver.create_node("test_node", auth=auth, ...)
If you don't provide the ``auth`` argument libcloud will assign
a password::
>>> driver = AzureNodeDriver(...)
>>> node = driver.create_node("test_node", ...)
>>> password = node.extra["properties"] \
["osProfile"]["adminPassword"]
:param name: String with a name for this new node (required)
:type name: ``str``
:param size: The size of resources allocated to this node.
(required)
:type size: :class:`.NodeSize`
:param image: OS Image to boot on node. (required)
:type image: :class:`.AzureImage`
:param location: Which data center to create a node in.
(if None, use default location specified as 'region' in __init__)
:type location: :class:`.NodeLocation`
:param auth: Initial authentication information for the node
(optional)
:type auth: :class:`.NodeAuthSSHKey` or :class:`NodeAuthPassword`
:param ex_resource_group: The resource group in which to create the
node
:type ex_resource_group: ``str``
:param ex_storage_account: The storage account id in which to store
the node's disk image.
Note: when booting from a user image (AzureVhdImage)
the source image and the node image must use the same storage account.
:type ex_storage_account: ``str``
:param ex_blob_container: The name of the blob container on the
storage account in which to store the node's disk image (optional,
default "vhds")
:type ex_blob_container: ``str``
:param ex_user_name: User name for the initial admin user
(optional, default "azureuser")
:type ex_user_name: ``str``
:param ex_network: The virtual network the node will be attached to.
Must provide either `ex_network` (to create a default NIC for the
node on the given network) or `ex_nic` (to supply the NIC explicitly).
:type ex_network: ``str``
:param ex_subnet: If ex_network is provided, the subnet of the
virtual network the node will be attached to. Optional, default
is the "default" subnet.
:type ex_subnet: ``str``
:param ex_nic: A virtual NIC to attach to this Node, from
`ex_create_network_interface` or `ex_get_nic`.
Must provide either `ex_nic` (to supply the NIC explicitly) or
ex_network (to create a default NIC for the node on the
given network).
:type ex_nic: :class:`AzureNic`
:param ex_tags: Optional tags to associate with this node.
:type ex_tags: ``dict``
:param ex_customdata: Custom data that will
be placed in the file /var/lib/waagent/CustomData
https://azure.microsoft.com/en-us/documentation/ \
articles/virtual-machines-how-to-inject-custom-data/
:type ex_customdata: ``str``
:return: The newly created node.
:rtype: :class:`.Node`
"""
if location is None:
location = self.default_location
if ex_nic is None:
if ex_network is None:
raise ValueError("Must provide either ex_network or ex_nic")
if ex_subnet is None:
ex_subnet = "default"
subnet_id = "/subscriptions/%s/resourceGroups/%s/providers" \
"/Microsoft.Network/virtualnetworks/%s/subnets/%s" % \
(self.subscription_id, ex_resource_group,
ex_network, ex_subnet)
subnet = AzureSubnet(subnet_id, ex_subnet, {})
ex_nic = self.ex_create_network_interface(name + "-nic",
subnet,
ex_resource_group,
location)
auth = self._get_and_check_auth(auth)
target = "/subscriptions/%s/resourceGroups/%s/providers" \
"/Microsoft.Compute/virtualMachines/%s" % \
(self.subscription_id, ex_resource_group, name)
instance_vhd = "https://%s.blob.core.windows.net" \
"/%s/%s-os.vhd" \
% (ex_storage_account,
ex_blob_container,
name
)
if isinstance(image, AzureVhdImage):
storageProfile = {
"osDisk": {
"name": "virtualmachine-osDisk",
"osType": "linux",
"caching": "ReadWrite",
"createOption": "FromImage",
"image": {
"uri": image.id
},
"vhd": {
"uri": instance_vhd
}
}
}
elif isinstance(image, AzureImage):
storageProfile = {
"imageReference": {
"publisher": image.publisher,
"offer": image.offer,
"sku": image.sku,
"version": image.version
},
"osDisk": {
"name": "virtualmachine-osDisk",
"vhd": {
"uri": instance_vhd
},
"caching": "ReadWrite",
"createOption": "FromImage"
}
}
else:
raise LibcloudError(
"Unknown image type %s,"
"expected one of AzureImage, AzureVhdImage",
type(image))
data = {
"id": target,
"name": name,
"type": "Microsoft.Compute/virtualMachines",
"location": location.id,
"tags": ex_tags,
"properties": {
"hardwareProfile": {
"vmSize": size.id
},
"storageProfile": storageProfile,
"osProfile": {
"computerName": name
},
"networkProfile": {
"networkInterfaces": [
{
"id": ex_nic.id
}
]
}
}
}
if ex_customdata:
data["properties"]["osProfile"]["customData"] = \
base64.b64encode(ex_customdata)
data["properties"]["osProfile"]["adminUsername"] = ex_user_name
if isinstance(auth, NodeAuthSSHKey):
data["properties"]["osProfile"]["adminPassword"] = \
binascii.hexlify(os.urandom(20)).decode("utf-8")
data["properties"]["osProfile"]["linuxConfiguration"] = {
"disablePasswordAuthentication": "true",
"ssh": {
"publicKeys": [
{
"path": '/home/%s/.ssh/authorized_keys' % (
ex_user_name),
"keyData": auth.pubkey
}
]
}
}
elif isinstance(auth, NodeAuthPassword):
data["properties"]["osProfile"]["linuxConfiguration"] = {
"disablePasswordAuthentication": "false"
}
data["properties"]["osProfile"]["adminPassword"] = auth.password
else:
raise ValueError(
"Must provide NodeAuthSSHKey or NodeAuthPassword in auth")
r = self.connection.request(target,
params={"api-version": "2015-06-15"},
data=data,
method="PUT")
while r.object is None:
time.sleep(1)
node = self._to_node(r.object)
node.size = size
node.image = image
return node
def ex_iterate_multipart_uploads(self,
container,
prefix=None,
delimiter=None,
max_uploads=MAX_UPLOADS_PER_RESPONSE):
"""
Extension method for listing all in-progress OSS multipart uploads.
Each multipart upload which has not been committed or aborted is
considered in-progress.
:param container: The container holding the uploads
:type container: :class:`Container`
:keyword prefix: Print only uploads of objects with this prefix
:type prefix: ``str``
:keyword delimiter: The object/key names are grouped based on
being split by this delimiter
:type delimiter: ``str``
:keyword max_uploads: The max uplod items returned for one request
:type max_uploads: ``int``
:return: A generator of OSSMultipartUpload instances.
:rtype: ``generator`` of :class:`OSSMultipartUpload`
"""
if not self.supports_multipart_upload:
raise LibcloudError('Feature not supported', driver=self)
request_path = '/?uploads'
params = {'max-uploads': max_uploads}
if prefix:
params['prefix'] = prefix
if delimiter:
params['delimiter'] = delimiter
def finder(node, text):
return node.findtext(fixxpath(xpath=text,
namespace=self.namespace))
while True:
response = self.connection.request(request_path,
params=params,
container=container)
if response.status != httplib.OK:
raise LibcloudError('Error fetching multipart uploads. '
'Got code: %s' % response.status,
driver=self)
body = response.parse_body()
# pylint: disable=maybe-no-member
for node in body.findall(
fixxpath(xpath='Upload', namespace=self.namespace)):
key = finder(node, 'Key')
upload_id = finder(node, 'UploadId')
initiated = finder(node, 'Initiated')
yield OSSMultipartUpload(key, upload_id, initiated)
# Check if this is the last entry in the listing
# pylint: disable=maybe-no-member
is_truncated = body.findtext(
fixxpath(xpath='IsTruncated', namespace=self.namespace))
if is_truncated.lower() == 'false':
break
# Provide params for the next request
upload_marker = body.findtext(
fixxpath(xpath='NextUploadIdMarker', namespace=self.namespace))
key_marker = body.findtext(
fixxpath(xpath='NextKeyMarker', namespace=self.namespace))
params['key-marker'] = key_marker
params['upload-id-marker'] = upload_marker
def _upload_object(self,
object_name,
content_type,
upload_func,
upload_func_kwargs,
request_path,
request_method='PUT',
headers=None,
file_path=None,
iterator=None):
"""
Helper function for setting common request headers and calling the
passed in callback which uploads an object.
"""
headers = headers or {}
if file_path and not os.path.exists(file_path):
raise OSError('File %s does not exist' % (file_path))
if iterator is not None and not hasattr(iterator, 'next'):
raise AttributeError('iterator object must implement next() ' +
'method.')
if not content_type:
if file_path:
name = file_path
else:
name = object_name
content_type, _ = utils.guess_file_mime_type(name)
if not content_type:
raise AttributeError(
'File content-type could not be guessed and' +
' no content_type value provided')
file_size = None
if iterator:
if self.supports_chunked_encoding:
headers['Transfer-Encoding'] = 'chunked'
upload_func_kwargs['chunked'] = True
else:
# Chunked transfer encoding is not supported. Need to buffer all
# the data in memory so we can determine file size.
iterator = utils.read_in_chunks(iterator=iterator)
data = utils.exhaust_iterator(iterator=iterator)
file_size = len(data)
upload_func_kwargs['data'] = data
else:
file_size = os.path.getsize(file_path)
upload_func_kwargs['chunked'] = False
if file_size:
headers['Content-Length'] = file_size
headers['Content-Type'] = content_type
response = self.connection.request(request_path,
method=request_method,
data=None,
headers=headers,
raw=True)
upload_func_kwargs['response'] = response
success, data_hash, bytes_transferred = upload_func(
**upload_func_kwargs)
if not success:
raise LibcloudError(
value='Object upload failed, Perhaps a timeout?', driver=self)
result_dict = {
'response': response,
'data_hash': data_hash,
'bytes_transferred': bytes_transferred
}
return result_dict
def _put_object(self,
container,
object_name,
upload_func,
upload_func_kwargs,
method='PUT',
query_args=None,
extra=None,
file_path=None,
iterator=None,
verify_hash=False):
"""
Create an object and upload data using the given function.
"""
headers = {}
extra = extra or {}
content_type = extra.get('content_type', None)
meta_data = extra.get('meta_data', None)
acl = extra.get('acl', None)
if meta_data:
for key, value in list(meta_data.items()):
key = self.http_vendor_prefix + 'meta-%s' % (key)
headers[key] = value
if acl:
if acl not in ['public-read', 'private', 'public-read-write']:
raise AttributeError('invalid acl value: %s' % acl)
headers[self.http_vendor_prefix + 'object-acl'] = acl
request_path = self._get_object_path(container, object_name)
if query_args:
request_path = '?'.join((request_path, query_args))
# TODO: Let the underlying exceptions bubble up and capture the SIGPIPE
# here.
# SIGPIPE is thrown if the provided container does not exist or the
# user does not have correct permission
result_dict = self._upload_object(
object_name=object_name,
content_type=content_type,
upload_func=upload_func,
upload_func_kwargs=upload_func_kwargs,
request_path=request_path,
request_method=method,
headers=headers,
file_path=file_path,
iterator=iterator,
container=container)
response = result_dict['response']
bytes_transferred = result_dict['bytes_transferred']
headers = response.headers
response = response.response
server_hash = headers['etag'].replace('"', '')
if (verify_hash and result_dict['data_hash'].upper() != server_hash):
raise ObjectHashMismatchError(
value='MD5 hash checksum does not match',
object_name=object_name,
driver=self)
elif response.status == httplib.OK:
obj = Object(name=object_name,
size=bytes_transferred,
hash=server_hash,
extra={'acl': acl},
meta_data=meta_data,
container=container,
driver=self)
return obj
else:
raise LibcloudError('Unexpected status code, status_code=%s' %
(response.status),
driver=self)
def _upload_in_chunks(self,
response,
data,
iterator,
object_path,
blob_type,
lease,
calculate_hash=True):
"""
Uploads data from an interator in fixed sized chunks to S3
:param response: Response object from the initial POST request
:type response: :class:`RawResponse`
:param data: Any data from the initial POST request
:type data: ``str``
:param iterator: The generator for fetching the upload data
:type iterator: ``generator``
:param object_path: The path of the object to which we are uploading
:type object_name: ``str``
:param blob_type: The blob type being uploaded
:type blob_type: ``str``
:param lease: The lease object to be used for renewal
:type lease: :class:`AzureBlobLease`
:keyword calculate_hash: Indicates if we must calculate the data hash
:type calculate_hash: ``bool``
:return: A tuple of (status, checksum, bytes transferred)
:rtype: ``tuple``
"""
# Get the upload id from the response xml
if response.status != httplib.CREATED:
raise LibcloudError('Error initializing upload. Code: %d' %
(response.status),
driver=self)
data_hash = None
if calculate_hash:
data_hash = self._get_hash_function()
bytes_transferred = 0
count = 1
chunks = []
headers = {}
lease.update_headers(headers)
if blob_type == 'BlockBlob':
params = {'comp': 'block'}
else:
params = {'comp': 'page'}
# Read the input data in chunk sizes suitable for AWS
for data in read_in_chunks(iterator, AZURE_CHUNK_SIZE):
data = b(data)
content_length = len(data)
offset = bytes_transferred
bytes_transferred += content_length
if calculate_hash:
data_hash.update(data)
chunk_hash = self._get_hash_function()
chunk_hash.update(data)
chunk_hash = base64.b64encode(b(chunk_hash.digest()))
headers['Content-MD5'] = chunk_hash.decode('utf-8')
headers['Content-Length'] = str(content_length)
if blob_type == 'BlockBlob':
# Block id can be any unique string that is base64 encoded
# A 10 digit number can hold the max value of 50000 blocks
# that are allowed for azure
block_id = base64.b64encode(b('%10d' % (count)))
block_id = block_id.decode('utf-8')
params['blockid'] = block_id
# Keep this data for a later commit
chunks.append(block_id)
else:
headers['x-ms-page-write'] = 'update'
headers['x-ms-range'] = 'bytes=%d-%d' % \
(offset, (bytes_transferred - 1))
# Renew lease before updating
lease.renew()
resp = self.connection.request(object_path,
method='PUT',
data=data,
headers=headers,
params=params)
if resp.status != httplib.CREATED:
resp.parse_error()
raise LibcloudError('Error uploading chunk %d. Code: %d' %
(count, resp.status),
driver=self)
count += 1
if calculate_hash:
data_hash = data_hash.hexdigest()
if blob_type == 'BlockBlob':
self._commit_blocks(object_path, chunks, lease)
# The Azure service does not return a hash immediately for
# chunked uploads. It takes some time for the data to get synced
response.headers['content-md5'] = None
return (True, data_hash, bytes_transferred)
def _vcl_request(self, method, *args):
res = self.connection.request(method, *args)
if (res['status'] == 'error'):
raise LibcloudError(res['errormsg'], driver=self)
return res
def wait_until_running(self, nodes, wait_period=3, timeout=600,
ssh_interface='public_ips', force_ipv4=True):
"""
Block until the given nodes are fully booted and have an IP address
assigned.
:param nodes: list of node instances.
:type nodes: ``List`` of :class:`.Node`
:param wait_period: How many seconds to between each loop
iteration (default is 3)
:type wait_period: ``int``
:param timeout: How many seconds to wait before timing out
(default is 600)
:type timeout: ``int``
:param ssh_interface: The interface to wait for.
Default is 'public_ips', other option is
'private_ips'.
:type ssh_interface: ``str``
:param force_ipv4: Ignore ipv6 IP addresses (default is True).
:type force_ipv4: ``bool``
:return: ``[(Node, ip_addresses)]`` list of tuple of Node instance and
list of ip_address on success.
:rtype: ``list`` of ``tuple``
"""
def is_supported(address):
"""Return True for supported address"""
if force_ipv4 and not is_valid_ip_address(address=address,
family=socket.AF_INET):
return False
return True
def filter_addresses(addresses):
"""Return list of supported addresses"""
return [a for a in addresses if is_supported(a)]
start = time.time()
end = start + timeout
if ssh_interface not in ['public_ips', 'private_ips']:
raise ValueError('ssh_interface argument must either be' +
'public_ips or private_ips')
uuids = set([n.uuid for n in nodes])
while time.time() < end:
nodes = self.list_nodes()
nodes = list([n for n in nodes if n.uuid in uuids])
if len(nodes) > len(uuids):
found_uuids = [n.uuid for n in nodes]
msg = ('Unable to match specified uuids ' +
'(%s) with existing nodes. Found ' % (uuids) +
'multiple nodes with same uuid: (%s)' % (found_uuids))
raise LibcloudError(value=msg, driver=self)
running_nodes = [n for n in nodes if n.state == NodeState.RUNNING]
addresses = [filter_addresses(getattr(n, ssh_interface)) for n in
running_nodes]
if len(running_nodes) == len(uuids) == len(addresses):
return list(zip(running_nodes, addresses))
else:
time.sleep(wait_period)
continue
raise LibcloudError(value='Timed out after %s seconds' % (timeout),
driver=self)
def get_balancer(self, balancer_id):
balancers = self.list_balancers(ex_balancer_ids=[balancer_id])
if len(balancers) != 1:
raise LibcloudError('could not find load balancer with id %s' %
balancer_id)
return balancers[0]
def _put_object(self, container, object_name, upload_func,
upload_func_kwargs, method='PUT', query_args=None,
extra=None, file_path=None, iterator=None,
verify_hash=True, storage_class=None):
headers = {}
extra = extra or {}
storage_class = storage_class or 'standard'
if storage_class not in ['standard', 'reduced_redundancy']:
raise ValueError(
'Invalid storage class value: %s' % (storage_class))
headers['x-amz-storage-class'] = storage_class.upper()
content_type = extra.get('content_type', None)
meta_data = extra.get('meta_data', None)
acl = extra.get('acl', None)
if meta_data:
for key, value in list(meta_data.items()):
key = 'x-amz-meta-%s' % (key)
headers[key] = value
if acl:
headers['x-amz-acl'] = acl
request_path = self._get_object_path(container, object_name)
if query_args:
request_path = '?'.join((request_path, query_args))
# TODO: Let the underlying exceptions bubble up and capture the SIGPIPE
# here.
# SIGPIPE is thrown if the provided container does not exist or the
# user does not have correct permission
result_dict = self._upload_object(
object_name=object_name, content_type=content_type,
upload_func=upload_func, upload_func_kwargs=upload_func_kwargs,
request_path=request_path, request_method=method,
headers=headers, file_path=file_path, iterator=iterator)
response = result_dict['response']
bytes_transferred = result_dict['bytes_transferred']
headers = response.headers
response = response.response
server_hash = headers['etag'].replace('"', '')
if (verify_hash and result_dict['data_hash'] != server_hash):
raise ObjectHashMismatchError(
value='MD5 hash checksum does not match',
object_name=object_name, driver=self)
elif response.status == httplib.OK:
obj = Object(
name=object_name, size=bytes_transferred, hash=server_hash,
extra={'acl': acl}, meta_data=meta_data, container=container,
driver=self)
return obj
else:
raise LibcloudError(
'Unexpected status code, status_code=%s' % (response.status),
driver=self)
def _put_object(self,
container,
object_name,
method='PUT',
query_args=None,
extra=None,
file_path=None,
stream=None,
verify_hash=False):
"""
Create an object and upload data using the given function.
"""
headers = {}
extra = extra or {}
content_type = extra.get('content_type', None)
meta_data = extra.get('meta_data', None)
acl = extra.get('acl', None)
if meta_data:
for key, value in list(meta_data.items()):
key = self.http_vendor_prefix + 'meta-%s' % (key)
headers[key] = value
if acl:
if acl not in ['public-read', 'private', 'public-read-write']:
raise AttributeError('invalid acl value: %s' % acl)
headers[self.http_vendor_prefix + 'object-acl'] = acl
request_path = self._get_object_path(container, object_name)
if query_args:
request_path = '?'.join((request_path, query_args))
result_dict = self._upload_object(object_name=object_name,
content_type=content_type,
request_path=request_path,
request_method=method,
headers=headers,
file_path=file_path,
stream=stream)
response = result_dict['response']
bytes_transferred = result_dict['bytes_transferred']
headers = response.headers
server_hash = headers['etag'].replace('"', '')
if (verify_hash and result_dict['data_hash'] != server_hash):
raise ObjectHashMismatchError(
value='MD5 hash {0} checksum does not match {1}'.format(
server_hash, result_dict['data_hash']),
object_name=object_name,
driver=self)
elif response.status == httplib.OK:
obj = Object(name=object_name,
size=bytes_transferred,
hash=server_hash,
extra={'acl': acl},
meta_data=meta_data,
container=container,
driver=self)
return obj
else:
raise LibcloudError('Unexpected status code, status_code=%s' %
(response.status),
driver=self)
def async_request(self, action, params=None, data=None, headers=None,
method='GET', context=None):
"""
Perform an 'async' request to the specified path. Keep in mind that
this function is *blocking* and 'async' in this case means that the
hit URL only returns a job ID which is the periodically polled until
the job has completed.
This function works like this:
- Perform a request to the specified path. Response should contain a
'job_id'.
- Returned 'job_id' is then used to construct a URL which is used for
retrieving job status. Constructed URL is then periodically polled
until the response indicates that the job has completed or the
timeout of 'self.timeout' seconds has been reached.
:type action: ``str``
:param action: A path
:type params: ``dict``
:param params: Optional mapping of additional parameters to send. If
None, leave as an empty ``dict``.
:type data: ``unicode``
:param data: A body of data to send with the request.
:type headers: ``dict``
:param headers: Extra headers to add to the request
None, leave as an empty ``dict``.
:type method: ``str``
:param method: An HTTP method such as "GET" or "POST".
:type context: ``dict``
:param context: Context dictionary which is passed to the functions
which construct initial and poll URL.
:return: An :class:`Response` instance.
:rtype: :class:`Response` instance
"""
request = getattr(self, self.request_method)
kwargs = self.get_request_kwargs(action=action, params=params,
data=data, headers=headers,
method=method,
context=context)
response = request(**kwargs)
kwargs = self.get_poll_request_kwargs(response=response,
context=context,
request_kwargs=kwargs)
end = time.time() + self.timeout
completed = False
while time.time() < end and not completed:
response = request(**kwargs)
completed = self.has_completed(response=response)
if not completed:
time.sleep(self.poll_interval)
if not completed:
raise LibcloudError('Job did not complete in %s seconds' %
(self.timeout))
return response
def wait_until_running(self,
nodes,
wait_period=3,
timeout=600,
ssh_interface='public_ips',
force_ipv4=True):
"""
Block until the provided nodes are considered running.
Node is considered running when it's state is "running" and when it has
at least one IP address assigned.
:param nodes: List of nodes to wait for.
:type nodes: ``list`` of :class:`.Node`
:param wait_period: How many seconds to wait between each loop
iteration. (default is 3)
:type wait_period: ``int``
:param timeout: How many seconds to wait before giving up.
(default is 600)
:type timeout: ``int``
:param ssh_interface: Which attribute on the node to use to obtain
an IP address. Valid options: public_ips,
private_ips. Default is public_ips.
:type ssh_interface: ``str``
:param force_ipv4: Ignore IPv6 addresses (default is True).
:type force_ipv4: ``bool``
:return: ``[(Node, ip_addresses)]`` list of tuple of Node instance and
list of ip_address on success.
:rtype: ``list`` of ``tuple``
"""
def is_supported(address):
"""
Return True for supported address.
"""
if force_ipv4 and not is_valid_ip_address(address=address,
family=socket.AF_INET):
return False
return True
def filter_addresses(addresses):
"""
Return list of supported addresses.
"""
return [address for address in addresses if is_supported(address)]
if ssh_interface not in ['public_ips', 'private_ips']:
raise ValueError('ssh_interface argument must either be' +
'public_ips or private_ips')
start = time.time()
end = start + timeout
uuids = set([node.uuid for node in nodes])
while time.time() < end:
all_nodes = self.list_nodes()
matching_nodes = list(
[node for node in all_nodes if node.uuid in uuids])
if len(matching_nodes) > len(uuids):
found_uuids = [node.uuid for node in matching_nodes]
msg = ('Unable to match specified uuids ' +
'(%s) with existing nodes. Found ' % (uuids) +
'multiple nodes with same uuid: (%s)' % (found_uuids))
raise LibcloudError(value=msg, driver=self)
running_nodes = [
node for node in matching_nodes
if node.state == NodeState.RUNNING
]
addresses = [
filter_addresses(getattr(node, ssh_interface))
for node in running_nodes
]
if len(running_nodes) == len(uuids) == len(addresses):
return list(zip(running_nodes, addresses))
else:
time.sleep(wait_period)
continue
raise LibcloudError(value='Timed out after %s seconds' % (timeout),
driver=self)
def _get_first_ip(self, location=None):
ips = self.ex_list_ips(public=True, assigned=False, location=location)
try:
return ips[0].ip
except IndexError:
raise LibcloudError('No public unassigned IPs left', self.driver)
def parse_error(self):
if self.status in [httplib.UNAUTHORIZED]:
raise InvalidCredsError(self.body)
raise LibcloudError('Unknown error. Status code: %d' % (self.status),
driver=self.connection.driver)
def _upload_in_chunks(self, stream, object_path, lease, meta_data,
content_type, object_name, file_path, verify_hash,
headers):
"""
Uploads data from an interator in fixed sized chunks to Azure Storage
"""
data_hash = None
if verify_hash:
data_hash = self._get_hash_function()
bytes_transferred = 0
count = 1
chunks = []
headers = headers or {}
lease.update_headers(headers)
params = {'comp': 'block'}
# Read the input data in chunk sizes suitable for Azure
for data in read_in_chunks(stream,
AZURE_UPLOAD_CHUNK_SIZE,
fill_size=True):
data = b(data)
content_length = len(data)
bytes_transferred += content_length
if verify_hash:
data_hash.update(data)
chunk_hash = self._get_hash_function()
chunk_hash.update(data)
chunk_hash = base64.b64encode(b(chunk_hash.digest()))
headers['Content-MD5'] = chunk_hash.decode('utf-8')
headers['Content-Length'] = str(content_length)
# Block id can be any unique string that is base64 encoded
# A 10 digit number can hold the max value of 50000 blocks
# that are allowed for azure
block_id = base64.b64encode(b('%10d' % (count)))
block_id = block_id.decode('utf-8')
params['blockid'] = block_id
# Keep this data for a later commit
chunks.append(block_id)
# Renew lease before updating
lease.renew()
resp = self.connection.request(object_path,
method='PUT',
data=data,
headers=headers,
params=params)
if resp.status != httplib.CREATED:
resp.parse_error()
raise LibcloudError('Error uploading chunk %d. Code: %d' %
(count, resp.status),
driver=self)
count += 1
if verify_hash:
data_hash = base64.b64encode(b(data_hash.digest()))
data_hash = data_hash.decode('utf-8')
response = self._commit_blocks(object_path=object_path,
chunks=chunks,
lease=lease,
meta_data=meta_data,
content_type=content_type,
data_hash=data_hash,
object_name=object_name,
file_path=file_path)
# According to the Azure docs:
# > This header refers to the content of the request, meaning, in this
# > case, the list of blocks, and not the content of the blob itself.
# However, the validation code assumes that the content-md5 in the
# server response refers to the object so we must discard the value
response.headers['content-md5'] = None
return {
'response': response,
'data_hash': data_hash,
'bytes_transferred': bytes_transferred,
}
def _put_object(self,
container,
object_name,
method='PUT',
query_args=None,
extra=None,
file_path=None,
stream=None,
verify_hash=True,
storage_class=None):
headers = {}
extra = extra or {}
headers.update(self._to_storage_class_headers(storage_class))
content_type = extra.get('content_type', None)
meta_data = extra.get('meta_data', None)
acl = extra.get('acl', None)
if meta_data:
for key, value in list(meta_data.items()):
key = self.http_vendor_prefix + '-meta-%s' % (key)
headers[key] = value
if acl:
headers[self.http_vendor_prefix + '-acl'] = acl
request_path = self._get_object_path(container, object_name)
if query_args:
request_path = '?'.join((request_path, query_args))
result_dict = self._upload_object(object_name=object_name,
content_type=content_type,
request_path=request_path,
request_method=method,
headers=headers,
file_path=file_path,
stream=stream)
response = result_dict['response']
bytes_transferred = result_dict['bytes_transferred']
headers = response.headers
response = response
server_hash = headers.get('etag', '').replace('"', '')
if (verify_hash and result_dict['data_hash'] != server_hash):
raise ObjectHashMismatchError(
value='MD5 hash {0} checksum does not match {1}'.format(
server_hash, result_dict['data_hash']),
object_name=object_name,
driver=self)
elif response.status == httplib.OK:
obj = Object(name=object_name,
size=bytes_transferred,
hash=server_hash,
extra={'acl': acl},
meta_data=meta_data,
container=container,
driver=self)
return obj
else:
raise LibcloudError('Unexpected status code, status_code=%s' %
(response.status),
driver=self)
def _put_object(self,
container,
object_name,
upload_func,
upload_func_kwargs,
extra=None,
file_path=None,
iterator=None,
verify_hash=True):
extra = extra or {}
container_name_cleaned = self._clean_container_name(container.name)
object_name_cleaned = self._clean_object_name(object_name)
content_type = extra.get('content_type', None)
meta_data = extra.get('meta_data', None)
headers = {}
if meta_data:
for key, value in list(meta_data.items()):
key = 'X-Object-Meta-%s' % (key)
headers[key] = value
request_path = '/%s/%s' % (container_name_cleaned, object_name_cleaned)
result_dict = self._upload_object(
object_name=object_name,
content_type=content_type,
upload_func=upload_func,
upload_func_kwargs=upload_func_kwargs,
request_path=request_path,
request_method='PUT',
headers=headers,
file_path=file_path,
iterator=iterator)
response = result_dict['response'].response
bytes_transferred = result_dict['bytes_transferred']
server_hash = result_dict['response'].headers.get('etag', None)
if response.status == httplib.EXPECTATION_FAILED:
raise LibcloudError(value='Missing content-type header',
driver=self)
elif verify_hash and not server_hash:
raise LibcloudError(value='Server didn\'t return etag',
driver=self)
elif (verify_hash and result_dict['data_hash'] != server_hash):
raise ObjectHashMismatchError(
value=('MD5 hash checksum does not match (expected=%s, ' +
'actual=%s)') % (result_dict['data_hash'], server_hash),
object_name=object_name,
driver=self)
elif response.status == httplib.CREATED:
obj = Object(name=object_name,
size=bytes_transferred,
hash=server_hash,
extra=None,
meta_data=meta_data,
container=container,
driver=self)
return obj
else:
# @TODO: Add test case for this condition (probably 411)
raise LibcloudError('status_code=%s' % (response.status),
driver=self)
def ex_configure_load_balancer(self, balancer, port=80,
protocol='http',
algorithm=DEFAULT_ALGORITHM,
ex_allocation=100):
"""
Configure the loadbalancer by adding it with a front-end port (aka
a service group in the Softlayer loadbalancer model).
Softlayer loadbalancer may be defined with multiple service
groups (front-end ports) each defined with a unique port number.
:param balancer: The loadbalancer.
:type balancer: :class:`LoadBalancer`
:param port: Port of the service group, defaults to 80.
:type port: ``int``
:param protocol: Loadbalancer protocol, defaults to http.
:type protocol: ``str``
:param algorithm: Load balancing algorithm, defaults to
Algorithm.ROUND_ROBIN
:type algorithm: :class:`Algorithm`
:param ex_allocation: The percentage of the total connection
allocations to allocate for this group.
:type ex_allocation: ``int``
:return: ``True`` if ex_add_service_group was successful.
:rtype: ``bool``
"""
_types = self._get_routing_types()
_methods = self._get_routing_methods()
rt = find(_types, lambda t: t['keyname'] == protocol.upper())
if not rt:
raise LibcloudError(value='Invalid protocol %s' % protocol,
driver=self)
value = self._algorithm_to_value(algorithm)
meth = find(_methods, lambda m: m['keyname'] == value)
if not meth:
raise LibcloudError(value='Invalid algorithm %s' % algorithm,
driver=self)
service_group_template = {
'port': port,
'allocation': ex_allocation,
'serviceGroups': [{
'routingTypeId': rt['id'],
'routingMethodId': meth['id']
}]
}
lb = self._get_balancer_model(balancer.id)
if len(lb['virtualServers']) > 0:
port = lb['virtualServers'][0]['port']
raise LibcloudError(value='Loadbalancer already configured with '
'a service group (front-end port)' % port,
driver=self)
lb['virtualServers'].append(service_group_template)
self.connection.request(lb_service, 'editObject', lb, id=balancer.id)
return True
def _put_object(
self,
container,
object_name,
extra=None,
file_path=None,
stream=None,
verify_hash=True,
headers=None,
):
extra = extra or {}
container_name_encoded = self._encode_container_name(container.name)
object_name_encoded = self._encode_object_name(object_name)
content_type = extra.get("content_type", None)
meta_data = extra.get("meta_data", None)
content_disposition = extra.get("content_disposition", None)
headers = headers or {}
if meta_data:
for key, value in list(meta_data.items()):
key = "X-Object-Meta-%s" % (key)
headers[key] = value
if content_disposition is not None:
headers["Content-Disposition"] = content_disposition
request_path = "/%s/%s" % (container_name_encoded, object_name_encoded)
result_dict = self._upload_object(
object_name=object_name,
content_type=content_type,
request_path=request_path,
request_method="PUT",
headers=headers,
file_path=file_path,
stream=stream,
)
response = result_dict["response"]
bytes_transferred = result_dict["bytes_transferred"]
server_hash = result_dict["response"].headers.get("etag", None)
if response.status == httplib.EXPECTATION_FAILED:
raise LibcloudError(value="Missing content-type header",
driver=self)
elif verify_hash and not server_hash:
raise LibcloudError(value="Server didn't return etag", driver=self)
elif verify_hash and result_dict["data_hash"] != server_hash:
raise ObjectHashMismatchError(
value=("MD5 hash checksum does not match (expected=%s, " +
"actual=%s)") % (result_dict["data_hash"], server_hash),
object_name=object_name,
driver=self,
)
elif response.status == httplib.CREATED:
obj = Object(
name=object_name,
size=bytes_transferred,
hash=server_hash,
extra=None,
meta_data=meta_data,
container=container,
driver=self,
)
return obj
else:
# @TODO: Add test case for this condition (probably 411)
raise LibcloudError("status_code=%s" % (response.status),
driver=self)
def _upload_from_iterator(self, iterator, object_path, upload_id,
calculate_hash=True):
"""
Uploads data from an interator in fixed sized chunks to S3
:param iterator: The generator for fetching the upload data
:type iterator: ``generator``
:param object_path: The path of the object to which we are uploading
:type object_name: ``str``
:param upload_id: The upload id allocated for this multipart upload
:type upload_id: ``str``
:keyword calculate_hash: Indicates if we must calculate the data hash
:type calculate_hash: ``bool``
:return: A tuple of (chunk info, checksum, bytes transferred)
:rtype: ``tuple``
"""
data_hash = None
if calculate_hash:
data_hash = self._get_hash_function()
bytes_transferred = 0
count = 1
chunks = []
params = {'uploadId': upload_id}
# Read the input data in chunk sizes suitable for AWS
for data in read_in_chunks(iterator, chunk_size=CHUNK_SIZE,
fill_size=True, yield_empty=True):
bytes_transferred += len(data)
if calculate_hash:
data_hash.update(data)
chunk_hash = self._get_hash_function()
chunk_hash.update(data)
chunk_hash = base64.b64encode(chunk_hash.digest()).decode('utf-8')
# This provides an extra level of data check and is recommended
# by amazon
headers = {'Content-MD5': chunk_hash}
params['partNumber'] = count
request_path = '?'.join((object_path, urlencode(params)))
resp = self.connection.request(request_path, method='PUT',
data=data, headers=headers)
if resp.status != httplib.OK:
raise LibcloudError('Error uploading chunk', driver=self)
server_hash = resp.headers['etag']
# Keep this data for a later commit
chunks.append((count, server_hash))
count += 1
if calculate_hash:
data_hash = data_hash.hexdigest()
return (chunks, data_hash, bytes_transferred)
def __enter__(self):
try:
self.lock.acquire(timeout=0.1)
except LockTimeout:
raise LibcloudError('Lock timeout')
def _put_object(self,
container,
object_name,
object_size,
upload_func,
upload_func_kwargs,
file_path=None,
extra=None,
verify_hash=True,
blob_type=None,
use_lease=False):
"""
Control function that does the real job of uploading data to a blob
"""
extra = extra or {}
meta_data = extra.get('meta_data', {})
content_type = extra.get('content_type', None)
headers = self._prepare_upload_headers(object_name, object_size, extra,
meta_data, blob_type)
object_path = self._get_object_path(container, object_name)
# Get a lease if required and do the operations
with AzureBlobLease(self, object_path, use_lease) as lease:
if 'lease' in upload_func_kwargs:
upload_func_kwargs['lease'] = lease
lease.update_headers(headers)
iterator = iter('')
result_dict = self._upload_object(object_name,
content_type,
upload_func,
upload_func_kwargs,
object_path,
headers=headers,
file_path=file_path,
iterator=iterator)
response = result_dict['response']
bytes_transferred = result_dict['bytes_transferred']
data_hash = result_dict['data_hash']
headers = response.headers
response = response.response
if response.status != httplib.CREATED:
raise LibcloudError('Unexpected status code, status_code=%s' %
(response.status),
driver=self)
server_hash = headers['content-md5']
if server_hash:
server_hash = binascii.hexlify(base64.b64decode(b(server_hash)))
server_hash = server_hash.decode('utf-8')
else:
# TODO: HACK - We could poll the object for a while and get
# the hash
pass
if (verify_hash and server_hash and data_hash != server_hash):
raise ObjectHashMismatchError(
value='MD5 hash checksum does not match',
object_name=object_name,
driver=self)
return Object(name=object_name,
size=bytes_transferred,
hash=headers['etag'],
extra=None,
meta_data=meta_data,
container=container,
driver=self)
