def _select_datastore_summary(self, size_bytes, datastores):
"""Get best summary from datastore list that can accomodate volume.
The implementation selects datastore based on maximum relative
free space, which is (free_space/total_space) and has free space to
store the volume backing.
:param size_bytes: Size in bytes of the volume
:param datastores: Datastores from which a choice is to be made
for the volume
:return: Best datastore summary to be picked for the volume
"""
best_summary = None
best_ratio = 0
for datastore in datastores:
summary = self.volumeops.get_summary(datastore)
if summary.freeSpace > size_bytes:
ratio = float(summary.freeSpace) / summary.capacity
if ratio > best_ratio:
best_ratio = ratio
best_summary = summary
if not best_summary:
msg = _("Unable to pick datastore to accomodate %(size)s bytes "
"from the datastores: %(dss)s.")
LOG.error(msg % {'size': size_bytes, 'dss': datastores})
raise error_util.VimException(msg % {
'size': size_bytes,
'dss': datastores
})
LOG.debug(_("Selected datastore: %s for the volume.") % best_summary)
return best_summary
def _create_backing_in_inventory(self, volume):
"""Creates backing under any suitable host.
The method tries to pick datastore that can fit the volume under
any host in the inventory.
:param volume: Volume object
:return: Reference to the created backing
"""
retrv_result = self.volumeops.get_hosts()
while retrv_result:
hosts = retrv_result.objects
if not hosts:
break
backing = None
for host in hosts:
try:
backing = self._create_backing(volume, host.obj)
if backing:
break
except error_util.VimException as excep:
LOG.warn(_("Unable to find suitable datastore for "
"volume: %(vol)s under host: %(host)s. "
"More details: %(excep)s") %
{'vol': volume['name'],
'host': host.obj, 'excep': excep})
if backing:
self.volumeops.cancel_retrieval(retrv_result)
return backing
retrv_result = self.volumeops.continue_retrieval(retrv_result)
msg = _("Unable to create volume: %s in the inventory.")
LOG.error(msg % volume['name'])
raise error_util.VimException(msg % volume['name'])
def get_dss_rp(self, host):
"""Get datastores and resource pool of the host.
:param host: Managed object reference of the host
:return: Datastores mounted to the host and resource pool to which
the host belongs to
"""
props = self._session.invoke_api(vim_util, 'get_object_properties',
self._session.vim, host,
['datastore', 'parent'])
# Get datastores and compute resource or cluster compute resource
datastores = None
compute_resource = None
for elem in props:
for prop in elem.propSet:
if prop.name == 'datastore':
datastores = prop.val.ManagedObjectReference
elif prop.name == 'parent':
compute_resource = prop.val
# Get resource pool from compute resource or cluster compute resource
resource_pool = self._session.invoke_api(vim_util,
'get_object_property',
self._session.vim,
compute_resource,
'resourcePool')
if not datastores:
msg = _("There are no datastores present under %s.")
LOG.error(msg % host)
raise error_util.VimException(msg % host)
return (datastores, resource_pool)
def __init__(self, session, host, rp_ref, vm_folder_ref, vm_create_spec,
vmdk_size):
"""Initialize a writer for vmdk file.
:param session: a valid api session to ESX/VC server
:param host: the ESX or VC host IP
:param rp_ref: resource pool into which backing VM is imported
:param vm_folder_ref: VM folder in ESX/VC inventory to use as parent
of backing VM
:param vm_create_spec: backing VM created using this create spec
:param vmdk_size: VMDK size to be imported into backing VM
"""
self._session = session
self._vmdk_size = vmdk_size
self._progress = 0
lease = session.invoke_api(session.vim,
'ImportVApp',
rp_ref,
spec=vm_create_spec,
folder=vm_folder_ref)
session.wait_for_lease_ready(lease)
self._lease = lease
lease_info = session.invoke_api(vim_util, 'get_object_property',
session.vim, lease, 'info')
self._vm_ref = lease_info.entity
# Find the url for vmdk device
url = self.find_vmdk_url(lease_info, host)
if not url:
msg = _("Could not retrieve URL from lease.")
LOG.exception(msg)
raise error_util.VimException(msg)
LOG.info(_LI("Opening vmdk url: %s for write.") % url)
# Prepare the http connection to the vmdk url
cookies = session.vim.client.options.transport.cookiejar
_urlparse = urlparse.urlparse(url)
scheme, netloc, path, _params, query, _fragment = _urlparse
if scheme == 'http':
conn = httplib.HTTPConnection(netloc)
elif scheme == 'https':
conn = httplib.HTTPSConnection(netloc)
if query:
path = path + '?' + query
conn.putrequest('PUT', path)
conn.putheader('User-Agent', USER_AGENT)
conn.putheader('Content-Length', str(vmdk_size))
conn.putheader('Overwrite', 't')
conn.putheader('Cookie', self._build_vim_cookie_headers(cookies))
conn.putheader('Content-Type', 'binary/octet-stream')
conn.endheaders()
self.conn = conn
VMwareHTTPFile.__init__(self, conn)
def _create_backing_in_inventory(self, volume):
"""Creates backing under any suitable host.
The method tries to pick datastore that can fit the volume under
any host in the inventory.
:param volume: Volume object
:return: Reference to the created backing
"""
# Get all hosts
hosts = self.volumeops.get_hosts()
if not hosts:
msg = _("There are no hosts in the inventory.")
LOG.error(msg)
raise error_util.VimException(msg)
backing = None
for host in hosts:
try:
host = hosts[0].obj
backing = self._create_backing(volume, host)
break
except error_util.VimException as excep:
LOG.warn(
_("Unable to find suitable datastore for "
"volume: %(vol)s under host: %(host)s. "
"More details: %(excep)s") % {
'vol': volume['name'],
'host': host,
'excep': excep
})
if backing:
return backing
msg = _("Unable to create volume: %(vol)s on the hosts: %(hosts)s.")
LOG.error(msg % {'vol': volume['name'], 'hosts': hosts})
raise error_util.VimException(msg % {
'vol': volume['name'],
'hosts': hosts
})
def _select_datastore_summary(self, size_bytes, datastores):
"""Get the best datastore summary from the given datastore list.
The implementation selects a datastore which is connected to maximum
number of hosts, provided there is enough space to accommodate the
volume. Ties are broken based on space utilization; datastore with
low space utilization is preferred.
:param size_bytes: Size in bytes of the volume
:param datastores: Datastores from which a choice is to be made
for the volume
:return: Summary of the best datastore selected for volume
"""
best_summary = None
max_host_count = 0
best_space_utilization = 1.0
for datastore in datastores:
summary = self.volumeops.get_summary(datastore)
if summary.freeSpace > size_bytes:
host_count = len(self.volumeops.get_connected_hosts(datastore))
if host_count > max_host_count:
max_host_count = host_count
best_space_utilization = self._compute_space_utilization(
summary)
best_summary = summary
elif host_count == max_host_count:
# break the tie based on space utilization
space_utilization = self._compute_space_utilization(
summary)
if space_utilization < best_space_utilization:
best_space_utilization = space_utilization
best_summary = summary
if not best_summary:
msg = _("Unable to pick datastore to accommodate %(size)s bytes "
"from the datastores: %(dss)s.") % {
'size': size_bytes,
'dss': datastores
}
LOG.error(msg)
raise error_util.VimException(msg)
LOG.debug(
_("Selected datastore: %(datastore)s with %(host_count)d "
"connected host(s) for the volume.") % {
'datastore': best_summary,
'host_count': max_host_count
})
return best_summary
def test_invoke_api_with_expected_exception(self):
api_session = self._create_api_session(True)
ret = mock.Mock()
responses = [error_util.VimException(None), ret]
def api(*args, **kwargs):
response = responses.pop(0)
if isinstance(response, Exception):
raise response
return response
module = mock.Mock()
module.api = api
with mock.patch.object(greenthread, 'sleep'):
self.assertEqual(ret, api_session.invoke_api(module, 'api'))
def get_dss_rp(self, host):
"""Get accessible datastores and resource pool of the host.
:param host: Managed object reference of the host
:return: Datastores accessible to the host and resource pool to which
the host belongs to
"""
props = self._session.invoke_api(vim_util, 'get_object_properties',
self._session.vim, host,
['datastore', 'parent'])
# Get datastores and compute resource or cluster compute resource
datastores = []
compute_resource = None
for elem in props:
for prop in elem.propSet:
if prop.name == 'datastore' and prop.val:
# Consider only if datastores are present under host
datastores = prop.val.ManagedObjectReference
elif prop.name == 'parent':
compute_resource = prop.val
LOG.debug(_("Datastores attached to host %(host)s are: %(ds)s."), {
'host': host,
'ds': datastores
})
# Filter datastores based on if it is accessible, mounted and writable
valid_dss = []
for datastore in datastores:
if self._is_valid(datastore, host):
valid_dss.append(datastore)
# Get resource pool from compute resource or cluster compute resource
resource_pool = self._session.invoke_api(vim_util,
'get_object_property',
self._session.vim,
compute_resource,
'resourcePool')
if not valid_dss:
msg = _("There are no valid datastores attached to %s.") % host
LOG.error(msg)
raise error_util.VimException(msg)
else:
LOG.debug(_("Valid datastores are: %s"), valid_dss)
return (valid_dss, resource_pool)
def _select_ds_for_volume(self, size_gb):
"""Select datastore that can accommodate a volume of given size.
Returns the selected datastore summary along with a compute host and
its resource pool and folder where the volume can be created
:return: (host, rp, folder, summary)
"""
retrv_result = self.volumeops.get_hosts()
while retrv_result:
hosts = retrv_result.objects
if not hosts:
break
(selected_host, rp, folder, summary) = (None, None, None, None)
for host in hosts:
host = host.obj
try:
(dss, rp) = self.volumeops.get_dss_rp(host)
(folder,
summary) = self._get_folder_ds_summary(size_gb, rp, dss)
selected_host = host
break
except error_util.VimException as excep:
LOG.warn(
_("Unable to find suitable datastore for volume "
"of size: %(vol)s GB under host: %(host)s. "
"More details: %(excep)s") % {
'vol': size_gb,
'host': host.obj,
'excep': excep
})
if selected_host:
self.volumeops.cancel_retrieval(retrv_result)
return (selected_host, rp, folder, summary)
retrv_result = self.volumeops.continue_retrieval(retrv_result)
msg = _("Unable to find host to accommodate a disk of size: %s "
"in the inventory.") % size_gb
LOG.error(msg)
raise error_util.VimException(msg)
def __init__(self, session, host, vm_ref, vmdk_path, vmdk_size):
"""Initialize a writer for vmdk file.
During an export operation the vmdk disk is converted to a
stream-optimized sparse disk format. So the size of the VMDK
after export may be smaller than the current vmdk disk size.
:param session: a valid api session to ESX/VC server
:param host: the ESX or VC host IP
:param vm_ref: backing VM whose vmdk is to be exported
:param vmdk_path: datastore relative path to vmdk file to be exported
:param vmdk_size: current disk size of vmdk file to be exported
"""
self._session = session
self._vmdk_size = vmdk_size
self._progress = 0
lease = session.invoke_api(session.vim, 'ExportVm', vm_ref)
session.wait_for_lease_ready(lease)
self._lease = lease
lease_info = session.invoke_api(vim_util, 'get_object_property',
session.vim, lease, 'info')
# find the right disk url corresponding to given vmdk_path
url = self.find_vmdk_url(lease_info, host)
if not url:
msg = _("Could not retrieve URL from lease.")
LOG.exception(msg)
raise error_util.VimException(msg)
LOG.info(_("Opening vmdk url: %s for read.") % url)
cookies = session.vim.client.options.transport.cookiejar
headers = {
'User-Agent': USER_AGENT,
'Cookie': self._build_vim_cookie_headers(cookies)
}
request = urllib2.Request(url, None, headers)
conn = urllib2.urlopen(request)
VMwareHTTPFile.__init__(self, conn)
def func(*args, **kwargs):
raise error_util.VimException(None)
def vim_request_handler(managed_object, **kwargs):
"""Handler for VI SDK calls.
Builds the SOAP message and parses the response for fault
checking and other errors.
:param managed_object:Managed object reference
:param kwargs: Keyword arguments of the call
:return: Response of the API call
"""
try:
if isinstance(managed_object, str):
# For strings use string value for value and type
# of the managed object.
managed_object = get_moref(managed_object, managed_object)
request = getattr(self.client.service, attr_name)
response = request(managed_object, **kwargs)
if (attr_name.lower() == 'retrievepropertiesex'):
retrieve_properties_ex_fault_checker(response)
return response
except error_util.VimFaultException as excep:
raise
except suds.WebFault as excep:
doc = excep.document
detail = doc.childAtPath('/Envelope/Body/Fault/detail')
fault_list = []
for child in detail.getChildren():
fault_list.append(child.get('type'))
raise error_util.VimFaultException(fault_list, excep)
except AttributeError as excep:
raise error_util.VimAttributeException(
_("No such SOAP method "
"%(attr)s. Detailed "
"error: %(excep)s.") % {
'attr': attr_name,
'excep': excep
})
except (httplib.CannotSendRequest, httplib.ResponseNotReady,
httplib.CannotSendHeader) as excep:
raise error_util.SessionOverLoadException(
_("httplib error in "
"%(attr)s: "
"%(excep)s.") % {
'attr': attr_name,
'excep': excep
})
except (urllib2.URLError, urllib2.HTTPError) as excep:
raise error_util.VimConnectionException(
_("urllib2 error in %(attr)s: %(excep)s.") % {
'attr': attr_name,
'excep': excep
})
except Exception as excep:
# Socket errors which need special handling for they
# might be caused by server API call overload
if (str(excep).find(ADDRESS_IN_USE_ERROR) != -1
or str(excep).find(CONN_ABORT_ERROR)) != -1:
raise error_util.SessionOverLoadException(
_("Socket error "
"in %(attr)s: "
"%(excep)s.") % {
'attr': attr_name,
'excep': excep
})
# Type error that needs special handling for it might be
# caused by server API call overload
elif str(excep).find(RESP_NOT_XML_ERROR) != -1:
raise error_util.SessionOverLoadException(
_("Type error "
"in %(attr)s: "
"%(excep)s.") % {
'attr': attr_name,
'excep': excep
})
else:
raise error_util.VimException(
_("Error in %(attr)s. "
"Detailed error: "
"%(excep)s.") % {
'attr': attr_name,
'excep': excep
})
def test_select_datastore(self, filter_datastores, get_profile_id):
# Test with no hosts.
size_bytes = units.Ki
req = {self._ds_sel.SIZE_BYTES: size_bytes}
self._vops.get_hosts.return_value = mock.Mock(objects=[])
self.assertEqual((), self._ds_sel.select_datastore(req))
self._vops.get_hosts.assert_called_once_with()
# Test with single host with no valid datastores.
host_1 = mock.sentinel.host_1
self._vops.get_hosts.return_value = mock.Mock(
objects=[mock.Mock(obj=host_1)])
self._vops.continue_retrieval.return_value = None
self._vops.get_dss_rp.side_effect = error_util.VimException('error')
self.assertEqual((), self._ds_sel.select_datastore(req))
self._vops.get_dss_rp.assert_called_once_with(host_1)
# Test with three hosts and vCenter connection problem while fetching
# datastores for the second host.
self._vops.get_dss_rp.reset_mock()
host_2 = mock.sentinel.host_2
host_3 = mock.sentinel.host_3
self._vops.get_hosts.return_value = mock.Mock(objects=[
mock.Mock(obj=host_1),
mock.Mock(obj=host_2),
mock.Mock(obj=host_3)
])
self._vops.get_dss_rp.side_effect = [
error_util.VimException('no valid datastores'),
error_util.VimConnectionException('connection error')
]
self.assertRaises(error_util.VimConnectionException,
self._ds_sel.select_datastore, req)
get_dss_rp_exp_calls = [mock.call(host_1), mock.call(host_2)]
self.assertEqual(get_dss_rp_exp_calls,
self._vops.get_dss_rp.call_args_list)
# Modify previous case to return datastores for second and third host,
# where none of them meet the requirements which include a storage
# profile and affinity requirements.
aff_ds_types = [ds_sel.DatastoreType.VMFS]
req[ds_sel.DatastoreSelector.HARD_AFFINITY_DS_TYPE] = aff_ds_types
ds_1a = mock.sentinel.ds_1a
anti_affinity_ds = [ds_1a]
req[ds_sel.DatastoreSelector.HARD_ANTI_AFFINITY_DS] = anti_affinity_ds
profile_name = mock.sentinel.profile_name
req[ds_sel.DatastoreSelector.PROFILE_NAME] = profile_name
profile_id = mock.sentinel.profile_id
get_profile_id.return_value = profile_id
ds_2a = mock.sentinel.ds_2a
ds_2b = mock.sentinel.ds_2b
ds_3a = mock.sentinel.ds_3a
self._vops.get_dss_rp.reset_mock()
rp_2 = mock.sentinel.rp_2
rp_3 = mock.sentinel.rp_3
self._vops.get_dss_rp.side_effect = [
error_util.VimException('no valid datastores'),
([ds_2a, ds_2b], rp_2), ([ds_3a], rp_3)
]
filter_datastores.return_value = []
self.assertEqual((), self._ds_sel.select_datastore(req))
get_profile_id.assert_called_once_with(profile_name)
get_dss_rp_exp_calls.append(mock.call(host_3))
self.assertEqual(get_dss_rp_exp_calls,
self._vops.get_dss_rp.call_args_list)
filter_datastores_exp_calls = [
mock.call([ds_2a, ds_2b], size_bytes, profile_id, anti_affinity_ds,
aff_ds_types),
mock.call([ds_3a], size_bytes, profile_id, anti_affinity_ds,
aff_ds_types)
]
self.assertEqual(filter_datastores_exp_calls,
filter_datastores.call_args_list)
# Modify previous case to have a non-empty summary list after filtering
# with preferred utilization threshold unset.
self._vops.get_dss_rp.side_effect = [
error_util.VimException('no valid datastores'),
([ds_2a, ds_2b], rp_2), ([ds_3a], rp_3)
]
summary_2b = self._create_summary(ds_2b,
free_space=0.5 * units.Mi,
capacity=units.Mi)
filter_datastores.side_effect = [[summary_2b]]
self._vops.get_connected_hosts.return_value = [host_1]
self.assertEqual((host_2, rp_2, summary_2b),
self._ds_sel.select_datastore(req))
# Modify previous case to have a preferred utilization threshold
# satsified by one datastore.
self._vops.get_dss_rp.side_effect = [
error_util.VimException('no valid datastores'),
([ds_2a, ds_2b], rp_2), ([ds_3a], rp_3)
]
req[ds_sel.DatastoreSelector.PREF_UTIL_THRESH] = 0.4
summary_3a = self._create_summary(ds_3a,
free_space=0.7 * units.Mi,
capacity=units.Mi)
filter_datastores.side_effect = [[summary_2b], [summary_3a]]
self.assertEqual((host_3, rp_3, summary_3a),
self._ds_sel.select_datastore(req))
# Modify previous case to have a preferred utilization threshold
# which cannot be satisfied.
self._vops.get_dss_rp.side_effect = [
error_util.VimException('no valid datastores'),
([ds_2a, ds_2b], rp_2), ([ds_3a], rp_3)
]
filter_datastores.side_effect = [[summary_2b], [summary_3a]]
req[ds_sel.DatastoreSelector.PREF_UTIL_THRESH] = 0.2
summary_2b.freeSpace = 0.75 * units.Mi
self.assertEqual((host_2, rp_2, summary_2b),
self._ds_sel.select_datastore(req))
# Clear side effects.
self._vops.get_dss_rp.side_effect = None
