def _reallocate(self, allocation):
"""Allocate an alternative host.
:param: allocation: allocation to change.
:return: True if an alternative host was successfully allocated.
"""
reservation = db_api.reservation_get(allocation['reservation_id'])
pool = nova.ReservationPool()
# Remove the failed host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(allocation['compute_host_id'])
pool.remove_computehost(reservation['aggregate_id'],
host['service_name'])
try:
self.placement_client.delete_reservation_inventory(
host['service_name'], reservation['id'])
except openstack_ex.ResourceProviderNotFound:
pass
# Allocate an alternative host.
values = {}
lease = db_api.lease_get(reservation['lease_id'])
values['start_date'] = max(datetime.datetime.utcnow(),
lease['start_date'])
values['end_date'] = lease['end_date']
specs = ['vcpus', 'memory_mb', 'disk_gb', 'affinity', 'amount',
'resource_properties']
for key in specs:
values[key] = reservation[key]
changed_hosts = self.pickup_hosts(reservation['id'], values)
if len(changed_hosts['added']) == 0:
db_api.host_allocation_destroy(allocation['id'])
LOG.warn('Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
return False
else:
new_host_id = changed_hosts['added'].pop()
db_api.host_allocation_update(
allocation['id'], {'compute_host_id': new_host_id})
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_host_id)
pool.add_computehost(reservation['aggregate_id'],
new_host['service_name'],
stay_in=True)
self.placement_client.update_reservation_inventory(
new_host['service_name'], reservation['id'], 1)
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
return True
def delete_computehost(self, host_id):
host = db_api.host_get(host_id)
if not host:
raise manager_ex.HostNotFound(host=host_id)
with trusts.create_ctx_from_trust(host['trust_id']):
if db_api.host_allocation_get_all_by_values(
compute_host_id=host_id):
raise manager_ex.CantDeleteHost(
host=host_id,
msg='The host is reserved.'
)
inventory = nova.NovaInventory()
servers = inventory.get_servers_per_host(
host['hypervisor_hostname'])
if servers:
raise manager_ex.HostHavingServers(
host=host['hypervisor_hostname'], servers=servers)
try:
pool = nova.ReservationPool()
pool.remove_computehost(self.freepool_name,
host['service_name'])
self.placement_client.delete_reservation_provider(
host['hypervisor_hostname'])
# NOTE(sbauza): Extracapabilities will be destroyed thanks to
# the DB FK.
db_api.host_destroy(host_id)
except db_ex.BlazarDBException as e:
# Nothing so bad, but we need to alert admins
# they have to rerun
raise manager_ex.CantDeleteHost(host=host_id, msg=str(e))
def on_end(self, resource_id):
instance_reservation = db_api.instance_reservation_get(resource_id)
reservation_id = instance_reservation['reservation_id']
ctx = context.current()
try:
self.nova.flavor_access.remove_tenant_access(
reservation_id, ctx.project_id)
except nova_exceptions.NotFound:
pass
allocations = db_api.host_allocation_get_all_by_values(
reservation_id=reservation_id)
for allocation in allocations:
host = db_api.host_get(allocation['compute_host_id'])
db_api.host_allocation_destroy(allocation['id'])
try:
self.placement_client.delete_reservation_inventory(
host['service_name'], reservation_id)
except openstack_ex.ResourceProviderNotFound:
pass
for server in self.nova.servers.list(search_opts={
'flavor': reservation_id,
'all_tenants': 1}, detailed=False):
server.delete()
self.cleanup_resources(instance_reservation)
self.placement_client.delete_reservation_class(reservation_id)
def on_start(self, resource_id):
ctx = context.current()
instance_reservation = db_api.instance_reservation_get(resource_id)
reservation_id = instance_reservation['reservation_id']
try:
self.nova.flavor_access.add_tenant_access(reservation_id,
ctx.project_id)
except nova_exceptions.ClientException:
LOG.info(
'Failed to associate flavor %(reservation_id)s '
'to project %(project_id)s', {
'reservation_id': reservation_id,
'project_id': ctx.project_id
})
raise mgr_exceptions.EventError()
pool = nova.ReservationPool()
# Dict of number of instances to reserve on a host keyed by the
# host id
allocation_map = collections.defaultdict(lambda: 0)
for allocation in db_api.host_allocation_get_all_by_values(
reservation_id=reservation_id):
host_id = allocation['compute_host_id']
allocation_map[host_id] += 1
for host_id, num in allocation_map.items():
host = db_api.host_get(host_id)
pool.add_computehost(instance_reservation['aggregate_id'],
host['service_name'],
stay_in=True)
self.placement_client.update_reservation_inventory(
host['hypervisor_hostname'], reservation_id, num)
def on_start(self, resource_id):
"""Add the hosts in the pool."""
host_reservation = db_api.host_reservation_get(resource_id)
pool = nova.ReservationPool()
hosts = []
for allocation in db_api.host_allocation_get_all_by_values(
reservation_id=host_reservation['reservation_id']):
host = db_api.host_get(allocation['compute_host_id'])
hosts.append(host['service_name'])
pool.add_computehost(host_reservation['aggregate_id'], hosts)
for host in hosts:
for server in self.nova.servers.list(search_opts={
"host": host,
"all_tenants": 1
}):
try:
LOG.info('Terminating preemptible instance %s (%s)',
server.name, server.id)
self.nova.servers.delete(server=server)
except nova_exceptions.NotFound:
LOG.info(
'Could not find server %s, may have been deleted '
'concurrently.', server)
except Exception as e:
LOG.exception('Failed to delete %s: %s.', server, str(e))
def on_start(self, resource_id):
"""Add the hosts in the pool."""
host_reservation = db_api.host_reservation_get(resource_id)
pool = nova.ReservationPool()
for allocation in db_api.host_allocation_get_all_by_values(
reservation_id=host_reservation['reservation_id']):
host = db_api.host_get(allocation['compute_host_id'])
pool.add_computehost(host_reservation['aggregate_id'],
host['service_name'])
def get_computehost(self, host_id):
host = db_api.host_get(host_id)
extra_capabilities = self._get_extra_capabilities(host_id)
if host is not None and extra_capabilities:
res = host.copy()
res.update(extra_capabilities)
return res
else:
return host
def _reallocate(self, allocation):
"""Allocate an alternative host.
:param: allocation: allocation to change.
:return: True if an alternative host was successfully allocated.
"""
reservation = db_api.reservation_get(allocation['reservation_id'])
h_reservation = db_api.host_reservation_get(
reservation['resource_id'])
lease = db_api.lease_get(reservation['lease_id'])
pool = nova.ReservationPool()
# Remove the old host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(allocation['compute_host_id'])
pool.remove_computehost(h_reservation['aggregate_id'],
host['service_name'])
# Allocate an alternative host.
start_date = max(datetime.datetime.utcnow(), lease['start_date'])
new_hostids = self._matching_hosts(
reservation['hypervisor_properties'],
reservation['resource_properties'],
'1-1', start_date, lease['end_date']
)
if not new_hostids:
db_api.host_allocation_destroy(allocation['id'])
LOG.warn('Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
return False
else:
new_hostid = new_hostids.pop()
db_api.host_allocation_update(allocation['id'],
{'compute_host_id': new_hostid})
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_hostid)
pool.add_computehost(h_reservation['aggregate_id'],
new_host['service_name'])
return True
def _update_allocations(self, dates_before, dates_after, reservation_id,
reservation_status, host_reservation, values):
min_hosts = self._convert_int_param(
values.get('min', host_reservation['count_range'].split('-')[0]),
'min')
max_hosts = self._convert_int_param(
values.get('max', host_reservation['count_range'].split('-')[1]),
'max')
if min_hosts < 0 or max_hosts < min_hosts:
raise manager_ex.InvalidRange()
hypervisor_properties = values.get(
'hypervisor_properties', host_reservation['hypervisor_properties'])
resource_properties = values.get(
'resource_properties', host_reservation['resource_properties'])
allocs = db_api.host_allocation_get_all_by_values(
reservation_id=reservation_id)
allocs_to_remove = self._allocations_to_remove(dates_before,
dates_after, max_hosts,
hypervisor_properties,
resource_properties,
allocs)
if (allocs_to_remove
and reservation_status == status.reservation.ACTIVE):
raise manager_ex.NotEnoughHostsAvailable()
kept_hosts = len(allocs) - len(allocs_to_remove)
if kept_hosts < max_hosts:
min_hosts = min_hosts - kept_hosts \
if (min_hosts - kept_hosts) > 0 else 0
max_hosts = max_hosts - kept_hosts
host_ids = self._matching_hosts(
hypervisor_properties, resource_properties,
str(min_hosts) + '-' + str(max_hosts),
dates_after['start_date'], dates_after['end_date'])
if len(host_ids) >= min_hosts:
pool = nova.ReservationPool()
for host_id in host_ids:
db_api.host_allocation_create({
'compute_host_id':
host_id,
'reservation_id':
reservation_id
})
if reservation_status == status.reservation.ACTIVE:
# Add new host into the aggregate.
new_host = db_api.host_get(host_id)
pool.add_computehost(host_reservation['aggregate_id'],
new_host['service_name'])
else:
raise manager_ex.NotEnoughHostsAvailable()
for allocation in allocs_to_remove:
db_api.host_allocation_destroy(allocation['id'])
def _pre_reallocate(self, reservation, host_id):
"""Delete the reservation inventory/aggregates for the host."""
pool = nova.ReservationPool()
# Remove the failed host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(host_id)
pool.remove_computehost(reservation['aggregate_id'],
host['service_name'])
try:
self.placement_client.delete_reservation_inventory(
host['hypervisor_hostname'], reservation['id'])
except openstack_ex.ResourceProviderNotFound:
pass
def on_end(self, resource_id):
instance_reservation = db_api.instance_reservation_get(resource_id)
reservation_id = instance_reservation['reservation_id']
ctx = context.current()
try:
self.nova.flavor_access.remove_tenant_access(
reservation_id, ctx.project_id)
except nova_exceptions.NotFound:
pass
hostnames = []
allocations = db_api.host_allocation_get_all_by_values(
reservation_id=reservation_id)
for allocation in allocations:
host = db_api.host_get(allocation['compute_host_id'])
db_api.host_allocation_destroy(allocation['id'])
hostnames.append(host['hypervisor_hostname'])
for server in self.nova.servers.list(search_opts={
'flavor': reservation_id,
'all_tenants': 1
},
detailed=False):
try:
self.nova.servers.delete(server=server)
except nova_exceptions.NotFound:
LOG.info(
"Could not find server '%s', may have been deleted "
"concurrently.", server.id)
except Exception as e:
LOG.exception("Failed to delete server '%s': %s.", server.id,
str(e))
# We need to check the deletion is complete before deleting the
# reservation inventory. See the bug #1813252 for details.
if not self._check_server_deletion(reservation_id):
LOG.error('Timed out while deleting servers on reservation %s',
reservation_id)
raise mgr_exceptions.ServerDeletionTimeout()
self.cleanup_resources(instance_reservation)
for host_name in hostnames:
try:
self.placement_client.delete_reservation_inventory(
host_name, reservation_id)
except openstack_ex.ResourceProviderNotFound:
pass
self.placement_client.delete_reservation_class(reservation_id)
def _post_reallocate(self, reservation, lease, host_id, num):
"""Add the reservation inventory/aggregates for the host."""
pool = nova.ReservationPool()
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(host_id)
pool.add_computehost(reservation['aggregate_id'],
new_host['service_name'],
stay_in=True)
# Here we use "additional=True" not to break the existing
# inventory(allocations) on the new host
self.placement_client.update_reservation_inventory(
new_host['hypervisor_hostname'], reservation['id'], num,
additional=True)
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
def update_resources(self, reservation_id):
"""Updates reserved resources in Nova.
This method updates reserved resources in Compute service. If the
reservation is in active status, it adds new allocated hosts into
a reserved aggregate. If the reservation is not started yet, it
updates a reserved flavor.
"""
reservation = db_api.reservation_get(reservation_id)
if reservation['status'] == 'active':
pool = nova.ReservationPool()
# Dict of number of instances to reserve on a host keyed by the
# host id
allocation_map = collections.defaultdict(lambda: 0)
for allocation in db_api.host_allocation_get_all_by_values(
reservation_id=reservation['id']):
host_id = allocation['compute_host_id']
allocation_map[host_id] += 1
for host_id, num in allocation_map.items():
host = db_api.host_get(host_id)
try:
pool.add_computehost(reservation['aggregate_id'],
host['service_name'],
stay_in=True)
except mgr_exceptions.AggregateAlreadyHasHost:
pass
except nova_exceptions.ClientException:
err_msg = ('Fail to add host %s to aggregate %s.' %
(host, reservation['aggregate_id']))
raise mgr_exceptions.NovaClientError(err_msg)
self.placement_client.update_reservation_inventory(
host['hypervisor_hostname'], reservation['id'], num)
else:
try:
self.nova.nova.flavors.delete(reservation['id'])
self._create_flavor(reservation['id'], reservation['vcpus'],
reservation['memory_mb'],
reservation['disk_gb'],
reservation['server_group_id'])
except nova_exceptions.ClientException:
LOG.exception(
"Failed to update Nova resources "
"for reservation %s", reservation['id'])
raise mgr_exceptions.NovaClientError()
def on_start(self, resource_id):
ctx = context.current()
instance_reservation = db_api.instance_reservation_get(resource_id)
reservation_id = instance_reservation['reservation_id']
try:
self.nova.flavor_access.add_tenant_access(reservation_id,
ctx.project_id)
except nova_exceptions.ClientException:
LOG.info('Failed to associate flavor %(reservation_id)s '
'to project %(project_id)s',
{'reservation_id': reservation_id,
'project_id': ctx.project_id})
raise mgr_exceptions.EventError()
pool = nova.ReservationPool()
for allocation in db_api.host_allocation_get_all_by_values(
reservation_id=reservation_id):
host = db_api.host_get(allocation['compute_host_id'])
pool.add_computehost(instance_reservation['aggregate_id'],
host['service_name'], stay_in=True)
def update_resources(self, reservation_id):
"""Updates reserved resources in Nova.
This method updates reserved resources in Compute service. If the
reservation is in active status, it adds new allocated hosts into
a reserved aggregate. If the reservation is not started yet, it
updates a reserved flavor.
"""
reservation = db_api.reservation_get(reservation_id)
if reservation['status'] == 'active':
pool = nova.ReservationPool()
for allocation in db_api.host_allocation_get_all_by_values(
reservation_id=reservation['id']):
host = db_api.host_get(allocation['compute_host_id'])
try:
pool.add_computehost(reservation['aggregate_id'],
host['service_name'],
stay_in=True)
except mgr_exceptions.AggregateAlreadyHasHost:
pass
except nova_exceptions.ClientException:
err_msg = ('Fail to add host %s to aggregate %s.' %
(host, reservation['aggregate_id']))
raise mgr_exceptions.NovaClientError(err_msg)
else:
try:
self.nova.nova.flavors.delete(reservation['id'])
self._create_flavor(reservation['id'], reservation['vcpus'],
reservation['memory_mb'],
reservation['disk_gb'],
reservation['server_group_id'])
except nova_exceptions.ClientException:
LOG.exception(
"Failed to update Nova resources "
"for reservation %s", reservation['id'])
raise mgr_exceptions.NovaClientError()
def heal_reservations(self, failed_resources):
"""Heal reservations which suffer from resource failures.
:param: failed_resources: a list of failed hosts.
:return: a dictionary of {reservation id: flags to update}
e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
{'missing_resources': True}}
"""
reservation_flags = {}
failed_allocs = []
for host in failed_resources:
failed_allocs += db_api.host_allocation_get_all_by_values(
compute_host_id=host['id'])
for alloc in failed_allocs:
reservation = db_api.reservation_get(alloc['reservation_id'])
if reservation['resource_type'] != RESOURCE_TYPE:
continue
pool = None
# Remove the failed host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(alloc['compute_host_id'])
pool = nova.ReservationPool()
pool.remove_computehost(reservation['aggregate_id'],
host['service_name'])
# Allocate alternative resource.
values = {}
lease = db_api.lease_get(reservation['lease_id'])
values['start_date'] = max(datetime.datetime.utcnow(),
lease['start_date'])
values['end_date'] = lease['end_date']
specs = ['vcpus', 'memory_mb', 'disk_gb', 'affinity', 'amount']
for key in specs:
values[key] = reservation[key]
changed_hosts = self.pickup_hosts(reservation['id'], values)
if len(changed_hosts['added']) == 0:
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'missing_resources': True})
db_api.host_allocation_destroy(alloc['id'])
LOG.warn(
'Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
else:
new_host_id = changed_hosts['added'].pop()
db_api.host_allocation_update(alloc['id'],
{'compute_host_id': new_host_id})
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_host_id)
pool.add_computehost(reservation['aggregate_id'],
new_host['service_name'],
stay_in=True)
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'resources_changed': True})
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
return reservation_flags
def heal_reservations(self, failed_resources):
"""Heal reservations which suffer from resource failures.
:param: failed_resources: a list of failed hosts.
:return: a dictionary of {reservation id: flags to update}
e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
{'missing_resources': True}}
"""
reservation_flags = {}
failed_allocs = []
for host in failed_resources:
failed_allocs += db_api.host_allocation_get_all_by_values(
compute_host_id=host['id'])
for alloc in failed_allocs:
reservation = db_api.reservation_get(alloc['reservation_id'])
if reservation['resource_type'] != plugin.RESOURCE_TYPE:
continue
lease = db_api.lease_get(reservation['lease_id'])
host_reservation = None
pool = None
# Remove the failed host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(alloc['compute_host_id'])
host_reservation = db_api.host_reservation_get(
reservation['resource_id'])
with trusts.create_ctx_from_trust(lease['trust_id']):
pool = nova.ReservationPool()
pool.remove_computehost(host_reservation['aggregate_id'],
host['service_name'])
# Allocate alternative resource.
start_date = max(datetime.datetime.utcnow(), lease['start_date'])
new_hostids = self._matching_hosts(
reservation['hypervisor_properties'],
reservation['resource_properties'], '1-1', start_date,
lease['end_date'])
if not new_hostids:
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'missing_resources': True})
db_api.host_allocation_destroy(alloc['id'])
LOG.warn(
'Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
else:
new_hostid = new_hostids.pop()
db_api.host_allocation_update(alloc['id'],
{'compute_host_id': new_hostid})
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_hostid)
with trusts.create_ctx_from_trust(lease['trust_id']):
pool.add_computehost(host_reservation['aggregate_id'],
new_host['service_name'])
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'resources_changed': True})
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
return reservation_flags
