def on_end(self, resource_id):
"""Remove the hosts from the pool."""
host_reservation = db_api.host_reservation_get(resource_id)
db_api.host_reservation_update(host_reservation['id'],
{'status': 'completed'})
allocations = db_api.host_allocation_get_all_by_values(
reservation_id=host_reservation['reservation_id'])
for allocation in allocations:
db_api.host_allocation_destroy(allocation['id'])
pool = nova.ReservationPool()
for host in pool.get_computehosts(host_reservation['aggregate_id']):
for server in self.nova.servers.list(search_opts={
"host": host,
"all_tenants": 1
}):
try:
self.nova.servers.delete(server=server)
except nova_exceptions.NotFound:
LOG.info(
'Could not find server %s, may have been deleted '
'concurrently.', server)
except Exception as e:
LOG.exception('Failed to delete %s: %s.', server, str(e))
try:
pool.delete(host_reservation['aggregate_id'])
except manager_ex.AggregateNotFound:
pass
def on_end(self, resource_id):
instance_reservation = db_api.instance_reservation_get(resource_id)
reservation_id = instance_reservation['reservation_id']
ctx = context.current()
try:
self.nova.flavor_access.remove_tenant_access(
reservation_id, ctx.project_id)
except nova_exceptions.NotFound:
pass
allocations = db_api.host_allocation_get_all_by_values(
reservation_id=reservation_id)
for allocation in allocations:
host = db_api.host_get(allocation['compute_host_id'])
db_api.host_allocation_destroy(allocation['id'])
try:
self.placement_client.delete_reservation_inventory(
host['service_name'], reservation_id)
except openstack_ex.ResourceProviderNotFound:
pass
for server in self.nova.servers.list(search_opts={
'flavor': reservation_id,
'all_tenants': 1}, detailed=False):
server.delete()
self.cleanup_resources(instance_reservation)
self.placement_client.delete_reservation_class(reservation_id)
def on_end(self, resource_id):
instance_reservation = db_api.instance_reservation_get(resource_id)
ctx = context.current()
try:
self.nova.flavor_access.remove_tenant_access(
instance_reservation['reservation_id'], ctx.project_id)
except nova_exceptions.NotFound:
pass
allocations = db_api.host_allocation_get_all_by_values(
reservation_id=instance_reservation['reservation_id'])
for allocation in allocations:
db_api.host_allocation_destroy(allocation['id'])
for server in self.nova.servers.list(search_opts={
'flavor':
instance_reservation['reservation_id'],
'all_tenants':
1
},
detailed=False):
server.delete()
self.cleanup_resources(instance_reservation)
def _heal_reservation(self, reservation, host_ids):
"""Allocate alternative host(s) for the given reservation.
:param reservation: A reservation that has allocations to change
:param host_ids: Failed host ids
:return: True if all the allocations in the given reservation
are successfully allocated
"""
lease = db_api.lease_get(reservation['lease_id'])
ret = True
allocations = [
alloc for alloc in reservation['computehost_allocations']
if alloc['compute_host_id'] in host_ids]
if reservation['affinity']:
old_host_id = allocations[0]['compute_host_id']
new_host_id = self._select_host(reservation, lease)
self._pre_reallocate(reservation, old_host_id)
if new_host_id is None:
for allocation in allocations:
db_api.host_allocation_destroy(allocation['id'])
LOG.warn('Could not find alternative host for '
'reservation %s (lease: %s).',
reservation['id'], lease['name'])
ret = False
else:
for allocation in allocations:
db_api.host_allocation_update(
allocation['id'], {'compute_host_id': new_host_id})
self._post_reallocate(
reservation, lease, new_host_id, len(allocations))
else:
new_host_ids = []
for allocation in allocations:
old_host_id = allocation['compute_host_id']
new_host_id = self._select_host(reservation, lease)
self._pre_reallocate(reservation, old_host_id)
if new_host_id is None:
db_api.host_allocation_destroy(allocation['id'])
LOG.warn('Could not find alternative host for '
'reservation %s (lease: %s).',
reservation['id'], lease['name'])
ret = False
continue
db_api.host_allocation_update(
allocation['id'], {'compute_host_id': new_host_id})
new_host_ids.append(new_host_id)
for new_host, num in collections.Counter(new_host_ids).items():
self._post_reallocate(reservation, lease, new_host, num)
return ret
def _update_allocations(self, dates_before, dates_after, reservation_id,
reservation_status, host_reservation, values):
min_hosts = self._convert_int_param(
values.get('min', host_reservation['count_range'].split('-')[0]),
'min')
max_hosts = self._convert_int_param(
values.get('max', host_reservation['count_range'].split('-')[1]),
'max')
if min_hosts < 0 or max_hosts < min_hosts:
raise manager_ex.InvalidRange()
hypervisor_properties = values.get(
'hypervisor_properties', host_reservation['hypervisor_properties'])
resource_properties = values.get(
'resource_properties', host_reservation['resource_properties'])
allocs = db_api.host_allocation_get_all_by_values(
reservation_id=reservation_id)
allocs_to_remove = self._allocations_to_remove(dates_before,
dates_after, max_hosts,
hypervisor_properties,
resource_properties,
allocs)
if (allocs_to_remove
and reservation_status == status.reservation.ACTIVE):
raise manager_ex.NotEnoughHostsAvailable()
kept_hosts = len(allocs) - len(allocs_to_remove)
if kept_hosts < max_hosts:
min_hosts = min_hosts - kept_hosts \
if (min_hosts - kept_hosts) > 0 else 0
max_hosts = max_hosts - kept_hosts
host_ids = self._matching_hosts(
hypervisor_properties, resource_properties,
str(min_hosts) + '-' + str(max_hosts),
dates_after['start_date'], dates_after['end_date'])
if len(host_ids) >= min_hosts:
pool = nova.ReservationPool()
for host_id in host_ids:
db_api.host_allocation_create({
'compute_host_id':
host_id,
'reservation_id':
reservation_id
})
if reservation_status == status.reservation.ACTIVE:
# Add new host into the aggregate.
new_host = db_api.host_get(host_id)
pool.add_computehost(host_reservation['aggregate_id'],
new_host['service_name'])
else:
raise manager_ex.NotEnoughHostsAvailable()
for allocation in allocs_to_remove:
db_api.host_allocation_destroy(allocation['id'])
def update_host_allocations(self, added, removed, reservation_id):
allocations = db_api.host_allocation_get_all_by_values(
reservation_id=reservation_id)
removed_allocs = [a for a in allocations
if a['compute_host_id'] in removed]
for alloc in removed_allocs:
db_api.host_allocation_destroy(alloc['id'])
for added_host in added:
db_api.host_allocation_create({'compute_host_id': added_host,
'reservation_id': reservation_id})
def _reallocate(self, allocation):
"""Allocate an alternative host.
:param: allocation: allocation to change.
:return: True if an alternative host was successfully allocated.
"""
reservation = db_api.reservation_get(allocation['reservation_id'])
pool = nova.ReservationPool()
# Remove the failed host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(allocation['compute_host_id'])
pool.remove_computehost(reservation['aggregate_id'],
host['service_name'])
try:
self.placement_client.delete_reservation_inventory(
host['service_name'], reservation['id'])
except openstack_ex.ResourceProviderNotFound:
pass
# Allocate an alternative host.
values = {}
lease = db_api.lease_get(reservation['lease_id'])
values['start_date'] = max(datetime.datetime.utcnow(),
lease['start_date'])
values['end_date'] = lease['end_date']
specs = ['vcpus', 'memory_mb', 'disk_gb', 'affinity', 'amount',
'resource_properties']
for key in specs:
values[key] = reservation[key]
changed_hosts = self.pickup_hosts(reservation['id'], values)
if len(changed_hosts['added']) == 0:
db_api.host_allocation_destroy(allocation['id'])
LOG.warn('Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
return False
else:
new_host_id = changed_hosts['added'].pop()
db_api.host_allocation_update(
allocation['id'], {'compute_host_id': new_host_id})
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_host_id)
pool.add_computehost(reservation['aggregate_id'],
new_host['service_name'],
stay_in=True)
self.placement_client.update_reservation_inventory(
new_host['service_name'], reservation['id'], 1)
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
return True
def on_end(self, resource_id):
instance_reservation = db_api.instance_reservation_get(resource_id)
reservation_id = instance_reservation['reservation_id']
ctx = context.current()
try:
self.nova.flavor_access.remove_tenant_access(
reservation_id, ctx.project_id)
except nova_exceptions.NotFound:
pass
hostnames = []
allocations = db_api.host_allocation_get_all_by_values(
reservation_id=reservation_id)
for allocation in allocations:
host = db_api.host_get(allocation['compute_host_id'])
db_api.host_allocation_destroy(allocation['id'])
hostnames.append(host['hypervisor_hostname'])
for server in self.nova.servers.list(search_opts={
'flavor': reservation_id,
'all_tenants': 1
},
detailed=False):
try:
self.nova.servers.delete(server=server)
except nova_exceptions.NotFound:
LOG.info(
"Could not find server '%s', may have been deleted "
"concurrently.", server.id)
except Exception as e:
LOG.exception("Failed to delete server '%s': %s.", server.id,
str(e))
# We need to check the deletion is complete before deleting the
# reservation inventory. See the bug #1813252 for details.
if not self._check_server_deletion(reservation_id):
LOG.error('Timed out while deleting servers on reservation %s',
reservation_id)
raise mgr_exceptions.ServerDeletionTimeout()
self.cleanup_resources(instance_reservation)
for host_name in hostnames:
try:
self.placement_client.delete_reservation_inventory(
host_name, reservation_id)
except openstack_ex.ResourceProviderNotFound:
pass
self.placement_client.delete_reservation_class(reservation_id)
def _reallocate(self, allocation):
"""Allocate an alternative host.
:param: allocation: allocation to change.
:return: True if an alternative host was successfully allocated.
"""
reservation = db_api.reservation_get(allocation['reservation_id'])
h_reservation = db_api.host_reservation_get(
reservation['resource_id'])
lease = db_api.lease_get(reservation['lease_id'])
pool = nova.ReservationPool()
# Remove the old host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(allocation['compute_host_id'])
pool.remove_computehost(h_reservation['aggregate_id'],
host['service_name'])
# Allocate an alternative host.
start_date = max(datetime.datetime.utcnow(), lease['start_date'])
new_hostids = self._matching_hosts(
reservation['hypervisor_properties'],
reservation['resource_properties'],
'1-1', start_date, lease['end_date']
)
if not new_hostids:
db_api.host_allocation_destroy(allocation['id'])
LOG.warn('Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
return False
else:
new_hostid = new_hostids.pop()
db_api.host_allocation_update(allocation['id'],
{'compute_host_id': new_hostid})
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_hostid)
pool.add_computehost(h_reservation['aggregate_id'],
new_host['service_name'])
return True
def on_end(self, resource_id):
"""Remove the hosts from the pool."""
host_reservation = db_api.host_reservation_get(resource_id)
db_api.host_reservation_update(host_reservation['id'],
{'status': 'completed'})
allocations = db_api.host_allocation_get_all_by_values(
reservation_id=host_reservation['reservation_id'])
for allocation in allocations:
db_api.host_allocation_destroy(allocation['id'])
pool = nova.ReservationPool()
for host in pool.get_computehosts(host_reservation['aggregate_id']):
for server in self.nova.servers.list(search_opts={
"host": host,
"all_tenants": 1
}):
self.nova.servers.delete(server=server)
try:
pool.delete(host_reservation['aggregate_id'])
except manager_ex.AggregateNotFound:
pass
def update_host_allocations(self, added, removed, reservation_id):
allocations = db_api.host_allocation_get_all_by_values(
reservation_id=reservation_id)
removed_allocs = []
for host_id in removed:
for allocation in allocations:
if allocation['compute_host_id'] == host_id:
removed_allocs.append(allocation['id'])
break
# TODO(tetsuro): It would be nice to have something like
# db_api.host_allocation_replace() to process the following
# deletion and addition in *one* DB transaction.
for alloc_id in removed_allocs:
db_api.host_allocation_destroy(alloc_id)
for added_host in added:
db_api.host_allocation_create({'compute_host_id': added_host,
'reservation_id': reservation_id})
def heal_reservations(self, failed_resources):
"""Heal reservations which suffer from resource failures.
:param: failed_resources: a list of failed hosts.
:return: a dictionary of {reservation id: flags to update}
e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
{'missing_resources': True}}
"""
reservation_flags = {}
failed_allocs = []
for host in failed_resources:
failed_allocs += db_api.host_allocation_get_all_by_values(
compute_host_id=host['id'])
for alloc in failed_allocs:
reservation = db_api.reservation_get(alloc['reservation_id'])
if reservation['resource_type'] != plugin.RESOURCE_TYPE:
continue
lease = db_api.lease_get(reservation['lease_id'])
host_reservation = None
pool = None
# Remove the failed host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(alloc['compute_host_id'])
host_reservation = db_api.host_reservation_get(
reservation['resource_id'])
with trusts.create_ctx_from_trust(lease['trust_id']):
pool = nova.ReservationPool()
pool.remove_computehost(host_reservation['aggregate_id'],
host['service_name'])
# Allocate alternative resource.
start_date = max(datetime.datetime.utcnow(), lease['start_date'])
new_hostids = self._matching_hosts(
reservation['hypervisor_properties'],
reservation['resource_properties'], '1-1', start_date,
lease['end_date'])
if not new_hostids:
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'missing_resources': True})
db_api.host_allocation_destroy(alloc['id'])
LOG.warn(
'Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
else:
new_hostid = new_hostids.pop()
db_api.host_allocation_update(alloc['id'],
{'compute_host_id': new_hostid})
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_hostid)
with trusts.create_ctx_from_trust(lease['trust_id']):
pool.add_computehost(host_reservation['aggregate_id'],
new_host['service_name'])
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'resources_changed': True})
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
return reservation_flags
def heal_reservations(self, failed_resources):
"""Heal reservations which suffer from resource failures.
:param: failed_resources: a list of failed hosts.
:return: a dictionary of {reservation id: flags to update}
e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
{'missing_resources': True}}
"""
reservation_flags = {}
failed_allocs = []
for host in failed_resources:
failed_allocs += db_api.host_allocation_get_all_by_values(
compute_host_id=host['id'])
for alloc in failed_allocs:
reservation = db_api.reservation_get(alloc['reservation_id'])
if reservation['resource_type'] != RESOURCE_TYPE:
continue
pool = None
# Remove the failed host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(alloc['compute_host_id'])
pool = nova.ReservationPool()
pool.remove_computehost(reservation['aggregate_id'],
host['service_name'])
# Allocate alternative resource.
values = {}
lease = db_api.lease_get(reservation['lease_id'])
values['start_date'] = max(datetime.datetime.utcnow(),
lease['start_date'])
values['end_date'] = lease['end_date']
specs = ['vcpus', 'memory_mb', 'disk_gb', 'affinity', 'amount']
for key in specs:
values[key] = reservation[key]
changed_hosts = self.pickup_hosts(reservation['id'], values)
if len(changed_hosts['added']) == 0:
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'missing_resources': True})
db_api.host_allocation_destroy(alloc['id'])
LOG.warn(
'Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
else:
new_host_id = changed_hosts['added'].pop()
db_api.host_allocation_update(alloc['id'],
{'compute_host_id': new_host_id})
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_host_id)
pool.add_computehost(reservation['aggregate_id'],
new_host['service_name'],
stay_in=True)
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'resources_changed': True})
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
return reservation_flags
