def _heal_reservation(self, reservation, host_ids):
"""Allocate alternative host(s) for the given reservation.
:param reservation: A reservation that has allocations to change
:param host_ids: Failed host ids
:return: True if all the allocations in the given reservation
are successfully allocated
"""
lease = db_api.lease_get(reservation['lease_id'])
ret = True
allocations = [
alloc for alloc in reservation['computehost_allocations']
if alloc['compute_host_id'] in host_ids]
if reservation['affinity']:
old_host_id = allocations[0]['compute_host_id']
new_host_id = self._select_host(reservation, lease)
self._pre_reallocate(reservation, old_host_id)
if new_host_id is None:
for allocation in allocations:
db_api.host_allocation_destroy(allocation['id'])
LOG.warn('Could not find alternative host for '
'reservation %s (lease: %s).',
reservation['id'], lease['name'])
ret = False
else:
for allocation in allocations:
db_api.host_allocation_update(
allocation['id'], {'compute_host_id': new_host_id})
self._post_reallocate(
reservation, lease, new_host_id, len(allocations))
else:
new_host_ids = []
for allocation in allocations:
old_host_id = allocation['compute_host_id']
new_host_id = self._select_host(reservation, lease)
self._pre_reallocate(reservation, old_host_id)
if new_host_id is None:
db_api.host_allocation_destroy(allocation['id'])
LOG.warn('Could not find alternative host for '
'reservation %s (lease: %s).',
reservation['id'], lease['name'])
ret = False
continue
db_api.host_allocation_update(
allocation['id'], {'compute_host_id': new_host_id})
new_host_ids.append(new_host_id)
for new_host, num in collections.Counter(new_host_ids).items():
self._post_reallocate(reservation, lease, new_host, num)
return ret
def _reallocate(self, allocation):
"""Allocate an alternative host.
:param: allocation: allocation to change.
:return: True if an alternative host was successfully allocated.
"""
reservation = db_api.reservation_get(allocation['reservation_id'])
pool = nova.ReservationPool()
# Remove the failed host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(allocation['compute_host_id'])
pool.remove_computehost(reservation['aggregate_id'],
host['service_name'])
try:
self.placement_client.delete_reservation_inventory(
host['service_name'], reservation['id'])
except openstack_ex.ResourceProviderNotFound:
pass
# Allocate an alternative host.
values = {}
lease = db_api.lease_get(reservation['lease_id'])
values['start_date'] = max(datetime.datetime.utcnow(),
lease['start_date'])
values['end_date'] = lease['end_date']
specs = ['vcpus', 'memory_mb', 'disk_gb', 'affinity', 'amount',
'resource_properties']
for key in specs:
values[key] = reservation[key]
changed_hosts = self.pickup_hosts(reservation['id'], values)
if len(changed_hosts['added']) == 0:
db_api.host_allocation_destroy(allocation['id'])
LOG.warn('Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
return False
else:
new_host_id = changed_hosts['added'].pop()
db_api.host_allocation_update(
allocation['id'], {'compute_host_id': new_host_id})
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_host_id)
pool.add_computehost(reservation['aggregate_id'],
new_host['service_name'],
stay_in=True)
self.placement_client.update_reservation_inventory(
new_host['service_name'], reservation['id'], 1)
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
return True
def _reallocate(self, allocation):
"""Allocate an alternative host.
:param: allocation: allocation to change.
:return: True if an alternative host was successfully allocated.
"""
reservation = db_api.reservation_get(allocation['reservation_id'])
h_reservation = db_api.host_reservation_get(
reservation['resource_id'])
lease = db_api.lease_get(reservation['lease_id'])
pool = nova.ReservationPool()
# Remove the old host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(allocation['compute_host_id'])
pool.remove_computehost(h_reservation['aggregate_id'],
host['service_name'])
# Allocate an alternative host.
start_date = max(datetime.datetime.utcnow(), lease['start_date'])
new_hostids = self._matching_hosts(
reservation['hypervisor_properties'],
reservation['resource_properties'],
'1-1', start_date, lease['end_date']
)
if not new_hostids:
db_api.host_allocation_destroy(allocation['id'])
LOG.warn('Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
return False
else:
new_hostid = new_hostids.pop()
db_api.host_allocation_update(allocation['id'],
{'compute_host_id': new_hostid})
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_hostid)
pool.add_computehost(h_reservation['aggregate_id'],
new_host['service_name'])
return True
def heal_reservations(self, failed_resources):
"""Heal reservations which suffer from resource failures.
:param: failed_resources: a list of failed hosts.
:return: a dictionary of {reservation id: flags to update}
e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
{'missing_resources': True}}
"""
reservation_flags = {}
failed_allocs = []
for host in failed_resources:
failed_allocs += db_api.host_allocation_get_all_by_values(
compute_host_id=host['id'])
for alloc in failed_allocs:
reservation = db_api.reservation_get(alloc['reservation_id'])
if reservation['resource_type'] != plugin.RESOURCE_TYPE:
continue
lease = db_api.lease_get(reservation['lease_id'])
host_reservation = None
pool = None
# Remove the failed host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(alloc['compute_host_id'])
host_reservation = db_api.host_reservation_get(
reservation['resource_id'])
with trusts.create_ctx_from_trust(lease['trust_id']):
pool = nova.ReservationPool()
pool.remove_computehost(host_reservation['aggregate_id'],
host['service_name'])
# Allocate alternative resource.
start_date = max(datetime.datetime.utcnow(), lease['start_date'])
new_hostids = self._matching_hosts(
reservation['hypervisor_properties'],
reservation['resource_properties'], '1-1', start_date,
lease['end_date'])
if not new_hostids:
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'missing_resources': True})
db_api.host_allocation_destroy(alloc['id'])
LOG.warn(
'Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
else:
new_hostid = new_hostids.pop()
db_api.host_allocation_update(alloc['id'],
{'compute_host_id': new_hostid})
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_hostid)
with trusts.create_ctx_from_trust(lease['trust_id']):
pool.add_computehost(host_reservation['aggregate_id'],
new_host['service_name'])
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'resources_changed': True})
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
return reservation_flags
def heal_reservations(self, failed_resources):
"""Heal reservations which suffer from resource failures.
:param: failed_resources: a list of failed hosts.
:return: a dictionary of {reservation id: flags to update}
e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
{'missing_resources': True}}
"""
reservation_flags = {}
failed_allocs = []
for host in failed_resources:
failed_allocs += db_api.host_allocation_get_all_by_values(
compute_host_id=host['id'])
for alloc in failed_allocs:
reservation = db_api.reservation_get(alloc['reservation_id'])
if reservation['resource_type'] != RESOURCE_TYPE:
continue
pool = None
# Remove the failed host from the aggregate.
if reservation['status'] == status.reservation.ACTIVE:
host = db_api.host_get(alloc['compute_host_id'])
pool = nova.ReservationPool()
pool.remove_computehost(reservation['aggregate_id'],
host['service_name'])
# Allocate alternative resource.
values = {}
lease = db_api.lease_get(reservation['lease_id'])
values['start_date'] = max(datetime.datetime.utcnow(),
lease['start_date'])
values['end_date'] = lease['end_date']
specs = ['vcpus', 'memory_mb', 'disk_gb', 'affinity', 'amount']
for key in specs:
values[key] = reservation[key]
changed_hosts = self.pickup_hosts(reservation['id'], values)
if len(changed_hosts['added']) == 0:
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'missing_resources': True})
db_api.host_allocation_destroy(alloc['id'])
LOG.warn(
'Could not find alternative host for reservation %s '
'(lease: %s).', reservation['id'], lease['name'])
else:
new_host_id = changed_hosts['added'].pop()
db_api.host_allocation_update(alloc['id'],
{'compute_host_id': new_host_id})
if reservation['status'] == status.reservation.ACTIVE:
# Add the alternative host into the aggregate.
new_host = db_api.host_get(new_host_id)
pool.add_computehost(reservation['aggregate_id'],
new_host['service_name'],
stay_in=True)
if reservation['id'] not in reservation_flags:
reservation_flags[reservation['id']] = {}
reservation_flags[reservation['id']].update(
{'resources_changed': True})
LOG.warn('Resource changed for reservation %s (lease: %s).',
reservation['id'], lease['name'])
return reservation_flags