def on_end(self, resource_id): """Remove the hosts from the pool.""" host_reservation = db_api.host_reservation_get(resource_id) db_api.host_reservation_update(host_reservation['id'], {'status': 'completed'}) allocations = db_api.host_allocation_get_all_by_values( reservation_id=host_reservation['reservation_id']) for allocation in allocations: db_api.host_allocation_destroy(allocation['id']) pool = nova.ReservationPool() for host in pool.get_computehosts(host_reservation['aggregate_id']): for server in self.nova.servers.list(search_opts={ "host": host, "all_tenants": 1 }): try: self.nova.servers.delete(server=server) except nova_exceptions.NotFound: LOG.info( 'Could not find server %s, may have been deleted ' 'concurrently.', server) except Exception as e: LOG.exception('Failed to delete %s: %s.', server, str(e)) try: pool.delete(host_reservation['aggregate_id']) except manager_ex.AggregateNotFound: pass
def on_end(self, resource_id): instance_reservation = db_api.instance_reservation_get(resource_id) reservation_id = instance_reservation['reservation_id'] ctx = context.current() try: self.nova.flavor_access.remove_tenant_access( reservation_id, ctx.project_id) except nova_exceptions.NotFound: pass allocations = db_api.host_allocation_get_all_by_values( reservation_id=reservation_id) for allocation in allocations: host = db_api.host_get(allocation['compute_host_id']) db_api.host_allocation_destroy(allocation['id']) try: self.placement_client.delete_reservation_inventory( host['service_name'], reservation_id) except openstack_ex.ResourceProviderNotFound: pass for server in self.nova.servers.list(search_opts={ 'flavor': reservation_id, 'all_tenants': 1}, detailed=False): server.delete() self.cleanup_resources(instance_reservation) self.placement_client.delete_reservation_class(reservation_id)
def on_end(self, resource_id): instance_reservation = db_api.instance_reservation_get(resource_id) ctx = context.current() try: self.nova.flavor_access.remove_tenant_access( instance_reservation['reservation_id'], ctx.project_id) except nova_exceptions.NotFound: pass allocations = db_api.host_allocation_get_all_by_values( reservation_id=instance_reservation['reservation_id']) for allocation in allocations: db_api.host_allocation_destroy(allocation['id']) for server in self.nova.servers.list(search_opts={ 'flavor': instance_reservation['reservation_id'], 'all_tenants': 1 }, detailed=False): server.delete() self.cleanup_resources(instance_reservation)
def _heal_reservation(self, reservation, host_ids): """Allocate alternative host(s) for the given reservation. :param reservation: A reservation that has allocations to change :param host_ids: Failed host ids :return: True if all the allocations in the given reservation are successfully allocated """ lease = db_api.lease_get(reservation['lease_id']) ret = True allocations = [ alloc for alloc in reservation['computehost_allocations'] if alloc['compute_host_id'] in host_ids] if reservation['affinity']: old_host_id = allocations[0]['compute_host_id'] new_host_id = self._select_host(reservation, lease) self._pre_reallocate(reservation, old_host_id) if new_host_id is None: for allocation in allocations: db_api.host_allocation_destroy(allocation['id']) LOG.warn('Could not find alternative host for ' 'reservation %s (lease: %s).', reservation['id'], lease['name']) ret = False else: for allocation in allocations: db_api.host_allocation_update( allocation['id'], {'compute_host_id': new_host_id}) self._post_reallocate( reservation, lease, new_host_id, len(allocations)) else: new_host_ids = [] for allocation in allocations: old_host_id = allocation['compute_host_id'] new_host_id = self._select_host(reservation, lease) self._pre_reallocate(reservation, old_host_id) if new_host_id is None: db_api.host_allocation_destroy(allocation['id']) LOG.warn('Could not find alternative host for ' 'reservation %s (lease: %s).', reservation['id'], lease['name']) ret = False continue db_api.host_allocation_update( allocation['id'], {'compute_host_id': new_host_id}) new_host_ids.append(new_host_id) for new_host, num in collections.Counter(new_host_ids).items(): self._post_reallocate(reservation, lease, new_host, num) return ret
def _update_allocations(self, dates_before, dates_after, reservation_id, reservation_status, host_reservation, values): min_hosts = self._convert_int_param( values.get('min', host_reservation['count_range'].split('-')[0]), 'min') max_hosts = self._convert_int_param( values.get('max', host_reservation['count_range'].split('-')[1]), 'max') if min_hosts < 0 or max_hosts < min_hosts: raise manager_ex.InvalidRange() hypervisor_properties = values.get( 'hypervisor_properties', host_reservation['hypervisor_properties']) resource_properties = values.get( 'resource_properties', host_reservation['resource_properties']) allocs = db_api.host_allocation_get_all_by_values( reservation_id=reservation_id) allocs_to_remove = self._allocations_to_remove(dates_before, dates_after, max_hosts, hypervisor_properties, resource_properties, allocs) if (allocs_to_remove and reservation_status == status.reservation.ACTIVE): raise manager_ex.NotEnoughHostsAvailable() kept_hosts = len(allocs) - len(allocs_to_remove) if kept_hosts < max_hosts: min_hosts = min_hosts - kept_hosts \ if (min_hosts - kept_hosts) > 0 else 0 max_hosts = max_hosts - kept_hosts host_ids = self._matching_hosts( hypervisor_properties, resource_properties, str(min_hosts) + '-' + str(max_hosts), dates_after['start_date'], dates_after['end_date']) if len(host_ids) >= min_hosts: pool = nova.ReservationPool() for host_id in host_ids: db_api.host_allocation_create({ 'compute_host_id': host_id, 'reservation_id': reservation_id }) if reservation_status == status.reservation.ACTIVE: # Add new host into the aggregate. new_host = db_api.host_get(host_id) pool.add_computehost(host_reservation['aggregate_id'], new_host['service_name']) else: raise manager_ex.NotEnoughHostsAvailable() for allocation in allocs_to_remove: db_api.host_allocation_destroy(allocation['id'])
def update_host_allocations(self, added, removed, reservation_id): allocations = db_api.host_allocation_get_all_by_values( reservation_id=reservation_id) removed_allocs = [a for a in allocations if a['compute_host_id'] in removed] for alloc in removed_allocs: db_api.host_allocation_destroy(alloc['id']) for added_host in added: db_api.host_allocation_create({'compute_host_id': added_host, 'reservation_id': reservation_id})
def _reallocate(self, allocation): """Allocate an alternative host. :param: allocation: allocation to change. :return: True if an alternative host was successfully allocated. """ reservation = db_api.reservation_get(allocation['reservation_id']) pool = nova.ReservationPool() # Remove the failed host from the aggregate. if reservation['status'] == status.reservation.ACTIVE: host = db_api.host_get(allocation['compute_host_id']) pool.remove_computehost(reservation['aggregate_id'], host['service_name']) try: self.placement_client.delete_reservation_inventory( host['service_name'], reservation['id']) except openstack_ex.ResourceProviderNotFound: pass # Allocate an alternative host. values = {} lease = db_api.lease_get(reservation['lease_id']) values['start_date'] = max(datetime.datetime.utcnow(), lease['start_date']) values['end_date'] = lease['end_date'] specs = ['vcpus', 'memory_mb', 'disk_gb', 'affinity', 'amount', 'resource_properties'] for key in specs: values[key] = reservation[key] changed_hosts = self.pickup_hosts(reservation['id'], values) if len(changed_hosts['added']) == 0: db_api.host_allocation_destroy(allocation['id']) LOG.warn('Could not find alternative host for reservation %s ' '(lease: %s).', reservation['id'], lease['name']) return False else: new_host_id = changed_hosts['added'].pop() db_api.host_allocation_update( allocation['id'], {'compute_host_id': new_host_id}) if reservation['status'] == status.reservation.ACTIVE: # Add the alternative host into the aggregate. new_host = db_api.host_get(new_host_id) pool.add_computehost(reservation['aggregate_id'], new_host['service_name'], stay_in=True) self.placement_client.update_reservation_inventory( new_host['service_name'], reservation['id'], 1) LOG.warn('Resource changed for reservation %s (lease: %s).', reservation['id'], lease['name']) return True
def on_end(self, resource_id): instance_reservation = db_api.instance_reservation_get(resource_id) reservation_id = instance_reservation['reservation_id'] ctx = context.current() try: self.nova.flavor_access.remove_tenant_access( reservation_id, ctx.project_id) except nova_exceptions.NotFound: pass hostnames = [] allocations = db_api.host_allocation_get_all_by_values( reservation_id=reservation_id) for allocation in allocations: host = db_api.host_get(allocation['compute_host_id']) db_api.host_allocation_destroy(allocation['id']) hostnames.append(host['hypervisor_hostname']) for server in self.nova.servers.list(search_opts={ 'flavor': reservation_id, 'all_tenants': 1 }, detailed=False): try: self.nova.servers.delete(server=server) except nova_exceptions.NotFound: LOG.info( "Could not find server '%s', may have been deleted " "concurrently.", server.id) except Exception as e: LOG.exception("Failed to delete server '%s': %s.", server.id, str(e)) # We need to check the deletion is complete before deleting the # reservation inventory. See the bug #1813252 for details. if not self._check_server_deletion(reservation_id): LOG.error('Timed out while deleting servers on reservation %s', reservation_id) raise mgr_exceptions.ServerDeletionTimeout() self.cleanup_resources(instance_reservation) for host_name in hostnames: try: self.placement_client.delete_reservation_inventory( host_name, reservation_id) except openstack_ex.ResourceProviderNotFound: pass self.placement_client.delete_reservation_class(reservation_id)
def _reallocate(self, allocation): """Allocate an alternative host. :param: allocation: allocation to change. :return: True if an alternative host was successfully allocated. """ reservation = db_api.reservation_get(allocation['reservation_id']) h_reservation = db_api.host_reservation_get( reservation['resource_id']) lease = db_api.lease_get(reservation['lease_id']) pool = nova.ReservationPool() # Remove the old host from the aggregate. if reservation['status'] == status.reservation.ACTIVE: host = db_api.host_get(allocation['compute_host_id']) pool.remove_computehost(h_reservation['aggregate_id'], host['service_name']) # Allocate an alternative host. start_date = max(datetime.datetime.utcnow(), lease['start_date']) new_hostids = self._matching_hosts( reservation['hypervisor_properties'], reservation['resource_properties'], '1-1', start_date, lease['end_date'] ) if not new_hostids: db_api.host_allocation_destroy(allocation['id']) LOG.warn('Could not find alternative host for reservation %s ' '(lease: %s).', reservation['id'], lease['name']) return False else: new_hostid = new_hostids.pop() db_api.host_allocation_update(allocation['id'], {'compute_host_id': new_hostid}) LOG.warn('Resource changed for reservation %s (lease: %s).', reservation['id'], lease['name']) if reservation['status'] == status.reservation.ACTIVE: # Add the alternative host into the aggregate. new_host = db_api.host_get(new_hostid) pool.add_computehost(h_reservation['aggregate_id'], new_host['service_name']) return True
def on_end(self, resource_id): """Remove the hosts from the pool.""" host_reservation = db_api.host_reservation_get(resource_id) db_api.host_reservation_update(host_reservation['id'], {'status': 'completed'}) allocations = db_api.host_allocation_get_all_by_values( reservation_id=host_reservation['reservation_id']) for allocation in allocations: db_api.host_allocation_destroy(allocation['id']) pool = nova.ReservationPool() for host in pool.get_computehosts(host_reservation['aggregate_id']): for server in self.nova.servers.list(search_opts={ "host": host, "all_tenants": 1 }): self.nova.servers.delete(server=server) try: pool.delete(host_reservation['aggregate_id']) except manager_ex.AggregateNotFound: pass
def update_host_allocations(self, added, removed, reservation_id): allocations = db_api.host_allocation_get_all_by_values( reservation_id=reservation_id) removed_allocs = [] for host_id in removed: for allocation in allocations: if allocation['compute_host_id'] == host_id: removed_allocs.append(allocation['id']) break # TODO(tetsuro): It would be nice to have something like # db_api.host_allocation_replace() to process the following # deletion and addition in *one* DB transaction. for alloc_id in removed_allocs: db_api.host_allocation_destroy(alloc_id) for added_host in added: db_api.host_allocation_create({'compute_host_id': added_host, 'reservation_id': reservation_id})
def heal_reservations(self, failed_resources): """Heal reservations which suffer from resource failures. :param: failed_resources: a list of failed hosts. :return: a dictionary of {reservation id: flags to update} e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': {'missing_resources': True}} """ reservation_flags = {} failed_allocs = [] for host in failed_resources: failed_allocs += db_api.host_allocation_get_all_by_values( compute_host_id=host['id']) for alloc in failed_allocs: reservation = db_api.reservation_get(alloc['reservation_id']) if reservation['resource_type'] != plugin.RESOURCE_TYPE: continue lease = db_api.lease_get(reservation['lease_id']) host_reservation = None pool = None # Remove the failed host from the aggregate. if reservation['status'] == status.reservation.ACTIVE: host = db_api.host_get(alloc['compute_host_id']) host_reservation = db_api.host_reservation_get( reservation['resource_id']) with trusts.create_ctx_from_trust(lease['trust_id']): pool = nova.ReservationPool() pool.remove_computehost(host_reservation['aggregate_id'], host['service_name']) # Allocate alternative resource. start_date = max(datetime.datetime.utcnow(), lease['start_date']) new_hostids = self._matching_hosts( reservation['hypervisor_properties'], reservation['resource_properties'], '1-1', start_date, lease['end_date']) if not new_hostids: if reservation['id'] not in reservation_flags: reservation_flags[reservation['id']] = {} reservation_flags[reservation['id']].update( {'missing_resources': True}) db_api.host_allocation_destroy(alloc['id']) LOG.warn( 'Could not find alternative host for reservation %s ' '(lease: %s).', reservation['id'], lease['name']) else: new_hostid = new_hostids.pop() db_api.host_allocation_update(alloc['id'], {'compute_host_id': new_hostid}) if reservation['status'] == status.reservation.ACTIVE: # Add the alternative host into the aggregate. new_host = db_api.host_get(new_hostid) with trusts.create_ctx_from_trust(lease['trust_id']): pool.add_computehost(host_reservation['aggregate_id'], new_host['service_name']) if reservation['id'] not in reservation_flags: reservation_flags[reservation['id']] = {} reservation_flags[reservation['id']].update( {'resources_changed': True}) LOG.warn('Resource changed for reservation %s (lease: %s).', reservation['id'], lease['name']) return reservation_flags
def heal_reservations(self, failed_resources): """Heal reservations which suffer from resource failures. :param: failed_resources: a list of failed hosts. :return: a dictionary of {reservation id: flags to update} e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': {'missing_resources': True}} """ reservation_flags = {} failed_allocs = [] for host in failed_resources: failed_allocs += db_api.host_allocation_get_all_by_values( compute_host_id=host['id']) for alloc in failed_allocs: reservation = db_api.reservation_get(alloc['reservation_id']) if reservation['resource_type'] != RESOURCE_TYPE: continue pool = None # Remove the failed host from the aggregate. if reservation['status'] == status.reservation.ACTIVE: host = db_api.host_get(alloc['compute_host_id']) pool = nova.ReservationPool() pool.remove_computehost(reservation['aggregate_id'], host['service_name']) # Allocate alternative resource. values = {} lease = db_api.lease_get(reservation['lease_id']) values['start_date'] = max(datetime.datetime.utcnow(), lease['start_date']) values['end_date'] = lease['end_date'] specs = ['vcpus', 'memory_mb', 'disk_gb', 'affinity', 'amount'] for key in specs: values[key] = reservation[key] changed_hosts = self.pickup_hosts(reservation['id'], values) if len(changed_hosts['added']) == 0: if reservation['id'] not in reservation_flags: reservation_flags[reservation['id']] = {} reservation_flags[reservation['id']].update( {'missing_resources': True}) db_api.host_allocation_destroy(alloc['id']) LOG.warn( 'Could not find alternative host for reservation %s ' '(lease: %s).', reservation['id'], lease['name']) else: new_host_id = changed_hosts['added'].pop() db_api.host_allocation_update(alloc['id'], {'compute_host_id': new_host_id}) if reservation['status'] == status.reservation.ACTIVE: # Add the alternative host into the aggregate. new_host = db_api.host_get(new_host_id) pool.add_computehost(reservation['aggregate_id'], new_host['service_name'], stay_in=True) if reservation['id'] not in reservation_flags: reservation_flags[reservation['id']] = {} reservation_flags[reservation['id']].update( {'resources_changed': True}) LOG.warn('Resource changed for reservation %s (lease: %s).', reservation['id'], lease['name']) return reservation_flags