def reserve_resource(self, reservation_id, values): """Create reservation.""" self._check_params(values) host_ids = self._matching_hosts( values['hypervisor_properties'], values['resource_properties'], values['count_range'], values['start_date'], values['end_date'], ) if not host_ids: raise manager_ex.NotEnoughHostsAvailable() pool = nova.ReservationPool() pool_name = reservation_id az_name = "%s%s" % (CONF[self.resource_type].blazar_az_prefix, pool_name) pool_instance = pool.create(name=pool_name, az=az_name) host_rsrv_values = { 'reservation_id': reservation_id, 'aggregate_id': pool_instance.id, 'resource_properties': values['resource_properties'], 'hypervisor_properties': values['hypervisor_properties'], 'count_range': values['count_range'], 'status': 'pending', 'before_end': values['before_end'] } host_reservation = db_api.host_reservation_create(host_rsrv_values) for host_id in host_ids: db_api.host_allocation_create({'compute_host_id': host_id, 'reservation_id': reservation_id}) return host_reservation['id']
def _create_resources(self, inst_reservation): reservation_id = inst_reservation['reservation_id'] ctx = context.current() user_client = nova.NovaClientWrapper() reserved_group = user_client.nova.server_groups.create( RESERVATION_PREFIX + ':' + reservation_id, 'affinity' if inst_reservation['affinity'] else 'anti-affinity') reserved_flavor = self._create_flavor(reservation_id, inst_reservation['vcpus'], inst_reservation['memory_mb'], inst_reservation['disk_gb'], reserved_group.id) pool = nova.ReservationPool() pool_metadata = { RESERVATION_PREFIX: reservation_id, 'filter_tenant_id': ctx.project_id, 'affinity_id': reserved_group.id } agg = pool.create(name=reservation_id, metadata=pool_metadata) self.placement_client.create_reservation_class(reservation_id) return reserved_flavor, reserved_group, agg
def setUp(self): super(ReservationPoolTestCase, self).setUp() self.pool_name = 'pool-name-xxx' self.project_id = 'project-uuid' self.fake_aggregate = AggregateFake(i=123, name='fooname', hosts=['host1', 'host2']) physical_host_conf = cfg.CONF[host_plugin.RESOURCE_TYPE] nova_conf = cfg.CONF.nova self.freepool_name = nova_conf.aggregate_freepool_name self.project_id_key = nova_conf.project_id_key self.blazar_owner = nova_conf.blazar_owner self.blazar_az_prefix = physical_host_conf.blazar_az_prefix self.fake_freepool = AggregateFake(i=456, name=self.freepool_name, hosts=['host3']) self.set_context(context.BlazarContext(project_id=self.project_id)) self.nova_client = nova_client self.nova = self.patch(self.nova_client, 'Client').return_value self.patch(self.nova.aggregates, 'set_metadata') self.patch(self.nova.aggregates, 'remove_host') self.patch(base, 'url_for').return_value = 'http://foo.bar' self.pool = nova.ReservationPool() self.p_name = self.patch(self.pool, '_generate_aggregate_name') self.p_name.return_value = self.pool_name
def on_end(self, resource_id): """Remove the hosts from the pool.""" host_reservation = db_api.host_reservation_get(resource_id) db_api.host_reservation_update(host_reservation['id'], {'status': 'completed'}) allocations = db_api.host_allocation_get_all_by_values( reservation_id=host_reservation['reservation_id']) for allocation in allocations: db_api.host_allocation_destroy(allocation['id']) pool = nova.ReservationPool() for host in pool.get_computehosts(host_reservation['aggregate_id']): for server in self.nova.servers.list(search_opts={ "host": host, "all_tenants": 1 }): try: self.nova.servers.delete(server=server) except nova_exceptions.NotFound: LOG.info( 'Could not find server %s, may have been deleted ' 'concurrently.', server) except Exception as e: LOG.exception('Failed to delete %s: %s.', server, str(e)) try: pool.delete(host_reservation['aggregate_id']) except manager_ex.AggregateNotFound: pass
def on_start(self, resource_id): ctx = context.current() instance_reservation = db_api.instance_reservation_get(resource_id) reservation_id = instance_reservation['reservation_id'] try: self.nova.flavor_access.add_tenant_access(reservation_id, ctx.project_id) except nova_exceptions.ClientException: LOG.info( 'Failed to associate flavor %(reservation_id)s ' 'to project %(project_id)s', { 'reservation_id': reservation_id, 'project_id': ctx.project_id }) raise mgr_exceptions.EventError() pool = nova.ReservationPool() # Dict of number of instances to reserve on a host keyed by the # host id allocation_map = collections.defaultdict(lambda: 0) for allocation in db_api.host_allocation_get_all_by_values( reservation_id=reservation_id): host_id = allocation['compute_host_id'] allocation_map[host_id] += 1 for host_id, num in allocation_map.items(): host = db_api.host_get(host_id) pool.add_computehost(instance_reservation['aggregate_id'], host['service_name'], stay_in=True) self.placement_client.update_reservation_inventory( host['hypervisor_hostname'], reservation_id, num)
def on_start(self, resource_id): """Add the hosts in the pool.""" host_reservation = db_api.host_reservation_get(resource_id) pool = nova.ReservationPool() hosts = [] for allocation in db_api.host_allocation_get_all_by_values( reservation_id=host_reservation['reservation_id']): host = db_api.host_get(allocation['compute_host_id']) hosts.append(host['service_name']) pool.add_computehost(host_reservation['aggregate_id'], hosts) for host in hosts: for server in self.nova.servers.list(search_opts={ "host": host, "all_tenants": 1 }): try: LOG.info('Terminating preemptible instance %s (%s)', server.name, server.id) self.nova.servers.delete(server=server) except nova_exceptions.NotFound: LOG.info( 'Could not find server %s, may have been deleted ' 'concurrently.', server) except Exception as e: LOG.exception('Failed to delete %s: %s.', server, str(e))
def delete_computehost(self, host_id): host = db_api.host_get(host_id) if not host: raise manager_ex.HostNotFound(host=host_id) with trusts.create_ctx_from_trust(host['trust_id']): if db_api.host_allocation_get_all_by_values( compute_host_id=host_id): raise manager_ex.CantDeleteHost( host=host_id, msg='The host is reserved.' ) inventory = nova.NovaInventory() servers = inventory.get_servers_per_host( host['hypervisor_hostname']) if servers: raise manager_ex.HostHavingServers( host=host['hypervisor_hostname'], servers=servers) try: pool = nova.ReservationPool() pool.remove_computehost(self.freepool_name, host['service_name']) self.placement_client.delete_reservation_provider( host['hypervisor_hostname']) # NOTE(sbauza): Extracapabilities will be destroyed thanks to # the DB FK. db_api.host_destroy(host_id) except db_ex.BlazarDBException as e: # Nothing so bad, but we need to alert admins # they have to rerun raise manager_ex.CantDeleteHost(host=host_id, msg=str(e))
def on_start(self, resource_id): """Add the hosts in the pool.""" host_reservation = db_api.host_reservation_get(resource_id) pool = nova.ReservationPool() for allocation in db_api.host_allocation_get_all_by_values( reservation_id=host_reservation['reservation_id']): host = db_api.host_get(allocation['compute_host_id']) pool.add_computehost(host_reservation['aggregate_id'], host['service_name'])
def _update_allocations(self, dates_before, dates_after, reservation_id, reservation_status, host_reservation, values): min_hosts = self._convert_int_param( values.get('min', host_reservation['count_range'].split('-')[0]), 'min') max_hosts = self._convert_int_param( values.get('max', host_reservation['count_range'].split('-')[1]), 'max') if min_hosts < 0 or max_hosts < min_hosts: raise manager_ex.InvalidRange() hypervisor_properties = values.get( 'hypervisor_properties', host_reservation['hypervisor_properties']) resource_properties = values.get( 'resource_properties', host_reservation['resource_properties']) allocs = db_api.host_allocation_get_all_by_values( reservation_id=reservation_id) allocs_to_remove = self._allocations_to_remove(dates_before, dates_after, max_hosts, hypervisor_properties, resource_properties, allocs) if (allocs_to_remove and reservation_status == status.reservation.ACTIVE): raise manager_ex.NotEnoughHostsAvailable() kept_hosts = len(allocs) - len(allocs_to_remove) if kept_hosts < max_hosts: min_hosts = min_hosts - kept_hosts \ if (min_hosts - kept_hosts) > 0 else 0 max_hosts = max_hosts - kept_hosts host_ids = self._matching_hosts( hypervisor_properties, resource_properties, str(min_hosts) + '-' + str(max_hosts), dates_after['start_date'], dates_after['end_date']) if len(host_ids) >= min_hosts: pool = nova.ReservationPool() for host_id in host_ids: db_api.host_allocation_create({ 'compute_host_id': host_id, 'reservation_id': reservation_id }) if reservation_status == status.reservation.ACTIVE: # Add new host into the aggregate. new_host = db_api.host_get(host_id) pool.add_computehost(host_reservation['aggregate_id'], new_host['service_name']) else: raise manager_ex.NotEnoughHostsAvailable() for allocation in allocs_to_remove: db_api.host_allocation_destroy(allocation['id'])
def _reallocate(self, allocation): """Allocate an alternative host. :param: allocation: allocation to change. :return: True if an alternative host was successfully allocated. """ reservation = db_api.reservation_get(allocation['reservation_id']) pool = nova.ReservationPool() # Remove the failed host from the aggregate. if reservation['status'] == status.reservation.ACTIVE: host = db_api.host_get(allocation['compute_host_id']) pool.remove_computehost(reservation['aggregate_id'], host['service_name']) try: self.placement_client.delete_reservation_inventory( host['service_name'], reservation['id']) except openstack_ex.ResourceProviderNotFound: pass # Allocate an alternative host. values = {} lease = db_api.lease_get(reservation['lease_id']) values['start_date'] = max(datetime.datetime.utcnow(), lease['start_date']) values['end_date'] = lease['end_date'] specs = ['vcpus', 'memory_mb', 'disk_gb', 'affinity', 'amount', 'resource_properties'] for key in specs: values[key] = reservation[key] changed_hosts = self.pickup_hosts(reservation['id'], values) if len(changed_hosts['added']) == 0: db_api.host_allocation_destroy(allocation['id']) LOG.warn('Could not find alternative host for reservation %s ' '(lease: %s).', reservation['id'], lease['name']) return False else: new_host_id = changed_hosts['added'].pop() db_api.host_allocation_update( allocation['id'], {'compute_host_id': new_host_id}) if reservation['status'] == status.reservation.ACTIVE: # Add the alternative host into the aggregate. new_host = db_api.host_get(new_host_id) pool.add_computehost(reservation['aggregate_id'], new_host['service_name'], stay_in=True) self.placement_client.update_reservation_inventory( new_host['service_name'], reservation['id'], 1) LOG.warn('Resource changed for reservation %s (lease: %s).', reservation['id'], lease['name']) return True
def cleanup_resources(self, instance_reservation): def check_and_delete_resource(client, id): try: client.delete(id) except nova_exceptions.NotFound: pass reservation_id = instance_reservation['reservation_id'] check_and_delete_resource(self.nova.nova.server_groups, instance_reservation['server_group_id']) check_and_delete_resource(self.nova.nova.flavors, reservation_id) check_and_delete_resource(nova.ReservationPool(), reservation_id)
def _pre_reallocate(self, reservation, host_id): """Delete the reservation inventory/aggregates for the host.""" pool = nova.ReservationPool() # Remove the failed host from the aggregate. if reservation['status'] == status.reservation.ACTIVE: host = db_api.host_get(host_id) pool.remove_computehost(reservation['aggregate_id'], host['service_name']) try: self.placement_client.delete_reservation_inventory( host['hypervisor_hostname'], reservation['id']) except openstack_ex.ResourceProviderNotFound: pass
def before_end(self, resource_id): """Take an action before the end of a lease.""" host_reservation = db_api.host_reservation_get(resource_id) action = host_reservation['before_end'] if action == 'default': action = CONF[plugin.RESOURCE_TYPE].before_end if action == 'snapshot': pool = nova.ReservationPool() client = nova.BlazarNovaClient() for host in pool.get_computehosts( host_reservation['aggregate_id']): for server in client.servers.list( search_opts={"host": host, "all_tenants": 1}): client.servers.create_image(server=server)
def _post_reallocate(self, reservation, lease, host_id, num): """Add the reservation inventory/aggregates for the host.""" pool = nova.ReservationPool() if reservation['status'] == status.reservation.ACTIVE: # Add the alternative host into the aggregate. new_host = db_api.host_get(host_id) pool.add_computehost(reservation['aggregate_id'], new_host['service_name'], stay_in=True) # Here we use "additional=True" not to break the existing # inventory(allocations) on the new host self.placement_client.update_reservation_inventory( new_host['hypervisor_hostname'], reservation['id'], num, additional=True) LOG.warn('Resource changed for reservation %s (lease: %s).', reservation['id'], lease['name'])
def update_resources(self, reservation_id): """Updates reserved resources in Nova. This method updates reserved resources in Compute service. If the reservation is in active status, it adds new allocated hosts into a reserved aggregate. If the reservation is not started yet, it updates a reserved flavor. """ reservation = db_api.reservation_get(reservation_id) if reservation['status'] == 'active': pool = nova.ReservationPool() # Dict of number of instances to reserve on a host keyed by the # host id allocation_map = collections.defaultdict(lambda: 0) for allocation in db_api.host_allocation_get_all_by_values( reservation_id=reservation['id']): host_id = allocation['compute_host_id'] allocation_map[host_id] += 1 for host_id, num in allocation_map.items(): host = db_api.host_get(host_id) try: pool.add_computehost(reservation['aggregate_id'], host['service_name'], stay_in=True) except mgr_exceptions.AggregateAlreadyHasHost: pass except nova_exceptions.ClientException: err_msg = ('Fail to add host %s to aggregate %s.' % (host, reservation['aggregate_id'])) raise mgr_exceptions.NovaClientError(err_msg) self.placement_client.update_reservation_inventory( host['hypervisor_hostname'], reservation['id'], num) else: try: self.nova.nova.flavors.delete(reservation['id']) self._create_flavor(reservation['id'], reservation['vcpus'], reservation['memory_mb'], reservation['disk_gb'], reservation['server_group_id']) except nova_exceptions.ClientException: LOG.exception( "Failed to update Nova resources " "for reservation %s", reservation['id']) raise mgr_exceptions.NovaClientError()
def _reallocate(self, allocation): """Allocate an alternative host. :param: allocation: allocation to change. :return: True if an alternative host was successfully allocated. """ reservation = db_api.reservation_get(allocation['reservation_id']) h_reservation = db_api.host_reservation_get( reservation['resource_id']) lease = db_api.lease_get(reservation['lease_id']) pool = nova.ReservationPool() # Remove the old host from the aggregate. if reservation['status'] == status.reservation.ACTIVE: host = db_api.host_get(allocation['compute_host_id']) pool.remove_computehost(h_reservation['aggregate_id'], host['service_name']) # Allocate an alternative host. start_date = max(datetime.datetime.utcnow(), lease['start_date']) new_hostids = self._matching_hosts( reservation['hypervisor_properties'], reservation['resource_properties'], '1-1', start_date, lease['end_date'] ) if not new_hostids: db_api.host_allocation_destroy(allocation['id']) LOG.warn('Could not find alternative host for reservation %s ' '(lease: %s).', reservation['id'], lease['name']) return False else: new_hostid = new_hostids.pop() db_api.host_allocation_update(allocation['id'], {'compute_host_id': new_hostid}) LOG.warn('Resource changed for reservation %s (lease: %s).', reservation['id'], lease['name']) if reservation['status'] == status.reservation.ACTIVE: # Add the alternative host into the aggregate. new_host = db_api.host_get(new_hostid) pool.add_computehost(h_reservation['aggregate_id'], new_host['service_name']) return True
def on_end(self, resource_id): """Remove the hosts from the pool.""" host_reservation = db_api.host_reservation_get(resource_id) db_api.host_reservation_update(host_reservation['id'], {'status': 'completed'}) allocations = db_api.host_allocation_get_all_by_values( reservation_id=host_reservation['reservation_id']) for allocation in allocations: db_api.host_allocation_destroy(allocation['id']) pool = nova.ReservationPool() for host in pool.get_computehosts(host_reservation['aggregate_id']): for server in self.nova.servers.list(search_opts={ "host": host, "all_tenants": 1 }): self.nova.servers.delete(server=server) try: pool.delete(host_reservation['aggregate_id']) except manager_ex.AggregateNotFound: pass
def on_start(self, resource_id): ctx = context.current() instance_reservation = db_api.instance_reservation_get(resource_id) reservation_id = instance_reservation['reservation_id'] try: self.nova.flavor_access.add_tenant_access(reservation_id, ctx.project_id) except nova_exceptions.ClientException: LOG.info('Failed to associate flavor %(reservation_id)s ' 'to project %(project_id)s', {'reservation_id': reservation_id, 'project_id': ctx.project_id}) raise mgr_exceptions.EventError() pool = nova.ReservationPool() for allocation in db_api.host_allocation_get_all_by_values( reservation_id=reservation_id): host = db_api.host_get(allocation['compute_host_id']) pool.add_computehost(instance_reservation['aggregate_id'], host['service_name'], stay_in=True)
def update_resources(self, reservation_id): """Updates reserved resources in Nova. This method updates reserved resources in Compute service. If the reservation is in active status, it adds new allocated hosts into a reserved aggregate. If the reservation is not started yet, it updates a reserved flavor. """ reservation = db_api.reservation_get(reservation_id) if reservation['status'] == 'active': pool = nova.ReservationPool() for allocation in db_api.host_allocation_get_all_by_values( reservation_id=reservation['id']): host = db_api.host_get(allocation['compute_host_id']) try: pool.add_computehost(reservation['aggregate_id'], host['service_name'], stay_in=True) except mgr_exceptions.AggregateAlreadyHasHost: pass except nova_exceptions.ClientException: err_msg = ('Fail to add host %s to aggregate %s.' % (host, reservation['aggregate_id'])) raise mgr_exceptions.NovaClientError(err_msg) else: try: self.nova.nova.flavors.delete(reservation['id']) self._create_flavor(reservation['id'], reservation['vcpus'], reservation['memory_mb'], reservation['disk_gb'], reservation['server_group_id']) except nova_exceptions.ClientException: LOG.exception( "Failed to update Nova resources " "for reservation %s", reservation['id']) raise mgr_exceptions.NovaClientError()
def heal_reservations(self, failed_resources): """Heal reservations which suffer from resource failures. :param: failed_resources: a list of failed hosts. :return: a dictionary of {reservation id: flags to update} e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': {'missing_resources': True}} """ reservation_flags = {} failed_allocs = [] for host in failed_resources: failed_allocs += db_api.host_allocation_get_all_by_values( compute_host_id=host['id']) for alloc in failed_allocs: reservation = db_api.reservation_get(alloc['reservation_id']) if reservation['resource_type'] != RESOURCE_TYPE: continue pool = None # Remove the failed host from the aggregate. if reservation['status'] == status.reservation.ACTIVE: host = db_api.host_get(alloc['compute_host_id']) pool = nova.ReservationPool() pool.remove_computehost(reservation['aggregate_id'], host['service_name']) # Allocate alternative resource. values = {} lease = db_api.lease_get(reservation['lease_id']) values['start_date'] = max(datetime.datetime.utcnow(), lease['start_date']) values['end_date'] = lease['end_date'] specs = ['vcpus', 'memory_mb', 'disk_gb', 'affinity', 'amount'] for key in specs: values[key] = reservation[key] changed_hosts = self.pickup_hosts(reservation['id'], values) if len(changed_hosts['added']) == 0: if reservation['id'] not in reservation_flags: reservation_flags[reservation['id']] = {} reservation_flags[reservation['id']].update( {'missing_resources': True}) db_api.host_allocation_destroy(alloc['id']) LOG.warn( 'Could not find alternative host for reservation %s ' '(lease: %s).', reservation['id'], lease['name']) else: new_host_id = changed_hosts['added'].pop() db_api.host_allocation_update(alloc['id'], {'compute_host_id': new_host_id}) if reservation['status'] == status.reservation.ACTIVE: # Add the alternative host into the aggregate. new_host = db_api.host_get(new_host_id) pool.add_computehost(reservation['aggregate_id'], new_host['service_name'], stay_in=True) if reservation['id'] not in reservation_flags: reservation_flags[reservation['id']] = {} reservation_flags[reservation['id']].update( {'resources_changed': True}) LOG.warn('Resource changed for reservation %s (lease: %s).', reservation['id'], lease['name']) return reservation_flags
def heal_reservations(self, failed_resources): """Heal reservations which suffer from resource failures. :param: failed_resources: a list of failed hosts. :return: a dictionary of {reservation id: flags to update} e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': {'missing_resources': True}} """ reservation_flags = {} failed_allocs = [] for host in failed_resources: failed_allocs += db_api.host_allocation_get_all_by_values( compute_host_id=host['id']) for alloc in failed_allocs: reservation = db_api.reservation_get(alloc['reservation_id']) if reservation['resource_type'] != plugin.RESOURCE_TYPE: continue lease = db_api.lease_get(reservation['lease_id']) host_reservation = None pool = None # Remove the failed host from the aggregate. if reservation['status'] == status.reservation.ACTIVE: host = db_api.host_get(alloc['compute_host_id']) host_reservation = db_api.host_reservation_get( reservation['resource_id']) with trusts.create_ctx_from_trust(lease['trust_id']): pool = nova.ReservationPool() pool.remove_computehost(host_reservation['aggregate_id'], host['service_name']) # Allocate alternative resource. start_date = max(datetime.datetime.utcnow(), lease['start_date']) new_hostids = self._matching_hosts( reservation['hypervisor_properties'], reservation['resource_properties'], '1-1', start_date, lease['end_date']) if not new_hostids: if reservation['id'] not in reservation_flags: reservation_flags[reservation['id']] = {} reservation_flags[reservation['id']].update( {'missing_resources': True}) db_api.host_allocation_destroy(alloc['id']) LOG.warn( 'Could not find alternative host for reservation %s ' '(lease: %s).', reservation['id'], lease['name']) else: new_hostid = new_hostids.pop() db_api.host_allocation_update(alloc['id'], {'compute_host_id': new_hostid}) if reservation['status'] == status.reservation.ACTIVE: # Add the alternative host into the aggregate. new_host = db_api.host_get(new_hostid) with trusts.create_ctx_from_trust(lease['trust_id']): pool.add_computehost(host_reservation['aggregate_id'], new_host['service_name']) if reservation['id'] not in reservation_flags: reservation_flags[reservation['id']] = {} reservation_flags[reservation['id']].update( {'resources_changed': True}) LOG.warn('Resource changed for reservation %s (lease: %s).', reservation['id'], lease['name']) return reservation_flags
def create_computehost(self, host_values): # TODO(sbauza): # - Exception handling for HostNotFound host_id = host_values.pop('id', None) host_name = host_values.pop('name', None) try: trust_id = host_values.pop('trust_id') except KeyError: raise manager_ex.MissingTrustId() host_ref = host_id or host_name if host_ref is None: raise manager_ex.InvalidHost(host=host_values) with trusts.create_ctx_from_trust(trust_id): inventory = nova.NovaInventory() servers = inventory.get_servers_per_host(host_ref) if servers: raise manager_ex.HostHavingServers(host=host_ref, servers=servers) host_details = inventory.get_host_details(host_ref) # NOTE(sbauza): Only last duplicate name for same extra capability # will be stored to_store = set(host_values.keys()) - set(host_details.keys()) extra_capabilities_keys = to_store extra_capabilities = dict( (key, host_values[key]) for key in extra_capabilities_keys ) if any([len(key) > 64 for key in extra_capabilities_keys]): raise manager_ex.ExtraCapabilityTooLong() self.placement_client.create_reservation_provider( host_details['hypervisor_hostname']) pool = nova.ReservationPool() pool.add_computehost(self.freepool_name, host_details['service_name']) host = None cantaddextracapability = [] try: if trust_id: host_details.update({'trust_id': trust_id}) host = db_api.host_create(host_details) except db_ex.BlazarDBException as e: # We need to rollback # TODO(sbauza): Investigate use of Taskflow for atomic # transactions pool.remove_computehost(self.freepool_name, host_details['service_name']) self.placement_client.delete_reservation_provider( host_details['hypervisor_hostname']) raise e for key in extra_capabilities: values = {'computehost_id': host['id'], 'capability_name': key, 'capability_value': extra_capabilities[key], } try: db_api.host_extra_capability_create(values) except db_ex.BlazarDBException: cantaddextracapability.append(key) if cantaddextracapability: raise manager_ex.CantAddExtraCapability( keys=cantaddextracapability, host=host['id']) return self.get_computehost(host['id'])