def get_detailed_tenant_quotas(context, resources, tenant_id): """Given a list of resources and a sepecific tenant, retrieve the detailed quotas (limit, used, reserved). :param context: The request context, for access checks. :param resources: A dictionary of the registered resource keys. :return dict: mapping resource name in dict to its corresponding limit used and reserved. Reserved currently returns default value of 0 """ res_reserve_info = quota_api.get_reservations_for_resources( context, tenant_id, resources.keys()) tenant_quota_ext = {} for key, resource in resources.items(): if isinstance(resource, res.TrackedResource): used = resource.count_used(context, tenant_id, resync_usage=False) else: # NOTE(ihrachys) .count won't use the plugin we pass, but we # pass it regardless to keep the quota driver API intact plugins = directory.get_plugins() plugin = plugins.get(key, plugins[constants.CORE]) used = resource.count(context, plugin, tenant_id) tenant_quota_ext[key] = { 'limit': resource.default, 'used': used, 'reserved': res_reserve_info.get(key, 0), } # update with specific tenant limits quota_objs = quota_obj.Quota.get_objects(context, project_id=tenant_id) for item in quota_objs: tenant_quota_ext[item['resource']]['limit'] = item['limit'] return tenant_quota_ext
def count_reserved(self, context, tenant_id): """Return the current reservation count for the resource.""" # NOTE(princenana) Current implementation of reservations # is ephemeral and returns the default value reservations = quota_api.get_reservations_for_resources( context, tenant_id, [self.name]) reserved = reservations.get(self.name, 0) return reserved
def count(self, context, _plugin, tenant_id, resync_usage=True): """Return the current usage count for the resource. This method will fetch aggregate information for resource usage data, unless usage data are marked as "dirty". In the latter case resource usage will be calculated counting rows for tenant_id in the resource's database model. Active reserved amount are instead always calculated by summing amounts for matching records in the 'reservations' database model. The _plugin and _resource parameters are unused but kept for compatibility with the signature of the count method for CountableResource instances. """ # Load current usage data, setting a row-level lock on the DB usage_info = quota_api.get_quota_usage_by_resource_and_tenant( context, self.name, tenant_id, lock_for_update=True) # Always fetch reservations, as they are not tracked by usage counters reservations = quota_api.get_reservations_for_resources( context, tenant_id, [self.name]) reserved = reservations.get(self.name, 0) # If dirty or missing, calculate actual resource usage querying # the database and set/create usage info data # NOTE: this routine "trusts" usage counters at service startup. This # assumption is generally valid, but if the database is tampered with, # or if data migrations do not take care of usage counters, the # assumption will not hold anymore if (tenant_id in self._dirty_tenants or not usage_info or usage_info.dirty): LOG.debug(("Usage tracker for resource:%(resource)s and tenant:" "%(tenant_id)s is out of sync, need to count used " "quota"), { 'resource': self.name, 'tenant_id': tenant_id }) in_use = context.session.query( self._model_class).filter_by(tenant_id=tenant_id).count() # Update quota usage, if requested (by default do not do that, as # typically one counts before adding a record, and that would mark # the usage counter as dirty again) if resync_usage: usage_info = self._resync(context, tenant_id, in_use) else: resource = usage_info.resource if usage_info else self.name tenant_id = usage_info.tenant_id if usage_info else tenant_id dirty = usage_info.dirty if usage_info else True usage_info = quota_api.QuotaUsageInfo(resource, tenant_id, in_use, dirty) LOG.debug(("Quota usage for %(resource)s was recalculated. " "Used quota:%(used)d."), { 'resource': self.name, 'used': usage_info.used }) return usage_info.used + reserved
def count(self, context, _plugin, tenant_id, resync_usage=False): """Return the current usage count for the resource. This method will fetch aggregate information for resource usage data, unless usage data are marked as "dirty". In the latter case resource usage will be calculated counting rows for tenant_id in the resource's database model. Active reserved amount are instead always calculated by summing amounts for matching records in the 'reservations' database model. The _plugin and _resource parameters are unused but kept for compatibility with the signature of the count method for CountableResource instances. """ # Load current usage data, setting a row-level lock on the DB usage_info = quota_api.get_quota_usage_by_resource_and_tenant( context, self.name, tenant_id, lock_for_update=True) # Always fetch reservations, as they are not tracked by usage counters reservations = quota_api.get_reservations_for_resources( context, tenant_id, [self.name]) reserved = reservations.get(self.name, 0) # If dirty or missing, calculate actual resource usage querying # the database and set/create usage info data # NOTE: this routine "trusts" usage counters at service startup. This # assumption is generally valid, but if the database is tampered with, # or if data migrations do not take care of usage counters, the # assumption will not hold anymore if (tenant_id in self._dirty_tenants or not usage_info or usage_info.dirty): LOG.debug(("Usage tracker for resource:%(resource)s and tenant:" "%(tenant_id)s is out of sync, need to count used " "quota"), {'resource': self.name, 'tenant_id': tenant_id}) in_use = context.session.query(self._model_class).filter_by( tenant_id=tenant_id).count() # Update quota usage, if requested (by default do not do that, as # typically one counts before adding a record, and that would mark # the usage counter as dirty again) if resync_usage or not usage_info: usage_info = self._resync(context, tenant_id, in_use) else: # NOTE(salv-orlando): Passing 0 for reserved amount as # reservations are currently not supported usage_info = quota_api.QuotaUsageInfo(usage_info.resource, usage_info.tenant_id, in_use, usage_info.dirty) LOG.debug(("Quota usage for %(resource)s was recalculated. " "Used quota:%(used)d."), {'resource': self.name, 'used': usage_info.used}) return usage_info.used + reserved
def test_get_expired_reservations_for_resources(self): with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow: mock_utcnow.return_value = datetime.datetime(2015, 5, 20, 0, 0) self._get_reservations_for_resource_helper() deltas = quota_api.get_reservations_for_resources( self.context, self.project_id, ['goals', 'assists', 'bookings'], expired=True) self.assertIn('assists', deltas) self.assertEqual(2, deltas['assists']) self.assertIn('bookings', deltas) self.assertEqual(2, deltas['bookings']) self.assertEqual(2, len(deltas))
def resync(self, context, tenant_id): if tenant_id not in self._out_of_sync_tenants: return LOG.debug(("Synchronizing usage tracker for tenant:%(tenant_id)s on " "resource:%(resource)s"), {'tenant_id': tenant_id, 'resource': self.name}) in_use = context.session.query(self._model_class).filter_by( tenant_id=tenant_id).count() reservations = quota_api.get_reservations_for_resources( context, tenant_id, [self.name]) reserved = reservations.get(self.name, 0) # Update quota usage return self._resync(context, tenant_id, in_use, reserved)
def test_get_expired_reservations_for_resources(self): with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow: mock_utcnow.return_value = datetime.datetime( 2015, 5, 20, 0, 0) self._get_reservations_for_resource_helper() deltas = quota_api.get_reservations_for_resources( self.context, self.tenant_id, ['goals', 'assists', 'bookings'], expired=True) self.assertIn('assists', deltas) self.assertEqual(2, deltas['assists']) self.assertIn('bookings', deltas) self.assertEqual(2, deltas['bookings']) self.assertEqual(2, len(deltas))
def resync(self, context, tenant_id): if tenant_id not in self._out_of_sync_tenants: return LOG.debug(("Synchronizing usage tracker for tenant:%(tenant_id)s on " "resource:%(resource)s"), { 'tenant_id': tenant_id, 'resource': self.name }) in_use = context.session.query( self._model_class).filter_by(tenant_id=tenant_id).count() reservations = quota_api.get_reservations_for_resources( context, tenant_id, [self.name]) reserved = reservations.get(self.name, 0) # Update quota usage return self._resync(context, tenant_id, in_use, reserved)
def make_reservation(self, context, tenant_id, resources, deltas, plugin): # Lock current reservation table # NOTE(salv-orlando): This routine uses DB write locks. # These locks are acquired by the count() method invoked on resources. # Please put your shotguns aside. # A non locking algorithm for handling reservation is feasible, however # it will require two database writes even in cases when there are not # concurrent reservations. # For this reason it might be advisable to handle contention using # this kind of locks and paying the cost of a write set certification # failure when a MySQL Galera cluster is employed. Also, this class of # locks should be ok to use when support for sending "hotspot" writes # to a single node will be available. requested_resources = deltas.keys() with db_api.autonested_transaction(context.session): # get_tenant_quotes needs in input a dictionary mapping resource # name to BaseResosurce instances so that the default quota can be # retrieved current_limits = self.get_tenant_quotas( context, resources, tenant_id) unlimited_resources = set([resource for (resource, limit) in current_limits.items() if limit < 0]) # Do not even bother counting resources and calculating headroom # for resources with unlimited quota LOG.debug(("Resources %s have unlimited quota limit. It is not " "required to calculated headroom "), ",".join(unlimited_resources)) requested_resources = (set(requested_resources) - unlimited_resources) # Gather current usage information # TODO(salv-orlando): calling count() for every resource triggers # multiple queries on quota usage. This should be improved, however # this is not an urgent matter as the REST API currently only # allows allocation of a resource at a time # NOTE: pass plugin too for compatibility with CountableResource # instances current_usages = dict( (resource, resources[resource].count( context, plugin, tenant_id, resync_usage=False)) for resource in requested_resources) # Adjust for expired reservations. Apparently it is cheaper than # querying every time for active reservations and counting overall # quantity of resources reserved expired_deltas = quota_api.get_reservations_for_resources( context, tenant_id, requested_resources, expired=True) # Verify that the request can be accepted with current limits resources_over_limit = [] for resource in requested_resources: expired_reservations = expired_deltas.get(resource, 0) total_usage = current_usages[resource] - expired_reservations res_headroom = current_limits[resource] - total_usage LOG.debug(("Attempting to reserve %(delta)d items for " "resource %(resource)s. Total usage: %(total)d; " "quota limit: %(limit)d; headroom:%(headroom)d"), {'resource': resource, 'delta': deltas[resource], 'total': total_usage, 'limit': current_limits[resource], 'headroom': res_headroom}) if res_headroom < deltas[resource]: resources_over_limit.append(resource) if expired_reservations: self._handle_expired_reservations(context, tenant_id) if resources_over_limit: raise exceptions.OverQuota(overs=sorted(resources_over_limit)) # Success, store the reservation # TODO(salv-orlando): Make expiration time configurable return quota_api.create_reservation( context, tenant_id, deltas)
def make_reservation(self, context, tenant_id, resources, deltas, plugin): # Lock current reservation table # NOTE(salv-orlando): This routine uses DB write locks. # These locks are acquired by the count() method invoked on resources. # Please put your shotguns aside. # A non locking algorithm for handling reservation is feasible, however # it will require two database writes even in cases when there are not # concurrent reservations. # For this reason it might be advisable to handle contention using # this kind of locks and paying the cost of a write set certification # failure when a MySQL Galera cluster is employed. Also, this class of # locks should be ok to use when support for sending "hotspot" writes # to a single node will be available. requested_resources = deltas.keys() with db_api.context_manager.writer.using(context): # get_tenant_quotes needs in input a dictionary mapping resource # name to BaseResosurce instances so that the default quota can be # retrieved current_limits = self.get_tenant_quotas(context, resources, tenant_id) unlimited_resources = set([ resource for (resource, limit) in current_limits.items() if limit < 0 ]) # Do not even bother counting resources and calculating headroom # for resources with unlimited quota LOG.debug( "Resources %s have unlimited quota limit. It is not " "required to calculate headroom ", ",".join(unlimited_resources)) requested_resources = (set(requested_resources) - unlimited_resources) # Gather current usage information # TODO(salv-orlando): calling count() for every resource triggers # multiple queries on quota usage. This should be improved, however # this is not an urgent matter as the REST API currently only # allows allocation of a resource at a time # NOTE: pass plugin too for compatibility with CountableResource # instances current_usages = dict((resource, resources[resource].count( context, plugin, tenant_id, resync_usage=False)) for resource in requested_resources) # Adjust for expired reservations. Apparently it is cheaper than # querying every time for active reservations and counting overall # quantity of resources reserved expired_deltas = quota_api.get_reservations_for_resources( context, tenant_id, requested_resources, expired=True) # Verify that the request can be accepted with current limits resources_over_limit = [] for resource in requested_resources: expired_reservations = expired_deltas.get(resource, 0) total_usage = current_usages[resource] - expired_reservations res_headroom = current_limits[resource] - total_usage LOG.debug( ("Attempting to reserve %(delta)d items for " "resource %(resource)s. Total usage: %(total)d; " "quota limit: %(limit)d; headroom:%(headroom)d"), { 'resource': resource, 'delta': deltas[resource], 'total': total_usage, 'limit': current_limits[resource], 'headroom': res_headroom }) if res_headroom < deltas[resource]: resources_over_limit.append(resource) if expired_reservations: self._handle_expired_reservations(context, tenant_id) if resources_over_limit: raise exceptions.OverQuota(overs=sorted(resources_over_limit)) # Success, store the reservation # TODO(salv-orlando): Make expiration time configurable return quota_api.create_reservation(context, tenant_id, deltas)
def test_get_reservation_for_resources_with_empty_list(self): self.assertIsNone(quota_api.get_reservations_for_resources( self.context, self.tenant_id, []))