def reduceL1Cache(self, sp, cacheL1, cacheL2, assigned):
logger.debug("Reducing L1 cache erasing a service in cache for {0}".format(sp))
# We will try to destroy the newest cacheL1 element that is USABLE if the deployer can't cancel a new service creation
cacheItems = sp.cachedUserServices().filter(UserServiceManager.getCacheStateFilter(services.UserDeployment.L1_CACHE)).order_by('-creation_date')
if len(cacheItems) == 0:
logger.debug('There is more services than configured, but could not reduce cache cause its already empty')
return
if cacheL2 < sp.cache_l2_srvs:
valid = None
for n in cacheItems:
if n.needsOsManager():
if State.isUsable(n.state) is False or State.isUsable(n.os_state):
valid = n
break
else:
valid = n
break
if valid is not None:
valid.moveToLevel(services.UserDeployment.L2_CACHE)
return
cache = cacheItems[0]
cache.removeOrCancel()
def reduceL2Cache(self, sp, cacheL1, cacheL2, assigned):
logger.debug("Reducing L2 cache erasing a service in cache for {0}".format(sp))
if cacheL2 > 0:
cacheItems = sp.cachedUserServices().filter(UserServiceManager.getCacheStateFilter(services.UserDeployment.L2_CACHE)).order_by('creation_date')
# TODO: Look first for non finished cache items and cancel them
cache = cacheItems[0]
cache.removeOrCancel()
def growL1Cache(self, sp, cacheL1, cacheL2, assigned):
'''
This method tries to enlarge L1 cache.
If for some reason the number of deployed services (Counting all, ACTIVE
and PREPARING, assigned, L1 and L2) is over max allowed service deployments,
this method will not grow the L1 cache
'''
logger.debug("Growing L1 cache creating a new service for {0}".format(sp))
# First, we try to assign from L2 cache
if cacheL2 > 0:
valid = None
with transaction.atomic():
for n in sp.cachedUserServices().select_for_update().filter(UserServiceManager.getCacheStateFilter(services.UserDeployment.L2_CACHE)).order_by('creation_date'):
if n.needsOsManager():
if State.isUsable(n.state) is False or State.isUsable(n.os_state):
valid = n
break
else:
valid = n
break
if valid is not None:
valid.moveToLevel(services.UserDeployment.L1_CACHE)
return
try:
UserServiceManager.manager().createCacheFor(sp.activePublication(), services.UserDeployment.L1_CACHE)
except MaxServicesReachedException as e:
log.doLog(sp, log.ERROR, 'Max number of services reached for this service', log.INTERNAL)
logger.error(str(e))
except:
logger.exception('Exception')
def reduceL1Cache(self, sp, cacheL1, cacheL2, assigned):
logger.debug(
"Reducing L1 cache erasing a service in cache for {0}".format(sp))
# We will try to destroy the newest cacheL1 element that is USABLE if the deployer can't cancel a new service creation
cacheItems = sp.cachedUserServices().filter(
UserServiceManager.getCacheStateFilter(
services.UserDeployment.L1_CACHE)).order_by('-creation_date')
if len(cacheItems) == 0:
logger.debug(
'There is more services than configured, but could not reduce cache cause its already empty'
)
return
if cacheL2 < sp.cache_l2_srvs:
valid = None
for n in cacheItems:
if n.needsOsManager():
if State.isUsable(n.state) is False or State.isUsable(
n.os_state):
valid = n
break
else:
valid = n
break
if valid is not None:
valid.moveToLevel(services.UserDeployment.L2_CACHE)
return
cache = cacheItems[0]
cache.removeOrCancel()
def reduceL2Cache(self, sp, cacheL1, cacheL2, assigned):
logger.debug(
"Reducing L2 cache erasing a service in cache for {0}".format(sp))
if cacheL2 > 0:
cacheItems = sp.cachedUserServices().filter(
UserServiceManager.getCacheStateFilter(
services.UserDeployment.L2_CACHE)).order_by(
'creation_date')
# TODO: Look first for non finished cache items and cancel them
cache = cacheItems[0]
cache.removeOrCancel()
def growL1Cache(self, sp, cacheL1, cacheL2, assigned):
"""
This method tries to enlarge L1 cache.
If for some reason the number of deployed services (Counting all, ACTIVE
and PREPARING, assigned, L1 and L2) is over max allowed service deployments,
this method will not grow the L1 cache
"""
logger.debug(
"Growing L1 cache creating a new service for {0}".format(sp))
# First, we try to assign from L2 cache
if cacheL2 > 0:
valid = None
with transaction.atomic():
for n in sp.cachedUserServices().select_for_update().filter(
UserServiceManager.getCacheStateFilter(
services.UserDeployment.L2_CACHE)).order_by(
'creation_date'):
if n.needsOsManager():
if State.isUsable(n.state) is False or State.isUsable(
n.os_state):
valid = n
break
else:
valid = n
break
if valid is not None:
valid.moveToLevel(services.UserDeployment.L1_CACHE)
return
try:
UserServiceManager.manager().createCacheFor(
sp.activePublication(), services.UserDeployment.L1_CACHE)
except MaxServicesReachedError as e:
log.doLog(sp, log.ERROR,
'Max number of services reached for this service',
log.INTERNAL)
logger.error(str(e))
except:
logger.exception('Exception')
def servicesPoolsNeedingCacheUpdate(self):
# State filter for cached and inAssigned objects
# First we get all deployed services that could need cache generation
DeployedService.objects.update()
# We start filtering out the deployed services that do not need caching at all.
whichNeedsCaching = DeployedService.objects.filter(Q(initial_srvs__gte=0) | Q(cache_l1_srvs__gte=0)).filter(max_srvs__gt=0, state=State.ACTIVE,
service__provider__maintenance_mode=False)[:]
# We will get the one that proportionally needs more cache
servicesPools = []
for sp in whichNeedsCaching:
sp.userServices.update() # Cleans cached queries
# If this deployedService don't have a publication active and needs it, ignore it
if sp.activePublication() is None and sp.service.getInstance().publicationType is not None:
logger.debug('Needs publication but do not have one, cache test ignored')
continue
# If it has any running publication, do not generate cache anymore
if sp.publications.filter(state=State.PREPARING).count() > 0:
logger.debug('Stopped cache generation for deployed service with publication running: {0}'.format(sp))
continue
if sp.isRestrained():
ServiceCacheUpdater.__notifyRestrain(sp)
continue
# Get data related to actual state of cache
inCacheL1 = sp.cachedUserServices().filter(UserServiceManager.getCacheStateFilter(services.UserDeployment.L1_CACHE)).count()
inCacheL2 = sp.cachedUserServices().filter(UserServiceManager.getCacheStateFilter(services.UserDeployment.L2_CACHE)).count()
inAssigned = sp.assignedUserServices().filter(UserServiceManager.getStateFilter()).count()
# if we bypasses max cache, we will reduce it in first place. This is so because this will free resources on service provider
logger.debug("Examining {0} with {1} in cache L1 and {2} in cache L2, {3} inAssigned".format(
sp, inCacheL1, inCacheL2, inAssigned))
totalL1Assigned = inCacheL1 + inAssigned
# We have more than we want
if totalL1Assigned > sp.max_srvs:
logger.debug('We have more services than max configured')
servicesPools.append((sp, inCacheL1, inCacheL2, inAssigned))
continue
# We have more in L1 cache than needed
if totalL1Assigned > sp.initial_srvs and inCacheL1 > sp.cache_l1_srvs:
logger.debug('We have more services in cache L1 than configured')
servicesPools.append((sp, inCacheL1, inCacheL2, inAssigned))
continue
# If we have more in L2 cache than needed, decrease L2 cache, but int this case, we continue checking cause L2 cache removal
# has less priority than l1 creations or removals, but higher. In this case, we will simply take last l2 oversized found and reduce it
if inCacheL2 > sp.cache_l2_srvs:
logger.debug('We have more services in L2 cache than configured, decreasing it')
servicesPools.append((sp, inCacheL1, inCacheL2, inAssigned))
continue
# If this service don't allows more starting user services, continue
if UserServiceManager.manager().canInitiateServiceFromDeployedService(sp) is False:
logger.debug('This provider has the max allowed starting services running: {0}'.format(sp))
continue
# If wee need to grow l2 cache, annotate it
# Whe check this before checking the total, because the l2 cache is independent of max services or l1 cache.
# It reflects a value that must be keeped in cache for futre fast use.
if inCacheL2 < sp.cache_l2_srvs:
logger.debug('Needs to grow L2 cache for {}'.format(sp))
servicesPools.append((sp, inCacheL1, inCacheL2, inAssigned))
continue
# We skip it if already at max
if totalL1Assigned == sp.max_srvs:
continue
if totalL1Assigned < sp.initial_srvs or inCacheL1 < sp.cache_l1_srvs:
logger.debug('Needs to grow L1 cache for {}'.format(sp))
servicesPools.append((sp, inCacheL1, inCacheL2, inAssigned))
# We also return calculated values so we can reuse then
return servicesPools
def servicesPoolsNeedingCacheUpdate(self):
# State filter for cached and inAssigned objects
# First we get all deployed services that could need cache generation
DeployedService.objects.update()
# We start filtering out the deployed services that do not need caching at all.
whichNeedsCaching = DeployedService.objects.filter(
Q(initial_srvs__gte=0) | Q(cache_l1_srvs__gte=0)).filter(
max_srvs__gt=0,
state=State.ACTIVE,
service__provider__maintenance_mode=False)[:]
# We will get the one that proportionally needs more cache
servicesPools = []
for sp in whichNeedsCaching:
sp.userServices.update() # Cleans cached queries
# If this deployedService don't have a publication active and needs it, ignore it
spServiceInstance = sp.service.getInstance()
if sp.activePublication(
) is None and spServiceInstance.publicationType is not None:
logger.debug(
'{} Needs publication but do not have one, cache test ignored'
.format(sp))
continue
# If it has any running publication, do not generate cache anymore
if sp.publications.filter(state=State.PREPARING).count() > 0:
logger.debug(
'Stopped cache generation for deployed service with publication running: {0}'
.format(sp))
continue
if sp.isRestrained():
ServiceCacheUpdater.__notifyRestrain(sp)
continue
# Get data related to actual state of cache
inCacheL1 = sp.cachedUserServices().filter(
UserServiceManager.getCacheStateFilter(
services.UserDeployment.L1_CACHE)).count()
inCacheL2 = sp.cachedUserServices().filter(
UserServiceManager.getCacheStateFilter(
services.UserDeployment.L2_CACHE)).count()
inAssigned = sp.assignedUserServices().filter(
UserServiceManager.getStateFilter()).count()
# if we bypasses max cache, we will reduce it in first place. This is so because this will free resources on service provider
logger.debug(
"Examining {0} with {1} in cache L1 and {2} in cache L2, {3} inAssigned"
.format(sp, inCacheL1, inCacheL2, inAssigned))
totalL1Assigned = inCacheL1 + inAssigned
# We have more than we want
if totalL1Assigned > sp.max_srvs:
logger.debug('We have more services than max configured')
servicesPools.append((sp, inCacheL1, inCacheL2, inAssigned))
continue
# We have more in L1 cache than needed
if totalL1Assigned > sp.initial_srvs and inCacheL1 > sp.cache_l1_srvs:
logger.debug(
'We have more services in cache L1 than configured')
servicesPools.append((sp, inCacheL1, inCacheL2, inAssigned))
continue
# If we have more in L2 cache than needed, decrease L2 cache, but int this case, we continue checking cause L2 cache removal
# has less priority than l1 creations or removals, but higher. In this case, we will simply take last l2 oversized found and reduce it
if inCacheL2 > sp.cache_l2_srvs:
logger.debug(
'We have more services in L2 cache than configured, decreasing it'
)
servicesPools.append((sp, inCacheL1, inCacheL2, inAssigned))
continue
# If this service don't allows more starting user services, continue
if UserServiceManager.manager(
).canInitiateServiceFromDeployedService(sp) is False:
logger.debug(
'This provider has the max allowed starting services running: {0}'
.format(sp))
continue
# If wee need to grow l2 cache, annotate it
# Whe check this before checking the total, because the l2 cache is independent of max services or l1 cache.
# It reflects a value that must be keeped in cache for futre fast use.
if inCacheL2 < sp.cache_l2_srvs:
logger.debug('Needs to grow L2 cache for {}'.format(sp))
servicesPools.append((sp, inCacheL1, inCacheL2, inAssigned))
continue
# We skip it if already at max
if totalL1Assigned == sp.max_srvs:
continue
if totalL1Assigned < sp.initial_srvs or inCacheL1 < sp.cache_l1_srvs:
logger.debug('Needs to grow L1 cache for {}'.format(sp))
servicesPools.append((sp, inCacheL1, inCacheL2, inAssigned))
# We also return calculated values so we can reuse then
return servicesPools
