def publish(self, parent: models.ServicePool):
"""
Custom method "publish", provided to initiate a publication of a deployed service
:param parent: Parent service pool
"""
changeLog = self._params[
'changelog'] if 'changelog' in self._params else None
if permissions.checkPermissions(
self._user, parent,
permissions.PERMISSION_MANAGEMENT) is False:
logger.debug('Management Permission failed for user %s',
self._user)
raise self.accessDenied()
logger.debug('Custom "publish" invoked for %s', parent)
parent.publish(
changeLog
) # Can raise exceptions that will be processed on response
log.doLog(
parent, log.INFO, "Initated publication v{} by {}".format(
parent.current_pub_revision, self._user.pretty_name),
log.ADMIN)
return self.success()
def growL1Cache(self, servicePool: ServicePool, cacheL1: int, cacheL2: int, assigned: int) -> None:
"""
This method tries to enlarge L1 cache.
If for some reason the number of deployed services (Counting all, ACTIVE
and PREPARING, assigned, L1 and L2) is over max allowed service deployments,
this method will not grow the L1 cache
"""
logger.debug('Growing L1 cache creating a new service for %s', servicePool.name)
# First, we try to assign from L2 cache
if cacheL2 > 0:
valid = None
with transaction.atomic():
for n in servicePool.cachedUserServices().select_for_update().filter(userServiceManager().getCacheStateFilter(services.UserDeployment.L2_CACHE)).order_by('creation_date'):
if n.needsOsManager():
if State.isUsable(n.state) is False or State.isUsable(n.os_state):
valid = n
break
else:
valid = n
break
if valid is not None:
valid.moveToLevel(services.UserDeployment.L1_CACHE)
return
try:
# This has a velid publication, or it will not be here
userServiceManager().createCacheFor(
typing.cast(ServicePoolPublication, servicePool.activePublication()), services.UserDeployment.L1_CACHE
)
except MaxServicesReachedError:
log.doLog(servicePool, log.ERROR, 'Max number of services reached for this service', log.INTERNAL)
logger.warning('Max user services reached for %s: %s. Cache not created', servicePool.name, servicePool.max_srvs)
except Exception:
logger.exception('Exception')
def startRemovalOf(self, servicePool: ServicePool):
if (
servicePool.service is None
): # Maybe an inconsistent value? (must not, but if no ref integrity in db, maybe someone "touched.. ;)")
logger.error('Found service pool %s without service',
servicePool.name)
servicePool.delete(
) # Just remove it "a las bravas", the best we can do
return
# Get publications in course...., that only can be one :)
logger.debug('Removal process of %s', servicePool)
publishing = servicePool.publications.filter(state=State.PREPARING)
for pub in publishing:
pub.cancel()
# Now all publishments are canceling, let's try to cancel cache and assigned
uServices: typing.Iterable[
UserService] = servicePool.userServices.filter(
state=State.PREPARING)
for userService in uServices:
logger.debug('Canceling %s', userService)
userService.cancel()
# Nice start of removal, maybe we need to do some limitation later, but there should not be too much services nor publications cancelable at once
servicePool.state = State.REMOVING
servicePool.name += ' (removed)'
servicePool.save()
def deleteItem(self, item: ServicePool) -> None:
try:
logger.debug('Deleting %s', item)
item.remove(
) # This will mark it for deletion, but in fact will not delete it directly
except Exception:
# Eat it and logit
logger.exception('deleting service pool')
def setFallbackAccess(self, item: ServicePool):
self.ensureAccess(item, permissions.PERMISSION_MANAGEMENT)
fallback = self._params.get('fallbackAccess')
if fallback != '':
logger.debug('Setting fallback of %s to %s', item.name, fallback)
item.fallbackAccess = fallback
item.save()
return item.fallbackAccess
def continueRemovalOf(self, servicePool: ServicePool):
# Recheck that there is no publication created in "bad moment"
try:
for pub in servicePool.publications.filter(state=State.PREPARING):
pub.cancel()
except Exception:
pass
try:
# Now all publications are canceling, let's try to cancel cache and assigned also
uServices: typing.Iterable[
UserService] = servicePool.userServices.filter(
state=State.PREPARING)
for userService in uServices:
logger.debug('Canceling %s', userService)
userService.cancel()
except Exception:
pass
# First, we remove all publications and user services in "info_state"
with transaction.atomic():
servicePool.userServices.select_for_update().filter(
state__in=State.INFO_STATES).delete()
# Mark usable user services as removable
now = getSqlDatetime()
with transaction.atomic():
servicePool.userServices.select_for_update().filter(
state=State.USABLE).update(state=State.REMOVABLE,
state_date=now)
# When no service is at database, we start with publications
if servicePool.userServices.all().count() == 0:
try:
logger.debug(
'All services removed, checking active publication')
if servicePool.activePublication() is not None:
logger.debug('Active publication found, unpublishing it')
servicePool.unpublish()
else:
logger.debug(
'No active publication found, removing info states and checking if removal is done'
)
servicePool.publications.filter(
state__in=State.INFO_STATES).delete()
if servicePool.publications.count() == 0:
servicePool.removed(
) # Mark it as removed, clean later from database
except Exception:
logger.exception(
'Cought unexpected exception at continueRemovalOf: ')
def getItems(self, parent: models.ServicePool, item: typing.Optional[str]):
# Extract provider
try:
if not item:
return [
AssignedService.itemToDict(k, True) for k in
parent.cachedUserServices().all().prefetch_related(
'properties', 'deployed_service', 'publication')
]
cachedService: models.UserService = parent.cachedUserServices(
).get(uuid=processUuid(item))
return AssignedService.itemToDict(cachedService, True)
except Exception:
logger.exception('getItems')
raise self.invalidItemException()
def get(self):
logger.debug('args: {0}'.format(self._args))
if len(self._args) == 1:
if self._args[0] == 'overview': # System overview
users = User.objects.count()
groups = Group.objects.count()
services = Service.objects.count()
service_pools = ServicePool.objects.count()
meta_pools = MetaPool.objects.count()
user_services = UserService.objects.exclude(state__in=(State.REMOVED, State.ERROR)).count()
restrained_services_pools = len(ServicePool.getRestraineds())
return {
'users': users,
'groups': groups,
'services': services,
'service_pools': service_pools,
'meta_pools': meta_pools,
'user_services': user_services,
'restrained_services_pools': restrained_services_pools,
}
if len(self._args) == 2:
if self._args[0] == 'stats':
if self._args[1] == 'assigned':
return getServicesPoolsCounters(None, counters.CT_ASSIGNED)
if self._args[1] == 'inuse':
return getServicesPoolsCounters(None, counters.CT_INUSE)
raise RequestError('invalid request')
def publish(self, servicePool: ServicePool, changeLog: typing.Optional[str] = None): # pylint: disable=no-self-use
"""
Initiates the publication of a service pool, or raises an exception if this cannot be done
:param servicePool: Service pool object (db object)
:param changeLog: if not None, store change log string on "change log" table
"""
if servicePool.publications.filter(state__in=State.PUBLISH_STATES).count() > 0:
raise PublishException(_('Already publishing. Wait for previous publication to finish and try again'))
if servicePool.isInMaintenance():
raise PublishException(_('Service is in maintenance mode and new publications are not allowed'))
publication: typing.Optional[ServicePoolPublication] = None
try:
now = getSqlDatetime()
publication = servicePool.publications.create(state=State.LAUNCHING, state_date=now, publish_date=now, revision=servicePool.current_pub_revision)
if changeLog:
servicePool.changelog.create(revision=servicePool.current_pub_revision, log=changeLog, stamp=now)
if publication:
DelayedTaskRunner.runner().insert(PublicationLauncher(publication), 4, PUBTAG + str(publication.id))
except Exception as e:
logger.debug('Caught exception at publish: %s', e)
if publication is not None:
try:
publication.delete()
except Exception:
logger.info('Could not delete %s', publication)
raise PublishException(str(e))
def createAssignedFor(self, servicePool: ServicePool,
user: User) -> UserService:
"""
Creates a new assigned deployed service for the current publication (if any) of service pool and user indicated
"""
# First, honor maxPreparingServices
if self.canInitiateServiceFromDeployedService(servicePool) is False:
# Cannot create new
logger.info(
'Too many preparing services. Creation of assigned service denied by max preparing services parameter. (login storm with insufficient cache?).'
)
raise ServiceNotReadyError()
if servicePool.service.getType().publicationType is not None:
publication = servicePool.activePublication()
logger.debug(
'Creating a new assigned element for user %s por publication %s',
user, publication)
if publication:
assigned = self.__createAssignedAtDb(publication, user)
else:
raise Exception(
'Invalid publication creating service assignation: {} {}'.
format(servicePool, user))
else:
logger.debug('Creating a new assigned element for user %s', user)
assigned = self.__createAssignedAtDbForNoPublication(
servicePool, user)
assignedInstance = assigned.getInstance()
state = assignedInstance.deployForUser(user)
UserServiceOpChecker.makeUnique(assigned, assignedInstance, state)
return assigned
def createFromAssignable(self, servicePool: ServicePool, user: User, assignableId: str) -> UserService:
"""
Creates an assigned service from an "assignable" id
"""
serviceInstance = servicePool.service.getInstance()
if not serviceInstance.canAssign():
raise Exception('This service type cannot assign asignables')
if servicePool.service.getType().publicationType is not None:
publication = servicePool.activePublication()
logger.debug('Creating an assigned element from assignable %s for user %s por publication %s', user, assignableId, publication)
if publication:
assigned = self.__createAssignedAtDb(publication, user)
else:
raise Exception('Invalid publication creating service assignation: {} {}'.format(servicePool, user))
else:
logger.debug('Creating an assigned element from assignable %s for user %s', assignableId, user)
assigned = self.__createAssignedAtDbForNoPublication(servicePool, user)
# Now, get from serviceInstance the data
assignedInstance = assigned.getInstance()
state = serviceInstance.assignFromAssignables(assignableId, user, assignedInstance)
# assigned.updateData(assignedInstance)
UserServiceOpChecker.makeUnique(assigned, assignedInstance, state)
return assigned
def reduceL1Cache(self, servicePool: ServicePool, cacheL1: int, cacheL2: int, assigned: int):
logger.debug("Reducing L1 cache erasing a service in cache for %s", servicePool)
# We will try to destroy the newest cacheL1 element that is USABLE if the deployer can't cancel a new service creation
cacheItems: typing.List[UserService] = list(
servicePool.cachedUserServices().filter(
userServiceManager().getCacheStateFilter(services.UserDeployment.L1_CACHE)
).exclude(
Q(properties__name='destroy_after') & Q(properties__value='y')
).order_by(
'-creation_date'
).iterator()
)
if not cacheItems:
logger.debug('There is more services than max configured, but could not reduce cache L1 cause its already empty')
return
if cacheL2 < servicePool.cache_l2_srvs:
valid = None
for n in cacheItems:
if n.needsOsManager():
if State.isUsable(n.state) is False or State.isUsable(n.os_state):
valid = n
break
else:
valid = n
break
if valid is not None:
valid.moveToLevel(services.UserDeployment.L2_CACHE)
return
cache = cacheItems[0]
cache.removeOrCancel()
def getExistingAssignationForUser(self, servicePool: ServicePool, user: User) -> typing.Optional[UserService]:
existing = servicePool.assignedUserServices().filter(user=user, state__in=State.VALID_STATES) # , deployed_service__visible=True
lenExisting = existing.count()
if lenExisting > 0: # Already has 1 assigned
logger.debug('Found assigned service from %s to user %s', servicePool, user.name)
return existing[0]
return None
def getServicesPoolsCounters(servicePool, counter_type):
# pylint: disable=no-value-for-parameter
try:
cacheKey = (servicePool and servicePool.id or 'all') + str(counter_type) + str(POINTS) + str(SINCE)
to = getSqlDatetime()
since = to - timedelta(days=SINCE)
val = cache.get(cacheKey)
if val is None:
if servicePool is None:
us = ServicePool()
complete = True # Get all deployed services stats
else:
us = servicePool
complete = False
val = []
for x in counters.getCounters(us, counter_type, since=since, to=to, limit=POINTS, use_max=USE_MAX, all=complete):
val.append({'stamp': x[0], 'value': int(x[1])})
if len(val) > 2:
cache.put(cacheKey, encoders.encode(pickle.dumps(val), 'zip') , 600)
else:
val = [{'stamp': since, 'value': 0}, {'stamp': to, 'value': 0}]
else:
val = pickle.loads(encoders.decode(val, 'zip'))
return val
except:
logger.exception('exception')
raise ResponseError('can\'t create stats for objects!!!')
def getLogs(self, parent: models.ServicePool,
item: str) -> typing.List[typing.Any]:
try:
item = parent.cachedUserServices().get(uuid=processUuid(item))
logger.debug('Getting logs for %s', item)
return log.getLogs(item)
except Exception:
raise self.invalidItemException()
def reduceL2Cache(self, servicePool: ServicePool, cacheL1: int, cacheL2: int, assigned: int):
logger.debug("Reducing L2 cache erasing a service in cache for %s", servicePool.name)
if cacheL2 > 0:
cacheItems: typing.List[UserService] = servicePool.cachedUserServices().filter(
userServiceManager().getCacheStateFilter(services.UserDeployment.L2_CACHE)
).order_by('creation_date')
# TODO: Look first for non finished cache items and cancel them?
cache = cacheItems[0]
cache.removeOrCancel()
def growL2Cache(self, servicePool: ServicePool, cacheL1: int, cacheL2: int, assigned: int) -> None:
"""
Tries to grow L2 cache of service.
If for some reason the number of deployed services (Counting all, ACTIVE
and PREPARING, assigned, L1 and L2) is over max allowed service deployments,
this method will not grow the L1 cache
"""
logger.debug("Growing L2 cache creating a new service for %s", servicePool.name)
try:
# This has a velid publication, or it will not be here
userServiceManager().createCacheFor(
typing.cast(ServicePoolPublication, servicePool.activePublication()), services.UserDeployment.L2_CACHE
)
except MaxServicesReachedError:
logger.warning('Max user services reached for %s: %s. Cache not created', servicePool.name, servicePool.max_srvs)
def getPoolsForGroups(groups):
for servicePool in ServicePool.getDeployedServicesForGroups(groups):
yield servicePool
def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: # pylint: disable=too-many-locals, too-many-branches, too-many-statements
"""Obtains the service data dictionary will all available services for this request
Arguments:
request {HttpRequest} -- request from where to xtract credentials
Returns:
typing.Dict[str, typing.Any] -- Keys has this:
'services': services,
'ip': request.ip,
'nets': nets,
'transports': validTrans,
'autorun': autorun
"""
# Session data
os: typing.Dict[str, str] = request.os
# We look for services for this authenticator groups. User is logged in in just 1 authenticator, so his groups must coincide with those assigned to ds
groups = list(request.user.getGroups())
availServicePools = ServicePool.getDeployedServicesForGroups(groups)
availMetaPools = MetaPool.getForGroups(groups)
# Information for administrators
nets = ''
validTrans = ''
logger.debug('OS: %s', os['OS'])
if request.user.isStaff():
nets = ','.join([n.name for n in Network.networksFor(request.ip)])
tt = []
for t in Transport.objects.all():
if t.validForIp(request.ip):
tt.append(t.name)
validTrans = ','.join(tt)
logger.debug('Checking meta pools: %s', availMetaPools)
services = []
# Add meta pools data first
for meta in availMetaPools:
# Check that we have access to at least one transport on some of its children
hasUsablePools = False
in_use = False
for pool in meta.pools.all():
# if pool.isInMaintenance():
# continue
for t in pool.transports.all():
typeTrans = t.getType()
if t.getType() and t.validForIp(request.ip) and typeTrans.supportsOs(os['OS']) and t.validForOs(os['OS']):
hasUsablePools = True
break
if not in_use:
assignedUserService = userServiceManager().getExistingAssignationForUser(pool, request.user)
if assignedUserService:
in_use = assignedUserService.in_use
# Stop when 1 usable pool is found
if hasUsablePools:
break
# If no usable pools, this is not visible
if hasUsablePools:
group = meta.servicesPoolGroup.as_dict if meta.servicesPoolGroup else ServicePoolGroup.default().as_dict
services.append({
'id': 'M' + meta.uuid,
'name': meta.name,
'visual_name': meta.visual_name,
'description': meta.comments,
'group': group,
'transports': [{
'id': 'meta',
'name': 'meta',
'link': html.udsMetaLink(request, 'M' + meta.uuid),
'priority': 0
}],
'imageId': meta.image and meta.image.uuid or 'x',
'show_transports': False,
'allow_users_remove': False,
'allow_users_reset': False,
'maintenance': meta.isInMaintenance(),
'not_accesible': not meta.isAccessAllowed(),
'in_use': in_use,
'to_be_replaced': None,
'to_be_replaced_text': '',
})
# Now generic user service
for svr in availServicePools:
# Skip pools that are part of meta pools
if svr.is_meta:
continue
trans = []
for t in svr.transports.all().order_by('priority'):
typeTrans = t.getType()
if typeTrans is None: # This may happen if we "remove" a transport type but we have a transport of that kind on DB
continue
if t.validForIp(request.ip) and typeTrans.supportsOs(os['OS']) and t.validForOs(os['OS']):
if typeTrans.ownLink is True:
link = reverse('TransportOwnLink', args=('F' + svr.uuid, t.uuid))
else:
link = html.udsAccessLink(request, 'F' + svr.uuid, t.uuid)
trans.append(
{
'id': t.uuid,
'name': t.name,
'link': link,
'priority': t.priority
}
)
# If empty transports, do not include it on list
if not trans:
continue
if svr.image is not None:
imageId = svr.image.uuid
else:
imageId = 'x'
# Locate if user service has any already assigned user service for this
ads = userServiceManager().getExistingAssignationForUser(svr, request.user)
if ads is None:
in_use = False
else:
in_use = ads.in_use
group = svr.servicesPoolGroup.as_dict if svr.servicesPoolGroup else ServicePoolGroup.default().as_dict
tbr = svr.toBeReplaced(request.user)
if tbr:
tbr = formats.date_format(tbr, "SHORT_DATETIME_FORMAT")
tbrt = ugettext('This service is about to be replaced by a new version. Please, close the session before {} and save all your work to avoid loosing it.').format(tbr)
else:
tbrt = ''
services.append({
'id': 'F' + svr.uuid,
'name': svr.name,
'visual_name': svr.visual_name,
'description': svr.comments,
'group': group,
'transports': trans,
'imageId': imageId,
'show_transports': svr.show_transports,
'allow_users_remove': svr.allow_users_remove,
'allow_users_reset': svr.allow_users_reset,
'maintenance': svr.isInMaintenance(),
'not_accesible': not svr.isAccessAllowed(),
'in_use': in_use,
'to_be_replaced': tbr,
'to_be_replaced_text': tbrt,
})
logger.debug('Services: %s', services)
# Sort services and remove services with no transports...
services = [s for s in sorted(services, key=lambda s: s['name'].upper()) if s['transports']]
autorun = False
if len(services) == 1 and GlobalConfig.AUTORUN_SERVICE.getBool(True) and services[0]['transports']:
if request.session.get('autorunDone', '0') == '0':
request.session['autorunDone'] = '1'
autorun = True
# return redirect('uds.web.views.service', idService=services[0]['id'], idTransport=services[0]['transports'][0]['id'])
return {
'services': services,
'ip': request.ip,
'nets': nets,
'transports': validTrans,
'autorun': autorun
}
def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: # pylint: disable=too-many-locals, too-many-branches, too-many-statements
"""Obtains the service data dictionary will all available services for this request
Arguments:
request {HttpRequest} -- request from where to xtract credentials
Returns:
typing.Dict[str, typing.Any] -- Keys has this:
'services': services,
'ip': request.ip,
'nets': nets,
'transports': validTrans,
'autorun': autorun
"""
# Session data
os: typing.Dict[str, str] = request.os
# We look for services for this authenticator groups. User is logged in in just 1 authenticator, so his groups must coincide with those assigned to ds
groups = list(request.user.getGroups())
availServicePools = list(
ServicePool.getDeployedServicesForGroups(
groups,
request.user)) # Pass in user to get "number_assigned" to optimize
availMetaPools = list(MetaPool.getForGroups(
groups,
request.user)) # Pass in user to get "number_assigned" to optimize
now = getSqlDatetime()
# Information for administrators
nets = ''
validTrans = ''
logger.debug('OS: %s', os['OS'])
if request.user.isStaff():
nets = ','.join([n.name for n in Network.networksFor(request.ip)])
tt = []
t: Transport
for t in Transport.objects.all().prefetch_related('networks'):
if t.validForIp(request.ip):
tt.append(t.name)
validTrans = ','.join(tt)
logger.debug('Checking meta pools: %s', availMetaPools)
services = []
meta: MetaPool
# Preload all assigned user services for this user
# Add meta pools data first
for meta in availMetaPools:
# Check that we have access to at least one transport on some of its children
hasUsablePools = False
in_use = meta.number_in_use > 0 # False
for pool in meta.pools.all():
# if pool.isInMaintenance():
# continue
for t in pool.transports.all():
typeTrans = t.getType()
if t.getType() and t.validForIp(
request.ip) and typeTrans.supportsOs(
os['OS']) and t.validForOs(os['OS']):
hasUsablePools = True
break
# if not in_use and meta.number_in_use: # Only look for assignation on possible used
# assignedUserService = userServiceManager().getExistingAssignationForUser(pool, request.user)
# if assignedUserService:
# in_use = assignedUserService.in_use
# Stop when 1 usable pool is found
if hasUsablePools:
break
# If no usable pools, this is not visible
if hasUsablePools:
group = meta.servicesPoolGroup.as_dict if meta.servicesPoolGroup else ServicePoolGroup.default(
).as_dict
services.append({
'id':
'M' + meta.uuid,
'name':
meta.name,
'visual_name':
meta.visual_name,
'description':
meta.comments,
'group':
group,
'transports': [{
'id':
'meta',
'name':
'meta',
'link':
html.udsMetaLink(request, 'M' + meta.uuid),
'priority':
0
}],
'imageId':
meta.image and meta.image.uuid or 'x',
'show_transports':
False,
'allow_users_remove':
False,
'allow_users_reset':
False,
'maintenance':
meta.isInMaintenance(),
'not_accesible':
not meta.isAccessAllowed(now),
'in_use':
in_use,
'to_be_replaced':
None,
'to_be_replaced_text':
'',
'custom_calendar_text':
meta.calendar_message,
})
# Now generic user service
svr: ServicePool
for svr in availServicePools:
# Skip pools that are part of meta pools
if svr.is_meta:
continue
use = str(svr.usage(svr.usage_count)) + '%'
trans = []
for t in sorted(
svr.transports.all(), key=lambda x: x.priority
): # In memory sort, allows reuse prefetched and not too big array
try:
typeTrans = t.getType()
except Exception:
continue
if t.validForIp(request.ip) and typeTrans.supportsOs(
os['OS']) and t.validForOs(os['OS']):
if typeTrans.ownLink:
link = reverse('TransportOwnLink',
args=('F' + svr.uuid, t.uuid))
else:
link = html.udsAccessLink(request, 'F' + svr.uuid, t.uuid)
trans.append({
'id': t.uuid,
'name': t.name,
'link': link,
'priority': t.priority
})
# If empty transports, do not include it on list
if not trans:
continue
if svr.image:
imageId = svr.image.uuid
else:
imageId = 'x'
# Locate if user service has any already assigned user service for this. Use "pre cached" number of assignations in this pool to optimize
in_use = svr.number_in_use > 0
# if svr.number_in_use: # Anotated value got from getDeployedServicesForGroups(...). If 0, no assignation for this user
# ads = userServiceManager().getExistingAssignationForUser(svr, request.user)
# if ads:
# in_use = ads.in_use
group = svr.servicesPoolGroup.as_dict if svr.servicesPoolGroup else ServicePoolGroup.default(
).as_dict
# Only add toBeReplaced info in case we allow it. This will generate some "overload" on the services
toBeReplaced = svr.toBeReplaced(
request.user
) if svr.pubs_active > 0 and GlobalConfig.NOTIFY_REMOVAL_BY_PUB.getBool(
False) else None
# tbr = False
if toBeReplaced:
toBeReplaced = formats.date_format(toBeReplaced,
"SHORT_DATETIME_FORMAT")
toBeReplacedTxt = ugettext(
'This service is about to be replaced by a new version. Please, close the session before {} and save all your work to avoid loosing it.'
).format(toBeReplaced)
else:
toBeReplacedTxt = ''
datator = lambda x: x.replace('{use}', use).replace(
'{total}', str(svr.max_srvs))
services.append({
'id':
'F' + svr.uuid,
'name':
datator(svr.name),
'visual_name':
datator(
svr.visual_name.replace('{use}',
use).replace('{total}',
str(svr.max_srvs))),
'description':
svr.comments,
'group':
group,
'transports':
trans,
'imageId':
imageId,
'show_transports':
svr.show_transports,
'allow_users_remove':
svr.allow_users_remove,
'allow_users_reset':
svr.allow_users_reset,
'maintenance':
svr.isInMaintenance(),
'not_accesible':
not svr.isAccessAllowed(now),
'in_use':
in_use,
'to_be_replaced':
toBeReplaced,
'to_be_replaced_text':
toBeReplacedTxt,
'custom_calendar_text':
svr.calendar_message,
})
# logger.debug('Services: %s', services)
# Sort services and remove services with no transports...
services = [
s for s in sorted(services, key=lambda s: s['name'].upper())
if s['transports']
]
autorun = False
if len(services) == 1 and GlobalConfig.AUTORUN_SERVICE.getBool(
False) and services[0]['transports']:
if request.session.get('autorunDone', '0') == '0':
request.session['autorunDone'] = '1'
autorun = True
# return redirect('uds.web.views.service', idService=services[0]['id'], idTransport=services[0]['transports'][0]['id'])
return {
'services': services,
'ip': request.ip,
'nets': nets,
'transports': validTrans,
'autorun': autorun
}
def afterSave(self, item: ServicePool) -> None:
if self._params.get('publish_on_save', False) is True:
try:
item.publish()
except Exception:
pass
def getAssignationForUser(
self, servicePool: ServicePool, user: User
) -> typing.Optional[UserService]: # pylint: disable=too-many-branches
if servicePool.service.getInstance().spawnsNew is False:
assignedUserService = self.getExistingAssignationForUser(
servicePool, user)
else:
assignedUserService = None
# If has an assigned user service, returns this without any more work
if assignedUserService:
return assignedUserService
if servicePool.isRestrained():
raise InvalidServiceException(
_('The requested service is restrained'))
cache: typing.Optional[UserService] = None
# Now try to locate 1 from cache already "ready" (must be usable and at level 1)
with transaction.atomic():
caches = typing.cast(
typing.List[UserService],
servicePool.cachedUserServices().select_for_update().filter(
cache_level=services.UserDeployment.L1_CACHE,
state=State.USABLE,
os_state=State.USABLE)[:1])
if caches:
cache = caches[0]
# Ensure element is reserved correctly on DB
if servicePool.cachedUserServices().select_for_update().filter(
user=None, uuid=typing.cast(
UserService, cache).uuid).update(
user=user, cache_level=0) != 1:
cache = None
else:
cache = None
# Out of previous atomic
if not cache:
with transaction.atomic():
caches = typing.cast(
typing.List[UserService],
servicePool.cachedUserServices().select_for_update().
filter(cache_level=services.UserDeployment.L1_CACHE,
state=State.USABLE)[:1])
if cache:
cache = caches[0]
if servicePool.cachedUserServices().select_for_update(
).filter(user=None,
uuid=typing.cast(UserService, cache).uuid).update(
user=user, cache_level=0) != 1:
cache = None
else:
cache = None
# Out of atomic transaction
if cache:
# Early assign
cache.assignToUser(user)
logger.debug(
'Found a cached-ready service from %s for user %s, item %s',
servicePool, user, cache)
events.addEvent(servicePool,
events.ET_CACHE_HIT,
fld1=servicePool.cachedUserServices().filter(
cache_level=services.UserDeployment.L1_CACHE,
state=State.USABLE).count())
return cache
# Cache missed
# Now find if there is a preparing one
with transaction.atomic():
caches = servicePool.cachedUserServices().select_for_update(
).filter(cache_level=services.UserDeployment.L1_CACHE,
state=State.PREPARING)[:1]
if caches:
cache = caches[0]
if servicePool.cachedUserServices().select_for_update().filter(
user=None, uuid=typing.cast(
UserService, cache).uuid).update(
user=user, cache_level=0) != 1:
cache = None
else:
cache = None
# Out of atomic transaction
if cache:
cache.assignToUser(user)
logger.debug(
'Found a cached-preparing service from %s for user %s, item %s',
servicePool, user, cache)
events.addEvent(servicePool,
events.ET_CACHE_MISS,
fld1=servicePool.cachedUserServices().filter(
cache_level=services.UserDeployment.L1_CACHE,
state=State.PREPARING).count())
return cache
# Can't assign directly from L2 cache... so we check if we can create e new service in the limits requested
serviceType = servicePool.service.getType()
if serviceType.usesCache:
# inCacheL1 = ds.cachedUserServices().filter(UserServiceManager.getCacheStateFilter(services.UserDeployment.L1_CACHE)).count()
inAssigned = servicePool.assignedUserServices().filter(
UserServiceManager.getStateFilter()).count()
# totalL1Assigned = inCacheL1 + inAssigned
if inAssigned >= servicePool.max_srvs: # cacheUpdater will drop unnecesary L1 machines, so it's not neccesary to check against inCacheL1
log.doLog(
servicePool, log.WARN,
'Max number of services reached: {}'.format(
servicePool.max_srvs), log.INTERNAL)
raise MaxServicesReachedError()
# Can create new service, create it
events.addEvent(servicePool, events.ET_CACHE_MISS, fld1=0)
return self.createAssignedFor(servicePool, user)
def getPoolsForGroups(groups):
for servicePool in ServicePool.getDeployedServicesForGroups(groups):
yield servicePool
def getServicesData(
request: 'ExtendedHttpRequestWithUser',
) -> typing.Dict[str, typing.Any]: # pylint: disable=too-many-locals, too-many-branches, too-many-statements
"""Obtains the service data dictionary will all available services for this request
Arguments:
request {ExtendedHttpRequest} -- request from where to xtract credentials
Returns:
typing.Dict[str, typing.Any] -- Keys has this:
'services': services,
'ip': request.ip,
'nets': nets,
'transports': validTrans,
'autorun': autorun
"""
# We look for services for this authenticator groups. User is logged in in just 1 authenticator, so his groups must coincide with those assigned to ds
groups = list(request.user.getGroups())
availServicePools = list(
ServicePool.getDeployedServicesForGroups(
groups,
request.user)) # Pass in user to get "number_assigned" to optimize
availMetaPools = list(MetaPool.getForGroups(
groups,
request.user)) # Pass in user to get "number_assigned" to optimize
now = getSqlDatetime()
# Information for administrators
nets = ''
validTrans = ''
osName = request.os['OS']
logger.debug('OS: %s', osName)
if request.user.isStaff():
nets = ','.join([n.name for n in Network.networksFor(request.ip)])
tt = []
t: Transport
for t in Transport.objects.all().prefetch_related('networks'):
if t.validForIp(request.ip):
tt.append(t.name)
validTrans = ','.join(tt)
logger.debug('Checking meta pools: %s', availMetaPools)
services = []
# Metapool helpers
def transportIterator(member) -> typing.Iterable[Transport]:
for t in member.pool.transports.all().order_by('priority'):
typeTrans = t.getType()
if (typeTrans and t.validForIp(request.ip)
and typeTrans.supportsOs(osName) and t.validForOs(osName)):
yield t
def buildMetaTransports(
transports: typing.Iterable[Transport],
isLabel: bool,
) -> typing.List[typing.Mapping[str, typing.Any]]:
idd = lambda i: i.uuid if not isLabel else 'LABEL:' + i.label
return [{
'id': idd(i),
'name': i.name,
'link': html.udsAccessLink(request, 'M' + meta.uuid, idd(i)),
'priority': 0,
} for i in transports]
# Preload all assigned user services for this user
# Add meta pools data first
for meta in availMetaPools:
# Check that we have access to at least one transport on some of its children
metaTransports: typing.List[typing.Mapping[str, typing.Any]] = []
in_use = meta.number_in_use > 0 # type: ignore # anotated value
inAll: typing.Optional[typing.Set[str]] = None
tmpSet: typing.Set[str]
if (meta.transport_grouping == MetaPool.COMMON_TRANSPORT_SELECT
): # If meta.use_common_transports
# only keep transports that are in ALL members
for member in meta.members.all().order_by('priority'):
tmpSet = set()
# if first pool, get all its transports and check that are valid
for t in transportIterator(member):
if inAll is None:
tmpSet.add(t.uuid)
elif t.uuid in inAll: # For subsequent, reduce...
tmpSet.add(t.uuid)
inAll = tmpSet
# tmpSet has ALL common transports
metaTransports = buildMetaTransports(
Transport.objects.filter(uuid__in=inAll or []), isLabel=False)
elif meta.transport_grouping == MetaPool.LABEL_TRANSPORT_SELECT:
ltrans: typing.MutableMapping[str, Transport] = {}
for member in meta.members.all().order_by('priority'):
tmpSet = set()
# if first pool, get all its transports and check that are valid
for t in transportIterator(member):
if not t.label:
continue
if t.label not in ltrans or ltrans[
t.label].priority > t.priority:
ltrans[t.label] = t
if inAll is None:
tmpSet.add(t.label)
elif t.label in inAll: # For subsequent, reduce...
tmpSet.add(t.label)
inAll = tmpSet
# tmpSet has ALL common transports
metaTransports = buildMetaTransports(
(v for k, v in ltrans.items() if k in (inAll or set())),
isLabel=True)
else:
for member in meta.members.all():
# if pool.isInMaintenance():
# continue
for t in member.pool.transports.all():
typeTrans = t.getType()
if (typeTrans and t.validForIp(request.ip)
and typeTrans.supportsOs(osName)
and t.validForOs(osName)):
metaTransports = [{
'id':
'meta',
'name':
'meta',
'link':
html.udsAccessLink(request, 'M' + meta.uuid, None),
'priority':
0,
}]
break
# if not in_use and meta.number_in_use: # Only look for assignation on possible used
# assignedUserService = userServiceManager().getExistingAssignationForUser(pool, request.user)
# if assignedUserService:
# in_use = assignedUserService.in_use
# Stop when 1 usable pool is found (metaTransports is filled)
if metaTransports:
break
# If no usable pools, this is not visible
if metaTransports:
group = (meta.servicesPoolGroup.as_dict if meta.servicesPoolGroup
else ServicePoolGroup.default().as_dict)
services.append({
'id': 'M' + meta.uuid,
'name': meta.name,
'visual_name': meta.visual_name,
'description': meta.comments,
'group': group,
'transports': metaTransports,
'imageId': meta.image and meta.image.uuid or 'x',
'show_transports': len(metaTransports) > 1,
'allow_users_remove': False,
'allow_users_reset': False,
'maintenance': meta.isInMaintenance(),
'not_accesible': not meta.isAccessAllowed(now),
'in_use': in_use,
'to_be_replaced': None,
'to_be_replaced_text': '',
'custom_calendar_text': meta.calendar_message,
})
# Now generic user service
for sPool in availServicePools:
# Skip pools that are part of meta pools
if sPool.owned_by_meta:
continue
use_percent = str(sPool.usage(
sPool.usage_count)) + '%' # type: ignore # anotated value
use_count = str(sPool.usage_count) # type: ignore # anotated value
left_count = str(sPool.max_srvs -
sPool.usage_count) # type: ignore # anotated value
trans: typing.List[typing.MutableMapping[str, typing.Any]] = []
for t in sorted(
sPool.transports.all(), key=lambda x: x.priority
): # In memory sort, allows reuse prefetched and not too big array
try:
typeTrans = t.getType()
except Exception:
continue
if (t.validForIp(request.ip) and typeTrans.supportsOs(osName)
and t.validForOs(osName)):
if typeTrans.ownLink:
link = reverse('TransportOwnLink',
args=('F' + sPool.uuid, t.uuid))
else:
link = html.udsAccessLink(request, 'F' + sPool.uuid,
t.uuid)
trans.append({
'id': t.uuid,
'name': t.name,
'link': link,
'priority': t.priority
})
# If empty transports, do not include it on list
if not trans:
continue
if sPool.image:
imageId = sPool.image.uuid
else:
imageId = 'x'
# Locate if user service has any already assigned user service for this. Use "pre cached" number of assignations in this pool to optimize
in_use = typing.cast(typing.Any, sPool).number_in_use > 0
# if svr.number_in_use: # Anotated value got from getDeployedServicesForGroups(...). If 0, no assignation for this user
# ads = userServiceManager().getExistingAssignationForUser(svr, request.user)
# if ads:
# in_use = ads.in_use
group = (sPool.servicesPoolGroup.as_dict if sPool.servicesPoolGroup
else ServicePoolGroup.default().as_dict)
# Only add toBeReplaced info in case we allow it. This will generate some "overload" on the services
toBeReplaced = (sPool.toBeReplaced(request.user)
if typing.cast(typing.Any, sPool).pubs_active > 0
and GlobalConfig.NOTIFY_REMOVAL_BY_PUB.getBool(False)
else None)
# tbr = False
if toBeReplaced:
toBeReplaced = formats.date_format(toBeReplaced,
'SHORT_DATETIME_FORMAT')
toBeReplacedTxt = ugettext(
'This service is about to be replaced by a new version. Please, close the session before {} and save all your work to avoid loosing it.'
).format(toBeReplaced)
else:
toBeReplacedTxt = ''
# Calculate max deployed
maxDeployed = str(sPool.max_srvs)
# if sPool.service.getType().usesCache is False:
# maxDeployed = sPool.service.getInstance().maxDeployed
def datator(x) -> str:
return (x.replace('{use}', use_percent).replace(
'{total}',
str(sPool.max_srvs)).replace('{usec}', use_count).replace(
'{left}', left_count))
services.append({
'id':
'F' + sPool.uuid,
'name':
datator(sPool.name),
'visual_name':
datator(
sPool.visual_name.replace('{use}', use_percent).replace(
'{total}', maxDeployed)),
'description':
sPool.comments,
'group':
group,
'transports':
trans,
'imageId':
imageId,
'show_transports':
sPool.show_transports,
'allow_users_remove':
sPool.allow_users_remove,
'allow_users_reset':
sPool.allow_users_reset,
'maintenance':
sPool.isInMaintenance(),
'not_accesible':
not sPool.isAccessAllowed(now),
'in_use':
in_use,
'to_be_replaced':
toBeReplaced,
'to_be_replaced_text':
toBeReplacedTxt,
'custom_calendar_text':
sPool.calendar_message,
})
# logger.debug('Services: %s', services)
# Sort services and remove services with no transports...
services = [
s for s in sorted(services, key=lambda s: s['name'].upper())
if s['transports']
]
autorun = False
if (len(services) == 1 and GlobalConfig.AUTORUN_SERVICE.getBool(False)
and services[0]['transports']):
if request.session.get('autorunDone', '0') == '0':
request.session['autorunDone'] = '1'
autorun = True
return {
'services': services,
'ip': request.ip,
'nets': nets,
'transports': validTrans,
'autorun': autorun
}
def item_as_dict(self, item: ServicePool) -> typing.Dict[str, typing.Any]:
summary = 'summarize' in self._params
# if item does not have an associated service, hide it (the case, for example, for a removed service)
# Access from dict will raise an exception, and item will be skipped
poolGroupId: typing.Optional[str] = None
poolGroupName: str = _('Default')
poolGroupThumb: str = DEFAULT_THUMB_BASE64
if item.servicesPoolGroup:
poolGroupId = item.servicesPoolGroup.uuid
poolGroupName = item.servicesPoolGroup.name
if item.servicesPoolGroup.image:
poolGroupThumb = item.servicesPoolGroup.image.thumb64
state = item.state
if item.isInMaintenance():
state = State.MAINTENANCE
elif userServiceManager().canInitiateServiceFromDeployedService(
item) is False:
state = State.SLOWED_DOWN
val = {
'id':
item.uuid,
'name':
item.name,
'short_name':
item.short_name,
'tags': [tag.tag for tag in item.tags.all()],
'parent':
item.service.name,
'parent_type':
item.service.data_type,
'comments':
item.comments,
'state':
state,
'thumb':
item.image.thumb64
if item.image is not None else DEFAULT_THUMB_BASE64,
'account':
item.account.name if item.account is not None else '',
'account_id':
item.account.uuid if item.account is not None else None,
'service_id':
item.service.uuid,
'provider_id':
item.service.provider.uuid,
'image_id':
item.image.uuid if item.image is not None else None,
'initial_srvs':
item.initial_srvs,
'cache_l1_srvs':
item.cache_l1_srvs,
'cache_l2_srvs':
item.cache_l2_srvs,
'max_srvs':
item.max_srvs,
'show_transports':
item.show_transports,
'visible':
item.visible,
'allow_users_remove':
item.allow_users_remove,
'allow_users_reset':
item.allow_users_reset,
'ignores_unused':
item.ignores_unused,
'fallbackAccess':
item.fallbackAccess,
'meta_member': [{
'id': i.uuid,
'name': i.name
} for i in item.meta.all()],
}
# Extended info
if not summary:
state = item.state
if item.isInMaintenance():
state = State.MAINTENANCE
elif userServiceManager().canInitiateServiceFromDeployedService(
item) is False:
state = State.SLOWED_DOWN
poolGroupId = None
poolGroupName = _('Default')
poolGroupThumb = DEFAULT_THUMB_BASE64
if item.servicesPoolGroup is not None:
poolGroupId = item.servicesPoolGroup.uuid
poolGroupName = item.servicesPoolGroup.name
if item.servicesPoolGroup.image is not None:
poolGroupThumb = item.servicesPoolGroup.image.thumb64
val['state'] = state
val['thumb'] = item.image.thumb64 if item.image is not None else DEFAULT_THUMB_BASE64
val['user_services_count'] = item.userServices.exclude(
state__in=State.INFO_STATES).count()
val['user_services_in_preparation'] = item.userServices.filter(
state=State.PREPARING).count()
val['tags'] = [tag.tag for tag in item.tags.all()]
val['restrained'] = item.isRestrained()
val['permission'] = permissions.getEffectivePermission(
self._user, item)
val['info'] = Services.serviceInfo(item.service)
val['pool_group_id'] = poolGroupId
val['pool_group_name'] = poolGroupName
val['pool_group_thumb'] = poolGroupThumb
val['usage'] = item.usage()
if item.osmanager:
val['osmanager_id'] = item.osmanager.uuid
return val