def publish(self, parent: models.ServicePool): """ Custom method "publish", provided to initiate a publication of a deployed service :param parent: Parent service pool """ changeLog = self._params[ 'changelog'] if 'changelog' in self._params else None if permissions.checkPermissions( self._user, parent, permissions.PERMISSION_MANAGEMENT) is False: logger.debug('Management Permission failed for user %s', self._user) raise self.accessDenied() logger.debug('Custom "publish" invoked for %s', parent) parent.publish( changeLog ) # Can raise exceptions that will be processed on response log.doLog( parent, log.INFO, "Initated publication v{} by {}".format( parent.current_pub_revision, self._user.pretty_name), log.ADMIN) return self.success()
def growL1Cache(self, servicePool: ServicePool, cacheL1: int, cacheL2: int, assigned: int) -> None: """ This method tries to enlarge L1 cache. If for some reason the number of deployed services (Counting all, ACTIVE and PREPARING, assigned, L1 and L2) is over max allowed service deployments, this method will not grow the L1 cache """ logger.debug('Growing L1 cache creating a new service for %s', servicePool.name) # First, we try to assign from L2 cache if cacheL2 > 0: valid = None with transaction.atomic(): for n in servicePool.cachedUserServices().select_for_update().filter(userServiceManager().getCacheStateFilter(services.UserDeployment.L2_CACHE)).order_by('creation_date'): if n.needsOsManager(): if State.isUsable(n.state) is False or State.isUsable(n.os_state): valid = n break else: valid = n break if valid is not None: valid.moveToLevel(services.UserDeployment.L1_CACHE) return try: # This has a velid publication, or it will not be here userServiceManager().createCacheFor( typing.cast(ServicePoolPublication, servicePool.activePublication()), services.UserDeployment.L1_CACHE ) except MaxServicesReachedError: log.doLog(servicePool, log.ERROR, 'Max number of services reached for this service', log.INTERNAL) logger.warning('Max user services reached for %s: %s. Cache not created', servicePool.name, servicePool.max_srvs) except Exception: logger.exception('Exception')
def startRemovalOf(self, servicePool: ServicePool): if ( servicePool.service is None ): # Maybe an inconsistent value? (must not, but if no ref integrity in db, maybe someone "touched.. ;)") logger.error('Found service pool %s without service', servicePool.name) servicePool.delete( ) # Just remove it "a las bravas", the best we can do return # Get publications in course...., that only can be one :) logger.debug('Removal process of %s', servicePool) publishing = servicePool.publications.filter(state=State.PREPARING) for pub in publishing: pub.cancel() # Now all publishments are canceling, let's try to cancel cache and assigned uServices: typing.Iterable[ UserService] = servicePool.userServices.filter( state=State.PREPARING) for userService in uServices: logger.debug('Canceling %s', userService) userService.cancel() # Nice start of removal, maybe we need to do some limitation later, but there should not be too much services nor publications cancelable at once servicePool.state = State.REMOVING servicePool.name += ' (removed)' servicePool.save()
def deleteItem(self, item: ServicePool) -> None: try: logger.debug('Deleting %s', item) item.remove( ) # This will mark it for deletion, but in fact will not delete it directly except Exception: # Eat it and logit logger.exception('deleting service pool')
def setFallbackAccess(self, item: ServicePool): self.ensureAccess(item, permissions.PERMISSION_MANAGEMENT) fallback = self._params.get('fallbackAccess') if fallback != '': logger.debug('Setting fallback of %s to %s', item.name, fallback) item.fallbackAccess = fallback item.save() return item.fallbackAccess
def continueRemovalOf(self, servicePool: ServicePool): # Recheck that there is no publication created in "bad moment" try: for pub in servicePool.publications.filter(state=State.PREPARING): pub.cancel() except Exception: pass try: # Now all publications are canceling, let's try to cancel cache and assigned also uServices: typing.Iterable[ UserService] = servicePool.userServices.filter( state=State.PREPARING) for userService in uServices: logger.debug('Canceling %s', userService) userService.cancel() except Exception: pass # First, we remove all publications and user services in "info_state" with transaction.atomic(): servicePool.userServices.select_for_update().filter( state__in=State.INFO_STATES).delete() # Mark usable user services as removable now = getSqlDatetime() with transaction.atomic(): servicePool.userServices.select_for_update().filter( state=State.USABLE).update(state=State.REMOVABLE, state_date=now) # When no service is at database, we start with publications if servicePool.userServices.all().count() == 0: try: logger.debug( 'All services removed, checking active publication') if servicePool.activePublication() is not None: logger.debug('Active publication found, unpublishing it') servicePool.unpublish() else: logger.debug( 'No active publication found, removing info states and checking if removal is done' ) servicePool.publications.filter( state__in=State.INFO_STATES).delete() if servicePool.publications.count() == 0: servicePool.removed( ) # Mark it as removed, clean later from database except Exception: logger.exception( 'Cought unexpected exception at continueRemovalOf: ')
def getItems(self, parent: models.ServicePool, item: typing.Optional[str]): # Extract provider try: if not item: return [ AssignedService.itemToDict(k, True) for k in parent.cachedUserServices().all().prefetch_related( 'properties', 'deployed_service', 'publication') ] cachedService: models.UserService = parent.cachedUserServices( ).get(uuid=processUuid(item)) return AssignedService.itemToDict(cachedService, True) except Exception: logger.exception('getItems') raise self.invalidItemException()
def get(self): logger.debug('args: {0}'.format(self._args)) if len(self._args) == 1: if self._args[0] == 'overview': # System overview users = User.objects.count() groups = Group.objects.count() services = Service.objects.count() service_pools = ServicePool.objects.count() meta_pools = MetaPool.objects.count() user_services = UserService.objects.exclude(state__in=(State.REMOVED, State.ERROR)).count() restrained_services_pools = len(ServicePool.getRestraineds()) return { 'users': users, 'groups': groups, 'services': services, 'service_pools': service_pools, 'meta_pools': meta_pools, 'user_services': user_services, 'restrained_services_pools': restrained_services_pools, } if len(self._args) == 2: if self._args[0] == 'stats': if self._args[1] == 'assigned': return getServicesPoolsCounters(None, counters.CT_ASSIGNED) if self._args[1] == 'inuse': return getServicesPoolsCounters(None, counters.CT_INUSE) raise RequestError('invalid request')
def publish(self, servicePool: ServicePool, changeLog: typing.Optional[str] = None): # pylint: disable=no-self-use """ Initiates the publication of a service pool, or raises an exception if this cannot be done :param servicePool: Service pool object (db object) :param changeLog: if not None, store change log string on "change log" table """ if servicePool.publications.filter(state__in=State.PUBLISH_STATES).count() > 0: raise PublishException(_('Already publishing. Wait for previous publication to finish and try again')) if servicePool.isInMaintenance(): raise PublishException(_('Service is in maintenance mode and new publications are not allowed')) publication: typing.Optional[ServicePoolPublication] = None try: now = getSqlDatetime() publication = servicePool.publications.create(state=State.LAUNCHING, state_date=now, publish_date=now, revision=servicePool.current_pub_revision) if changeLog: servicePool.changelog.create(revision=servicePool.current_pub_revision, log=changeLog, stamp=now) if publication: DelayedTaskRunner.runner().insert(PublicationLauncher(publication), 4, PUBTAG + str(publication.id)) except Exception as e: logger.debug('Caught exception at publish: %s', e) if publication is not None: try: publication.delete() except Exception: logger.info('Could not delete %s', publication) raise PublishException(str(e))
def createAssignedFor(self, servicePool: ServicePool, user: User) -> UserService: """ Creates a new assigned deployed service for the current publication (if any) of service pool and user indicated """ # First, honor maxPreparingServices if self.canInitiateServiceFromDeployedService(servicePool) is False: # Cannot create new logger.info( 'Too many preparing services. Creation of assigned service denied by max preparing services parameter. (login storm with insufficient cache?).' ) raise ServiceNotReadyError() if servicePool.service.getType().publicationType is not None: publication = servicePool.activePublication() logger.debug( 'Creating a new assigned element for user %s por publication %s', user, publication) if publication: assigned = self.__createAssignedAtDb(publication, user) else: raise Exception( 'Invalid publication creating service assignation: {} {}'. format(servicePool, user)) else: logger.debug('Creating a new assigned element for user %s', user) assigned = self.__createAssignedAtDbForNoPublication( servicePool, user) assignedInstance = assigned.getInstance() state = assignedInstance.deployForUser(user) UserServiceOpChecker.makeUnique(assigned, assignedInstance, state) return assigned
def createFromAssignable(self, servicePool: ServicePool, user: User, assignableId: str) -> UserService: """ Creates an assigned service from an "assignable" id """ serviceInstance = servicePool.service.getInstance() if not serviceInstance.canAssign(): raise Exception('This service type cannot assign asignables') if servicePool.service.getType().publicationType is not None: publication = servicePool.activePublication() logger.debug('Creating an assigned element from assignable %s for user %s por publication %s', user, assignableId, publication) if publication: assigned = self.__createAssignedAtDb(publication, user) else: raise Exception('Invalid publication creating service assignation: {} {}'.format(servicePool, user)) else: logger.debug('Creating an assigned element from assignable %s for user %s', assignableId, user) assigned = self.__createAssignedAtDbForNoPublication(servicePool, user) # Now, get from serviceInstance the data assignedInstance = assigned.getInstance() state = serviceInstance.assignFromAssignables(assignableId, user, assignedInstance) # assigned.updateData(assignedInstance) UserServiceOpChecker.makeUnique(assigned, assignedInstance, state) return assigned
def reduceL1Cache(self, servicePool: ServicePool, cacheL1: int, cacheL2: int, assigned: int): logger.debug("Reducing L1 cache erasing a service in cache for %s", servicePool) # We will try to destroy the newest cacheL1 element that is USABLE if the deployer can't cancel a new service creation cacheItems: typing.List[UserService] = list( servicePool.cachedUserServices().filter( userServiceManager().getCacheStateFilter(services.UserDeployment.L1_CACHE) ).exclude( Q(properties__name='destroy_after') & Q(properties__value='y') ).order_by( '-creation_date' ).iterator() ) if not cacheItems: logger.debug('There is more services than max configured, but could not reduce cache L1 cause its already empty') return if cacheL2 < servicePool.cache_l2_srvs: valid = None for n in cacheItems: if n.needsOsManager(): if State.isUsable(n.state) is False or State.isUsable(n.os_state): valid = n break else: valid = n break if valid is not None: valid.moveToLevel(services.UserDeployment.L2_CACHE) return cache = cacheItems[0] cache.removeOrCancel()
def getExistingAssignationForUser(self, servicePool: ServicePool, user: User) -> typing.Optional[UserService]: existing = servicePool.assignedUserServices().filter(user=user, state__in=State.VALID_STATES) # , deployed_service__visible=True lenExisting = existing.count() if lenExisting > 0: # Already has 1 assigned logger.debug('Found assigned service from %s to user %s', servicePool, user.name) return existing[0] return None
def getServicesPoolsCounters(servicePool, counter_type): # pylint: disable=no-value-for-parameter try: cacheKey = (servicePool and servicePool.id or 'all') + str(counter_type) + str(POINTS) + str(SINCE) to = getSqlDatetime() since = to - timedelta(days=SINCE) val = cache.get(cacheKey) if val is None: if servicePool is None: us = ServicePool() complete = True # Get all deployed services stats else: us = servicePool complete = False val = [] for x in counters.getCounters(us, counter_type, since=since, to=to, limit=POINTS, use_max=USE_MAX, all=complete): val.append({'stamp': x[0], 'value': int(x[1])}) if len(val) > 2: cache.put(cacheKey, encoders.encode(pickle.dumps(val), 'zip') , 600) else: val = [{'stamp': since, 'value': 0}, {'stamp': to, 'value': 0}] else: val = pickle.loads(encoders.decode(val, 'zip')) return val except: logger.exception('exception') raise ResponseError('can\'t create stats for objects!!!')
def getLogs(self, parent: models.ServicePool, item: str) -> typing.List[typing.Any]: try: item = parent.cachedUserServices().get(uuid=processUuid(item)) logger.debug('Getting logs for %s', item) return log.getLogs(item) except Exception: raise self.invalidItemException()
def reduceL2Cache(self, servicePool: ServicePool, cacheL1: int, cacheL2: int, assigned: int): logger.debug("Reducing L2 cache erasing a service in cache for %s", servicePool.name) if cacheL2 > 0: cacheItems: typing.List[UserService] = servicePool.cachedUserServices().filter( userServiceManager().getCacheStateFilter(services.UserDeployment.L2_CACHE) ).order_by('creation_date') # TODO: Look first for non finished cache items and cancel them? cache = cacheItems[0] cache.removeOrCancel()
def growL2Cache(self, servicePool: ServicePool, cacheL1: int, cacheL2: int, assigned: int) -> None: """ Tries to grow L2 cache of service. If for some reason the number of deployed services (Counting all, ACTIVE and PREPARING, assigned, L1 and L2) is over max allowed service deployments, this method will not grow the L1 cache """ logger.debug("Growing L2 cache creating a new service for %s", servicePool.name) try: # This has a velid publication, or it will not be here userServiceManager().createCacheFor( typing.cast(ServicePoolPublication, servicePool.activePublication()), services.UserDeployment.L2_CACHE ) except MaxServicesReachedError: logger.warning('Max user services reached for %s: %s. Cache not created', servicePool.name, servicePool.max_srvs)
def getPoolsForGroups(groups): for servicePool in ServicePool.getDeployedServicesForGroups(groups): yield servicePool
def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: # pylint: disable=too-many-locals, too-many-branches, too-many-statements """Obtains the service data dictionary will all available services for this request Arguments: request {HttpRequest} -- request from where to xtract credentials Returns: typing.Dict[str, typing.Any] -- Keys has this: 'services': services, 'ip': request.ip, 'nets': nets, 'transports': validTrans, 'autorun': autorun """ # Session data os: typing.Dict[str, str] = request.os # We look for services for this authenticator groups. User is logged in in just 1 authenticator, so his groups must coincide with those assigned to ds groups = list(request.user.getGroups()) availServicePools = ServicePool.getDeployedServicesForGroups(groups) availMetaPools = MetaPool.getForGroups(groups) # Information for administrators nets = '' validTrans = '' logger.debug('OS: %s', os['OS']) if request.user.isStaff(): nets = ','.join([n.name for n in Network.networksFor(request.ip)]) tt = [] for t in Transport.objects.all(): if t.validForIp(request.ip): tt.append(t.name) validTrans = ','.join(tt) logger.debug('Checking meta pools: %s', availMetaPools) services = [] # Add meta pools data first for meta in availMetaPools: # Check that we have access to at least one transport on some of its children hasUsablePools = False in_use = False for pool in meta.pools.all(): # if pool.isInMaintenance(): # continue for t in pool.transports.all(): typeTrans = t.getType() if t.getType() and t.validForIp(request.ip) and typeTrans.supportsOs(os['OS']) and t.validForOs(os['OS']): hasUsablePools = True break if not in_use: assignedUserService = userServiceManager().getExistingAssignationForUser(pool, request.user) if assignedUserService: in_use = assignedUserService.in_use # Stop when 1 usable pool is found if hasUsablePools: break # If no usable pools, this is not visible if hasUsablePools: group = meta.servicesPoolGroup.as_dict if meta.servicesPoolGroup else ServicePoolGroup.default().as_dict services.append({ 'id': 'M' + meta.uuid, 'name': meta.name, 'visual_name': meta.visual_name, 'description': meta.comments, 'group': group, 'transports': [{ 'id': 'meta', 'name': 'meta', 'link': html.udsMetaLink(request, 'M' + meta.uuid), 'priority': 0 }], 'imageId': meta.image and meta.image.uuid or 'x', 'show_transports': False, 'allow_users_remove': False, 'allow_users_reset': False, 'maintenance': meta.isInMaintenance(), 'not_accesible': not meta.isAccessAllowed(), 'in_use': in_use, 'to_be_replaced': None, 'to_be_replaced_text': '', }) # Now generic user service for svr in availServicePools: # Skip pools that are part of meta pools if svr.is_meta: continue trans = [] for t in svr.transports.all().order_by('priority'): typeTrans = t.getType() if typeTrans is None: # This may happen if we "remove" a transport type but we have a transport of that kind on DB continue if t.validForIp(request.ip) and typeTrans.supportsOs(os['OS']) and t.validForOs(os['OS']): if typeTrans.ownLink is True: link = reverse('TransportOwnLink', args=('F' + svr.uuid, t.uuid)) else: link = html.udsAccessLink(request, 'F' + svr.uuid, t.uuid) trans.append( { 'id': t.uuid, 'name': t.name, 'link': link, 'priority': t.priority } ) # If empty transports, do not include it on list if not trans: continue if svr.image is not None: imageId = svr.image.uuid else: imageId = 'x' # Locate if user service has any already assigned user service for this ads = userServiceManager().getExistingAssignationForUser(svr, request.user) if ads is None: in_use = False else: in_use = ads.in_use group = svr.servicesPoolGroup.as_dict if svr.servicesPoolGroup else ServicePoolGroup.default().as_dict tbr = svr.toBeReplaced(request.user) if tbr: tbr = formats.date_format(tbr, "SHORT_DATETIME_FORMAT") tbrt = ugettext('This service is about to be replaced by a new version. Please, close the session before {} and save all your work to avoid loosing it.').format(tbr) else: tbrt = '' services.append({ 'id': 'F' + svr.uuid, 'name': svr.name, 'visual_name': svr.visual_name, 'description': svr.comments, 'group': group, 'transports': trans, 'imageId': imageId, 'show_transports': svr.show_transports, 'allow_users_remove': svr.allow_users_remove, 'allow_users_reset': svr.allow_users_reset, 'maintenance': svr.isInMaintenance(), 'not_accesible': not svr.isAccessAllowed(), 'in_use': in_use, 'to_be_replaced': tbr, 'to_be_replaced_text': tbrt, }) logger.debug('Services: %s', services) # Sort services and remove services with no transports... services = [s for s in sorted(services, key=lambda s: s['name'].upper()) if s['transports']] autorun = False if len(services) == 1 and GlobalConfig.AUTORUN_SERVICE.getBool(True) and services[0]['transports']: if request.session.get('autorunDone', '0') == '0': request.session['autorunDone'] = '1' autorun = True # return redirect('uds.web.views.service', idService=services[0]['id'], idTransport=services[0]['transports'][0]['id']) return { 'services': services, 'ip': request.ip, 'nets': nets, 'transports': validTrans, 'autorun': autorun }
def getServicesData(request: 'HttpRequest') -> typing.Dict[str, typing.Any]: # pylint: disable=too-many-locals, too-many-branches, too-many-statements """Obtains the service data dictionary will all available services for this request Arguments: request {HttpRequest} -- request from where to xtract credentials Returns: typing.Dict[str, typing.Any] -- Keys has this: 'services': services, 'ip': request.ip, 'nets': nets, 'transports': validTrans, 'autorun': autorun """ # Session data os: typing.Dict[str, str] = request.os # We look for services for this authenticator groups. User is logged in in just 1 authenticator, so his groups must coincide with those assigned to ds groups = list(request.user.getGroups()) availServicePools = list( ServicePool.getDeployedServicesForGroups( groups, request.user)) # Pass in user to get "number_assigned" to optimize availMetaPools = list(MetaPool.getForGroups( groups, request.user)) # Pass in user to get "number_assigned" to optimize now = getSqlDatetime() # Information for administrators nets = '' validTrans = '' logger.debug('OS: %s', os['OS']) if request.user.isStaff(): nets = ','.join([n.name for n in Network.networksFor(request.ip)]) tt = [] t: Transport for t in Transport.objects.all().prefetch_related('networks'): if t.validForIp(request.ip): tt.append(t.name) validTrans = ','.join(tt) logger.debug('Checking meta pools: %s', availMetaPools) services = [] meta: MetaPool # Preload all assigned user services for this user # Add meta pools data first for meta in availMetaPools: # Check that we have access to at least one transport on some of its children hasUsablePools = False in_use = meta.number_in_use > 0 # False for pool in meta.pools.all(): # if pool.isInMaintenance(): # continue for t in pool.transports.all(): typeTrans = t.getType() if t.getType() and t.validForIp( request.ip) and typeTrans.supportsOs( os['OS']) and t.validForOs(os['OS']): hasUsablePools = True break # if not in_use and meta.number_in_use: # Only look for assignation on possible used # assignedUserService = userServiceManager().getExistingAssignationForUser(pool, request.user) # if assignedUserService: # in_use = assignedUserService.in_use # Stop when 1 usable pool is found if hasUsablePools: break # If no usable pools, this is not visible if hasUsablePools: group = meta.servicesPoolGroup.as_dict if meta.servicesPoolGroup else ServicePoolGroup.default( ).as_dict services.append({ 'id': 'M' + meta.uuid, 'name': meta.name, 'visual_name': meta.visual_name, 'description': meta.comments, 'group': group, 'transports': [{ 'id': 'meta', 'name': 'meta', 'link': html.udsMetaLink(request, 'M' + meta.uuid), 'priority': 0 }], 'imageId': meta.image and meta.image.uuid or 'x', 'show_transports': False, 'allow_users_remove': False, 'allow_users_reset': False, 'maintenance': meta.isInMaintenance(), 'not_accesible': not meta.isAccessAllowed(now), 'in_use': in_use, 'to_be_replaced': None, 'to_be_replaced_text': '', 'custom_calendar_text': meta.calendar_message, }) # Now generic user service svr: ServicePool for svr in availServicePools: # Skip pools that are part of meta pools if svr.is_meta: continue use = str(svr.usage(svr.usage_count)) + '%' trans = [] for t in sorted( svr.transports.all(), key=lambda x: x.priority ): # In memory sort, allows reuse prefetched and not too big array try: typeTrans = t.getType() except Exception: continue if t.validForIp(request.ip) and typeTrans.supportsOs( os['OS']) and t.validForOs(os['OS']): if typeTrans.ownLink: link = reverse('TransportOwnLink', args=('F' + svr.uuid, t.uuid)) else: link = html.udsAccessLink(request, 'F' + svr.uuid, t.uuid) trans.append({ 'id': t.uuid, 'name': t.name, 'link': link, 'priority': t.priority }) # If empty transports, do not include it on list if not trans: continue if svr.image: imageId = svr.image.uuid else: imageId = 'x' # Locate if user service has any already assigned user service for this. Use "pre cached" number of assignations in this pool to optimize in_use = svr.number_in_use > 0 # if svr.number_in_use: # Anotated value got from getDeployedServicesForGroups(...). If 0, no assignation for this user # ads = userServiceManager().getExistingAssignationForUser(svr, request.user) # if ads: # in_use = ads.in_use group = svr.servicesPoolGroup.as_dict if svr.servicesPoolGroup else ServicePoolGroup.default( ).as_dict # Only add toBeReplaced info in case we allow it. This will generate some "overload" on the services toBeReplaced = svr.toBeReplaced( request.user ) if svr.pubs_active > 0 and GlobalConfig.NOTIFY_REMOVAL_BY_PUB.getBool( False) else None # tbr = False if toBeReplaced: toBeReplaced = formats.date_format(toBeReplaced, "SHORT_DATETIME_FORMAT") toBeReplacedTxt = ugettext( 'This service is about to be replaced by a new version. Please, close the session before {} and save all your work to avoid loosing it.' ).format(toBeReplaced) else: toBeReplacedTxt = '' datator = lambda x: x.replace('{use}', use).replace( '{total}', str(svr.max_srvs)) services.append({ 'id': 'F' + svr.uuid, 'name': datator(svr.name), 'visual_name': datator( svr.visual_name.replace('{use}', use).replace('{total}', str(svr.max_srvs))), 'description': svr.comments, 'group': group, 'transports': trans, 'imageId': imageId, 'show_transports': svr.show_transports, 'allow_users_remove': svr.allow_users_remove, 'allow_users_reset': svr.allow_users_reset, 'maintenance': svr.isInMaintenance(), 'not_accesible': not svr.isAccessAllowed(now), 'in_use': in_use, 'to_be_replaced': toBeReplaced, 'to_be_replaced_text': toBeReplacedTxt, 'custom_calendar_text': svr.calendar_message, }) # logger.debug('Services: %s', services) # Sort services and remove services with no transports... services = [ s for s in sorted(services, key=lambda s: s['name'].upper()) if s['transports'] ] autorun = False if len(services) == 1 and GlobalConfig.AUTORUN_SERVICE.getBool( False) and services[0]['transports']: if request.session.get('autorunDone', '0') == '0': request.session['autorunDone'] = '1' autorun = True # return redirect('uds.web.views.service', idService=services[0]['id'], idTransport=services[0]['transports'][0]['id']) return { 'services': services, 'ip': request.ip, 'nets': nets, 'transports': validTrans, 'autorun': autorun }
def afterSave(self, item: ServicePool) -> None: if self._params.get('publish_on_save', False) is True: try: item.publish() except Exception: pass
def getAssignationForUser( self, servicePool: ServicePool, user: User ) -> typing.Optional[UserService]: # pylint: disable=too-many-branches if servicePool.service.getInstance().spawnsNew is False: assignedUserService = self.getExistingAssignationForUser( servicePool, user) else: assignedUserService = None # If has an assigned user service, returns this without any more work if assignedUserService: return assignedUserService if servicePool.isRestrained(): raise InvalidServiceException( _('The requested service is restrained')) cache: typing.Optional[UserService] = None # Now try to locate 1 from cache already "ready" (must be usable and at level 1) with transaction.atomic(): caches = typing.cast( typing.List[UserService], servicePool.cachedUserServices().select_for_update().filter( cache_level=services.UserDeployment.L1_CACHE, state=State.USABLE, os_state=State.USABLE)[:1]) if caches: cache = caches[0] # Ensure element is reserved correctly on DB if servicePool.cachedUserServices().select_for_update().filter( user=None, uuid=typing.cast( UserService, cache).uuid).update( user=user, cache_level=0) != 1: cache = None else: cache = None # Out of previous atomic if not cache: with transaction.atomic(): caches = typing.cast( typing.List[UserService], servicePool.cachedUserServices().select_for_update(). filter(cache_level=services.UserDeployment.L1_CACHE, state=State.USABLE)[:1]) if cache: cache = caches[0] if servicePool.cachedUserServices().select_for_update( ).filter(user=None, uuid=typing.cast(UserService, cache).uuid).update( user=user, cache_level=0) != 1: cache = None else: cache = None # Out of atomic transaction if cache: # Early assign cache.assignToUser(user) logger.debug( 'Found a cached-ready service from %s for user %s, item %s', servicePool, user, cache) events.addEvent(servicePool, events.ET_CACHE_HIT, fld1=servicePool.cachedUserServices().filter( cache_level=services.UserDeployment.L1_CACHE, state=State.USABLE).count()) return cache # Cache missed # Now find if there is a preparing one with transaction.atomic(): caches = servicePool.cachedUserServices().select_for_update( ).filter(cache_level=services.UserDeployment.L1_CACHE, state=State.PREPARING)[:1] if caches: cache = caches[0] if servicePool.cachedUserServices().select_for_update().filter( user=None, uuid=typing.cast( UserService, cache).uuid).update( user=user, cache_level=0) != 1: cache = None else: cache = None # Out of atomic transaction if cache: cache.assignToUser(user) logger.debug( 'Found a cached-preparing service from %s for user %s, item %s', servicePool, user, cache) events.addEvent(servicePool, events.ET_CACHE_MISS, fld1=servicePool.cachedUserServices().filter( cache_level=services.UserDeployment.L1_CACHE, state=State.PREPARING).count()) return cache # Can't assign directly from L2 cache... so we check if we can create e new service in the limits requested serviceType = servicePool.service.getType() if serviceType.usesCache: # inCacheL1 = ds.cachedUserServices().filter(UserServiceManager.getCacheStateFilter(services.UserDeployment.L1_CACHE)).count() inAssigned = servicePool.assignedUserServices().filter( UserServiceManager.getStateFilter()).count() # totalL1Assigned = inCacheL1 + inAssigned if inAssigned >= servicePool.max_srvs: # cacheUpdater will drop unnecesary L1 machines, so it's not neccesary to check against inCacheL1 log.doLog( servicePool, log.WARN, 'Max number of services reached: {}'.format( servicePool.max_srvs), log.INTERNAL) raise MaxServicesReachedError() # Can create new service, create it events.addEvent(servicePool, events.ET_CACHE_MISS, fld1=0) return self.createAssignedFor(servicePool, user)
def getPoolsForGroups(groups): for servicePool in ServicePool.getDeployedServicesForGroups(groups): yield servicePool
def getServicesData( request: 'ExtendedHttpRequestWithUser', ) -> typing.Dict[str, typing.Any]: # pylint: disable=too-many-locals, too-many-branches, too-many-statements """Obtains the service data dictionary will all available services for this request Arguments: request {ExtendedHttpRequest} -- request from where to xtract credentials Returns: typing.Dict[str, typing.Any] -- Keys has this: 'services': services, 'ip': request.ip, 'nets': nets, 'transports': validTrans, 'autorun': autorun """ # We look for services for this authenticator groups. User is logged in in just 1 authenticator, so his groups must coincide with those assigned to ds groups = list(request.user.getGroups()) availServicePools = list( ServicePool.getDeployedServicesForGroups( groups, request.user)) # Pass in user to get "number_assigned" to optimize availMetaPools = list(MetaPool.getForGroups( groups, request.user)) # Pass in user to get "number_assigned" to optimize now = getSqlDatetime() # Information for administrators nets = '' validTrans = '' osName = request.os['OS'] logger.debug('OS: %s', osName) if request.user.isStaff(): nets = ','.join([n.name for n in Network.networksFor(request.ip)]) tt = [] t: Transport for t in Transport.objects.all().prefetch_related('networks'): if t.validForIp(request.ip): tt.append(t.name) validTrans = ','.join(tt) logger.debug('Checking meta pools: %s', availMetaPools) services = [] # Metapool helpers def transportIterator(member) -> typing.Iterable[Transport]: for t in member.pool.transports.all().order_by('priority'): typeTrans = t.getType() if (typeTrans and t.validForIp(request.ip) and typeTrans.supportsOs(osName) and t.validForOs(osName)): yield t def buildMetaTransports( transports: typing.Iterable[Transport], isLabel: bool, ) -> typing.List[typing.Mapping[str, typing.Any]]: idd = lambda i: i.uuid if not isLabel else 'LABEL:' + i.label return [{ 'id': idd(i), 'name': i.name, 'link': html.udsAccessLink(request, 'M' + meta.uuid, idd(i)), 'priority': 0, } for i in transports] # Preload all assigned user services for this user # Add meta pools data first for meta in availMetaPools: # Check that we have access to at least one transport on some of its children metaTransports: typing.List[typing.Mapping[str, typing.Any]] = [] in_use = meta.number_in_use > 0 # type: ignore # anotated value inAll: typing.Optional[typing.Set[str]] = None tmpSet: typing.Set[str] if (meta.transport_grouping == MetaPool.COMMON_TRANSPORT_SELECT ): # If meta.use_common_transports # only keep transports that are in ALL members for member in meta.members.all().order_by('priority'): tmpSet = set() # if first pool, get all its transports and check that are valid for t in transportIterator(member): if inAll is None: tmpSet.add(t.uuid) elif t.uuid in inAll: # For subsequent, reduce... tmpSet.add(t.uuid) inAll = tmpSet # tmpSet has ALL common transports metaTransports = buildMetaTransports( Transport.objects.filter(uuid__in=inAll or []), isLabel=False) elif meta.transport_grouping == MetaPool.LABEL_TRANSPORT_SELECT: ltrans: typing.MutableMapping[str, Transport] = {} for member in meta.members.all().order_by('priority'): tmpSet = set() # if first pool, get all its transports and check that are valid for t in transportIterator(member): if not t.label: continue if t.label not in ltrans or ltrans[ t.label].priority > t.priority: ltrans[t.label] = t if inAll is None: tmpSet.add(t.label) elif t.label in inAll: # For subsequent, reduce... tmpSet.add(t.label) inAll = tmpSet # tmpSet has ALL common transports metaTransports = buildMetaTransports( (v for k, v in ltrans.items() if k in (inAll or set())), isLabel=True) else: for member in meta.members.all(): # if pool.isInMaintenance(): # continue for t in member.pool.transports.all(): typeTrans = t.getType() if (typeTrans and t.validForIp(request.ip) and typeTrans.supportsOs(osName) and t.validForOs(osName)): metaTransports = [{ 'id': 'meta', 'name': 'meta', 'link': html.udsAccessLink(request, 'M' + meta.uuid, None), 'priority': 0, }] break # if not in_use and meta.number_in_use: # Only look for assignation on possible used # assignedUserService = userServiceManager().getExistingAssignationForUser(pool, request.user) # if assignedUserService: # in_use = assignedUserService.in_use # Stop when 1 usable pool is found (metaTransports is filled) if metaTransports: break # If no usable pools, this is not visible if metaTransports: group = (meta.servicesPoolGroup.as_dict if meta.servicesPoolGroup else ServicePoolGroup.default().as_dict) services.append({ 'id': 'M' + meta.uuid, 'name': meta.name, 'visual_name': meta.visual_name, 'description': meta.comments, 'group': group, 'transports': metaTransports, 'imageId': meta.image and meta.image.uuid or 'x', 'show_transports': len(metaTransports) > 1, 'allow_users_remove': False, 'allow_users_reset': False, 'maintenance': meta.isInMaintenance(), 'not_accesible': not meta.isAccessAllowed(now), 'in_use': in_use, 'to_be_replaced': None, 'to_be_replaced_text': '', 'custom_calendar_text': meta.calendar_message, }) # Now generic user service for sPool in availServicePools: # Skip pools that are part of meta pools if sPool.owned_by_meta: continue use_percent = str(sPool.usage( sPool.usage_count)) + '%' # type: ignore # anotated value use_count = str(sPool.usage_count) # type: ignore # anotated value left_count = str(sPool.max_srvs - sPool.usage_count) # type: ignore # anotated value trans: typing.List[typing.MutableMapping[str, typing.Any]] = [] for t in sorted( sPool.transports.all(), key=lambda x: x.priority ): # In memory sort, allows reuse prefetched and not too big array try: typeTrans = t.getType() except Exception: continue if (t.validForIp(request.ip) and typeTrans.supportsOs(osName) and t.validForOs(osName)): if typeTrans.ownLink: link = reverse('TransportOwnLink', args=('F' + sPool.uuid, t.uuid)) else: link = html.udsAccessLink(request, 'F' + sPool.uuid, t.uuid) trans.append({ 'id': t.uuid, 'name': t.name, 'link': link, 'priority': t.priority }) # If empty transports, do not include it on list if not trans: continue if sPool.image: imageId = sPool.image.uuid else: imageId = 'x' # Locate if user service has any already assigned user service for this. Use "pre cached" number of assignations in this pool to optimize in_use = typing.cast(typing.Any, sPool).number_in_use > 0 # if svr.number_in_use: # Anotated value got from getDeployedServicesForGroups(...). If 0, no assignation for this user # ads = userServiceManager().getExistingAssignationForUser(svr, request.user) # if ads: # in_use = ads.in_use group = (sPool.servicesPoolGroup.as_dict if sPool.servicesPoolGroup else ServicePoolGroup.default().as_dict) # Only add toBeReplaced info in case we allow it. This will generate some "overload" on the services toBeReplaced = (sPool.toBeReplaced(request.user) if typing.cast(typing.Any, sPool).pubs_active > 0 and GlobalConfig.NOTIFY_REMOVAL_BY_PUB.getBool(False) else None) # tbr = False if toBeReplaced: toBeReplaced = formats.date_format(toBeReplaced, 'SHORT_DATETIME_FORMAT') toBeReplacedTxt = ugettext( 'This service is about to be replaced by a new version. Please, close the session before {} and save all your work to avoid loosing it.' ).format(toBeReplaced) else: toBeReplacedTxt = '' # Calculate max deployed maxDeployed = str(sPool.max_srvs) # if sPool.service.getType().usesCache is False: # maxDeployed = sPool.service.getInstance().maxDeployed def datator(x) -> str: return (x.replace('{use}', use_percent).replace( '{total}', str(sPool.max_srvs)).replace('{usec}', use_count).replace( '{left}', left_count)) services.append({ 'id': 'F' + sPool.uuid, 'name': datator(sPool.name), 'visual_name': datator( sPool.visual_name.replace('{use}', use_percent).replace( '{total}', maxDeployed)), 'description': sPool.comments, 'group': group, 'transports': trans, 'imageId': imageId, 'show_transports': sPool.show_transports, 'allow_users_remove': sPool.allow_users_remove, 'allow_users_reset': sPool.allow_users_reset, 'maintenance': sPool.isInMaintenance(), 'not_accesible': not sPool.isAccessAllowed(now), 'in_use': in_use, 'to_be_replaced': toBeReplaced, 'to_be_replaced_text': toBeReplacedTxt, 'custom_calendar_text': sPool.calendar_message, }) # logger.debug('Services: %s', services) # Sort services and remove services with no transports... services = [ s for s in sorted(services, key=lambda s: s['name'].upper()) if s['transports'] ] autorun = False if (len(services) == 1 and GlobalConfig.AUTORUN_SERVICE.getBool(False) and services[0]['transports']): if request.session.get('autorunDone', '0') == '0': request.session['autorunDone'] = '1' autorun = True return { 'services': services, 'ip': request.ip, 'nets': nets, 'transports': validTrans, 'autorun': autorun }
def item_as_dict(self, item: ServicePool) -> typing.Dict[str, typing.Any]: summary = 'summarize' in self._params # if item does not have an associated service, hide it (the case, for example, for a removed service) # Access from dict will raise an exception, and item will be skipped poolGroupId: typing.Optional[str] = None poolGroupName: str = _('Default') poolGroupThumb: str = DEFAULT_THUMB_BASE64 if item.servicesPoolGroup: poolGroupId = item.servicesPoolGroup.uuid poolGroupName = item.servicesPoolGroup.name if item.servicesPoolGroup.image: poolGroupThumb = item.servicesPoolGroup.image.thumb64 state = item.state if item.isInMaintenance(): state = State.MAINTENANCE elif userServiceManager().canInitiateServiceFromDeployedService( item) is False: state = State.SLOWED_DOWN val = { 'id': item.uuid, 'name': item.name, 'short_name': item.short_name, 'tags': [tag.tag for tag in item.tags.all()], 'parent': item.service.name, 'parent_type': item.service.data_type, 'comments': item.comments, 'state': state, 'thumb': item.image.thumb64 if item.image is not None else DEFAULT_THUMB_BASE64, 'account': item.account.name if item.account is not None else '', 'account_id': item.account.uuid if item.account is not None else None, 'service_id': item.service.uuid, 'provider_id': item.service.provider.uuid, 'image_id': item.image.uuid if item.image is not None else None, 'initial_srvs': item.initial_srvs, 'cache_l1_srvs': item.cache_l1_srvs, 'cache_l2_srvs': item.cache_l2_srvs, 'max_srvs': item.max_srvs, 'show_transports': item.show_transports, 'visible': item.visible, 'allow_users_remove': item.allow_users_remove, 'allow_users_reset': item.allow_users_reset, 'ignores_unused': item.ignores_unused, 'fallbackAccess': item.fallbackAccess, 'meta_member': [{ 'id': i.uuid, 'name': i.name } for i in item.meta.all()], } # Extended info if not summary: state = item.state if item.isInMaintenance(): state = State.MAINTENANCE elif userServiceManager().canInitiateServiceFromDeployedService( item) is False: state = State.SLOWED_DOWN poolGroupId = None poolGroupName = _('Default') poolGroupThumb = DEFAULT_THUMB_BASE64 if item.servicesPoolGroup is not None: poolGroupId = item.servicesPoolGroup.uuid poolGroupName = item.servicesPoolGroup.name if item.servicesPoolGroup.image is not None: poolGroupThumb = item.servicesPoolGroup.image.thumb64 val['state'] = state val['thumb'] = item.image.thumb64 if item.image is not None else DEFAULT_THUMB_BASE64 val['user_services_count'] = item.userServices.exclude( state__in=State.INFO_STATES).count() val['user_services_in_preparation'] = item.userServices.filter( state=State.PREPARING).count() val['tags'] = [tag.tag for tag in item.tags.all()] val['restrained'] = item.isRestrained() val['permission'] = permissions.getEffectivePermission( self._user, item) val['info'] = Services.serviceInfo(item.service) val['pool_group_id'] = poolGroupId val['pool_group_name'] = poolGroupName val['pool_group_thumb'] = poolGroupThumb val['usage'] = item.usage() if item.osmanager: val['osmanager_id'] = item.osmanager.uuid return val