class PlotCache: def __init__( self, plotsLocation = False ): self.plotsLocation = plotsLocation self.alive = True self.__graphCache = DictCache( deleteFunction = _deleteGraph ) self.__graphLifeTime = 600 self.purgeThread = threading.Thread( target = self.purgeExpired ) self.purgeThread.start() def setPlotsLocation( self, plotsDir ): self.plotsLocation = plotsDir for plot in os.listdir( self.plotsLocation ): if plot.find( ".png" ) > 0: plotLocation = "%s/%s" % ( self.plotsLocation, plot ) gLogger.verbose( "Purging %s" % plotLocation ) os.unlink( plotLocation ) def purgeExpired( self ): while self.alive: time.sleep( self.__graphLifeTime ) self.__graphCache.purgeExpired() def getPlot( self, plotHash, plotData, plotMetadata, subplotMetadata ): """ Get plot from the cache if exists, else generate it """ plotDict = self.__graphCache.get( plotHash ) if plotDict == False: basePlotFileName = "%s/%s.png" % ( self.plotsLocation, plotHash ) if subplotMetadata: retVal = graph( plotData, basePlotFileName, plotMetadata, metadata = subplotMetadata ) else: retVal = graph( plotData, basePlotFileName, plotMetadata ) if not retVal[ 'OK' ]: return retVal plotDict = retVal[ 'Value' ] if plotDict[ 'plot' ]: plotDict[ 'plot' ] = os.path.basename( basePlotFileName ) self.__graphCache.add( plotHash, self.__graphLifeTime, plotDict ) return S_OK( plotDict ) def getPlotData( self, plotFileName ): filename = "%s/%s" % ( self.plotsLocation, plotFileName ) try: fd = file( filename, "rb" ) data = fd.read() fd.close() except Exception, v: return S_ERROR( "Can't open file %s: %s" % ( plotFileName, str( v ) ) ) return S_OK( data )
class DataCache: def __init__( self ): self.graphsLocation = os.path.join( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), 'data', 'accountingPlots' ) self.cachedGraphs = {} self.alive = True self.purgeThread = threading.Thread( target = self.purgeExpired ) self.purgeThread.start() self.__dataCache = DictCache() self.__graphCache = DictCache( deleteFunction = _deleteGraph ) self.__dataLifeTime = 600 self.__graphLifeTime = 3600 def setGraphsLocation( self, graphsDir ): self.graphsLocation = graphsDir for graphName in os.listdir( self.graphsLocation ): if graphName.find( ".png" ) > 0: graphLocation = "%s/%s" % ( self.graphsLocation, graphName ) gLogger.verbose( "Purging %s" % graphLocation ) os.unlink( graphLocation ) def purgeExpired( self ): while self.alive: time.sleep( 600 ) self.__graphCache.purgeExpired() self.__dataCache.purgeExpired() def getReportData( self, reportRequest, reportHash, dataFunc ): """ Get report data from cache if exists, else generate it """ reportData = self.__dataCache.get( reportHash ) if reportData == False: retVal = dataFunc( reportRequest ) if not retVal[ 'OK' ]: return retVal reportData = retVal[ 'Value' ] self.__dataCache.add( reportHash, self.__dataLifeTime, reportData ) return S_OK( reportData ) def getReportPlot( self, reportRequest, reportHash, reportData, plotFunc ): """ Get report data from cache if exists, else generate it """ plotDict = self.__graphCache.get( reportHash ) if plotDict == False: basePlotFileName = "%s/%s" % ( self.graphsLocation, reportHash ) retVal = plotFunc( reportRequest, reportData, basePlotFileName ) if not retVal[ 'OK' ]: return retVal plotDict = retVal[ 'Value' ] if plotDict[ 'plot' ]: plotDict[ 'plot' ] = "%s.png" % reportHash if plotDict[ 'thumbnail' ]: plotDict[ 'thumbnail' ] = "%s.thb.png" % reportHash self.__graphCache.add( reportHash, self.__graphLifeTime, plotDict ) return S_OK( plotDict ) def getPlotData( self, plotFileName ): filename = "%s/%s" % ( self.graphsLocation, plotFileName ) try: fd = file( filename, "rb" ) data = fd.read() fd.close() except Exception, e: return S_ERROR( "Can't open file %s: %s" % ( plotFileName, str( e ) ) ) return S_OK( data )
class ProxyManagerClient: def __init__( self ): self.__usersCache = DictCache() self.__proxiesCache = DictCache() self.__vomsProxiesCache = DictCache() self.__pilotProxiesCache = DictCache() self.__filesCache = DictCache( self.__deleteTemporalFile ) def __deleteTemporalFile( self, filename ): try: os.unlink( filename ) except: pass def clearCaches( self ): self.__usersCache.purgeAll() self.__proxiesCache.purgeAll() self.__vomsProxiesCache.purgeAll() self.__pilotProxiesCache.purgeAll() def __getSecondsLeftToExpiration( self, expiration, utc = True ): if utc: td = expiration - datetime.datetime.utcnow() else: td = expiration - datetime.datetime.now() return td.days * 86400 + td.seconds def __refreshUserCache( self, validSeconds = 0 ): rpcClient = RPCClient( "Framework/ProxyManager", timeout = 120 ) retVal = rpcClient.getRegisteredUsers( validSeconds ) if not retVal[ 'OK' ]: return retVal data = retVal[ 'Value' ] #Update the cache for record in data: cacheKey = ( record[ 'DN' ], record[ 'group' ] ) self.__usersCache.add( cacheKey, self.__getSecondsLeftToExpiration( record[ 'expirationtime' ] ), record ) return S_OK() @gUsersSync def userHasProxy( self, userDN, userGroup, validSeconds = 0 ): """ Check if a user(DN-group) has a proxy in the proxy management - Updates internal cache if needed to minimize queries to the service """ cacheKey = ( userDN, userGroup ) if self.__usersCache.exists( cacheKey, validSeconds ): return S_OK( True ) #Get list of users from the DB with proxys at least 300 seconds gLogger.verbose( "Updating list of users in proxy management" ) retVal = self.__refreshUserCache( validSeconds ) if not retVal[ 'OK' ]: return retVal return S_OK( self.__usersCache.exists( cacheKey, validSeconds ) ) @gUsersSync def getUserPersistence( self, userDN, userGroup, validSeconds = 0 ): """ Check if a user(DN-group) has a proxy in the proxy management - Updates internal cache if needed to minimize queries to the service """ cacheKey = ( userDN, userGroup ) userData = self.__usersCache.get( cacheKey, validSeconds ) if userData: if userData[ 'persistent' ]: return S_OK( True ) #Get list of users from the DB with proxys at least 300 seconds gLogger.verbose( "Updating list of users in proxy management" ) retVal = self.__refreshUserCache( validSeconds ) if not retVal[ 'OK' ]: return retVal userData = self.__usersCache.get( cacheKey, validSeconds ) if userData: return S_OK( userData[ 'persistent' ] ) return S_OK( False ) def setPersistency( self, userDN, userGroup, persistent ): """ Set the persistency for user/group """ #Hack to ensure bool in the rpc call persistentFlag = True if not persistent: persistentFlag = False rpcClient = RPCClient( "Framework/ProxyManager", timeout = 120 ) retVal = rpcClient.setPersistency( userDN, userGroup, persistentFlag ) if not retVal[ 'OK' ]: return retVal #Update internal persistency cache cacheKey = ( userDN, userGroup ) record = self.__usersCache.get( cacheKey, 0 ) if record: record[ 'persistent' ] = persistentFlag self.__usersCache.add( cacheKey, self.__getSecondsLeftToExpiration( record[ 'expirationtime' ] ), record ) return retVal def uploadProxy( self, proxy = False, diracGroup = False, chainToConnect = False, restrictLifeTime = 0 ): """ Upload a proxy to the proxy management service using delgation """ #Discover proxy location if type( proxy ) == g_X509ChainType: chain = proxy proxyLocation = "" else: if not proxy: proxyLocation = Locations.getProxyLocation() if not proxyLocation: return S_ERROR( "Can't find a valid proxy" ) elif type( proxy ) in ( types.StringType, types.UnicodeType ): proxyLocation = proxy else: return S_ERROR( "Can't find a valid proxy" ) chain = X509Chain() retVal = chain.loadProxyFromFile( proxyLocation ) if not retVal[ 'OK' ]: return S_ERROR( "Can't load %s: %s " % ( proxyLocation, retVal[ 'Message' ] ) ) if not chainToConnect: chainToConnect = chain #Make sure it's valid if chain.hasExpired()[ 'Value' ]: return S_ERROR( "Proxy %s has expired" % proxyLocation ) #rpcClient = RPCClient( "Framework/ProxyManager", proxyChain = chainToConnect ) rpcClient = RPCClient( "Framework/ProxyManager", timeout = 120 ) #Get a delegation request retVal = rpcClient.requestDelegationUpload( chain.getRemainingSecs()['Value'], diracGroup ) if not retVal[ 'OK' ]: return retVal #Check if the delegation has been granted if 'Value' not in retVal or not retVal[ 'Value' ]: return S_OK() reqDict = retVal[ 'Value' ] #Generate delegated chain chainLifeTime = chain.getRemainingSecs()[ 'Value' ] - 60 if restrictLifeTime and restrictLifeTime < chainLifeTime: chainLifeTime = restrictLifeTime retVal = chain.generateChainFromRequestString( reqDict[ 'request' ], lifetime = chainLifeTime, diracGroup = diracGroup ) if not retVal[ 'OK' ]: return retVal #Upload! return rpcClient.completeDelegationUpload( reqDict[ 'id' ], retVal[ 'Value' ] ) @gProxiesSync def downloadProxy( self, userDN, userGroup, limited = False, requiredTimeLeft = 43200, proxyToConnect = False, token = False ): """ Get a proxy Chain from the proxy management """ cacheKey = ( userDN, userGroup ) if self.__proxiesCache.exists( cacheKey, requiredTimeLeft ): return S_OK( self.__proxiesCache.get( cacheKey ) ) req = X509Request() req.generateProxyRequest( limited = limited ) if proxyToConnect: rpcClient = RPCClient( "Framework/ProxyManager", proxyChain = proxyToConnect, timeout = 120 ) else: rpcClient = RPCClient( "Framework/ProxyManager", timeout = 120 ) if token: retVal = rpcClient.getProxyWithToken( userDN, userGroup, req.dumpRequest()['Value'], long( requiredTimeLeft ), token ) else: retVal = rpcClient.getProxy( userDN, userGroup, req.dumpRequest()['Value'], long( requiredTimeLeft ) ) if not retVal[ 'OK' ]: return retVal chain = X509Chain( keyObj = req.getPKey() ) retVal = chain.loadChainFromString( retVal[ 'Value' ] ) if not retVal[ 'OK' ]: return retVal self.__proxiesCache.add( cacheKey, chain.getRemainingSecs()['Value'], chain ) return S_OK( chain ) def downloadProxyToFile( self, userDN, userGroup, limited = False, requiredTimeLeft = 43200, filePath = False, proxyToConnect = False, token = False ): """ Get a proxy Chain from the proxy management and write it to file """ retVal = self.downloadProxy( userDN, userGroup, limited, requiredTimeLeft, proxyToConnect, token ) if not retVal[ 'OK' ]: return retVal chain = retVal[ 'Value' ] retVal = self.dumpProxyToFile( chain, filePath ) if not retVal[ 'OK' ]: return retVal retVal[ 'chain' ] = chain return retVal @gVOMSProxiesSync def downloadVOMSProxy( self, userDN, userGroup, limited = False, requiredTimeLeft = 43200, requiredVOMSAttribute = False, proxyToConnect = False, token = False ): """ Download a proxy if needed and transform it into a VOMS one """ cacheKey = ( userDN, userGroup, requiredVOMSAttribute, limited ) if self.__vomsProxiesCache.exists( cacheKey, requiredTimeLeft ): return S_OK( self.__vomsProxiesCache.get( cacheKey ) ) req = X509Request() req.generateProxyRequest( limited = limited ) if proxyToConnect: rpcClient = RPCClient( "Framework/ProxyManager", proxyChain = proxyToConnect, timeout = 120 ) else: rpcClient = RPCClient( "Framework/ProxyManager", timeout = 120 ) if token: retVal = rpcClient.getVOMSProxyWithToken( userDN, userGroup, req.dumpRequest()['Value'], long( requiredTimeLeft ), token, requiredVOMSAttribute ) else: retVal = rpcClient.getVOMSProxy( userDN, userGroup, req.dumpRequest()['Value'], long( requiredTimeLeft ), requiredVOMSAttribute ) if not retVal[ 'OK' ]: return retVal chain = X509Chain( keyObj = req.getPKey() ) retVal = chain.loadChainFromString( retVal[ 'Value' ] ) if not retVal[ 'OK' ]: return retVal self.__vomsProxiesCache.add( cacheKey, chain.getRemainingSecs()['Value'], chain ) return S_OK( chain ) def downloadVOMSProxyToFile( self, userDN, userGroup, limited = False, requiredTimeLeft = 43200, requiredVOMSAttribute = False, filePath = False, proxyToConnect = False, token = False ): """ Download a proxy if needed, transform it into a VOMS one and write it to file """ retVal = self.downloadVOMSProxy( userDN, userGroup, limited, requiredTimeLeft, requiredVOMSAttribute, proxyToConnect, token ) if not retVal[ 'OK' ]: return retVal chain = retVal[ 'Value' ] retVal = self.dumpProxyToFile( chain, filePath ) if not retVal[ 'OK' ]: return retVal retVal[ 'chain' ] = chain return retVal def getPilotProxyFromDIRACGroup( self, userDN, userGroup, requiredTimeLeft = 43200, proxyToConnect = False ): """ Download a pilot proxy with VOMS extensions depending on the group """ #Assign VOMS attribute vomsAttr = CS.getVOMSAttributeForGroup( userGroup ) if not vomsAttr: gLogger.verbose( "No voms attribute assigned to group %s when requested pilot proxy" % userGroup ) return self.downloadProxy( userDN, userGroup, limited = False, requiredTimeLeft = requiredTimeLeft, proxyToConnect = proxyToConnect ) else: return self.downloadVOMSProxy( userDN, userGroup, limited = False, requiredTimeLeft = requiredTimeLeft, requiredVOMSAttribute = vomsAttr, proxyToConnect = proxyToConnect ) def getPilotProxyFromVOMSGroup( self, userDN, vomsAttr, requiredTimeLeft = 43200, proxyToConnect = False ): """ Download a pilot proxy with VOMS extensions depending on the group """ groups = CS.getGroupsWithVOMSAttribute( vomsAttr ) if not groups: return S_ERROR( "No group found that has %s as voms attrs" % vomsAttr ) for userGroup in groups: result = self.downloadVOMSProxy( userDN, userGroup, limited = False, requiredTimeLeft = requiredTimeLeft, requiredVOMSAttribute = vomsAttr, proxyToConnect = proxyToConnect ) if result['OK']: return result return result def getPayloadProxyFromDIRACGroup( self, userDN, userGroup, requiredTimeLeft, token = False, proxyToConnect = False ): """ Download a payload proxy with VOMS extensions depending on the group """ #Assign VOMS attribute vomsAttr = CS.getVOMSAttributeForGroup( userGroup ) if not vomsAttr: gLogger.verbose( "No voms attribute assigned to group %s when requested payload proxy" % userGroup ) return self.downloadProxy( userDN, userGroup, limited = True, requiredTimeLeft = requiredTimeLeft, proxyToConnect = proxyToConnect, token = token ) else: return self.downloadVOMSProxy( userDN, userGroup, limited = True, requiredTimeLeft = requiredTimeLeft, requiredVOMSAttribute = vomsAttr, proxyToConnect = proxyToConnect, token = token ) def getPayloadProxyFromVOMSGroup( self, userDN, vomsAttr, token, requiredTimeLeft, proxyToConnect = False ): """ Download a payload proxy with VOMS extensions depending on the VOMS attr """ groups = CS.getGroupsWithVOMSAttribute( vomsAttr ) if not groups: return S_ERROR( "No group found that has %s as voms attrs" % vomsAttr ) userGroup = groups[0] return self.downloadVOMSProxy( userDN, userGroup, limited = True, requiredTimeLeft = requiredTimeLeft, requiredVOMSAttribute = vomsAttr, proxyToConnect = proxyToConnect, token = token ) def dumpProxyToFile( self, chain, destinationFile = False, requiredTimeLeft = 600 ): """ Dump a proxy to a file. It's cached so multiple calls won't generate extra files """ if self.__filesCache.exists( chain, requiredTimeLeft ): filepath = self.__filesCache.get( chain ) if os.path.isfile( filepath ): return S_OK( filepath ) self.__filesCache.delete( filepath ) retVal = chain.dumpAllToFile( destinationFile ) if not retVal[ 'OK' ]: return retVal filename = retVal[ 'Value' ] self.__filesCache.add( chain, chain.getRemainingSecs()['Value'], filename ) return S_OK( filename ) def deleteGeneratedProxyFile( self, chain ): """ Delete a file generated by a dump """ self.__filesCache.delete( chain ) return S_OK() def requestToken( self, requesterDN, requesterGroup, numUses = 1 ): """ Request a number of tokens. usesList must be a list of integers and each integer is the number of uses a token must have """ rpcClient = RPCClient( "Framework/ProxyManager", timeout = 120 ) return rpcClient.generateToken( requesterDN, requesterGroup, numUses ) def renewProxy( self, proxyToBeRenewed = False, minLifeTime = 3600, newProxyLifeTime = 43200, proxyToConnect = False ): """ Renew a proxy using the ProxyManager Arguments: proxyToBeRenewed : proxy to renew minLifeTime : if proxy life time is less than this, renew. Skip otherwise newProxyLifeTime : life time of new proxy proxyToConnect : proxy to use for connecting to the service """ retVal = File.multiProxyArgument( proxyToBeRenewed ) if not retVal[ 'Value' ]: return retVal proxyToRenewDict = retVal[ 'Value' ] secs = proxyToRenewDict[ 'chain' ].getRemainingSecs()[ 'Value' ] if secs > minLifeTime: File.deleteMultiProxy( proxyToRenewDict ) return S_OK() if not proxyToConnect: proxyToConnectDict = proxyToRenewDict else: retVal = File.multiProxyArgument( proxyToConnect ) if not retVal[ 'Value' ]: File.deleteMultiProxy( proxyToRenewDict ) return retVal proxyToConnectDict = retVal[ 'Value' ] userDN = proxyToRenewDict[ 'chain' ].getIssuerCert()[ 'Value' ].getSubjectDN()[ 'Value' ] retVal = proxyToRenewDict[ 'chain' ].getDIRACGroup() if not retVal[ 'OK' ]: File.deleteMultiProxy( proxyToRenewDict ) File.deleteMultiProxy( proxyToConnectDict ) return retVal userGroup = retVal[ 'Value' ] limited = proxyToRenewDict[ 'chain' ].isLimitedProxy()[ 'Value' ] voms = VOMS() retVal = voms.getVOMSAttributes( proxyToRenewDict[ 'chain' ] ) if not retVal[ 'OK' ]: File.deleteMultiProxy( proxyToRenewDict ) File.deleteMultiProxy( proxyToConnectDict ) return retVal vomsAttrs = retVal[ 'Value' ] if vomsAttrs: retVal = self.downloadVOMSProxy( userDN, userGroup, limited = limited, requiredTimeLeft = newProxyLifeTime, requiredVOMSAttribute = vomsAttrs[0], proxyToConnect = proxyToConnectDict[ 'chain' ] ) else: retVal = self.downloadProxy( userDN, userGroup, limited = limited, requiredTimeLeft = newProxyLifeTime, proxyToConnect = proxyToConnectDict[ 'chain' ] ) File.deleteMultiProxy( proxyToRenewDict ) File.deleteMultiProxy( proxyToConnectDict ) if not retVal[ 'OK' ]: return retVal if not proxyToRenewDict[ 'tempFile' ]: return proxyToRenewDict[ 'chain' ].dumpAllToFile( proxyToRenewDict[ 'file' ] ) return S_OK( proxyToRenewDict[ 'chain' ] ) def getDBContents( self, condDict = {} ): """ Get the contents of the db """ rpcClient = RPCClient( "Framework/ProxyManager", timeout = 120 ) return rpcClient.getContents( condDict, [ [ 'UserDN', 'DESC' ] ], 0, 0 ) def getVOMSAttributes( self, chain ): """ Get the voms attributes for a chain """ return VOMS().getVOMSAttributes( chain ) def getUploadedProxyLifeTime( self, DN, group ): """ Get the remaining seconds for an uploaded proxy """ result = self.getDBContents( { 'UserDN' : [ DN ], 'UserGroup' : [ group ] } ) if not result[ 'OK' ]: return result data = result[ 'Value' ] if len( data[ 'Records' ] ) == 0: return S_OK( 0 ) pNames = list( data[ 'ParameterNames' ] ) dnPos = pNames.index( 'UserDN' ) groupPos = pNames.index( 'UserGroup' ) expiryPos = pNames.index( 'ExpirationTime' ) for row in data[ 'Records' ]: if DN == row[ dnPos ] and group == row[ groupPos ]: td = row[ expiryPos ] - datetime.datetime.utcnow() secondsLeft = td.days * 86400 + td.seconds return S_OK( max( 0, secondsLeft ) ) return S_OK( 0 )
class DataCache: def __init__(self): self.graphsLocation = os.path.join( gConfig.getValue('/LocalSite/InstancePath', rootPath), 'data', 'accountingPlots') self.cachedGraphs = {} self.alive = True self.purgeThread = threading.Thread(target=self.purgeExpired) self.purgeThread.setDaemon(1) self.purgeThread.start() self.__dataCache = DictCache() self.__graphCache = DictCache(deleteFunction=self._deleteGraph) self.__dataLifeTime = 600 self.__graphLifeTime = 3600 def setGraphsLocation(self, graphsDir): self.graphsLocation = graphsDir for graphName in os.listdir(self.graphsLocation): if graphName.find(".png") > 0: graphLocation = "%s/%s" % (self.graphsLocation, graphName) gLogger.verbose("Purging %s" % graphLocation) os.unlink(graphLocation) def purgeExpired(self): while self.alive: time.sleep(600) self.__graphCache.purgeExpired() self.__dataCache.purgeExpired() def getReportData(self, reportRequest, reportHash, dataFunc): """ Get report data from cache if exists, else generate it """ reportData = self.__dataCache.get(reportHash) if reportData == False: retVal = dataFunc(reportRequest) if not retVal['OK']: return retVal reportData = retVal['Value'] self.__dataCache.add(reportHash, self.__dataLifeTime, reportData) return S_OK(reportData) def getReportPlot(self, reportRequest, reportHash, reportData, plotFunc): """ Get report data from cache if exists, else generate it """ plotDict = self.__graphCache.get(reportHash) if plotDict == False: basePlotFileName = "%s/%s" % (self.graphsLocation, reportHash) retVal = plotFunc(reportRequest, reportData, basePlotFileName) if not retVal['OK']: return retVal plotDict = retVal['Value'] if plotDict['plot']: plotDict['plot'] = "%s.png" % reportHash if plotDict['thumbnail']: plotDict['thumbnail'] = "%s.thb.png" % reportHash self.__graphCache.add(reportHash, self.__graphLifeTime, plotDict) return S_OK(plotDict) def getPlotData(self, plotFileName): filename = "%s/%s" % (self.graphsLocation, plotFileName) try: fd = file(filename, "rb") data = fd.read() fd.close() except Exception, e: return S_ERROR("Can't open file %s: %s" % (plotFileName, str(e))) return S_OK(data)