def export_getPilotStatistics(attribute, selectDict): """ Get pilot statistics distribution per attribute value with a given selection """ startDate = selectDict.get('FromDate', None) if startDate: del selectDict['FromDate'] if startDate is None: startDate = selectDict.get('LastUpdate', None) if startDate: del selectDict['LastUpdate'] endDate = selectDict.get('ToDate', None) if endDate: del selectDict['ToDate'] result = pilotDB.getCounters('PilotAgents', [attribute], selectDict, newer=startDate, older=endDate, timeStamp='LastUpdateTime') statistics = {} if result['OK']: for status, count in result['Value']: if "OwnerDN" in status: userName = getUsernameForDN(status['OwnerDN']) if userName['OK']: status['OwnerDN'] = userName['Value'] statistics[status['OwnerDN']] = count else: statistics[status[attribute]] = count return S_OK(statistics)
def getUserNameAndGroup(info): """ Get the user name and group from the DN and VOMS role """ global dnCache, roleCache owner = {} username = dnCache.get(info['OwnerDN']) if not username: result = getUsernameForDN(info['OwnerDN']) if result['OK']: username = result['Value'] dnCache[info['OwnerDN']] = username elif "No username" in result['Message']: username = '******' dnCache[info['OwnerDN']] = username if username and username != 'Unknown': groups = roleCache.get('/'+info['OwnerRole']) if not groups: groups = getGroupsWithVOMSAttribute('/'+info['OwnerRole']) roleCache['/'+info['OwnerRole']] = groups if groups: owner['username'] = username owner['group'] = groups[0] return owner
def getPilotMonitorSelectors(self): """ Get distinct values for the Pilot Monitor page selectors """ paramNames = [ 'OwnerDN', 'OwnerGroup', 'GridType', 'Broker', 'Status', 'DestinationSite', 'GridSite' ] resultDict = {} for param in paramNames: result = self.getDistinctAttributeValues('PilotAgents', param) if result['OK']: resultDict[param] = result['Value'] else: resultDict = [] if param == "OwnerDN": userList = [] for dn in result['Value']: resultUser = getUsernameForDN(dn) if resultUser['OK']: userList.append(resultUser['Value']) resultDict["Owner"] = userList return S_OK(resultDict)
def getUserRightsForJob(self, jobID): """ Get access rights to job with jobID for the user specified by userDN/userGroup """ result = self.jobDB.getJobAttributes(jobID, ['OwnerDN', 'OwnerGroup']) if not result['OK']: return result elif result['Value']: owner = result['Value']['OwnerDN'] group = result['Value']['OwnerGroup'] result = getUsernameForDN(owner) ownerName = '' if result['OK']: ownerName = result['Value'] result = self.getJobPolicy(owner, group) if self.userName and self.userName == ownerName and self.userGroup == group: result['UserIsOwner'] = True else: result['UserIsOwner'] = False return result else: return S_ERROR('Job not found')
def getPilotMonitorSelectors(self): """Get distinct values for the Pilot Monitor page selectors""" paramNames = [ "OwnerDN", "OwnerGroup", "GridType", "Broker", "Status", "DestinationSite", "GridSite" ] resultDict = {} for param in paramNames: result = self.getDistinctAttributeValues("PilotAgents", param) if result["OK"]: resultDict[param] = result["Value"] else: resultDict = [] if param == "OwnerDN": userList = [] for dn in result["Value"]: resultUser = getUsernameForDN(dn) if resultUser["OK"]: userList.append(resultUser["Value"]) resultDict["Owner"] = userList return S_OK(resultDict)
def __sendAccounting( ftsJob, ownerDN ): """ prepare and send DataOperation to AccouringDB """ dataOp = DataOperation() dataOp.setStartTime( fromString( ftsJob.SubmitTime ) ) dataOp.setEndTime( fromString( ftsJob.LastUpdate ) ) accountingDict = dict() accountingDict["OperationType"] = "ReplicateAndRegister" username = getUsernameForDN( ownerDN ) if not username["OK"]: username = ownerDN else: username = username["Value"] accountingDict["User"] = username accountingDict["Protocol"] = "FTS" # accountingDict['RegistrationTime'] = 0 # accountingDict['RegistrationOK'] = 0 # accountingDict['RegistrationTotal'] = 0 accountingDict["TransferOK"] = len( [ f for f in ftsJob if f.Status == "Finished" ] ) accountingDict["TransferTotal"] = len( ftsJob ) accountingDict["TransferSize"] = ftsJob.Size accountingDict["FinalStatus"] = ftsJob.Status accountingDict["Source"] = ftsJob.SourceSE accountingDict["Destination"] = ftsJob.TargetSE dt = ftsJob.LastUpdate - ftsJob.SubmitTime transferTime = dt.days * 86400 + dt.seconds accountingDict["TransferTime"] = transferTime dataOp.setValuesFromDict( accountingDict ) dataOp.commit()
def _addOperationForTransformations(operationsOnTransformationDict, operation, transformations, owner=None, ownerGroup=None, ownerDN=None): """Fill the operationsOnTransformationDict""" transformationIDsAndBodies = [ (transformation['TransformationID'], transformation['Body'], transformation['AuthorDN'], transformation['AuthorGroup']) for transformation in transformations['Value'] ] for transID, body, t_ownerDN, t_ownerGroup in transformationIDsAndBodies: if transID in operationsOnTransformationDict: operationsOnTransformationDict[transID]['Operations'].append( operation) else: operationsOnTransformationDict[transID] = { 'Body': body, 'Operations': [operation], 'Owner': owner if owner else getUsernameForDN(t_ownerDN)['Value'], 'OwnerGroup': ownerGroup if owner else t_ownerGroup, 'OwnerDN': ownerDN if owner else t_ownerDN }
def _addOperationForTransformations( operationsOnTransformationDict, operation, transformations, owner=None, ownerGroup=None, ownerDN=None, ): """Fill the operationsOnTransformationDict""" transformationIDsAndBodies = ( ( transformation["TransformationID"], transformation["Body"], transformation["AuthorDN"], transformation["AuthorGroup"], ) for transformation in transformations["Value"] ) for transID, body, t_ownerDN, t_ownerGroup in transformationIDsAndBodies: if transID in operationsOnTransformationDict: operationsOnTransformationDict[transID]["Operations"].append(operation) else: operationsOnTransformationDict[transID] = { "TransformationID": transID, "Body": body, "Operations": [operation], "Owner": owner if owner else getUsernameForDN(t_ownerDN)["Value"], "OwnerGroup": ownerGroup if owner else t_ownerGroup, "OwnerDN": ownerDN if owner else t_ownerDN, }
def getUserNameAndGroup(info): """ Get the user name and group from the DN and VOMS role """ global dnCache, roleCache owner = {} if "OwnerDN" not in info: return owner username = dnCache.get(info.get('OwnerDN')) if not username: result = getUsernameForDN(info.get('OwnerDN', 'Unknown')) if result['OK']: username = result['Value'] dnCache[info['OwnerDN']] = username elif "No username" in result['Message']: username = '******' dnCache[info['OwnerDN']] = username if username and username != 'Unknown': groups = roleCache.get('/' + info.get('OwnerRole')) if not groups: groups = getGroupsWithVOMSAttribute('/' + info['OwnerRole']) roleCache['/' + info['OwnerRole']] = groups if groups: owner['username'] = username owner['group'] = groups[0] return owner
def export_getPilotStatistics ( attribute, selectDict ): """ Get pilot statistics distribution per attribute value with a given selection """ startDate = selectDict.get( 'FromDate', None ) if startDate: del selectDict['FromDate'] if startDate is None: startDate = selectDict.get( 'LastUpdate', None ) if startDate: del selectDict['LastUpdate'] endDate = selectDict.get( 'ToDate', None ) if endDate: del selectDict['ToDate'] result = pilotDB.getCounters( 'PilotAgents', [attribute], selectDict, newer = startDate, older = endDate, timeStamp = 'LastUpdateTime' ) statistics = {} if result['OK']: for status, count in result['Value']: if "OwnerDN" in status: userName = getUsernameForDN( status['OwnerDN'] ) if userName['OK']: status['OwnerDN'] = userName['Value'] statistics[status[selector]] = count return S_OK( statistics )
def getUserNameAndGroup(info): """Get the user name and group from the DN and VOMS role""" global dnCache, roleCache owner = {} if "OwnerDN" not in info: return owner username = dnCache.get(info.get("OwnerDN")) if not username: result = getUsernameForDN(info.get("OwnerDN", "Unknown")) if result["OK"]: username = result["Value"] dnCache[info["OwnerDN"]] = username elif "No username" in result["Message"]: username = "******" dnCache[info["OwnerDN"]] = username if username and username != "Unknown": groups = roleCache.get("/" + info.get("OwnerRole")) if not groups: groups = getGroupsWithVOMSAttribute("/" + info["OwnerRole"]) roleCache["/" + info["OwnerRole"]] = groups if groups: owner["username"] = username owner["group"] = groups[0] return owner
def getUserRightsForJob( self, jobID ): """ Get access rights to job with jobID for the user specified by userDN/userGroup """ result = self.jobDB.getJobAttributes( jobID, [ 'OwnerDN', 'OwnerGroup' ] ) if not result['OK']: return result elif result['Value']: owner = result['Value']['OwnerDN'] group = result['Value']['OwnerGroup'] result = getUsernameForDN(owner) ownerName = '' if result['OK']: ownerName = result['Value'] result = self.getJobPolicy( owner, group ) if self.userName and self.userName == ownerName and self.userGroup == group: result[ 'UserIsOwner' ] = True else: result[ 'UserIsOwner' ] = False return result else: return S_ERROR('Job not found')
def __init__(self, userDN, userGroup, userProperties): self.userDN = userDN self.userName = '' result = getUsernameForDN(userDN) if result['OK']: self.userName = result['Value'] self.userGroup = userGroup self.userProperties = userProperties self.jobDB = None
def __init__( self, userDN, userGroup, userProperties ): self.userDN = userDN self.userName = '' result = getUsernameForDN(userDN) if result['OK']: self.userName = result['Value'] self.userGroup = userGroup self.userProperties = userProperties self.jobDB = None
def authenticate_user(self, credential): """Authorize user :param dict credential: credential (token payload) :return: str or bool """ result = getUsernameForDN(wrapIDAsDN(credential["sub"])) if not result["OK"]: self.server.log.error(result["Message"]) return result.get("Value")
def __init__(self, userDN, userGroup, allInfo=True): self.userDN = userDN self.userName = '' result = getUsernameForDN(userDN) if result['OK']: self.userName = result['Value'] self.userGroup = userGroup self.userProperties = getPropertiesForGroup(userGroup, []) self.jobDB = None self.allInfo = allInfo self.__permissions = {} self.__getUserJobPolicy()
def __init__( self, userDN, userGroup, allInfo=True ): self.userDN = userDN self.userName = '' result = getUsernameForDN(userDN) if result['OK']: self.userName = result['Value'] self.userGroup = userGroup self.userProperties = getPropertiesForGroup( userGroup, [] ) self.jobDB = None self.allInfo = allInfo self.__permissions = {} self.__getUserJobPolicy()
def export_banSite(self, site, comment='No comment'): """ Ban the given site in the site mask """ result = self.getRemoteCredentials() dn = result['DN'] result = getUsernameForDN(dn) if result['OK']: author = result['Value'] else: author = dn result = jobDB.banSiteInMask(site, author, comment) return result
def export_banSite(self, site,comment='No comment'): """ Ban the given site in the site mask """ result = self.getRemoteCredentials() dn = result['DN'] result = getUsernameForDN(dn) if result['OK']: author = result['Value'] else: author = dn result = jobDB.banSiteInMask(site,author,comment) return result
def parseIdPAuthorizationResponse(self, response, session): """Fill session by user profile, tokens, comment, OIDC authorize status, etc. Prepare dict with user parameters, if DN is absent there try to get it. Create new or modify existing DIRAC user and store the session :param dict response: authorization response :param str session: session :return: S_OK(dict)/S_ERROR() """ providerName = session.pop("Provider") sLog.debug("Try to parse authentification response from %s:\n" % providerName, pprint.pformat(response)) # Parse response result = self.idps.getIdProvider(providerName) if not result["OK"]: return result idpObj = result["Value"] result = idpObj.parseAuthResponse(response, session) if not result["OK"]: return result # FINISHING with IdP # As a result of authentication we will receive user credential dictionary credDict, payload = result["Value"] sLog.debug("Read profile:", pprint.pformat(credDict)) # Is ID registred? result = getUsernameForDN(credDict["DN"]) if not result["OK"]: comment = f"ID {credDict['ID']} is not registred in DIRAC. " payload.update(idpObj.getUserProfile().get("Value", {})) result = self.__registerNewUser(providerName, payload) if result["OK"]: comment += "Administrators have been notified about you." else: comment += "Please, contact the DIRAC administrators." # Notify user about problem html = getHTML("unregistered user!", info=comment, theme="warning") return S_ERROR(html) credDict["username"] = result["Value"] # Update token for user. This token will be stored separately in the database and # updated from time to time. This token will never be transmitted, # it will be used to make exchange token requests. result = self.tokenCli.updateToken(idpObj.token, credDict["ID"], idpObj.name) return S_OK(credDict) if result["OK"] else result
def _addOperationForTransformations(operationsOnTransformationDict, operation, transformations, owner=None, ownerGroup=None, ownerDN=None): """Fill the operationsOnTransformationDict""" transformationIDsAndBodies = [(transformation['TransformationID'], transformation['Body'], transformation['AuthorDN'], transformation['AuthorGroup']) for transformation in transformations['Value']] for transID, body, t_ownerDN, t_ownerGroup in transformationIDsAndBodies: if transID in operationsOnTransformationDict: operationsOnTransformationDict[transID]['Operations'].append(operation) else: operationsOnTransformationDict[transID] = {'Body': body, 'Operations': [operation], 'Owner': owner if owner else getUsernameForDN(t_ownerDN)['Value'], 'OwnerGroup': ownerGroup if owner else t_ownerGroup, 'OwnerDN': ownerDN if owner else t_ownerDN}
def export_getPilotStatistics(cls, attribute, selectDict): """Get pilot statistics distribution per attribute value with a given selection""" startDate = selectDict.get("FromDate", None) if startDate: del selectDict["FromDate"] if startDate is None: startDate = selectDict.get("LastUpdate", None) if startDate: del selectDict["LastUpdate"] endDate = selectDict.get("ToDate", None) if endDate: del selectDict["ToDate"] # Owner attribute is not part of PilotAgentsDB # It has to be converted into a OwnerDN owners = selectDict.get("Owner") if owners: ownerDNs = [] for owner in owners: result = getDNForUsername(owner) if not result["OK"]: return result ownerDNs.append(result["Value"]) selectDict["OwnerDN"] = ownerDNs del selectDict["Owner"] result = cls.pilotAgentsDB.getCounters("PilotAgents", [attribute], selectDict, newer=startDate, older=endDate, timeStamp="LastUpdateTime") statistics = {} if result["OK"]: for status, count in result["Value"]: if "OwnerDN" in status: userName = getUsernameForDN(status["OwnerDN"]) if userName["OK"]: status["OwnerDN"] = userName["Value"] statistics[status["OwnerDN"]] = count else: statistics[status[attribute]] = count return S_OK(statistics)
def getJobPolicy(self, jobOwnerDN='', jobOwnerGroup=''): """ Get the job operations rights for a job owned by jobOwnerDN/jobOwnerGroup for a user with userDN/userGroup. Returns a dictionary of various operations rights """ # Can not do anything by default permDict = {} for r in ALL_RIGHTS: permDict[r] = False # Anybody can get info about the jobs permDict[RIGHT_GET_INFO] = True # Give JobAdmin permission if needed if Properties.JOB_ADMINISTRATOR in self.userProperties: for r in PROPERTY_RIGHTS[Properties.JOB_ADMINISTRATOR]: permDict[r] = True # Give JobAdmin permission if needed if Properties.NORMAL_USER in self.userProperties: for r in PROPERTY_RIGHTS[Properties.NORMAL_USER]: permDict[r] = True # Give permissions of the generic pilot if Properties.GENERIC_PILOT in self.userProperties: for r in PROPERTY_RIGHTS[Properties.GENERIC_PILOT]: permDict[r] = True # Job Owner can do everything with his jobs result = getUsernameForDN(jobOwnerDN) jobOwnerName = '' if result['OK']: jobOwnerName = result['Value'] if jobOwnerName and self.userName and jobOwnerName == self.userName: for r in OWNER_RIGHTS: permDict[r] = True # Members of the same group sharing their jobs can do everything if jobOwnerGroup == self.userGroup: if Properties.JOB_SHARING in self.userProperties: for right in GROUP_RIGHTS: permDict[right] = True return S_OK(permDict)
def getJobPolicy( self, jobOwnerDN = '', jobOwnerGroup = '' ): """ Get the job operations rights for a job owned by jobOwnerDN/jobOwnerGroup for a user with userDN/userGroup. Returns a dictionary of various operations rights """ # Can not do anything by default permDict = {} for r in ALL_RIGHTS: permDict[r] = False # Anybody can get info about the jobs permDict[ RIGHT_GET_INFO ] = True # Give JobAdmin permission if needed if Properties.JOB_ADMINISTRATOR in self.userProperties: for r in PROPERTY_RIGHTS[ Properties.JOB_ADMINISTRATOR ]: permDict[ r ] = True # Give JobAdmin permission if needed if Properties.NORMAL_USER in self.userProperties: for r in PROPERTY_RIGHTS[ Properties.NORMAL_USER ]: permDict[ r ] = True # Give permissions of the generic pilot if Properties.GENERIC_PILOT in self.userProperties: for r in PROPERTY_RIGHTS[ Properties.GENERIC_PILOT ]: permDict[ r ] = True # Job Owner can do everything with his jobs result = getUsernameForDN(jobOwnerDN) jobOwnerName = '' if result['OK']: jobOwnerName = result['Value'] if jobOwnerName and self.userName and jobOwnerName == self.userName: for r in OWNER_RIGHTS: permDict[r] = True # Members of the same group sharing their jobs can do everything if jobOwnerGroup == self.userGroup: if Properties.JOB_SHARING in self.userProperties: for right in GROUP_RIGHTS: permDict[right] = True return S_OK( permDict )
def __sendAccounting(ftsJob, ownerDN): """ prepare and send DataOperation to AccouringDB """ dataOp = DataOperation() dataOp.setStartTime(fromString(ftsJob.SubmitTime)) dataOp.setEndTime(fromString(ftsJob.LastUpdate)) accountingDict = dict() accountingDict["OperationType"] = "ReplicateAndRegister" username = getUsernameForDN(ownerDN) if not username["OK"]: username = ownerDN else: username = username["Value"] accountingDict["User"] = username accountingDict[ "Protocol"] = "FTS3" if 'fts3' in ftsJob.FTSServer.lower( ) else 'FTS' accountingDict['ExecutionSite'] = ftsJob.FTSServer accountingDict['RegistrationTime'] = ftsJob._regTime accountingDict['RegistrationOK'] = ftsJob._regSuccess accountingDict['RegistrationTotal'] = ftsJob._regTotal accountingDict["TransferOK"] = len( [f for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES]) accountingDict["TransferTotal"] = len(ftsJob) accountingDict["TransferSize"] = ftsJob.Size - ftsJob.FailedSize accountingDict["FinalStatus"] = ftsJob.Status accountingDict["Source"] = ftsJob.SourceSE accountingDict["Destination"] = ftsJob.TargetSE dt = ftsJob.LastUpdate - ftsJob.SubmitTime transferTime = dt.days * 86400 + dt.seconds accountingDict["TransferTime"] = transferTime # accountingDict['TransferTime'] = sum( [f._duration for f in ftsJob]) dataOp.setValuesFromDict(accountingDict) dataOp.commit()
def setUp(self): self.host = "localhost" self.notificationPort = 9154 self.rootPwd = "" self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI(self.host) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result["OK"]: raise Exception(result["Message"]) cfg = result["Value"] setup = cfg.getOption("DIRAC/Setup", "dirac-JenkinsSetup") self.frameworkSetup = cfg.getOption("DIRAC/Setups/" + setup + "/Framework") self.rootPwd = cfg.getOption("Systems/Databases/Password") self.diracPwd = self.rootPwd result = getProxyInfo() if not result["OK"]: raise Exception(result["Message"]) chain = result["Value"]["chain"] result = chain.getCertInChain(-1) if not result["OK"]: raise Exception(result["Message"]) result = result["Value"].getSubjectDN() if not result["OK"]: raise Exception(result["Message"]) userDN = result["Value"] result = getUsernameForDN(userDN) if not result["OK"]: raise Exception(result["Message"]) self.user = result["Value"] if not self.user: self.user = "******"
def setUp(self): self.host = 'localhost' self.notificationPort = 9154 self.rootPwd = '' self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI(self.host) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result['OK']: raise Exception(result['Message']) cfg = result['Value'] setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup') self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework') self.rootPwd = cfg.getOption('Systems/Databases/Password') self.diracPwd = self.rootPwd result = getProxyInfo() if not result['OK']: raise Exception(result['Message']) chain = result['Value']['chain'] result = chain.getCertInChain(-1) if not result['OK']: raise Exception(result['Message']) result = result['Value'].getSubjectDN() if not result['OK']: raise Exception(result['Message']) userDN = result['Value'] result = getUsernameForDN(userDN) if not result['OK']: raise Exception(result['Message']) self.user = result['Value'] if not self.user: self.user = '******'
def setUp(self): self.host = 'localhost' self.notificationPort = 9154 self.rootPwd = '' self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI(self.host) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result['OK']: raise Exception(result['Message']) cfg = result['Value'] setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup') self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework') self.rootPwd = cfg.getOption('Systems/Databases/Password') self.diracPwd = self.rootPwd result = getProxyInfo() if not result['OK']: raise Exception(result['Message']) chain = result['Value']['chain'] result = chain.getCertInChain(-1) if not result['OK']: raise Exception(result['Message']) result = result['Value'].getSubjectDN() if not result['OK']: raise Exception(result['Message']) userDN = result['Value'] result = getUsernameForDN(userDN) if not result['OK']: raise Exception(result['Message']) self.user = result['Value'] if not self.user: self.user = '******'
def __sendAccounting( ftsJob, ownerDN ): """ prepare and send DataOperation to AccouringDB """ dataOp = DataOperation() dataOp.setStartTime( fromString( ftsJob.SubmitTime ) ) dataOp.setEndTime( fromString( ftsJob.LastUpdate ) ) accountingDict = dict() accountingDict["OperationType"] = "ReplicateAndRegister" username = getUsernameForDN( ownerDN ) if not username["OK"]: username = ownerDN else: username = username["Value"] accountingDict["User"] = username accountingDict["Protocol"] = "FTS3" if 'fts3' in ftsJob.FTSServer.lower() else 'FTS' accountingDict['ExecutionSite'] = ftsJob.FTSServer accountingDict['RegistrationTime'] = ftsJob._regTime accountingDict['RegistrationOK'] = ftsJob._regSuccess accountingDict['RegistrationTotal'] = ftsJob._regTotal accountingDict["TransferOK"] = len( [ f for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES ] ) accountingDict["TransferTotal"] = len( ftsJob ) accountingDict["TransferSize"] = ftsJob.Size - ftsJob.FailedSize accountingDict["FinalStatus"] = ftsJob.Status accountingDict["Source"] = ftsJob.SourceSE accountingDict["Destination"] = ftsJob.TargetSE # dt = ftsJob.LastUpdate - ftsJob.SubmitTime # transferTime = dt.days * 86400 + dt.seconds # accountingDict["TransferTime"] = transferTime accountingDict['TransferTime'] = sum( [int( f._duration ) for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES ] ) dataOp.setValuesFromDict( accountingDict ) dataOp.commit()
def getPilotMonitorSelectors( self ): """ Get distinct values for the Pilot Monitor page selectors """ paramNames = ['OwnerDN', 'OwnerGroup', 'GridType', 'Broker', 'Status', 'DestinationSite', 'GridSite'] resultDict = {} for param in paramNames: result = self.getDistinctAttributeValues( 'PilotAgents', param ) if result['OK']: resultDict[param] = result['Value'] else: resultDict = [] if param == "OwnerDN": userList = [] for dn in result['Value']: resultUser = getUsernameForDN( dn ) if resultUser['OK']: userList.append( resultUser['Value'] ) resultDict["Owner"] = userList return S_OK( resultDict )
def __sendAccounting(ftsJob, ownerDN): """ prepare and send DataOperation to AccouringDB """ dataOp = DataOperation() dataOp.setStartTime(fromString(ftsJob.SubmitTime)) dataOp.setEndTime(fromString(ftsJob.LastUpdate)) accountingDict = dict() accountingDict["OperationType"] = "ReplicateAndRegister" username = getUsernameForDN(ownerDN) if not username["OK"]: username = ownerDN else: username = username["Value"] accountingDict["User"] = username accountingDict["Protocol"] = "FTS" # accountingDict['RegistrationTime'] = 0 # accountingDict['RegistrationOK'] = 0 # accountingDict['RegistrationTotal'] = 0 accountingDict["TransferOK"] = len( [f for f in ftsJob if f.Status == "Finished"]) accountingDict["TransferTotal"] = len(ftsJob) accountingDict["TransferSize"] = ftsJob.Size accountingDict["FinalStatus"] = ftsJob.Status accountingDict["Source"] = ftsJob.SourceSE accountingDict["Destination"] = ftsJob.TargetSE dt = ftsJob.LastUpdate - ftsJob.SubmitTime transferTime = dt.days * 86400 + dt.seconds accountingDict["TransferTime"] = transferTime dataOp.setValuesFromDict(accountingDict) dataOp.commit()
def web_getStatisticsData( self ): req = self.__request() paletteColor = Palette() RPC = RPCClient( "WorkloadManagement/WMSAdministrator" ) selector = self.request.arguments["statsField"][0] if selector == 'Site': selector = "GridSite" if selector == "Computing Element": selector = "DestinationSite" elif selector == "Owner Group": selector = "OwnerGroup" elif selector == "Owner": selector = "OwnerDN" result = yield self.threadTask( RPC.getPilotStatistics, selector, req ) if not result['OK']: if 'FromDate' in req: del req['FromDate'] if 'LastUpdate' in req: del req['LastUpdate'] if 'ToDate' in req: del req['ToDate'] result = yield self.threadTask( RPC.getCounters, "PilotAgents", [selector], req ) statistics = {} if result['OK']: for status, count in result['Value']: if "OwnerDN" in status: userName = getUsernameForDN( status['OwnerDN'] ) if userName['OK']: status['OwnerDN'] = userName['Value'] statistics[status[selector]] = count result = S_OK(statistics) if result["OK"]: callback = [] result = dict( result["Value"] ) keylist = result.keys() keylist.sort() if selector == "Site": tier1 = gConfig.getValue( "/WebApp/PreferredSites", [] ) if len( tier1 ) > 0: tier1.sort() for i in tier1: if result.has_key( i ): countryCode = i.rsplit( ".", 1 )[1] callback.append( {"key":i, "value":result[i], "code":countryCode, "color": paletteColor.getColor( countryCode ) } ) for key in keylist: if selector == "Site" and tier1: if key not in tier1: try: countryCode = key.rsplit( ".", 1 )[1] except: countryCode = "Unknown" callback.append( {"key":key, "value":result[key], "code":countryCode, "color": paletteColor.getColor( key ) } ) elif selector == "Site" and not tier1: try: countryCode = key.rsplit( ".", 1 )[1] except: countryCode = "Unknown" callback.append( {"key":key, "value":result[key], "code":countryCode, "color": paletteColor.getColor( key ) } ) else: callback.append( {"key":key, "value":result[key], "code":"", "color": paletteColor.getColor( key ) } ) callback = {"success":"true", "result":callback} else: callback = {"success":"false", "error":result["Message"]} self.finish( callback )
def getUsersGroupsAndSEs( queryQueue, name_id_se ): """ This dumps the users, groups and SEs from the LFC, converts them into DIRAC group, and create the query to insert them into the DFC. WATCH OUT !! The DIRAC group is evaluated from the VOMS attribute, since this is what is used in the LFC. So if you have several DIRAC groups that have the same voms role, the assigned group will be the first one... :param queryQueue : queue in which to put the SQL statement to be executed against the DFC :param name_id_se : cache to be filled in with the mapping {seName:id in the DFC} """ connection = cx_Oracle.Connection( prodDbAccount, prodDbAccountPassword, prodDbTNS, threaded = True ) fromdbR_ = connection.cursor() fromdbR_.arraysize = 1000 # First, fetch all the UID and DN from the LFC if fromdbR_.execute( "SELECT USERID, USERNAME from CNS_USERINFO" ): rows = fromdbR_.fetchall() for row in rows: uid = row[0] dn = row[1] name = uid_name.get( uid ) # If the name is not in cache, if not name: # We do some mangling of the DN to try to have sensitive name whatever happen # We also include the UID in case the name is not unique, or several DN are associated namePart = 'name' idPart = '' dnSplit = dn.split( 'CN=' ) try: namePart = dnSplit[-1].replace( "'", " " ) except Exception as e: pass try: idPart = dnSplit[2].replace( "/", "" ) except Exception as e: pass idPart += ' uid_%s' % uid name = "Unknown (%s %s)" % ( namePart, idPart ) # Now, we do try to convert the DN into a DIRAC username if "Unknown" in name: result = getUsernameForDN( dn ) if result['OK']: name = result['Value'] uid_name[uid] = name # Now we prepare the SQL statement to insert them into the DFC for uid in uid_name: username = uid_name[uid] queryQueue.put( "INSERT INTO FC_Users(UID, UserName) values (%s, '%s');\n" % ( uid, username ) ) # Now, same principle on the group if fromdbR_.execute( "SELECT GID, GROUPNAME from CNS_GROUPINFO" ): rows = fromdbR_.fetchall() for row in rows: gid = row[0] gn = row[1] groupname = gn # CAUTION: as explained in the docstring, if multiple groups share the same voms role # we take the first one groups = getGroupsWithVOMSAttribute( '/' + gn ) if groups: groupname = groups[0] queryQueue.put( "INSERT INTO FC_Groups(GID, GroupName) values (%s, '%s');\n" % ( gid, groupname ) ) # We build a cache that contains the mapping between the name and its ID in the DFC DB # The ID starts at 2 because ID=1 is taken by FakeSe seCounter = 2 # Fetch the name from the LFC if fromdbR_.execute( "select unique HOST from CNS_FILE_REPLICA" ): rows = fromdbR_.fetchall() for row in rows: seName = row[0] # Populate the SE cache name_id_se[seName] = seCounter # Create the query for the DFC queryQueue.put( "INSERT INTO FC_StorageElements(SEID, SEName) values (%s, '%s');\n" % ( seCounter, seName ) ) seCounter += 1 # Also here we just insert all the statuses defined above for status in dirac_status: queryQueue.put( "INSERT INTO FC_Statuses (Status) values ('%s');\n" % status ) fromdbR_.close() connection.close() # Set the poison pill in the queue queryQueue.put( None ) return
def web_getStatisticsData(self): req = self.__request() paletteColor = Palette() RPC = RPCClient("WorkloadManagement/WMSAdministrator") selector = self.request.arguments["statsField"][0] if selector == 'Site': selector = "GridSite" if selector == "Computing Element": selector = "DestinationSite" elif selector == "Owner Group": selector = "OwnerGroup" elif selector == "Owner": selector = "OwnerDN" result = yield self.threadTask(RPC.getPilotStatistics, selector, req) if not result['OK']: if 'FromDate' in req: del req['FromDate'] if 'LastUpdate' in req: del req['LastUpdate'] if 'ToDate' in req: del req['ToDate'] result = yield self.threadTask(RPC.getCounters, "PilotAgents", [selector], req) statistics = {} if result['OK']: for status, count in result['Value']: if "OwnerDN" in status: userName = getUsernameForDN(status['OwnerDN']) if userName['OK']: status['OwnerDN'] = userName['Value'] statistics[status[selector]] = count result = S_OK(statistics) if result["OK"]: callback = [] result = dict(result["Value"]) keylist = result.keys() keylist.sort() if selector == "Site": tier1 = gConfig.getValue("/WebApp/PreferredSites", []) if len(tier1) > 0: tier1.sort() for i in tier1: if result.has_key(i): countryCode = i.rsplit(".", 1)[1] callback.append({ "key": i, "value": result[i], "code": countryCode, "color": paletteColor.getColor(countryCode) }) for key in keylist: if selector == "Site" and tier1: if key not in tier1: try: countryCode = key.rsplit(".", 1)[1] except: countryCode = "Unknown" callback.append({ "key": key, "value": result[key], "code": countryCode, "color": paletteColor.getColor(key) }) elif selector == "Site" and not tier1: try: countryCode = key.rsplit(".", 1)[1] except: countryCode = "Unknown" callback.append({ "key": key, "value": result[key], "code": countryCode, "color": paletteColor.getColor(key) }) else: callback.append({ "key": key, "value": result[key], "code": "", "color": paletteColor.getColor(key) }) callback = {"success": "true", "result": callback} else: callback = {"success": "false", "error": result["Message"]} self.finish(callback)
def getUsersGroupsAndSEs(queryQueue, name_id_se): """ This dumps the users, groups and SEs from the LFC, converts them into DIRAC group, and create the query to insert them into the DFC. WATCH OUT !! The DIRAC group is evaluated from the VOMS attribute, since this is what is used in the LFC. So if you have several DIRAC groups that have the same voms role, the assigned group will be the first one... :param queryQueue : queue in which to put the SQL statement to be executed against the DFC :param name_id_se : cache to be filled in with the mapping {seName:id in the DFC} """ connection = cx_Oracle.Connection(prodDbAccount, prodDbAccountPassword, prodDbTNS, threaded=True) fromdbR_ = connection.cursor() fromdbR_.arraysize = 1000 # First, fetch all the UID and DN from the LFC if fromdbR_.execute("SELECT USERID, USERNAME from CNS_USERINFO"): rows = fromdbR_.fetchall() for row in rows: uid = row[0] dn = row[1] name = uid_name.get(uid) # If the name is not in cache, if not name: # We do some mangling of the DN to try to have sensitive name whatever happen # We also include the UID in case the name is not unique, or several DN are associated namePart = 'name' idPart = '' dnSplit = dn.split('CN=') try: namePart = dnSplit[-1].replace("'", " ") except Exception as e: pass try: idPart = dnSplit[2].replace("/", "") except Exception as e: pass idPart += ' uid_%s' % uid name = "Unknown (%s %s)" % (namePart, idPart) # Now, we do try to convert the DN into a DIRAC username if "Unknown" in name: result = getUsernameForDN(dn) if result['OK']: name = result['Value'] uid_name[uid] = name # Now we prepare the SQL statement to insert them into the DFC for uid in uid_name: username = uid_name[uid] queryQueue.put( "INSERT INTO FC_Users(UID, UserName) values (%s, '%s');\n" % (uid, username)) # Now, same principle on the group if fromdbR_.execute("SELECT GID, GROUPNAME from CNS_GROUPINFO"): rows = fromdbR_.fetchall() for row in rows: gid = row[0] gn = row[1] groupname = gn # CAUTION: as explained in the docstring, if multiple groups share the same voms role # we take the first one groups = getGroupsWithVOMSAttribute('/' + gn) if groups: groupname = groups[0] queryQueue.put( "INSERT INTO FC_Groups(GID, GroupName) values (%s, '%s');\n" % (gid, groupname)) # We build a cache that contains the mapping between the name and its ID in the DFC DB # The ID starts at 2 because ID=1 is taken by FakeSe seCounter = 2 # Fetch the name from the LFC if fromdbR_.execute("select unique HOST from CNS_FILE_REPLICA"): rows = fromdbR_.fetchall() for row in rows: seName = row[0] # Populate the SE cache name_id_se[seName] = seCounter # Create the query for the DFC queryQueue.put( "INSERT INTO FC_StorageElements(SEID, SEName) values (%s, '%s');\n" % (seCounter, seName)) seCounter += 1 # Also here we just insert all the statuses defined above for status in dirac_status: queryQueue.put("INSERT INTO FC_Statuses (Status) values ('%s');\n" % status) fromdbR_.close() connection.close() # Set the poison pill in the queue queryQueue.put(None) return
def main(): global verbose global taskQueueID Script.registerSwitch("v", "verbose", "give max details about task queues", setVerbose) Script.registerSwitch("t:", "taskQueue=", "show this task queue only", setTaskQueueID) Script.parseCommandLine(initializeMonitor=False) result = MatcherClient().getActiveTaskQueues() if not result['OK']: gLogger.error(result['Message']) sys.exit(1) tqDict = result['Value'] if not verbose: fields = [ 'TaskQueue', 'Jobs', 'CPUTime', 'Owner', 'OwnerGroup', 'Sites', 'Platforms', 'SubmitPools', 'Setup', 'Priority' ] records = [] for tqId in sorted(tqDict): if taskQueueID and tqId != taskQueueID: continue record = [str(tqId)] tqData = tqDict[tqId] for key in fields[1:]: if key == 'Owner': value = tqData.get('OwnerDN', '-') if value != '-': result = getUsernameForDN(value) if not result['OK']: value = 'Unknown' else: value = result['Value'] else: value = tqData.get(key, '-') if isinstance(value, list): if len(value) > 1: record.append(str(value[0]) + '...') else: record.append(str(value[0])) else: record.append(str(value)) records.append(record) printTable(fields, records) else: fields = ['Key', 'Value'] for tqId in sorted(tqDict): if taskQueueID and tqId != taskQueueID: continue gLogger.notice("\n==> TQ %s" % tqId) records = [] tqData = tqDict[tqId] for key in sorted(tqData): value = tqData[key] if isinstance(value, list): records.append([key, {"Value": value, 'Just': 'L'}]) else: value = str(value) records.append([key, {"Value": value, 'Just': 'L'}]) printTable(fields, records, numbering=False)
if not verbose: fields = ['TaskQueue','Jobs','CPUTime','Owner','OwnerGroup','Sites', 'Platforms','SubmitPools','Setup','Priority'] records = [] print for tqId in sorted( tqDict ): if taskQueueID and tqId != taskQueueID: continue record = [str(tqId)] tqData = tqDict[ tqId ] for key in fields[1:]: if key == 'Owner': value = tqData.get( 'OwnerDN', '-' ) if value != '-': result = getUsernameForDN( value ) if not result['OK']: value = 'Unknown' else: value = result['Value'] else: value = tqData.get( key, '-' ) if type( value ) == types.ListType: if len( value ) > 1: record.append( str( value[0] ) + '...' ) else: record.append( str( value[0] ) ) else: record.append( str( value ) ) records.append( record )
def generateProxyOrToken( self, client, grant_type, user=None, scope=None, expires_in=None, include_refresh_token=True ): """Generate proxy or tokens after authorization :param client: instance of the IdP client :param grant_type: authorization grant type (unused) :param str user: user identificator :param str scope: requested scope :param expires_in: when the token should expire (unused) :param bool include_refresh_token: to include refresh token (unused) :return: dict or str -- will return tokens as dict or proxy as string """ # Read requested scopes group = self._getScope(scope, "g") lifetime = self._getScope(scope, "lifetime") # Found provider name for group provider = getIdPForGroup(group) # Search DIRAC username by user ID result = getUsernameForDN(wrapIDAsDN(user)) if not result["OK"]: raise OAuth2Error(result["Message"]) userName = result["Value"] # User request a proxy if "proxy" in scope_to_list(scope): # Try to return user proxy if proxy scope present in the authorization request if not isDownloadProxyAllowed(): raise OAuth2Error("You can't get proxy, configuration(allowProxyDownload) not allow to do that.") sLog.debug( "Try to query %s@%s proxy%s" % (user, group, ("with lifetime:%s" % lifetime) if lifetime else "") ) # Get user DNs result = getDNForUsername(userName) if not result["OK"]: raise OAuth2Error(result["Message"]) userDNs = result["Value"] err = [] # Try every DN to generate a proxy for dn in userDNs: sLog.debug("Try to get proxy for %s" % dn) params = {} if lifetime: params["requiredTimeLeft"] = int(lifetime) # if the configuration describes adding a VOMS extension, we will do so if getGroupOption(group, "AutoAddVOMS", False): result = self.proxyCli.downloadVOMSProxy(dn, group, **params) else: # otherwise we will return the usual proxy result = self.proxyCli.downloadProxy(dn, group, **params) if not result["OK"]: err.append(result["Message"]) else: sLog.info("Proxy was created.") result = result["Value"].dumpAllToString() if not result["OK"]: raise OAuth2Error(result["Message"]) # Proxy generated return { "proxy": result["Value"].decode() if isinstance(result["Value"], bytes) else result["Value"] } # Proxy cannot be generated or not found raise OAuth2Error("; ".join(err)) # User request a tokens else: # Ask TokenManager to generate new tokens for user result = self.tokenCli.getToken(userName, group) if not result["OK"]: raise OAuth2Error(result["Message"]) token = result["Value"] # Wrap the refresh token and register it to protect against reuse result = self.registerRefreshToken( dict(sub=user, scope=scope, provider=provider, azp=client.get_client_id()), token ) if not result["OK"]: raise OAuth2Error(result["Message"]) # Return tokens as dictionary return result["Value"]
fields = [ 'TaskQueue', 'Jobs', 'CPUTime', 'Owner', 'OwnerGroup', 'Sites', 'Platforms', 'SubmitPools', 'Setup', 'Priority' ] records = [] for tqId in sorted(tqDict): if taskQueueID and tqId != taskQueueID: continue record = [str(tqId)] tqData = tqDict[tqId] for key in fields[1:]: if key == 'Owner': value = tqData.get('OwnerDN', '-') if value != '-': result = getUsernameForDN(value) if not result['OK']: value = 'Unknown' else: value = result['Value'] else: value = tqData.get(key, '-') if isinstance(value, list): if len(value) > 1: record.append(str(value[0]) + '...') else: record.append(str(value[0])) else: record.append(str(value)) records.append(record)
def main(): global verbose global taskQueueID Script.registerSwitch("v", "verbose", "give max details about task queues", setVerbose) Script.registerSwitch("t:", "taskQueue=", "show this task queue only", setTaskQueueID) Script.parseCommandLine(initializeMonitor=False) result = MatcherClient().getActiveTaskQueues() if not result["OK"]: gLogger.error(result["Message"]) sys.exit(1) tqDict = result["Value"] if not verbose: fields = [ "TaskQueue", "Jobs", "CPUTime", "Owner", "OwnerGroup", "Sites", "Platforms", "Priority", ] records = [] for tqId in sorted(tqDict): if taskQueueID and tqId != taskQueueID: continue record = [str(tqId)] tqData = tqDict[tqId] for key in fields[1:]: if key == "Owner": value = tqData.get("OwnerDN", "-") if value != "-": result = getUsernameForDN(value) if not result["OK"]: value = "Unknown" else: value = result["Value"] else: value = tqData.get(key, "-") if isinstance(value, list): if len(value) > 1: record.append(str(value[0]) + "...") else: record.append(str(value[0])) else: record.append(str(value)) records.append(record) printTable(fields, records) else: fields = ["Key", "Value"] for tqId in sorted(tqDict): if taskQueueID and tqId != taskQueueID: continue gLogger.notice("\n==> TQ %s" % tqId) records = [] tqData = tqDict[tqId] for key in sorted(tqData): value = tqData[key] if isinstance(value, list): records.append([key, {"Value": value, "Just": "L"}]) else: value = str(value) records.append([key, {"Value": value, "Just": "L"}]) printTable(fields, records, numbering=False)