def __addPilotsAccountingReport(self, pilotsData): """ fill accounting data """ for pRef in pilotsData: pData = pilotsData[pRef] pA = PilotAccounting() pA.setEndTime(pData['LastUpdateTime']) pA.setStartTime(pData['SubmissionTime']) retVal = CS.getUsernameForDN(pData['OwnerDN']) if not retVal['OK']: userName = '******' self.log.error("Can't determine username for dn:", pData['OwnerDN']) else: userName = retVal['Value'] pA.setValueByKey('User', userName) pA.setValueByKey('UserGroup', pData['OwnerGroup']) result = getSiteForCE(pData['DestinationSite']) if result['OK'] and result['Value'].strip(): pA.setValueByKey('Site', result['Value'].strip()) else: pA.setValueByKey('Site', 'Unknown') pA.setValueByKey('GridCE', pData['DestinationSite']) pA.setValueByKey('GridMiddleware', pData['GridType']) pA.setValueByKey('GridResourceBroker', pData['Broker']) pA.setValueByKey('GridStatus', pData['Status']) if 'Jobs' not in pData: pA.setValueByKey('Jobs', 0) else: pA.setValueByKey('Jobs', len(pData['Jobs'])) self.log.verbose("Added accounting record for pilot %s" % pData['PilotID']) retVal = gDataStoreClient.addRegister(pA) if not retVal['OK']: return retVal return S_OK()
def __addPilotsAccountingReport( self, pilotsData ): """ fill accounting data """ for pRef in pilotsData: pData = pilotsData[pRef] pA = PilotAccounting() pA.setEndTime( pData[ 'LastUpdateTime' ] ) pA.setStartTime( pData[ 'SubmissionTime' ] ) retVal = CS.getUsernameForDN( pData[ 'OwnerDN' ] ) if not retVal[ 'OK' ]: userName = '******' self.log.error( "Can't determine username for dn:", pData[ 'OwnerDN' ] ) else: userName = retVal[ 'Value' ] pA.setValueByKey( 'User', userName ) pA.setValueByKey( 'UserGroup', pData[ 'OwnerGroup' ] ) result = getSiteForCE( pData[ 'DestinationSite' ] ) if result['OK'] and result[ 'Value' ].strip(): pA.setValueByKey( 'Site', result['Value'].strip() ) else: pA.setValueByKey( 'Site', 'Unknown' ) pA.setValueByKey( 'GridCE', pData[ 'DestinationSite' ] ) pA.setValueByKey( 'GridMiddleware', pData[ 'GridType' ] ) pA.setValueByKey( 'GridResourceBroker', pData[ 'Broker' ] ) pA.setValueByKey( 'GridStatus', pData[ 'Status' ] ) if not 'Jobs' in pData: pA.setValueByKey( 'Jobs', 0 ) else: pA.setValueByKey( 'Jobs', len( pData['Jobs'] ) ) self.log.verbose( "Added accounting record for pilot %s" % pData[ 'PilotID' ] ) retVal = gDataStoreClient.addRegister( pA ) if not retVal[ 'OK' ]: return retVal return S_OK()
def __addPilotsAccountingReport(self, pilotsData): """ fill accounting data """ for pRef in pilotsData: pData = pilotsData[pRef] pA = PilotAccounting() pA.setEndTime(pData["LastUpdateTime"]) pA.setStartTime(pData["SubmissionTime"]) retVal = CS.getUsernameForDN(pData["OwnerDN"]) if not retVal["OK"]: userName = "******" self.log.error("Can't determine username for dn:", pData["OwnerDN"]) else: userName = retVal["Value"] pA.setValueByKey("User", userName) pA.setValueByKey("UserGroup", pData["OwnerGroup"]) result = getSiteForCE(pData["DestinationSite"]) if result["OK"] and result["Value"].strip(): pA.setValueByKey("Site", result["Value"].strip()) else: pA.setValueByKey("Site", "Unknown") pA.setValueByKey("GridCE", pData["DestinationSite"]) pA.setValueByKey("GridMiddleware", pData["GridType"]) pA.setValueByKey("GridResourceBroker", pData["Broker"]) pA.setValueByKey("GridStatus", pData["Status"]) if not "Jobs" in pData: pA.setValueByKey("Jobs", 0) else: pA.setValueByKey("Jobs", len(pData["Jobs"])) self.log.verbose("Added accounting record for pilot %s" % pData["PilotID"]) retVal = gDataStoreClient.addRegister(pA) if not retVal["OK"]: return retVal return S_OK()
def setPilotStatus( self, pilotRef, status, destination=None, statusReason=None, gridSite=None, queue=None, updateTime=None, conn = False ): """ Set pilot job LCG status """ setList = [] setList.append("Status='%s'" % status) if updateTime: setList.append("LastUpdateTime='%s'" % updateTime) else: setList.append("LastUpdateTime=UTC_TIMESTAMP()") if not statusReason: statusReason = "Not given" setList.append("StatusReason='%s'" % statusReason) if gridSite: setList.append("GridSite='%s'" % gridSite) if queue: setList.append("Queue='%s'" % queue) if destination: setList.append("DestinationSite='%s'" % destination) if not gridSite: result = getSiteForCE(destination) if result['OK']: gridSite = result['Value'] setList.append("GridSite='%s'" % gridSite) set_string = ','.join(setList) req = "UPDATE PilotAgents SET "+set_string+" WHERE PilotJobReference='%s'" % pilotRef result = self._update( req, conn = conn ) if not result['OK']: return result return S_OK()
def run(self): ''' Checks it has the parameters it needs and tries to send an sms to the users that apply. ''' if self.url is None: return S_ERROR('Slack URL not set') # Minor security checks element = self.decisionParams['element'] if element is None: return S_ERROR('element should not be None') name = self.decisionParams['name'] if name is None: return S_ERROR('name should not be None') statusType = self.decisionParams['statusType'] if statusType is None: return S_ERROR('statusType should not be None') previousStatus = self.decisionParams['status'] if previousStatus is None: return S_ERROR('status should not be None') status = self.enforcementResult['Status'] if status is None: return S_ERROR('status should not be None') reason = self.enforcementResult['Reason'] if reason is None: return S_ERROR('reason should not be None') if self.decisionParams['element'] == 'Site': siteName = self.decisionParams['name'] else: elementType = self.decisionParams['elementType'] if elementType == 'StorageElement': siteName = getSitesForSE(name) elif elementType == 'ComputingElement': siteName = getSiteForCE(name) else: siteName = {'OK': True, 'Value': 'Unassigned'} if not siteName['OK']: self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message'])) siteName = "Unassigned Resources" elif not siteName['Value']: siteName = "Unassigned Resources" else: siteName = siteName['Value'] if isinstance(siteName['Value'], basestring) else siteName['Value'][0] message = "*{name}* _{statusType}_ --> _{status}_ \n{reason}".format(name=name, statusType=statusType, status=status, reason=reason) return self.sendSlackMessage(message)
def sendPilotAccounting(self, pilotDict): """ Send pilot accounting record """ for pRef in pilotDict: self.log.verbose('Preparing accounting record for pilot %s' % pRef) pA = PilotAccounting() pA.setEndTime(pilotDict[pRef]['LastUpdateTime']) pA.setStartTime(pilotDict[pRef]['SubmissionTime']) retVal = CS.getUsernameForDN(pilotDict[pRef]['OwnerDN']) if not retVal['OK']: userName = '******' self.log.error("Can't determine username for dn:", pilotDict[pRef]['OwnerDN']) else: userName = retVal['Value'] pA.setValueByKey('User', userName) pA.setValueByKey('UserGroup', pilotDict[pRef]['OwnerGroup']) result = getSiteForCE(pilotDict[pRef]['DestinationSite']) if result['OK'] and result['Value'].strip(): pA.setValueByKey('Site', result['Value'].strip()) else: pA.setValueByKey('Site', 'Unknown') pA.setValueByKey('GridCE', pilotDict[pRef]['DestinationSite']) pA.setValueByKey('GridMiddleware', pilotDict[pRef]['GridType']) pA.setValueByKey('GridResourceBroker', pilotDict[pRef]['Broker']) pA.setValueByKey('GridStatus', pilotDict[pRef]['Status']) if not 'Jobs' in pilotDict[pRef]: pA.setValueByKey('Jobs', 0) else: pA.setValueByKey('Jobs', len(pilotDict[pRef]['Jobs'])) self.log.info("Adding accounting record for pilot %s" % pilotDict[pRef]['PilotID']) retVal = gDataStoreClient.addRegister(pA) if not retVal['OK']: self.log.error('Failed to send accounting info for pilot ', pRef) else: # Set up AccountingSent flag result = pilotAgentsDB.setAccountingFlag(pRef) if not result['OK']: self.log.error('Failed to set accounting flag for pilot ', pRef) self.log.info('Committing accounting records for %d pilots' % len(pilotDict)) result = gDataStoreClient.commit() if result['OK']: for pRef in pilotDict: self.log.verbose('Setting AccountingSent flag for pilot %s' % pRef) result = pilotAgentsDB.setAccountingFlag(pRef) if not result['OK']: self.log.error('Failed to set accounting flag for pilot ', pRef) else: return result return S_OK()
def run(self): ''' Checks it has the parameters it needs and writes the date to a cache file. ''' # Minor security checks element = self.decisionParams['element'] if element is None: return S_ERROR('element should not be None') name = self.decisionParams['name'] if name is None: return S_ERROR('name should not be None') statusType = self.decisionParams['statusType'] if statusType is None: return S_ERROR('statusType should not be None') previousStatus = self.decisionParams['status'] if previousStatus is None: return S_ERROR('status should not be None') status = self.enforcementResult['Status'] if status is None: return S_ERROR('status should not be None') reason = self.enforcementResult['Reason'] if reason is None: return S_ERROR('reason should not be None') if self.decisionParams['element'] == 'Site': siteName = self.decisionParams['name'] else: elementType = self.decisionParams['elementType'] if elementType == 'StorageElement': siteName = getSitesForSE(name) elif elementType == 'ComputingElement': siteName = getSiteForCE(name) else: siteName = {'OK': True, 'Value': 'Unassigned'} if not siteName['OK']: self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message'])) siteName = "Unassigned Resources" elif not siteName['Value']: siteName = "Unassigned Resources" else: siteName = siteName['Value'] if isinstance(siteName['Value'], basestring) else siteName['Value'][0] # create record for insertion recordDict = {} recordDict['SiteName'] = siteName recordDict['ResourceName'] = name recordDict['Status'] = status recordDict['PreviousStatus'] = previousStatus recordDict['StatusType'] = statusType return self.rsClient.insert('ResourceStatusCache', recordDict)
def __getXRSLExtraString(self, multiprocessor=False): # For the XRSL additional string from configuration - only done at initialisation time # If this string changes, the corresponding (ARC) site directors have to be restarted # # Variable = XRSLExtraString (or XRSLMPExtraString for multi processor mode) # Default value = '' # If you give a value, I think it should be of the form # (aaa = "xxx") # Otherwise the ARC job description parser will have a fit # Locations searched in order : # Top priority : Resources/Sites/<Grid>/<Site>/CEs/<CE>/XRSLExtraString # Second priority : Resources/Sites/<Grid>/<Site>/XRSLExtraString # Default : Resources/Computing/CEDefaults/XRSLExtraString # xrslExtraString = '' # Start with the default value result = getSiteForCE(self.ceHost) self.site = '' if result['OK']: self.site = result['Value'] else: gLogger.error("Unknown Site ...") return # Now we know the site. Get the grid grid = self.site.split(".")[0] # The different possibilities that we have agreed upon if multiprocessor: xtraVariable = "XRSLMPExtraString" else: xtraVariable = "XRSLExtraString" firstOption = "Resources/Sites/%s/%s/CEs/%s/%s" % (grid, self.site, self.ceHost, xtraVariable) secondOption = "Resources/Sites/%s/%s/%s" % (grid, self.site, xtraVariable) defaultOption = "Resources/Computing/CEDefaults/%s" % xtraVariable # Now go about getting the string in the agreed order gLogger.debug("Trying to get %s : first option %s" % (xtraVariable, firstOption)) result = gConfig.getValue(firstOption, defaultValue='') if result != '': xrslExtraString = result gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString)) else: gLogger.debug("Trying to get %s : second option %s" % (xtraVariable, secondOption)) result = gConfig.getValue(secondOption, defaultValue='') if result != '': xrslExtraString = result gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString)) else: gLogger.debug("Trying to get %s : default option %s" % (xtraVariable, defaultOption)) result = gConfig.getValue(defaultOption, defaultValue='') if result != '': xrslExtraString = result gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString)) if xrslExtraString == '': gLogger.always("No %s found in configuration for %s" % (xtraVariable, self.ceHost)) else: gLogger.always("%s : %s" % (xtraVariable, xrslExtraString)) gLogger.always(" --- to be added to pilots going to CE : %s" % self.ceHost) return xrslExtraString
def __getXRSLExtraString(self, multiprocessor=False): # For the XRSL additional string from configuration - only done at initialisation time # If this string changes, the corresponding (ARC) site directors have to be restarted # # Variable = XRSLExtraString (or for multi processor mode) # Default value = '' # If you give a value, I think it should be of the form # (aaa = "xxx") # Otherwise the ARC job description parser will have a fit # Locations searched in order : # Top priority : Resources/Sites/<Grid>/<Site>/CEs/<CE>/XRSLExtraString # Second priority : Resources/Sites/<Grid>/<Site>/XRSLExtraString # Default : Resources/Computing/CEDefaults/XRSLExtraString # xrslExtraString = '' # Start with the default value result = getSiteForCE(self.ceHost) self.site = '' if result['OK']: self.site = result['Value'] else: gLogger.error("Unknown Site ...") return # Now we know the site. Get the grid grid = self.site.split(".")[0] # The different possibilities that we have agreed upon if multiprocessor: xtraVariable = "XRSLMPExtraString" else: xtraVariable = "XRSLExtraString" firstOption = "Resources/Sites/%s/%s/CEs/%s/%s" % (grid, self.site, self.ceHost, xtraVariable) secondOption = "Resources/Sites/%s/%s/%s" % (grid, self.site, xtraVariable) defaultOption = "Resources/Computing/CEDefaults/%s" % xtraVariable # Now go about getting the string in the agreed order gLogger.debug("Trying to get %s : first option %s" % (xtraVariable, firstOption)) result = gConfig.getValue(firstOption, defaultValue='') if result != '': xrslExtraString = result gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString)) else: gLogger.debug("Trying to get %s : second option %s" % (xtraVariable, secondOption)) result = gConfig.getValue(secondOption, defaultValue='') if result != '': xrslExtraString = result gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString)) else: gLogger.debug("Trying to get %s : default option %s" % (xtraVariable, defaultOption)) result = gConfig.getValue(defaultOption, defaultValue='') if result != '': xrslExtraString = result gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString)) if xrslExtraString: gLogger.always("%s : %s" % (xtraVariable, xrslExtraString)) gLogger.always(" --- to be added to pilots going to CE : %s" % self.ceHost) return xrslExtraString
def sendPilotAccounting( self, pilotDict ): """ Send pilot accounting record """ for pRef in pilotDict: self.log.verbose( 'Preparing accounting record for pilot %s' % pRef ) pA = PilotAccounting() pA.setEndTime( pilotDict[pRef][ 'LastUpdateTime' ] ) pA.setStartTime( pilotDict[pRef][ 'SubmissionTime' ] ) retVal = CS.getUsernameForDN( pilotDict[pRef][ 'OwnerDN' ] ) if not retVal[ 'OK' ]: userName = '******' self.log.error( "Can't determine username for dn:", pilotDict[pRef][ 'OwnerDN' ] ) else: userName = retVal[ 'Value' ] pA.setValueByKey( 'User', userName ) pA.setValueByKey( 'UserGroup', pilotDict[pRef][ 'OwnerGroup' ] ) result = getSiteForCE( pilotDict[pRef][ 'DestinationSite' ] ) if result['OK'] and result[ 'Value' ].strip(): pA.setValueByKey( 'Site', result['Value'].strip() ) else: pA.setValueByKey( 'Site', 'Unknown' ) pA.setValueByKey( 'GridCE', pilotDict[pRef][ 'DestinationSite' ] ) pA.setValueByKey( 'GridMiddleware', pilotDict[pRef][ 'GridType' ] ) pA.setValueByKey( 'GridResourceBroker', pilotDict[pRef][ 'Broker' ] ) pA.setValueByKey( 'GridStatus', pilotDict[pRef][ 'Status' ] ) if not 'Jobs' in pilotDict[pRef]: pA.setValueByKey( 'Jobs', 0 ) else: pA.setValueByKey( 'Jobs', len( pilotDict[pRef]['Jobs'] ) ) self.log.verbose( "Adding accounting record for pilot %s" % pilotDict[pRef][ 'PilotID' ] ) retVal = gDataStoreClient.addRegister( pA ) if not retVal[ 'OK' ]: self.log.error( 'Failed to send accounting info for pilot ', pRef ) else: # Set up AccountingSent flag result = pilotAgentsDB.setAccountingFlag( pRef ) if not result['OK']: self.log.error( 'Failed to set accounting flag for pilot ', pRef ) self.log.info( 'Committing accounting records for %d pilots' % len( pilotDict ) ) result = gDataStoreClient.commit() if result['OK']: for pRef in pilotDict: self.log.verbose( 'Setting AccountingSent flag for pilot %s' % pRef ) result = pilotAgentsDB.setAccountingFlag( pRef ) if not result['OK']: self.log.error( 'Failed to set accounting flag for pilot ', pRef ) else: return result return S_OK()
def sendPilotAccounting(self, pilotDict): """ Send pilot accounting record """ for pRef in pilotDict: self.log.verbose("Preparing accounting record for pilot %s" % pRef) pA = PilotAccounting() pA.setEndTime(pilotDict[pRef]["LastUpdateTime"]) pA.setStartTime(pilotDict[pRef]["SubmissionTime"]) retVal = CS.getUsernameForDN(pilotDict[pRef]["OwnerDN"]) if not retVal["OK"]: userName = "******" self.log.error("Can't determine username for dn:", pilotDict[pRef]["OwnerDN"]) else: userName = retVal["Value"] pA.setValueByKey("User", userName) pA.setValueByKey("UserGroup", pilotDict[pRef]["OwnerGroup"]) result = getSiteForCE(pilotDict[pRef]["DestinationSite"]) if result["OK"] and result["Value"].strip(): pA.setValueByKey("Site", result["Value"].strip()) else: pA.setValueByKey("Site", "Unknown") pA.setValueByKey("GridCE", pilotDict[pRef]["DestinationSite"]) pA.setValueByKey("GridMiddleware", pilotDict[pRef]["GridType"]) pA.setValueByKey("GridResourceBroker", pilotDict[pRef]["Broker"]) pA.setValueByKey("GridStatus", pilotDict[pRef]["Status"]) if not "Jobs" in pilotDict[pRef]: pA.setValueByKey("Jobs", 0) else: pA.setValueByKey("Jobs", len(pilotDict[pRef]["Jobs"])) self.log.info("Adding accounting record for pilot %s" % pilotDict[pRef]["PilotID"]) retVal = gDataStoreClient.addRegister(pA) if not retVal["OK"]: self.log.error("Failed to send accounting info for pilot ", pRef) else: # Set up AccountingSent flag result = pilotAgentsDB.setAccountingFlag(pRef) if not result["OK"]: self.log.error("Failed to set accounting flag for pilot ", pRef) self.log.info("Committing accounting records for %d pilots" % len(pilotDict)) result = gDataStoreClient.commit() if result["OK"]: for pRef in pilotDict: self.log.verbose("Setting AccountingSent flag for pilot %s" % pRef) result = pilotAgentsDB.setAccountingFlag(pRef) if not result["OK"]: self.log.error("Failed to set accounting flag for pilot ", pRef) else: return result return S_OK()
def setDestinationCE( self, ceName ): """ Developer function. Allows to direct a job to a particular Grid CE. """ kwargs = {'ceName':ceName} diracSite = getSiteForCE( ceName ) if not diracSite['OK']: return self._reportError( diracSite['Message'], **kwargs ) if not diracSite['Value']: return self._reportError( 'No DIRAC site name found for CE %s' % ( ceName ), **kwargs ) diracSite = diracSite['Value'] self.setDestination( diracSite ) self._addJDLParameter( 'GridRequiredCEs', ceName ) return S_OK()
def setPilotDestinationSite(self,pilotRef,destination, conn=False): """ Set the pilot agent destination site """ gridSite = 'Unknown' result = getSiteForCE(destination) if result['OK']: gridSite = result['Value'] if not gridSite: gridSite = 'Unknown' req = "UPDATE PilotAgents SET DestinationSite='%s', GridSite='%s' WHERE PilotJobReference='%s'" req = req % (destination,gridSite,pilotRef) result = self._update(req, conn = conn) return result
def setPilotDestinationSite(self, pilotRef, destination, conn=False): """ Set the pilot agent destination site """ gridSite = 'Unknown' result = getSiteForCE(destination) if result['OK']: gridSite = result['Value'] if not gridSite: gridSite = 'Unknown' req = "UPDATE PilotAgents SET DestinationSite='%s', GridSite='%s' WHERE PilotJobReference='%s'" req = req % (destination, gridSite, pilotRef) result = self._update(req, conn=conn) return result
def setDestinationCE( self, ceName ): """ Developer function. Allows to direct a job to a particular Grid CE. """ kwargs = {'ceName':ceName} diracSite = getSiteForCE( ceName ) if not diracSite['OK']: return self._reportError( diracSite['Message'], **kwargs ) if not diracSite['Value']: return self._reportError( 'No DIRAC site name found for CE %s' % ( ceName ), **kwargs ) diracSite = diracSite['Value'] self.setDestination( diracSite ) self._addJDLParameter( 'GridRequiredCEs', ceName ) return S_OK()
def setPilotStatus(self, pilotRef, status, destination=None, statusReason=None, gridSite=None, queue=None, benchmark=None, currentJob=None, updateTime=None, conn=False): """ Set pilot job status """ setList = [] setList.append("Status='%s'" % status) if updateTime: setList.append("LastUpdateTime='%s'" % updateTime) else: setList.append("LastUpdateTime=UTC_TIMESTAMP()") if not statusReason: statusReason = "Not given" setList.append("StatusReason='%s'" % statusReason) if gridSite: setList.append("GridSite='%s'" % gridSite) if queue: setList.append("Queue='%s'" % queue) if benchmark: setList.append("BenchMark='%s'" % float(benchmark)) if currentJob: setList.append("CurrentJobID='%s'" % int(currentJob)) if destination: setList.append("DestinationSite='%s'" % destination) if not gridSite: result = getSiteForCE(destination) if result['OK']: gridSite = result['Value'] setList.append("GridSite='%s'" % gridSite) set_string = ','.join(setList) req = "UPDATE PilotAgents SET " + set_string + " WHERE PilotJobReference='%s'" % pilotRef result = self._update(req, conn=conn) if not result['OK']: return result return S_OK()
def setDestinationCE(self, ceName, diracSite=None): """ Developer function. Allows to direct a job to a particular Grid CE. """ kwargs = {"ceName": ceName} if not diracSite: diracSite = getSiteForCE(ceName) if not diracSite["OK"]: return self._reportError(diracSite["Message"], **kwargs) if not diracSite["Value"]: return self._reportError("No DIRAC site name found for CE %s" % (ceName), **kwargs) diracSite = diracSite["Value"] self.setDestination(diracSite) # Keep GridRequiredCEs for backward compatibility self._addJDLParameter("GridRequiredCEs", ceName) self._addJDLParameter("GridCE", ceName) return S_OK()
def __purgeSites(self, ceBdiiDict): """Remove all sites that are not in self.selectedSites. Modifies the ceBdiiDict! """ if not self.selectedSites: return for site in list(ceBdiiDict): ces = list(ceBdiiDict[site]['CEs']) if not ces: self.log.error("No CE information for site:", site) continue diracSiteName = getSiteForCE(ces[0]) if not diracSiteName['OK']: self.log.error("Failed to get DIRAC site name for ce", "%s: %s" % (ces[0], diracSiteName['Message'])) continue self.log.debug("Checking site %s (%s), aka %s" % (site, ces, diracSiteName['Value'])) if diracSiteName['Value'] in self.selectedSites: continue self.log.info("Dropping site %s, aka %s" % (site, diracSiteName)) ceBdiiDict.pop(site) return
def __purgeSites(self, ceBdiiDict): """Remove all sites that are not in self.selectedSites. Modifies the ceBdiiDict! """ if not self.selectedSites: return for site in list(ceBdiiDict): ces = list(ceBdiiDict[site]['CEs']) if not ces: self.log.error("No CE information for site:", site) continue diracSiteName = getSiteForCE(ces[0]) if not diracSiteName['OK']: self.log.error("Failed to get DIRAC site name for ce", "%s: %s" % (ces[0], diracSiteName['Message'])) continue self.log.debug("Checking site %s (%s), aka %s" % (site, ces, diracSiteName['Value'])) if diracSiteName['Value'] in self.selectedSites: continue self.log.info("Dropping site %s, aka %s" % (site, diracSiteName)) ceBdiiDict.pop(site) return
def run(self): ''' Checks it has the parameters it needs and writes the date to a cache file. ''' # Minor security checks element = self.decisionParams['element'] if element is None: return S_ERROR('element should not be None') name = self.decisionParams['name'] if name is None: return S_ERROR('name should not be None') statusType = self.decisionParams['statusType'] if statusType is None: return S_ERROR('statusType should not be None') previousStatus = self.decisionParams['status'] if previousStatus is None: return S_ERROR('status should not be None') status = self.enforcementResult['Status'] if status is None: return S_ERROR('status should not be None') reason = self.enforcementResult['Reason'] if reason is None: return S_ERROR('reason should not be None') if self.decisionParams['element'] == 'Site': siteName = self.decisionParams['name'] else: elementType = self.decisionParams['elementType'] if elementType == 'StorageElement': siteName = getSitesForSE(name) elif elementType == 'ComputingElement': siteName = getSiteForCE(name) else: siteName = {'OK': True, 'Value': 'Unassigned'} if not siteName['OK']: self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message'])) siteName = "Unassigned Resources" elif not siteName['Value']: siteName = "Unassigned Resources" else: siteName = siteName['Value'][0] with sqlite3.connect(self.cacheFile) as conn: try: conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache( SiteName VARCHAR(64) NOT NULL, ResourceName VARCHAR(64) NOT NULL, Status VARCHAR(8) NOT NULL DEFAULT "", PreviousStatus VARCHAR(8) NOT NULL DEFAULT "", StatusType VARCHAR(128) NOT NULL DEFAULT "all", Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP );''') insertQuery = "INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)" insertQuery += " VALUES ('%s', '%s', '%s', '%s', '%s' ); " % (siteName, name, status, previousStatus, statusType) conn.execute(insertQuery) conn.commit() except sqlite3.OperationalError: self.log.error('Email cache database is locked') return S_OK()
def run(self): ''' Checks it has the parameters it needs and writes the date to a cache file. ''' # Minor security checks element = self.decisionParams['element'] if element is None: return S_ERROR('element should not be None') name = self.decisionParams['name'] if name is None: return S_ERROR('name should not be None') statusType = self.decisionParams['statusType'] if statusType is None: return S_ERROR('statusType should not be None') previousStatus = self.decisionParams['status'] if previousStatus is None: return S_ERROR('status should not be None') status = self.enforcementResult['Status'] if status is None: return S_ERROR('status should not be None') reason = self.enforcementResult['Reason'] if reason is None: return S_ERROR('reason should not be None') if self.decisionParams['element'] == 'Site': siteName = self.decisionParams['name'] else: elementType = self.decisionParams['elementType'] if elementType == 'StorageElement': siteName = getSitesForSE(name) elif elementType == 'ComputingElement': siteName = getSiteForCE(name) else: siteName = {'OK': True, 'Value': 'Unassigned'} if not siteName['OK']: self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message'])) siteName = "Unassigned Resources" elif not siteName['Value']: siteName = "Unassigned Resources" else: siteName = siteName['Value'][0] with sqlite3.connect(self.cacheFile) as conn: try: conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache( SiteName VARCHAR(64) NOT NULL, ResourceName VARCHAR(64) NOT NULL, Status VARCHAR(8) NOT NULL DEFAULT "", PreviousStatus VARCHAR(8) NOT NULL DEFAULT "", StatusType VARCHAR(128) NOT NULL DEFAULT "all", Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP );''') except sqlite3.OperationalError: self.log.error('Email cache database is locked') conn.execute( "INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)" " VALUES ('" + siteName + "', '" + name + "', '" + status + "', '" + previousStatus + "', '" + statusType + "' ); ") conn.commit() return S_OK()
def getPilotMonitorWeb(self,selectDict,sortList,startItem,maxItems): """ Get summary of the pilot job information in a standard structure """ resultDict = {} last_update = None if selectDict.has_key('LastUpdateTime'): last_update = selectDict['LastUpdateTime'] del selectDict['LastUpdateTime'] if selectDict.has_key('Owner'): userList = selectDict['Owner'] if type(userList) != type([]): userList = [userList] dnList = [] for uName in userList: uList = getDNForUsername(uName)['Value'] dnList += uList selectDict['OwnerDN'] = dnList del selectDict['Owner'] startDate = selectDict.get('FromDate',None) if startDate: del selectDict['FromDate'] # For backward compatibility if startDate is None: startDate = selectDict.get('LastUpdateTime',None) if startDate: del selectDict['LastUpdateTime'] endDate = selectDict.get('ToDate',None) if endDate: del selectDict['ToDate'] # Sorting instructions. Only one for the moment. if sortList: orderAttribute = sortList[0][0]+":"+sortList[0][1] else: orderAttribute = None # Select pilots for the summary result = self.selectPilots(selectDict, orderAttribute=orderAttribute, newer=startDate, older=endDate, timeStamp='LastUpdateTime' ) if not result['OK']: return S_ERROR('Failed to select pilots: '+result['Message']) pList = result['Value'] nPilots = len(pList) resultDict['TotalRecords'] = nPilots if nPilots == 0: return S_OK(resultDict) ini = startItem last = ini + maxItems if ini >= nPilots: return S_ERROR('Item number out of range') if last > nPilots: last = nPilots pilotList = pList[ini:last] paramNames = ['PilotJobReference','OwnerDN','OwnerGroup','GridType','Broker', 'Status','DestinationSite','BenchMark','ParentID', 'SubmissionTime', 'PilotID', 'LastUpdateTime','CurrentJobID','TaskQueueID', 'GridSite'] result = self.getPilotInfo(pilotList,paramNames=paramNames) if not result['OK']: return S_ERROR('Failed to get pilot info: '+result['Message']) pilotDict = result['Value'] records = [] for pilot in pilotList: parList = [] for parameter in paramNames: if type(pilotDict[pilot][parameter]) not in [IntType,LongType]: parList.append(str(pilotDict[pilot][parameter])) else: parList.append(pilotDict[pilot][parameter]) if parameter=='GridSite': gridSite = pilotDict[pilot][parameter] # If the Grid Site is unknown try to recover it in the last moment if gridSite == "Unknown": ce = pilotDict[pilot]['DestinationSite'] result = getSiteForCE(ce) if result['OK']: gridSite = result['Value'] del parList[-1] parList.append(gridSite) records.append(parList) resultDict['ParameterNames'] = paramNames resultDict['Records'] = records return S_OK(resultDict)
def getPilotMonitorWeb(self, selectDict, sortList, startItem, maxItems): """ Get summary of the pilot job information in a standard structure """ resultDict = {} if 'LastUpdateTime' in selectDict: del selectDict['LastUpdateTime'] if 'Owner' in selectDict: userList = selectDict['Owner'] if not isinstance(userList, type([])): userList = [userList] dnList = [] for uName in userList: uList = getDNForUsername(uName)['Value'] dnList += uList selectDict['OwnerDN'] = dnList del selectDict['Owner'] startDate = selectDict.get('FromDate', None) if startDate: del selectDict['FromDate'] # For backward compatibility if startDate is None: startDate = selectDict.get('LastUpdateTime', None) if startDate: del selectDict['LastUpdateTime'] endDate = selectDict.get('ToDate', None) if endDate: del selectDict['ToDate'] # Sorting instructions. Only one for the moment. if sortList: orderAttribute = sortList[0][0] + ":" + sortList[0][1] else: orderAttribute = None # Select pilots for the summary result = self.selectPilots(selectDict, orderAttribute=orderAttribute, newer=startDate, older=endDate, timeStamp='LastUpdateTime') if not result['OK']: return S_ERROR('Failed to select pilots: ' + result['Message']) pList = result['Value'] nPilots = len(pList) resultDict['TotalRecords'] = nPilots if nPilots == 0: return S_OK(resultDict) ini = startItem last = ini + maxItems if ini >= nPilots: return S_ERROR('Item number out of range') if last > nPilots: last = nPilots pilotList = pList[ini:last] paramNames = [ 'PilotJobReference', 'OwnerDN', 'OwnerGroup', 'GridType', 'Broker', 'Status', 'DestinationSite', 'BenchMark', 'ParentID', 'SubmissionTime', 'PilotID', 'LastUpdateTime', 'CurrentJobID', 'TaskQueueID', 'GridSite' ] result = self.getPilotInfo(pilotList, paramNames=paramNames) if not result['OK']: return S_ERROR('Failed to get pilot info: ' + result['Message']) pilotDict = result['Value'] records = [] for pilot in pilotList: parList = [] for parameter in paramNames: if not isinstance(pilotDict[pilot][parameter], (int, long)): parList.append(str(pilotDict[pilot][parameter])) else: parList.append(pilotDict[pilot][parameter]) if parameter == 'GridSite': gridSite = pilotDict[pilot][parameter] # If the Grid Site is unknown try to recover it in the last moment if gridSite == "Unknown": ce = pilotDict[pilot]['DestinationSite'] result = getSiteForCE(ce) if result['OK']: gridSite = result['Value'] del parList[-1] parList.append(gridSite) records.append(parList) resultDict['ParameterNames'] = paramNames resultDict['Records'] = records return S_OK(resultDict)