class PilotCommand(Command): """ Pilot "master" Command. """ def __init__(self, args=None, clients=None): super(PilotCommand, self).__init__(args, clients) if 'WMSAdministrator' in self.apis: self.wmsAdmin = self.apis['WMSAdministrator'] else: self.wmsAdmin = WMSAdministratorClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient() def _storeCommand(self, result): """ Stores the results of doNew method on the database. """ for pilotDict in result: resQuery = self.rmClient.addOrModifyPilotCache( pilotDict['Site'], pilotDict['CE'], pilotDict['PilotsPerJob'], pilotDict['PilotJobEff'], pilotDict['Status']) if not resQuery['OK']: return resQuery return S_OK() def _prepareCommand(self): """ JobCommand requires one arguments: - name : <str> """ if 'name' not in self.args: return S_ERROR('"name" not found in self.args') name = self.args['name'] if 'element' not in self.args: return S_ERROR('element is missing') element = self.args['element'] if element not in ['Site', 'Resource']: return S_ERROR('"%s" is not Site nor Resource' % element) return S_OK((element, name)) def doNew(self, masterParams=None): if masterParams is not None: element, name = masterParams else: params = self._prepareCommand() if not params['OK']: return params element, name = params['Value'] wmsDict = {} if element == 'Site': wmsDict = {'GridSite': name} elif element == 'Resource': wmsDict = {'ExpandSite': name} else: # You should never see this error return S_ERROR('"%s" is not Site nor Resource' % element) wmsResults = self.wmsAdmin.getPilotSummaryWeb(wmsDict, [], 0, 0) if not wmsResults['OK']: return wmsResults wmsResults = wmsResults['Value'] if 'ParameterNames' not in wmsResults: return S_ERROR('Wrong result dictionary, missing "ParameterNames"') params = wmsResults['ParameterNames'] if 'Records' not in wmsResults: return S_ERROR('Wrong formed result dictionary, missing "Records"') records = wmsResults['Records'] uniformResult = [] for record in records: # This returns a dictionary with the following keys: # 'Site', 'CE', 'Submitted', 'Ready', 'Scheduled', 'Waiting', 'Running', # 'Done', 'Aborted', 'Done_Empty', 'Aborted_Hour', 'Total', 'PilotsPerJob', # 'PilotJobEff', 'Status', 'InMask' pilotDict = dict(zip(params, record)) pilotDict['PilotsPerJob'] = float(pilotDict['PilotsPerJob']) pilotDict['PilotJobEff'] = float(pilotDict['PilotJobEff']) uniformResult.append(pilotDict) storeRes = self._storeCommand(uniformResult) if not storeRes['OK']: return storeRes return S_OK(uniformResult) def doCache(self): params = self._prepareCommand() if not params['OK']: return params element, name = params['Value'] if element == 'Site': # WMS returns Site entries with CE = 'Multiple' site, ce = name, 'Multiple' elif element == 'Resource': site, ce = None, name else: # You should never see this error return S_ERROR('"%s" is not Site nor Resource' % element) result = self.rmClient.selectPilotCache(site, ce) if result['OK']: result = S_OK( [dict(zip(result['Columns'], res)) for res in result['Value']]) return result def doMaster(self): siteNames = getSites() if not siteNames['OK']: return siteNames siteNames = siteNames['Value'] ces = CSHelpers.getComputingElements() if not ces['OK']: return ces ces = ces['Value'] pilotResults = self.doNew(('Site', siteNames)) if not pilotResults['OK']: self.metrics['failed'].append(pilotResults['Message']) pilotResults = self.doNew(('Resource', ces)) if not pilotResults['OK']: self.metrics['failed'].append(pilotResults['Message']) return S_OK(self.metrics)
class PilotCommand( Command ): """ Pilot "master" Command. """ def __init__( self, args = None, clients = None ): super( PilotCommand, self ).__init__( args, clients ) if 'WMSAdministrator' in self.apis: self.wmsAdmin = self.apis[ 'WMSAdministrator' ] else: self.wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient() def _storeCommand( self, result ): """ Stores the results of doNew method on the database. """ for pilotDict in result: resQuery = self.rmClient.addOrModifyPilotCache( pilotDict[ 'Site' ], pilotDict[ 'CE' ], pilotDict[ 'PilotsPerJob' ], pilotDict[ 'PilotJobEff' ], pilotDict[ 'Status' ] ) if not resQuery[ 'OK' ]: return resQuery return S_OK() def _prepareCommand( self ): """ JobCommand requires one arguments: - name : <str> """ if not 'name' in self.args: return S_ERROR( '"name" not found in self.args' ) name = self.args[ 'name' ] if not 'element' in self.args: return S_ERROR( 'element is missing' ) element = self.args[ 'element' ] if element not in [ 'Site', 'Resource' ]: return S_ERROR( '"%s" is not Site nor Resource' % element ) return S_OK( ( element, name ) ) def doNew( self, masterParams = None ): if masterParams is not None: element, name = masterParams else: params = self._prepareCommand() if not params[ 'OK' ]: return params element, name = params[ 'Value' ] wmsDict = {} if element == 'Site': wmsDict = { 'GridSite' : name } elif element == 'Resource': wmsDict = { 'ExpandSite' : name } else: # You should never see this error return S_ERROR( '"%s" is not Site nor Resource' % element ) wmsResults = self.wmsAdmin.getPilotSummaryWeb( wmsDict, [], 0, 0 ) if not wmsResults[ 'OK' ]: return wmsResults wmsResults = wmsResults[ 'Value' ] if not 'ParameterNames' in wmsResults: return S_ERROR( 'Wrong result dictionary, missing "ParameterNames"' ) params = wmsResults[ 'ParameterNames' ] if not 'Records' in wmsResults: return S_ERROR( 'Wrong formed result dictionary, missing "Records"' ) records = wmsResults[ 'Records' ] uniformResult = [] for record in records: # This returns a dictionary with the following keys: # 'Site', 'CE', 'Submitted', 'Ready', 'Scheduled', 'Waiting', 'Running', # 'Done', 'Aborted', 'Done_Empty', 'Aborted_Hour', 'Total', 'PilotsPerJob', # 'PilotJobEff', 'Status', 'InMask' pilotDict = dict( zip( params, record ) ) pilotDict[ 'PilotsPerJob' ] = float( pilotDict[ 'PilotsPerJob' ] ) pilotDict[ 'PilotJobEff' ] = float( pilotDict[ 'PilotJobEff' ] ) uniformResult.append( pilotDict ) storeRes = self._storeCommand( uniformResult ) if not storeRes[ 'OK' ]: return storeRes return S_OK( uniformResult ) def doCache( self ): params = self._prepareCommand() if not params[ 'OK' ]: return params element, name = params[ 'Value' ] if element == 'Site': # WMS returns Site entries with CE = 'Multiple' site, ce = name, 'Multiple' elif element == 'Resource': site, ce = None, name else: # You should never see this error return S_ERROR( '"%s" is not Site nor Resource' % element ) result = self.rmClient.selectPilotCache( site, ce ) if result[ 'OK' ]: result = S_OK( [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ] ) return result def doMaster( self ): siteNames = CSHelpers.getSites() if not siteNames[ 'OK' ]: return siteNames siteNames = siteNames[ 'Value' ] ces = CSHelpers.getComputingElements() if not ces[ 'OK' ]: return ces ces = ces[ 'Value' ] pilotResults = self.doNew( ( 'Site', siteNames ) ) if not pilotResults[ 'OK' ]: self.metrics[ 'failed' ].append( pilotResults[ 'Message' ] ) pilotResults = self.doNew( ( 'Resource', ces ) ) if not pilotResults[ 'OK' ]: self.metrics[ 'failed' ].append( pilotResults[ 'Message' ] ) return S_OK( self.metrics )
class PilotCommand(Command): """ Pilot "master" Command. """ def __init__(self, args=None, clients=None): super(PilotCommand, self).__init__(args, clients) if "Pilots" in self.apis: self.pilots = self.apis["Pilots"] else: self.pilots = PilotManagerClient() if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient() def _storeCommand(self, result): """ Stores the results of doNew method on the database. """ for pilotDict in result: resQuery = self.rmClient.addOrModifyPilotCache( site=pilotDict["Site"], cE=pilotDict["CE"], vO=pilotDict.get("OwnerGroup", None), pilotsPerJob=pilotDict["PilotsPerJob"], pilotJobEff=pilotDict["PilotJobEff"], status=pilotDict["Status"], ) if not resQuery["OK"]: return resQuery return S_OK() def _prepareCommand(self): """ JobCommand requires one arguments: - name : <str> """ self.log.debug("_prepareCommand: args:", self.args) if "name" not in self.args: return S_ERROR('"name" not found in self.args') name = self.args["name"] if "element" not in self.args: return S_ERROR("element is missing") element = self.args["element"] if "vO" not in self.args: return S_ERROR('_prepareCommand: "vO" not found in self.args') vo = self.args["vO"] if element not in ["Site", "Resource"]: return S_ERROR('"%s" is not Site nor Resource' % element) return S_OK((element, name, vo)) def doNew(self, masterParams=None): self.log.debug("PilotCommand doNew") if masterParams is not None: element, name = masterParams else: params = self._prepareCommand() if not params["OK"]: return params element, name = params["Value"] wmsDict = {} if element == "Site": wmsDict = {"GridSite": name} elif element == "Resource": wmsDict = {"ExpandSite": name} else: # You should never see this error return S_ERROR('"%s" is not Site nor Resource' % element) if element == "Resource": pilotsResultPivot = self.pilots.getGroupedPilotSummary( {}, ["GridSite", "DestinationSite", "OwnerGroup"]) elif element == "Site": pilotsResultPivot = self.pilots.getGroupedPilotSummary( {}, ["GridSite", "OwnerGroup"]) else: # You should never see this error return S_ERROR('"%s" is not Site nor Resource' % element) if not pilotsResultPivot["OK"]: return pilotsResultPivot pilotsResults = pilotsResultPivot["Value"] if "ParameterNames" not in pilotsResults: return S_ERROR('Wrong result dictionary, missing "ParameterNames"') params = pilotsResults["ParameterNames"] if "Records" not in pilotsResults: return S_ERROR('Wrong formed result dictionary, missing "Records"') records = pilotsResults["Records"] uniformResult = [] for record in records: # This returns a dictionary with the following keys: # 'Site', 'CE', 'Submitted', 'Ready', 'Scheduled', 'Waiting', 'Running', # 'Done', 'Aborted', 'Done_Empty', 'Aborted_Hour', 'Total', 'PilotsPerJob', # 'PilotJobEff', 'Status', 'InMask' pilotDict = dict(zip(params, record)) pilotDict["PilotsPerJob"] = float(pilotDict["PilotsPerJob"]) pilotDict["PilotJobEff"] = float(pilotDict["PilotJobEff"]) uniformResult.append(pilotDict) storeRes = self._storeCommand(uniformResult) if not storeRes["OK"]: return storeRes return S_OK(uniformResult) def doCache(self): self.log.debug("PilotCommand doCache") params = self._prepareCommand() if not params["OK"]: return params element, name, vo = params["Value"] if element == "Site": # WMS returns Site entries with CE = 'Multiple' site, ce = name, "Multiple" elif element == "Resource": site, ce = None, name else: # You should never see this error return S_ERROR('"%s" is not Site nor Resource' % element) result = self.rmClient.selectPilotCache(site=site, cE=ce) if result["OK"]: result = S_OK( [dict(zip(result["Columns"], res)) for res in result["Value"]]) self.log.debug("PilotCommand doCache result: ", result) return result def doMaster(self): self.log.debug("PilotCommand doMaster") siteNames = getSites() if not siteNames["OK"]: return siteNames siteNames = siteNames["Value"] res = getCESiteMapping() if not res["OK"]: return res ces = list(res["Value"]) pilotResults = self.doNew(("Site", siteNames)) if not pilotResults["OK"]: self.metrics["failed"].append(pilotResults["Message"]) pilotResults = self.doNew(("Resource", ces)) if not pilotResults["OK"]: self.metrics["failed"].append(pilotResults["Message"]) return S_OK(self.metrics)
class PilotCommand(Command): """ Pilot 'master' Command. """ def __init__(self, args=None, clients=None): """ Constructor. :Parameters: **args** - [, `dict` ] arguments to be passed to be used in the _prepareCommand method ( name and timespan are the expected ones ) **clients - [, `dict` ] clients from where information is fetched. Mainly used to avoid creating new connections on agents looping over clients. ResourceManagementClient and PilotsDB are most welcome. """ super(PilotCommand, self).__init__(args, clients) if 'PilotsDB' in self.apis: self.pilotsDB = self.apis['PilotsDB'] else: self.pilotsDB = PilotAgentsDB() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient() def _storeCommand(self, result): """ Stores the results of doNew method on the database. :Parameters: **result** - `list( dict )` list of dictionaries to be inserted on the DB. Unfortunately, there is no bulk insertion method on the database. The dictionaries are sanitized in doNew method so that they match the column names in the database. :return: S_OK / S_ERROR """ for pilotDict in result: lowerCasePilotDict = {} for key, value in pilotDict.iteritems(): lowerCasePilotDict[key[0].lower() + key[1:]] = value # I do not care about the **magic, it makes it cleaner resQuery = self.rmClient.addOrModifyPilotCache( **lowerCasePilotDict) if not resQuery['OK']: return resQuery return S_OK() def _prepareCommand(self): """ Method that parses command arguments to extract the ones needed: name : name of the computing element timespan ( seconds ) : time window :return: : S_OK( name, timespan ) / S_ERROR """ if not 'name' in self.args: return S_ERROR('"name" not found in self.args') name = self.args['name'] if not 'timespan' in self.args: return S_ERROR('"timespan" not found in self.args') timespan = self.args['timespan'] return S_OK((name, timespan)) def doNew(self, masterParams=None): """ doNew method. If is master execution, name is declared as '' so that all ce's are asked. Once values are obtained, they are stored on the Database. The entries with name Unknown, NotAssigned and Total are skipped. :Parameters: **masterParams** - [, bool ] if True, it queries for all elements in the database for the given timespan :return: S_OK( list ( dict ) ) / S_ERROR """ # Ask for all CEs if masterParams is True: self.args['name'] = '' params = self._prepareCommand() if not params['OK']: return params computingElement, timespan = params['Value'] # Calculate time window from timespan and utcnow endTimeWindow = datetime.utcnow() startTimeWindow = endTimeWindow - timedelta(seconds=timespan) # Get pilots information from DB pilotsRes = self.pilotsDB.getPilotSummaryShort(startTimeWindow, endTimeWindow, computingElement) if not pilotsRes['OK']: return pilotsRes # This list matches the database schema in ResourceManagemntDB. It is used # to have a perfect match even it there are no pilots on a particular state pilotStatuses = [ 'Scheduled', 'Waiting', 'Submitted', 'Running', 'Done', 'Aborted', 'Cancelled', 'Deleted', 'Failed', 'Held', 'Killed', 'Stalled' ] uniformResult = [] for ceName, pilotDict in pilotsRes['Value'].items(): if ceName in ['Total', 'Unknown', 'NotAssigned']: continue uniformPilotDict = dict.fromkeys(pilotStatuses, 0) uniformPilotDict.update(pilotDict) uniformPilotDict['Timespan'] = timespan uniformPilotDict['CE'] = ceName uniformResult.append(uniformPilotDict) # Store results storeRes = self._storeCommand(uniformResult) if not storeRes['OK']: return storeRes return S_OK(uniformResult) def doCache(self): """ doCache gets values from the database instead from the PilotsDB tables. If successful, returns a list of dictionaries with the database records. :return: S_OK( list( dict ) ) / S_ERROR """ params = self._prepareCommand() if not params['OK']: return params computingElement, timespan = params['Value'] # Make sure the records we obtain are NOT out of date lastValidRecord = datetime.utcnow() - timedelta(seconds=timespan) result = self.rmClient.selectPilotCache( cE=computingElement, timespan=timespan, meta={'older': ('LastCheckTime', lastValidRecord)}) if result['OK']: result = S_OK( [dict(zip(result['Columns'], res)) for res in result['Value']]) return result def doMaster(self): """ Master method, asks for all information in the database for the given timespan ( see _prepareCommand ). :return: : S_OK( failedMessages ) """ pilotResults = self.doNew(masterParams=True) if not pilotResults['OK']: self.metrics['failed'].append(pilotResults['Message']) return S_OK(self.metrics)
class PilotCommand( Command ): """ Pilot 'master' Command. """ def __init__( self, args = None, clients = None ): """ Constructor. :Parameters: **args** - [, `dict` ] arguments to be passed to be used in the _prepareCommand method ( name and timespan are the expected ones ) **clients - [, `dict` ] clients from where information is fetched. Mainly used to avoid creating new connections on agents looping over clients. ResourceManagementClient and PilotsDB are most welcome. """ super( PilotCommand, self ).__init__( args, clients ) if 'PilotsDB' in self.apis: self.pilotsDB = self.apis[ 'PilotsDB' ] else: self.pilotsDB = PilotAgentsDB() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient() def _storeCommand( self, result ): """ Stores the results of doNew method on the database. :Parameters: **result** - `list( dict )` list of dictionaries to be inserted on the DB. Unfortunately, there is no bulk insertion method on the database. The dictionaries are sanitized in doNew method so that they match the column names in the database. :return: S_OK / S_ERROR """ for pilotDict in result: lowerCasePilotDict = {} for key, value in pilotDict.iteritems(): lowerCasePilotDict[ key[0].lower() + key[1:] ] = value # I do not care about the **magic, it makes it cleaner resQuery = self.rmClient.addOrModifyPilotCache( **lowerCasePilotDict ) if not resQuery[ 'OK' ]: return resQuery return S_OK() def _prepareCommand( self ): """ Method that parses command arguments to extract the ones needed: name : name of the computing element timespan ( seconds ) : time window :return: : S_OK( name, timespan ) / S_ERROR """ if not 'name' in self.args: return S_ERROR( '"name" not found in self.args' ) name = self.args[ 'name' ] if not 'timespan' in self.args: return S_ERROR( '"timespan" not found in self.args' ) timespan = self.args[ 'timespan' ] return S_OK( ( name, timespan ) ) def doNew( self, masterParams = None ): """ doNew method. If is master execution, name is declared as '' so that all ce's are asked. Once values are obtained, they are stored on the Database. The entries with name Unknown, NotAssigned and Total are skipped. :Parameters: **masterParams** - [, bool ] if True, it queries for all elements in the database for the given timespan :return: S_OK( list ( dict ) ) / S_ERROR """ # Ask for all CEs if masterParams is True: self.args[ 'name' ] = '' params = self._prepareCommand() if not params[ 'OK' ]: return params computingElement, timespan = params[ 'Value' ] # Calculate time window from timespan and utcnow endTimeWindow = datetime.utcnow() startTimeWindow = endTimeWindow - timedelta( seconds = timespan ) # Get pilots information from DB pilotsRes = self.pilotsDB.getPilotSummaryShort( startTimeWindow, endTimeWindow, computingElement ) if not pilotsRes[ 'OK' ]: return pilotsRes # This list matches the database schema in ResourceManagemntDB. It is used # to have a perfect match even it there are no pilots on a particular state pilotStatuses = [ 'Scheduled', 'Waiting', 'Submitted', 'Running', 'Done', 'Aborted', 'Cancelled', 'Deleted', 'Failed', 'Held', 'Killed', 'Stalled' ] uniformResult = [] for ceName, pilotDict in pilotsRes[ 'Value' ].items(): if ceName in [ 'Total', 'Unknown', 'NotAssigned' ]: continue uniformPilotDict = dict.fromkeys( pilotStatuses, 0 ) uniformPilotDict.update( pilotDict ) uniformPilotDict[ 'Timespan' ] = timespan uniformPilotDict[ 'CE' ] = ceName uniformResult.append( uniformPilotDict ) # Store results storeRes = self._storeCommand( uniformResult ) if not storeRes[ 'OK' ]: return storeRes return S_OK( uniformResult ) def doCache( self ): """ doCache gets values from the database instead from the PilotsDB tables. If successful, returns a list of dictionaries with the database records. :return: S_OK( list( dict ) ) / S_ERROR """ params = self._prepareCommand() if not params[ 'OK' ]: return params computingElement, timespan = params[ 'Value' ] # Make sure the records we obtain are NOT out of date lastValidRecord = datetime.utcnow() - timedelta( seconds = timespan ) result = self.rmClient.selectPilotCache( cE = computingElement, timespan = timespan, meta = { 'older' : ( 'LastCheckTime', lastValidRecord ) } ) if result[ 'OK' ]: result = S_OK( [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ] ) return result def doMaster( self ): """ Master method, asks for all information in the database for the given timespan ( see _prepareCommand ). :return: : S_OK( failedMessages ) """ pilotResults = self.doNew( masterParams = True ) if not pilotResults[ 'OK' ]: self.metrics[ 'failed' ].append( pilotResults[ 'Message' ] ) return S_OK( self.metrics )