def execute(self): """ Main execution method """ result = gConfig.getSections("/DIRAC/Setups") if not result['OK']: return result validSetups = result['Value'] gLogger.info("Valid setups for this cycle are %s" % ", ".join(validSetups)) #Get the WMS Snapshot! result = self.jobDB.getSummarySnapshot(self.__jobDBFields) now = Time.dateTime() if not result['OK']: gLogger.error("Can't the the jobdb summary", result['Message']) else: values = result['Value'][1] for record in values: recordSetup = record[0] if recordSetup not in validSetups: gLogger.error("Setup %s is not valid" % recordSetup) continue if recordSetup not in self.dsClients: gLogger.info("Creating DataStore client for %s" % recordSetup) self.dsClients[recordSetup] = DataStoreClient( setup=recordSetup, retryGraceTime=900) record = record[1:] rD = {} for fV in self.__summaryDefinedFields: rD[fV[0]] = fV[1] for iP in range(len(self.__summaryKeyFieldsMapping)): fieldName = self.__summaryKeyFieldsMapping[iP] rD[self.__renameFieldsMapping.get(fieldName, fieldName)] = record[iP] record = record[len(self.__summaryKeyFieldsMapping):] for iP in range(len(self.__summaryValueFieldsMapping)): rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP]) acWMS = WMSHistory() acWMS.setStartTime(now) acWMS.setEndTime(now) acWMS.setValuesFromDict(rD) retVal = acWMS.checkValues() if not retVal['OK']: gLogger.error("Invalid accounting record ", "%s -> %s" % (retVal['Message'], rD)) else: self.dsClients[recordSetup].addRegister(acWMS) for setup in self.dsClients: gLogger.info("Sending records for setup %s" % setup) result = self.dsClients[setup].commit() if not result['OK']: gLogger.error( "Couldn't commit wms history for setup %s" % setup, result['Message']) else: gLogger.info("Sent %s records for setup %s" % (result['Value'], setup)) return S_OK()
def execute(self): """ Main execution method """ # Get the WMS Snapshot! result = JobDB().getSummarySnapshot(self.__jobDBFields) now = Time.dateTime() if not result['OK']: self.log.error( "Can't get the JobDB summary", "%s: won't commit at this cycle" % result['Message']) return S_ERROR() # Now we try to commit values = result['Value'][1] self.log.info("Start sending records") for record in values: record = record[1:] rD = {} for fV in self.__summaryDefinedFields: rD[fV[0]] = fV[1] for iP in range(len(self.__summaryKeyFieldsMapping)): fieldName = self.__summaryKeyFieldsMapping[iP] rD[self.__renameFieldsMapping.get(fieldName, fieldName)] = record[iP] record = record[len(self.__summaryKeyFieldsMapping):] for iP in range(len(self.__summaryValueFieldsMapping)): rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP]) for backend in self.datastores: if backend.lower() == 'monitoring': rD['timestamp'] = int(Time.toEpoch(now)) self.datastores['Monitoring'].addRecord(rD) elif backend.lower() == 'accounting': acWMS = WMSHistory() acWMS.setStartTime(now) acWMS.setEndTime(now) acWMS.setValuesFromDict(rD) retVal = acWMS.checkValues() if not retVal['OK']: self.log.error("Invalid accounting record ", "%s -> %s" % (retVal['Message'], rD)) else: self.datastores['Accounting'].addRegister(acWMS) for backend, datastore in self.datastores.items(): self.log.info("Committing to %s backend" % backend) result = datastore.commit() if not result['OK']: self.log.error("Couldn't commit WMS history to %s" % backend, result['Message']) return S_ERROR() self.log.verbose("Done committing to %s backend" % backend) return S_OK()
def execute( self ): """ Main execution method """ result = gConfig.getSections( "/DIRAC/Setups" ) if not result[ 'OK' ]: return result validSetups = result[ 'Value' ] gLogger.info( "Valid setups for this cycle are %s" % ", ".join( validSetups ) ) #Get the WMS Snapshot! result = self.jobDB.getSummarySnapshot( self.__jobDBFields ) now = Time.dateTime() if not result[ 'OK' ]: gLogger.error( "Can't the the jobdb summary", result[ 'Message' ] ) else: values = result[ 'Value' ][1] for record in values: recordSetup = record[0] if recordSetup not in validSetups: gLogger.error( "Setup %s is not valid" % recordSetup ) continue if recordSetup not in self.dsClients: gLogger.info( "Creating DataStore client for %s" % recordSetup ) self.dsClients[ recordSetup ] = DataStoreClient( setup = recordSetup, retryGraceTime = 900 ) record = record[1:] rD = {} for fV in self.__summaryDefinedFields: rD[ fV[0] ] = fV[1] for iP in range( len( self.__summaryKeyFieldsMapping ) ): fieldName = self.__summaryKeyFieldsMapping[iP] rD[ self.__renameFieldsMapping.get( fieldName, fieldName ) ] = record[iP] record = record[ len( self.__summaryKeyFieldsMapping ): ] for iP in range( len( self.__summaryValueFieldsMapping ) ): rD[ self.__summaryValueFieldsMapping[iP] ] = int( record[iP] ) acWMS = WMSHistory() acWMS.setStartTime( now ) acWMS.setEndTime( now ) acWMS.setValuesFromDict( rD ) retVal = acWMS.checkValues() if not retVal[ 'OK' ]: gLogger.error( "Invalid accounting record ", "%s -> %s" % ( retVal[ 'Message' ], rD ) ) else: self.dsClients[ recordSetup ].addRegister( acWMS ) for setup in self.dsClients: gLogger.info( "Sending records for setup %s" % setup ) result = self.dsClients[ setup ].commit() if not result[ 'OK' ]: gLogger.error( "Couldn't commit wms history for setup %s" % setup, result[ 'Message' ] ) else: gLogger.info( "Sent %s records for setup %s" % ( result[ 'Value' ], setup ) ) return S_OK()
def execute(self): """Main execution method""" # PilotsHistory to Monitoring if "Monitoring" in self.pilotMonitoringOption: self.log.info("Committing PilotsHistory to Monitoring") result = PilotAgentsDB().getSummarySnapshot() now = datetime.datetime.utcnow() if not result["OK"]: self.log.error( "Can't get the PilotAgentsDB summary", "%s: won't commit PilotsHistory at this cycle" % result["Message"], ) values = result["Value"][1] for record in values: rD = {} for iP, _ in enumerate(self.__pilotsMapping): rD[self.__pilotsMapping[iP]] = record[iP] rD["timestamp"] = int(TimeUtilities.toEpoch(now)) self.pilotReporter.addRecord(rD) self.log.info("Committing to Monitoring...") result = self.pilotReporter.commit() if not result["OK"]: self.log.error("Could not commit to Monitoring", result["Message"]) self.log.verbose("Done committing PilotsHistory to Monitoring") # WMSHistory to Monitoring or Accounting self.log.info("Committing WMSHistory to %s backend" % "and ".join(self.jobMonitoringOption)) result = JobDB().getSummarySnapshot(self.__jobDBFields) now = datetime.datetime.utcnow() if not result["OK"]: self.log.error( "Can't get the JobDB summary", "%s: won't commit WMSHistory at this cycle" % result["Message"]) return S_ERROR() values = result["Value"][1] self.log.info("Start sending WMSHistory records") for record in values: record = record[1:] rD = {} for fV in self.__summaryDefinedFields: rD[fV[0]] = fV[1] for iP, _ in enumerate(self.__summaryKeyFieldsMapping): fieldName = self.__summaryKeyFieldsMapping[iP] rD[self.__renameFieldsMapping.get(fieldName, fieldName)] = record[iP] record = record[len(self.__summaryKeyFieldsMapping):] for iP, _ in enumerate(self.__summaryValueFieldsMapping): rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP]) for backend in self.datastores: if backend.lower() == "monitoring": rD["timestamp"] = int(TimeUtilities.toEpoch(now)) self.datastores["Monitoring"].addRecord(rD) elif backend.lower() == "accounting": acWMS = WMSHistory() acWMS.setStartTime(now) acWMS.setEndTime(now) acWMS.setValuesFromDict(rD) retVal = acWMS.checkValues() if not retVal["OK"]: self.log.error("Invalid WMSHistory accounting record ", "%s -> %s" % (retVal["Message"], rD)) else: self.datastores["Accounting"].addRegister(acWMS) for backend, datastore in self.datastores.items(): self.log.info("Committing WMSHistory records to %s backend" % backend) result = datastore.commit() if not result["OK"]: self.log.error("Couldn't commit WMSHistory to %s" % backend, result["Message"]) return S_ERROR() self.log.verbose("Done committing WMSHistory to %s backend" % backend) return S_OK()
def execute(self): """ Main execution method """ result = gConfig.getSections("/DIRAC/Setups") if not result['OK']: return result validSetups = result['Value'] self.log.info("Valid setups for this cycle are %s" % ", ".join(validSetups)) # Get the WMS Snapshot! result = self.jobDB.getSummarySnapshot(self.__jobDBFields) now = Time.dateTime() if not result['OK']: self.log.error( "Can't get the JobDB summary", "%s: won't commit at this cycle" % result['Message']) else: values = result['Value'][1] if self.retryOnce: self.log.verbose( "Adding to records to commit those not committed within the previous cycle" ) acWMSListAdded = [] for record in values: recordSetup = record[0] if recordSetup not in validSetups: self.log.error("Setup %s is not valid" % recordSetup) continue if recordSetup not in self.dsClients: self.log.info("Creating DataStore client for %s" % recordSetup) self.dsClients[recordSetup] = DataStoreClient( retryGraceTime=900) record = record[1:] rD = {} for fV in self.__summaryDefinedFields: rD[fV[0]] = fV[1] for iP in range(len(self.__summaryKeyFieldsMapping)): fieldName = self.__summaryKeyFieldsMapping[iP] rD[self.__renameFieldsMapping.get(fieldName, fieldName)] = record[iP] record = record[len(self.__summaryKeyFieldsMapping):] for iP in range(len(self.__summaryValueFieldsMapping)): rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP]) acWMS = WMSHistory() acWMS.setStartTime(now) acWMS.setEndTime(now) acWMS.setValuesFromDict(rD) retVal = acWMS.checkValues() if not retVal['OK']: self.log.error("Invalid accounting record ", "%s -> %s" % (retVal['Message'], rD)) else: self.dsClients[recordSetup].addRegister(acWMS) acWMSListAdded.append(acWMS) if self.retryOnce and self.retryValues: for acWMSCumulated in self.retryValues: retVal = acWMSCumulated.checkValues() if not retVal['OK']: self.log.error("Invalid accounting record ", "%s" % (retVal['Message'])) else: self.dsClients[recordSetup].addRegister(acWMSCumulated) for setup in self.dsClients: self.log.info("Sending records for setup %s" % setup) result = self.dsClients[setup].commit() if not result['OK']: self.log.error( "Couldn't commit wms history for setup %s" % setup, result['Message']) # Re-creating the client: for new connection, and for avoiding accumulating too large of a backlog self.dsClients[setup] = DataStoreClient(retryGraceTime=900) if not self.retryOnce: self.log.info("Will try again at next cycle") self.retryOnce = True self.retryValues = acWMSListAdded else: self.log.warn("Won't retry one more time") self.retryOnce = False self.retryValues = [] else: self.log.info("Sent %s records for setup %s" % (result['Value'], setup)) self.retryOnce = False return S_OK()
def execute( self ): """ Main execution method """ result = gConfig.getSections( "/DIRAC/Setups" ) if not result[ 'OK' ]: return result validSetups = result[ 'Value' ] self.log.info( "Valid setups for this cycle are %s" % ", ".join( validSetups ) ) #Get the WMS Snapshot! result = self.jobDB.getSummarySnapshot( self.__jobDBFields ) now = Time.dateTime() if not result[ 'OK' ]: self.log.error( "Can't get the JobDB summary", "%s: won't commit at this cycle" % result[ 'Message' ] ) else: values = result[ 'Value' ][1] if self.retryOnce: self.log.verbose( "Adding to records to commit those not committed within the previous cycle" ) acWMSListAdded = [] for record in values: recordSetup = record[0] if recordSetup not in validSetups: self.log.error( "Setup %s is not valid" % recordSetup ) continue if recordSetup not in self.dsClients: self.log.info( "Creating DataStore client for %s" % recordSetup ) self.dsClients[ recordSetup ] = DataStoreClient( setup = recordSetup, retryGraceTime = 900 ) record = record[1:] rD = {} for fV in self.__summaryDefinedFields: rD[ fV[0] ] = fV[1] for iP in range( len( self.__summaryKeyFieldsMapping ) ): fieldName = self.__summaryKeyFieldsMapping[iP] rD[ self.__renameFieldsMapping.get( fieldName, fieldName ) ] = record[iP] record = record[ len( self.__summaryKeyFieldsMapping ): ] for iP in range( len( self.__summaryValueFieldsMapping ) ): rD[ self.__summaryValueFieldsMapping[iP] ] = int( record[iP] ) acWMS = WMSHistory() acWMS.setStartTime( now ) acWMS.setEndTime( now ) acWMS.setValuesFromDict( rD ) retVal = acWMS.checkValues() if not retVal[ 'OK' ]: self.log.error( "Invalid accounting record ", "%s -> %s" % ( retVal[ 'Message' ], rD ) ) else: self.dsClients[ recordSetup ].addRegister( acWMS ) acWMSListAdded.append( acWMS ) if self.retryOnce and self.retryValues: for acWMSCumulated in self.retryValues: retVal = acWMSCumulated.checkValues() if not retVal[ 'OK' ]: self.log.error( "Invalid accounting record ", "%s" % ( retVal[ 'Message' ] ) ) else: self.dsClients[ recordSetup ].addRegister( acWMSCumulated ) for setup in self.dsClients: self.log.info( "Sending records for setup %s" % setup ) result = self.dsClients[ setup ].commit() if not result[ 'OK' ]: self.log.error( "Couldn't commit wms history for setup %s" % setup, result[ 'Message' ] ) # Re-creating the client: for new connection, and for avoiding accumulating too large of a backlog self.dsClients[ setup ] = DataStoreClient( setup = setup, retryGraceTime = 900 ) if not self.retryOnce: self.log.info( "Will try again at next cycle" ) self.retryOnce = True self.retryValues = acWMSListAdded else: self.log.warn( "Won't retry one more time" ) self.retryOnce = False self.retryValues = [] else: self.log.info( "Sent %s records for setup %s" % ( result[ 'Value' ], setup ) ) self.retryOnce = False return S_OK()