def execute(self):
""" Main execution method
"""
result = gConfig.getSections("/DIRAC/Setups")
if not result['OK']:
return result
validSetups = result['Value']
gLogger.info("Valid setups for this cycle are %s" %
", ".join(validSetups))
#Get the WMS Snapshot!
result = self.jobDB.getSummarySnapshot(self.__jobDBFields)
now = Time.dateTime()
if not result['OK']:
gLogger.error("Can't the the jobdb summary", result['Message'])
else:
values = result['Value'][1]
for record in values:
recordSetup = record[0]
if recordSetup not in validSetups:
gLogger.error("Setup %s is not valid" % recordSetup)
continue
if recordSetup not in self.dsClients:
gLogger.info("Creating DataStore client for %s" %
recordSetup)
self.dsClients[recordSetup] = DataStoreClient(
setup=recordSetup, retryGraceTime=900)
record = record[1:]
rD = {}
for fV in self.__summaryDefinedFields:
rD[fV[0]] = fV[1]
for iP in range(len(self.__summaryKeyFieldsMapping)):
fieldName = self.__summaryKeyFieldsMapping[iP]
rD[self.__renameFieldsMapping.get(fieldName,
fieldName)] = record[iP]
record = record[len(self.__summaryKeyFieldsMapping):]
for iP in range(len(self.__summaryValueFieldsMapping)):
rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP])
acWMS = WMSHistory()
acWMS.setStartTime(now)
acWMS.setEndTime(now)
acWMS.setValuesFromDict(rD)
retVal = acWMS.checkValues()
if not retVal['OK']:
gLogger.error("Invalid accounting record ",
"%s -> %s" % (retVal['Message'], rD))
else:
self.dsClients[recordSetup].addRegister(acWMS)
for setup in self.dsClients:
gLogger.info("Sending records for setup %s" % setup)
result = self.dsClients[setup].commit()
if not result['OK']:
gLogger.error(
"Couldn't commit wms history for setup %s" % setup,
result['Message'])
else:
gLogger.info("Sent %s records for setup %s" %
(result['Value'], setup))
return S_OK()
def execute(self):
""" Main execution method
"""
# Get the WMS Snapshot!
result = JobDB().getSummarySnapshot(self.__jobDBFields)
now = Time.dateTime()
if not result['OK']:
self.log.error(
"Can't get the JobDB summary",
"%s: won't commit at this cycle" % result['Message'])
return S_ERROR()
# Now we try to commit
values = result['Value'][1]
self.log.info("Start sending records")
for record in values:
record = record[1:]
rD = {}
for fV in self.__summaryDefinedFields:
rD[fV[0]] = fV[1]
for iP in range(len(self.__summaryKeyFieldsMapping)):
fieldName = self.__summaryKeyFieldsMapping[iP]
rD[self.__renameFieldsMapping.get(fieldName,
fieldName)] = record[iP]
record = record[len(self.__summaryKeyFieldsMapping):]
for iP in range(len(self.__summaryValueFieldsMapping)):
rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP])
for backend in self.datastores:
if backend.lower() == 'monitoring':
rD['timestamp'] = int(Time.toEpoch(now))
self.datastores['Monitoring'].addRecord(rD)
elif backend.lower() == 'accounting':
acWMS = WMSHistory()
acWMS.setStartTime(now)
acWMS.setEndTime(now)
acWMS.setValuesFromDict(rD)
retVal = acWMS.checkValues()
if not retVal['OK']:
self.log.error("Invalid accounting record ",
"%s -> %s" % (retVal['Message'], rD))
else:
self.datastores['Accounting'].addRegister(acWMS)
for backend, datastore in self.datastores.items():
self.log.info("Committing to %s backend" % backend)
result = datastore.commit()
if not result['OK']:
self.log.error("Couldn't commit WMS history to %s" % backend,
result['Message'])
return S_ERROR()
self.log.verbose("Done committing to %s backend" % backend)
return S_OK()
def execute( self ):
""" Main execution method
"""
result = gConfig.getSections( "/DIRAC/Setups" )
if not result[ 'OK' ]:
return result
validSetups = result[ 'Value' ]
gLogger.info( "Valid setups for this cycle are %s" % ", ".join( validSetups ) )
#Get the WMS Snapshot!
result = self.jobDB.getSummarySnapshot( self.__jobDBFields )
now = Time.dateTime()
if not result[ 'OK' ]:
gLogger.error( "Can't the the jobdb summary", result[ 'Message' ] )
else:
values = result[ 'Value' ][1]
for record in values:
recordSetup = record[0]
if recordSetup not in validSetups:
gLogger.error( "Setup %s is not valid" % recordSetup )
continue
if recordSetup not in self.dsClients:
gLogger.info( "Creating DataStore client for %s" % recordSetup )
self.dsClients[ recordSetup ] = DataStoreClient( setup = recordSetup, retryGraceTime = 900 )
record = record[1:]
rD = {}
for fV in self.__summaryDefinedFields:
rD[ fV[0] ] = fV[1]
for iP in range( len( self.__summaryKeyFieldsMapping ) ):
fieldName = self.__summaryKeyFieldsMapping[iP]
rD[ self.__renameFieldsMapping.get( fieldName, fieldName ) ] = record[iP]
record = record[ len( self.__summaryKeyFieldsMapping ): ]
for iP in range( len( self.__summaryValueFieldsMapping ) ):
rD[ self.__summaryValueFieldsMapping[iP] ] = int( record[iP] )
acWMS = WMSHistory()
acWMS.setStartTime( now )
acWMS.setEndTime( now )
acWMS.setValuesFromDict( rD )
retVal = acWMS.checkValues()
if not retVal[ 'OK' ]:
gLogger.error( "Invalid accounting record ", "%s -> %s" % ( retVal[ 'Message' ], rD ) )
else:
self.dsClients[ recordSetup ].addRegister( acWMS )
for setup in self.dsClients:
gLogger.info( "Sending records for setup %s" % setup )
result = self.dsClients[ setup ].commit()
if not result[ 'OK' ]:
gLogger.error( "Couldn't commit wms history for setup %s" % setup, result[ 'Message' ] )
else:
gLogger.info( "Sent %s records for setup %s" % ( result[ 'Value' ], setup ) )
return S_OK()
def execute(self):
"""Main execution method"""
# PilotsHistory to Monitoring
if "Monitoring" in self.pilotMonitoringOption:
self.log.info("Committing PilotsHistory to Monitoring")
result = PilotAgentsDB().getSummarySnapshot()
now = datetime.datetime.utcnow()
if not result["OK"]:
self.log.error(
"Can't get the PilotAgentsDB summary",
"%s: won't commit PilotsHistory at this cycle" %
result["Message"],
)
values = result["Value"][1]
for record in values:
rD = {}
for iP, _ in enumerate(self.__pilotsMapping):
rD[self.__pilotsMapping[iP]] = record[iP]
rD["timestamp"] = int(TimeUtilities.toEpoch(now))
self.pilotReporter.addRecord(rD)
self.log.info("Committing to Monitoring...")
result = self.pilotReporter.commit()
if not result["OK"]:
self.log.error("Could not commit to Monitoring",
result["Message"])
self.log.verbose("Done committing PilotsHistory to Monitoring")
# WMSHistory to Monitoring or Accounting
self.log.info("Committing WMSHistory to %s backend" %
"and ".join(self.jobMonitoringOption))
result = JobDB().getSummarySnapshot(self.__jobDBFields)
now = datetime.datetime.utcnow()
if not result["OK"]:
self.log.error(
"Can't get the JobDB summary",
"%s: won't commit WMSHistory at this cycle" %
result["Message"])
return S_ERROR()
values = result["Value"][1]
self.log.info("Start sending WMSHistory records")
for record in values:
record = record[1:]
rD = {}
for fV in self.__summaryDefinedFields:
rD[fV[0]] = fV[1]
for iP, _ in enumerate(self.__summaryKeyFieldsMapping):
fieldName = self.__summaryKeyFieldsMapping[iP]
rD[self.__renameFieldsMapping.get(fieldName,
fieldName)] = record[iP]
record = record[len(self.__summaryKeyFieldsMapping):]
for iP, _ in enumerate(self.__summaryValueFieldsMapping):
rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP])
for backend in self.datastores:
if backend.lower() == "monitoring":
rD["timestamp"] = int(TimeUtilities.toEpoch(now))
self.datastores["Monitoring"].addRecord(rD)
elif backend.lower() == "accounting":
acWMS = WMSHistory()
acWMS.setStartTime(now)
acWMS.setEndTime(now)
acWMS.setValuesFromDict(rD)
retVal = acWMS.checkValues()
if not retVal["OK"]:
self.log.error("Invalid WMSHistory accounting record ",
"%s -> %s" % (retVal["Message"], rD))
else:
self.datastores["Accounting"].addRegister(acWMS)
for backend, datastore in self.datastores.items():
self.log.info("Committing WMSHistory records to %s backend" %
backend)
result = datastore.commit()
if not result["OK"]:
self.log.error("Couldn't commit WMSHistory to %s" % backend,
result["Message"])
return S_ERROR()
self.log.verbose("Done committing WMSHistory to %s backend" %
backend)
return S_OK()
def execute(self):
""" Main execution method
"""
result = gConfig.getSections("/DIRAC/Setups")
if not result['OK']:
return result
validSetups = result['Value']
self.log.info("Valid setups for this cycle are %s" %
", ".join(validSetups))
# Get the WMS Snapshot!
result = self.jobDB.getSummarySnapshot(self.__jobDBFields)
now = Time.dateTime()
if not result['OK']:
self.log.error(
"Can't get the JobDB summary",
"%s: won't commit at this cycle" % result['Message'])
else:
values = result['Value'][1]
if self.retryOnce:
self.log.verbose(
"Adding to records to commit those not committed within the previous cycle"
)
acWMSListAdded = []
for record in values:
recordSetup = record[0]
if recordSetup not in validSetups:
self.log.error("Setup %s is not valid" % recordSetup)
continue
if recordSetup not in self.dsClients:
self.log.info("Creating DataStore client for %s" %
recordSetup)
self.dsClients[recordSetup] = DataStoreClient(
retryGraceTime=900)
record = record[1:]
rD = {}
for fV in self.__summaryDefinedFields:
rD[fV[0]] = fV[1]
for iP in range(len(self.__summaryKeyFieldsMapping)):
fieldName = self.__summaryKeyFieldsMapping[iP]
rD[self.__renameFieldsMapping.get(fieldName,
fieldName)] = record[iP]
record = record[len(self.__summaryKeyFieldsMapping):]
for iP in range(len(self.__summaryValueFieldsMapping)):
rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP])
acWMS = WMSHistory()
acWMS.setStartTime(now)
acWMS.setEndTime(now)
acWMS.setValuesFromDict(rD)
retVal = acWMS.checkValues()
if not retVal['OK']:
self.log.error("Invalid accounting record ",
"%s -> %s" % (retVal['Message'], rD))
else:
self.dsClients[recordSetup].addRegister(acWMS)
acWMSListAdded.append(acWMS)
if self.retryOnce and self.retryValues:
for acWMSCumulated in self.retryValues:
retVal = acWMSCumulated.checkValues()
if not retVal['OK']:
self.log.error("Invalid accounting record ",
"%s" % (retVal['Message']))
else:
self.dsClients[recordSetup].addRegister(acWMSCumulated)
for setup in self.dsClients:
self.log.info("Sending records for setup %s" % setup)
result = self.dsClients[setup].commit()
if not result['OK']:
self.log.error(
"Couldn't commit wms history for setup %s" % setup,
result['Message'])
# Re-creating the client: for new connection, and for avoiding accumulating too large of a backlog
self.dsClients[setup] = DataStoreClient(retryGraceTime=900)
if not self.retryOnce:
self.log.info("Will try again at next cycle")
self.retryOnce = True
self.retryValues = acWMSListAdded
else:
self.log.warn("Won't retry one more time")
self.retryOnce = False
self.retryValues = []
else:
self.log.info("Sent %s records for setup %s" %
(result['Value'], setup))
self.retryOnce = False
return S_OK()
def execute( self ):
""" Main execution method
"""
result = gConfig.getSections( "/DIRAC/Setups" )
if not result[ 'OK' ]:
return result
validSetups = result[ 'Value' ]
self.log.info( "Valid setups for this cycle are %s" % ", ".join( validSetups ) )
#Get the WMS Snapshot!
result = self.jobDB.getSummarySnapshot( self.__jobDBFields )
now = Time.dateTime()
if not result[ 'OK' ]:
self.log.error( "Can't get the JobDB summary", "%s: won't commit at this cycle" % result[ 'Message' ] )
else:
values = result[ 'Value' ][1]
if self.retryOnce:
self.log.verbose( "Adding to records to commit those not committed within the previous cycle" )
acWMSListAdded = []
for record in values:
recordSetup = record[0]
if recordSetup not in validSetups:
self.log.error( "Setup %s is not valid" % recordSetup )
continue
if recordSetup not in self.dsClients:
self.log.info( "Creating DataStore client for %s" % recordSetup )
self.dsClients[ recordSetup ] = DataStoreClient( setup = recordSetup, retryGraceTime = 900 )
record = record[1:]
rD = {}
for fV in self.__summaryDefinedFields:
rD[ fV[0] ] = fV[1]
for iP in range( len( self.__summaryKeyFieldsMapping ) ):
fieldName = self.__summaryKeyFieldsMapping[iP]
rD[ self.__renameFieldsMapping.get( fieldName, fieldName ) ] = record[iP]
record = record[ len( self.__summaryKeyFieldsMapping ): ]
for iP in range( len( self.__summaryValueFieldsMapping ) ):
rD[ self.__summaryValueFieldsMapping[iP] ] = int( record[iP] )
acWMS = WMSHistory()
acWMS.setStartTime( now )
acWMS.setEndTime( now )
acWMS.setValuesFromDict( rD )
retVal = acWMS.checkValues()
if not retVal[ 'OK' ]:
self.log.error( "Invalid accounting record ", "%s -> %s" % ( retVal[ 'Message' ], rD ) )
else:
self.dsClients[ recordSetup ].addRegister( acWMS )
acWMSListAdded.append( acWMS )
if self.retryOnce and self.retryValues:
for acWMSCumulated in self.retryValues:
retVal = acWMSCumulated.checkValues()
if not retVal[ 'OK' ]:
self.log.error( "Invalid accounting record ", "%s" % ( retVal[ 'Message' ] ) )
else:
self.dsClients[ recordSetup ].addRegister( acWMSCumulated )
for setup in self.dsClients:
self.log.info( "Sending records for setup %s" % setup )
result = self.dsClients[ setup ].commit()
if not result[ 'OK' ]:
self.log.error( "Couldn't commit wms history for setup %s" % setup, result[ 'Message' ] )
# Re-creating the client: for new connection, and for avoiding accumulating too large of a backlog
self.dsClients[ setup ] = DataStoreClient( setup = setup, retryGraceTime = 900 )
if not self.retryOnce:
self.log.info( "Will try again at next cycle" )
self.retryOnce = True
self.retryValues = acWMSListAdded
else:
self.log.warn( "Won't retry one more time" )
self.retryOnce = False
self.retryValues = []
else:
self.log.info( "Sent %s records for setup %s" % ( result[ 'Value' ], setup ) )
self.retryOnce = False
return S_OK()