def execute(self):
     """ Main execution method
 """
     result = gConfig.getSections("/DIRAC/Setups")
     if not result['OK']:
         return result
     validSetups = result['Value']
     gLogger.info("Valid setups for this cycle are %s" %
                  ", ".join(validSetups))
     #Get the WMS Snapshot!
     result = self.jobDB.getSummarySnapshot(self.__jobDBFields)
     now = Time.dateTime()
     if not result['OK']:
         gLogger.error("Can't the the jobdb summary", result['Message'])
     else:
         values = result['Value'][1]
         for record in values:
             recordSetup = record[0]
             if recordSetup not in validSetups:
                 gLogger.error("Setup %s is not valid" % recordSetup)
                 continue
             if recordSetup not in self.dsClients:
                 gLogger.info("Creating DataStore client for %s" %
                              recordSetup)
                 self.dsClients[recordSetup] = DataStoreClient(
                     setup=recordSetup, retryGraceTime=900)
             record = record[1:]
             rD = {}
             for fV in self.__summaryDefinedFields:
                 rD[fV[0]] = fV[1]
             for iP in range(len(self.__summaryKeyFieldsMapping)):
                 fieldName = self.__summaryKeyFieldsMapping[iP]
                 rD[self.__renameFieldsMapping.get(fieldName,
                                                   fieldName)] = record[iP]
             record = record[len(self.__summaryKeyFieldsMapping):]
             for iP in range(len(self.__summaryValueFieldsMapping)):
                 rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP])
             acWMS = WMSHistory()
             acWMS.setStartTime(now)
             acWMS.setEndTime(now)
             acWMS.setValuesFromDict(rD)
             retVal = acWMS.checkValues()
             if not retVal['OK']:
                 gLogger.error("Invalid accounting record ",
                               "%s -> %s" % (retVal['Message'], rD))
             else:
                 self.dsClients[recordSetup].addRegister(acWMS)
         for setup in self.dsClients:
             gLogger.info("Sending records for setup %s" % setup)
             result = self.dsClients[setup].commit()
             if not result['OK']:
                 gLogger.error(
                     "Couldn't commit wms history for setup %s" % setup,
                     result['Message'])
             else:
                 gLogger.info("Sent %s records for setup %s" %
                              (result['Value'], setup))
     return S_OK()
    def execute(self):
        """ Main execution method
    """
        # Get the WMS Snapshot!
        result = JobDB().getSummarySnapshot(self.__jobDBFields)
        now = Time.dateTime()
        if not result['OK']:
            self.log.error(
                "Can't get the JobDB summary",
                "%s: won't commit at this cycle" % result['Message'])
            return S_ERROR()

        # Now we try to commit
        values = result['Value'][1]

        self.log.info("Start sending records")
        for record in values:
            record = record[1:]
            rD = {}
            for fV in self.__summaryDefinedFields:
                rD[fV[0]] = fV[1]
            for iP in range(len(self.__summaryKeyFieldsMapping)):
                fieldName = self.__summaryKeyFieldsMapping[iP]
                rD[self.__renameFieldsMapping.get(fieldName,
                                                  fieldName)] = record[iP]
            record = record[len(self.__summaryKeyFieldsMapping):]
            for iP in range(len(self.__summaryValueFieldsMapping)):
                rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP])

            for backend in self.datastores:
                if backend.lower() == 'monitoring':
                    rD['timestamp'] = int(Time.toEpoch(now))
                    self.datastores['Monitoring'].addRecord(rD)

                elif backend.lower() == 'accounting':
                    acWMS = WMSHistory()
                    acWMS.setStartTime(now)
                    acWMS.setEndTime(now)
                    acWMS.setValuesFromDict(rD)
                    retVal = acWMS.checkValues()
                    if not retVal['OK']:
                        self.log.error("Invalid accounting record ",
                                       "%s -> %s" % (retVal['Message'], rD))
                    else:
                        self.datastores['Accounting'].addRegister(acWMS)

        for backend, datastore in self.datastores.items():
            self.log.info("Committing to %s backend" % backend)
            result = datastore.commit()
            if not result['OK']:
                self.log.error("Couldn't commit WMS history to %s" % backend,
                               result['Message'])
                return S_ERROR()
            self.log.verbose("Done committing to %s backend" % backend)

        return S_OK()
 def execute( self ):
   """ Main execution method
   """
   result = gConfig.getSections( "/DIRAC/Setups" )
   if not result[ 'OK' ]:
     return result
   validSetups = result[ 'Value' ]
   gLogger.info( "Valid setups for this cycle are %s" % ", ".join( validSetups ) )
   #Get the WMS Snapshot!
   result = self.jobDB.getSummarySnapshot( self.__jobDBFields )
   now = Time.dateTime()
   if not result[ 'OK' ]:
     gLogger.error( "Can't the the jobdb summary", result[ 'Message' ] )
   else:
     values = result[ 'Value' ][1]
     for record in values:
       recordSetup = record[0]
       if recordSetup not in validSetups:
         gLogger.error( "Setup %s is not valid" % recordSetup )
         continue
       if recordSetup not in self.dsClients:
         gLogger.info( "Creating DataStore client for %s" % recordSetup )
         self.dsClients[ recordSetup ] = DataStoreClient( setup = recordSetup, retryGraceTime = 900 )
       record = record[1:]
       rD = {}
       for fV in self.__summaryDefinedFields:
         rD[ fV[0] ] = fV[1]
       for iP in range( len( self.__summaryKeyFieldsMapping ) ):
         fieldName = self.__summaryKeyFieldsMapping[iP]
         rD[ self.__renameFieldsMapping.get( fieldName, fieldName ) ] = record[iP]
       record = record[ len( self.__summaryKeyFieldsMapping ): ]
       for iP in range( len( self.__summaryValueFieldsMapping ) ):
         rD[ self.__summaryValueFieldsMapping[iP] ] = int( record[iP] )
       acWMS = WMSHistory()
       acWMS.setStartTime( now )
       acWMS.setEndTime( now )
       acWMS.setValuesFromDict( rD )
       retVal = acWMS.checkValues()
       if not retVal[ 'OK' ]:
         gLogger.error( "Invalid accounting record ", "%s -> %s" % ( retVal[ 'Message' ], rD ) )
       else:
         self.dsClients[ recordSetup ].addRegister( acWMS )
     for setup in self.dsClients:
       gLogger.info( "Sending records for setup %s" % setup )
       result = self.dsClients[ setup ].commit()
       if not result[ 'OK' ]:
         gLogger.error( "Couldn't commit wms history for setup %s" % setup, result[ 'Message' ] )
       else:
         gLogger.info( "Sent %s records for setup %s" % ( result[ 'Value' ], setup ) )
   return S_OK()
Exemple #4
0
class WMSHistoryPlotter(BaseReporter):

    _typeName = "WMSHistory"
    _typeKeyFields = [dF[0] for dF in WMSHistory().definitionKeyFields]

    def _translateGrouping(self, grouping):
        if grouping == "Country":
            sqlRepr = 'upper( substring( %s, locate( ".", %s, length( %s ) - 4 ) + 1 ) )'
            return (sqlRepr, ['Site', 'Site', 'Site'], sqlRepr)
        elif grouping == "Grid":
            return ('substring_index( %s, ".", 1 )', ['Site'])
        else:
            return ("%s", [grouping])

    def _reportNumberOfJobs(self, reportRequest):
        selectFields = (
            self._getSelectStringForGrouping(reportRequest['groupingFields']) +
            ", %s, %s, SUM(%s/%s)", reportRequest['groupingFields'][1] +
            ['startTime', 'bucketLength', 'Jobs', 'entriesInBucket'])
        retVal = self._getTimedData(reportRequest['startTime'],
                                    reportRequest['endTime'], selectFields,
                                    reportRequest['condDict'],
                                    reportRequest['groupingFields'], {
                                        'convertToGranularity': 'average',
                                        'checkNone': True
                                    })
        if not retVal['OK']:
            return retVal
        dataDict, granularity = retVal['Value']
        self.stripDataField(dataDict, 0)
        return S_OK({'data': dataDict, 'granularity': granularity})

    def _plotNumberOfJobs(self, reportRequest, plotInfo, filename):
        metadata = {
            'title': 'Jobs by %s' % reportRequest['grouping'],
            'starttime': reportRequest['startTime'],
            'endtime': reportRequest['endTime'],
            'span': plotInfo['granularity'],
            'skipEdgeColor': True,
            'ylabel': "jobs"
        }
        plotInfo['data'] = self._fillWithZero(plotInfo['granularity'],
                                              reportRequest['startTime'],
                                              reportRequest['endTime'],
                                              plotInfo['data'])
        return self._generateStackedLinePlot(filename, plotInfo['data'],
                                             metadata)

    def _reportNumberOfReschedules(self, reportRequest):
        selectFields = (
            self._getSelectStringForGrouping(reportRequest['groupingFields']) +
            ", %s, %s, SUM(%s/%s)", reportRequest['groupingFields'][1] +
            ['startTime', 'bucketLength', 'Reschedules', 'entriesInBucket'])
        retVal = self._getTimedData(reportRequest['startTime'],
                                    reportRequest['endTime'], selectFields,
                                    reportRequest['condDict'],
                                    reportRequest['groupingFields'], {
                                        'convertToGranularity': 'average',
                                        'checkNone': True
                                    })
        if not retVal['OK']:
            return retVal
        dataDict, granularity = retVal['Value']
        self.stripDataField(dataDict, 0)
        return S_OK({'data': dataDict, 'granularity': granularity})

    def _plotNumberOfReschedules(self, reportRequest, plotInfo, filename):
        metadata = {
            'title': 'Reschedules by %s' % reportRequest['grouping'],
            'starttime': reportRequest['startTime'],
            'endtime': reportRequest['endTime'],
            'span': plotInfo['granularity'],
            'skipEdgeColor': True,
            'ylabel': "reschedules"
        }
        plotInfo['data'] = self._fillWithZero(plotInfo['granularity'],
                                              reportRequest['startTime'],
                                              reportRequest['endTime'],
                                              plotInfo['data'])
        return self._generateStackedLinePlot(filename, plotInfo['data'],
                                             metadata)

    def _reportAverageNumberOfJobs(self, reportRequest):
        selectFields = (
            self._getSelectStringForGrouping(reportRequest['groupingFields']) +
            ", SUM(%s/%s)",
            reportRequest['groupingFields'][1] + ['Jobs', 'entriesInBucket'])
        retVal = self._getSummaryData(reportRequest['startTime'],
                                      reportRequest['endTime'], selectFields,
                                      reportRequest['condDict'],
                                      reportRequest['groupingFields'], {})
        if not retVal['OK']:
            return retVal
        dataDict = retVal['Value']
        bins = self._getBins(self._typeName, reportRequest['startTime'],
                             reportRequest['endTime'])
        numBins = len(bins)
        for key in dataDict:
            dataDict[key] = float(dataDict[key] / numBins)
        return S_OK({'data': dataDict})

    def _plotAverageNumberOfJobs(self, reportRequest, plotInfo, filename):
        metadata = {
            'title':
            'Average Number of Jobs by %s' % reportRequest['grouping'],
            'ylabel': 'Jobs',
            'starttime': reportRequest['startTime'],
            'endtime': reportRequest['endTime']
        }
        return self._generatePiePlot(filename, plotInfo['data'], metadata)
    def execute(self):
        """Main execution method"""
        # PilotsHistory to Monitoring
        if "Monitoring" in self.pilotMonitoringOption:
            self.log.info("Committing PilotsHistory to Monitoring")
            result = PilotAgentsDB().getSummarySnapshot()
            now = datetime.datetime.utcnow()
            if not result["OK"]:
                self.log.error(
                    "Can't get the PilotAgentsDB summary",
                    "%s: won't commit PilotsHistory at this cycle" %
                    result["Message"],
                )

            values = result["Value"][1]
            for record in values:
                rD = {}
                for iP, _ in enumerate(self.__pilotsMapping):
                    rD[self.__pilotsMapping[iP]] = record[iP]
                rD["timestamp"] = int(TimeUtilities.toEpoch(now))
                self.pilotReporter.addRecord(rD)

            self.log.info("Committing to Monitoring...")
            result = self.pilotReporter.commit()
            if not result["OK"]:
                self.log.error("Could not commit to Monitoring",
                               result["Message"])
            self.log.verbose("Done committing PilotsHistory to Monitoring")

        # WMSHistory to Monitoring or Accounting
        self.log.info("Committing WMSHistory to %s backend" %
                      "and ".join(self.jobMonitoringOption))
        result = JobDB().getSummarySnapshot(self.__jobDBFields)
        now = datetime.datetime.utcnow()
        if not result["OK"]:
            self.log.error(
                "Can't get the JobDB summary",
                "%s: won't commit WMSHistory at this cycle" %
                result["Message"])
            return S_ERROR()

        values = result["Value"][1]

        self.log.info("Start sending WMSHistory records")
        for record in values:
            record = record[1:]
            rD = {}
            for fV in self.__summaryDefinedFields:
                rD[fV[0]] = fV[1]
            for iP, _ in enumerate(self.__summaryKeyFieldsMapping):
                fieldName = self.__summaryKeyFieldsMapping[iP]
                rD[self.__renameFieldsMapping.get(fieldName,
                                                  fieldName)] = record[iP]
            record = record[len(self.__summaryKeyFieldsMapping):]
            for iP, _ in enumerate(self.__summaryValueFieldsMapping):
                rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP])

            for backend in self.datastores:
                if backend.lower() == "monitoring":
                    rD["timestamp"] = int(TimeUtilities.toEpoch(now))
                    self.datastores["Monitoring"].addRecord(rD)

                elif backend.lower() == "accounting":
                    acWMS = WMSHistory()
                    acWMS.setStartTime(now)
                    acWMS.setEndTime(now)
                    acWMS.setValuesFromDict(rD)
                    retVal = acWMS.checkValues()
                    if not retVal["OK"]:
                        self.log.error("Invalid WMSHistory accounting record ",
                                       "%s -> %s" % (retVal["Message"], rD))
                    else:
                        self.datastores["Accounting"].addRegister(acWMS)

        for backend, datastore in self.datastores.items():
            self.log.info("Committing WMSHistory records to %s backend" %
                          backend)
            result = datastore.commit()
            if not result["OK"]:
                self.log.error("Couldn't commit WMSHistory to %s" % backend,
                               result["Message"])
                return S_ERROR()
            self.log.verbose("Done committing WMSHistory to %s backend" %
                             backend)

        return S_OK()
Exemple #6
0
    def execute(self):
        """ Main execution method
    """
        result = gConfig.getSections("/DIRAC/Setups")
        if not result['OK']:
            return result
        validSetups = result['Value']
        self.log.info("Valid setups for this cycle are %s" %
                      ", ".join(validSetups))
        # Get the WMS Snapshot!
        result = self.jobDB.getSummarySnapshot(self.__jobDBFields)
        now = Time.dateTime()
        if not result['OK']:
            self.log.error(
                "Can't get the JobDB summary",
                "%s: won't commit at this cycle" % result['Message'])
        else:
            values = result['Value'][1]

            if self.retryOnce:
                self.log.verbose(
                    "Adding to records to commit those not committed within the previous cycle"
                )
            acWMSListAdded = []

            for record in values:
                recordSetup = record[0]
                if recordSetup not in validSetups:
                    self.log.error("Setup %s is not valid" % recordSetup)
                    continue
                if recordSetup not in self.dsClients:
                    self.log.info("Creating DataStore client for %s" %
                                  recordSetup)
                    self.dsClients[recordSetup] = DataStoreClient(
                        retryGraceTime=900)
                record = record[1:]
                rD = {}
                for fV in self.__summaryDefinedFields:
                    rD[fV[0]] = fV[1]
                for iP in range(len(self.__summaryKeyFieldsMapping)):
                    fieldName = self.__summaryKeyFieldsMapping[iP]
                    rD[self.__renameFieldsMapping.get(fieldName,
                                                      fieldName)] = record[iP]
                record = record[len(self.__summaryKeyFieldsMapping):]
                for iP in range(len(self.__summaryValueFieldsMapping)):
                    rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP])
                acWMS = WMSHistory()
                acWMS.setStartTime(now)
                acWMS.setEndTime(now)
                acWMS.setValuesFromDict(rD)
                retVal = acWMS.checkValues()
                if not retVal['OK']:
                    self.log.error("Invalid accounting record ",
                                   "%s -> %s" % (retVal['Message'], rD))
                else:
                    self.dsClients[recordSetup].addRegister(acWMS)
                    acWMSListAdded.append(acWMS)

            if self.retryOnce and self.retryValues:
                for acWMSCumulated in self.retryValues:
                    retVal = acWMSCumulated.checkValues()
                    if not retVal['OK']:
                        self.log.error("Invalid accounting record ",
                                       "%s" % (retVal['Message']))
                    else:
                        self.dsClients[recordSetup].addRegister(acWMSCumulated)

            for setup in self.dsClients:
                self.log.info("Sending records for setup %s" % setup)
                result = self.dsClients[setup].commit()
                if not result['OK']:
                    self.log.error(
                        "Couldn't commit wms history for setup %s" % setup,
                        result['Message'])
                    # Re-creating the client: for new connection, and for avoiding accumulating too large of a backlog
                    self.dsClients[setup] = DataStoreClient(retryGraceTime=900)
                    if not self.retryOnce:
                        self.log.info("Will try again at next cycle")
                        self.retryOnce = True
                        self.retryValues = acWMSListAdded
                    else:
                        self.log.warn("Won't retry one more time")
                        self.retryOnce = False
                        self.retryValues = []
                else:
                    self.log.info("Sent %s records for setup %s" %
                                  (result['Value'], setup))
                    self.retryOnce = False
        return S_OK()
Exemple #7
0
class WMSHistoryPlotter(BaseReporter):

    _typeName = "WMSHistory"
    _typeKeyFields = [dF[0] for dF in WMSHistory().definitionKeyFields]

    def _translateGrouping(self, grouping):
        if grouping == "Country":
            sqlRepr = 'upper( substring( %s, locate( ".", %s, length( %s ) - 4 ) + 1 ) )'
            return (sqlRepr, ["Site", "Site", "Site"], sqlRepr)
        elif grouping == "Grid":
            return ('substring_index( %s, ".", 1 )', ["Site"])
        else:
            return ("%s", [grouping])

    def _reportNumberOfJobs(self, reportRequest):
        selectFields = (
            self._getSelectStringForGrouping(reportRequest["groupingFields"]) +
            ", %s, %s, SUM(%s/%s)",
            reportRequest["groupingFields"][1] +
            ["startTime", "bucketLength", "Jobs", "entriesInBucket"],
        )
        retVal = self._getTimedData(
            reportRequest["startTime"],
            reportRequest["endTime"],
            selectFields,
            reportRequest["condDict"],
            reportRequest["groupingFields"],
            {
                "convertToGranularity": "average",
                "checkNone": True
            },
        )
        if not retVal["OK"]:
            return retVal
        dataDict, granularity = retVal["Value"]
        self.stripDataField(dataDict, 0)
        return S_OK({"data": dataDict, "granularity": granularity})

    def _plotNumberOfJobs(self, reportRequest, plotInfo, filename):
        metadata = {
            "title": "Jobs by %s" % reportRequest["grouping"],
            "starttime": reportRequest["startTime"],
            "endtime": reportRequest["endTime"],
            "span": plotInfo["granularity"],
            "skipEdgeColor": True,
            "ylabel": "jobs",
        }
        plotInfo["data"] = self._fillWithZero(plotInfo["granularity"],
                                              reportRequest["startTime"],
                                              reportRequest["endTime"],
                                              plotInfo["data"])
        return self._generateStackedLinePlot(filename, plotInfo["data"],
                                             metadata)

    def _reportNumberOfReschedules(self, reportRequest):
        selectFields = (
            self._getSelectStringForGrouping(reportRequest["groupingFields"]) +
            ", %s, %s, SUM(%s/%s)",
            reportRequest["groupingFields"][1] +
            ["startTime", "bucketLength", "Reschedules", "entriesInBucket"],
        )
        retVal = self._getTimedData(
            reportRequest["startTime"],
            reportRequest["endTime"],
            selectFields,
            reportRequest["condDict"],
            reportRequest["groupingFields"],
            {
                "convertToGranularity": "average",
                "checkNone": True
            },
        )
        if not retVal["OK"]:
            return retVal
        dataDict, granularity = retVal["Value"]
        self.stripDataField(dataDict, 0)
        return S_OK({"data": dataDict, "granularity": granularity})

    def _plotNumberOfReschedules(self, reportRequest, plotInfo, filename):
        metadata = {
            "title": "Reschedules by %s" % reportRequest["grouping"],
            "starttime": reportRequest["startTime"],
            "endtime": reportRequest["endTime"],
            "span": plotInfo["granularity"],
            "skipEdgeColor": True,
            "ylabel": "reschedules",
        }
        plotInfo["data"] = self._fillWithZero(plotInfo["granularity"],
                                              reportRequest["startTime"],
                                              reportRequest["endTime"],
                                              plotInfo["data"])
        return self._generateStackedLinePlot(filename, plotInfo["data"],
                                             metadata)

    def _reportAverageNumberOfJobs(self, reportRequest):
        selectFields = (
            self._getSelectStringForGrouping(reportRequest["groupingFields"]) +
            ", SUM(%s/%s)",
            reportRequest["groupingFields"][1] + ["Jobs", "entriesInBucket"],
        )
        retVal = self._getSummaryData(
            reportRequest["startTime"],
            reportRequest["endTime"],
            selectFields,
            reportRequest["condDict"],
            reportRequest["groupingFields"],
            {},
        )
        if not retVal["OK"]:
            return retVal
        dataDict = retVal["Value"]
        bins = self._getBins(self._typeName, reportRequest["startTime"],
                             reportRequest["endTime"])
        numBins = len(bins)
        for key in dataDict:
            dataDict[key] = float(dataDict[key] / numBins)
        return S_OK({"data": dataDict})

    def _plotAverageNumberOfJobs(self, reportRequest, plotInfo, filename):
        metadata = {
            "title":
            "Average Number of Jobs by %s" % reportRequest["grouping"],
            "ylabel": "Jobs",
            "starttime": reportRequest["startTime"],
            "endtime": reportRequest["endTime"],
        }
        return self._generatePiePlot(filename, plotInfo["data"], metadata)
  def execute( self ):
    """ Main execution method
    """
    result = gConfig.getSections( "/DIRAC/Setups" )
    if not result[ 'OK' ]:
      return result
    validSetups = result[ 'Value' ]
    self.log.info( "Valid setups for this cycle are %s" % ", ".join( validSetups ) )
    #Get the WMS Snapshot!
    result = self.jobDB.getSummarySnapshot( self.__jobDBFields )
    now = Time.dateTime()
    if not result[ 'OK' ]:
      self.log.error( "Can't get the JobDB summary", "%s: won't commit at this cycle" % result[ 'Message' ] )
    else:
      values = result[ 'Value' ][1]

      if self.retryOnce:
        self.log.verbose( "Adding to records to commit those not committed within the previous cycle" )
      acWMSListAdded = []

      for record in values:
        recordSetup = record[0]
        if recordSetup not in validSetups:
          self.log.error( "Setup %s is not valid" % recordSetup )
          continue
        if recordSetup not in self.dsClients:
          self.log.info( "Creating DataStore client for %s" % recordSetup )
          self.dsClients[ recordSetup ] = DataStoreClient( setup = recordSetup, retryGraceTime = 900 )
        record = record[1:]
        rD = {}
        for fV in self.__summaryDefinedFields:
          rD[ fV[0] ] = fV[1]
        for iP in range( len( self.__summaryKeyFieldsMapping ) ):
          fieldName = self.__summaryKeyFieldsMapping[iP]
          rD[ self.__renameFieldsMapping.get( fieldName, fieldName ) ] = record[iP]
        record = record[ len( self.__summaryKeyFieldsMapping ): ]
        for iP in range( len( self.__summaryValueFieldsMapping ) ):
          rD[ self.__summaryValueFieldsMapping[iP] ] = int( record[iP] )
        acWMS = WMSHistory()
        acWMS.setStartTime( now )
        acWMS.setEndTime( now )
        acWMS.setValuesFromDict( rD )
        retVal = acWMS.checkValues()
        if not retVal[ 'OK' ]:
          self.log.error( "Invalid accounting record ", "%s -> %s" % ( retVal[ 'Message' ], rD ) )
        else:
          self.dsClients[ recordSetup ].addRegister( acWMS )
          acWMSListAdded.append( acWMS )

      if self.retryOnce and self.retryValues:
        for acWMSCumulated in self.retryValues:
          retVal = acWMSCumulated.checkValues()
          if not retVal[ 'OK' ]:
            self.log.error( "Invalid accounting record ", "%s" % ( retVal[ 'Message' ] ) )
          else:
            self.dsClients[ recordSetup ].addRegister( acWMSCumulated )

      for setup in self.dsClients:
        self.log.info( "Sending records for setup %s" % setup )
        result = self.dsClients[ setup ].commit()
        if not result[ 'OK' ]:
          self.log.error( "Couldn't commit wms history for setup %s" % setup, result[ 'Message' ] )
          # Re-creating the client: for new connection, and for avoiding accumulating too large of a backlog
          self.dsClients[ setup ] = DataStoreClient( setup = setup, retryGraceTime = 900 )
          if not self.retryOnce:
            self.log.info( "Will try again at next cycle" )
            self.retryOnce = True
            self.retryValues = acWMSListAdded
          else:
            self.log.warn( "Won't retry one more time" )
            self.retryOnce = False
            self.retryValues = []
        else:
          self.log.info( "Sent %s records for setup %s" % ( result[ 'Value' ], setup ) )
          self.retryOnce = False
    return S_OK()