def monitorStageRequests(self): """ This is the third logical task manages the StageSubmitted->Staged transition of the Replicas """ res = self.__getStageSubmittedReplicas() if not res['OK']: gLogger.fatal( "StageMonitor.monitorStageRequests: Failed to get replicas from StorageManagementDB.", res['Message']) return res if not res['Value']: gLogger.info( "StageMonitor.monitorStageRequests: There were no StageSubmitted replicas found" ) return res seReplicas = res['Value']['SEReplicas'] replicaIDs = res['Value']['ReplicaIDs'] gLogger.info( "StageMonitor.monitorStageRequests: Obtained %s StageSubmitted replicas for monitoring." % len(replicaIDs)) for storageElement, seReplicaIDs in seReplicas.items(): self.__monitorStorageElementStageRequests(storageElement, seReplicaIDs, replicaIDs) gDataStoreClient.commit() return S_OK()
def test_addAndRemoveDataOperation(): # just inserting one record record = createDataOperationAccountingRecord() record.setStartTime() record.setEndTime() res = gDataStoreClient.addRegister(record) assert res['OK'] res = gDataStoreClient.commit() assert res['OK'] rc = ReportsClient() res = rc.listReports('DataOperation') assert res['OK'] res = rc.listUniqueKeyValues('DataOperation') assert res['OK'] res = rc.getReport('DataOperation', 'Successful transfers', datetime.datetime.utcnow(), datetime.datetime.utcnow(), {}, 'Destination') assert res['OK'] # now removing that record res = gDataStoreClient.remove(record) assert res['OK']
def test_addAndRemove(): # just inserting one record record = createAccountingRecord() record.setStartTime() record.setEndTime() res = gDataStoreClient.addRegister(record) assert res['OK'] res = gDataStoreClient.commit() assert res['OK'] rc = ReportsClient() res = rc.listReports('DataOperation') assert res['OK'] res = rc.listUniqueKeyValues('DataOperation') assert res['OK'] res = rc.getReport('DataOperation', 'Successful transfers', datetime.datetime.utcnow(), datetime.datetime.utcnow(), {}, 'Destination') assert res['OK'] # now removing that record res = gDataStoreClient.remove(record) assert res['OK']
def test_addAndRemoveStorageOccupancy(): # just inserting one record record = createStorageOccupancyAccountingRecord() record.setStartTime() record.setEndTime() res = gDataStoreClient.addRegister(record) assert res['OK'] res = gDataStoreClient.commit() assert res['OK'] rc = ReportsClient() res = rc.listReports('StorageOccupancy') assert res['OK'] res = rc.listUniqueKeyValues('StorageOccupancy') assert res['OK'] res = rc.getReport('StorageOccupancy', 'Free and Used Space', datetime.datetime.utcnow(), datetime.datetime.utcnow(), {}, 'StorageElement') assert res['OK'] # now removing that record res = gDataStoreClient.remove(record) assert res['OK']
def accountPilots(self, pilotsToAccount, connection): """ account for pilots """ accountingFlag = False pae = self.am_getOption('PilotAccountingEnabled', 'yes') if pae.lower() == "yes": accountingFlag = True if not pilotsToAccount: self.log.info('No pilots to Account') return S_OK() accountingSent = False if accountingFlag: retVal = self.pilotDB.getPilotInfo(pilotsToAccount.keys(), conn=connection) if not retVal['OK']: self.log.error('Fail to retrieve Info for pilots', retVal['Message']) return retVal dbData = retVal['Value'] for pref in dbData: if pref in pilotsToAccount: if dbData[pref]['Status'] not in self.finalStateList: dbData[pref]['Status'] = pilotsToAccount[pref][ 'Status'] dbData[pref]['DestinationSite'] = pilotsToAccount[ pref]['DestinationSite'] dbData[pref]['LastUpdateTime'] = pilotsToAccount[pref][ 'StatusDate'] retVal = self.__addPilotsAccountingReport(dbData) if not retVal['OK']: self.log.error('Fail to retrieve Info for pilots', retVal['Message']) return retVal self.log.info("Sending accounting records...") retVal = gDataStoreClient.commit() if not retVal['OK']: self.log.error("Can't send accounting reports", retVal['Message']) else: self.log.info("Accounting sent for %s pilots" % len(pilotsToAccount)) accountingSent = True if not accountingFlag or accountingSent: for pRef in pilotsToAccount: pDict = pilotsToAccount[pRef] self.log.verbose('Setting Status for %s to %s' % (pRef, pDict['Status'])) self.pilotDB.setPilotStatus(pRef, pDict['Status'], pDict['DestinationSite'], pDict['StatusDate'], conn=connection) return S_OK()
def accountPilots(self, pilotsToAccount, connection): """account for pilots""" accountingFlag = False pae = self.am_getOption("PilotAccountingEnabled", "yes") if pae.lower() == "yes": accountingFlag = True if not pilotsToAccount: self.log.info("No pilots to Account") return S_OK() accountingSent = False if accountingFlag: retVal = self.pilotDB.getPilotInfo(list(pilotsToAccount), conn=connection) if not retVal["OK"]: self.log.error("Fail to retrieve Info for pilots", retVal["Message"]) return retVal dbData = retVal["Value"] for pref in dbData: if pref in pilotsToAccount: if dbData[pref][ "Status"] not in PilotStatus.PILOT_FINAL_STATES: dbData[pref]["Status"] = pilotsToAccount[pref][ "Status"] dbData[pref]["DestinationSite"] = pilotsToAccount[ pref]["DestinationSite"] dbData[pref]["LastUpdateTime"] = pilotsToAccount[pref][ "StatusDate"] retVal = self._addPilotsAccountingReport(dbData) if not retVal["OK"]: self.log.error("Fail to retrieve Info for pilots", retVal["Message"]) return retVal self.log.info("Sending accounting records...") retVal = gDataStoreClient.commit() if not retVal["OK"]: self.log.error("Can't send accounting reports", retVal["Message"]) else: self.log.info("Accounting sent for %s pilots" % len(pilotsToAccount)) accountingSent = True if not accountingFlag or accountingSent: for pRef in pilotsToAccount: pDict = pilotsToAccount[pRef] self.log.verbose("Setting Status for %s to %s" % (pRef, pDict["Status"])) self.pilotDB.setPilotStatus(pRef, pDict["Status"], pDict["DestinationSite"], pDict["StatusDate"], conn=connection) return S_OK()
def commit( self ): """ Commit register to server """ retVal = gDataStoreClient.addRegister( self ) if not retVal[ 'OK' ]: return retVal return gDataStoreClient.commit()
def commit(self): """ Commit register to server """ retVal = gDataStoreClient.addRegister(self) if not retVal['OK']: return retVal return gDataStoreClient.commit()
def _storeCommand(self, results): """ _storeCommand Adding records to accounting, on top of what does the derived method. :param dict results: something like {'ElementName': 'CERN-HIST-EOS', 'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server', 'Free': 3264963586.10073, 'Total': 8000000000.0, 'SpaceReservation': 'LHCb-Disk'} :returns: S_OK/S_ERROR dict """ res = super(FreeDiskSpaceCommand, self)._storeCommand(results) if not res['OK']: return res siteRes = DMSHelpers().getLocalSiteForSE(results['ElementName']) if not siteRes['OK']: return siteRes if not siteRes['Value']: return S_OK() spaceReservation = results.get('SpaceReservation') accountingDict = { 'SpaceToken': spaceReservation, 'Endpoint': results['Endpoint'], 'Site': siteRes['Value'] } results['Used'] = results['Total'] - results['Free'] for sType in ['Total', 'Free', 'Used']: spaceTokenAccounting = SpaceToken() spaceTokenAccounting.setNowAsStartAndEndTime() spaceTokenAccounting.setValuesFromDict(accountingDict) spaceTokenAccounting.setValueByKey('SpaceType', sType) spaceTokenAccounting.setValueByKey( 'Space', int(convertSizeUnits(results[sType], 'MB', 'B'))) gDataStoreClient.addRegister(spaceTokenAccounting) gDataStoreClient.commit() return S_OK()
def sendPilotAccounting(self, pilotDict): """ Send pilot accounting record """ for pRef in pilotDict: self.log.verbose('Preparing accounting record for pilot %s' % pRef) pA = PilotAccounting() pA.setEndTime(pilotDict[pRef]['LastUpdateTime']) pA.setStartTime(pilotDict[pRef]['SubmissionTime']) retVal = CS.getUsernameForDN(pilotDict[pRef]['OwnerDN']) if not retVal['OK']: userName = '******' self.log.error("Can't determine username for dn:", pilotDict[pRef]['OwnerDN']) else: userName = retVal['Value'] pA.setValueByKey('User', userName) pA.setValueByKey('UserGroup', pilotDict[pRef]['OwnerGroup']) result = getSiteForCE(pilotDict[pRef]['DestinationSite']) if result['OK'] and result['Value'].strip(): pA.setValueByKey('Site', result['Value'].strip()) else: pA.setValueByKey('Site', 'Unknown') pA.setValueByKey('GridCE', pilotDict[pRef]['DestinationSite']) pA.setValueByKey('GridMiddleware', pilotDict[pRef]['GridType']) pA.setValueByKey('GridResourceBroker', pilotDict[pRef]['Broker']) pA.setValueByKey('GridStatus', pilotDict[pRef]['Status']) if not 'Jobs' in pilotDict[pRef]: pA.setValueByKey('Jobs', 0) else: pA.setValueByKey('Jobs', len(pilotDict[pRef]['Jobs'])) self.log.info("Adding accounting record for pilot %s" % pilotDict[pRef]['PilotID']) retVal = gDataStoreClient.addRegister(pA) if not retVal['OK']: self.log.error('Failed to send accounting info for pilot ', pRef) else: # Set up AccountingSent flag result = pilotAgentsDB.setAccountingFlag(pRef) if not result['OK']: self.log.error('Failed to set accounting flag for pilot ', pRef) self.log.info('Committing accounting records for %d pilots' % len(pilotDict)) result = gDataStoreClient.commit() if result['OK']: for pRef in pilotDict: self.log.verbose('Setting AccountingSent flag for pilot %s' % pRef) result = pilotAgentsDB.setAccountingFlag(pRef) if not result['OK']: self.log.error('Failed to set accounting flag for pilot ', pRef) else: return result return S_OK()
def __commitRecords( self ): if self.recordsToCommit: res = gDataStoreClient.commit() if not res[ 'OK' ]: self.log.error( "Accounting ERROR: commit returned %s" % res ) else: self.log.notice( "%d records committed " % self.recordsToCommit ) self.recordsToCommit = 0 self.log.notice( "commit for DataStorage returned: %s" % res )
def monitorStageRequests( self ): """ This is the third logical task manages the StageSubmitted->Staged transition of the Replicas """ res = self.__getStageSubmittedReplicas() if not res['OK']: gLogger.fatal( "StageMonitor.monitorStageRequests: Failed to get replicas from StorageManagementDB.", res['Message'] ) return res if not res['Value']: gLogger.info( "StageMonitor.monitorStageRequests: There were no StageSubmitted replicas found" ) return res seReplicas = res['Value']['SEReplicas'] replicaIDs = res['Value']['ReplicaIDs'] gLogger.info( "StageMonitor.monitorStageRequests: Obtained %s StageSubmitted replicas for monitoring." % len( replicaIDs ) ) for storageElement, seReplicaIDs in seReplicas.items(): self.__monitorStorageElementStageRequests( storageElement, seReplicaIDs, replicaIDs ) gDataStoreClient.commit() return S_OK()
def commitAccounting(): result = gDataStoreClient.commit() sLog.debug( "Concluding the sending and committing data operation to accounting" ) if not result["OK"]: sLog.error("Could not commit data operation to accounting", result["Message"]) sLog.debug("Committing to accounting concluded") return result
def __commitAccounting(self): res = gDataStoreClient.commit() if not res['OK']: self.log.error( "while committing %d Popularity records" % self.numPopRows, res['Message']) else: self.log.info( "%s records for Popularity type successfully committed" % self.numPopRows) self.numPopRows = 0 return res
def _storeCommand(self, results): """ Stores the results in the cache (SpaceTokenOccupancyCache), and adds records to the StorageOccupancy accounting. :param dict results: something like {'ElementName': 'CERN-HIST-EOS', 'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server', 'Free': 3264963586.10073, 'Total': 8000000000.0, 'SpaceReservation': 'LHCb-Disk'} :returns: S_OK/S_ERROR dict """ # Stores in cache res = self.rmClient.addOrModifySpaceTokenOccupancyCache( endpoint=results['Endpoint'], lastCheckTime=datetime.utcnow(), free=results['Free'], total=results['Total'], token=results['ElementName']) if not res['OK']: self.log.error("Error calling addOrModifySpaceTokenOccupancyCache", res['Message']) return res # Now proceed with the accounting siteRes = DMSHelpers().getLocalSiteForSE(results['ElementName']) if not siteRes['OK']: return siteRes accountingDict = { 'StorageElement': results['ElementName'], 'Endpoint': results['Endpoint'], 'Site': siteRes['Value'] if siteRes['Value'] else 'unassigned' } results['Used'] = results['Total'] - results['Free'] for sType in ['Total', 'Free', 'Used']: spaceTokenAccounting = StorageOccupancy() spaceTokenAccounting.setNowAsStartAndEndTime() spaceTokenAccounting.setValuesFromDict(accountingDict) spaceTokenAccounting.setValueByKey('SpaceType', sType) spaceTokenAccounting.setValueByKey( 'Space', int(convertSizeUnits(results[sType], 'MB', 'B'))) res = gDataStoreClient.addRegister(spaceTokenAccounting) if not res['OK']: self.log.warn("Could not commit register", res['Message']) continue return gDataStoreClient.commit()
def _storeCommand(self, results): """ Stores the results in the cache (SpaceTokenOccupancyCache), and adds records to the StorageOccupancy accounting. :param dict results: something like {'ElementName': 'CERN-HIST-EOS', 'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server', 'Free': 3264963586.10073, 'Total': 8000000000.0} :returns: S_OK/S_ERROR dict """ # Stores in cache res = self.rmClient.addOrModifySpaceTokenOccupancyCache( endpoint=results["Endpoint"], lastCheckTime=datetime.utcnow(), free=results["Free"], total=results["Total"], token=results["ElementName"], ) if not res["OK"]: self.log.error("Error calling addOrModifySpaceTokenOccupancyCache", res["Message"]) return res # Now proceed with the accounting siteRes = DMSHelpers().getLocalSiteForSE(results["ElementName"]) if not siteRes["OK"]: return siteRes accountingDict = { "StorageElement": results["ElementName"], "Endpoint": results["Endpoint"], "Site": siteRes["Value"] if siteRes["Value"] else "unassigned", } results["Used"] = results["Total"] - results["Free"] for sType in ["Total", "Free", "Used"]: spaceTokenAccounting = StorageOccupancy() spaceTokenAccounting.setNowAsStartAndEndTime() spaceTokenAccounting.setValuesFromDict(accountingDict) spaceTokenAccounting.setValueByKey("SpaceType", sType) spaceTokenAccounting.setValueByKey( "Space", int(convertSizeUnits(results[sType], "MB", "B"))) res = gDataStoreClient.addRegister(spaceTokenAccounting) if not res["OK"]: self.log.warn("Could not commit register", res["Message"]) continue return gDataStoreClient.commit()
def test_addAndRemoveStorageOccupancy(): # just inserting one record record = createStorageOccupancyAccountingRecord() record.setStartTime() record.setEndTime() res = gDataStoreClient.addRegister(record) assert res['OK'] res = gDataStoreClient.commit() assert res['OK'] # now removing that record res = gDataStoreClient.remove(record) assert res['OK']
def test_addAndRemoveDataperation(): # just inserting one record record = createDataOperationAccountingRecord() record.setStartTime() record.setEndTime() res = gDataStoreClient.addRegister(record) assert res["OK"] res = gDataStoreClient.commit() assert res["OK"] # now removing that record res = gDataStoreClient.remove(record) assert res["OK"]
def sendPilotAccounting( self, pilotDict ): """ Send pilot accounting record """ for pRef in pilotDict: self.log.verbose( 'Preparing accounting record for pilot %s' % pRef ) pA = PilotAccounting() pA.setEndTime( pilotDict[pRef][ 'LastUpdateTime' ] ) pA.setStartTime( pilotDict[pRef][ 'SubmissionTime' ] ) retVal = CS.getUsernameForDN( pilotDict[pRef][ 'OwnerDN' ] ) if not retVal[ 'OK' ]: userName = '******' self.log.error( "Can't determine username for dn:", pilotDict[pRef][ 'OwnerDN' ] ) else: userName = retVal[ 'Value' ] pA.setValueByKey( 'User', userName ) pA.setValueByKey( 'UserGroup', pilotDict[pRef][ 'OwnerGroup' ] ) result = getSiteForCE( pilotDict[pRef][ 'DestinationSite' ] ) if result['OK'] and result[ 'Value' ].strip(): pA.setValueByKey( 'Site', result['Value'].strip() ) else: pA.setValueByKey( 'Site', 'Unknown' ) pA.setValueByKey( 'GridCE', pilotDict[pRef][ 'DestinationSite' ] ) pA.setValueByKey( 'GridMiddleware', pilotDict[pRef][ 'GridType' ] ) pA.setValueByKey( 'GridResourceBroker', pilotDict[pRef][ 'Broker' ] ) pA.setValueByKey( 'GridStatus', pilotDict[pRef][ 'Status' ] ) if not 'Jobs' in pilotDict[pRef]: pA.setValueByKey( 'Jobs', 0 ) else: pA.setValueByKey( 'Jobs', len( pilotDict[pRef]['Jobs'] ) ) self.log.verbose( "Adding accounting record for pilot %s" % pilotDict[pRef][ 'PilotID' ] ) retVal = gDataStoreClient.addRegister( pA ) if not retVal[ 'OK' ]: self.log.error( 'Failed to send accounting info for pilot ', pRef ) else: # Set up AccountingSent flag result = pilotAgentsDB.setAccountingFlag( pRef ) if not result['OK']: self.log.error( 'Failed to set accounting flag for pilot ', pRef ) self.log.info( 'Committing accounting records for %d pilots' % len( pilotDict ) ) result = gDataStoreClient.commit() if result['OK']: for pRef in pilotDict: self.log.verbose( 'Setting AccountingSent flag for pilot %s' % pRef ) result = pilotAgentsDB.setAccountingFlag( pRef ) if not result['OK']: self.log.error( 'Failed to set accounting flag for pilot ', pRef ) else: return result return S_OK()
def sendPilotAccounting(self, pilotDict): """ Send pilot accounting record """ for pRef in pilotDict: self.log.verbose("Preparing accounting record for pilot %s" % pRef) pA = PilotAccounting() pA.setEndTime(pilotDict[pRef]["LastUpdateTime"]) pA.setStartTime(pilotDict[pRef]["SubmissionTime"]) retVal = CS.getUsernameForDN(pilotDict[pRef]["OwnerDN"]) if not retVal["OK"]: userName = "******" self.log.error("Can't determine username for dn:", pilotDict[pRef]["OwnerDN"]) else: userName = retVal["Value"] pA.setValueByKey("User", userName) pA.setValueByKey("UserGroup", pilotDict[pRef]["OwnerGroup"]) result = getSiteForCE(pilotDict[pRef]["DestinationSite"]) if result["OK"] and result["Value"].strip(): pA.setValueByKey("Site", result["Value"].strip()) else: pA.setValueByKey("Site", "Unknown") pA.setValueByKey("GridCE", pilotDict[pRef]["DestinationSite"]) pA.setValueByKey("GridMiddleware", pilotDict[pRef]["GridType"]) pA.setValueByKey("GridResourceBroker", pilotDict[pRef]["Broker"]) pA.setValueByKey("GridStatus", pilotDict[pRef]["Status"]) if not "Jobs" in pilotDict[pRef]: pA.setValueByKey("Jobs", 0) else: pA.setValueByKey("Jobs", len(pilotDict[pRef]["Jobs"])) self.log.info("Adding accounting record for pilot %s" % pilotDict[pRef]["PilotID"]) retVal = gDataStoreClient.addRegister(pA) if not retVal["OK"]: self.log.error("Failed to send accounting info for pilot ", pRef) else: # Set up AccountingSent flag result = pilotAgentsDB.setAccountingFlag(pRef) if not result["OK"]: self.log.error("Failed to set accounting flag for pilot ", pRef) self.log.info("Committing accounting records for %d pilots" % len(pilotDict)) result = gDataStoreClient.commit() if result["OK"]: for pRef in pilotDict: self.log.verbose("Setting AccountingSent flag for pilot %s" % pRef) result = pilotAgentsDB.setAccountingFlag(pRef) if not result["OK"]: self.log.error("Failed to set accounting flag for pilot ", pRef) else: return result return S_OK()
def accountPilots(self, pilotsToAccount, connection): """ account for pilots """ accountingFlag = False pae = self.am_getOption('PilotAccountingEnabled', 'yes') if pae.lower() == "yes": accountingFlag = True if not pilotsToAccount: self.log.info('No pilots to Account') return S_OK() accountingSent = False if accountingFlag: retVal = self.pilotDB.getPilotInfo(pilotsToAccount.keys(), conn=connection) if not retVal['OK']: self.log.error('Fail to retrieve Info for pilots', retVal['Message']) return retVal dbData = retVal['Value'] for pref in dbData: if pref in pilotsToAccount: if dbData[pref]['Status'] not in self.finalStateList: dbData[pref]['Status'] = pilotsToAccount[pref]['Status'] dbData[pref]['DestinationSite'] = pilotsToAccount[pref]['DestinationSite'] dbData[pref]['LastUpdateTime'] = pilotsToAccount[pref]['StatusDate'] retVal = self.__addPilotsAccountingReport(dbData) if not retVal['OK']: self.log.error('Fail to retrieve Info for pilots', retVal['Message']) return retVal self.log.info("Sending accounting records...") retVal = gDataStoreClient.commit() if not retVal['OK']: self.log.error("Can't send accounting reports", retVal['Message']) else: self.log.info("Accounting sent for %s pilots" % len(pilotsToAccount)) accountingSent = True if not accountingFlag or accountingSent: for pRef in pilotsToAccount: pDict = pilotsToAccount[pRef] self.log.verbose('Setting Status for %s to %s' % (pRef, pDict['Status'])) self.pilotDB.setPilotStatus(pRef, pDict['Status'], pDict['DestinationSite'], pDict['StatusDate'], conn=connection) return S_OK()
def test_addAndRemove(): # just inserting one record record = createAccountingRecord() record.setStartTime() record.setEndTime() res = gDataStoreClient.addRegister(record) assert res['OK'] res = gDataStoreClient.commit() assert res['OK'] # now removing that record res = gDataStoreClient.remove(record) assert res['OK']
def userStorageAccounting( self ): self.log.notice( "-------------------------------------------------------------------------------------\n" ) self.log.notice( "Generate accounting records for user directories " ) self.log.notice( "-------------------------------------------------------------------------------------\n" ) result = self.__stDB.getUserSummary() if not result[ 'OK' ]: return result userCatalogData = result[ 'Value' ] print userCatalogData self.log.notice( "Got summary for %s users" % ( len( userCatalogData ) ) ) result = self.__stDB.getUserSummaryPerSE() if not result[ 'OK' ]: return result userSEData = result[ 'Value' ] self.log.notice( "Got SE summary for %s users" % ( len( userSEData ) ) ) now = Time.dateTime() numRows = 0 for user in sorted( userSEData ): if user not in userCatalogData: self.log.error( "User has SE data but not Catalog data!", user ) continue for se in sorted( userSEData[ user ] ): seData = userSEData[ user ][ se ] usRecord = UserStorage() usRecord.setStartTime( now ) usRecord.setEndTime( now ) usRecord.setValueByKey( "User", user ) usRecord.setValueByKey( "StorageElement", se ) usRecord.setValueByKey( "LogicalSize", userCatalogData[ user ][ 'Size' ] ) usRecord.setValueByKey( "LogicalFiles", userCatalogData[ user ][ 'Files' ] ) usRecord.setValueByKey( "PhysicalSize", seData[ 'Size' ] ) usRecord.setValueByKey( "PhysicalFiles", seData[ 'Files' ] ) usRecord.setValueByKey( "StorageSize", 0 ) usRecord.setValueByKey( "StorageFiles", 0 ) gDataStoreClient.addRegister( usRecord ) numRows += 1 self.log.notice( " User %s is using %.2f GiB (%s files)" % ( user, userCatalogData[ user ][ 'Size' ] / ( 1024.0 ** 3 ), userCatalogData[ user ][ 'Files' ] ) ) self.log.notice( "Sending %s records to accounting for user storage" % numRows ) res = gDataStoreClient.commit() if not res[ 'OK' ]: self.log.notice( "ERROR: committing UserStorage records: %s " % res ) return S_ERROR( res ) else: self.log.notice( "%s records for UserStorage type successfully committed" % numRows )
def __updateMigrationAccounting( self, se, migratingFiles, matchingFiles, mismatchingFiles, assumedEndTime, previousMonitorTime ): """ Create accounting messages for the overall throughput observed and the total migration time for the files """ allMigrated = matchingFiles + mismatchingFiles gMonitor.addMark( "MigratedFiles%s" % se, len( allMigrated ) ) gMonitor.addMark( "TotalMigratedFiles%s" % se, len( allMigrated ) ) lfnFileID = {} sizesToObtain = [] for fileID in allMigrated: if not migratingFiles[fileID]['Size']: lfn = migratingFiles[fileID]['LFN'] sizesToObtain.append( lfn ) lfnFileID[lfn] = fileID if sizesToObtain: res = self.ReplicaManager.getCatalogFileSize( sizesToObtain ) if not res['OK']: gLogger.error( "[%s] __updateMigrationAccounting: Failed to obtain file sizes" % se ) return res for lfn, error in res['Value']['Failed'].items(): gLogger.error( "[%s] __updateAccounting: Failed to get file size" % se, "%s %s" % ( lfn, error ) ) migratingFiles[lfnFileID[lfn]]['Size'] = 0 for lfn, size in res['Value']['Successful'].items(): migratingFiles[lfnFileID[lfn]]['Size'] = size totalSize = 0 for fileID in allMigrated: size = migratingFiles[fileID]['Size'] totalSize += size submitTime = migratingFiles[fileID]['SubmitTime'] timeDiff = submitTime - assumedEndTime migrationTime = ( timeDiff.days * 86400 ) + ( timeDiff.seconds ) + ( timeDiff.microseconds / 1000000.0 ) gMonitor.addMark( "MigrationTime%s" % se, migrationTime ) gDataStoreClient.addRegister( self.__initialiseAccountingObject( 'MigrationTime', se, submitTime, assumedEndTime, size ) ) gDataStoreClient.addRegister( self.__initialiseAccountingObject( 'MigrationThroughput', se, previousMonitorTime, assumedEndTime, size ) ) oDataOperation = self.__initialiseAccountingObject( 'MigrationSuccess', se, submitTime, assumedEndTime, size ) if fileID in mismatchingFiles: oDataOperation.setValueByKey( 'TransferOK', 0 ) oDataOperation.setValueByKey( 'FinalStatus', 'Failed' ) gDataStoreClient.addRegister( oDataOperation ) gMonitor.addMark( "TotalMigratedSize%s" % se, totalSize ) gMonitor.addMark( "ChecksumMismatches%s" % se, len( mismatchingFiles ) ) gMonitor.addMark( "TotalChecksumMismatches%s" % se, len( mismatchingFiles ) ) gMonitor.addMark( "ChecksumMatches%s" % se, len( matchingFiles ) ) gMonitor.addMark( "TotalChecksumMatches%s" % se, len( matchingFiles ) ) if allMigrated: gLogger.info( '[%s] __updateMigrationAccounting: Attempting to send accounting message...' % se ) return gDataStoreClient.commit() return S_OK()
def sendAccounting(self): self.dataOp.setValuesFromDict(baseDict) if startTime: self.dataOp.setStartTime(startTime) self.dataOp.setEndTime(endTime) else: self.dataOp.setStartTime() self.dataOp.setEndTime() # Adding only to register if not commitFlag and not delayedCommit: return gDataStoreClient.addRegister(self.dataOp) # Adding to register and committing if commitFlag and not delayedCommit: gDataStoreClient.addRegister(self.dataOp) result = gDataStoreClient.commit() sLog.debug("Committing data operation to accounting") if not result["OK"]: sLog.error("Could not commit data operation to accounting", result["Message"]) return result sLog.debug("Done committing to accounting") # Only late committing else: result = self.dataOp.delayedCommit() if not result["OK"]: sLog.error( "Could not delay-commit data operation to accounting") return result # Send data and commit prioritizing the first monitoring option in the list for backend in self.monitoringOptions: func = locals()[f"send{backend}"] res = func() if not res["OK"]: return res
def __executeMethod(self, lfn, *args, **kwargs): """ Forward the call to each storage in turn until one works. The method to be executed is stored in self.methodName :param lfn : string, list or dictionary :param *args : variable amount of non-keyword arguments. SHOULD BE EMPTY :param **kwargs : keyword arguments :returns S_OK( { 'Failed': {lfn : reason} , 'Successful': {lfn : value} } ) The Failed dict contains the lfn only if the operation failed on all the storages The Successful dict contains the value returned by the successful storages. A special kwargs is 'inputProtocol', which can be specified for putFile. It describes the protocol used as source protocol, since there is in principle only one. """ removedArgs = {} log = self.log.getSubLogger('__executeMethod') log.verbose("preparing the execution of %s" % (self.methodName)) # args should normaly be empty to avoid problem... if args: log.verbose("args should be empty!%s" % args) # because there is normally only one kw argument, I can move it from args to kwargs methDefaultArgs = StorageElementItem.__defaultsArguments.get(self.methodName, {}).keys() if methDefaultArgs: kwargs[methDefaultArgs[0]] = args[0] args = args[1:] log.verbose( "put it in kwargs, but dirty and might be dangerous!args %s kwargs %s" % (args, kwargs)) # We check the deprecated arguments for depArg in StorageElementItem.__deprecatedArguments: if depArg in kwargs: log.verbose("%s is not an allowed argument anymore. Please change your code!" % depArg) removedArgs[depArg] = kwargs[depArg] del kwargs[depArg] # Set default argument if any methDefaultArgs = StorageElementItem.__defaultsArguments.get(self.methodName, {}) for argName in methDefaultArgs: if argName not in kwargs: log.debug("default argument %s for %s not present.\ Setting value %s." % (argName, self.methodName, methDefaultArgs[argName])) kwargs[argName] = methDefaultArgs[argName] res = checkArgumentFormat(lfn) if not res['OK']: errStr = "Supplied lfns must be string, list of strings or a dictionary." log.debug(errStr) return res lfnDict = res['Value'] log.verbose( "Attempting to perform '%s' operation with %s lfns." % (self.methodName, len(lfnDict))) res = self.isValid(operation=self.methodName) if not res['OK']: return res else: if not self.valid: return S_ERROR(self.errorReason) # In case executing putFile, we can assume that all the source urls # are from the same protocol. This optional parameter, if defined # can be used to ignore some storage plugins and thus save time # and avoid fake failures showing in the accounting inputProtocol = kwargs.pop('inputProtocol', None) successful = {} failed = {} filteredPlugins = self.__filterPlugins(self.methodName, kwargs.get('protocols'), inputProtocol) if not filteredPlugins: return S_ERROR(errno.EPROTONOSUPPORT, "No storage plugins matching the requirements\ (operation %s protocols %s inputProtocol %s)" % (self.methodName, kwargs.get('protocols'), inputProtocol)) # Try all of the storages one by one for storage in filteredPlugins: # Determine whether to use this storage object storageParameters = storage.getParameters() pluginName = storageParameters['PluginName'] if not lfnDict: log.debug("No lfns to be attempted for %s protocol." % pluginName) continue log.verbose("Generating %s protocol URLs for %s." % (len(lfnDict), pluginName)) replicaDict = kwargs.pop('replicaDict', {}) if storage.pluginName != "Proxy": res = self.__generateURLDict(lfnDict, storage, replicaDict=replicaDict) urlDict = res['Value']['Successful'] # url : lfn failed.update(res['Value']['Failed']) else: urlDict = dict([(lfn, lfn) for lfn in lfnDict]) if not urlDict: log.verbose("__executeMethod No urls generated for protocol %s." % pluginName) else: log.verbose( "Attempting to perform '%s' for %s physical files" % (self.methodName, len(urlDict))) fcn = None if hasattr(storage, self.methodName) and callable(getattr(storage, self.methodName)): fcn = getattr(storage, self.methodName) if not fcn: return S_ERROR( DErrno.ENOMETH, "SE.__executeMethod: unable to invoke %s, it isn't a member function of storage") urlsToUse = {} # url : the value of the lfn dictionary for the lfn of this url for url in urlDict: urlsToUse[url] = lfnDict[urlDict[url]] startDate = datetime.datetime.utcnow() startTime = time.time() res = fcn(urlsToUse, *args, **kwargs) elapsedTime = time.time() - startTime self.addAccountingOperation(urlsToUse, startDate, elapsedTime, storageParameters, res) if not res['OK']: errStr = "Completely failed to perform %s." % self.methodName log.debug(errStr, 'with plugin %s: %s' % (pluginName, res['Message'])) for lfn in urlDict.values(): if lfn not in failed: failed[lfn] = '' failed[lfn] = "%s %s" % (failed[lfn], res['Message']) if failed[lfn] else res['Message'] else: for url, lfn in urlDict.items(): if url not in res['Value']['Successful']: if lfn not in failed: failed[lfn] = '' if url in res['Value']['Failed']: self.log.debug(res['Value']['Failed'][url]) failed[lfn] = "%s %s" % (failed[lfn], res['Value']['Failed'][url] ) if failed[lfn] else res['Value']['Failed'][url] else: errStr = 'No error returned from plug-in' failed[lfn] = "%s %s" % (failed[lfn], errStr) if failed[lfn] else errStr else: successful[lfn] = res['Value']['Successful'][url] if lfn in failed: failed.pop(lfn) lfnDict.pop(lfn) gDataStoreClient.commit() return S_OK({'Failed': failed, 'Successful': successful})
def __updateMigrationAccounting(self, se, migratingFiles, matchingFiles, mismatchingFiles, assumedEndTime, previousMonitorTime): """ Create accounting messages for the overall throughput observed and the total migration time for the files """ allMigrated = matchingFiles + mismatchingFiles gMonitor.addMark("MigratedFiles%s" % se, len(allMigrated)) gMonitor.addMark("TotalMigratedFiles%s" % se, len(allMigrated)) lfnFileID = {} sizesToObtain = [] for fileID in allMigrated: if not migratingFiles[fileID]['Size']: lfn = migratingFiles[fileID]['LFN'] sizesToObtain.append(lfn) lfnFileID[lfn] = fileID if sizesToObtain: res = self.ReplicaManager.getCatalogFileSize(sizesToObtain) if not res['OK']: gLogger.error( "[%s] __updateMigrationAccounting: Failed to obtain file sizes" % se) return res for lfn, error in res['Value']['Failed'].items(): gLogger.error( "[%s] __updateAccounting: Failed to get file size" % se, "%s %s" % (lfn, error)) migratingFiles[lfnFileID[lfn]]['Size'] = 0 for lfn, size in res['Value']['Successful'].items(): migratingFiles[lfnFileID[lfn]]['Size'] = size totalSize = 0 for fileID in allMigrated: size = migratingFiles[fileID]['Size'] totalSize += size submitTime = migratingFiles[fileID]['SubmitTime'] timeDiff = submitTime - assumedEndTime migrationTime = (timeDiff.days * 86400) + (timeDiff.seconds) + ( timeDiff.microseconds / 1000000.0) gMonitor.addMark("MigrationTime%s" % se, migrationTime) gDataStoreClient.addRegister( self.__initialiseAccountingObject('MigrationTime', se, submitTime, assumedEndTime, size)) gDataStoreClient.addRegister( self.__initialiseAccountingObject('MigrationThroughput', se, previousMonitorTime, assumedEndTime, size)) oDataOperation = self.__initialiseAccountingObject( 'MigrationSuccess', se, submitTime, assumedEndTime, size) if fileID in mismatchingFiles: oDataOperation.setValueByKey('TransferOK', 0) oDataOperation.setValueByKey('FinalStatus', 'Failed') gDataStoreClient.addRegister(oDataOperation) gMonitor.addMark("TotalMigratedSize%s" % se, totalSize) gMonitor.addMark("ChecksumMismatches%s" % se, len(mismatchingFiles)) gMonitor.addMark("TotalChecksumMismatches%s" % se, len(mismatchingFiles)) gMonitor.addMark("ChecksumMatches%s" % se, len(matchingFiles)) gMonitor.addMark("TotalChecksumMatches%s" % se, len(matchingFiles)) if allMigrated: gLogger.info( '[%s] __updateMigrationAccounting: Attempting to send accounting message...' % se) return gDataStoreClient.commit() return S_OK()
def execute(self, production_id=None, prod_job_id=None, wms_job_id=None, workflowStatus=None, stepStatus=None, wf_commons=None, step_commons=None, step_number=None, step_id=None): """ Main execution method. Here we analyse what is written in the XML summary and the pool XML, and send accounting """ try: super(AnalyseFileAccess, self).execute(self.version, production_id, prod_job_id, wms_job_id, workflowStatus, stepStatus, wf_commons, step_commons, step_number, step_id) self._resolveInputVariables() self.log.info("Analyzing root access from %s and %s" % (self.XMLSummary, self.poolXMLCatName)) pfn_lfn = {} lfn_guid = {} lfn_pfn_fail = {} successful_lfn = set() for guid in self.poolXMLCatName_o.files: pFile = self.poolXMLCatName_o.files[guid] lfn = pFile.lfns[0] # there can be only one lfn_guid[lfn] = guid self.lfn_pfn[lfn] = [] for pfn, _ftype, se in pFile.pfns: pfn_lfn[pfn] = lfn self.pfn_se[pfn] = se self.lfn_pfn[lfn].append(pfn) for inputFile, status in self.XMLSummary_o.inputStatus: # The inputFile starts with 'LFN:' or 'PFN:' cleanedName = inputFile[4:] if status == 'full': # it is an LFN successful_lfn.add(cleanedName) elif status == 'fail': # it is a PFN lfn = pfn_lfn.get(cleanedName) if not lfn: self.log.error( "Failed pfn %s is not listed in the catalog" % cleanedName) continue lfn_pfn_fail.setdefault(lfn, []).append(cleanedName) else: # intermediate status, think of it... pass # The lfn in successful and not in lfn_pfn_failed succeeded immediately immediately_successful = successful_lfn - set(lfn_pfn_fail) for lfn in immediately_successful: # We take the first replica in the catalog pfn = self.__getNthPfnForLfn(lfn, 0) remoteSE = self.pfn_se.get(pfn) if not remoteSE: continue oDataOperation = self.__initialiseAccountingObject( remoteSE, True) gDataStoreClient.addRegister(oDataOperation) # For each file that had failure for lfn in lfn_pfn_fail: failedPfns = lfn_pfn_fail[lfn] # We add the accounting for the failure for pfn in failedPfns: remoteSE = self.pfn_se.get(pfn) if not remoteSE: continue oDataOperation = self.__initialiseAccountingObject( remoteSE, False) gDataStoreClient.addRegister(oDataOperation) # If there were more options to try, the next one is successful if len(failedPfns) < len(self.lfn_pfn[lfn]): pfn = self.__getNthPfnForLfn(lfn, len(failedPfns)) remoteSE = self.pfn_se.get(pfn) if not remoteSE: continue oDataOperation = self.__initialiseAccountingObject( remoteSE, True) gDataStoreClient.addRegister(oDataOperation) gDataStoreClient.commit() except Exception as e: #pylint:disable=broad-except self.log.warn(str(e)) finally: super(AnalyseFileAccess, self).finalize(self.version) return S_OK()
def __executeMethod(self, lfn, *args, **kwargs): """ Forward the call to each storage in turn until one works. The method to be executed is stored in self.methodName :param lfn : string, list or dictionary :param *args : variable amount of non-keyword arguments. SHOULD BE EMPTY :param **kwargs : keyword arguments :returns S_OK( { 'Failed': {lfn : reason} , 'Successful': {lfn : value} } ) The Failed dict contains the lfn only if the operation failed on all the storages The Successful dict contains the value returned by the successful storages. A special kwargs is 'inputProtocol', which can be specified for putFile. It describes the protocol used as source protocol, since there is in principle only one. """ removedArgs = {} log = self.log.getSubLogger('__executeMethod') log.verbose("preparing the execution of %s" % (self.methodName)) # args should normaly be empty to avoid problem... if len(args): log.verbose("args should be empty!%s" % args) # because there is normally only one kw argument, I can move it from args to kwargs methDefaultArgs = StorageElementItem.__defaultsArguments.get( self.methodName, {}).keys() if len(methDefaultArgs): kwargs[methDefaultArgs[0]] = args[0] args = args[1:] log.verbose( "put it in kwargs, but dirty and might be dangerous!args %s kwargs %s" % (args, kwargs)) # We check the deprecated arguments for depArg in StorageElementItem.__deprecatedArguments: if depArg in kwargs: log.verbose( "%s is not an allowed argument anymore. Please change your code!" % depArg) removedArgs[depArg] = kwargs[depArg] del kwargs[depArg] # Set default argument if any methDefaultArgs = StorageElementItem.__defaultsArguments.get( self.methodName, {}) for argName in methDefaultArgs: if argName not in kwargs: log.debug("default argument %s for %s not present.\ Setting value %s." % (argName, self.methodName, methDefaultArgs[argName])) kwargs[argName] = methDefaultArgs[argName] res = checkArgumentFormat(lfn) if not res['OK']: errStr = "Supplied lfns must be string, list of strings or a dictionary." log.debug(errStr) return res lfnDict = res['Value'] log.verbose("Attempting to perform '%s' operation with %s lfns." % (self.methodName, len(lfnDict))) res = self.isValid(operation=self.methodName) if not res['OK']: return res else: if not self.valid: return S_ERROR(self.errorReason) # In case executing putFile, we can assume that all the source urls # are from the same protocol. This optional parameter, if defined # can be used to ignore some storage plugins and thus save time # and avoid fake failures showing in the accounting inputProtocol = kwargs.pop('inputProtocol', None) successful = {} failed = {} filteredPlugins = self.__filterPlugins(self.methodName, kwargs.get('protocols'), inputProtocol) if not filteredPlugins: return S_ERROR( errno.EPROTONOSUPPORT, "No storage plugins matching the requirements\ (operation %s protocols %s inputProtocol %s)" \ % ( self.methodName, kwargs.get( 'protocols' ), inputProtocol ) ) # Try all of the storages one by one for storage in filteredPlugins: # Determine whether to use this storage object storageParameters = storage.getParameters() pluginName = storageParameters['PluginName'] if not lfnDict: log.debug("No lfns to be attempted for %s protocol." % pluginName) continue log.verbose("Generating %s protocol URLs for %s." % (len(lfnDict), pluginName)) replicaDict = kwargs.pop('replicaDict', {}) if storage.pluginName != "Proxy": res = self.__generateURLDict(lfnDict, storage, replicaDict=replicaDict) urlDict = res['Value']['Successful'] # url : lfn failed.update(res['Value']['Failed']) else: urlDict = dict([(lfn, lfn) for lfn in lfnDict]) if not len(urlDict): log.verbose( "__executeMethod No urls generated for protocol %s." % pluginName) else: log.verbose( "Attempting to perform '%s' for %s physical files" % (self.methodName, len(urlDict))) fcn = None if hasattr(storage, self.methodName) and callable( getattr(storage, self.methodName)): fcn = getattr(storage, self.methodName) if not fcn: return S_ERROR( DErrno.ENOMETH, "SE.__executeMethod: unable to invoke %s, it isn't a member function of storage" ) urlsToUse = { } # url : the value of the lfn dictionary for the lfn of this url for url in urlDict: urlsToUse[url] = lfnDict[urlDict[url]] startDate = datetime.datetime.utcnow() startTime = time.time() res = fcn(urlsToUse, *args, **kwargs) elapsedTime = time.time() - startTime self.addAccountingOperation(urlsToUse, startDate, elapsedTime, storageParameters, res) if not res['OK']: errStr = "Completely failed to perform %s." % self.methodName log.debug( errStr, 'with plugin %s: %s' % (pluginName, res['Message'])) for lfn in urlDict.values(): if lfn not in failed: failed[lfn] = '' failed[lfn] = "%s %s" % ( failed[lfn], res['Message']) if failed[lfn] else res['Message'] else: for url, lfn in urlDict.items(): if url not in res['Value']['Successful']: if lfn not in failed: failed[lfn] = '' if url in res['Value']['Failed']: self.log.debug(res['Value']['Failed'][url]) failed[lfn] = "%s %s" % ( failed[lfn], res['Value']['Failed'][url]) if failed[ lfn] else res['Value']['Failed'][url] else: errStr = 'No error returned from plug-in' failed[lfn] = "%s %s" % ( failed[lfn], errStr) if failed[lfn] else errStr else: successful[lfn] = res['Value']['Successful'][url] if lfn in failed: failed.pop(lfn) lfnDict.pop(lfn) gDataStoreClient.commit() return S_OK({'Failed': failed, 'Successful': successful})
def topDirectoryAccounting( self ): self.log.notice( "-------------------------------------------------------------------------------------\n" ) self.log.notice( "Generate accounting records for top directories " ) self.log.notice( "-------------------------------------------------------------------------------------\n" ) ftb = 1.0e12 # get info from the DB about the LOGICAL STORAGE USAGE (from the su_Directory table): result = self.__stDB.getSummary( '/lhcb/' ) if not result[ 'OK' ]: return result logicalUsage = result['Value'] topDirLogicalUsage = {} # build the list of first level directories for row in logicalUsage: # d, size, files = row splitDir = row.split( "/" ) if len( splitDir ) > 3: # skip the root directory "/lhcb/" firstLevelDir = '/' + splitDir[1] + '/' + splitDir[2] + '/' topDirLogicalUsage.setdefault( firstLevelDir, {'Files':0, 'Size':0} ) topDirLogicalUsage[ firstLevelDir ][ 'Files' ] += logicalUsage[ row ][ 'Files' ] topDirLogicalUsage[ firstLevelDir ][ 'Size' ] += logicalUsage[ row ][ 'Size' ] self.log.notice( "Summary on logical usage of top directories: " ) for row in topDirLogicalUsage: self.log.notice( "dir: %s size: %.4f TB files: %d" % ( row, topDirLogicalUsage[row]['Size'] / ftb, topDirLogicalUsage[row]['Files'] ) ) # loop on top level directories (/lhcb/data, /lhcb/user/, /lhcb/MC/, etc..) # to get the summary in terms of PHYSICAL usage grouped by SE: seData = {} for directory in topDirLogicalUsage: result = self.__stDB.getDirectorySummaryPerSE( directory ) # retrieve the PHYSICAL usage if not result[ 'OK' ]: return result seData[ directory ] = result[ 'Value' ] self.log.notice( "Got SE summary for %s directories " % ( len( seData ) ) ) self.log.debug( "SEData: %s" % seData ) # loop on top level directories to send the accounting records numRows = 0 now = Time.dateTime() for directory in seData: self.log.debug( "dir: %s SEData: %s " % ( directory, seData[ directory ] ) ) if directory not in topDirLogicalUsage: self.log.error( "Dir %s is in the summary per SE, but it is not in the logical files summary!" % directory ) continue for se in sorted( seData[ directory ] ): storageRecord = Storage() storageRecord.setStartTime( now ) storageRecord.setEndTime( now ) storageRecord.setValueByKey( "Directory", directory ) storageRecord.setValueByKey( "StorageElement", se ) storageRecord.setValueByKey( "LogicalFiles", topDirLogicalUsage[ directory ][ 'Files' ] ) storageRecord.setValueByKey( "LogicalSize", topDirLogicalUsage[ directory ][ 'Size' ] ) try: physicalFiles = seData[ directory ][ se ][ 'Files' ] except: self.log.error( "WARNING! no files replicas for directory %s on SE %s" % ( directory, se ) ) physicalFiles = 0 try: physicalSize = seData[ directory ][ se ][ 'Size' ] except: self.log.error( "WARNING! no size for replicas for directory %s on SE %s" % ( directory, se ) ) physicalSize = 0 storageRecord.setValueByKey( "PhysicalFiles", physicalFiles ) storageRecord.setValueByKey( "PhysicalSize", physicalSize ) gDataStoreClient.addRegister( storageRecord ) numRows += 1 self.log.debug( "Directory: %s SE: %s physical size: %.4f TB (%d files)" % ( directory, se, physicalSize / ftb, physicalFiles ) ) self.log.notice( "Sending %s records to accounting for top level directories storage" % numRows ) res = gDataStoreClient.commit() if not res[ 'OK' ]: self.log.notice( "ERROR: committing Storage records: %s " % res ) return S_ERROR( res ) else: self.log.notice( "%s records for Storage type successfully committed" % numRows )