def __sendAccounting( ftsJob, ownerDN ): """ prepare and send DataOperation to AccouringDB """ dataOp = DataOperation() dataOp.setStartTime( fromString( ftsJob.SubmitTime ) ) dataOp.setEndTime( fromString( ftsJob.LastUpdate ) ) accountingDict = dict() accountingDict["OperationType"] = "ReplicateAndRegister" username = getUsernameForDN( ownerDN ) if not username["OK"]: username = ownerDN else: username = username["Value"] accountingDict["User"] = username accountingDict["Protocol"] = "FTS" # accountingDict['RegistrationTime'] = 0 # accountingDict['RegistrationOK'] = 0 # accountingDict['RegistrationTotal'] = 0 accountingDict["TransferOK"] = len( [ f for f in ftsJob if f.Status == "Finished" ] ) accountingDict["TransferTotal"] = len( ftsJob ) accountingDict["TransferSize"] = ftsJob.Size accountingDict["FinalStatus"] = ftsJob.Status accountingDict["Source"] = ftsJob.SourceSE accountingDict["Destination"] = ftsJob.TargetSE dt = ftsJob.LastUpdate - ftsJob.SubmitTime transferTime = dt.days * 86400 + dt.seconds accountingDict["TransferTime"] = transferTime dataOp.setValuesFromDict( accountingDict ) dataOp.commit()
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ): """ send accounting record :param self: self reference :param regSuc: number of files successfully registered :param regTotal: number of files attepted to register :param regTime: time stamp at the end of registration :param transEndTime: time stamp at the end of FTS job :param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers """ submitTime = fromString( self.submitTime ) oAccounting = DataOperation() dt = transEndTime - submitTime transferTime = dt.days * 86400 + dt.seconds if 'fts3' in self.ftsServer and transferTime < 0: import datetime while transferTime < 0: # Shift by one hour until transfer time is positive (ugly fix for FTS3 bug) transferTime += 3600 submitTime -= datetime.timedelta( 0, 3600 ) self.log.verbose( 'Fixed UTC submit time... Submit: %s, end: %s' % ( submitTime, transEndTime ) ) oAccounting.setEndTime( transEndTime ) oAccounting.setStartTime( submitTime ) accountingDict = {} accountingDict['OperationType'] = 'replicateAndRegister' result = getProxyInfo() if not result['OK']: userName = '******' else: userName = result['Value'].get( 'username', 'unknown' ) accountingDict['User'] = userName accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3' accountingDict['RegistrationTime'] = regTime accountingDict['RegistrationOK'] = regSuc accountingDict['RegistrationTotal'] = regTotal accountingDict['TransferOK'] = transDict['transOK'] accountingDict['TransferTotal'] = transDict['transTotal'] accountingDict['TransferSize'] = transDict['transSize'] accountingDict['FinalStatus'] = self.requestStatus accountingDict['Source'] = self.sourceSE accountingDict['Destination'] = self.targetSE accountingDict['TransferTime'] = transferTime oAccounting.setValuesFromDict( accountingDict ) self.log.verbose( "Attempting to commit accounting message..." ) oAccounting.commit() self.log.verbose( "...committed." ) return S_OK()
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime ): transSuc = 0 transSize = 0 missingSize = [] for lfn in self.fileDict.keys(): if self.fileDict[lfn].get( 'Status' ) == 'Finished': transSuc += 1 if not self.catalogMetadata.has_key( lfn ): missingSize.append( lfn ) if missingSize: self.__updateMetadataCache( missingSize ) for lfn in self.fileDict.keys(): if self.fileDict[lfn].get( 'Status' ) == 'Finished': transSize += self.catalogMetadata[lfn]['Size'] transTotal = 0 for state in ( self.statusSummary.keys() ): transTotal += self.statusSummary[state] submitTime = fromString( self.submitTime ) endTime = fromString( transEndTime ) oAccounting = DataOperation() #oAccounting.setEndTime(endTime) oAccounting.setEndTime( transEndTime ) oAccounting.setStartTime( submitTime ) accountingDict = {} accountingDict['OperationType'] = 'replicateAndRegister' accountingDict['User'] = '******' accountingDict['Protocol'] = 'FTS' accountingDict['RegistrationTime'] = regTime accountingDict['RegistrationOK'] = regSuc accountingDict['RegistrationTotal'] = regTotal accountingDict['TransferOK'] = transSuc accountingDict['TransferTotal'] = transTotal accountingDict['TransferSize'] = transSize accountingDict['FinalStatus'] = self.requestStatus accountingDict['Source'] = self.sourceSE accountingDict['Destination'] = self.targetSE c = transEndTime - submitTime transferTime = c.days * 86400 + c.seconds accountingDict['TransferTime'] = transferTime oAccounting.setValuesFromDict( accountingDict ) gLogger.verbose( "Attempting to commit accounting message..." ) oAccounting.commit() gLogger.verbose( "...committed." ) return S_OK()
def __sendAccounting(self, regSuc, regTotal, regTime, transEndTime): transSuc = 0 transSize = 0 missingSize = [] for lfn in self.fileDict.keys(): if self.fileDict[lfn].get('Status') == 'Finished': transSuc += 1 if not self.catalogMetadata.has_key(lfn): missingSize.append(lfn) if missingSize: self.__updateMetadataCache(missingSize) for lfn in self.fileDict.keys(): if self.fileDict[lfn].get('Status') == 'Finished': transSize += self.catalogMetadata[lfn]['Size'] transTotal = 0 for state in (self.statusSummary.keys()): transTotal += self.statusSummary[state] submitTime = fromString(self.submitTime) endTime = fromString(transEndTime) oAccounting = DataOperation() #oAccounting.setEndTime(endTime) oAccounting.setEndTime(transEndTime) oAccounting.setStartTime(submitTime) accountingDict = {} accountingDict['OperationType'] = 'replicateAndRegister' accountingDict['User'] = '******' accountingDict['Protocol'] = 'FTS' accountingDict['RegistrationTime'] = regTime accountingDict['RegistrationOK'] = regSuc accountingDict['RegistrationTotal'] = regTotal accountingDict['TransferOK'] = transSuc accountingDict['TransferTotal'] = transTotal accountingDict['TransferSize'] = transSize accountingDict['FinalStatus'] = self.requestStatus accountingDict['Source'] = self.sourceSE accountingDict['Destination'] = self.targetSE c = transEndTime - submitTime transferTime = c.days * 86400 + c.seconds accountingDict['TransferTime'] = transferTime oAccounting.setValuesFromDict(accountingDict) gLogger.verbose("Attempting to commit accounting message...") oAccounting.commit() gLogger.verbose("...committed.") return S_OK()
def __sendAccounting(self, regSuc, regTotal, regTime, transEndTime, transDict): """ send accounting record :param self: self reference :param regSuc: number of files successfully registered :param regTotal: number of files attepted to register :param regTime: time stamp at the end of registration :param transEndTime: time stamp at the end of FTS job :param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers """ oAccounting = DataOperation() oAccounting.setEndTime(transEndTime) oAccounting.setStartTime(self.submitTime) accountingDict = {} accountingDict['OperationType'] = 'replicateAndRegister' result = getProxyInfo() if not result['OK']: userName = '******' else: userName = result['Value'].get('username', 'unknown') accountingDict['User'] = userName accountingDict[ 'Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3' accountingDict['RegistrationTime'] = regTime accountingDict['RegistrationOK'] = regSuc accountingDict['RegistrationTotal'] = regTotal accountingDict['TransferOK'] = transDict['transOK'] accountingDict['TransferTotal'] = transDict['transTotal'] accountingDict['TransferSize'] = transDict['transSize'] accountingDict['FinalStatus'] = self.requestStatus accountingDict['Source'] = self.sourceSE accountingDict['Destination'] = self.targetSE accountingDict['TransferTime'] = self.transferTime oAccounting.setValuesFromDict(accountingDict) self.log.verbose("Attempting to commit accounting message...") oAccounting.commit() self.log.verbose("...committed.") return S_OK()
def __sendAccounting(ftsJob, ownerDN): """ prepare and send DataOperation to AccouringDB """ dataOp = DataOperation() dataOp.setStartTime(fromString(ftsJob.SubmitTime)) dataOp.setEndTime(fromString(ftsJob.LastUpdate)) accountingDict = dict() accountingDict["OperationType"] = "ReplicateAndRegister" username = getUsernameForDN(ownerDN) if not username["OK"]: username = ownerDN else: username = username["Value"] accountingDict["User"] = username accountingDict[ "Protocol"] = "FTS3" if 'fts3' in ftsJob.FTSServer.lower( ) else 'FTS' accountingDict['ExecutionSite'] = ftsJob.FTSServer accountingDict['RegistrationTime'] = ftsJob._regTime accountingDict['RegistrationOK'] = ftsJob._regSuccess accountingDict['RegistrationTotal'] = ftsJob._regTotal accountingDict["TransferOK"] = len( [f for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES]) accountingDict["TransferTotal"] = len(ftsJob) accountingDict["TransferSize"] = ftsJob.Size - ftsJob.FailedSize accountingDict["FinalStatus"] = ftsJob.Status accountingDict["Source"] = ftsJob.SourceSE accountingDict["Destination"] = ftsJob.TargetSE dt = ftsJob.LastUpdate - ftsJob.SubmitTime transferTime = dt.days * 86400 + dt.seconds accountingDict["TransferTime"] = transferTime # accountingDict['TransferTime'] = sum( [f._duration for f in ftsJob]) dataOp.setValuesFromDict(accountingDict) dataOp.commit()
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ): """ send accounting record :param self: self reference :param regSuc: number of files successfully registered :param regTotal: number of files attepted to register :param regTime: time stamp at the end of registration :param transEndTime: time stamp at the end of FTS job :param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers """ oAccounting = DataOperation() oAccounting.setEndTime( transEndTime ) oAccounting.setStartTime( self.submitTime ) accountingDict = {} accountingDict['OperationType'] = 'replicateAndRegister' result = getProxyInfo() if not result['OK']: userName = '******' else: userName = result['Value'].get( 'username', 'unknown' ) accountingDict['User'] = userName accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3' accountingDict['RegistrationTime'] = regTime accountingDict['RegistrationOK'] = regSuc accountingDict['RegistrationTotal'] = regTotal accountingDict['TransferOK'] = transDict['transOK'] accountingDict['TransferTotal'] = transDict['transTotal'] accountingDict['TransferSize'] = transDict['transSize'] accountingDict['FinalStatus'] = self.requestStatus accountingDict['Source'] = self.sourceSE accountingDict['Destination'] = self.targetSE accountingDict['TransferTime'] = self.transferTime oAccounting.setValuesFromDict( accountingDict ) self.log.verbose( "Attempting to commit accounting message..." ) oAccounting.commit() self.log.verbose( "...committed." ) return S_OK()
def __sendAccounting(self, regSuc, regTotal, regTime, transEndTime, transDict): """ send accounting record :param self: self reference :param regSuc: number of files successfully registered :param regTotal: number of files attepted to register :param regTime: time stamp at the end of registration :param transEndTime: time stamp at the end of FTS job :param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers """ submitTime = fromString(self.submitTime) oAccounting = DataOperation() oAccounting.setEndTime(transEndTime) oAccounting.setStartTime(submitTime) accountingDict = {} accountingDict['OperationType'] = 'replicateAndRegister' accountingDict['User'] = '******' accountingDict['Protocol'] = 'FTS' accountingDict['RegistrationTime'] = regTime accountingDict['RegistrationOK'] = regSuc accountingDict['RegistrationTotal'] = regTotal accountingDict['TransferOK'] = transDict['transOK'] accountingDict['TransferTotal'] = transDict['transTotal'] accountingDict['TransferSize'] = transDict['transSize'] accountingDict['FinalStatus'] = self.requestStatus accountingDict['Source'] = self.sourceSE accountingDict['Destination'] = self.targetSE dt = transEndTime - submitTime transferTime = dt.days * 86400 + dt.seconds accountingDict['TransferTime'] = transferTime oAccounting.setValuesFromDict(accountingDict) self.log.verbose("Attempting to commit accounting message...") oAccounting.commit() self.log.verbose("...committed.") return S_OK()
def __sendAccounting( ftsJob, ownerDN ): """ prepare and send DataOperation to AccouringDB """ dataOp = DataOperation() dataOp.setStartTime( fromString( ftsJob.SubmitTime ) ) dataOp.setEndTime( fromString( ftsJob.LastUpdate ) ) accountingDict = dict() accountingDict["OperationType"] = "ReplicateAndRegister" username = getUsernameForDN( ownerDN ) if not username["OK"]: username = ownerDN else: username = username["Value"] accountingDict["User"] = username accountingDict["Protocol"] = "FTS3" if 'fts3' in ftsJob.FTSServer.lower() else 'FTS' accountingDict['ExecutionSite'] = ftsJob.FTSServer accountingDict['RegistrationTime'] = ftsJob._regTime accountingDict['RegistrationOK'] = ftsJob._regSuccess accountingDict['RegistrationTotal'] = ftsJob._regTotal accountingDict["TransferOK"] = len( [ f for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES ] ) accountingDict["TransferTotal"] = len( ftsJob ) accountingDict["TransferSize"] = ftsJob.Size - ftsJob.FailedSize accountingDict["FinalStatus"] = ftsJob.Status accountingDict["Source"] = ftsJob.SourceSE accountingDict["Destination"] = ftsJob.TargetSE # dt = ftsJob.LastUpdate - ftsJob.SubmitTime # transferTime = dt.days * 86400 + dt.seconds # accountingDict["TransferTime"] = transferTime accountingDict['TransferTime'] = sum( [int( f._duration ) for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES ] ) dataOp.setValuesFromDict( accountingDict ) dataOp.commit()
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ): """ send accounting record :param self: self reference :param regSuc: number of files successfully registered :param regTotal: number of files attepted to register :param regTime: time stamp at the end of registration :param transEndTime: time stamp at the end of FTS job :param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers """ submitTime = fromString( self.submitTime ) oAccounting = DataOperation() oAccounting.setEndTime( transEndTime ) oAccounting.setStartTime( submitTime ) accountingDict = {} accountingDict['OperationType'] = 'replicateAndRegister' accountingDict['User'] = '******' accountingDict['Protocol'] = 'FTS' accountingDict['RegistrationTime'] = regTime accountingDict['RegistrationOK'] = regSuc accountingDict['RegistrationTotal'] = regTotal accountingDict['TransferOK'] = transDict['transOK'] accountingDict['TransferTotal'] = transDict['transTotal'] accountingDict['TransferSize'] = transDict['transSize'] accountingDict['FinalStatus'] = self.requestStatus accountingDict['Source'] = self.sourceSE accountingDict['Destination'] = self.targetSE dt = transEndTime - submitTime transferTime = dt.days * 86400 + dt.seconds accountingDict['TransferTime'] = transferTime oAccounting.setValuesFromDict( accountingDict ) self.log.verbose( "Attempting to commit accounting message..." ) oAccounting.commit() self.log.verbose( "...committed." ) return S_OK()
def __sendAccounting(ftsJob, ownerDN): """ prepare and send DataOperation to AccouringDB """ dataOp = DataOperation() dataOp.setStartTime(fromString(ftsJob.SubmitTime)) dataOp.setEndTime(fromString(ftsJob.LastUpdate)) accountingDict = dict() accountingDict["OperationType"] = "ReplicateAndRegister" username = getUsernameForDN(ownerDN) if not username["OK"]: username = ownerDN else: username = username["Value"] accountingDict["User"] = username accountingDict["Protocol"] = "FTS" # accountingDict['RegistrationTime'] = 0 # accountingDict['RegistrationOK'] = 0 # accountingDict['RegistrationTotal'] = 0 accountingDict["TransferOK"] = len( [f for f in ftsJob if f.Status == "Finished"]) accountingDict["TransferTotal"] = len(ftsJob) accountingDict["TransferSize"] = ftsJob.Size accountingDict["FinalStatus"] = ftsJob.Status accountingDict["Source"] = ftsJob.SourceSE accountingDict["Destination"] = ftsJob.TargetSE dt = ftsJob.LastUpdate - ftsJob.SubmitTime transferTime = dt.days * 86400 + dt.seconds accountingDict["TransferTime"] = transferTime dataOp.setValuesFromDict(accountingDict) dataOp.commit()