def __fileRetry(self, prodid, mode): callback = {} tsClient = TransformationClient() if mode == "proc": res = tsClient.getTransformationFilesCount(prodid, "ErrorCount", {'Status': 'Processed'}) elif mode == "not": res = tsClient.getTransformationFilesCount( prodid, "ErrorCount", {'Status': ['Unused', 'Assigned', 'Failed']}) elif mode == "all": res = tsClient.getTransformationFilesCount(prodid, "ErrorCount") if not res['OK']: callback = {"success": "false", "error": res["Message"]} else: resList = [] total = res['Value'].pop('Total') if total == 0: callback = {"success": "false", "error": "No files found"} else: for status in sorted(res['Value'].keys()): count = res['Value'][status] percent = "%.1f" % ((count * 100.0) / total) resList.append((status, str(count), percent)) resList.append(('Total', total, '-')) callback = {"success": "true", "result": resList} gLogger.info("#######", res) return callback
def __fileRetry(self, prodid, mode): callback = {} tsClient = TransformationClient() if mode == "proc": res = tsClient.getTransformationFilesCount(prodid, "ErrorCount", {'Status': 'Processed'}) elif mode == "not": res = tsClient.getTransformationFilesCount(prodid, "ErrorCount", {'Status': ['Unused', 'Assigned', 'Failed']}) elif mode == "all": res = tsClient.getTransformationFilesCount(prodid, "ErrorCount") if not res['OK']: callback = {"success": "false", "error": res["Message"]} else: resList = [] total = res['Value'].pop('Total') if total == 0: callback = {"success": "false", "error": "No files found"} else: for status in sorted(res['Value'].keys()): count = res['Value'][status] percent = "%.1f" % ((count * 100.0) / total) resList.append((status, str(count), percent)) resList.append(('Total', total, '-')) callback = {"success": "true", "result": resList} gLogger.info("#######", res) return callback
def __transformationFileStatus(self, transid): callback = {} tsClient = TransformationClient() res = tsClient.getTransformationFilesCount(transid, "Status") if not res['OK']: callback = {"success": "false", "error": res["Message"]} else: resList = [] total = res['Value'].pop('Total') if total == 0: callback = {"success": "false", "error": "No files found"} else: for status in sorted(res['Value'].keys()): count = res['Value'][status] percent = "%.1f" % ((count * 100.0) / total) resList.append((status, str(count), percent)) resList.append(('Total', total, '-')) callback = {"success": "true", "result": resList} gLogger.info("#######", res) return callback
def __transformationFileStatus(self, transid): callback = {} tsClient = TransformationClient() res = tsClient.getTransformationFilesCount(transid, "Status") if not res['OK']: callback = {"success": "false", "error": res["Message"]} else: resList = [] total = res['Value'].pop('Total') if total == 0: callback = {"success": "false", "error": "No files found"} else: for status in sorted(res['Value'].keys()): count = res['Value'][status] percent = "%.1f" % ((count * 100.0) / total) resList.append((status, str(count), percent)) resList.append(('Total', total, '-')) callback = {"success": "true", "result": resList} gLogger.info("#######", res) return callback
class fakeClient: def __init__(self, trans, transID, lfns, asIfProd): self.trans = trans self.transID = transID from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient self.transClient = TransformationClient() from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient self.bk = BookkeepingClient() from DIRAC.DataManagementSystem.Client.DataManager import DataManager self.dm = DataManager() self.asIfProd = asIfProd (self.transFiles, self.transReplicas) = self.prepareForPlugin(lfns) def addFilesToTransformation(self, transID, lfns): return S_OK({ 'Failed': {}, 'Successful': dict([(lfn, 'Added') for lfn in lfns]) }) def getTransformation(self, transID, extraParams=False): if transID == self.transID and self.asIfProd: transID = self.asIfProd if transID != self.transID: return self.transClient.getTransformation(transID) res = self.trans.getType() return DIRAC.S_OK({'Type': res['Value']}) def getReplicas(self): return self.transReplicas def getFiles(self): return self.transFiles def getCounters(self, table, attrList, condDict): if condDict['TransformationID'] == self.transID and self.asIfProd: condDict['TransformationID'] = self.asIfProd if condDict['TransformationID'] != self.transID: return self.transClient.getCounters(table, attrList, condDict) possibleTargets = [ 'CERN-RAW', 'CNAF-RAW', 'GRIDKA-RAW', 'IN2P3-RAW', 'SARA-RAW', 'PIC-RAW', 'RAL-RAW', 'RRCKI-RAW' ] counters = [] for se in possibleTargets: counters.append(({'UsedSE': se}, 0)) return DIRAC.S_OK(counters) def getBookkeepingQuery(self, transID): if transID == self.transID and self.asIfProd: return self.transClient.getBookkeepingQuery(asIfProd) return self.trans.getBkQuery() def insertTransformationRun(self, transID, runID, xx): return DIRAC.S_OK() def getTransformationRuns(self, condDict): if condDict['TransformationID'] == self.transID and self.asIfProd: condDict['TransformationID'] = self.asIfProd if condDict['TransformationID'] == self.transID: transRuns = [] runs = condDict.get('RunNumber', []) if not runs and self.transFiles: res = self.bk.getFileMetadata( [fileDict['LFN'] for fileDict in self.transFiles]) if not res['OK']: return res runs = list( set(meta['RunNumber'] for meta in res['Value']['Successful'].itervalues())) for run in runs: transRuns.append({ 'RunNumber': run, 'Status': "Active", "SelectedSite": None }) return DIRAC.S_OK(transRuns) else: return self.transClient.getTransformationRuns(condDict) def getTransformationFiles(self, condDict=None): if condDict.get('TransformationID') == self.transID and self.asIfProd: condDict['TransformationID'] = self.asIfProd if condDict.get('TransformationID') == self.transID: transFiles = [] if 'Status' in condDict and 'Unused' not in condDict['Status']: return DIRAC.S_OK(transFiles) runs = None if 'RunNumber' in condDict: runs = condDict['RunNumber'] if not isinstance(runs, list): runs = [runs] for fileDict in self.transFiles: if not runs or fileDict['RunNumber'] in runs: transFiles.append({ 'LFN': fileDict['LFN'], 'Status': 'Unused', 'RunNumber': fileDict['RunNumber'] }) return DIRAC.S_OK(transFiles) else: return self.transClient.getTransformationFiles(condDict=condDict) def setParameterToTransformationFiles(self, transID, lfnDict): """ Update the transFiles with some parameters """ if transID == self.transID: for fileDict in self.transFiles: fileDict.update(lfnDict.get(fileDict['LFN'], {})) return S_OK() else: return self.transClient.setParameterToTransformationFiles( transID, lfnDict) def getTransformationFilesCount(self, transID, field, selection=None): if selection is None: selection = {} if transID == self.transID or selection.get( 'TransformationID') == self.transID: runs = selection.get('RunNumber') if runs and not isinstance(runs, list): runs = [runs] if field == 'Status': counters = {'Unused': 0} for fileDict in self.transFiles: if not runs or fileDict['RunNumber'] in runs: counters['Unused'] += 1 elif field == 'RunNumber': counters = {} for fileDict in self.transFiles: runID = fileDict['RunNumber'] if not runs or runID in runs: counters.setdefault(runID, 0) counters[runID] += 1 else: return DIRAC.S_ERROR('Not implemented for field ' + field) counters['Total'] = sum(count for count in counters.itervalues()) return DIRAC.S_OK(counters) else: return self.transClient.getTransformationFilesCount( transID, field, selection=selection) def getTransformationRunStats(self, transIDs): counters = {} for transID in transIDs: if transID == self.transID: for fileDict in self.transFiles: runID = fileDict['RunNumber'] counters[transID][runID]['Unused'] = counters.setdefault( transID, {}).setdefault(runID, {}).setdefault( 'Unused', 0) + 1 for runID in counters[transID]: counters[transID][runID]['Total'] = counters[transID][ runID]['Unused'] else: res = self.transClient.getTransformationRunStats(transIDs) if res['OK']: counters.update(res['Value']) else: return res return DIRAC.S_OK(counters) def addRunsMetadata(self, runID, val): return self.transClient.addRunsMetadata(runID, val) def getRunsMetadata(self, runID): return self.transClient.getRunsMetadata(runID) def setTransformationRunStatus(self, transID, runID, status): return DIRAC.S_OK() def setTransformationRunsSite(self, transID, runID, site): return DIRAC.S_OK() def setFileStatusForTransformation(self, transID, status, lfns): return DIRAC.S_OK() def addTransformationRunFiles(self, transID, run, lfns): return DIRAC.S_OK() def setDestinationForRun(self, runID, site): return DIRAC.S_OK() def getDestinationForRun(self, runID): return self.transClient.getDestinationForRun(runID) def prepareForPlugin(self, lfns): import time print "Preparing the plugin input data (%d files)" % len(lfns) type = self.trans.getType()['Value'] if not lfns: return (None, None) res = self.bk.getFileMetadata(lfns) if res['OK']: files = [] for lfn, metadata in res['Value']['Successful'].iteritems(): runID = metadata.get('RunNumber', 0) runDict = {"RunNumber": runID, "LFN": lfn} files.append(runDict) else: print "Error getting BK metadata", res['Message'] return ([], {}) replicas = {} startTime = time.time() from DIRAC.Core.Utilities.List import breakListIntoChunks for lfnChunk in breakListIntoChunks(lfns, 200): # print lfnChunk if type.lower() in ("replication", "removal"): res = self.dm.getReplicas(lfnChunk, getUrl=False) else: res = self.dm.getReplicasForJobs(lfnChunk, getUrl=False) # print res if res['OK']: for lfn, ses in res['Value']['Successful'].iteritems(): if ses: replicas[lfn] = sorted(ses) else: print "Error getting replicas of %d files:" % len( lfns), res['Message'] print "Obtained replicas of %d files in %.3f seconds" % ( len(lfns), time.time() - startTime) return (files, replicas)