def __fileRetry(self, prodid, mode):
callback = {}
tsClient = TransformationClient()
if mode == "proc":
res = tsClient.getTransformationFilesCount(prodid, "ErrorCount",
{'Status': 'Processed'})
elif mode == "not":
res = tsClient.getTransformationFilesCount(
prodid, "ErrorCount",
{'Status': ['Unused', 'Assigned', 'Failed']})
elif mode == "all":
res = tsClient.getTransformationFilesCount(prodid, "ErrorCount")
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
resList = []
total = res['Value'].pop('Total')
if total == 0:
callback = {"success": "false", "error": "No files found"}
else:
for status in sorted(res['Value'].keys()):
count = res['Value'][status]
percent = "%.1f" % ((count * 100.0) / total)
resList.append((status, str(count), percent))
resList.append(('Total', total, '-'))
callback = {"success": "true", "result": resList}
gLogger.info("#######", res)
return callback
def __fileRetry(self, prodid, mode):
callback = {}
tsClient = TransformationClient()
if mode == "proc":
res = tsClient.getTransformationFilesCount(prodid, "ErrorCount", {'Status': 'Processed'})
elif mode == "not":
res = tsClient.getTransformationFilesCount(prodid, "ErrorCount", {'Status': ['Unused', 'Assigned', 'Failed']})
elif mode == "all":
res = tsClient.getTransformationFilesCount(prodid, "ErrorCount")
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
resList = []
total = res['Value'].pop('Total')
if total == 0:
callback = {"success": "false", "error": "No files found"}
else:
for status in sorted(res['Value'].keys()):
count = res['Value'][status]
percent = "%.1f" % ((count * 100.0) / total)
resList.append((status, str(count), percent))
resList.append(('Total', total, '-'))
callback = {"success": "true", "result": resList}
gLogger.info("#######", res)
return callback
def __transformationFileStatus(self, transid):
callback = {}
tsClient = TransformationClient()
res = tsClient.getTransformationFilesCount(transid, "Status")
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
resList = []
total = res['Value'].pop('Total')
if total == 0:
callback = {"success": "false", "error": "No files found"}
else:
for status in sorted(res['Value'].keys()):
count = res['Value'][status]
percent = "%.1f" % ((count * 100.0) / total)
resList.append((status, str(count), percent))
resList.append(('Total', total, '-'))
callback = {"success": "true", "result": resList}
gLogger.info("#######", res)
return callback
def __transformationFileStatus(self, transid):
callback = {}
tsClient = TransformationClient()
res = tsClient.getTransformationFilesCount(transid, "Status")
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
resList = []
total = res['Value'].pop('Total')
if total == 0:
callback = {"success": "false", "error": "No files found"}
else:
for status in sorted(res['Value'].keys()):
count = res['Value'][status]
percent = "%.1f" % ((count * 100.0) / total)
resList.append((status, str(count), percent))
resList.append(('Total', total, '-'))
callback = {"success": "true", "result": resList}
gLogger.info("#######", res)
return callback
class fakeClient:
def __init__(self, trans, transID, lfns, asIfProd):
self.trans = trans
self.transID = transID
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
self.transClient = TransformationClient()
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
self.bk = BookkeepingClient()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self.asIfProd = asIfProd
(self.transFiles, self.transReplicas) = self.prepareForPlugin(lfns)
def addFilesToTransformation(self, transID, lfns):
return S_OK({
'Failed': {},
'Successful': dict([(lfn, 'Added') for lfn in lfns])
})
def getTransformation(self, transID, extraParams=False):
if transID == self.transID and self.asIfProd:
transID = self.asIfProd
if transID != self.transID:
return self.transClient.getTransformation(transID)
res = self.trans.getType()
return DIRAC.S_OK({'Type': res['Value']})
def getReplicas(self):
return self.transReplicas
def getFiles(self):
return self.transFiles
def getCounters(self, table, attrList, condDict):
if condDict['TransformationID'] == self.transID and self.asIfProd:
condDict['TransformationID'] = self.asIfProd
if condDict['TransformationID'] != self.transID:
return self.transClient.getCounters(table, attrList, condDict)
possibleTargets = [
'CERN-RAW', 'CNAF-RAW', 'GRIDKA-RAW', 'IN2P3-RAW', 'SARA-RAW',
'PIC-RAW', 'RAL-RAW', 'RRCKI-RAW'
]
counters = []
for se in possibleTargets:
counters.append(({'UsedSE': se}, 0))
return DIRAC.S_OK(counters)
def getBookkeepingQuery(self, transID):
if transID == self.transID and self.asIfProd:
return self.transClient.getBookkeepingQuery(asIfProd)
return self.trans.getBkQuery()
def insertTransformationRun(self, transID, runID, xx):
return DIRAC.S_OK()
def getTransformationRuns(self, condDict):
if condDict['TransformationID'] == self.transID and self.asIfProd:
condDict['TransformationID'] = self.asIfProd
if condDict['TransformationID'] == self.transID:
transRuns = []
runs = condDict.get('RunNumber', [])
if not runs and self.transFiles:
res = self.bk.getFileMetadata(
[fileDict['LFN'] for fileDict in self.transFiles])
if not res['OK']:
return res
runs = list(
set(meta['RunNumber']
for meta in res['Value']['Successful'].itervalues()))
for run in runs:
transRuns.append({
'RunNumber': run,
'Status': "Active",
"SelectedSite": None
})
return DIRAC.S_OK(transRuns)
else:
return self.transClient.getTransformationRuns(condDict)
def getTransformationFiles(self, condDict=None):
if condDict.get('TransformationID') == self.transID and self.asIfProd:
condDict['TransformationID'] = self.asIfProd
if condDict.get('TransformationID') == self.transID:
transFiles = []
if 'Status' in condDict and 'Unused' not in condDict['Status']:
return DIRAC.S_OK(transFiles)
runs = None
if 'RunNumber' in condDict:
runs = condDict['RunNumber']
if not isinstance(runs, list):
runs = [runs]
for fileDict in self.transFiles:
if not runs or fileDict['RunNumber'] in runs:
transFiles.append({
'LFN': fileDict['LFN'],
'Status': 'Unused',
'RunNumber': fileDict['RunNumber']
})
return DIRAC.S_OK(transFiles)
else:
return self.transClient.getTransformationFiles(condDict=condDict)
def setParameterToTransformationFiles(self, transID, lfnDict):
"""
Update the transFiles with some parameters
"""
if transID == self.transID:
for fileDict in self.transFiles:
fileDict.update(lfnDict.get(fileDict['LFN'], {}))
return S_OK()
else:
return self.transClient.setParameterToTransformationFiles(
transID, lfnDict)
def getTransformationFilesCount(self, transID, field, selection=None):
if selection is None:
selection = {}
if transID == self.transID or selection.get(
'TransformationID') == self.transID:
runs = selection.get('RunNumber')
if runs and not isinstance(runs, list):
runs = [runs]
if field == 'Status':
counters = {'Unused': 0}
for fileDict in self.transFiles:
if not runs or fileDict['RunNumber'] in runs:
counters['Unused'] += 1
elif field == 'RunNumber':
counters = {}
for fileDict in self.transFiles:
runID = fileDict['RunNumber']
if not runs or runID in runs:
counters.setdefault(runID, 0)
counters[runID] += 1
else:
return DIRAC.S_ERROR('Not implemented for field ' + field)
counters['Total'] = sum(count for count in counters.itervalues())
return DIRAC.S_OK(counters)
else:
return self.transClient.getTransformationFilesCount(
transID, field, selection=selection)
def getTransformationRunStats(self, transIDs):
counters = {}
for transID in transIDs:
if transID == self.transID:
for fileDict in self.transFiles:
runID = fileDict['RunNumber']
counters[transID][runID]['Unused'] = counters.setdefault(
transID, {}).setdefault(runID, {}).setdefault(
'Unused', 0) + 1
for runID in counters[transID]:
counters[transID][runID]['Total'] = counters[transID][
runID]['Unused']
else:
res = self.transClient.getTransformationRunStats(transIDs)
if res['OK']:
counters.update(res['Value'])
else:
return res
return DIRAC.S_OK(counters)
def addRunsMetadata(self, runID, val):
return self.transClient.addRunsMetadata(runID, val)
def getRunsMetadata(self, runID):
return self.transClient.getRunsMetadata(runID)
def setTransformationRunStatus(self, transID, runID, status):
return DIRAC.S_OK()
def setTransformationRunsSite(self, transID, runID, site):
return DIRAC.S_OK()
def setFileStatusForTransformation(self, transID, status, lfns):
return DIRAC.S_OK()
def addTransformationRunFiles(self, transID, run, lfns):
return DIRAC.S_OK()
def setDestinationForRun(self, runID, site):
return DIRAC.S_OK()
def getDestinationForRun(self, runID):
return self.transClient.getDestinationForRun(runID)
def prepareForPlugin(self, lfns):
import time
print "Preparing the plugin input data (%d files)" % len(lfns)
type = self.trans.getType()['Value']
if not lfns:
return (None, None)
res = self.bk.getFileMetadata(lfns)
if res['OK']:
files = []
for lfn, metadata in res['Value']['Successful'].iteritems():
runID = metadata.get('RunNumber', 0)
runDict = {"RunNumber": runID, "LFN": lfn}
files.append(runDict)
else:
print "Error getting BK metadata", res['Message']
return ([], {})
replicas = {}
startTime = time.time()
from DIRAC.Core.Utilities.List import breakListIntoChunks
for lfnChunk in breakListIntoChunks(lfns, 200):
# print lfnChunk
if type.lower() in ("replication", "removal"):
res = self.dm.getReplicas(lfnChunk, getUrl=False)
else:
res = self.dm.getReplicasForJobs(lfnChunk, getUrl=False)
# print res
if res['OK']:
for lfn, ses in res['Value']['Successful'].iteritems():
if ses:
replicas[lfn] = sorted(ses)
else:
print "Error getting replicas of %d files:" % len(
lfns), res['Message']
print "Obtained replicas of %d files in %.3f seconds" % (
len(lfns), time.time() - startTime)
return (files, replicas)