def _getLFNRoot(lfn,
namespace='',
configVersion=0,
bkClient=None,
quick=False):
"""
return the root path of a given lfn
eg : /lhcb/data/CCRC08/00009909 = getLFNRoot(/lhcb/data/CCRC08/00009909/DST/0000/00009909_00003456_2.dst)
eg : /lhcb/MC/<year>/ = getLFNRoot(None)
"""
LFN_ROOT = ''
if not lfn:
LFN_ROOT = '/lhcb/%s/%s' % (namespace, configVersion)
gLogger.debug('LFN_ROOT will be %s' % (LFN_ROOT))
return LFN_ROOT
lfn = [
fname.replace(' ', '').replace('LFN:', '') for fname in lfn.split(';')
]
lfnroot = [part for part in lfn[0].split('/') if part]
if quick:
for part in lfnroot[0:4]:
LFN_ROOT += '/%s' % (part)
else:
if not bkClient:
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
bkClient = BookkeepingClient()
dataTypes = bkClient.getFileTypes({})
if not dataTypes['OK']:
raise Exception(dataTypes['Message'])
dataTypes = [x[0] for x in dataTypes['Value']['Records']]
gLogger.verbose('DataTypes retrieved from the BKK are:\n%s' %
(', '.join(dataTypes)))
gLogger.verbose('wf lfn: %s, namespace: %s, configVersion: %s' %
(lfn, namespace, configVersion))
for part in lfnroot:
if part not in dataTypes:
LFN_ROOT += '/%s' % (part)
else:
break
if re.search('//', LFN_ROOT):
LFN_ROOT = LFN_ROOT.replace('//', '/')
if namespace.lower() in ('test', 'debug'):
tmpLfnRoot = LFN_ROOT.split(os.path.sep)
if len(tmpLfnRoot) > 2:
tmpLfnRoot[2] = namespace
else:
tmpLfnRoot[-1] = namespace
LFN_ROOT = (os.path.sep).join(tmpLfnRoot, )
return LFN_ROOT
class BKQuery():
"""
It used to build a dictionary using a given Bookkeeping path
which is used to query the Bookkeeping database.
"""
def __init__(self,
bkQuery=None,
prods=None,
runs=None,
fileTypes=None,
visible=True,
eventTypes=None):
prods = prods if prods is not None else []
runs = runs if runs is not None else []
fileTypes = fileTypes if fileTypes is not None else []
self.extraBKitems = ("StartRun", "EndRun", "Production", "RunNumber")
self.__bkClient = BookkeepingClient()
bkPath = ''
bkQueryDict = {}
self.__bkFileTypes = set()
self.__exceptFileTypes = set()
self.__fakeAllDST = 'ZZZZZZZZALL.DST'
self.__alreadyWarned = False
if isinstance(bkQuery, BKQuery):
bkQueryDict = bkQuery.getQueryDict().copy()
elif isinstance(bkQuery, dict):
bkQueryDict = bkQuery.copy()
elif isinstance(bkQuery, basestring):
bkPath = bkQuery
bkQueryDict = self.buildBKQuery(bkPath=bkPath,
bkQueryDict=bkQueryDict,
prods=prods,
runs=runs,
fileTypes=fileTypes,
eventTypes=eventTypes,
visible=visible)
self.__bkPath = bkPath
self.__bkQueryDict = bkQueryDict
if not bkQueryDict.get('Visible'):
self.setVisible(visible)
def __str__(self):
return str(self.__bkQueryDict)
def buildBKQuery(self,
bkPath='',
bkQueryDict=None,
prods=None,
runs=None,
fileTypes=None,
visible=True,
eventTypes=None):
""" it builds a dictionary using a path
"""
bkQueryDict = bkQueryDict if bkQueryDict is not None else {}
prods = prods if prods is not None else []
if not isinstance(prods, list):
prods = [prods]
runs = runs if runs is not None else []
fileTypes = fileTypes if fileTypes is not None else []
gLogger.verbose("BKQUERY.buildBKQuery: Path %s, Dict %s, \
Prods %s, Runs %s, FileTypes %s, EventTypes %s, Visible %s" %
(bkPath, str(bkQueryDict), str(prods), str(runs),
str(fileTypes), str(eventTypes), visible))
self.__bkQueryDict = {}
if not bkPath and not prods and not bkQueryDict and not runs:
return {}
if bkQueryDict:
bkQuery = bkQueryDict.copy()
else:
bkQuery = {}
###### Query given as a path /ConfigName/ConfigVersion/ConditionDescription/ProcessingPass/EventType/FileType ######
# or if prefixed with evt: /ConfigName/ConfigVersion/EventType/ConditionDescription/ProcessingPass/FileType
if bkPath:
self.__getAllBKFileTypes()
bkFields = ("ConfigName", "ConfigVersion", "ConditionDescription",
"ProcessingPass", "EventType", "FileType")
url = bkPath.split(':', 1)
if len(url) == 1:
bkPath = url[0]
else:
if url[0] == 'evt':
bkFields = ("ConfigName", "ConfigVersion", "EventType",
"ConditionDescription", "ProcessingPass",
"FileType")
elif url[0] == 'pp':
bkFields = ("ProcessingPass", "EventType", "FileType")
elif url[0] == 'prod':
bkFields = ("Production", "ProcessingPass", "EventType",
"FileType")
elif url[0] == 'runs':
bkFields = ("Runs", "ProcessingPass", "EventType",
"FileType")
elif url[0] not in ('sim', 'daq', 'cond'):
gLogger.error('Invalid BK path:%s' % bkPath)
return self.__bkQueryDict
bkPath = url[1]
if bkPath[0] != '/':
bkPath = '/' + bkPath
if bkPath[0:2] == '//':
bkPath = bkPath[1:]
bkPath = bkPath.replace("RealData", "Real Data")
i = 0
processingPass = '******'
defaultPP = False
bk = bkPath.split('/')[1:] + len(bkFields) * ['']
for bpath in bk:
gLogger.verbose(
'buildBKQuery.1. Item #%d, Field %s, From Path %s, ProcessingPass %s'
% (i, bkFields[i], bpath, processingPass))
if bkFields[i] == 'ProcessingPass':
if bpath != '' and bpath.upper() != 'ALL' and \
not bpath.split(',')[0].split(' ')[0].isdigit() and \
not bpath.upper() in self.__bkFileTypes:
processingPass = os.path.join(processingPass, bpath)
continue
# Set the PP
if processingPass != '/':
bkQuery['ProcessingPass'] = processingPass
else:
defaultPP = True
i += 1
gLogger.verbose(
'buildBKQuery.2. Item #%d, Field %s, From Path %s, ProcessingPass %s'
% (i, bkFields[i], bpath, processingPass))
if bkFields[i] == 'EventType' and bpath:
eventTypeList = []
# print b
if bpath.upper() == 'ALL':
bpath = 'ALL'
else:
for et in bpath.split(','):
try:
eventType = int(et.split(' ')[0])
eventTypeList.append(eventType)
except ValueError:
pass
if len(eventTypeList) == 1:
eventTypeList = eventTypeList[0]
bpath = eventTypeList
gLogger.verbose('buildBKQuery. Event types %s' %
eventTypeList)
# Set the BK dictionary item
if bpath != '':
bkQuery[bkFields[i]] = bpath
if defaultPP:
# PP was empty, try once more to get the Event Type
defaultPP = False
else:
# Go to next item
i += 1
if i == len(bkFields):
break
gLogger.verbose('buildBKQuery. Query dict %s' % str(bkQuery))
# Set default event type to real data
if bkQuery.get('ConfigName') != 'MC' and not bkQuery.get(
'EventType'):
bkQuery['EventType'] = '90000000'
if bkQuery.get('EventType') == 'ALL':
bkQuery.pop('EventType')
# Run limits are given
runs = bkQuery.pop('Runs', runs)
if runs:
try:
bkQuery = parseRuns(bkQuery, runs)
except BadRunRange:
return self.__bkQueryDict
###### Query given as a list of production ######
if prods and str(prods[0]).upper() != 'ALL':
try:
bkQuery.setdefault('Production',
[]).extend([int(prod) for prod in prods])
except ValueError as ex: # The prods list does not contains numbers
gLogger.warn(ex)
gLogger.error('Invalid production list', str(prods))
return self.__bkQueryDict
# If an event type is specified
if eventTypes:
bkQuery['EventType'] = eventTypes
# Set the file type(s) taking into account excludes file types
fileTypes = bkQuery.get('FileType', fileTypes)
bkQuery.pop('FileType', None)
self.__bkQueryDict = bkQuery.copy()
fileType = self.__fileType(fileTypes)
# print fileType
if fileType:
bkQuery['FileType'] = fileType
# Remove all "ALL"'s in the dict, if any
for i in self.__bkQueryDict:
if isinstance(bkQuery[i], basestring) and bkQuery[i] == 'ALL':
bkQuery.pop(i)
# If there is only one production, make it faster with a single value rather than a list
prodList = bkQuery.get('Production')
if isinstance(prodList, list) and len(prodList) == 1:
bkQuery['Production'] = prodList[0]
self.__bkQueryDict = bkQuery.copy()
self.setVisible(visible)
# Set both event type entries
# print "Before setEventType", self.__bkQueryDict
if not self.setEventType(bkQuery.get('EventType')):
self.__bkQueryDict = {}
return self.__bkQueryDict
# Set conditions
# print "Before setConditions", self.__bkQueryDict
self.setConditions(
bkQuery.get(
'ConditionDescription',
bkQuery.get('DataTakingConditions',
bkQuery.get('SimulationConditions'))))
# print "Returned value", self.__bkQueryDict
return self.__bkQueryDict
def setOption(self, key, val):
"""
It insert an item to the dictionary. The key is an bookkeeping attribute (condition).
"""
if val:
self.__bkQueryDict[key] = val
else:
self.__bkQueryDict.pop(key, None)
return self.__bkQueryDict
def setConditions(self, cond=None):
""" Set the dictionary items for a given condition, or remove it (cond=None) """
if 'ConfigName' not in self.__bkQueryDict and cond:
gLogger.warn(
"Impossible to set Conditions to a BK Query without Configuration"
)
return self.__bkQueryDict
# There are two items in the dictionary: ConditionDescription and Simulation/DataTaking-Conditions
eventType = self.__bkQueryDict.get('EventType', 'ALL')
if self.__bkQueryDict.get('ConfigName') == 'MC' or \
(isinstance(eventType, basestring) and eventType.upper() != 'ALL' and
eventType[0] != '9'):
conditionsKey = 'SimulationConditions'
else:
conditionsKey = 'DataTakingConditions'
self.setOption('ConditionDescription', cond)
return self.setOption(conditionsKey, cond)
def setFileType(self, fileTypes=None):
"""insert the file type to the Boookkeeping dictionary
"""
return self.setOption('FileType', self.__fileType(fileTypes))
def setDQFlag(self, dqFlag='OK'):
"""
Sets the data quality.
"""
if isinstance(dqFlag, basestring):
dqFlag = dqFlag.upper()
elif isinstance(dqFlag, list):
dqFlag = [dq.upper() for dq in dqFlag]
return self.setOption('DataQuality', dqFlag)
def setStartDate(self, startDate):
"""
Sets the start date.
"""
return self.setOption('StartDate', startDate)
def setEndDate(self, endDate):
"""
Sets the end date
"""
return self.setOption('EndDate', endDate)
def setProcessingPass(self, processingPass):
"""
Sets the processing pass
"""
return self.setOption('ProcessingPass', processingPass)
def setEventType(self, eventTypes=None):
"""
Sets the event type
"""
if eventTypes:
if isinstance(eventTypes, basestring):
eventTypes = eventTypes.split(',')
elif not isinstance(eventTypes, list):
eventTypes = [eventTypes]
try:
eventTypes = [str(int(et)) for et in eventTypes]
except ValueError as ex:
gLogger.warn(ex)
gLogger.error('Invalid list of event types', eventTypes)
return {}
if isinstance(eventTypes, list) and len(eventTypes) == 1:
eventTypes = eventTypes[0]
return self.setOption('EventType', eventTypes)
def setVisible(self, visible=None):
"""
Sets the visibility flag
"""
if visible is True or (isinstance(visible, basestring)
and visible[0].lower() == 'y'):
visible = 'Yes'
if visible is False:
visible = 'No'
return self.setOption('Visible', visible)
def setExceptFileTypes(self, fileTypes):
"""
Sets the expected file types
"""
if not isinstance(fileTypes, list):
fileTypes = [fileTypes]
self.__exceptFileTypes.update(fileTypes)
self.setFileType(
[t for t in self.getFileTypeList() if t not in fileTypes])
def getExceptFileTypes(self):
return list(self.__exceptFileTypes)
def getQueryDict(self):
"""
Returns the bookkeeping dictionary
"""
return self.__bkQueryDict
def getPath(self):
"""
Returns the Bookkeeping path
"""
return self.__bkPath
def makePath(self):
"""
Builds a path from the dictionary
"""
bk = self.__bkQueryDict
fileType = bk.get('FileType', '')
if isinstance(fileType, list):
fileType = ','.join(fileType)
path = os.path.join(
'/', bk.get('ConfigName', ''), bk.get('ConfigVersion', ''),
bk.get('ConditionDescription', '.'),
bk.get('ProcessingPass', '.')[1:],
str(bk.get('EventType', '.')).replace('90000000', '.'),
fileType).replace('/./', '//')
while True:
if path.endswith('/'):
path = path[:-1]
else:
return path
def getFileTypeList(self):
"""
Returns the file types
"""
fileTypes = self.__bkQueryDict.get('FileType', [])
if not isinstance(fileTypes, list):
fileTypes = [fileTypes]
return fileTypes
def getEventTypeList(self):
"""
Returns the event types
"""
eventType = self.__bkQueryDict.get("EventType", [])
if eventType:
if not isinstance(eventType, list):
eventType = [eventType]
return eventType
def getProcessingPass(self):
"""
Returns the processing pass
"""
return self.__bkQueryDict.get('ProcessingPass', '')
def getConditions(self):
"""
Returns the Simulation/data taking conditions
"""
return self.__bkQueryDict.get('ConditionDescription', '')
def getConfiguration(self):
"""
Returns the configuration name and configuration version
"""
configName = self.__bkQueryDict.get('ConfigName', '')
configVersion = self.__bkQueryDict.get('ConfigVersion', '')
if not configName or not configVersion:
return ''
return os.path.join('/', configName, configVersion)
def isVisible(self):
"""
Returns True/False depending on the visibility flag
"""
return self.__bkQueryDict.get('Visible', 'All')
def __fileType(self, fileType=None, returnList=False):
"""
return the file types taking into account the expected file types
"""
gLogger.verbose("BKQuery.__fileType: %s, fileType: %s" %
(self, fileType))
if not fileType:
return []
self.__getAllBKFileTypes()
if isinstance(fileType, list):
fileTypes = fileType
else:
fileTypes = fileType.split(',')
allRequested = None
if fileTypes[0].lower() == "all":
allRequested = True
bkTypes = self.getBKFileTypes()
gLogger.verbose('BKQuery.__fileType: bkTypes %s' % str(bkTypes))
if bkTypes:
fileTypes = list(set(bkTypes) - self.__exceptFileTypes)
else:
fileTypes = []
expandedTypes = set()
# print "Requested", fileTypes
for fileType in fileTypes:
if fileType.lower() == 'all.hist':
allRequested = False
expandedTypes.update([
t for t in self.__exceptFileTypes.union(self.__bkFileTypes)
if t.endswith('HIST')
])
elif fileType.lower().find("all.") == 0:
ext = '.' + fileType.split('.')[1]
fileType = []
if allRequested is None:
allRequested = True
expandedTypes.update([
t for t in set(self.getBKFileTypes()) -
self.__exceptFileTypes if t.endswith(ext)
])
else:
expandedTypes.add(fileType)
# Remove __exceptFileTypes only if not explicitly required
# print "Obtained", fileTypes, expandedTypes
gLogger.verbose(
"BKQuery.__fileType: requested %s, expanded %s, except %s" %
(allRequested, expandedTypes, self.__exceptFileTypes))
if expandedTypes - self.__bkFileTypes and not self.__alreadyWarned:
self.__alreadyWarned = True
gLogger.always(
"**** Take care: some requested file types do not exist!!",
str(sorted(expandedTypes - self.__bkFileTypes)))
if allRequested or not expandedTypes & self.__exceptFileTypes:
expandedTypes -= self.__exceptFileTypes
gLogger.verbose("BKQuery.__fileType: result %s" %
sorted(expandedTypes))
if len(expandedTypes) == 1 and not returnList:
return list(expandedTypes)[0]
else:
return list(expandedTypes)
def __getAllBKFileTypes(self):
"""
Returns the file types from the bookkeeping database
"""
if not self.__bkFileTypes:
self.__bkFileTypes = set([self.__fakeAllDST])
warned = False
while True:
res = self.__bkClient.getAvailableFileTypes()
if res['OK']:
dbresult = res['Value']
for record in dbresult['Records']:
if record[0].endswith('HIST') or \
record[0].endswith('ETC') or \
record[0] == 'LOG' or \
record[0].endswith('ROOT'):
self.__exceptFileTypes.add(record[0])
self.__bkFileTypes.add(record[0])
break
if not warned:
gLogger.always('Error getting BK file types, retrying',
res['Message'])
warned = True
def __getBKFiles(self, bkQueryDict, retries=5):
"""
Call BK getFiles() with some retries
"""
if not retries:
retries = sys.maxsize
errorLogged = False
while retries:
res = self.__bkClient.getFiles(bkQueryDict)
if res['OK']:
break
retries -= 1
if not errorLogged:
errorLogged = True
gLogger.warn("Error getting files from BK, retrying...",
res['Message'])
return res
def getLFNsAndSize(self, getSize=True):
"""
Returns the LFNs and their size for a given data set
"""
self.__getAllBKFileTypes()
res = self.__getBKFiles(self.__bkQueryDict)
lfns = []
lfnSize = 0
if not res['OK']:
gLogger.error("Error from BK for %s:" % self.__bkQueryDict,
res['Message'])
else:
lfns = set(res['Value'])
exceptFiles = list(self.__exceptFileTypes)
if exceptFiles and not self.__bkQueryDict.get('FileType'):
res = self.__getBKFiles(
BKQuery(self.__bkQueryDict).setOption(
'FileType', exceptFiles))
if res['OK']:
lfnsExcept = set(res['Value']) & lfns
else:
gLogger.error(
"***** ERROR ***** Error in getting dataset from BK for %s files:"
% exceptFiles, res['Message'])
lfnsExcept = set()
if lfnsExcept:
gLogger.warn(
"***** WARNING ***** Found %d files in BK query that will be \
excluded (file type in %s)!" % (len(lfnsExcept), str(exceptFiles)))
gLogger.warn(
" If creating a transformation, set '--FileType ALL'"
)
lfns = lfns - lfnsExcept
else:
exceptFiles = False
if getSize:
# Get size only if needed
query = BKQuery(self.__bkQueryDict)
query.setOption("FileSize", True)
res = self.__getBKFiles(query.getQueryDict())
if res['OK'] and isinstance(res['Value'],
list) and res['Value'][0]:
lfnSize = res['Value'][0]
if exceptFiles and not self.__bkQueryDict.get('FileType'):
res = self.__getBKFiles(
query.setOption('FileType', exceptFiles))
if res['OK'] and isinstance(res['Value'],
list) and res['Value'][0]:
lfnSize -= res['Value'][0]
lfnSize /= 1000000000000.
else:
lfnSize = 0.
return {'LFNs': list(lfns), 'LFNSize': lfnSize}
def getLFNSize(self, visible=None):
"""
Returns the size of a given data set
"""
if visible is None:
visible = self.isVisible()
res = self.__getBKFiles(
BKQuery(self.__bkQueryDict,
visible=visible).setOption('FileSize', True))
if res['OK'] and isinstance(res['Value'], list) and res['Value'][0]:
lfnSize = res['Value'][0]
else:
lfnSize = 0
return lfnSize
def getNumberOfLFNs(self, visible=None):
"""
Returns the number of LFNs correspond to a given data set
"""
if visible is None:
visible = self.isVisible()
if self.isVisible() != visible:
query = BKQuery(self.__bkQueryDict, visible=visible)
else:
query = self
fileTypes = query.getFileTypeList()
nbFiles = 0
size = 0
for fileType in fileTypes:
if fileType:
res = self.__bkClient.getFilesSummary(
query.setFileType(fileType))
# print query, res
if res['OK']:
res = res['Value']
ind = res['ParameterNames'].index('NbofFiles')
if res['Records'][0][ind]:
nbFiles += res['Records'][0][ind]
ind1 = res['ParameterNames'].index('FileSize')
size += res['Records'][0][ind1]
# print 'Visible',query.isVisible(),fileType, 'Files:',
# res['Records'][0][ind], 'Size:', res['Records'][0][ind1]
return {'NumberOfLFNs': nbFiles, 'LFNSize': size}
def getLFNs(self, printSEUsage=False, printOutput=True, visible=None):
"""
returns a list of lfns. It prints statistics about the data sets if it is requested.
"""
if visible is None:
visible = self.isVisible()
if self.isVisible() != visible:
query = BKQuery(self.__bkQueryDict, visible=visible)
else:
query = self
# Loop for each production or each event type rather than make a single query
loopItem = None
prods = self.__bkQueryDict.get('Production')
eventTypes = self.__bkQueryDict.get('EventType')
if prods and isinstance(prods, list):
loopItem = 'Production'
loopList = prods
elif eventTypes and isinstance(eventTypes, list):
loopItem = 'EventType'
loopList = eventTypes
if loopItem:
# It's faster to loop on a list of prods or event types than query the BK with a list as argument
lfns = []
lfnSize = 0
if query == self:
query = BKQuery(self.__bkQueryDict, visible=visible)
for item in loopList:
query.setOption(loopItem, item)
lfnsAndSize = query.getLFNsAndSize(getSize=printOutput)
lfns += lfnsAndSize['LFNs']
lfnSize += lfnsAndSize['LFNSize']
else:
lfnsAndSize = query.getLFNsAndSize(getSize=printOutput)
lfns = lfnsAndSize['LFNs']
lfnSize = lfnsAndSize['LFNSize']
if not lfns:
gLogger.verbose("No files found for BK query %s" %
str(self.__bkQueryDict))
else:
lfns.sort()
# Only for printing
if printOutput:
gLogger.notice("\n%d files (%.1f TB) in directories:" %
(len(lfns), lfnSize))
dirs = {}
for lfn in lfns:
directory = os.path.join(os.path.dirname(lfn), '')
dirs[directory] = dirs.setdefault(directory, 0) + 1
for directory in sorted(dirs):
gLogger.notice("%s %s files" %
(directory, dirs[directory]))
if printSEUsage:
rpc = RPCClient('DataManagement/StorageUsage')
totalUsage = {}
totalSize = 0
for directory in dirs:
res = rpc.getStorageSummary(directory, '', '', [])
if res['OK']:
for se in [
se for se in res['Value']
if not se.endswith("-ARCHIVE")
]:
totalUsage[se] = totalUsage.setdefault(
se, 0) + res['Value'][se]['Size']
totalSize += res['Value'][se]['Size']
ses = sorted(totalUsage)
totalUsage['Total'] = totalSize
ses.append('Total')
gLogger.notice("\n%s %s" % ("SE".ljust(20), "Size (TB)"))
for se in ses:
gLogger.notice("%s %s" %
(se.ljust(20),
('%.1f' %
(totalUsage[se] / 1000000000000.))))
return lfns
def getDirs(self, printOutput=False, visible=None):
"""
Returns the directories
"""
if visible is None:
visible = self.isVisible()
lfns = self.getLFNs(printSEUsage=True,
printOutput=printOutput,
visible=visible)
dirs = set()
for lfn in lfns:
dirs.add(os.path.dirname(lfn))
return sorted(dirs)
@staticmethod
def __getProdStatus(prod):
"""
Returns the status of a given transformation
"""
res = TransformationClient().getTransformation(prod, extraParams=False)
if not res['OK']:
gLogger.error("Couldn't get information on production %d" % prod)
return None
return res['Value']['Status']
def getBKRuns(self):
"""
It returns a list of runs from the bookkeeping.
"""
if self.getProcessingPass().replace('/', '') == 'Real Data':
return self.getBKProductions()
def getBKProductions(self, visible=None):
"""
It returns a list of productions
"""
if visible is None:
visible = self.isVisible()
prodList = self.__bkQueryDict.get('Production')
if prodList:
if not isinstance(prodList, list):
prodList = [prodList]
return sorted(prodList)
if not self.getProcessingPass():
gLogger.fatal(
'Impossible to get a list of productions without the Processing Pass'
)
return []
eventTypes = self.__bkQueryDict.get('EventType')
if not isinstance(eventTypes, list):
eventTypes = [eventTypes]
fullList = set()
for eventType in eventTypes:
bkQ = BKQuery(self.__bkQueryDict)
bkQ.setVisible(visible)
bkDict = bkQ.setEventType(eventType)
# gLogger.notice( 'Get productions for BK query', str( bkDict ) )
res = self.__bkClient.getProductions(bkDict)
if not res['OK']:
gLogger.error('Error getting productions from BK',
res['Message'])
return []
if self.getProcessingPass().replace('/', '') != 'Real Data':
fileTypes = self.getFileTypeList()
prodList = set(prod for prods in res['Value']['Records']
for prod in prods
if self.__getProdStatus(prod) != 'Deleted')
# print '\n', self.__bkQueryDict, res['Value']['Records'], '\nVisible:', visible, prodList
pList = set()
if fileTypes:
transClient = TransformationClient()
for prod in prodList:
res = transClient.getBookkeepingQuery(prod)
if res['OK'] and res['Value']['FileType'] in fileTypes:
pList.add(prod)
if not pList:
pList = prodList
else:
runList = sorted(
[-run for r in res['Value']['Records'] for run in r])
startRun = int(self.__bkQueryDict.get('StartRun', 0))
endRun = int(self.__bkQueryDict.get('EndRun', sys.maxsize))
pList = set(run for run in runList
if run >= startRun and run <= endRun)
fullList.update(pList)
return sorted(fullList)
def getBKConditions(self):
"""
It returns the data taking / simulation conditions
"""
conditions = self.__bkQueryDict.get('ConditionDescription')
if conditions:
if not isinstance(conditions, list):
conditions = [conditions]
return conditions
result = self.__bkClient.getConditions(self.__bkQueryDict)
if result['OK']:
resList = result['Value']
else:
return []
conditions = []
for res in resList:
ind = res['ParameterNames'].index('Description')
if res['Records']:
conditions += [par[ind] for par in res['Records']]
break
return sorted(conditions)
def getBKEventTypes(self):
"""
It returns the event types
"""
eventType = self.getEventTypeList()
if eventType:
return eventType
res = self.__bkClient.getEventTypes(self.__bkQueryDict)['Value']
ind = res['ParameterNames'].index('EventType')
eventTypes = sorted([rec[ind] for rec in res['Records']])
return eventTypes
def getBKFileTypes(self, bkDict=None):
"""
It returns the file types.
"""
fileTypes = self.getFileTypeList()
# print "Call getBKFileTypes:", self, fileTypes
if not fileTypes:
if not bkDict:
bkDict = self.__bkQueryDict.copy()
else:
bkDict = bkDict.copy()
bkDict.setdefault('Visible', 'All')
bkDict.pop('RunNumber', None)
fileTypes = []
eventTypes = bkDict.get('EventType')
if isinstance(eventTypes, list):
for et in eventTypes:
bkDict['EventType'] = et
fileTypes += self.getBKFileTypes(bkDict)
else:
res = self.__bkClient.getFileTypes(bkDict)
if res['OK']:
res = res['Value']
ind = res['ParameterNames'].index('FileTypes')
fileTypes = [
rec[ind] for rec in res['Records']
if rec[ind] not in self.__exceptFileTypes
]
if 'ALL.DST' in fileTypes:
fileTypes.remove('ALL.DST')
fileTypes.append(self.__fakeAllDST)
# print 'FileTypes1', fileTypes
fileTypes = self.__fileType(fileTypes, returnList=True)
# print 'FileTypes2', fileTypes
if self.__fakeAllDST in fileTypes:
fileTypes.remove(self.__fakeAllDST)
fileTypes.append('ALL.DST')
# print 'FileTypes3', fileTypes
return fileTypes
def getBKProcessingPasses(self, queryDict=None, depth=None):
"""
It returns the processing pass.
"""
if depth is None:
depth = sys.maxsize
processingPasses = {}
if not queryDict:
queryDict = self.__bkQueryDict.copy()
initialPP = queryDict.get('ProcessingPass', '/')
# print "Start", initialPP, queryDict
res = self.__bkClient.getProcessingPass(queryDict, initialPP)
if not res['OK']:
if 'Empty Directory' not in res['Message']:
gLogger.error(
"ERROR getting processing passes for %s" % queryDict,
res['Message'])
return {}
ppRecords = res['Value'][0]
if 'Name' in ppRecords['ParameterNames']:
ind = ppRecords['ParameterNames'].index('Name')
passes = sorted([
os.path.join(initialPP, rec[ind])
for rec in ppRecords['Records']
])
else:
passes = []
evtRecords = res['Value'][1]
if 'EventType' in evtRecords['ParameterNames']:
ind = evtRecords['ParameterNames'].index('EventType')
eventTypes = [str(rec[ind]) for rec in evtRecords['Records']]
else:
eventTypes = []
if passes and depth:
depth -= 1
nextProcessingPasses = {}
for pName in passes:
processingPasses[pName] = []
queryDict['ProcessingPass'] = pName
nextProcessingPasses.update(
self.getBKProcessingPasses(queryDict, depth=depth))
processingPasses.update(nextProcessingPasses)
if eventTypes:
processingPasses[initialPP] = eventTypes
for pName in ('/Real Data', '/'):
if pName in processingPasses:
processingPasses.pop(pName)
# print "End", initialPP, [( key, processingPasses[key] ) for key in sorted( processingPasses.keys() )]
return processingPasses