def _displaySetup(self, dsPath, head):
if os.path.exists(dsPath):
nickNames = set()
for block in DataProvider.loadFromFile(dsPath).getBlocks():
nickNames.add(block[DataProvider.Nickname])
utils.vprint('Mapping between nickname and other settings:\n', -1)
report = []
for nick in sorted(nickNames):
lumi_filter_str = formatLumi(
self._nmLumi.lookup(nick, '', is_selector=False))
if len(lumi_filter_str) > 4:
nice_lumi_filter = '%s ... %s (%d entries)' % (
lumi_filter_str[0], lumi_filter_str[-1],
len(lumi_filter_str))
else:
nice_lumi_filter = str.join(', ', lumi_filter_str)
config_files = self._nmCfg.lookup(nick, '', is_selector=False)
tmp = {
0: nick,
1: str.join(', ', imap(os.path.basename, config_files)),
2: nice_lumi_filter
}
lookupvars = {'DATASETNICK': nick}
for src in self._pm.lookupSources:
src.fillParameterInfo(None, lookupvars)
tmp.update(lookupvars)
report.append(tmp)
utils.printTabular(head, report, 'cl')
utils.vprint(level=-1)
def process(self, pNum, splitInfo, result):
if not self._lumi_filter.empty():
lumi_filter = self._lumi_filter.lookup(splitInfo[DataSplitter.Nickname], is_selector = False)
if lumi_filter:
idxRuns = splitInfo[DataSplitter.MetadataHeader].index("Runs")
iterRuns = ichain(imap(lambda m: m[idxRuns], splitInfo[DataSplitter.Metadata]))
short_lumi_filter = filterLumiFilter(list(iterRuns), lumi_filter)
result['LUMI_RANGE'] = str.join(',', imap(lambda lr: '"%s"' % lr, formatLumi(short_lumi_filter)))
def process(self, pNum, splitInfo, result):
if self.enabled():
lumi_filter = self._lumi_filter.lookup(splitInfo[DataSplitter.Nickname], is_selector = False)
if lumi_filter:
idxRuns = splitInfo[DataSplitter.MetadataHeader].index("Runs")
iterRuns = ichain(imap(lambda m: m[idxRuns], splitInfo[DataSplitter.Metadata]))
short_lumi_filter = filterLumiFilter(list(iterRuns), lumi_filter)
result['LUMI_RANGE'] = str.join(',', imap(lambda lr: '"%s"' % lr, formatLumi(short_lumi_filter)))
def __init__(self, config, name):
head = [(0, "Nickname")]
# Mapping between nickname and config files:
cfgList = config.get("nickname config", "")
self.nmCfg = config.getDict(
"nickname config", {}, parser=lambda x: map(str.strip, x.split(",")), str=lambda x: str.join(",", x)
)[0]
if cfgList:
if "config file" in config.getOptions():
raise ConfigError("Please use 'nickname config' instead of 'config file'")
allConfigFiles = utils.flatten(self.nmCfg.values())
config.set("config file", str.join("\n", allConfigFiles))
head.append((1, "Config file"))
# Mapping between nickname and constants:
self.nmCName = map(str.strip, config.get("nickname constants", "").split())
self.nmConst = {}
for var in self.nmCName:
tmp = config.getDict(var, {})[0]
for (nick, value) in tmp.items():
if value:
self.nmConst.setdefault(nick, {})[var] = value
else:
self.nmConst.setdefault(nick, {})[var] = ""
head.append((var, var))
# Mapping between nickname and lumi filter:
if "lumi filter" in config.getOptions():
raise ConfigError("Please use 'nickname lumi filter' instead of 'lumi filter'")
lumiParse = lambda x: formatLumi(parseLumiFilter(x))
self.nmLumi = config.getDict("nickname lumi filter", {}, parser=lumiParse)[0]
if self.nmLumi:
for dataset in config.get("dataset", "").splitlines():
(datasetNick, datasetProvider, datasetExpr) = DataProvider.parseDatasetExpr(config, dataset, None)
config.set(
"dataset %s" % datasetNick,
"lumi filter",
str.join(",", utils.flatten(fromNM(self.nmLumi, datasetNick, []))),
)
config.set("lumi filter", str.join(",", self.nmLumi.get(None, [])))
head.append((2, "Lumi filter"))
utils.vprint("Mapping between nickname and other settings:\n", -1)
def report():
for nick in sorted(set(self.nmCfg.keys() + self.nmConst.keys() + self.nmLumi.keys())):
tmp = {
0: nick,
1: str.join(", ", map(os.path.basename, self.nmCfg.get(nick, ""))),
2: self.displayLumi(self.nmLumi.get(nick, "")),
}
yield utils.mergeDicts([tmp, self.nmConst.get(nick, {})])
utils.printTabular(head, report(), "cl")
utils.vprint(level=-1)
CMSSW.__init__(self, config, name)
def getActiveLumiFilter(self, lumifilter, jobNum = None):
getLR = lambda x: str.join(',', map(lambda x: '"%s"' % x, formatLumi(x)))
return getLR(lumifilter) # TODO: Validate subset selection
try:
splitInfo = self.dataSplitter.getSplitInfo(jobNum)
runTag = splitInfo[DataSplitter.MetadataHeader].index("Runs")
runList = utils.listMapReduce(lambda m: m[runTag], splitInfo[DataSplitter.Metadata])
return getLR(filterLumiFilter(runList, lumifilter))
except Exception:
return getLR(lumifilter)
def _displaySetup(self, dsPath, head):
if os.path.exists(dsPath):
nickNames = set()
for block in DataProvider.loadFromFile(dsPath).getBlocks():
nickNames.add(block[DataProvider.Nickname])
utils.vprint('Mapping between nickname and other settings:\n', -1)
report = []
for nick in sorted(nickNames):
lumi_filter_str = formatLumi(self._nmLumi.lookup(nick, '', is_selector = False))
if len(lumi_filter_str) > 4:
nice_lumi_filter = '%s ... %s (%d entries)' % (lumi_filter_str[0], lumi_filter_str[-1], len(lumi_filter_str))
else:
nice_lumi_filter = str.join(', ', lumi_filter_str)
config_files = self._nmCfg.lookup(nick, '', is_selector = False)
tmp = {0: nick, 1: str.join(', ', imap(os.path.basename, config_files)), 2: nice_lumi_filter}
lookupvars = {'DATASETNICK': nick}
for src in self._pm.lookupSources:
src.fillParameterInfo(None, lookupvars)
tmp.update(lookupvars)
report.append(tmp)
utils.printTabular(head, report, 'cl')
utils.vprint(level = -1)
def __init__(self, config, datasetExpr, datasetNick, datasetID = 0):
DataProvider.__init__(self, config, datasetExpr, datasetNick, datasetID)
# PhEDex blacklist: 'T1_DE_KIT', 'T1_US_FNAL' and '*_Disk' allow user jobs - other T1's dont!
self.phedexBL = config.getList('phedex sites', ['-T3_US_FNALLPC'])
self.phedexWL = config.getList('phedex t1 accept', ['T1_DE_KIT', 'T1_US_FNAL'])
self.phedexT1 = config.get('phedex t1 mode', 'disk').lower()
self.onlyComplete = config.getBool('only complete sites', True)
self.locationFormat = config.get('location format', 'hostname').lower() # hostname or sitedb
if self.locationFormat not in ['hostname', 'sitedb', 'both']:
raise ConfigError('Invalid location format: %s' % self.locationFormat)
(self.datasetPath, self.url, self.datasetBlock) = utils.optSplit(datasetExpr, '@#')
self.url = utils.QM(self.url, self.url, config.get('dbs instance', ''))
self.datasetBlock = utils.QM(self.datasetBlock, self.datasetBlock, 'all')
self.includeLumi = config.getBool('keep lumi metadata', False)
self.onlyValid = config.getBool('only valid', True)
self.checkUnique = config.getBool('check unique', True)
# This works in tandem with active task module (cmssy.py supports only [section] lumi filter!)
self.selectedLumis = parseLumiFilter(config.get('lumi filter', ''))
if self.selectedLumis:
utils.vprint('Runs/lumi section filter enabled! (%d entries)' % len(self.selectedLumis), -1, once = True)
utils.vprint('\tThe following runs and lumi sections are selected:', 1, once = True)
utils.vprint('\t' + utils.wrapList(formatLumi(self.selectedLumis), 65, ',\n\t'), 1, once = True)
def formatLumiNice(lumis):
lumi_filter_str = formatLumi(lumis)
if len(lumi_filter_str) < 5:
return str.join(', ', lumi_filter_str)
return '%s ... %s (%d entries)' % (lumi_filter_str[0], lumi_filter_str[-1], len(lumi_filter_str))
def fmtLR(x):
if x:
return self.displayLumi(formatLumi(x))
return x
def outputGC(lumis, stream = sys.stdout):
stream.write('%s\n' % utils.wrapList(formatLumi(lumis), 60, ',\n'))
#!/usr/bin/env python
# | Copyright 2010-2016 Karlsruhe Institute of Technology
# |
# | Licensed under the Apache License, Version 2.0 (the "License");
# | you may not use this file except in compliance with the License.
# | You may obtain a copy of the License at
# |
# | http://www.apache.org/licenses/LICENSE-2.0
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
try:
from xmlrpclib import ServerProxy
except ImportError:
from xmlrpc.client import ServerProxy
import sys
from gcSupport import utils
from grid_control_cms.lumi_tools import formatLumi, mergeLumi, parseLumiFromJSON
server = ServerProxy('http://pccmsdqm04.cern.ch/runregistry/xmlrpc')
data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json', {'groupName': 'Collisions10'})
runs = parseLumiFromJSON(data)
sys.stdout.write('lumi filter = %s\n' % utils.wrapList(formatLumi(mergeLumi(runs)), 60, ',\n\t'))
def formatLumiNice(lumis):
lumi_filter_str = formatLumi(lumis)
if len(lumi_filter_str) < 5:
return str.join(', ', lumi_filter_str)
return '%s ... %s (%d entries)' % (lumi_filter_str[0], lumi_filter_str[-1],
len(lumi_filter_str))
def outputGC(lumis, stream=sys.stdout):
stream.write('%s\n' % utils.wrapList(formatLumi(lumis), 60, ',\n'))
#!/usr/bin/env python
#-# Copyright 2010 Karlsruhe Institute of Technology
#-#
#-# Licensed under the Apache License, Version 2.0 (the "License");
#-# you may not use this file except in compliance with the License.
#-# You may obtain a copy of the License at
#-#
#-# http://www.apache.org/licenses/LICENSE-2.0
#-#
#-# Unless required by applicable law or agreed to in writing, software
#-# distributed under the License is distributed on an "AS IS" BASIS,
#-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#-# See the License for the specific language governing permissions and
#-# limitations under the License.
from gcSupport import utils
from grid_control_cms.lumi_tools import formatLumi, parseLumiFromJSON, mergeLumi
import xmlrpclib
server = xmlrpclib.ServerProxy('http://pccmsdqm04.cern.ch/runregistry/xmlrpc')
data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json', {'groupName': 'Collisions10'})
runs = parseLumiFromJSON(data)
utils.vprint("lumi filter = %s" % utils.wrapList(formatLumi(mergeLumi(runs)), 60, ',\n\t'), -1)
if "#" in opts.remove:
eraseBlock(opts.remove)
else:
for block in api.listBlocks(opts.remove.split("#")[0]):
eraseBlock(block["Name"])
api.deleteProcDS(opts.remove.split("#")[0])
elif opts.listlumis:
allrl = []
for fileInfo in api.listFiles(opts.listlumis, retriveList=['retrive_lumi']):
lfn = fileInfo['LogicalFileName']
rl = []
for lumi in fileInfo['LumiList']:
rl.append(([int(lumi["RunNumber"]), int(lumi["LumiSectionNumber"])], [int(lumi["RunNumber"]), int(lumi["LumiSectionNumber"])]))
print lfn
print utils.wrapList(formatLumi(mergeLumi(rl)), 70, ',\n\t')
allrl.extend(rl)
print "\nComplete dataset:"
print utils.wrapList(formatLumi(mergeLumi(allrl)), 70, ',\n\t')
elif opts.list:
for block in api.listBlocks(opts.list):
print block["Name"]
elif opts.files:
for f in api.listFiles(opts.files, retriveList=['retrive_block', 'retrive_run', 'retrive_lumi']):
print f
print
elif opts.dump:
print api.listDatasetContents(opts.dump.split("#")[0], opts.dump)
#!/usr/bin/env python
# | Copyright 2010-2016 Karlsruhe Institute of Technology
# |
# | Licensed under the Apache License, Version 2.0 (the "License");
# | you may not use this file except in compliance with the License.
# | You may obtain a copy of the License at
# |
# | http://www.apache.org/licenses/LICENSE-2.0
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
try:
from xmlrpclib import ServerProxy
except ImportError:
from xmlrpc.client import ServerProxy
import sys
from gcSupport import utils
from grid_control_cms.lumi_tools import formatLumi, mergeLumi, parseLumiFromJSON
server = ServerProxy('http://pccmsdqm04.cern.ch/runregistry/xmlrpc')
data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json',
{'groupName': 'Collisions10'})
runs = parseLumiFromJSON(data)
sys.stdout.write('lumi filter = %s\n' %
utils.wrapList(formatLumi(mergeLumi(runs)), 60, ',\n\t'))
