def setup_dataset(config, dataset):
if dataset.lower() == 'true':
utils.vprint('Registering dummy data provider data')
dataSplitter = DummySplitter()
else:
dataSplitter = DataSplitter.loadStateForScript(dataset)
DataParameterSource = ParameterSource.getClass('DataParameterSource')
DataParameterSource.datasetsAvailable['data'] = DataParameterSource(
config.getWorkPath(), 'data', None, dataSplitter, DataSplitProcessorTest())
def list_parameters(opts, psource):
(result, needGCParam) = get_parameters(opts, psource)
enabledOutput = opts.output.split(',')
output = lfilter(lambda k: not opts.output or k in enabledOutput,
psource.getJobKeys())
stored = lfilter(lambda k: k.untracked == False, output)
untracked = lfilter(lambda k: k.untracked == True, output)
if opts.collapse > 0:
result_old = result
result = {}
result_nicks = {}
head = [('COLLATE_JOBS', '# of jobs')]
if 'DATASETSPLIT' in stored:
stored.remove('DATASETSPLIT')
if opts.collapse == 1:
stored.append('DATASETNICK')
head.append(('DATASETNICK', 'DATASETNICK'))
elif opts.collapse == 2:
head.append(('COLLATE_NICK', '# of nicks'))
for pset in result_old:
if ('DATASETSPLIT' in pset) and (opts.collapse == 1):
pset.pop('DATASETSPLIT')
nickname = None
if ('DATASETNICK' in pset) and (opts.collapse == 2):
nickname = pset.pop('DATASETNICK')
h = md5_hex(repr(lmap(lambda key: pset.get(str(key)), stored)))
result.setdefault(h, []).append(pset)
result_nicks.setdefault(h, set()).add(nickname)
def doCollate(h):
tmp = result[h][0]
tmp['COLLATE_JOBS'] = len(result[h])
tmp['COLLATE_NICK'] = len(result_nicks[h])
return tmp
result = lmap(doCollate, result)
else:
head = [('GC_JOB_ID', '#')]
if needGCParam:
head.append(('GC_PARAM', 'GC_PARAM'))
if opts.active:
head.append((ParameterInfo.ACTIVE, 'ACTIVE'))
if opts.visible:
stored = opts.visible.split(',')
head.extend(sorted(izip(stored, stored)))
if opts.untracked:
head.extend(
sorted(
imap(
lambda n: (n, '(%s)' % n),
ifilter(lambda n: n not in ['GC_PARAM', 'GC_JOB_ID'],
untracked))))
utils.vprint('')
utils.printTabular(head, result)
def get_parameters(opts, psource):
result = []
needGCParam = False
if psource.getMaxJobs() is not None:
countActive = 0
for info in psource.iterJobs():
if info[ParameterInfo.ACTIVE]:
countActive += 1
if opts.disabled or info[ParameterInfo.ACTIVE]:
if not info[ParameterInfo.ACTIVE]:
info['GC_PARAM'] = 'N/A'
if str(info['GC_PARAM']) != str(info['GC_JOB_ID']):
needGCParam = True
result.append(info)
if opts.parseable:
utils.vprint('Count,%d,%d' % (countActive, psource.getMaxJobs()))
else:
utils.vprint('Number of parameter points: %d' %
psource.getMaxJobs())
if countActive != psource.getMaxJobs():
utils.vprint('Number of active parameter points: %d' %
countActive)
else:
job = 123
if opts.job is not None:
job = int(opts.job)
utils.vprint(
'Unbounded parameter space found - showing parameters for job %d' %
job, -1)
result.append(psource.getJobInfo(job))
return (result, needGCParam)
def list_parameters(opts, psource):
(result, needGCParam) = get_parameters(opts, psource)
enabledOutput = opts.output.split(',')
output = lfilter(lambda k: not opts.output or k in enabledOutput, psource.getJobKeys())
stored = lfilter(lambda k: k.untracked == False, output)
untracked = lfilter(lambda k: k.untracked == True, output)
if opts.collapse > 0:
result_old = result
result = {}
result_nicks = {}
head = [('COLLATE_JOBS', '# of jobs')]
if 'DATASETSPLIT' in stored:
stored.remove('DATASETSPLIT')
if opts.collapse == 1:
stored.append('DATASETNICK')
head.append(('DATASETNICK', 'DATASETNICK'))
elif opts.collapse == 2:
head.append(('COLLATE_NICK', '# of nicks'))
for pset in result_old:
if ('DATASETSPLIT' in pset) and (opts.collapse == 1):
pset.pop('DATASETSPLIT')
nickname = None
if ('DATASETNICK' in pset) and (opts.collapse == 2):
nickname = pset.pop('DATASETNICK')
h = md5_hex(repr(lmap(pset.get, stored)))
result.setdefault(h, []).append(pset)
result_nicks.setdefault(h, set()).add(nickname)
def doCollate(h):
tmp = result[h][0]
tmp['COLLATE_JOBS'] = len(result[h])
tmp['COLLATE_NICK'] = len(result_nicks[h])
return tmp
result = lmap(doCollate, result)
else:
head = [('GC_JOB_ID', '#')]
if needGCParam:
head.append(('GC_PARAM', 'GC_PARAM'))
if opts.active:
head.append((ParameterInfo.ACTIVE, 'ACTIVE'))
if opts.visible:
stored = opts.visible.split(',')
head.extend(sorted(izip(stored, stored)))
if opts.untracked:
head.extend(sorted(imap(lambda n: (n, '(%s)' % n), ifilter(lambda n: n not in ['GC_PARAM', 'GC_JOB_ID'], untracked))))
utils.vprint('')
utils.printTabular(head, result)
def setup_dataset(config, dataset):
if dataset.lower() == 'true':
utils.vprint('Registering dummy data provider data')
dataSplitter = DummySplitter()
else:
dataSplitter = DataSplitter.loadPartitionsForScript(dataset)
config = config.changeView(setSections=None)
partProcessor = config.getCompositePlugin(
'partition processor',
'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor BasicPartitionProcessor',
'MultiPartitionProcessor',
cls='PartitionProcessor',
onChange=None)
DataParameterSource = ParameterSource.getClass('DataParameterSource')
DataParameterSource.datasetsAvailable['data'] = DataParameterSource(
config.getWorkPath(), 'data', None, dataSplitter, partProcessor)
def setup_config(opts, args):
# Set config based on settings from config file or command line
configFile = None
if os.path.exists(args[0]):
configFile = args[0]
config = getConfig(configFile, section='global')
if os.path.exists(config.getWorkPath('datamap.tar')):
opts.dataset = config.getWorkPath('datamap.tar')
config.changeView(setSections=['jobs']).set('nseeds', '1', '?=')
configParameters = config.changeView(setSections=['parameters'])
if opts.parameter:
utils.vprint('Provided options:')
for p in opts.parameter:
k, v = p.split('=', 1)
configParameters.set(k.strip(),
v.strip().replace('\\n', '\n'), '=')
utils.vprint('\t%s: %s' % (k.strip(), v.strip()))
utils.vprint('')
if configFile is None:
configParameters.set('parameters',
str.join(' ', args).replace('\\n', '\n'))
if opts.dataset:
configParameters.set('default lookup', 'DATASETNICK')
if utils.verbosity() > 2:
config.changeView(setSections=None).write(sys.stdout)
return config
def setup_config(opts, args):
# Set config based on settings from config file or command line
configFile = None
if os.path.exists(args[0]):
configFile = args[0]
config = getConfig(configFile, section = 'global')
config.changeView(setSections = ['jobs']).set('nseeds', '1', '?=')
configParameters = config.changeView(setSections = ['parameters'])
if opts.parameter:
utils.vprint('Provided options:')
for p in opts.parameter:
k, v = p.split('=', 1)
configParameters.set(k.strip(), v.strip().replace('\\n', '\n'), '=')
utils.vprint('\t%s: %s' % (k.strip(), v.strip()))
utils.vprint('')
if not os.path.exists(args[0]):
configParameters.set('parameters', str.join(' ', args).replace('\\n', '\n'))
if opts.dataset:
configParameters.set('default lookup', 'DATASETNICK')
if not opts.persistent:
configParameters.set('parameter adapter', 'BasicParameterAdapter', '=')
if utils.verbosity() > 2:
config.changeView(setSections = None).write(sys.stdout)
return config
def get_parameters(opts, psource):
result = []
needGCParam = False
if psource.getMaxJobs() is not None:
countActive = 0
for jobNum in irange(psource.getMaxJobs()):
info = psource.getJobInfo(jobNum)
if info[ParameterInfo.ACTIVE]:
countActive += 1
if opts.disabled or info[ParameterInfo.ACTIVE]:
if not info[ParameterInfo.ACTIVE]:
info['GC_PARAM'] = 'N/A'
if str(info['GC_PARAM']) != str(jobNum):
needGCParam = True
result.append(info)
if opts.parseable:
utils.vprint('Count,%d,%d' % (countActive, psource.getMaxJobs()))
else:
utils.vprint('Number of parameter points: %d' % psource.getMaxJobs())
if countActive != psource.getMaxJobs():
utils.vprint('Number of active parameter points: %d' % countActive)
else:
result.append(psource.getJobInfo(123))
return (result, needGCParam)
def save_parameters(psource, fn):
utils.vprint('')
ParameterSource.getClass('GCDumpParameterSource').write(fn, psource)
utils.vprint('Parameter information saved to ./%s' % fn)
def process_intervention(opts, psource):
utils.vprint('')
tmp = psource.getJobIntervention()
if tmp:
if opts.displaymode == 'parseable':
utils.vprint('R: %s' % str.join(',', imap(str, tmp[0])))
utils.vprint('D: %s' % str.join(',', imap(str, tmp[1])))
else:
utils.vprint(' Redo: %r' % tmp[0])
utils.vprint('Disable: %r' % tmp[1])
else:
if opts.displaymode == 'parseable':
utils.vprint('NOINT')
else:
utils.vprint('No intervention')
def save_parameters(psource, fn):
utils.vprint('')
ParameterSource.getClass('GCDumpParameterSource').write(fn, psource)
utils.vprint('Parameter information saved to ./%s' % fn)
def process_intervention(opts, psource):
utils.vprint('')
tmp = psource.getJobIntervention()
if tmp:
if opts.displaymode == 'parseable':
utils.vprint('R: %s' % str.join(',', imap(str, tmp[0])))
utils.vprint('D: %s' % str.join(',', imap(str, tmp[1])))
else:
utils.vprint(' Redo: %r' % tmp[0])
utils.vprint('Disable: %r' % tmp[1])
else:
if opts.displaymode == 'parseable':
utils.vprint('NOINT')
else:
utils.vprint('No intervention')
#!/usr/bin/env python
#-# Copyright 2010 Karlsruhe Institute of Technology
#-#
#-# Licensed under the Apache License, Version 2.0 (the "License");
#-# you may not use this file except in compliance with the License.
#-# You may obtain a copy of the License at
#-#
#-# http://www.apache.org/licenses/LICENSE-2.0
#-#
#-# Unless required by applicable law or agreed to in writing, software
#-# distributed under the License is distributed on an "AS IS" BASIS,
#-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#-# See the License for the specific language governing permissions and
#-# limitations under the License.
from gcSupport import utils
from grid_control_cms.lumi_tools import formatLumi, parseLumiFromJSON, mergeLumi
import xmlrpclib
server = xmlrpclib.ServerProxy('http://pccmsdqm04.cern.ch/runregistry/xmlrpc')
data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json', {'groupName': 'Collisions10'})
runs = parseLumiFromJSON(data)
utils.vprint("lumi filter = %s" % utils.wrapList(formatLumi(mergeLumi(runs)), 60, ',\n\t'), -1)
def main():
# Set config based on settings from config file or command line
configFile = None
if os.path.exists(args[0]):
configFile = args[0]
config = getConfig(configFile, section = 'global')
config.changeView(setSections = ['jobs']).set('nseeds', '1', '?=')
configParameters = config.changeView(setSections = ['parameters'])
if opts.parameters:
utils.vprint('Provided options:')
for p in opts.parameters:
k, v = p.split('=', 1)
configParameters.set(k.strip(), v.strip().replace('\\n', '\n'), '=')
utils.vprint('\t%s: %s' % (k.strip(), v.strip()))
utils.vprint('')
if not os.path.exists(args[0]):
configParameters.set('parameters', str.join(' ', args).replace('\\n', '\n'))
if opts.dataset:
configParameters.set('default lookup', 'DATASETNICK')
# configParameters.set('parameter adapter', 'BasicParameterAdapter', '=') # Don't track parameter changes
if opts.verbosity > 2:
config.changeView(setSections = None).write(sys.stdout)
# Initialize ParameterFactory
configTask = config.changeView(setSections = [config.get(['task', 'module'], 'DummyTask')])
pm = config.getPlugin('parameter factory', 'SimpleParameterFactory', cls = ParameterFactory).getInstance()
# Create dataset parameter source
class DummySplitter:
def getMaxJobs(self):
return 3
def getSplitInfo(self, pNum):
mkEntry = lambda ds, fl, n, nick: { DataSplitter.Dataset: ds, DataSplitter.Nickname: nick,
DataSplitter.FileList: fl, DataSplitter.NEntries: n }
rndStr = lambda: md5(str(random.random())).hexdigest()[:10]
tmp = [ mkEntry('ds1', ['a', 'b'], 23, 'data_1'), mkEntry('ds1', ['1'], 42, 'data_1'),
mkEntry('ds2', ['m', 'n'], 123, 'data_2'), mkEntry('ds2', ['x', 'y', 'z'], 987, 'data_3') ]
return tmp[pNum]
class DataSplitProcessorTest:
def getKeys(self):
return map(lambda k: ParameterMetadata(k, untracked=True),
['DATASETINFO', 'DATASETID', 'DATASETPATH', 'DATASETBLOCK', 'DATASETNICK'])
def process(self, pNum, splitInfo, result):
result.update({
'DATASETINFO': '',
'DATASETID': splitInfo.get(DataSplitter.DatasetID, None),
'DATASETPATH': splitInfo.get(DataSplitter.Dataset, None),
'DATASETBLOCK': splitInfo.get(DataSplitter.BlockName, None),
'DATASETNICK': splitInfo.get(DataSplitter.Nickname, None),
'DATASETSPLIT': pNum,
})
if opts.dataset.lower() == 'true':
utils.vprint('Registering dummy data provider data')
dataSplitter = DummySplitter()
elif opts.dataset:
dataSplitter = DataSplitter.loadState(opts.dataset)
if opts.dataset:
DataParameterSource.datasetsAvailable['data'] = DataParameterSource(
config.getWorkPath(), 'data', None, dataSplitter, DataSplitProcessorTest())
psource = pm.getSource(config)
if opts.forceiv:
for dp in DataParameterSource.datasetSources:
dp.intervention = (set([1]), set([0]), True)
if opts.listparams:
result = []
needGCParam = False
if psource.getMaxJobs() != None:
countActive = 0
for jobNum in range(psource.getMaxJobs()):
info = psource.getJobInfo(jobNum)
if info[ParameterInfo.ACTIVE]:
countActive += 1
if opts.inactive or info[ParameterInfo.ACTIVE]:
if not info[ParameterInfo.ACTIVE]:
info['GC_PARAM'] = 'N/A'
if str(info['GC_PARAM']) != str(jobNum):
needGCParam = True
result.append(info)
if opts.displaymode == 'parseable':
utils.vprint('Count,%d,%d' % (countActive, psource.getMaxJobs()))
else:
utils.vprint('Number of parameter points: %d' % psource.getMaxJobs())
if countActive != psource.getMaxJobs():
utils.vprint('Number of active parameter points: %d' % countActive)
else:
result.append(psource.getJobInfo(123))
enabledOutput = opts.output.split(',')
output = filter(lambda k: not opts.output or k in enabledOutput, psource.getJobKeys())
stored = filter(lambda k: k.untracked == False, output)
untracked = filter(lambda k: k.untracked == True, output)
if opts.collapse > 0:
result_old = result
result = {}
result_nicks = {}
head = [('COLLATE_JOBS', '# of jobs')]
if 'DATASETSPLIT' in stored:
stored.remove('DATASETSPLIT')
if (opts.collapse == 1):
stored.append('DATASETNICK')
head.append(('DATASETNICK', 'DATASETNICK'))
elif opts.collapse == 2:
head.append(('COLLATE_NICK', '# of nicks'))
for pset in result_old:
if ('DATASETSPLIT' in pset) and (opts.collapse == 1):
pset.pop('DATASETSPLIT')
nickname = None
if ('DATASETNICK' in pset) and (opts.collapse == 2):
nickname = pset.pop('DATASETNICK')
h = md5(repr(map(lambda key: pset.get(key), stored))).hexdigest()
result.setdefault(h, []).append(pset)
result_nicks.setdefault(h, set()).add(nickname)
def doCollate(h):
tmp = result[h][0]
tmp['COLLATE_JOBS'] = len(result[h])
tmp['COLLATE_NICK'] = len(result_nicks[h])
return tmp
result = map(doCollate, result)
else:
head = [('GC_JOB_ID', '#')]
if needGCParam:
head.append(('GC_PARAM', 'GC_PARAM'))
if opts.active:
head.append((ParameterInfo.ACTIVE, 'ACTIVE'))
if opts.visible:
stored = opts.visible.split(',')
head.extend(sorted(zip(stored, stored)))
if opts.untracked:
head.extend(sorted(map(lambda n: (n, '(%s)' % n), filter(lambda n: n not in ['GC_PARAM', 'GC_JOB_ID'], untracked))))
utils.vprint('')
utils.printTabular(head, result)
if opts.save:
utils.vprint('')
ParameterSource.getClass('GCDumpParameterSource').write(opts.save, psource)
utils.vprint('Parameter information saved to ./%s' % opts.save)
if opts.intervention:
utils.vprint('')
tmp = psource.getJobIntervention()
if tmp:
if opts.displaymode == 'parseable':
utils.vprint('R: %s' % str.join(',', map(str, tmp[0])))
utils.vprint('D: %s' % str.join(',', map(str, tmp[1])))
else:
utils.vprint(' Redo: %r' % tmp[0])
utils.vprint('Disable: %r' % tmp[1])
else:
if opts.displaymode == 'parseable':
utils.vprint('NOINT')
else:
utils.vprint('No intervention')
#!/usr/bin/env python
# | Copyright 2010-2016 Karlsruhe Institute of Technology
# |
# | Licensed under the Apache License, Version 2.0 (the "License");
# | you may not use this file except in compliance with the License.
# | You may obtain a copy of the License at
# |
# | http://www.apache.org/licenses/LICENSE-2.0
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
try:
from xmlrpclib import ServerProxy
except ImportError:
from xmlrpc.client import ServerProxy
from gcSupport import utils
from grid_control_cms.lumi_tools import formatLumi, mergeLumi, parseLumiFromJSON
server = ServerProxy('http://pccmsdqm04.cern.ch/runregistry/xmlrpc')
data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json',
{'groupName': 'Collisions10'})
runs = parseLumiFromJSON(data)
utils.vprint(
'lumi filter = %s' %
utils.wrapList(formatLumi(mergeLumi(runs)), 60, ',\n\t'), -1)