def __init__(self, config, datasetExpr, datasetNick=None):
ds_config = config.changeView(viewClass='TaggedConfigView',
addNames=[md5_hex(datasetExpr)])
if os.path.isdir(datasetExpr):
scan_pipeline = ['OutputDirsFromWork']
ds_config.set('source directory', datasetExpr)
datasetExpr = os.path.join(datasetExpr, 'work.conf')
else:
scan_pipeline = ['OutputDirsFromConfig', 'MetadataFromTask']
datasetExpr, selector = utils.optSplit(datasetExpr, '%')
ds_config.set('source config', datasetExpr)
ds_config.set('source job selector', selector)
ext_config = create_config(datasetExpr)
ext_task_name = ext_config.changeView(setSections=['global']).get(
['module', 'task'])
if 'ParaMod' in ext_task_name: # handle old config files
ext_task_name = ext_config.changeView(
setSections=['ParaMod']).get('module')
ext_task_cls = Plugin.getClass(ext_task_name)
for ext_task_cls in Plugin.getClass(ext_task_name).iterClassBases():
try:
scan_holder = GCProviderSetup.getClass('GCProviderSetup_' +
ext_task_cls.__name__)
except PluginError:
continue
scan_pipeline += scan_holder.scan_pipeline
break
ScanProviderBase.__init__(self, ds_config, datasetExpr, datasetNick,
scan_pipeline)
def _getPluginFactories(self,
option,
default=noDefault,
cls=Plugin,
tags=None,
inherit=False,
requirePlugin=True,
singlePlugin=False,
desc='plugin factories',
**kwargs):
if isinstance(cls, str):
cls = Plugin.getClass(cls)
def str2obj(value):
objList = list(
cls.bind(value, config=self, inherit=inherit, tags=tags or []))
if singlePlugin and len(objList) > 1:
raise ConfigError(
'This option only allows to specify a single plugin!')
if requirePlugin and not objList:
raise ConfigError(
'This option requires to specify a valid plugin!')
return objList
obj2str = lambda value: str.join(
'\n', imap(lambda obj: obj.bindValue(), value))
return self._getInternal(desc, obj2str, str2obj, str2obj, option,
default, **kwargs)
def _useAvailableDataSource(self, source):
DataParameterSource = Plugin.getClass('DataParameterSource')
if DataParameterSource.datasetsAvailable and not DataParameterSource.datasetsUsed:
if source is not None:
return ParameterSource.createInstance('CrossParameterSource', DataParameterSource.create(), source)
return DataParameterSource.create()
return source
def create(cls, pconfig = None, name = 'subspace', factory = 'SimpleParameterFactory'): # pylint:disable=arguments-differ
try:
ParameterFactory = Plugin.getClass('ParameterFactory')
config = pconfig.getConfig(viewClass = 'SimpleConfigView', addSections = [name])
return SubSpaceParameterSource(name, ParameterFactory.createInstance(factory, config))
except:
raise ParameterError('Unable to create subspace %r using factory %r' % (name, factory))
class OptsConfigFiller(Plugin.getClass('ConfigFiller')):
def __init__(self, cmd_line_args):
self._cmd_line_args = cmd_line_args
def fill(self, container):
combinedEntry = container.getEntry('cmdargs', lambda entry: entry.section == 'global')
newCmdLine = self._cmd_line_args
if combinedEntry:
newCmdLine = combinedEntry.value.split() + self._cmd_line_args
(opts, _) = gc_cmd_line_parser(newCmdLine)
def setConfigFromOpt(section, option, value):
if value is not None:
self._addEntry(container, section, option, str(value), '<cmdline>')
cmd_line_config_map = {
'state!': { '#init': opts.init, '#resync': opts.resync,
'#display config': opts.help_conf, '#display minimal config': opts.help_confmin },
'action': { 'delete': opts.delete, 'reset': opts.reset },
'global': { 'gui': opts.gui, 'submission': opts.submission },
'jobs': { 'max retry': opts.max_retry, 'selected': opts.job_selector },
'logging': { 'debug mode': opts.debug },
}
for section in cmd_line_config_map:
for (option, value) in cmd_line_config_map[section].items():
setConfigFromOpt(section, option, value)
for entry in opts.logging:
tmp = entry.replace(':', '=').split('=')
if len(tmp) == 1:
tmp.append('DEBUG')
setConfigFromOpt('logging', tmp[0] + ' level', tmp[1])
if opts.action is not None:
setConfigFromOpt('workflow', 'action', opts.action.replace(',', ' '))
if opts.continuous:
setConfigFromOpt('workflow', 'duration', -1)
Plugin.createInstance('StringConfigFiller', opts.override).fill(container)
def _getPluginFactories(self, option, default = noDefault,
cls = Plugin, tags = None, inherit = False, requirePlugin = True, singlePlugin = False,
desc = 'plugin factories', **kwargs):
if isinstance(cls, str):
cls = Plugin.getClass(cls)
def str2obj(value):
objList = list(cls.bind(value, config = self, inherit = inherit, tags = tags or []))
if singlePlugin and len(objList) > 1:
raise ConfigError('This option only allows to specify a single plugin!')
if requirePlugin and not objList:
raise ConfigError('This option requires to specify a valid plugin!')
return objList
obj2str = lambda value: str.join('\n', imap(lambda obj: obj.bindValue(), value))
return self._getInternal(desc, obj2str, str2obj, str2obj, option, default, **kwargs)
def create(cls,
pconfig=None,
name='subspace',
factory='SimpleParameterFactory'): # pylint:disable=arguments-differ
try:
ParameterFactory = Plugin.getClass('ParameterFactory')
config = pconfig.getConfig(viewClass='SimpleConfigView',
addSections=[name])
return SubSpaceParameterSource(
name, ParameterFactory.createInstance(factory, config))
except:
raise ParameterError(
'Unable to create subspace %r using factory %r' %
(name, factory))
def __init__(self, config, datasetExpr, datasetNick = None):
ds_config = config.changeView(viewClass = 'TaggedConfigView', addNames = [md5_hex(datasetExpr)])
if os.path.isdir(datasetExpr):
scan_pipeline = ['OutputDirsFromWork']
ds_config.set('source directory', datasetExpr)
datasetExpr = os.path.join(datasetExpr, 'work.conf')
else:
scan_pipeline = ['OutputDirsFromConfig', 'MetadataFromTask']
datasetExpr, selector = utils.optSplit(datasetExpr, '%')
ds_config.set('source config', datasetExpr)
ds_config.set('source job selector', selector)
ext_config = create_config(datasetExpr)
ext_task_name = ext_config.changeView(setSections = ['global']).get(['module', 'task'])
if 'ParaMod' in ext_task_name: # handle old config files
ext_task_name = ext_config.changeView(setSections = ['ParaMod']).get('module')
ext_task_cls = Plugin.getClass(ext_task_name)
for ext_task_cls in Plugin.getClass(ext_task_name).iterClassBases():
try:
scan_holder = GCProviderSetup.getClass('GCProviderSetup_' + ext_task_cls.__name__)
except PluginError:
continue
scan_pipeline += scan_holder.scan_pipeline
break
ScanProviderBase.__init__(self, ds_config, datasetExpr, datasetNick, scan_pipeline)
def getPluginList(pluginName):
aliasDict = {}
for entry in Plugin.getClass(pluginName).getClassList():
depth = entry.pop('depth', 0)
(alias, name) = entry.popitem()
aliasDict.setdefault(name, []).append((depth, alias))
aliasDict.pop(pluginName)
tableList = []
for name in aliasDict:
# sorted by length of name and depth
by_len_depth = sorted(aliasDict[name],
key=lambda d_a: (-len(d_a[1]), d_a[0]))
# sorted by depth and name
by_depth_name = sorted(aliasDict[name],
key=lambda d_a: (d_a[0], d_a[1]))
new_name = by_len_depth.pop()[1]
aliasList = lmap(lambda d_a: d_a[1], by_depth_name)
aliasList.remove(new_name)
entry = {'Name': new_name, 'Alias': str.join(', ', aliasList)}
if ('Multi' not in name) and ('Base' not in name):
tableList.append(entry)
return tableList
def getSource(self, config):
DataParameterSource = Plugin.getClass('DataParameterSource')
source = self._getRawSource(ParameterSource.createInstance('RNGParameterSource'))
if DataParameterSource.datasetsAvailable and not DataParameterSource.datasetsUsed:
source = ParameterSource.createInstance('CrossParameterSource', DataParameterSource.create(), source)
return ParameterAdapter.createInstance(self.adapter, config, source)
import re
try:
from hpfwk import Plugin
NickNameProducer = Plugin.getClass('NickNameProducer')
except:
from grid_control import datasets # FIXME: this line should be unnecessary
from grid_control.datasets import NickNameProducer
def lookup(data, keyMap, default = None):
for key in keyMap:
if key in data:
return keyMap[key]
return (default, '')[default == None]
def lookupBegin(data, keyMap, default = None):
print data
for key in keyMap:
if data.startswith(key):
return keyMap[key]
return (default, '')[default == None]
def addPart(data):
return ('_%s' % data, '')[data == '']
def parseCuts(data):
tmp = re.findall('([0-9]+)to([0-9]+)', data.lower())
if tmp:
return tmp[0]
return filter(lambda x: int(x) > 0, re.findall('pt[-_]*([0-9]+)', data.lower()))
class MyNick(Plugin.getClass('NickNameProducer')):
def getName(self, oldnick, dataset, block):
if oldnick:
return oldnick + '_changed'
return 'newnick'