def wrapper(*args):
try:
parameterClass = ParameterSource.getClass(clsName)
except Exception:
raise ParameterError('Unable to create parameter source "%r"!' % clsName)
try:
return parameterClass.create(self._paramConfig, *args)
except Exception:
raise ParameterError('Error while creating "%r" with arguments "%r"' % (parameterClass.__name__, args))
def _init_psp_field(self):
result = []
def _add_psp_entry(pnum):
tmp = {
ParameterInfo.ACTIVE: True,
ParameterInfo.REQS: [],
'GC_JOB_ID': pnum,
'GC_PARAM': pnum
}
self._psrc.fill_parameter_content(pnum, tmp)
output_tuple = self._helper.lookup(tmp)
if output_tuple:
for (lookup_idx, tmp) in enumerate(output_tuple):
result.append((pnum, lookup_idx))
if self._psrc.get_parameter_len() is None:
error_msg = 'Unable to use %r with an infinite parameter space!'
raise ParameterError(error_msg % self.__class__.__name__)
else:
for pnum in irange(self._psrc.get_parameter_len()):
_add_psp_entry(pnum)
if len(result) == 0:
self._log.critical(
'Lookup parameter "%s" has no matching entries!',
self._output_vn)
return result
def __init__(self, fn, format='sniffed'):
(self._fn, self._format) = (fn, format)
fp = open(fn)
try:
first_line = fp.readline()
sniffed = csv.Sniffer().sniff(first_line)
csv.register_dialect('sniffed', sniffed)
csv_header = first_line.strip().split(sniffed.delimiter) + [None]
psp_list = list(csv.DictReader(fp, csv_header, dialect=format))
finally:
fp.close()
for psp in psp_list:
psp.pop(None, None)
if None in psp.values():
raise ParameterError('Malformed entry in csv file %r: {%s}' %
(fn, str_dict_linear(psp)))
def _cleanup_dict(
mapping
): # strip all key value entries and filter empty parameters
tmp = tuple(
imap(lambda item: lmap(str.strip, item), mapping.items()))
return dict(lfilter(lambda k_v: k_v[0] != '', tmp))
output_vn_list = sorted(imap(ParameterMetadata, lidfilter(csv_header)),
key=lambda k: k.value)
InternalParameterSource.__init__(self, lmap(_cleanup_dict, psp_list),
output_vn_list)
def initPSpace(self):
result = []
def addEntry(pNum):
tmp = {
ParameterInfo.ACTIVE: True,
ParameterInfo.REQS: [],
'GC_JOB_ID': pNum,
'GC_PARAM': pNum
}
self._psource.fillParameterInfo(pNum, tmp)
lookupResult = self._matcher.lookup(tmp)
if lookupResult:
for (lookupIdx, tmp) in enumerate(lookupResult):
result.append((pNum, lookupIdx))
if self._psource.getMaxParameters() is None:
raise ParameterError(
'Unable to use %r with an infinite parameter space!' %
self.__class__.__name__)
else:
for pNum in irange(self._psource.getMaxParameters()):
addEntry(pNum)
if len(result) == 0:
self._log.critical(
'Lookup parameter "%s" has no matching entries!', self._key)
return result
def wrapper(*args):
parameterClass = ParameterSource.getClass(clsName)
try:
return parameterClass.create(self._paramConfig,
self._repository, *args)
except Exception:
raise ParameterError(
'Error while creating %r with arguments %r' %
(parameterClass.__name__, args))
def getSource(self):
if not self._pExpr:
return NullParameterSource()
self._log.debug('Parsing parameter expression: %s', repr(self._pExpr))
try:
source = self._getUserSource(self._pExpr)
except:
raise ParameterError('Unable to parse parameter expression %r' % self._pExpr)
self._log.debug('Parsed parameter source: %s', repr(source))
return source
def _initMaxParameters(self):
if None in self._psourceMaxList:
prob_sources = lfilter(lambda p: p.getMaxParameters() is None,
self._psourceList)
raise ParameterError(
'Unable to chain unlimited sources: %s' %
repr(str.join(', ', imap(repr, prob_sources))))
self._offsetList = lmap(lambda pIdx: sum(self._psourceMaxList[:pIdx]),
irange(len(self._psourceList)))
return sum(self._psourceMaxList)
def __init__(self, name, factory, repository):
(self._name, self._factory_name) = (name, factory.__class__.__name__)
try:
if 'subspace:%s' % name in repository:
raise ParameterError('Recusive parameter space detected! %s' %
repr(repository))
repository['subspace:%s' % name] = self
ForwardingParameterSource.__init__(self,
factory.get_psrc(repository))
finally:
repository.pop('subspace:%s' % name, None)
def get_psrc(self, respository):
if not self._pexpr:
return _create_psrc('NullParameterSource')
self._log.debug('Parsing parameter expression: %s', repr(self._pexpr))
try:
source = self._get_psrc_user(self._pexpr, respository)
except:
raise ParameterError('Unable to parse parameter expression %r' %
self._pexpr)
source = _create_psrc('InternalResolveParameterSource', source)
self._log.debug('Parsed parameter source: %s', repr(source))
return source
def fill_parameter_metadata(self, result):
map_vn2psrc = {}
for psrc in self._psrc_list:
metadata_list = []
psrc.fill_parameter_metadata(metadata_list)
for metadata in metadata_list:
vn = metadata.value
if vn in map_vn2psrc:
raise ParameterError(
'Collisions of parameter %s between %s and %s' %
(metadata.value, psrc, map_vn2psrc[vn]))
map_vn2psrc[vn] = psrc
psrc.fill_parameter_metadata(result)
def create(cls,
pconfig=None,
name='subspace',
factory='SimpleParameterFactory'): # pylint:disable=arguments-differ
try:
ParameterFactory = Plugin.getClass('ParameterFactory')
config = pconfig.getConfig(viewClass='SimpleConfigView',
addSections=[name])
return SubSpaceParameterSource(
name, ParameterFactory.createInstance(factory, config))
except:
raise ParameterError(
'Unable to create subspace %r using factory %r' %
(name, factory))
def resync(self, force = False): # Do not overwrite resync results - eg. from external or init trigger
source_hash = self._source.getHash()
if (self._resync_state == ParameterSource.EmptyResyncResult()) and ((source_hash != self._source_hash) or force):
activity = Activity('Syncronizing parameter information')
t_start = time.time()
try:
self._resync_state = self._resync()
except Exception:
raise ParameterError('Unable to resync parameters!')
self._source_hash = self._source.getHash()
activity.finish()
self._log.log(logging.INFO, 'Finished resync of parameter source (%s)', strTimeShort(time.time() - t_start))
result = self._resync_state
self._resync_state = ParameterSource.EmptyResyncResult()
return result
def create_psrc(cls, pconfig, repository, *args): # pylint:disable=arguments-differ
name = 'subspace'
factory = 'SimpleParameterFactory'
if len(args) == 1:
name = args[0]
elif len(args) == 2:
(factory, name) = (args[0], args[1])
try:
config = pconfig.get_config(view_class='SimpleConfigView',
add_sections=[name])
pfactory = Plugin.get_class('ParameterFactory').create_instance(
factory, config)
return SubSpaceParameterSource(name, pfactory, repository)
except:
raise ParameterError(
'Unable to create subspace %r using factory %r' %
(name, factory))
def fill_parameter_metadata(self, result):
map_vn2tracking_status = {}
map_vn2psrc_list = {}
for psrc in self._psrc_list:
metadata_list = []
psrc.fill_parameter_metadata(metadata_list)
for metadata in metadata_list:
vn = metadata.value
tracking_status = map_vn2tracking_status.setdefault(
vn, metadata.untracked)
if tracking_status != metadata.untracked:
raise ParameterError(
'Collisions of tracking status for parameter %s between %s and %s'
% (metadata.value, psrc,
str.join('; ', imap(repr, map_vn2psrc_list[vn]))))
if vn not in map_vn2psrc_list:
result.append(metadata)
map_vn2psrc_list.setdefault(vn, []).append(psrc)