def expand_parent_params(self, parent, param_values, origin):
"""Replace parameters with specific values in inherited parent names.
If a value is NOT specified, e.g.:
inherit = parent<m>
then it must be given in param_values (as defined by expansion of the
enclosing namespace name).
If a value IS specified, e.g.:
inherit = parent<m=3>
then it must be a legal value for that parameter.
"""
head, p_list_str, tail = REC_P_ALL.match(parent).groups()
if not p_list_str:
return head
used = {}
for item in (i.strip() for i in p_list_str.split(',')):
if '-' in item or '+' in item:
raise ParamExpandError(
"parameter offsets illegal here: '%s'" % origin)
elif '=' in item:
# Specific value given.
pname, pval = [val.strip() for val in item.split('=', 1)]
try:
pval = int(pval)
except ValueError:
pass
if pname not in self.param_cfg:
raise ParamExpandError(
"parameter '%s' undefined in '%s'" % (
pname, origin))
elif pval not in self.param_cfg[pname]:
raise ParamExpandError(
"illegal value '%s=%s' in '%s'" % (
pname, pval, origin))
used[pname] = pval
else:
# Non-specific; value must be supplied in param_values.
try:
used[item] = param_values[item]
except KeyError:
raise ParamExpandError(
"parameter '%s' undefined in '%s'" % (
item, origin))
if head:
tmpl = head
else:
tmpl = ''
for pname in used:
tmpl += self.param_tmpl_cfg[pname]
if tail:
tmpl += tail
return tmpl % used
def expand(self, line):
"""Expand a graph line for subset of suite parameters.
Input line is a string that may contain multiple parameterized node
names, e.g. "pre=>init<m>=>sim<m,n>=>post<m,n>=>done".
Unlike NameExpander this supports offsets like "foo<m-1,n>", which
means (because the parameter substitutions have to be computed on the
fly) we have shift creation of the expansion string template into the
inner loop of the recursive expansion function.
Returns a set containing lines expanded for all used parameters, e.g.
for "foo=>bar<m,n>" with m=2 and n=2 the result would be:
set([foo=>bar_m0_n0,
foo=>bar_m0_n1,
foo=>bar_m1_n0,
foo=>bar_m1_n1])
Specific parameter values can be singled out like this:
"sim<m=0,n>=>sim<m,n>"
Offset (negative only) values can be specified like this:
"sim<m-1,n>=>sim<m,n>"
(Here the offset node must be the first in a line, and if m-1 evaluates
to less than 0 the node will be removed to leave just "sim<m,n>").
"""
line_set = set()
used_pnames = []
for p_group in set(REC_P_GROUP.findall(line)):
for item in p_group.split(','):
pname, offs = REC_P_OFFS.match(item).groups()
if not self.param_cfg.get(pname, None):
raise ParamExpandError(
"parameter %s is not defined in <%s>: %s" % (
pname, p_group, line))
if offs and offs.startswith('='):
# Check that specific parameter values exist.
val = offs[1:]
try:
nval = int(val)
except ValueError:
nval = val
if not item_in_iterable(nval, self.param_cfg[pname]):
raise ParamExpandError(
"parameter %s out of range: %s" % (
pname, p_group))
if pname not in used_pnames:
used_pnames.append(pname)
used_params = [(p, self.param_cfg[p]) for p in used_pnames]
self._expand_graph(line, dict(used_params), used_params, line_set)
return line_set
def _expand_name(self, results, tmpl, params, spec_vals=None):
"""Recursively expand tmpl for any number of parameters.
tmpl is a string template, e.g. 'foo_m%(m)s_n%(n)s' for two
parameters m and n.
params is a list of tuples (name, max-val) for each parameter
to be looped over.
spec_vals is a map of values for parameters that are not to be looped
over because they've been assigned a specific value.
E.g. for "foo<m=0,n>" tmpl is "foo_m%(m)s_n%(n)s", params is
[('n', 2)], and spec_values {'m': 0}.
results contains the expanded names and corresponding parameter values,
as described above in the calling method.
"""
if spec_vals is None:
spec_vals = {}
if not params:
# Inner loop.
current_values = dict(spec_vals)
try:
results.append((tmpl % current_values, current_values))
except KeyError as exc:
raise ParamExpandError('parameter %s is not '
'defined.' % str(exc.args[0]))
else:
for param_val in params[0][1]:
spec_vals[params[0][0]] = param_val
self._expand_name(results, tmpl, params[1:], spec_vals)
def _expand_graph(self,
line,
all_params,
param_list,
line_set,
values=None):
"""Expand line into line_set for any number of parameters.
line is a graph string line as described above in the calling method.
param_list is a list of tuples (name, max-val) for each parameter.
results is a set to hold each expanded line.
"""
if values is None:
values = {}
if not param_list:
# Inner loop.
for p_group in set(REC_P_GROUP.findall(line)):
# Parameters must be expanded in the order found.
param_values = OrderedDictWithDefaults()
tmpl = ''
for item in p_group.split(','):
pname, offs = REC_P_OFFS.match(item).groups()
if offs is None:
param_values[pname] = values[pname]
elif offs.startswith('='):
# Specific value.
try:
# Template may require an integer
param_values[pname] = int(offs[1:])
except ValueError:
param_values[pname] = offs[1:]
else:
# Index offset.
plist = all_params[pname]
cur_idx = plist.index(values[pname])
off_idx = cur_idx + int(offs)
if 0 <= off_idx < len(plist):
offval = plist[off_idx]
else:
offval = self._REMOVE
param_values[pname] = offval
for pname in param_values:
tmpl += self.param_tmpl_cfg[pname]
try:
repl = tmpl % param_values
except KeyError as exc:
raise ParamExpandError('parameter %s is not '
'defined.' % str(exc.args[0]))
line = line.replace('<' + p_group + '>', repl)
# Remove out-of-range nodes
line = self._REMOVE_REC.sub('', line)
if line:
line_set.add(line)
else:
# Recurse through index ranges.
for param_val in param_list[0][1]:
values[param_list[0][0]] = param_val
self._expand_graph(line, all_params, param_list[1:], line_set,
values)
def expand(self, runtime_heading):
"""Expand runtime namespace names for a subset of workflow parameters.
Input runtime_heading is a string that may contain comma-separated
parameterized namespace names, e.g. for "foo<m,n>, bar<m,n>".
Unlike GraphExpander this does not support offsets like "foo<m-1,n>",
but it does support specific parameter values like "foo<m=0,n>".
Returns a list of tuples, each with an expanded name and its parameter
values (to be passed to the corresponding tasks), e.g.:
[('foo_i0_j0', {i:'0', j:'0'}),
('foo_i0_j1', {i:'0', j:'1'}),
('foo_i1_j0', {i:'1', j:'0'}),
('foo_i1_j1', {i:'1', j:'1'})]
"""
# Create a string template and values to pass to the expansion method.
results = []
for name in REC_NAMES.findall(runtime_heading):
tmpl = ''
spec_vals = {}
used_params = []
while name:
head, p_list_str, tail = REC_P_ALL.match(name.strip()).groups()
if not p_list_str:
break
if head:
tmpl += head
# Get the subset of parameters used in this case.
for item in (i.strip() for i in p_list_str.split(',')):
pname, sval = REC_P_OFFS.match(item.strip()).groups()
if not self.param_cfg.get(pname, None):
raise ParamExpandError(
"parameter %s is not defined in %s" %
(pname, runtime_heading))
if sval:
if sval.startswith('+') or sval.startswith('-'):
raise ParamExpandError(
"parameter index offsets are not"
" supported in name expansion: %s%s" %
(pname, sval))
elif sval.startswith('='):
# Check that specific parameter values exist.
val = sval[1:].strip()
# Pad integer values here.
try:
nval = int(val)
except ValueError:
nval = val
if not item_in_iterable(nval,
self.param_cfg[pname]):
raise ParamExpandError(
"parameter %s out of range: %s" %
(pname, p_list_str))
spec_vals[pname] = nval
else:
used_params.append((pname, self.param_cfg[pname]))
tmpl += self.param_tmpl_cfg[pname]
if tail:
name = tail
else:
name = ''
if tmpl:
tmpl += name
self._expand_name(results, tmpl, used_params, spec_vals)
else:
results.append((name.strip(), {}))
return results
