def _result(self,globals,locals):
assert('tools' in globals)
assert('doc' in globals)
( keys, values, otherwise_idx ) = \
self._gather_keys_and_values(globals,locals)
if self._require_an_otherwise_clause() and \
otherwise_idx is Conditional.MISSING:
raise ConditionalMissingOtherwise(
f'{self._path}: no "otherwise" clause provided')
idx=self._index(keys)
if idx is None:
if otherwise_idx is Conditional.MISSING:
raise ConditionalMissingOtherwise(
f'{self._path}: no clauses match and no '
f'"otherwise" value was given. {keys} {values}')
self.__result=self[otherwise_idx]._raw('otherwise')
if superdebug: _logger.debug(f'{self._path}: result=otherwise: {self.__result!r}')
idx=otherwise_idx
else:
self.__result=values[idx]
if superdebug: _logger.debug(f'{self._path}: result index {idx}: {self.__result!r}')
if 'message' in self[idx]:
message=from_config('message',self[idx].message,globals,locals,self._path)
_logger.info(f'{self._path}[{idx}]: {message}')
assert(self.__result is not Conditional.MISSING)
return self.__result
def _gather_keys_and_values(self,globals,locals):
keys=list()
values=list()
otherwise_idx=Conditional.MISSING
for i in range(len(self)):
vk=self[i]
has_otherwise = vk._has_raw('otherwise')
has_when = vk._has_raw('when')
has_do = vk._has_raw('do')
has_take = vk._has_raw('take')
if has_do and has_take:
raise ConditionalOverspecified(
f'{self._path}[{i}]: cannot have "do" and "take" in one entry')
if has_otherwise and ( has_when or has_do or has_take ):
raise ConditionalOverspecified(
f'{self._path}[{i}]: cannot have "otherwise" in the same entry '
'as "when," "take," or "do"')
elif has_otherwise and i!=len(self)-1:
raise ConditionalInvalidOtherwise(
f'{self._path}[{i}]: "otherwise" must be the last item')
elif has_otherwise:
otherwise_idx=i
elif has_when and ( has_do or has_take ):
values.append(vk._raw('do') if has_do else vk._raw('take'))
vk_locals=multidict(vk,locals)
raw_when=vk._raw('when')
keys.append(from_config('when',raw_when,globals,vk_locals,
f'{self._path}[{i}]'))
else:
raise ConditionalMissingDoWhen(
f'{self._path}[{i}]: entries must have both "take" and "when"'
'or "otherwise" (or "message"). You can use "do" in place of '
'"take" for backward compatibility. Saw keys: '+
', '.join(list(vk.keys())))
return keys, values, otherwise_idx
def _result(self,globals,locals):
rank_specs=list()
for i in range(len(self)):
spec=from_config('JobResourceSpecMaker',self._raw(i),globals,locals,f'{self._path}[{i}]')
if superdebug: _logger.debug(f'Look at spec #{i} in {self._path}...')
if not hasattr(spec,'_raw_child'):
rank_specs.append(spec)
continue
# Get the value, from that new dict_eval, of all keys in spec.
# Store it in the rank_specs list for the later constructor.
with_parent_scope=multidict(spec,self._get_locals(),locals)
ranks=dict()
for key in spec.keys():
ranks[key]=from_config(key,with_parent_scope._raw(key),globals,locals,f'{self._path}[{i}].{key}')
rank_specs.append(ranks)
return crow.sysenv.JobResourceSpec(rank_specs)
def _result(self,globals,locals):
result={}
for i in range(len(self)):
d=from_config('MergeMapping',self._raw(i),globals,locals,f'{self._path}[{i}]')
if not isinstance(d,collections.Mapping): continue
if not d: continue
if hasattr(d,'_raw_child'):
result.update(d._raw_child())
else:
result.update(d)
result=dict_eval(result,self._path,self._get_globals())
return result
def _result(self,globals,locals):
if 'select' not in self or 'otherwise' not in self or 'cases' not in self:
raise KeyError(f'{self._path}: !Select must contain select, otherwise, and cases.')
if not isinstance(self.cases,collections.Mapping):
raise TypeError(f'{self._path}.cases: !Select cases must be a map')
value=from_config('select',self._raw('select'),
globals,locals,self._path)
if value in self.cases:
if hasattr(self.cases,'_raw'):
return self.cases._raw(value)
return self.cases[value]
return self._raw('otherwise')
def _result(self,globals,locals):
result=[]
for i in range(len(self)):
d=from_config('AppendSequence',self._raw(i),globals,locals,f'{self._path}[{i}]')
if not isinstance(d,collections.Sequence) or isinstance(d,str):
raise TypeError(f'{self._path}: can only append lists.')
if not len(d):
continue
if hasattr(d,'_raw_child'):
result.extend(d._raw_child())
else:
result.extend(d)
result=list_eval(result,self._path,self._get_globals())
return result
def _check_scope(self, scope, stage, memo):
if self.__my_id in memo:
_logger.debug(
f'{scope._path}: do not re-validate with {self._path}')
return
memo.add(self.__my_id)
_logger.debug(f'{scope._path}: validate with {self._path}')
checked = set()
errors = list()
template = copy(self)
did_something = True
# Main validation loop. Iteratively validate, adding new
# Templates as they become available via is_present.
for var in template:
_logger.debug(f'{scope._path}.{var}: validate...')
try:
scheme = template[var]
if not isinstance(scheme, Mapping): continue # not a template
if stage and 'stages' in scheme:
if stage not in scheme.stages:
continue # skip validation; wrong stage
elif 'stages' in scheme:
continue # skip validation of stage-specific schemes
if 'precheck' in scheme:
scope[var] = scheme.precheck
if var in scope:
validate_var(scope._path, scheme, var, scope[var])
elif 'default' in scheme:
scope[var] = from_config(var, scheme._raw('default'),
self._globals(), scope,
f'{scope._path}.{var}')
_logger.debug(
f'{scope._path}.{var}: insert default {scope._raw(var)}'
)
if var not in scope and 'if_present' in scheme:
_logger.debug(
f'{scope._path}.{var}: not present; skip if_present')
if var in scope and 'if_present' in scheme:
_logger.debug(f'{scope._path}.{var}: evaluate if_present '
f'{scheme._raw("if_present")._path}')
ip = from_config(var, scheme._raw('if_present'),
self._globals(), scope,
f'{scope._path}.{var}')
_logger.debug(f'{scope._path}.{var}: result = {ip!r}')
if not ip: continue
if not isinstance(ip, Template):
if not isinstance(ip, Mapping): continue
ip = Template(ip._raw_child(), ip._path,
ip._get_globals())
_logger.debug(
f'{scope._path}.{var}: present ({scope._raw(var)!r}); '
f'add {ip._path} to validation')
ip._check_scope(scope, stage, memo)
if 'override' in scheme:
override = from_config(
'override', template[var]._raw('override'),
scope._globals(), scope,
f'{scope._path}.Template.{var}.override')
if override is not None: scope[var] = override
except (IndexError, AttributeError, TypeError, ValueError) as pye:
errors.append(
f'{scope._path}.{var}: {type(pye).__name__}: {pye}')
_logger.debug(f'{scope._path}.{var}: {pye}', exc_info=True)
except ConfigError as ce:
errors.append(str(ce))
_logger.debug(
f'{scope._path}.{var}: {type(ce).__name__}: {ce}',
exc_info=True)
# Insert default values for all templates found thus far and
# detect any missing, non-optional, variables
missing = list()
for var in template:
if var not in scope:
tmpl = template[var]
if not hasattr(tmpl, '__getitem__') or not hasattr(
tmpl, 'update'):
raise TypeError(
f'{self._path}.{var}: All entries in a !Template must be maps not {type(tmpl).__name__}'
)
if 'default' not in tmpl and not tmpl.get('optional', False):
missing.append(var)
# Second pass checking for required variables that have no
# values. This second pass deals with variables that were
# updated by an "override" clause.
reported_missing = set(missing)
in_scope = set([k for k in scope.keys()])
still_missing = reported_missing - in_scope
if still_missing:
raise VariableMissing(f'{scope._path}: missing: ' +
', '.join(still_missing) + ' in: ' +
', '.join([k for k in scope.keys()]))
# Check for variables that evaluate to an error
for key, expr in scope._raw_child().items():
if hasattr(expr, '_is_error'):
try:
scope[key]
except ConfigUserError as ce:
errors.append(f'{scope._path}.{key}: {ce}')
if errors: raise TemplateErrors(errors)
def __init__(self,
suite,
viewed,
path,
parent,
task_array_dimensions=None,
task_array_dimval=None,
task_array_dimidx=None):
# assert(isinstance(suite,Suite))
# assert(isinstance(viewed,dict_eval))
assert (hasattr(self, '_iter_raw'))
assert (isinstance(parent, SuiteView))
assert (not isinstance(viewed, SuiteView))
if task_array_dimensions:
self.task_array_dimensions = OrderedDict(task_array_dimensions)
else:
self.task_array_dimensions = OrderedDict()
if task_array_dimidx:
self.task_array_dimidx = OrderedDict(task_array_dimidx)
else:
self.task_array_dimidx = OrderedDict()
if task_array_dimval:
self.task_array_dimval = OrderedDict(task_array_dimval)
else:
self.task_array_dimval = OrderedDict()
self.suite = suite
self.viewed = viewed
self.viewed.task_path_list = path[1:]
self.viewed.task_path_str = '/' + '/'.join(path[1:])
self.viewed.task_path_var = '.'.join(path[1:])
if self.viewed.task_path_var:
self.viewed._path = self.viewed.task_path_var
if type(self.viewed) in SUITE_CLASS_MAP:
self.viewed.up = parent
self.viewed.this = self.viewed
elif not isinstance(self.viewed, Cycle):
assert (False)
self.path = SuitePath(path)
self.parent = parent
self.__cache = {}
assert (isinstance(self.viewed, Cycle) or 'this' in self.viewed)
if isinstance(self.viewed, Slot):
locals = multidict(self.parent, self.viewed)
globals = self.viewed._get_globals()
for k, v in self.viewed._raw_child().items():
if hasattr(v, '_as_dependency'): continue
self.viewed[k] = from_config(k, v, globals, locals,
self.viewed._path)
if isinstance(self.viewed, Task):
assert (isinstance(self.viewed, Cycle) or 'this' in self.viewed)
for k, v in self.viewed.items():
copied = False
if hasattr(v, "_validate"):
copied = True
v = copy(v)
v._validate('suite')
if self.__can_wrap(v):
if not copied:
v = copy(v)
self.viewed[k] = v
assert (isinstance(viewed, Cycle)
or self.viewed.task_path_var != parent.task_path_var)