def test_expand_identities(self):
root = {0:'x'}
l1 = [{1:'a'}, {1:'b'}, {1:'c'}]
l2 = [{2:'a'}, {2:'b'}, {2:'c'}]
l1cp = copy.deepcopy(l1)
d = {'root': root, 'l1':l1, 'l2':l2}
try:
next(namespaces.expand_fuzzyspec_partial(d, ('root', 'l1', 'l2')))
except StopIteration as e:
specs = e.value
else:
raise RuntimeError()
self.assertEqual(len(specs), 3*3)
for i, spec in enumerate(specs):
res = namespaces.resolve(d, spec)
namespaces.push_nslevel(res, 'container', {'i':i})
for i, spec in enumerate(specs):
res = namespaces.resolve(d, (*spec, 'container'))
self.assertEqual(res['i'], i)
self.assertEqual(res[1], l1cp[i//3][1])
self.assertEqual(res[2], l2[i%3][2])
self.assertEqual(res[0], 'x')
def test_parse_complex_dicts(self):
inp = {
'A': OrderedDict([
('sum', None),
('three', 33),
('y', 10),
]),
'B': OrderedDict([
('y', 5),
('three', 333),
('sum', None),
]),
'four': 4,
}
C = BaseConfig(inp)
ns = ChainMap()
C.process_fuzzyspec(('A',), ns=ns)
C.process_fuzzyspec(('B',), ns=ns)
self.assertNotIn('three', ns)
self.assertEqual(namespaces.resolve(ns, ('A',))['sum'], 47)
self.assertEqual(namespaces.resolve(ns, ('B',))['sum'], 342)
def test_from_(self):
inp = {
'd': {'y': 1},
'ys': [4, 5, {'from_':'d'}],
'inner': {'y': {'from_':'d'}, 'three':33},
'three': 3,
'four': 4,
'sum': None,
'z': {'from_':'d'},
}
c = BaseConfig(inp)
ns = ChainMap()
specs = c.process_fuzzyspec(('ys',), ns=ns)
ns_ = namespaces.resolve(ns, specs[2])
self.assertEqual(ns_['y'], 1)
self.assertEqual(c.resolve_key('sum', ns=ns_)[1], 8)
#ns = ChainMap()
spec, = c.process_fuzzyspec(('inner',), ns=ns)
ns_ = namespaces.resolve(ns, spec)
self.assertEqual(ns_['y'], 1)
self.assertEqual(c.resolve_key('sum', ns=ns_)[1], 38)
with self.assertRaises(ConfigError):
c.resolve_key('z', ns)
def test_resolve(self):
rem, ns = namespaces.resolve_partial(d, ('a',))
self.assertFalse(rem)
self.assertEqual(ns, ChainMap(a,d))
rem, ns = namespaces.resolve_partial(d, (('c',1),))
self.assertFalse(rem)
self.assertEqual(ns, ChainMap(c[1],d))
spec = ('a','b', ('c',1))
rem, ns = namespaces.resolve_partial(d, spec)
self.assertFalse(rem)
self.assertEqual(ns, namespaces.resolve(d,spec))
rem, ns = namespaces.resolve_partial(d, ('x','a', 'b'))
self.assertEqual(list(rem), ['x', 'a', 'b'])
self.assertEqual(len(ns.maps), 1)
spec = ('a', 'b', ('x', 0))
rem, ns = namespaces.resolve_partial(d, spec)
self.assertEqual(list(rem), [('x', 0)])
self.assertEqual(len(ns.maps), 3)
with self.assertRaises(KeyError):
namespaces.resolve(d, spec)
def test_identities(self):
a = {1:'a'}
alta = {1:'aa'}
b = {2:'b'}
d = {'a':a, 'alta':alta, 'b':b}
m1 = namespaces.resolve(d, ('a','b'))
m2 = namespaces.resolve(d, ('a','b'))
m3 = namespaces.resolve(d, ('alta','b'))
self.assertIs(m1.maps[0], m2.maps[0])
self.assertFalse(m3.maps[0] is m2.maps[0])
def test_fuzzy(self):
inp = {'four': 4, 'ys': [-1,-2,-3,-4], 'sum':None}
c = BaseConfig(inp)
ns = ChainMap()
ret = c.process_fuzzyspec(('ys',), ns=ns)
for spec, s in zip(ret, (6,5,4,3)):
ns_ = namespaces.resolve(ns, spec)
c.resolve_key('sum', ns=ns_)
self.assertEqual(ns_['sum'], s)
#specs have to be persistent
self.assertEqual(namespaces.resolve(ns, spec).maps[0],
{'sum': 3, 'y': -4})
async def _run_parallel(self, deps, completed_spec):
try:
runnable_specs = deps.send(completed_spec)
except StopIteration:
return
pending_tasks = {}
for pending_spec in runnable_specs:
if isinstance(pending_spec, (CollectSpec, CollectMapSpec)):
remote_coro = _async_identity(
pending_spec.function, namespaces.resolve(self.rootns, pending_spec.nsspec)
)
else:
remote_coro = curio.run_in_process(
self.get_result, pending_spec.function, *self.resolve_callargs(pending_spec)
)
pending_task = await curio.spawn(remote_coro)
pending_tasks[pending_task] = pending_spec
next_runs = []
waiter = curio.wait(pending_tasks)
async for completed_task in waiter:
try:
result = await completed_task.join()
except curio.TaskError as e:
raise curio.KernelExit() from e
new_completed_spec = pending_tasks.pop(completed_task)
self.set_result(result, new_completed_spec)
next_runs_coro = self._run_parallel(deps, new_completed_spec)
next_runs.append(await curio.spawn(next_runs_coro))
for wait_run in next_runs:
await wait_run.join()
def get_nice_name(ns, nsspec, suffix=None):
"""Get a name by quering the parts of a namespace specification.
``ns`` should be a namespace ChainMap and ``nsspec`` a
tuple with a valid specification
(see the ``namespaces`` documentation for more details)"""
parts = []
currspec = []
currns = ns
for ele in nsspec:
currspec.append(ele)
val = namespaces.value_from_spcec_ele(currns, ele)
# kind of ugly, but we don't want to dumpt compound types, and too long
# filenames)
if isinstance(val, (list, dict, set, tuple, frozenset)):
val = str(ele)
else:
try:
val = str(val)
except Exception as e:
log.debug("Could not convert a value (%r) to string: %s" % (val, e))
val = str(ele)
else:
if len(val) > 25:
val = str(ele)
parts.append(normalize_name(val))
currns = namespaces.resolve(ns, currspec)
if suffix:
parts.append(suffix)
return "_".join(parts)
def execute_sequential(self):
for node in self.graph:
callspec = node.value
if isinstance(callspec, (CollectSpec, CollectMapSpec)):
result = callspec.function(namespaces.resolve(self.rootns, callspec.nsspec))
else:
result = self.get_result(callspec.function, *self.resolve_callargs(callspec))
self.set_result(result, callspec)
def resolve_kwargs(self, nsspec, kwargs):
namespace = namespaces.resolve(self.rootns, nsspec)
kwdict = {}
put_index = len(namespace.maps) - 1
for kw in kwargs:
index, kwdict[kw] = namespace.get_where(kw)
if index < put_index:
put_index = index
kwdict = {kw: namespace[kw] for kw in kwargs}
return kwdict, put_index
def resolve_callargs(self, callspec):
function, kwargs, resultname, nsspec = callspec
namespace = namespaces.resolve(self.rootns, nsspec)
kwdict = {kw: namespace[kw] for kw in kwargs}
if hasattr(function, "prepare"):
prepare_args = function.prepare(spec=callspec, namespace=namespace, environment=self.environment)
else:
prepare_args = {}
return kwdict, prepare_args
def test_remove_outer(self):
targets = [Target('fruit', (['inner']), ())]
c = Config({'apple': True, 'inner':{'orange':False}})
provider = Provider()
builder = ResourceBuilder(targets=targets, providers=provider,
input_parser=c)
builder.resolve_targets()
builder.execute_sequential()
ns = namespaces.resolve(builder.rootns, ('inner',))
self.assertEqual(ns['fruit'], (None, False))
def set_result(self, result, spec):
function, _, resultname, nsspec = spec
namespace = namespaces.resolve(self.rootns, nsspec)
put_map = namespace.maps[1]
log.debug("Setting result for %s %s", spec, nsspec)
if resultname in put_map:
raise ValueError("Resource already set: %s" % resultname)
put_map[resultname] = result
for action, args in self._node_flags[spec]:
action(result, self.rootns, spec, **dict(args))
def _process_requirement(self, name, nsspec, *, extraargs=None, default=EMPTY, parents=None):
"""Create nodes so as to satisfy the requirement specified by the
arguments."""
if parents is None:
parents = []
log.debug("Processing requirement: %s" % (name,))
ns = namespaces.resolve(self.rootns, nsspec)
if extraargs is None:
extraargs = ()
# First try to find the name in the namespace
try:
put_index, val = self.input_parser.resolve_key(name, ns, parents=parents)
log.debug("Found %s for spec %s at %s" % (name, nsspec, put_index))
except InputNotFoundError as e:
# See https://www.python.org/dev/peps/pep-3110/
saved_exception = e
# Handle this case later
pass
else:
if extraargs:
raise ResourceNotUnderstood(
name,
"The resource %s name is "
"already present in the input, but some arguments were "
"passed to compute it: %s" % (name, extraargs),
parents[-1],
)
if isinstance(val, ExplicitNode):
yield from self._make_node((name, val.value), nsspec, extraargs, parents)
else:
yield put_index, val
return
# If the name is not in the providers, either it is an extra argument
# or is missing
if not self.is_provider_func(name):
if default is EMPTY:
raise saved_exception
else:
put_index = None
yield put_index, default
return
# here we handle the case where the requirement is a provider and
# make a new node for it.
yield from self._make_node(name, nsspec, extraargs, parents)
def _create_default_key(self, name, nsspec, put_index=None, defaults=None):
"""Push a namespace level for a node to store input values.
Return the nsspec of that node."""
if defaults is None:
defaults = {}
if put_index is None:
put_index = 0
defaults_label = "_" + name + "_defaults"
nsspec = (*nsspec[: len(nsspec) - put_index], defaults_label)
parent_ns = namespaces.resolve(self.rootns, nsspec[:-1])
namespaces.push_nslevel(parent_ns, defaults_label, defaults)
return nsspec
def test_nsexpand(self):
spec = ('pdfsets', 'theories', 'datasets')
c = Config(inp)
ns = utils.ChainMap()
specs = c.process_fuzzyspec(spec, ns=ns)
self.assertEqual(len(specs), 8)
datasets = [
'ds: d1 (theory: th 1)',
'ds: d2 (theory: th 1)',
'ds: d1 (theory: th 2)',
'ds: d2 (theory: th 2)',
'ds: d1 (theory: th 1)',
'ds: d2 (theory: th 1)',
'ds: d1 (theory: th 2)',
'ds: d2 (theory: th 2)']
for spec, ds in zip(specs, datasets):
self.assertEqual(namespaces.resolve(ns, spec)['dataset'], ds)
def _make_collect_targets(self, colltargets, name, nsspec, parents):
newparents = [name, *parents]
myspec = self._create_default_key(name, nsspec)
my_node = CollectMapSpec(colltargets, (), name, myspec)
required_by = yield 0, my_node
if required_by is None:
outputs = set()
else:
outputs = set([required_by])
self.graph.add_or_update_node(my_node, outputs=outputs)
myns = namespaces.resolve(self.rootns, myspec)
myns[collect.resultkey] = {}
tlens = myns[target_map.targetlenskey] = {}
for target in colltargets.targets:
target_specs = self.expand_target_spec(target)
tlens[target] = len(target_specs)
for i, tspec in enumerate(target_specs):
gen = self._process_requirement(
name=target.name, nsspec=tspec, extraargs=target.extraargs, parents=newparents
)
index, tnode = gen.send(None)
try:
gen.send(my_node)
except StopIteration:
pass
else:
raise RuntimeError()
# This allows to map directly inputs regardless of whether they
# are nodes or not.
if isinstance(tnode, Node):
flagargs = (("target", my_node), ("index", (target, i)))
self._node_flags[tnode].add((add_to_dict_flag, flagargs))
else:
myns[collect.resultkey][(target, i)] = tnode
def set_result(self, result, spec, put_index):
function, kwargs, resultname, execmode, nsspec = spec
log.debug("Setting %s in %s" % (str(spec), str(nsspec)))
namespace = namespaces.resolve(self.rootns, nsspec)
put_map = namespace.maps[put_index]
if not execmode in ExecModes:
raise TypeError("Callspecmode must be an ExecMode")
if execmode == ExecModes.SET_UNIQUE:
if resultname in put_map:
raise ValueError("Resource already set: %s" % resultname)
put_map[resultname] = result
elif execmode == ExecModes.SET_OR_UPDATE:
put_map[resultname] = result
elif execmode == ExecModes.APPEND_UNORDERED:
if not resultname in namespace:
put_map[resultname] = []
put_map[resultname].append(result)
else:
raise NotImplementedError(execmode)
def process_requirement(self, name, nsspec, extraargs=None, required_by=None,
default=EMPTY):
ns = namespaces.resolve(self.rootns, nsspec)
if extraargs is None:
extraargs = ()
try:
self.input_parser.resolve_key(name, ns, parents=[required_by])
except KeyError as e:
if hasattr(self.providers, name):
f = getattr(self.providers, name)
s = inspect.signature(f)
if(extraargs):
ns.update(dict(extraargs))
cs = CallSpec(f, tuple(s.parameters.keys()), name, ExecModes.SET_UNIQUE,
nsspec)
self.graph.add_or_update_node(cs)
for param_name, param in s.parameters.items():
self.process_requirement(param_name, nsspec, None,
required_by=cs,
default=param.default)
if required_by is None:
outputs = set()
else:
outputs = set([required_by])
self.graph.add_or_update_node(cs, outputs=outputs)
if hasattr(f, 'checks'):
for check in f.checks:
check(cs, ns, self.graph)
else:
if default is EMPTY:
raise e
else:
ns[name] = default
else:
if extraargs:
raise ResourceNotUnderstood("The resource %s name is "
"already present in the input, but some arguments were "
"passed to compute it: %s" % (name, extraargs))
def _make_collect(self, f, name, nsspec, parents):
"""Make a node that spans a function over the values in a list and
collects them in another list."""
newparents = [name, *parents]
myspec = self._create_default_key(name, nsspec)
collspec = CollectSpec(f, (), name, myspec)
log.debug("Appending node {}".format(collspec))
required_by = yield 0, collspec
if required_by is None:
outputs = set()
else:
outputs = set([required_by])
self.graph.add_or_update_node(collspec, outputs=outputs)
total_fuzzyspec = nsspec + f.fuzzyspec
specs = self.input_parser.process_fuzzyspec(total_fuzzyspec, self.rootns, newparents)
myns = namespaces.resolve(self.rootns, myspec)
myns[collect.resultkey] = OrderedDict.fromkeys(range(len(specs)))
for i, spec in enumerate(specs):
gen = self._make_node(f.function.__name__, spec, None, parents=newparents)
index, newcs = gen.send(None)
try:
gen.send(collspec)
except StopIteration:
pass
else:
raise RuntimeError()
flagargs = (("target", collspec), ("index", i))
self._node_flags[newcs].add((add_to_dict_flag, flagargs))
def _make_callspec(self, f, name, nsspec, extraargs, parents):
"""Make a normal node that calls a function."""
defaults = {}
s = inspect.signature(f)
if extraargs:
defaults.update(dict(extraargs))
# Note that this is the latest possible put_index and not len - 1
# because there is also the root namespace.
put_index = len(nsspec)
gens = []
for param_name, param in s.parameters.items():
default = defaults.get(param_name, param.default)
gen = self._process_requirement(
param_name, nsspec, extraargs=None, default=default, parents=[name, *parents]
)
index, _ = gen.send(None)
log.debug("put_index for %s is %s" % (param_name, index))
if index is None:
defaults[param_name] = default
elif index < put_index:
put_index = index
gens.append(gen)
# The namespace stack (put_index) goes in the opposite direction
# of the nsspec. put_index==len(nsspec)==len(ns.maps)-1
# corresponds to the root namespace, and put_index=0 to the current
# spec.
# We need the len bit for the case put_index==0
newnsspec = self._create_default_key(name, nsspec, put_index, defaults)
log.debug("New spec for %s is: %s" % (name, newnsspec))
ns = namespaces.resolve(self.rootns, newnsspec)
cs = CallSpec(f, tuple(s.parameters.keys()), name, newnsspec)
already_exists = cs in self.graph
if already_exists:
log.debug("Node '%s' already in the graph.", cs)
else:
log.debug("Appending node '%s'." % (cs,))
self.graph.add_or_update_node(cs)
for gen in gens:
try:
gen.send(cs)
except StopIteration:
pass
else:
raise RuntimeError()
required_by = yield put_index, cs
if required_by is None:
outputs = set()
else:
outputs = set([required_by])
self.graph.add_or_update_node(cs, outputs=outputs)
# Do not repeat the checks for the same node
if already_exists:
return
try:
check_types(f, ns)
except BadInputType as e:
raise ResourceError(name, e, parents) from e
if hasattr(f, "checks"):
for check in f.checks:
try:
check(callspec=cs, ns=ns, graph=self.graph, environment=self.environment)
except CheckError as e:
raise ResourceError(name, e, parents) from e
def nsspec(x, beginning=()):
ns = namespaces.resolve(self.rootns, beginning)
default_label = '_default' + str(x)
namespaces.push_nslevel(ns, default_label)
return beginning + (default_label,)
def add_to_dict_flag(result, ns, origin, target, index):
log.debug("Setting element %s of %r from %r", index, target, origin)
namespaces.resolve(ns, target.nsspec)[collect.resultkey][index] = result