def test_sideffect_not_canceled_if_not_resched(exemethod):
# Check op without any provides
#
an_sfx = sfx("b")
op1 = operation(lambda: {an_sfx: False},
name="op1",
provides=an_sfx,
returns_dict=True)
op2 = operation(lambda: 1, name="op2", needs=an_sfx, provides="b")
pipeline = compose("t", op1, op2, parallel=exemethod)
# sol = pipeline.compute()
# assert sol == {an_sfx: False, "b": 1}
sol = pipeline.compute(outputs="b")
assert sol == {"b": 1}
# Check also op with some provides
#
an_sfx = sfx("b")
op1 = operation(
lambda: {
"a": 1,
an_sfx: False
},
name="op1",
provides=["a", an_sfx],
returns_dict=True,
)
op2 = operation(lambda: 1, name="op2", needs=an_sfx, provides="b")
pipeline = compose("t", op1, op2, parallel=exemethod)
sol = pipeline.compute()
assert sol == {"a": 1, an_sfx: False, "b": 1}
sol = pipeline.compute(outputs="b")
assert sol == {"b": 1}
def test_sideffect_real_input(reverse, exemethod):
sidefx_fail = is_marshal_tasks() and not isinstance(
get_execution_pool(),
types.FunctionType # mp_dummy.Pool
)
ops = [
operation(name="extend", needs=["box", "a"],
provides=[sfx("b")])(_box_extend),
operation(name="increment", needs=["box", sfx("b")],
provides="c")(_box_increment),
]
if reverse:
ops = reversed(ops)
# Designate `a`, `b` as sideffect inp/out arguments.
graph = compose("mygraph", *ops, parallel=exemethod)
box_orig = [0]
assert graph(**{
"box": [0],
"a": True
}) == {
"a": True,
"box": box_orig if sidefx_fail else [1, 2, 3],
"c": None,
}
assert graph.compute({
"box": [0],
"a": True
}, ["box", "c"]) == {
"box": box_orig if sidefx_fail else [1, 2, 3],
"c": None,
}
def pipeline_sideffect1(request, exemethod) -> Pipeline:
ops = [
operation(name="extend", needs=["box", sfx("a")],
provides=[sfx("b")])(_box_extend),
operation(name="increment", needs=["box", sfx("b")],
provides=sfx("c"))(_box_increment),
]
if request.param:
ops = reversed(ops)
# Designate `a`, `b` as sideffect inp/out arguments.
graph = compose("sideffect1", *ops, parallel=exemethod)
return graph
def test_compose_nest_dict(caplog):
pipe = compose(
"t",
compose(
"p1",
operation(
str,
name="op1",
needs=[sfx("a"), "aa"],
provides=[sfxed("S1", "g"), sfxed("S2", "h")],
),
),
compose(
"p2",
operation(
str,
name="op2",
needs=sfx("a"),
provides=["a", sfx("b")],
aliases=[("a", "b")],
),
),
nest={
"op1": True,
"op2": lambda n: "p2.op2",
"aa": False,
sfx("a"): True,
"b": lambda n: f"PP.{n}",
sfxed("S1", "g"): True,
sfxed("S2", "h"): lambda n: dep_renamed(n, "ss2"),
sfx("b"): True,
},
)
got = str(pipe.ops)
print(got)
assert got == re.sub(
r"[\n ]{2,}", # collapse all space-chars into a single space
" ",
"""
[FnOp(name='p1.op1', needs=[sfx('p1.a'), 'aa'],
provides=[sfxed('p1.S1', 'g'), sfxed('ss2', 'h')], fn='str'),
FnOp(name='p2.op2', needs=[sfx('p2.a')],
provides=['a', sfx('p2.b'), 'PP.b'], aliases=[('a', 'PP.b')], fn='str')]
""".strip(),
)
for record in caplog.records:
assert record.levelname != "WARNING"
def test_compose_rename_dict(caplog):
pip = compose(
"t",
operation(str, "op1", provides=["a", "aa"]),
operation(
str,
"op2",
needs="a",
provides=["b", sfx("c")],
aliases=[("b", "B"), ("b", "p")],
),
nest={
"op1": "OP1",
"op2": lambda n: "OP2",
"a": "A",
"b": "bb"
},
)
print(str(pip))
assert str(pip) == (
"Pipeline('t', needs=['A'], "
"provides=['A', 'aa', 'bb', sfx('c'), 'B', 'p'], x2 ops: OP1, OP2)")
print(str(pip.ops))
assert (str(pip.ops) == dedent("""
[FnOp(name='OP1', provides=['A', 'aa'], fn='str'),
FnOp(name='OP2', needs=['A'], provides=['bb', sfx('c'), 'B', 'p'],
aliases=[('bb', 'B'), ('bb', 'p')], fn='str')]
""").replace("\n", ""))
def test_sideffect_steps(exemethod, pipeline_sideffect1: Pipeline):
sidefx_fail = is_marshal_tasks() and not isinstance(
get_execution_pool(),
types.FunctionType # mp_dummy.Pool
)
pipeline = pipeline_sideffect1.withset(parallel=exemethod)
box_orig = [0]
sol = pipeline.compute({"box": [0], sfx("a"): True}, ["box", sfx("c")])
assert sol == {"box": box_orig if sidefx_fail else [1, 2, 3]}
assert len(sol.plan.steps) == 4
## Check sideffect links plotted as blue
# (assumes color used only for this!).
dot = pipeline.net.plot()
assert "blue" in str(dot)
def test_pipe_rename():
pipe = compose(
"t",
operation(str, name="op1", needs=sfx("a")),
operation(
str,
name="op2",
needs=sfx("a"),
provides=["a", sfx("b")],
aliases=[("a", "b")],
),
)
def renamer(na):
assert na.op
assert not na.parent
return dep_renamed(na.name, lambda name: f"PP.{name}")
ren = pipe.withset(renamer=renamer)
got = str(ren)
assert got == ("""
Pipeline('t', needs=[sfx('PP.a')], provides=['PP.a', sfx('PP.b'), 'PP.b'], x2 ops: PP.op1, PP.op2)
""".strip())
got = str(ren.ops)
assert got == oneliner("""
[FnOp(name='PP.op1', needs=[sfx('PP.a')], fn='str'),
FnOp(name='PP.op2', needs=[sfx('PP.a')], provides=['PP.a', sfx('PP.b'), 'PP.b'],
aliases=[('PP.a', 'PP.b')], fn='str')]
""")
## Check dictionary with callables
#
ren = pipe.withset(renamer={
"op1": lambda n: "OP1",
"op2": False,
"a": optional("a"),
"b": "B",
})
got = str(ren.ops)
assert got == oneliner("""
[FnOp(name='OP1', needs=[sfx('a')], fn='str'),
FnOp(name='op2', needs=[sfx('a')], provides=['a'(?), sfx('b'), 'B'],
aliases=[('a'(?), 'B')], fn='str')]
""")
def test_op_rename():
op = operation(
str,
name="op1",
needs=sfx("a"),
provides=["a", sfx("b")],
aliases=[("a", "b")],
)
def renamer(na):
assert na.op
assert not na.parent
return dep_renamed(na.name, lambda name: f"PP.{name}")
ren = op.withset(renamer=renamer)
got = str(ren)
assert got == ("""
FnOp(name='PP.op1', needs=[sfx('PP.a')], provides=['PP.a', sfx('PP.b'), 'PP.b'], aliases=[('PP.a', 'PP.b')], fn='str')
""".strip())
def test_sideffect_no_real_data(pipeline_sideffect1: Pipeline):
sidefx_fail = is_marshal_tasks() and not isinstance(
get_execution_pool(),
types.FunctionType # mp_dummy.Pool
)
graph = pipeline_sideffect1
inp = {"box": [0], "a": True}
## Normal data must not match sideffects.
#
# with pytest.raises(ValueError, match="Unknown output node"):
# graph.compute(inp, ["a"])
# with pytest.raises(ValueError, match="Unknown output node"):
# graph.compute(inp, ["b"])
## Cannot compile due to missing inputs/outputs
#
with pytest.raises(ValueError, match="Unsolvable graph"):
graph(**inp)
with pytest.raises(ValueError, match="Unsolvable graph"):
graph.compute(inp)
with pytest.raises(ValueError, match="Unreachable outputs"):
graph.compute(inp, ["box", sfx("b")])
with pytest.raises(ValueError, match="Unsolvable graph"):
# Cannot run, since no sideffect inputs given.
graph.compute(inp)
box_orig = [0]
## OK INPUT SIDEFFECTS
#
# ok, no asked out
sol = graph.compute({"box": [0], sfx("a"): True})
assert sol == {
"box": box_orig if sidefx_fail else [1, 2, 3],
sfx("a"): True
}
#
# Although no out-sideffect asked (like regular data).
assert graph.compute({"box": [0], sfx("a"): True}, "box") == {"box": [0]}
#
# ok, asked the 1st out-sideffect
sol = graph.compute({"box": [0], sfx("a"): True}, ["box", sfx("b")])
assert sol == {"box": box_orig if sidefx_fail else [0, 1, 2]}
#
# ok, asked the 2nd out-sideffect
sol = graph.compute({"box": [0], sfx("a"): True}, ["box", sfx("c")])
assert sol == {"box": box_orig if sidefx_fail else [1, 2, 3]}
def test_sideffect_cancel_sfx_only_operation(exemethod):
an_sfx = sfx("b")
op1 = operation(
lambda: {an_sfx: False},
name="op1",
provides=an_sfx,
returns_dict=True,
rescheduled=True,
)
op2 = operation(lambda: 1, name="op2", needs=an_sfx, provides="a")
pipeline = compose("t", op1, op2, parallel=exemethod)
sol = pipeline.compute({})
assert sol == {an_sfx: False}
sol = pipeline.compute(outputs=an_sfx)
assert sol == {an_sfx: False}
def test_cwd_fnop():
op = operation(
str,
None,
needs=[
"a",
"a/b",
"/r/b",
optional("o"),
keyword("k"),
implicit("i"),
vararg("v1"),
varargs("v2"),
sfx("s1"),
sfxed("s2", "s22"),
vcat("vc"),
],
provides=["A/B", "C", "/R"],
aliases=[("A/B", "aa"), ("C", "CC"), ("/R", "RR")],
cwd="root",
)
exp = """
FnOp(name='str',
needs=['root/a'($),
'root/a/b'($),
'/r/b'($),
'root/o'($?'o'),
'root/k'($>'k'),
'root/i'($),
'root/v1'($*),
'root/v2'($+),
sfx('s1'),
sfxed('root/s2'($),
's22'),
'root/vc'($)],
provides=['root/A/B'($),
'root/C'($),
'/R'($),
'root/aa'($),
'root/CC'($),
'root/RR'($)],
aliases=[('root/A/B'($), 'root/aa'($)),
('root/C'($), 'root/CC'($)),
('/R'($), 'root/RR'($))],
fn='str')
"""
assert oneliner(op) == oneliner(exp)
def test_jetsam_n_plot_with_DEBUG():
pipe = compose(
"mix",
operation(
str,
"FUNC",
needs=[
"a",
sfxed("b", "foo", keyword="bb"),
implicit("c"),
sfxed("d", "bar"),
vararg("e"),
varargs("f"),
],
provides=[
"A",
sfxed("b", "FOO", keyword="bb"),
implicit("C"),
sfxed("d", "BAR", optional=True),
sfx("FOOBAR"),
],
aliases={
"A": "aaa",
"b": "bbb",
"d": "ddd"
}, # FIXME: "D" is implicit!
),
)
with debug_enabled(True), pytest.raises(ValueError, match="^Unsolvable"):
pipe.compute()
with debug_enabled(True), pytest.raises(
ValueError, match="^Failed matching inputs <=> needs") as exc:
pipe.compute({
"a": 1,
sfxed("b", "foo"): 2,
"c": 3,
sfxed("d", "bar"): 4,
"e": 5,
"f": [6, 7],
})
exc.value.jetsam.plot_fpath.unlink()
def test_sideffect_cancel(exemethod):
an_sfx = sfx("b")
op1 = operation(
lambda: {
"a": 1,
an_sfx: False
},
name="op1",
provides=["a", an_sfx],
returns_dict=True,
rescheduled=True,
)
op2 = operation(lambda: 1, name="op2", needs=an_sfx, provides="b")
pipeline = compose("t", op1, op2, parallel=exemethod)
sol = pipeline.compute()
assert sol == {"a": 1, an_sfx: False}
sol = pipeline.compute(outputs="a")
assert sol == {"a": 1} # an_sfx evicted
## SFX both pruned & evicted
#
assert an_sfx not in sol.dag.nodes
assert an_sfx in sol.plan.steps
def test_op_rename_parts():
op = operation(
str,
name="op1",
needs=[sfx("a/b"), "/a/b"],
provides=["b/c", sfxed("d/e/f", "k/l")],
aliases=[("b/c", "/b/t")],
)
def renamer(na):
if na.name and na.typ.endswith(".jsonpart"):
return f"PP.{na.name}"
ren = op.withset(renamer=renamer)
got = str(ren)
print(got)
assert got == oneliner(
"""
FnOp(name='op1',
needs=[sfx('a/b'), '/PP.a/PP.b'($)],
provides=['PP.b/PP.c'($), sfxed('PP.d/PP.e/PP.f'($), 'k/l'), '/PP.b/PP.t'($)],
aliases=[('PP.b/PP.c'($), '/PP.b/PP.t'($))], fn='str')
""", )
def yield_wrapped_ops(
self,
fn: Union[Callable, Tuple[Union[str, Collection[str]],
Union[Callable, Collection[Callable]]], ],
exclude=(),
domain: Union[str, int, Collection] = None,
) -> Iterable[FnOp]:
"""
Convert a (possibly **@autographed**) function into an graphtik **FnOperations**,
respecting any configured overrides
:param fn:
either a callable, or a 2-tuple(`name-path`, `fn-path`) for::
[module[, class, ...]] callable
- If `fn` is an operation, yielded as is (found also in 2-tuple).
- Both tuple elements may be singulars, and are auto-tuple-zed.
- The `name-path` may (or may not) correspond to the given `fn-path`,
and is used to derrive the operation-name; If not given, the function
name is inspected.
- The last elements of the `name-path` are overridden by names in decorations;
if the decor-name is the "default" (`None`), the `name-path` becomes
the op-name.
- The `name-path` is not used when matching overrides.
:param exclude:
a list of decor-names to exclude, as stored in decors.
Ignored if `fn` already an operation.
:param domain:
if given, overrides :attr:`domain` for :func:`.autographed` decorators
to search.
List-ified if a single str, :func:`autographed` decors for the 1st one
matching are used.
:return:
one or more :class:`FnOp` instances (if more than one name is defined
when the given function was :func:`autographed`).
Overriddes order: my-args, self.overrides, autograph-decorator, inspection
See also: David Brubeck Quartet, "40 days"
"""
if isinstance(fn, tuple):
name_path, fn_path = fn
else:
name_path, fn_path = (), fn
fun_path = cast(Tuple[Callable, ...], astuple(fn_path, None))
fun = fun_path[-1]
if isinstance(fun, Operation):
## pass-through operations
yield fun
return
def param_to_modifier(name: str, param: inspect.Parameter) -> str:
return (optional(name)
# is optional?
if param.default is not inspect._empty # type: ignore
else keyword(name)
if param.kind == Parameter.KEYWORD_ONLY else name)
given_name_path = astuple(name_path, None)
decors_by_name = get_autograph_decors(fun, {}, domain or self.domain)
for decor_name, decors in decors_by_name.items() or ((None, {}), ):
if given_name_path and not decor_name:
name_path = decor_path = given_name_path
else: # Name in decors was "default"(None).
name_path = decor_path = astuple(
(decor_name
if decor_name else func_name(fun, fqdn=1)).split("."),
None,
)
assert decor_path, locals()
if given_name_path:
# Overlay `decor_path` over `named_path`, right-aligned.
name_path = tuple(*name_path[:-len(decor_path)],
*decor_path)
fn_name = str(name_path[-1])
if fn_name in exclude:
continue
overrides = self._from_overrides(decor_path)
op_data = (ChainMap(overrides, decors) if (overrides and decors)
else overrides if overrides else decors)
if op_data:
log.debug("Autograph overrides for %r: %s", name_path, op_data)
op_props = "needs provides renames, inp_sideffects out_sideffects".split(
)
needs, provides, override_renames, inp_sideffects, out_sideffects = (
op_data.get(a, _unset) for a in op_props)
sig = None
if needs is _unset:
sig = inspect.signature(fun)
needs = [
param_to_modifier(name, param)
for name, param in sig.parameters.items() if name != "self"
and param.kind is not Parameter.VAR_KEYWORD
]
## Insert object as 1st need for object-methods.
#
if len(fun_path) > 1:
clazz = fun_path[-2]
# TODO: respect autograph decorator for object-names.
class_name = name_path[-2] if len(
name_path) > 1 else clazz.__name__
if is_regular_class(class_name, clazz):
log.debug("Object-method %s.%s", class_name, fn_name)
needs.insert(0, camel_2_snake_case(class_name))
needs = aslist(needs, "needs")
if ... in needs:
if sig is None:
sig = inspect.signature(fun)
needs = [
arg_name if n is ... else n
for n, arg_name in zip(needs, sig.parameters)
]
if provides is _unset:
if is_regular_class(fn_name, fun):
## Convert class-name into object variable.
provides = camel_2_snake_case(fn_name)
elif self.out_patterns:
provides = self._deduce_provides_from_fn_name(
fn_name) or _unset
if provides is _unset:
provides = ()
provides = aslist(provides, "provides")
needs, provides = self._apply_renames(
(override_renames, self.renames), (needs, provides))
if inp_sideffects is not _unset:
needs.extend((i if is_sfx(i) else sfxed(
*i) if isinstance(i, tuple) else sfx(i))
for i in aslist(inp_sideffects, "inp_sideffects"))
if out_sideffects is not _unset:
provides.extend(
(i if is_sfx(i) else sfxed(
*i) if isinstance(i, tuple) else sfx(i))
for i in aslist(out_sideffects, "out_sideffects"))
if self.full_path_names:
fn_name = self._join_path_names(*name_path)
op_kws = self._collect_rest_op_args(decors)
yield FnOp(fn=fun,
name=fn_name,
needs=needs,
provides=provides,
**op_kws)
def test_sideffect_NO_RESULT(caplog, exemethod):
# NO_RESULT does not cancel sideffects unless op-rescheduled
#
an_sfx = sfx("b")
op1 = operation(lambda: NO_RESULT, name="do-SFX", provides=an_sfx)
op2 = operation(lambda: 1, name="ask-SFX", needs=an_sfx, provides="a")
pipeline = compose("t", op1, op2, parallel=exemethod)
sol = pipeline.compute({}, outputs=an_sfx)
assert op1 in sol.executed
assert op2 not in sol.executed
assert sol == {}
sol = pipeline.compute({})
assert op1 in sol.executed
assert op2 in sol.executed
assert sol == {"a": 1}
sol = pipeline.compute({}, outputs="a")
assert op1 in sol.executed
assert op2 in sol.executed
assert sol == {"a": 1}
# NO_RESULT cancels sideffects of rescheduled ops.
#
pipeline = compose("t", op1, op2, rescheduled=True, parallel=exemethod)
sol = pipeline.compute({})
assert op1 in sol.executed
assert op2 not in sol.executed
assert sol == {an_sfx: False}
sol = pipeline.compute({}, outputs="a")
assert op2 not in sol.executed
assert op1 in sol.executed
assert sol == {} # an_sfx evicted
# NO_RESULT_BUT_SFX cancels sideffects of rescheduled ops.
#
op11 = operation(lambda: NO_RESULT_BUT_SFX, name="do-SFX", provides=an_sfx)
pipeline = compose("t", op11, op2, rescheduled=True, parallel=exemethod)
sol = pipeline.compute({}, outputs=an_sfx)
assert op11 in sol.executed
assert op2 not in sol.executed
assert sol == {}
sol = pipeline.compute({})
assert op11 in sol.executed
assert op2 in sol.executed
assert sol == {"a": 1}
sol = pipeline.compute({}, outputs="a")
assert op11 in sol.executed
assert op2 in sol.executed
assert sol == {"a": 1}
## If NO_RESULT were not translated,
# a warning of unknown out might have emerged.
caplog.clear()
pipeline = compose("t",
operation(lambda: 1, provides=an_sfx),
parallel=exemethod)
pipeline.compute({}, outputs=an_sfx)
for record in caplog.records:
if record.levelname == "WARNING":
assert "Ignoring result(1) because no `provides`" in record.message
caplog.clear()
pipeline = compose("t",
operation(lambda: NO_RESULT, provides=an_sfx),
parallel=exemethod)
pipeline.compute({}, outputs=an_sfx)
for record in caplog.records:
assert record.levelname != "WARNING"
return n2v_g_vmax * V.max()
def calc_n_max_vehicle(n2v_g_vmax, v_max):
"""Calc `n_max3` of Annex 2-2.g from `v_max` (Annex 2-2.i). """
return n2v_g_vmax * v_max
def calc_n_max(n95_high, n_max_cycle, n_max_vehicle):
n_max = max(n95_high, n_max_cycle, n_max_vehicle)
assert np.isfinite(n_max), (
"All `n_max` are NANs?",
n95_high,
n_max_cycle,
n_max_vehicle,
n_max,
)
return n_max
@autog.autographed(needs=["wot", ...], provides=sfx("valid_n_max"))
def validate_n_max(wot: pd.DataFrame, n_max: int) -> None:
"""Simply check the last N wot point is >= `n_max`. """
w = wio.pstep_factory.get().wot
if wot[w.n].iloc[-1] < n_max:
raise ValueError(
f"Last wot point ({wot[w.n].iloc[-1]}min⁻¹, {wot[w.n].iloc[-1]:.02f}kW) is below n_max({n_max})!"
)
}),
ValueError(
r"The `aliases` \['a'-->'A'\] rename non-existent provides in \[\]"
),
),
(
("a", {
"a": "A",
"b": "B"
}),
ValueError(
r"The `aliases` \['b'-->'B'\] rename non-existent provides in \['a'\]"
),
),
(
(sfx("a"), {
sfx("a"): "a"
}),
ValueError("must not contain `sideffects"),
),
(
("a", {
"a": sfx("AA")
}),
ValueError("must not contain `sideffects"),
),
(
(["a", "b"], {
"a": "b"
}),
ValueError(r"clash with existing provides in \['a', 'b'\]"),
