def test_operator_parse_brackets() -> None:
assert [
BinOp(
start=Pos(line=0, col=4, filename=FNAME),
end=Pos(line=0, col=24, filename=FNAME),
op=logical_or,
left=Lookup(
start=Pos(line=0, col=4, filename=FNAME),
end=Pos(line=0, col=7, filename=FNAME),
root="foo",
trailer=[],
),
right=BinOp(
start=Pos(line=0, col=12, filename=FNAME),
end=Pos(line=0, col=24, filename=FNAME),
op=operator.eq,
left=Lookup(
start=Pos(line=0, col=12, filename=FNAME),
end=Pos(line=0, col=15, filename=FNAME),
root="bar",
trailer=[],
),
right=Literal(
start=Pos(line=0, col=19, filename=FNAME),
end=Pos(line=0, col=24, filename=FNAME),
val="baz",
),
),
)
] == PARSER.parse(list(tokenize("""${{ foo or (bar == "baz") }}""", START)))
def test_expr_validation_not_context_field() -> None:
expr = StrExpr(
Pos(0, 0, LocalPath("<default>")),
Pos(0, 0, LocalPath("<default>")),
pattern="${{ flow.foo }}",
)
errors = validate_expr(expr, BatchContext)
assert errors[0].args[0] == "'BatchFlowCtx' has no attribute 'foo'"
def test_expr_validation_set_indexing() -> None:
expr = StrExpr(
Pos(0, 0, LocalPath("<default>")),
Pos(0, 0, LocalPath("<default>")),
pattern="${{ tags['flow_id'] }}",
)
errors = validate_expr(expr, BatchContext)
assert errors[0].args[0] == "'TagsCtx' is not subscriptable"
def test_expr_validation_unknown_context() -> None:
expr = StrExpr(
Pos(0, 0, LocalPath("<default>")),
Pos(0, 0, LocalPath("<default>")),
pattern="${{ something_new }}",
)
errors = validate_expr(expr, BatchContext)
assert errors[0].args[0] == "Unknown context 'something_new'"
def test_expr_validation_bad_subcontext_lookup() -> None:
expr = StrExpr(
Pos(0, 0, LocalPath("<default>")),
Pos(0, 0, LocalPath("<default>")),
pattern="${{ needs.task.foo }}",
)
errors = validate_expr(expr, BatchTaskContext, known_needs={"task"})
assert errors[0].args[0] == "'DepCtx' has no attribute 'foo'"
def test_expr_validation_bad_set_attr() -> None:
expr = StrExpr(
Pos(0, 0, LocalPath("<default>")),
Pos(0, 0, LocalPath("<default>")),
pattern="${{ tags.anything }}",
)
errors = validate_expr(expr, BatchContext)
assert errors[0].args[0] == "'TagsCtx' has no attribute 'anything'"
def test_expr_validation_ok_for_property_access() -> None:
expr = StrExpr(
Pos(0, 0, LocalPath("<default>")),
Pos(0, 0, LocalPath("<default>")),
pattern="${{ volumes.volume.ref_rw }}",
)
errors = validate_expr(expr, BatchContext)
assert errors == []
def test_expr_validation_list_comp_ok() -> None:
expr = StrExpr(
Pos(0, 0, LocalPath("<default>")),
Pos(0, 0, LocalPath("<default>")),
pattern="${{ [x * x for x in range(5) ] }}",
)
errors = validate_expr(expr, BatchContext)
assert errors == []
def test_expr_validation_ok() -> None:
expr = StrExpr(
Pos(0, 0, LocalPath("<default>")),
Pos(0, 0, LocalPath("<default>")),
pattern="${{ flow.flow_id }}",
)
errors = validate_expr(expr, BatchContext)
assert errors == []
def test_expr_validation_invalid_need() -> None:
expr = StrExpr(
Pos(0, 0, LocalPath("<default>")),
Pos(0, 0, LocalPath("<default>")),
pattern="${{ needs.fff }}",
)
errors = validate_expr(expr, BatchTaskContext, known_needs=set())
assert errors[0].args[
0] == "Task 'fff' is not available under needs context"
def test_non_template() -> None:
assert [
Token(
"TEXT",
"abc def jik",
Pos(0, 0, LocalPath("<test>")),
Pos(0, 13, LocalPath("<test>")),
)
] == list(tokenize("abc def jik", Pos(0, 0, LocalPath("<test>"))))
def test_non_template_with_unknown_chars() -> None:
assert [
Token(
"TEXT",
"abc!jik",
Pos(0, 0, LocalPath("<test>")),
Pos(0, 7, LocalPath("<test>")),
)
] == list(tokenize("abc!jik", Pos(0, 0, LocalPath("<test>"))))
def test_func_call_single_arg() -> None:
assert [
Call(
Pos(0, 4, FNAME),
Pos(0, 13, FNAME),
FUNCTIONS["len"],
[Literal(Pos(0, 8, FNAME), Pos(0, 13, FNAME), "abc")],
[],
)
] == PARSER.parse(list(tokenize("${{ len('abc') }}", START)))
def test_expr_validation_invalid_input() -> None:
expr = StrExpr(
Pos(0, 0, LocalPath("<default>")),
Pos(0, 0, LocalPath("<default>")),
pattern="${{ inputs.fff }}",
)
errors = validate_expr(expr,
BatchActionContext[EmptyRoot],
known_inputs=set())
assert errors[0].args[0] == "Input 'fff' is undefined"
def test_unary_operator_parse(op_str: str, op_func: Any) -> None:
assert [
UnaryOp(
Pos(0, 4, FNAME),
Pos(0, 10 + len(op_str), FNAME),
op_func,
Literal(
Pos(0, 5 + len(op_str), FNAME),
Pos(0, 10 + len(op_str), FNAME),
"bar",
),
)
] == PARSER.parse(list(tokenize(f"""${{{{ {op_str} "bar" }}}}""", START)))
async def test_dict_trailing_comma(client: Client) -> None:
pat = "${{ {'a': 1, 'b': 2,} }}"
expr = MappingExpr(START, Pos(0, len(pat), FNAME), pat,
int) # type: ignore
assert {
"a": 1,
"b": 2
} == await expr.eval(DictContext({}, client)) # type: ignore
def test_parser1() -> None:
assert [
Text(Pos(0, 0, FNAME), Pos(0, 5, FNAME), "some "),
Lookup(
Pos(0, 9, FNAME),
Pos(0, 16, FNAME),
"var",
[AttrGetter(Pos(0, 13, FNAME), Pos(0, 16, FNAME), "arg")],
),
Text(Pos(0, 19, FNAME), Pos(0, 24, FNAME), " text"),
] == PARSER.parse(list(tokenize("some ${{ var.arg }} text", START)))
def _make_ast_call(
args: Mapping[str, Union[bool, int, float,
str]]) -> ast.BaseActionCall:
def _make_simple_str_expr(res: Optional[str]) -> SimpleStrExpr:
return SimpleStrExpr(Pos(0, 0, LocalPath("fake")),
Pos(0, 0, LocalPath("fake")), res)
def _make_primitive_expr(
res: Union[bool, int, float, str]) -> PrimitiveExpr:
return PrimitiveExpr(Pos(0, 0, LocalPath("fake")),
Pos(0, 0, LocalPath("fake")), res)
return ast.BaseActionCall(
_start=Pos(0, 0, LocalPath("fake")),
_end=Pos(0, 0, LocalPath("fake")),
action=_make_simple_str_expr("ws:test"),
args={key: _make_primitive_expr(value)
for key, value in args.items()},
)
def test_func_call_multiple_args() -> None:
assert [
Call(
Pos(0, 4, FNAME),
Pos(0, 27, FNAME),
FUNCTIONS["fmt"],
[
Literal(Pos(0, 8, FNAME), Pos(0, 15, FNAME), "{} {}"),
Literal(Pos(0, 17, FNAME), Pos(0, 22, FNAME), "abc"),
Literal(Pos(0, 24, FNAME), Pos(0, 27, FNAME), 123),
],
[],
)
] == PARSER.parse(list(tokenize('${{ fmt("{} {}", "abc", 123) }}', START)))
def test_corner_case1() -> None:
s = dedent(
"""\
jupyter notebook
--no-browser
--ip=0.0.0.0
--allow-root
--NotebookApp.token=
--notebook-dir=${{ volumes.notebooks.mount }}
"""
)
assert (
[
Text(
Pos(0, 0, FNAME),
Pos(5, 17, FNAME),
dedent(
"""\
jupyter notebook
--no-browser
--ip=0.0.0.0
--allow-root
--NotebookApp.token=
--notebook-dir="""
),
),
Lookup(
Pos(5, 21, FNAME),
Pos(5, 44, FNAME),
"volumes",
[
AttrGetter(Pos(5, 29, FNAME), Pos(5, 38, FNAME), "notebooks"),
AttrGetter(Pos(5, 39, FNAME), Pos(5, 44, FNAME), "mount"),
],
),
Text(Pos(5, 47, FNAME), Pos(6, 0, FNAME), "\n"),
]
== PARSER.parse(list(tokenize(s, START)))
)
def _make_ast_inputs(
args: Mapping[str, Tuple[Optional[Union[bool, int, float, str]],
InputType]]
) -> Mapping[str, ast.Input]:
def _make_opt_primitive_expr(
res: Optional[Union[bool, int, float,
str]]) -> SimpleOptPrimitiveExpr:
return SimpleOptPrimitiveExpr(Pos(0, 0, LocalPath("fake")),
Pos(0, 0, LocalPath("fake")), res)
def _make_opt_str_expr(res: Optional[str]) -> SimpleOptStrExpr:
return SimpleOptStrExpr(Pos(0, 0, LocalPath("fake")),
Pos(0, 0, LocalPath("fake")), res)
return {
key: ast.Input(
_start=Pos(0, 0, LocalPath("fake")),
_end=Pos(0, 0, LocalPath("fake")),
default=_make_opt_primitive_expr(value[0]),
descr=_make_opt_str_expr(None),
type=value[1],
)
for key, value in args.items()
}
def test_func_call_with_trailer_item() -> None:
assert [
Call(
Pos(0, 4, FNAME),
Pos(0, 41, FNAME),
FUNCTIONS["from_json"],
[Literal(Pos(0, 14, FNAME), Pos(0, 36, FNAME), '{"a": 1, "b": "val"}')],
[
ItemGetter(
Pos(0, 38, FNAME),
Pos(0, 41, FNAME),
Literal(Pos(0, 38, FNAME), Pos(0, 41, FNAME), "a"),
)
],
)
] == PARSER.parse(
list(tokenize("""${{ from_json('{"a": 1, "b": "val"}')['a'] }}""", START))
)
def test_tmpl_ok5() -> None:
assert [
Lookup(
Pos(0, 4, FNAME),
Pos(0, 18, FNAME),
"name",
[
AttrGetter(Pos(0, 9, FNAME), Pos(0, 12, FNAME), "sub"),
AttrGetter(Pos(0, 13, FNAME), Pos(0, 18, FNAME), "param"),
],
)
] == PARSER.parse(list(tokenize("$[[ name.sub.param ]]", START)))
def test_func_call_arg_lookup() -> None:
assert [
Call(
Pos(0, 4, FNAME),
Pos(0, 30, FNAME),
FUNCTIONS["len"],
[
Lookup(
Pos(0, 8, FNAME),
Pos(0, 30, FNAME),
"images",
[
AttrGetter(Pos(0, 15, FNAME), Pos(0, 19, FNAME), "name"),
AttrGetter(Pos(0, 20, FNAME), Pos(0, 30, FNAME), "build_args"),
],
)
],
[],
)
] == PARSER.parse(list(tokenize("${{ len(images.name.build_args) }}", START)))
def test_operator_parse(op_str: str, op_func: Any) -> None:
assert [
BinOp(
Pos(0, 4, FNAME),
Pos(0, 14 + len(op_str), FNAME),
op_func,
Lookup(Pos(0, 4, FNAME), Pos(0, 7, FNAME), "foo", []),
Literal(
Pos(0, 9 + len(op_str), FNAME),
Pos(0, 14 + len(op_str), FNAME),
"bar",
),
)
] == PARSER.parse(list(tokenize(f"""${{{{ foo {op_str} "bar" }}}}""", START)))
def test_func_nested_calls() -> None:
assert [
Call(
Pos(0, 4, FNAME),
Pos(0, 16, FNAME),
FUNCTIONS["len"],
[
Call(
Pos(0, 8, FNAME),
Pos(0, 16, FNAME),
FUNCTIONS["keys"],
[Lookup(Pos(0, 13, FNAME), Pos(0, 16, FNAME), "abc", [])],
[],
)
],
[],
)
] == PARSER.parse(list(tokenize("${{ len(keys(abc)) }}", START)))
def test_dict_short() -> None:
assert [
DictMaker(
start=Pos(0, 5, FNAME),
end=Pos(0, 11, FNAME),
items=[
(
Literal(
start=Pos(0, 5, FNAME),
end=Pos(0, 6, FNAME),
val=1,
),
Literal(
start=Pos(0, 8, FNAME),
end=Pos(0, 11, FNAME),
val="2",
),
),
],
)
] == PARSER.parse(list(tokenize("${{ {1: '2'} }}", START)))
def test_parse_batch_call(assets: LocalPath) -> None:
workspace = assets
config_file = workspace / "batch-action-call.yml"
flow = parse_batch(workspace, config_file)
assert flow == ast.BatchFlow(
Pos(0, 0, config_file),
Pos(6, 0, config_file),
params=None,
id=SimpleOptIdExpr(
Pos(0, 0, config_file),
Pos(0, 0, config_file),
None,
),
kind=ast.FlowKind.BATCH,
title=SimpleOptStrExpr(
Pos(0, 0, config_file),
Pos(0, 0, config_file),
None,
),
life_span=OptTimeDeltaExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
images=None,
volumes=None,
defaults=None,
mixins=None,
tasks=[
ast.TaskActionCall(
Pos(2, 2, config_file),
Pos(6, 0, config_file),
id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
"test"),
needs=None,
strategy=None,
cache=None,
enable=EnableExpr(
Pos(0, 0, config_file),
Pos(0, 0, config_file),
"${{ success() }}",
),
action=SimpleStrExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file),
"ws:batch-action"),
args={
"arg1":
PrimitiveExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), "val 1")
},
)
],
)
def test_parse_live_module_call(assets: LocalPath) -> None:
workspace = assets
config_file = workspace / "live-module-call.yml"
flow = parse_live(workspace, config_file)
assert flow == ast.LiveFlow(
Pos(0, 0, config_file),
Pos(16, 0, config_file),
id=SimpleOptIdExpr(
Pos(0, 0, config_file),
Pos(0, 0, config_file),
None,
),
kind=ast.FlowKind.LIVE,
title=SimpleOptStrExpr(
Pos(0, 0, config_file),
Pos(0, 0, config_file),
None,
),
images=None,
volumes=None,
defaults=ANY,
mixins=None,
jobs={
"test":
ast.JobModuleCall(
Pos(13, 4, config_file),
Pos(16, 0, config_file),
module=SimpleStrExpr(
Pos(13, 4, config_file),
Pos(15, 0, config_file),
"workspace:live-module",
),
args={
"arg1":
PrimitiveExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), "val 1")
},
params=None,
)
},
)
def test_parse_stateful_action(assets: LocalPath) -> None:
config_file = assets / "stateful_actions/parser-test.yml"
action = parse_action(config_file)
assert action == ast.StatefulAction(
Pos(0, 0, config_file),
Pos(19, 0, config_file),
kind=ast.ActionKind.STATEFUL,
name=SimpleOptStrExpr(
Pos(0, 0, config_file),
Pos(0, 0, config_file),
"Test stateful Action",
),
author=SimpleOptStrExpr(
Pos(0, 0, config_file),
Pos(0, 0, config_file),
"Andrew Svetlov",
),
descr=SimpleOptStrExpr(
Pos(0, 0, config_file),
Pos(0, 0, config_file),
"description of test action",
),
inputs={
"arg1":
ast.Input(
Pos(6, 4, config_file),
Pos(7, 2, config_file),
descr=SimpleOptStrExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), "param 1"),
default=SimpleOptPrimitiveExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
),
"arg2":
ast.Input(
Pos(8, 4, config_file),
Pos(10, 0, config_file),
descr=SimpleOptStrExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), "param 2"),
default=SimpleOptPrimitiveExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file),
"value 2"),
),
},
outputs={
"res":
ast.Output(
Pos(12, 4, config_file),
Pos(13, 0, config_file),
descr=SimpleOptStrExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file),
"action result"),
value=OptStrExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
)
},
main=ast.ExecUnit(
Pos(14, 2, config_file),
Pos(16, 0, config_file),
title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
None),
name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
None),
image=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
"ubuntu"),
preset=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
None),
schedule_timeout=OptTimeDeltaExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
entrypoint=OptStrExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
cmd=OptBashExpr(
Pos(0, 0, config_file),
Pos(0, 0, config_file),
"echo ::save-state name=state::State",
),
workdir=OptRemotePathExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
env=None,
volumes=None,
tags=None,
life_span=OptTimeDeltaExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
http_port=OptIntExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
http_auth=OptBoolExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
pass_config=OptBoolExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
),
post=ast.ExecUnit(
Pos(17, 2, config_file),
Pos(19, 0, config_file),
title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
None),
name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
None),
image=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
"ubuntu"),
preset=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
None),
schedule_timeout=OptTimeDeltaExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
entrypoint=OptStrExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
cmd=OptBashExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
"echo End"),
workdir=OptRemotePathExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
env=None,
volumes=None,
tags=None,
life_span=OptTimeDeltaExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
http_port=OptIntExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
http_auth=OptBoolExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
pass_config=OptBoolExpr(Pos(0, 0, config_file),
Pos(0, 0, config_file), None),
),
post_if=EnableExpr(Pos(0, 0, config_file), Pos(0, 0, config_file),
"${{ always() }}"),
)