async def test_default_workflow_triggers() -> None:
workflows = {wf.name: wf for wf in TaskHandlerService.known_workflows(empty_config())}
assert workflows["collect"].triggers == [EventTrigger("start_collect_workflow")]
assert workflows["cleanup"].triggers == [EventTrigger("start_cleanup_workflow")]
assert workflows["metrics"].triggers == [EventTrigger("start_metrics_workflow")]
assert workflows["collect_and_cleanup"].triggers == [
EventTrigger("start_collect_and_cleanup_workflow"),
TimeTrigger("0 * * * *"),
]
async def test_parse_job_line_time_trigger(task_handler: TaskHandler) -> None:
job = await task_handler.parse_job_line(
"test", '0 5 * * sat match t2 == "node" | clean')
assert job.trigger == TimeTrigger("0 5 * * sat")
assert job.command.command == 'match t2 == "node" | clean'
assert job.wait is None
with pytest.raises(ParseError):
await task_handler.parse_job_line(
"test", '0 5 invalid_cron_expr * sat : match t2 == "node" | clean')
def test_workflow() -> Workflow:
return Workflow(
TaskDescriptorId("test_workflow"),
"Speakable name of workflow",
[
Step("start", PerformAction("start_collect"), timedelta(seconds=10)),
Step("act", PerformAction("collect"), timedelta(seconds=10)),
Step("done", PerformAction("collect_done"), timedelta(seconds=10), StepErrorBehaviour.Stop),
],
[EventTrigger("start me up"), TimeTrigger("1 1 1 1 1")],
)
async def test_parse_job_line_event_and_time_trigger(
task_handler: TaskHandler) -> None:
job = await task_handler.parse_job_line(
"test", '0 5 * * sat cleanup_plan : match t2 == "node" | clean')
assert job.trigger == TimeTrigger("0 5 * * sat")
assert job.command.command == 'match t2 == "node" | clean'
assert job.wait[0] == EventTrigger("cleanup_plan") # type: ignore
with pytest.raises(ParseError):
await task_handler.parse_job_line(
"test",
'0 5 invalid_cron_expr * sat cleanup_plan:match t2 == "node" | clean'
)
async def parse_with_cron() -> Job:
parts = re.split("\\s+", stripped, 5)
if len(parts) != 6:
raise ValueError(f"Invalid job {stripped}")
wait: Optional[Tuple[EventTrigger, timedelta]] = None
trigger = TimeTrigger(" ".join(parts[0:5]))
command = strip_quotes(parts[5], "'")
# check if we also need to wait for an event: name_of_event : command
if self.event_re.match(command):
event, command = re.split("\\s*:\\s*", command, 1)
command = strip_quotes(command, "'")
wait = EventTrigger(event), wait_timeout
await self.cli.evaluate_cli_command(command,
ctx,
replace_place_holder=False)
uid = uuid_str(f"{command}{trigger}{wait}")[0:8]
return Job(uid, ExecuteCommand(command), timeout, trigger, wait,
ctx.env, mutable)
async def test_jobs_command(cli: CLI, task_handler: TaskHandlerService,
job_db: JobDb) -> None:
async def execute(cmd: str) -> List[List[JsonElement]]:
ctx = CLIContext(cli.cli_env)
return await cli.execute_cli_command(cmd, stream.list, ctx)
# add job with schedule
result = await execute(
'jobs add --id hello --schedule "23 1 * * *" echo Hello World @NOW@')
assert result == [["Job hello added."]]
job = await job_db.get("hello")
assert job is not None
assert job.command.command == "echo Hello World @NOW@"
assert job.trigger == TimeTrigger("23 1 * * *")
assert job.wait is None
assert job in task_handler.task_descriptions
assert job.environment == {"graph": "ns", "section": "reported"}
# add job with schedule and event
with_event = await execute(
'jobs add --id timed_hi --schedule "23 1 * * *" --wait-for-event foo echo Hello World'
)
assert with_event == [["Job timed_hi added."]]
job_with_event: Job = await job_db.get("timed_hi") # type: ignore
assert job_with_event.wait is not None
event_trigger, timeout = job_with_event.wait
assert event_trigger.message_type == "foo"
assert timeout == timedelta(hours=1)
assert job_with_event.environment == {"graph": "ns", "section": "reported"}
assert job_with_event in task_handler.task_descriptions
# add job with event
only_event = await execute(
"jobs add --id only_event --wait-for-event foo echo Hello World")
assert only_event == [["Job only_event added."]]
job_only_event: Job = await job_db.get("only_event") # type: ignore
assert job_only_event.trigger == EventTrigger("foo")
assert job_only_event.wait is None
assert job_only_event.environment == {"graph": "ns", "section": "reported"}
assert job_only_event in task_handler.task_descriptions
# add job without any trigger
no_trigger = await execute("jobs add --id no_trigger echo Hello World")
assert no_trigger == [["Job no_trigger added."]]
job_no_trigger: Job = await job_db.get("no_trigger") # type: ignore
assert job_no_trigger.wait is None
assert job_no_trigger.environment == {"graph": "ns", "section": "reported"}
assert job_no_trigger in task_handler.task_descriptions
# deactivate timed_hi
deactivated = await execute("jobs deactivate timed_hi")
assert deactivated[0][0]["active"] is False # type: ignore
# activate timed_hi
activated = await execute("jobs activate timed_hi")
assert activated[0][0]["active"] is True # type: ignore
# show specific job
no_trigger_show = await execute("jobs show no_trigger")
assert len(no_trigger_show[0]) == 1
# show all jobs
all_jobs = await execute("jobs list")
assert len(all_jobs[0]) == 4
# start the job
run_hello = await execute("jobs run timed_hi")
assert run_hello[0][0].startswith(
"Job timed_hi started with id") # type: ignore
assert [
t for t in await task_handler.running_tasks()
if t.descriptor.id == "timed_hi"
]
# list all running jobs
all_running = await execute("jobs running")
assert [r["job"] for r in all_running[0]] == ["timed_hi"] # type: ignore
# delete a job
deleted = await execute("jobs delete timed_hi")
assert deleted == [["Job timed_hi deleted."]]
def test_marshalling_trigger() -> None:
roundtrip(EventTrigger("test", {"foo": True}))
roundtrip(TimeTrigger("* * * * *"))
def known_workflows() -> List[Workflow]:
collect_steps = [
Step("pre_collect", PerformAction("pre_collect"),
timedelta(seconds=10)),
Step("collect", PerformAction("collect"), timedelta(seconds=10)),
Step("post_collect", PerformAction("post_collect"),
timedelta(seconds=10)),
]
cleanup_steps = [
Step("pre_cleanup_plan", PerformAction("pre_cleanup_plan"),
timedelta(seconds=10)),
Step("cleanup_plan", PerformAction("cleanup_plan"),
timedelta(seconds=10)),
Step("post_cleanup_plan", PerformAction("post_cleanup_plan"),
timedelta(seconds=10)),
Step("pre_cleanup", PerformAction("pre_cleanup"),
timedelta(seconds=10)),
Step("cleanup", PerformAction("cleanup"), timedelta(seconds=10)),
Step("post_cleanup", PerformAction("post_cleanup"),
timedelta(seconds=10)),
]
metrics_steps = [
Step("pre_generate_metrics", PerformAction("pre_generate_metrics"),
timedelta(seconds=10)),
Step("generate_metrics", PerformAction("generate_metrics"),
timedelta(seconds=10)),
Step("post_generate_metrics",
PerformAction("post_generate_metrics"),
timedelta(seconds=10)),
]
return [
Workflow(
uid="collect",
name="collect",
steps=collect_steps + metrics_steps,
triggers=[EventTrigger("start_collect_workflow")],
on_surpass=TaskSurpassBehaviour.Wait,
),
Workflow(
uid="cleanup",
name="cleanup",
steps=cleanup_steps + metrics_steps,
triggers=[EventTrigger("start_cleanup_workflow")],
on_surpass=TaskSurpassBehaviour.Wait,
),
Workflow(
uid="metrics",
name="metrics",
steps=metrics_steps,
triggers=[EventTrigger("start_metrics_workflow")],
on_surpass=TaskSurpassBehaviour.Wait,
),
Workflow(
uid="collect_and_cleanup",
name="collect_and_cleanup",
steps=collect_steps + cleanup_steps + metrics_steps,
triggers=[
EventTrigger("start_collect_and_cleanup_workflow"),
TimeTrigger("0 * * * *")
],
on_surpass=TaskSurpassBehaviour.Wait,
),
]
def workflow(name: TaskDescriptorId, steps: List[Step]) -> Workflow:
trigger: List[Trigger] = [EventTrigger(f"start_{name}_workflow")]
wf_config = config.workflows.get(name)
if wf_config:
trigger.append(TimeTrigger(wf_config.schedule))
return Workflow(uid=name, name=name, steps=steps, triggers=trigger, on_surpass=TaskSurpassBehaviour.Wait)