def test_check_xtriggers(xtrigger_mgr_procpool):
"""Test check_xtriggers call.
check_xtriggers does pretty much the same as collate. The
difference is that besides tallying on all the xtriggers and
clock xtriggers available, it then proceeds to trying to
satisfy them."""
# add a xtrigger
# that will cause all_xtrig to be populated, but not all_xclock
get_name = SubFuncContext(
label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={}
)
xtrigger_mgr_procpool.add_trig("get_name", get_name, 'fdir')
get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef1 = TaskDef(
name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False
)
init()
sequence = ISO8601Sequence('P1D', '2019')
tdef1.xtrig_labels[sequence] = ["get_name"]
start_point = ISO8601Point('2019')
itask1 = TaskProxy(tdef=tdef1, start_point=start_point)
itask1.state.xtriggers["get_name"] = False # satisfied?
# add a clock xtrigger
# that will cause both all_xclock to be populated but not all_xtrig
wall_clock = SubFuncContext(
label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={}
)
wall_clock.out = "[\"True\", \"1\"]"
xtrigger_mgr_procpool.add_trig("wall_clock", wall_clock, "fdir")
# create a task
tdef2 = TaskDef(
name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False
)
tdef2.xtrig_labels[sequence] = ["wall_clock"]
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask2 = TaskProxy(tdef=tdef2, start_point=start_point)
xtrigger_mgr_procpool.check_xtriggers([itask1, itask2])
# won't be satisfied, as it is async, we are are not calling callback
assert not xtrigger_mgr_procpool.sat_xtrig
assert xtrigger_mgr_procpool.all_xtrig
def test_collate(xtrigger_mgr):
"""Test that collate properly tallies the totals of current xtriggers."""
xtrigger_mgr.collate(itasks=[])
assert not xtrigger_mgr.all_xtrig
# add a xtrigger
# that will cause all_xtrig to be populated
get_name = SubFuncContext(
label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={}
)
xtrigger_mgr.add_trig("get_name", get_name, 'fdir')
get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef = TaskDef(
name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False
)
init()
sequence = ISO8601Sequence('P1D', '20190101T00Z')
tdef.xtrig_labels[sequence] = ["get_name"]
start_point = ISO8601Point('2019')
itask = TaskProxy(tdef=tdef, start_point=start_point)
itask.state.xtriggers["get_name"] = get_name
xtrigger_mgr.collate([itask])
assert xtrigger_mgr.all_xtrig
# add a clock xtrigger
# that will cause both all_xclock to be populated but not all_xtrig
wall_clock = SubFuncContext(
label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={}
)
wall_clock.out = "[\"True\", \"1\"]"
xtrigger_mgr.add_trig("wall_clock", wall_clock, "fdir")
# create a task
tdef = TaskDef(
name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False
)
tdef.xtrig_labels[sequence] = ["wall_clock"]
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask = TaskProxy(tdef=tdef, start_point=start_point)
xtrigger_mgr.collate([itask])
assert not xtrigger_mgr.all_xtrig
def test_check_xtriggers():
"""Test check_xtriggers call.
check_xtriggers does pretty much the same as collate. The
difference is that besides tallying on all the xtriggers and
clock xtriggers available, it then proceeds to trying to
satisfy them."""
xtrigger_mgr = XtriggerManager(suite="sample_suite",
user="******",
proc_pool=MockedProcPool())
# add a xtrigger
# that will cause all_xtrig to be populated, but not all_xclock
get_name = SubFuncContext(label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={})
xtrigger_mgr.add_trig("get_name", get_name)
get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef1 = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False)
tdef1.xtrig_labels.add("get_name")
start_point = ISO8601Point('20000101T0000+05')
itask1 = TaskProxy(tdef=tdef1, start_point=start_point)
itask1.state.xtriggers["get_name"] = False # satisfied?
# add a clock xtrigger
# that will cause both all_xclock to be populated but not all_xtrig
wall_clock = SubFuncContext(label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={})
wall_clock.out = "[\"True\", \"1\"]"
xtrigger_mgr.add_clock("wall_clock", wall_clock)
# create a task
tdef2 = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False)
tdef2.xclock_label = "wall_clock"
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask2 = TaskProxy(tdef=tdef2, start_point=start_point)
itask2.state.xclock = "wall_clock", False # satisfied?
xtrigger_mgr.check_xtriggers([itask1, itask2])
assert xtrigger_mgr.sat_xclock
assert xtrigger_mgr.all_xclock
# won't be satisfied, as it is async, we are are not calling callback
assert not xtrigger_mgr.sat_xtrig
assert xtrigger_mgr.all_xtrig
def test_collate():
"""Test that collate properly tallies the totals of current xtriggers."""
xtrigger_mgr = XtriggerManager(suite="sample_suite", user="******")
xtrigger_mgr.collate(itasks=[])
assert not xtrigger_mgr.all_xclock
assert not xtrigger_mgr.all_xtrig
# add a xtrigger
# that will cause all_xtrig to be populated, but not all_xclock
get_name = SubFuncContext(label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={})
xtrigger_mgr.add_trig("get_name", get_name)
get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False)
tdef.xtrig_labels.add("get_name")
start_point = ISO8601Point('20000101T0000+05')
itask = TaskProxy(tdef=tdef, start_point=start_point)
itask.state.xtriggers["get_name"] = get_name
xtrigger_mgr.collate([itask])
assert not xtrigger_mgr.all_xclock
assert xtrigger_mgr.all_xtrig
# add a clock xtrigger
# that will cause both all_xclock to be populated but not all_xtrig
wall_clock = SubFuncContext(label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={})
wall_clock.out = "[\"True\", \"1\"]"
xtrigger_mgr.add_clock("wall_clock", wall_clock)
# create a task
tdef = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False)
tdef.xclock_label = "wall_clock"
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask = TaskProxy(tdef=tdef, start_point=start_point)
itask.state.xclock = "wall_clock", True
xtrigger_mgr.collate([itask])
assert xtrigger_mgr.all_xclock
assert not xtrigger_mgr.all_xtrig
def test_check_xtriggers(xtrigger_mgr):
"""Test process_xtriggers call."""
xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation
# add an xtrigger
get_name = SubFuncContext(
label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={}
)
xtrigger_mgr.add_trig("get_name", get_name, 'fdir')
get_name.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef1 = TaskDef(
name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
initial_point=1
)
init()
sequence = ISO8601Sequence('P1D', '2019')
tdef1.xtrig_labels[sequence] = ["get_name"]
start_point = ISO8601Point('2019')
itask1 = TaskProxy(tdef1, start_point)
itask1.state.xtriggers["get_name"] = False # satisfied?
# add a clock xtrigger
wall_clock = SubFuncContext(
label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={}
)
wall_clock.out = "[\"True\", \"1\"]"
xtrigger_mgr.add_trig("wall_clock", wall_clock, "fdir")
# create a task
tdef2 = TaskDef(
name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
initial_point=1
)
tdef2.xtrig_labels[sequence] = ["wall_clock"]
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
TaskProxy(tdef2, start_point)
xtrigger_mgr.check_xtriggers(itask1, lambda foo: None)
# won't be satisfied, as it is async, we are are not calling callback
assert not xtrigger_mgr.sat_xtrig
def test_satisfy_xclock_satisfied_xclock():
"""Test satisfy_xclock for a satisfied clock trigger."""
xtrigger_mgr = XtriggerManager(suite="sample_suite", user="******")
# the clock xtrigger
xtrig = SubFuncContext(label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={})
xtrig.out = "[\"True\", \"1\"]"
xtrigger_mgr.add_clock("wall_clock", xtrig)
# create a task
tdef = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False)
tdef.xclock_label = "wall_clock"
# cycle point for task proxy
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask = TaskProxy(tdef=tdef, start_point=start_point)
itask.state.xclock = "wall_clock", True # satisfied?
# we are defining in the state of the TaskProxy. that its xclock trigger
# has been satisfied, without actually adding it to the right dict.
assert not xtrigger_mgr.sat_xclock
xtrigger_mgr.satisfy_xclock(itask)
# as it was already satisfied, the function should return immediately,
# without touching sat_xclock, therefore, it must remain empty
assert not xtrigger_mgr.sat_xclock
def test_point_match(itask_point: PointBase, point_str: Optional[str],
expected: bool, set_cycling_type: Callable) -> None:
"""Test TaskProxy.point_match()."""
set_cycling_type(itask_point.TYPE)
mock_itask = Mock(point=itask_point.standardise())
assert TaskProxy.point_match(mock_itask, point_str) is expected
def test_housekeeping_with_xtrigger_satisfied(xtrigger_mgr):
"""The housekeeping method makes sure only satisfied xtrigger function
are kept."""
xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation
xtrig = SubFuncContext(
label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={}
)
xtrigger_mgr.add_trig("get_name", xtrig, 'fdir')
xtrig.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef = TaskDef(
name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
initial_point=1
)
init()
sequence = ISO8601Sequence('P1D', '2019')
tdef.xtrig_labels[sequence] = ["get_name"]
start_point = ISO8601Point('2019')
itask = TaskProxy(tdef, start_point)
# pretend the function has been activated
xtrigger_mgr.active.append(xtrig.get_signature())
xtrigger_mgr.callback(xtrig)
assert xtrigger_mgr.sat_xtrig
xtrigger_mgr.housekeep([itask])
# here we still have the same number as before
assert xtrigger_mgr.sat_xtrig
def test_housekeeping_with_xtrigger_satisfied():
"""The housekeeping method makes sure only satisfied xtrigger function
are kept."""
xtrigger_mgr = XtriggerManager(suite="sample_suite", user="******")
xtrig = SubFuncContext(label="get_name",
func_name="get_name",
func_args=[],
func_kwargs={})
xtrigger_mgr.add_trig("get_name", xtrig)
xtrig.out = "[\"True\", {\"name\": \"Yossarian\"}]"
tdef = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False)
tdef.xtrig_labels.add("get_name")
start_point = ISO8601Point('20000101T0000+05')
itask = TaskProxy(tdef=tdef, start_point=start_point)
xtrigger_mgr.collate([itask])
# pretend the function has been activated
xtrigger_mgr.active.append(xtrig.get_signature())
xtrigger_mgr.callback(xtrig)
assert xtrigger_mgr.sat_xtrig
xtrigger_mgr.housekeep()
# here we still have the same number as before
assert xtrigger_mgr.sat_xtrig
# however, we have no xclock trigger satisfied
assert not xtrigger_mgr.sat_xclock
def test_housekeeping_with_xclock_satisfied():
"""The housekeeping method makes sure only satisfied xclock function
are kept."""
xtrigger_mgr = XtriggerManager(suite="sample_suite", user="******")
# the clock xtrigger
xtrig = SubFuncContext(label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={})
xtrig.out = "[\"True\", \"1\"]"
xtrigger_mgr.add_clock("wall_clock", xtrig)
# create a task
tdef = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False)
tdef.xclock_label = "wall_clock"
# cycle point for task proxy
# TODO: we need to call init, before we can use ISO8601 points in Cylc,
# why?
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask = TaskProxy(tdef=tdef, start_point=start_point)
itask.state.xclock = "wall_clock", False # satisfied?
# satisfy xclock
xtrigger_mgr.satisfy_xclock(itask)
# tally
xtrigger_mgr.collate([itask])
assert xtrigger_mgr.sat_xclock
xtrigger_mgr.housekeep()
# here we still have the same number as before
assert xtrigger_mgr.sat_xclock
def test_satisfy_xclock_unsatisfied_xclock():
"""Test satisfy_xclock for an unsatisfied clock trigger."""
xtrigger_mgr = XtriggerManager(suite="sample_suite", user="******")
# the clock xtrigger
xtrig = SubFuncContext(label="wall_clock",
func_name="wall_clock",
func_args=[],
func_kwargs={})
xtrig.out = "[\"True\", \"1\"]"
xtrigger_mgr.add_clock("wall_clock", xtrig)
# create a task
tdef = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False)
tdef.xclock_label = "wall_clock"
# cycle point for task proxy
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask = TaskProxy(tdef=tdef, start_point=start_point)
itask.state.xclock = "wall_clock", False # satisfied?
# we are defining in the state of the TaskProxy. that its xclock trigger
# has **not** been satisfied.
assert not xtrigger_mgr.sat_xclock
xtrigger_mgr.satisfy_xclock(itask)
# as it was satisfied by the satisfy_clock function, we must have its
# signature in the dict. NB the signature is not the same as
# get_signature(), as it is actually the signature for that moment
# when it was satisfied.
assert xtrigger_mgr.sat_xclock
def test_get_clock_trigger_time(itask_point: PointBase, offset_str: str,
expected: int,
set_cycling_type: Callable) -> None:
"""Test get_clock_trigger_time() for exact and inexact offsets."""
set_cycling_type(itask_point.TYPE)
mock_itask = Mock(point=itask_point.standardise(), clock_trigger_time=None)
assert TaskProxy.get_clock_trigger_time(mock_itask, offset_str) == expected
def test__call_xtriggers_async(xtrigger_mgr):
"""Test _call_xtriggers_async"""
xtrigger_mgr.validate_xtrigger = lambda *a, **k: True # Ignore validation
# the echo1 xtrig (not satisfied)
echo1_xtrig = SubFuncContext(label="echo1",
func_name="echo1",
func_args=[],
func_kwargs={})
echo1_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]"
xtrigger_mgr.add_trig("echo1", echo1_xtrig, "fdir")
# the echo2 xtrig (satisfied through callback later)
echo2_xtrig = SubFuncContext(label="echo2",
func_name="echo2",
func_args=[],
func_kwargs={})
echo2_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]"
xtrigger_mgr.add_trig("echo2", echo2_xtrig, "fdir")
# create a task
tdef = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
initial_point=1)
init()
sequence = ISO8601Sequence('P1D', '2000')
tdef.xtrig_labels[sequence] = ["echo1", "echo2"]
# cycle point for task proxy
init()
start_point = ISO8601Point('2019')
# create task proxy
itask = TaskProxy(tdef, start_point, FlowLabelMgr().get_new_label())
# we start with no satisfied xtriggers, and nothing active
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 0
# after calling the first time, we get two active
xtrigger_mgr.call_xtriggers_async(itask)
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 2
# calling again does not change anything
xtrigger_mgr.call_xtriggers_async(itask)
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 2
# now we call callback manually as the proc_pool we passed is a mock
# then both should be satisfied
xtrigger_mgr.callback(echo1_xtrig)
xtrigger_mgr.callback(echo2_xtrig)
# so both were satisfied, and nothing is active
assert len(xtrigger_mgr.sat_xtrig) == 2
assert len(xtrigger_mgr.active) == 0
# calling satisfy_xtriggers again still does not change anything
xtrigger_mgr.call_xtriggers_async(itask)
assert len(xtrigger_mgr.sat_xtrig) == 2
assert len(xtrigger_mgr.active) == 0
def test_status_match(status_str: Optional[str], expected: bool):
"""Test TaskProxy.status_match().
For a task with status "waiting".
"""
mock_itask = Mock(state=Mock(status='waiting'))
assert TaskProxy.status_match(mock_itask, status_str) is expected
def test_name_match(name_str: str, expected: bool):
"""Test TaskProxy.name_match().
For a task named "beer" in family "FAM".
"""
mock_tdef = Mock(namespace_hierarchy=['root', 'FAM', 'beer'])
mock_tdef.name = 'beer'
mock_itask = Mock(tdef=mock_tdef)
assert TaskProxy.name_match(mock_itask, name_str) is expected
def call_xtriggers_async(self, itask: TaskProxy):
"""Call itask's xtrigger functions via the process pool...
...if previous call not still in-process and retry period is up.
Args:
itask: task proxy to check.
"""
for label, sig, ctx, _ in self._get_xtrigs(itask, unsat_only=True):
if sig.startswith("wall_clock"):
# Special case: quick synchronous clock check.
if 'absolute_as_seconds' not in ctx.func_kwargs:
ctx.func_kwargs.update(
{
'point_as_seconds': itask.get_point_as_seconds()
}
)
if wall_clock(*ctx.func_args, **ctx.func_kwargs):
itask.state.xtriggers[label] = True
self.sat_xtrig[sig] = {}
self.data_store_mgr.delta_task_xtrigger(sig, True)
LOG.info('xtrigger satisfied: %s = %s', label, sig)
continue
# General case: potentially slow asynchronous function call.
if sig in self.sat_xtrig:
if not itask.state.xtriggers[label]:
itask.state.xtriggers[label] = True
res = {}
for key, val in self.sat_xtrig[sig].items():
res["%s_%s" % (label, key)] = val
if res:
xtrigger_env = [{'environment': {key: val}} for
key, val in res.items()]
self.broadcast_mgr.put_broadcast(
[str(itask.point)],
[itask.tdef.name],
xtrigger_env
)
continue
if sig in self.active:
# Already waiting on this result.
continue
now = time()
if sig in self.t_next_call and now < self.t_next_call[sig]:
# Too soon to call this one again.
continue
self.t_next_call[sig] = now + ctx.intvl
# Queue to the process pool, and record as active.
self.active.append(sig)
self.proc_pool.put_command(ctx, self.callback)
def create_task_proxy(task_name: str,
suite_config: SuiteConfig,
is_startup=False) -> TaskProxy:
"""Create a Task Proxy based on a TaskDef loaded from the SuiteConfig.
Args:
task_name (str): task name
suite_config (SuiteConfig): SuiteConfig object that holds task
definitions
is_startup (bool): whether we are starting the workflow or not
"""
task_def = suite_config.get_taskdef(task_name)
return TaskProxy(tdef=task_def,
start_point=suite_config.start_point,
is_startup=is_startup)
def get_xtrig_ctx(self, itask: TaskProxy, label: str) -> SubFuncContext:
"""Get a real function context from the template.
Args:
itask: task proxy
label: xtrigger label
Returns:
function context
"""
farg_templ = {
TemplateVariables.CyclePoint.value: str(itask.point),
TemplateVariables.TaskName.value: str(itask.tdef.name),
TemplateVariables.TaskID.value: str(itask.identity)
}
farg_templ.update(self.farg_templ)
ctx = deepcopy(self.functx_map[label])
args = []
kwargs = {}
if ctx.func_name == "wall_clock":
if "trigger_time" in ctx.func_kwargs:
# Internal (retry timer): trigger_time already set.
kwargs["trigger_time"] = ctx.func_kwargs["trigger_time"]
elif "offset" in ctx.func_kwargs: # noqa: SIM106
# External (clock xtrigger): convert offset to trigger_time.
# Datetime cycling only.
kwargs["trigger_time"] = itask.get_clock_trigger_time(
ctx.func_kwargs["offset"])
else:
# Should not happen!
raise ValueError(
"wall_clock function kwargs needs trigger time or offset")
else:
# Other xtrig functions: substitute template values.
for val in ctx.func_args:
with suppress(TypeError):
val = val % farg_templ
args.append(val)
for key, val in ctx.func_kwargs.items():
with suppress(TypeError):
val = val % farg_templ
kwargs[key] = val
ctx.func_args = args
ctx.func_kwargs = kwargs
ctx.update_command(self.workflow_run_dir)
return ctx
def satisfy_xtriggers(self, itask: TaskProxy):
"""Attempt to satisfy itask's xtriggers.
Args:
itask (TaskProxy): TaskProxy
"""
for label, sig, ctx, _ in self._get_xtrigs(itask, unsat_only=True):
if sig.startswith("wall_clock"):
# Special case: synchronous clock check.
ctx.func_kwargs.update({
'point_as_seconds':
itask.get_point_as_seconds(),
})
if wall_clock(*ctx.func_args, **ctx.func_kwargs):
itask.state.xtriggers[label] = True
self.sat_xtrig[sig] = {}
LOG.info('xtrigger satisfied: %s = %s', label, sig)
continue
# General case: asynchronous xtrigger function call.
if sig in self.sat_xtrig:
if not itask.state.xtriggers[label]:
itask.state.xtriggers[label] = True
res = {}
for key, val in self.sat_xtrig[sig].items():
res["%s_%s" % (label, key)] = val
if res:
xtrigger_env = [{
'environment': {
key: val
}
} for key, val in res.items()]
self.broadcast_mgr.put_broadcast([str(ctx.point)],
[itask.tdef.name],
xtrigger_env)
continue
if sig in self.active:
# Already waiting on this result.
continue
now = time()
if sig in self.t_next_call and now < self.t_next_call[sig]:
# Too soon to call this one again.
continue
self.t_next_call[sig] = now + ctx.intvl
# Queue to the process pool, and record as active.
self.active.append(sig)
self.proc_pool.put_command(ctx, self.callback)
def _get_xclock(self, itask: TaskProxy, sig_only: bool = False) ->\
Union[str, Tuple[str, str, SubFuncContext, bool]]:
"""(Internal helper method.)
Args:
itask (TaskProxy): TaskProxy
sig_only (bool): whether to return the signature only or not
Returns:
Union[str, Tuple[str, str, SubFuncContext, bool]]: the signature
of the function (if sigs_only True) or a tuple with
label, signature, function context, and flag for satisfied.
"""
label, satisfied = itask.state.xclock
ctx = deepcopy(self.clockx_map[label])
ctx.func_kwargs.update(
{
'point_as_seconds': itask.get_point_as_seconds(),
}
)
sig = ctx.get_signature()
if sig_only:
return sig
else:
return label, sig, ctx, satisfied
def main(_, options, reg):
"""cylc validate CLI."""
profiler = Profiler(None, options.profile_mode)
profiler.start()
if not cylc.flow.flags.debug:
# for readability omit timestamps from logging unless in debug mode
for handler in LOG.handlers:
if isinstance(handler.formatter, CylcLogFormatter):
handler.formatter.configure(timestamp=False)
suite, flow_file = parse_suite_arg(options, reg)
cfg = SuiteConfig(
suite,
flow_file,
options,
load_template_vars(options.templatevars, options.templatevars_file),
output_fname=options.output, mem_log_func=profiler.log_memory)
# Check bounds of sequences
out_of_bounds = [str(seq) for seq in cfg.sequences
if seq.get_first_point(cfg.start_point) is None]
if out_of_bounds:
if len(out_of_bounds) > 1:
# avoid spamming users with multiple warnings
msg = ('multiple sequences out of bounds for initial cycle point '
'%s:\n%s' % (
cfg.start_point,
'\n'.join(textwrap.wrap(', '.join(out_of_bounds), 70))))
else:
msg = '%s: sequence out of bounds for initial cycle point %s' % (
out_of_bounds[0], cfg.start_point)
if options.strict:
LOG.warning(msg)
elif cylc.flow.flags.verbose:
sys.stderr.write(' + %s\n' % msg)
# Instantiate tasks and force evaluation of trigger expressions.
# (Taken from config.py to avoid circular import problems.)
# TODO - This is not exhaustive, it only uses the initial cycle point.
if cylc.flow.flags.verbose:
print('Instantiating tasks to check trigger expressions')
flow_label = FlowLabelMgr().get_new_label()
for name, taskdef in cfg.taskdefs.items():
try:
itask = TaskProxy(taskdef, cfg.start_point, flow_label)
except TaskProxySequenceBoundsError:
# Should already failed above in strict mode.
mesg = 'Task out of bounds for %s: %s\n' % (cfg.start_point, name)
if cylc.flow.flags.verbose:
sys.stderr.write(' + %s\n' % mesg)
continue
except Exception as exc:
raise SuiteConfigError(
'failed to instantiate task %s: %s' % (name, exc))
# force trigger evaluation now
try:
itask.state.prerequisites_eval_all()
except TriggerExpressionError as exc:
err = str(exc)
if '@' in err:
print(f"ERROR, {name}: xtriggers can't be in conditional"
f" expressions: {err}",
file=sys.stderr)
else:
print('ERROR, %s: bad trigger: %s' % (name, err),
file=sys.stderr)
raise SuiteConfigError("ERROR: bad trigger")
except Exception as exc:
print(str(exc), file=sys.stderr)
raise SuiteConfigError(
'%s: failed to evaluate triggers.' % name)
if cylc.flow.flags.verbose:
print(' + %s ok' % itask.identity)
print(cparse('<green>Valid for cylc-%s</green>' % CYLC_VERSION))
profiler.stop()
def test_satisfy_xtrigger(xtrigger_mgr_procpool_broadcast):
"""Test satisfy_xtriggers"""
# the echo1 xtrig (not satisfied)
echo1_xtrig = SubFuncContext(
label="echo1",
func_name="echo1",
func_args=[],
func_kwargs={}
)
echo1_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]"
xtrigger_mgr_procpool_broadcast.add_trig("echo1", echo1_xtrig, "fdir")
# the echo2 xtrig (satisfied through callback later)
echo2_xtrig = SubFuncContext(
label="echo2",
func_name="echo2",
func_args=[],
func_kwargs={}
)
echo2_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]"
xtrigger_mgr_procpool_broadcast.add_trig("echo2", echo2_xtrig, "fdir")
# create a task
tdef = TaskDef(
name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False
)
init()
sequence = ISO8601Sequence('P1D', '2000')
tdef.xtrig_labels[sequence] = ["echo1", "echo2"]
# cycle point for task proxy
init()
start_point = ISO8601Point('2019')
# create task proxy
itask = TaskProxy(tdef=tdef, start_point=start_point)
# we start with no satisfied xtriggers, and nothing active
assert len(xtrigger_mgr_procpool_broadcast.sat_xtrig) == 0
assert len(xtrigger_mgr_procpool_broadcast.active) == 0
# after calling satisfy_xtriggers the first time, we get two active
xtrigger_mgr_procpool_broadcast.satisfy_xtriggers(itask)
assert len(xtrigger_mgr_procpool_broadcast.sat_xtrig) == 0
assert len(xtrigger_mgr_procpool_broadcast.active) == 2
# calling satisfy_xtriggers again does not change anything
xtrigger_mgr_procpool_broadcast.satisfy_xtriggers(itask)
assert len(xtrigger_mgr_procpool_broadcast.sat_xtrig) == 0
assert len(xtrigger_mgr_procpool_broadcast.active) == 2
# now we call callback manually as the proc_pool we passed is a mock
# then both should be satisfied
xtrigger_mgr_procpool_broadcast.callback(echo1_xtrig)
xtrigger_mgr_procpool_broadcast.callback(echo2_xtrig)
# so both were satisfied, and nothing is active
assert len(xtrigger_mgr_procpool_broadcast.sat_xtrig) == 2
assert len(xtrigger_mgr_procpool_broadcast.active) == 0
# calling satisfy_xtriggers again still does not change anything
xtrigger_mgr_procpool_broadcast.satisfy_xtriggers(itask)
assert len(xtrigger_mgr_procpool_broadcast.sat_xtrig) == 2
assert len(xtrigger_mgr_procpool_broadcast.active) == 0
def test_satisfy_xtrigger():
"""Test satisfy_xtriggers"""
# the XtriggerManager instance
xtrigger_mgr = XtriggerManager(
suite="sample_suite",
user="******",
proc_pool=MockedProcPool(),
broadcast_mgr=MockedBroadcastMgr(suite_db_mgr=None))
# the echo1 xtrig (not satisfied)
echo1_xtrig = SubFuncContext(label="echo1",
func_name="echo1",
func_args=[],
func_kwargs={})
echo1_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]"
xtrigger_mgr.add_trig("echo1", echo1_xtrig)
# the echo2 xtrig (satisfied through callback later)
echo2_xtrig = SubFuncContext(label="echo2",
func_name="echo2",
func_args=[],
func_kwargs={})
echo2_xtrig.out = "[\"True\", {\"name\": \"herminia\"}]"
xtrigger_mgr.add_trig("echo2", echo2_xtrig)
# create a task
tdef = TaskDef(name="foo",
rtcfg=None,
run_mode="live",
start_point=1,
spawn_ahead=False)
tdef.xtrig_labels.add("echo1")
tdef.xtrig_labels.add("echo2")
# cycle point for task proxy
init()
start_point = ISO8601Point('20000101T0000+05')
# create task proxy
itask = TaskProxy(tdef=tdef, start_point=start_point)
# we start with no satisfied xtriggers, and nothing active
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 0
# after calling satisfy_xtriggers the first time, we get two active
xtrigger_mgr.satisfy_xtriggers(itask)
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 2
# calling satisfy_xtriggers again does not change anything
xtrigger_mgr.satisfy_xtriggers(itask)
assert len(xtrigger_mgr.sat_xtrig) == 0
assert len(xtrigger_mgr.active) == 2
# now we call callback manually as the proc_pool we passed is a mock
# then both should be satisfied
xtrigger_mgr.callback(echo1_xtrig)
xtrigger_mgr.callback(echo2_xtrig)
# so both were satisfied, and nothing is active
assert len(xtrigger_mgr.sat_xtrig) == 2
assert len(xtrigger_mgr.active) == 0
# calling satisfy_xtriggers again still does not change anything
xtrigger_mgr.satisfy_xtriggers(itask)
assert len(xtrigger_mgr.sat_xtrig) == 2
assert len(xtrigger_mgr.active) == 0
def main(parser: COP, options: 'Values', workflow_id: str) -> None:
"""cylc validate CLI."""
profiler = Profiler(None, options.profile_mode)
profiler.start()
if cylc.flow.flags.verbosity < 2:
disable_timestamps(LOG)
workflow_id, _, flow_file = parse_id(
workflow_id,
src=True,
constraint='workflows',
)
cfg = WorkflowConfig(workflow_id,
flow_file,
options,
get_template_vars(options),
output_fname=options.output,
mem_log_func=profiler.log_memory)
# Instantiate tasks and force evaluation of trigger expressions.
# (Taken from config.py to avoid circular import problems.)
# TODO - This is not exhaustive, it only uses the initial cycle point.
if cylc.flow.flags.verbosity > 0:
print('Instantiating tasks to check trigger expressions')
for name, taskdef in cfg.taskdefs.items():
try:
itask = TaskProxy(taskdef, cfg.start_point)
except TaskProxySequenceBoundsError:
# Should already failed above
mesg = 'Task out of bounds for %s: %s\n' % (cfg.start_point, name)
if cylc.flow.flags.verbosity > 0:
sys.stderr.write(' + %s\n' % mesg)
continue
except Exception as exc:
raise WorkflowConfigError('failed to instantiate task %s: %s' %
(name, exc))
# force trigger evaluation now
try:
itask.state.prerequisites_eval_all()
except TriggerExpressionError as exc:
err = str(exc)
if '@' in err:
print(
f"ERROR, {name}: xtriggers can't be in conditional"
f" expressions: {err}",
file=sys.stderr)
else:
print('ERROR, %s: bad trigger: %s' % (name, err),
file=sys.stderr)
raise WorkflowConfigError("ERROR: bad trigger")
except Exception as exc:
print(str(exc), file=sys.stderr)
raise WorkflowConfigError('%s: failed to evaluate triggers.' %
name)
if cylc.flow.flags.verbosity > 0:
print(' + %s ok' % itask.identity)
print(cparse('<green>Valid for cylc-%s</green>' % CYLC_VERSION))
profiler.stop()
def main(parser, options, suite, *task_ids):
"""cylc submit CLI.
No TASK EVENT HOOKS are set for the submit command because there is
no scheduler instance watching for task failure etc.
Note: a suite contact env file is not written by this command (it
would overwrite the real one if the suite is running).
"""
if not options.verbose and not options.debug:
LOG.setLevel(WARNING)
for task_id in task_ids:
if not TaskID.is_valid_id(task_id):
raise UserInputError("Invalid task ID %s" % task_id)
suiterc = get_suite_rc(suite)
suite_dir = os.path.dirname(suiterc)
# For user-defined batch system handlers
sys.path.append(os.path.join(suite_dir, 'python'))
# Load suite config and tasks
config = SuiteConfig(
suite, suiterc, options,
load_template_vars(options.templatevars, options.templatevars_file))
itasks = []
for task_id in task_ids:
name_str, point_str = TaskID.split(task_id)
taskdefs = config.find_taskdefs(name_str)
if not taskdefs:
raise UserInputError("No task found for %s" % task_id)
for taskdef in taskdefs:
itasks.append(
TaskProxy(taskdef,
get_point(point_str).standardise(),
is_startup=True))
# Initialise job submit environment
make_suite_run_tree(suite)
# Extract job.sh from library, for use in job scripts.
extract_resources(get_suite_srv_dir(suite), ['etc/job.sh'])
pool = SubProcPool()
owner = get_user()
job_pool = JobPool(suite, owner)
db_mgr = SuiteDatabaseManager()
task_job_mgr = TaskJobManager(
suite, pool, db_mgr,
TaskEventsManager(suite, pool, db_mgr, BroadcastMgr(db_mgr), job_pool),
job_pool)
task_job_mgr.task_remote_mgr.single_task_mode = True
task_job_mgr.job_file_writer.set_suite_env({
'CYLC_UTC':
str(config.cfg['cylc']['UTC mode']),
'CYLC_DEBUG':
str(cylc.flow.flags.debug).lower(),
'CYLC_VERBOSE':
str(cylc.flow.flags.verbose).lower(),
'CYLC_SUITE_NAME':
suite,
'CYLC_CYCLING_MODE':
str(config.cfg['scheduling']['cycling mode']),
'CYLC_SUITE_INITIAL_CYCLE_POINT':
str(config.cfg['scheduling']['initial cycle point']),
'CYLC_SUITE_FINAL_CYCLE_POINT':
str(config.cfg['scheduling']['final cycle point']),
})
ret_code = 0
waiting_tasks = list(itasks)
if options.dry_run:
while waiting_tasks:
prep_tasks, bad_tasks = task_job_mgr.prep_submit_task_jobs(
suite, waiting_tasks, dry_run=True)
for itask in prep_tasks + bad_tasks:
waiting_tasks.remove(itask)
if waiting_tasks:
task_job_mgr.proc_pool.process()
sleep(1.0)
for itask in itasks:
if itask.local_job_file_path:
print(('JOB SCRIPT=%s' % itask.local_job_file_path))
else:
print(('Unable to prepare job file for %s' % itask.identity),
file=sys.stderr)
ret_code = 1
else:
while waiting_tasks:
for itask in task_job_mgr.submit_task_jobs(suite, waiting_tasks):
waiting_tasks.remove(itask)
if waiting_tasks:
task_job_mgr.proc_pool.process()
sleep(1.0)
while task_job_mgr.proc_pool.is_not_done():
task_job_mgr.proc_pool.process()
for itask in itasks:
if itask.summary.get('submit_method_id') is not None:
print(('[%s] Job ID: %s' %
(itask.identity, itask.summary['submit_method_id'])))
if itask.state(TASK_STATUS_SUBMIT_FAILED):
ret_code = 1
sys.exit(ret_code)
def main(parser: COP, options: 'Values', reg: str) -> None:
"""cylc validate CLI."""
profiler = Profiler(None, options.profile_mode)
profiler.start()
if cylc.flow.flags.verbosity < 2:
disable_timestamps(LOG)
workflow, flow_file = parse_reg(reg, src=True)
cfg = WorkflowConfig(
workflow,
flow_file,
options,
get_template_vars(options),
output_fname=options.output,
mem_log_func=profiler.log_memory
)
# Check bounds of sequences
out_of_bounds = [str(seq) for seq in cfg.sequences
if seq.get_first_point(cfg.start_point) is None]
if out_of_bounds:
if len(out_of_bounds) > 1:
# avoid spamming users with multiple warnings
out_of_bounds_str = '\n'.join(
textwrap.wrap(', '.join(out_of_bounds), 70))
msg = (
"multiple sequences out of bounds for initial cycle point "
f"{cfg.start_point}:\n{out_of_bounds_str}")
else:
msg = (
f"{out_of_bounds[0]}: sequence out of bounds for "
f"initial cycle point {cfg.start_point}")
LOG.warning(msg)
# Instantiate tasks and force evaluation of trigger expressions.
# (Taken from config.py to avoid circular import problems.)
# TODO - This is not exhaustive, it only uses the initial cycle point.
if cylc.flow.flags.verbosity > 0:
print('Instantiating tasks to check trigger expressions')
for name, taskdef in cfg.taskdefs.items():
try:
itask = TaskProxy(taskdef, cfg.start_point)
except TaskProxySequenceBoundsError:
# Should already failed above
mesg = 'Task out of bounds for %s: %s\n' % (cfg.start_point, name)
if cylc.flow.flags.verbosity > 0:
sys.stderr.write(' + %s\n' % mesg)
continue
except Exception as exc:
raise WorkflowConfigError(
'failed to instantiate task %s: %s' % (name, exc))
# force trigger evaluation now
try:
itask.state.prerequisites_eval_all()
except TriggerExpressionError as exc:
err = str(exc)
if '@' in err:
print(f"ERROR, {name}: xtriggers can't be in conditional"
f" expressions: {err}",
file=sys.stderr)
else:
print('ERROR, %s: bad trigger: %s' % (name, err),
file=sys.stderr)
raise WorkflowConfigError("ERROR: bad trigger")
except Exception as exc:
print(str(exc), file=sys.stderr)
raise WorkflowConfigError(
'%s: failed to evaluate triggers.' % name)
if cylc.flow.flags.verbosity > 0:
print(' + %s ok' % itask.identity)
print(cparse('<green>Valid for cylc-%s</green>' % CYLC_VERSION))
profiler.stop()
