def family_limit():
""" example """
return (
Family("limits").add(
Limits({"total": 15,
"prio": 10,
"other": 20}), # use dictionnary
Defstatus("complete")),
Limit("record", 50), # record active/submit tasks
Inlimit("record"),
Limit("total", 2), # limiter
Family("limit").add(
Defstatus("complete"),
Family("prio").add( # on top: submitted first
Inlimit("../limits:prio"),
[Family("%03d" % step).add(
Task("process"), Variables(STEP=step))
for step in range(0, 120 + 1, 3)]),
Family("side").add( # below: take remaining tokens
Inlimit("../limits:other"),
[Family("%03d" % step).add(
Task("process"), Variables(STEP=step))
for step in range(0, 120, 3)])))
def family_limiter():
""" alternative example """
return (
Family("limiter").add(
Limit("total", 10),
Inlimit("total"),
Task("alarm").add(
Complete("limits eq complete"),
Trigger("../limiter:total gt 8")), # relative path
Family("limits").add(
Defstatus("complete"),
Family("weaker").add( # weaker is located above, not to be shadowed
Trigger("../limiter:total le 10"),
[Family("%03d" % step).add(
Task("process"), Variables(STEP=step))
for step in range(0, 120, 3)]),
Family("prio").add( # favourite shall not lead weaker to starve
Trigger("../limiter:total le 15"),
[Family("%03d" % step).add(
Task("process"), Variables(STEP=step))
for step in range(0, 120, 3)]))))
def family_for():
""" for family """
return (
Family("for").add(
process(),
Repeat(kind="integer", name="STEP", start=1, end=240, step=3)),
Family("loop").add(process(), Repeat("PARAM", PARAMS, kind="string")),
Family("parallel").add(Limit("lim", 2), Inlimit("lim"), [
Family(param).add(Edit(PARAM=param),
process().add(Label("info", param)))
for param in PARAMS
]),
Family("explode").add(
Limit("lim", 2),
Inlimit("lim"),
# LIST COMPREHENSION:
[Task("t%d" % num) for num in range(1, 5 + 1)]))
#!/usr/bin/env python2.7
""" back archive example """
from __future__ import print_function
import os
import ecf as ecflow
from ecf import (Defs, Defstatus, Suite, Family, Task, Variables, Label,
Limit, Inlimit, Repeat, Trigger)
HOME = os.getenv("HOME") + "/ecflow_server"
NAME = "back_archiving"; FILES = HOME + "/back"; DEFS = Defs()
DEFS.add(Suite(NAME).add(
Defstatus("suspended"), # so that jobs do not start immediately
Repeat(kind="day", step=1),
Variables(ECF_HOME=HOME, ECF_INCLUDE=HOME, ECF_FILES=FILES, SLEEP=2),
Limit("access", 2),
[Family(kind).add(
Repeat("DATE", 19900101, 19950712, kind="date"),
Variables(KIND=kind),
Task("get_old").add(Inlimit("access"), Label("info", "")),
Task("convert").add(Trigger("get_old == complete")),
Task("save_new").add(Trigger("convert eq complete"))
) for kind in ("analysis", "forecast", "climatology", "observations", "images")]))
# print(DEFS); DEFS.generate_scripts();
RESULT = DEFS.simulate(); print(RESULT)
CLIENT = ecflow.Client(os.getenv("ECF_HOST", "localhost"),
os.getenv("ECF_PORT", 2500))
CLIENT.replace("/%s" % NAME, DEFS)
def call_consumer(selection):
lead = "/%s/consumer/admin/leader:1" % selection
prod = "/%s/consumer/produce" % selection
return Family("consumer").add(
Defstatus("suspended"),
Edit(SLEEP=10,
PRODUCE="no",
CONSUME="no"),
Family("limit").add(
Defstatus("complete"),
Limit("consume", 7),),
Family("admin").add(
# set manually with the GUI or alter the event 1 so
# that producer 1 becomes leader
# default: producer0 leads
Task("leader").add(
Event("1"), # text this task is dummy task not designed to run
Defstatus("complete"))),
Edit(PRODUCE="yes", # default : task does both produce/consume
CONSUME="yes"),
call_task("produce", beg, fin, by).add(
# this task will do both produde/consume serially
Label("info", "both produce and consume in one task")),
Family("produce0").add(
# loop inside the task, report progress with a Meter
not_consumer(),
Label("info", "loop inside the job"),
call_task("produce", beg, fin, by)),
Family("produce1").add(
# choice is to submit a new job for each step, here
Label("info", "loop in the suite (repeat), submit one job per step"),
not_consumer(),
call_task("produce", '%STEP%', "%STEP%", by).add(
Repeat("STEP", beg, fin, by, kind="integer"))),
Family("consume").add(
Label("info", "use ecflow_client --wait %TRIGGER% in the job"),
not_producer(),
Inlimit("limit:consume"),
Edit(CALL_WAITER=1,
# $step will be interpreted in the job!
TRIGGER="../produce:step gt consume:$step or " +
"../produce eq complete"),
call_task("consume", beg, fin, by)),
Family("consume0or1").add(
Label("info", "an event may indicate the leader to trigger from"),
not_producer(),
Inlimit("limit:consume"),
call_task("consume", "%STEP%", "%STEP%", by),
Repeat("STEP", beg, fin, by, kind="integer"),
trigger("(%s and (consume0or1:STEP lt %s1/produce:STEP" % (
lead, prod) + " or %s==complete)) or " % prod +
"(not %s and (consume0or1:STEP lt %s0/produce:step" % (
lead, prod) + " or %s0/produce==complete))" % prod)),
Family("consume1").add(
Label("info", "spread consumer in multiple families"),
not_producer(),
Inlimit("limit:consume"),
consume1(producer=prod)),
Family("consume2").add(
# change manually with the GUI the limit to reduce/increase the load
Label("info", "one family per step to consume"),
Inlimit("limit:consume"),
not_producer(),
consume2(beg, fin, prod)))
def create_family_f5():
return Family("f5").add(Limit(
"l1", 2), Inlimit("l1"), Variables(SLEEP=2), [
Task("t%d" % idn).add(Late("-s 00:03 -a 00:10"))
for idn in xrange(1, 10)
])
def create_family_f5():
""" provider """
return Family("f5").add(Limit("l1", 2), Inlimit("l1"), Variables(SLEEP=2),
[Task("t%d" % tid) for tid in range(1, 10)])