def family_limit(): """ example """ return ( Family("limits").add( Limits({"total": 15, "prio": 10, "other": 20}), # use dictionnary Defstatus("complete")), Limit("record", 50), # record active/submit tasks Inlimit("record"), Limit("total", 2), # limiter Family("limit").add( Defstatus("complete"), Family("prio").add( # on top: submitted first Inlimit("../limits:prio"), [Family("%03d" % step).add( Task("process"), Variables(STEP=step)) for step in range(0, 120 + 1, 3)]), Family("side").add( # below: take remaining tokens Inlimit("../limits:other"), [Family("%03d" % step).add( Task("process"), Variables(STEP=step)) for step in range(0, 120, 3)])))
def family_case(): """ case block as a family example """ return ( Family("case_var").add( Task("case").add( Defstatus("complete"), Edit(VAR=1)), Task("when_1").add( Trigger("case:VAR == 1"), Complete("case:VAR != 1")), Task("when_2").add( Trigger("case:VAR eq 2"), Complete("case:VAR ne 2"))), Family("case_meter").add( Task("case").add( Meter("STEP", -1, 48)), Task("when_1").add( Trigger("case:STEP eq 1"), Complete("case==complete")), Task("when_2").add( Trigger("case:STEP eq 2"), Complete("case eq complete"))))
def create(name=os.getenv("SUITE", "elearning")): return Suite(name).add( Defstatus("suspended"), # start immediately or not for this demo Clock("real"), Edit(ECF_INCLUDE=HOME, # header files ECF_FILES=HOME, # script template .ecf ECF_HOME=HOME), # job + local output files Family("f1").add( Task("t1").add( Event(1), Label("info", ""), Meter("step", -1, 100)), Task("t2").add( Late("-c 01:00"), Meter("step", -1, 100), Event("a"), Event("b"), Trigger("t1:step gt 0")), Task("t3").add( Trigger("t2:a")), Task("t4").add( Complete("t2:b"), Trigger("t2 eq complete and not t2:b"))), Family("f2").add( Task("t1").add( Time("00:30 23:30 00:30")), Task("t2").add( Day("sunday")), Task("t3").add( Time("12:00"), Date("1.*.*")), Task("t4").add( Time("+00:02")), Task("t5").add( Time("00:02"))))
def family_limiter(): """ alternative example """ return ( Family("limiter").add( Limit("total", 10), Inlimit("total"), Task("alarm").add( Complete("limits eq complete"), Trigger("../limiter:total gt 8")), # relative path Family("limits").add( Defstatus("complete"), Family("weaker").add( # weaker is located above, not to be shadowed Trigger("../limiter:total le 10"), [Family("%03d" % step).add( Task("process"), Variables(STEP=step)) for step in range(0, 120, 3)]), Family("prio").add( # favourite shall not lead weaker to starve Trigger("../limiter:total le 15"), [Family("%03d" % step).add( Task("process"), Variables(STEP=step)) for step in range(0, 120, 3)]))))
def create(name): """ suite provider """ return Suite(name).add( Defstatus("suspended"), Edit(ECF_INCLUDE=HOME, # header files ECF_FILES=HOME, # script template .ecf ECF_HOME=HOME), # job + local output files Family("f1").add( Task("t1").add( Label("info", ""), Late("-c 01:00"), Meter("step", -1, 100)), Task("t2").add( Meter("step", -1, 100), Event("a"), Event("b"), Trigger("t1:step gt 0")), Task("t3").add( Trigger("t2:a")), Task("t4").add( Complete("t2:b"), Trigger("t2 eq complete and not t2:b"))), Family("f2").add( Task("t1").add( Time("00:30 23:30 00:30")), Task("t2").add( Day("sunday")), Task("t3").add( Time("12:00"), Date("1.*.*")), Task("t4").add( Time("+00:02")), Task("t5").add( Time("00:02"))))
#!/usr/bin/env python2.7 """ back archive example """ from __future__ import print_function import os import ecf as ecflow from ecf import (Defs, Defstatus, Suite, Family, Task, Variables, Label, Limit, Inlimit, Repeat, Trigger) HOME = os.getenv("HOME") + "/ecflow_server" NAME = "back_archiving"; FILES = HOME + "/back"; DEFS = Defs() DEFS.add(Suite(NAME).add( Defstatus("suspended"), # so that jobs do not start immediately Repeat(kind="day", step=1), Variables(ECF_HOME=HOME, ECF_INCLUDE=HOME, ECF_FILES=FILES, SLEEP=2), Limit("access", 2), [Family(kind).add( Repeat("DATE", 19900101, 19950712, kind="date"), Variables(KIND=kind), Task("get_old").add(Inlimit("access"), Label("info", "")), Task("convert").add(Trigger("get_old == complete")), Task("save_new").add(Trigger("convert eq complete")) ) for kind in ("analysis", "forecast", "climatology", "observations", "images")])) # print(DEFS); DEFS.generate_scripts(); RESULT = DEFS.simulate(); print(RESULT) CLIENT = ecflow.Client(os.getenv("ECF_HOST", "localhost"), os.getenv("ECF_PORT", 2500)) CLIENT.replace("/%s" % NAME, DEFS)
def call_consumer(selection): lead = "/%s/consumer/admin/leader:1" % selection prod = "/%s/consumer/produce" % selection return Family("consumer").add( Defstatus("suspended"), Edit(SLEEP=10, PRODUCE="no", CONSUME="no"), Family("limit").add( Defstatus("complete"), Limit("consume", 7),), Family("admin").add( # set manually with the GUI or alter the event 1 so # that producer 1 becomes leader # default: producer0 leads Task("leader").add( Event("1"), # text this task is dummy task not designed to run Defstatus("complete"))), Edit(PRODUCE="yes", # default : task does both produce/consume CONSUME="yes"), call_task("produce", beg, fin, by).add( # this task will do both produde/consume serially Label("info", "both produce and consume in one task")), Family("produce0").add( # loop inside the task, report progress with a Meter not_consumer(), Label("info", "loop inside the job"), call_task("produce", beg, fin, by)), Family("produce1").add( # choice is to submit a new job for each step, here Label("info", "loop in the suite (repeat), submit one job per step"), not_consumer(), call_task("produce", '%STEP%', "%STEP%", by).add( Repeat("STEP", beg, fin, by, kind="integer"))), Family("consume").add( Label("info", "use ecflow_client --wait %TRIGGER% in the job"), not_producer(), Inlimit("limit:consume"), Edit(CALL_WAITER=1, # $step will be interpreted in the job! TRIGGER="../produce:step gt consume:$step or " + "../produce eq complete"), call_task("consume", beg, fin, by)), Family("consume0or1").add( Label("info", "an event may indicate the leader to trigger from"), not_producer(), Inlimit("limit:consume"), call_task("consume", "%STEP%", "%STEP%", by), Repeat("STEP", beg, fin, by, kind="integer"), trigger("(%s and (consume0or1:STEP lt %s1/produce:STEP" % ( lead, prod) + " or %s==complete)) or " % prod + "(not %s and (consume0or1:STEP lt %s0/produce:step" % ( lead, prod) + " or %s0/produce==complete))" % prod)), Family("consume1").add( Label("info", "spread consumer in multiple families"), not_producer(), Inlimit("limit:consume"), consume1(producer=prod)), Family("consume2").add( # change manually with the GUI the limit to reduce/increase the load Label("info", "one family per step to consume"), Inlimit("limit:consume"), not_producer(), consume2(beg, fin, prod)))
else: deployed.append(pathname) if extn in pathname: # surround .ecf with head, tail: script = "%include <head.h>\n" + script + "\n%include <tail.h>" with open(pathname, "w") as destination: # overwrite! print(script, file=destination) print("#MSG: created", pathname) ##################################################################### # suite definition acq = "acquisition" deploy("echo acq %TASK%", files + acq + extn) # create wrapper post = "postproc" deploy("ecflow_client --label info %TASK%", files + post + extn) suite = Suite("course").add( Defstatus("suspended"), Repeat("YMD", 20180101, 20321212, kind="date"), Variables(ECF_HOME=home, # jobs files are created there by ecflow ECF_FILES=home + "/files", # task-wrappers location ECF_INCLUDE=home + "/include", # include files location # ECF_OUT=home, # output files location on remote systems, # no directory created there by ecflow... ECF_EXTN=extn, ), # task wrapper extension Task(acq).add(Event(1)), Family("ensemble").add( # ENS Complete(acq + ":1"), [Family("%02d" % num).add( Variables(ID=num), model(360, dependencies="../../" + acq)) # relative path... for num in xrange(0, 10)]), model(240, dependencies=acq), # HRES