Beispiel #1
0
def create(name=os.getenv("SUITE", "elearning")):
    return Suite(name).add(
        Defstatus("suspended"),  # start immediately or not for this demo
        Clock("real"),
        Edit(ECF_INCLUDE=HOME,  # header files
             ECF_FILES=HOME,  # script template .ecf
             ECF_HOME=HOME),  # job + local output files
        Family("f1").add(
            Task("t1").add(
                Event(1),
                Label("info", ""),
                Meter("step", -1, 100)),
            Task("t2").add(
                Late("-c 01:00"),
                Meter("step", -1, 100),
                Event("a"),
                Event("b"),
                Trigger("t1:step gt 0")),
            Task("t3").add(
                Trigger("t2:a")),
            Task("t4").add(
                Complete("t2:b"),
                Trigger("t2 eq complete and not t2:b"))),
        Family("f2").add(
            Task("t1").add(
                Time("00:30 23:30 00:30")),
            Task("t2").add(
                Day("sunday")),
            Task("t3").add(
                Time("12:00"),
                Date("1.*.*")),
            Task("t4").add(
                Time("+00:02")),
            Task("t5").add(
                Time("00:02"))))
Beispiel #2
0
def create(name):
    """ suite provider """
    return Suite(name).add(
        Defstatus("suspended"),
        Edit(ECF_INCLUDE=HOME,  # header files
             ECF_FILES=HOME,    # script template .ecf
             ECF_HOME=HOME),    # job + local output files
        Family("f1").add(
            Task("t1").add(
                Label("info", ""),
                Late("-c 01:00"),
                Meter("step", -1, 100)),
            Task("t2").add(
                Meter("step", -1, 100),
                Event("a"),
                Event("b"),
                Trigger("t1:step gt 0")),
            Task("t3").add(
                Trigger("t2:a")),
            Task("t4").add(
                Complete("t2:b"),
                Trigger("t2 eq complete and not t2:b"))),
        Family("f2").add(
            Task("t1").add(
                Time("00:30 23:30 00:30")),
            Task("t2").add(
                Day("sunday")),
            Task("t3").add(
                Time("12:00"),
                Date("1.*.*")),
            Task("t4").add(
                Time("+00:02")),
            Task("t5").add(
            Time("00:02"))))
Beispiel #3
0
#!/usr/bin/env python2.7
""" back archive example """
from __future__ import print_function
import os
import ecf as ecflow
from ecf import (Defs, Defstatus, Suite, Family, Task, Variables, Label,
                 Limit, Inlimit, Repeat, Trigger)
HOME = os.getenv("HOME") + "/ecflow_server"
NAME = "back_archiving"; FILES = HOME + "/back"; DEFS = Defs()
DEFS.add(Suite(NAME).add(
    Defstatus("suspended"),  # so that jobs do not start immediately
    Repeat(kind="day", step=1),
    Variables(ECF_HOME=HOME, ECF_INCLUDE=HOME, ECF_FILES=FILES, SLEEP=2),
    Limit("access", 2),
    [Family(kind).add(
        Repeat("DATE", 19900101, 19950712, kind="date"),
        Variables(KIND=kind),
        Task("get_old").add(Inlimit("access"), Label("info", "")),
        Task("convert").add(Trigger("get_old == complete")),
        Task("save_new").add(Trigger("convert eq complete"))
    ) for kind in ("analysis", "forecast", "climatology", "observations", "images")]))
# print(DEFS); DEFS.generate_scripts(); 
RESULT = DEFS.simulate(); print(RESULT)
CLIENT = ecflow.Client(os.getenv("ECF_HOST", "localhost"),
                       os.getenv("ECF_PORT", 2500))
CLIENT.replace("/%s" % NAME, DEFS)
        Family("f4").add(
            Variables(SLEEP=2),
            Repeat("NAME", ["a", "b", "c", "d", "e", "f"], kind="enumerated"),
            Task("t1")),
        Family("f5").add(
            Repeat("DATE", 20170101, 20200105, kind="date"),
            Task("t1").add(
                Repeat("PARAM", 1, 10, kind="integer"),
                Label("info", ""))))


DEFS.add(  # suite definition
    Suite(NAME).add(
        Defstatus("suspended"),  # so that jobs do not start immediately
        Variables(  # we can add multiple variables at once
            ECF_HOME=ECF_HOME,  # where job files are created by ecflow
            ECF_FILES=ECF_HOME + "/files",  # where to find script templates
            ECF_INCLUDE=ECF_HOME + "/include",  # where to find head.h tail.h
            SLEEP=2),
        create_families()))

SCRIPT_TEMPLATE = """#!/bin/bash
%include <head.h>
STEP=0
while [[ $STEP -le 240 ]] ; do
  sleep %SLEEP:1%; ecflow_client --meter step $STEP  # share progress
  msg="The date is %DATE:$(date)%. PARAM is %PARAM:% NAME is %NAME:%"
  ecflow_client --label info "$msg"
  (( STEP = STEP + 1))
done
%include <tail.h>
"""
#!/usr/bin/env python
from __future__ import print_function
import os
import ecf as ecflow
from ecf import (Defstatus, Suite, Family, Task, Variables)
print("Creating suite definition")
ECF_HOME = os.path.join(os.getenv("HOME"), "ecflow_server")
NAME = os.getenv("SUITE", "elearning")
DEFS = ecflow.Defs()
DEFS.add(  # suite definition
    Suite(NAME).add(
        Defstatus("suspended"),  # so that jobs do not start immediately
        Variables(ECF_HOME=ECF_HOME,
                  ECF_FILES=ECF_HOME + "/files",
                  ECF_INCLUDE=ECF_HOME + "/include"),
        Family("f1").add(  # hosting family
            Task("t1"),  # a first task
            Task("t2"),  # a second task
        )))

if __name__ == '__main__':
    HOST = os.getenv("ECF_HOST", "localhost")
    PORT = int(os.getenv("ECF_PORT", "%d" % (1500 + os.getuid())))
    CLIENT = ecflow.Client(HOST, PORT)

    NODE = "/" + NAME  # replace suite node: add f1, delete t1 t2
    CLIENT.replace(NODE, DEFS)
    print("replaced node %s into" % NODE, HOST, PORT)
import os
import ecf as ecflow
from ecf import (Defstatus, Suite, Family, Task, Variables, Trigger, Event)
ECF_HOME = os.path.join(os.getenv("HOME"), "ecflow_server")
NAME = os.getenv("SUITE", "elearning")
DEFS = ecflow.Defs()
DEFS.add(  # suite definition
    Suite(NAME).add(
        Defstatus("suspended"),  # so that jobs do not start immediately
        Variables(  # add multiple variables at once:
            ECF_HOME=ECF_HOME,  # where job files are created by ecflow
            ECF_FILES=ECF_HOME + "/files",      # where to find script template
            ECF_INCLUDE=ECF_HOME + "/include",  # where to find head.h tail.h
            SLEEP=5),
        Family("f1").add(
            Task("t1"),
            Task("t2").add(
                Trigger("t1 eq complete"),
                Event("a"),
                Event("1")),
            Task("t3").add(
                Trigger("t2:a")),
            Task("t4").add(
                Trigger("t2:1")))))

SCRIPT_TEMPLATE = """#!/bin/bash
%include <head.h>
echo "I will now sleep for %SLEEP:1% seconds"
sleep %SLEEP:1%; ecflow_client --event a  # set first event
sleep %SLEEP:!%; ecflow_client --event 1  # set a second event
sleep %SLEEP:1%;  # don't sleep too much anyway
#!/usr/bin/env python2.7
from __future__ import print_function
""" add another task, another manual """
import os
from ecf import (Defs, Defstatus, Suite, Variable, Task, Client)
print("Creating suite definition")
ECF_HOME = os.path.join(os.getenv("HOME"), "ecflow_server")
NAME = os.getenv("SUITE", "elearning")
DEFS = Defs()
DEFS.add(  # suite definition
    Suite(NAME).add(
        Defstatus("suspended"),  # so that jobs do not start immediately
        Variable("ECF_HOME", ECF_HOME),
        Task("t1"),  # first task
        Task("t2"),  # second task
    ))
SCRIPT_TEMPLATE = """%manual
  Manual for task t2
  Operations: if this task fails, set it to complete, report next working day
  Analyst:    Check something ...
%end
%include <head.h>
echo "I am part of a suite that lives in %ECF_HOME%"
%include <tail.h>

%manual
  There can be multiple manual pages in the same file.
  When viewed they are simply concatenated.
%end
"""
from __future__ import print_function
import os
import ecf as ecflow
from ecf import (Defstatus, Suite, Family, Task, Variables, Trigger)
TASK3 = Task("t3")  # a python object can be set, and added later to the suite
ECF_HOME = os.path.join(os.getenv("HOME"), "ecflow_server")
NAME = os.getenv("SUITE", "elearning")
DEFS = ecflow.Defs()
DEFS.add(  # suite definition
    Suite(NAME).add(
        Defstatus("suspended"),  # so that jobs do not start immediately
        Variables(
            ECF_HOME=ECF_HOME,  # where job files are created by ecflow
            ECF_FILES=ECF_HOME + "/files",      # where to find script template
            ECF_INCLUDE=ECF_HOME + "/include",  # where to find head.h tail.h
            SLEEP=1, ),
        Family("f1").add(
            Task("t1").add(Variables(SLEEP=5)),
            Task("t2").add(Variables(SLEEP=7),
                           Trigger("t1 eq complete")),
            TASK3,
            Task("t4").add(Trigger(["t1", "t2"])))))

SCRIPT_TEMPLATE = """#!/bin/bash
%include <head.h>
echo "I will now sleep for %SLEEP:1% seconds"; sleep %SLEEP:1%
%include <tail.h>
"""

if __name__ == '__main__':
    ecflow.USE_TRIGGER = True;  # ecflow.USE_TRIGGER = False;  # DEBUG
import os
import ecf as ecflow
from ecf import (Defstatus, Suite, Family, Task, Variables, Trigger, Event,
                 Meter)
ECF_HOME = os.path.join(os.getenv("HOME"), "ecflow_server")
NAME = os.getenv("SUITE", "elearning")
DEFS = ecflow.Defs()
DEFS.add(  # suite definition
    Suite(NAME).add(
        Defstatus("suspended"),  # so that jobs do not start immediately
        Variables(  # add multiple variables at once
            ECF_HOME=ECF_HOME,  # where job files are created by ecflow
            ECF_FILES=ECF_HOME + "/files",  # where to find script templates
            ECF_INCLUDE=ECF_HOME + "/include",  # where to find head.h tail.h
            SLEEP=2),
        Family("f2").add(
            Task("t1").add(Meter("step", -1, 240)),
            Task("t2").add(Trigger("t1 eq complete"), Event("a"), Event("b")),
            Task("t3").add(Trigger("t2:a")),
            Task("t4").add(Trigger("t2:b")),
            Task("t5").add(Trigger("t1:step ge 24")),
            Task("t6").add(Trigger("t1:step ge 48")),
            Task("t7").add(Trigger("t1:step ge 120")))))

SCRIPT_TEMPLATE = """#!/bin/bash
%include <head.h>
STEP=0
while [[ $STEP -le 240 ]] ; do
  ecflow_client --meter step $STEP  # share progress
  msg="The date is now $(date)"; sleep %SLEEP:1%
  (( STEP = STEP + 1))
Beispiel #10
0
#####################################################################
# suite definition
acq = "acquisition"
deploy("echo acq %TASK%", files + acq + extn)  # create wrapper
post = "postproc"
deploy("ecflow_client --label info %TASK%", files + post + extn)
suite = Suite("course").add(
    Defstatus("suspended"),
    Repeat("YMD", 20180101, 20321212, kind="date"),
    Variables(ECF_HOME=home,  # jobs files are created there by ecflow
              ECF_FILES=home + "/files",  # task-wrappers location
              ECF_INCLUDE=home + "/include",  # include files location
              # ECF_OUT=home,  # output files location on remote systems, 
                               # no directory created there by ecflow...
              ECF_EXTN=extn, ),  # task wrapper extension
    Task(acq).add(Event(1)),
    Family("ensemble").add(  # ENS
        Complete(acq + ":1"),
        [Family("%02d" % num).add(
            Variables(ID=num),
            model(360, dependencies="../../" + acq))  # relative path...
         for num in xrange(0, 10)]),
    model(240, dependencies=acq),  # HRES
    Task(post).add(
        Trigger("model eq complete"),
        Label("info", "")))


#####################################################################
head = """#!/bin/bash
set -e # stop the shell on first error
set -u # fail when using an undefined variable
from __future__ import print_function
import os
import ecf as ecflow
from ecf import (Date, Day, Defs, Defstatus, Suite, Family, Task,
                 If,  # If attribute in use example
                 Edit, Label, Repeat, Time, Trigger, Defs, Client)
HOME = os.getenv("HOME") + "/ecflow_course"; NAME = "data_acquisition"; DEFS = Defs()
DEFS.add(Suite(NAME).add(
    Defstatus("suspended"),  # so that jobs do not start immediately
    Repeat(kind="day", step=1),
    Edit(ECF_HOME=HOME, ECF_INCLUDE=HOME, ECF_FILES=HOME + "/acq", SLEEP=2),
    [Family(city).add(
        Family("archive").add(
            [Family(obs_type).add(
                If(test=city in ("Exeter", "Toulouse", "Offenbach"), then=Time("00:00 23:00 01:00")),
                If(city in ("Washington"), Time("00:00 23:00 03:00")),
                If(city in ("Tokyo"), Time("12:00")),
                If(city in ("Melbourne"), Day("monday")),
                If(city in ("Montreal"), Date("1.*.*")),
                Task("get").add(Label("info", "")),
                Task("process").add(Trigger("get eq complete")),
                Task("store").add(Trigger("get eq complete")))
             for obs_type in ("observations", "fields", "images")]))
        for city in ("Exeter", "Toulouse", "Offenbach", "Washington",
                     "Tokyo", "Melbourne", "Montreal")]))
# print(DEFS); DEFS.generate_scripts(); 
RESULT = DEFS.simulate(); # print(RESULT)
CLIENT = Client(os.getenv("ECF_HOST", "localhost"),
                       os.getenv("ECF_PORT", 2500))
CLIENT.replace("/%s" % NAME, DEFS)
import os
import ecf as ecflow
from ecf import (Defstatus, Suite, Family, Task, Edit,
                 Time, Date, Day, Clock, Defs, Client)
ECF_HOME = os.path.join(os.getenv("HOME"), "ecflow_server")
NAME = os.getenv("SUITE", "elearning")
DEFS = Defs()
DEFS.add(  # suite definition
    Suite(NAME).add(
        Clock("real"),
        Defstatus("suspended"),  # so that jobs do not start immediately
        Edit(  # we can add multiple variables at once
            ECF_HOME=ECF_HOME,  # where job files are created by ecflow
            ECF_FILES=ECF_HOME + "/files",  # where to find script templates
            ECF_INCLUDE=ECF_HOME + "/include",  # where to find head.h tail.h
            SLEEP=2, ),
        Family("f2").add(
            Task("t1").add(Time("00:30 23:30 00:30")),
            Task("t2").add(Day("sunday")),
            Task("t3").add(Date("16.01.2018"),
                           Time("12:00")),
            Task("t4").add(Time("+00:02")),
            Task("t5").add(Time("00:02")))))

SCRIPT_TEMPLATE = """#!/bin/bash
%include <head.h>
STEP=0
while [[ $STEP -le 240 ]] ; do
  sleep %SLEEP:1%; ecflow_client --meter step $STEP  # share progress
  (( STEP = STEP + 1))
done
#!/usr/bin/env python2.7
from __future__ import print_function
import os
import ecf as ecflow
from ecf import (Defstatus, Suite, Family, Task, Variables, Label, Meter)

ECF_HOME = os.path.join(os.getenv("HOME"), "ecflow_server")
ECF_INCLUDE = ECF_HOME + "/include"
NAME = os.getenv("SUITE", "elearning")
DEFS = ecflow.Defs()
DEFS.add(  # suite definition
    Suite(NAME).add(
        Defstatus("suspended"),  # so that jobs do not start immediately
        Variables(  # we can add multiple variables at once
            ECF_HOME=ECF_HOME,  # where job files are created by ecflow
            ECF_FILES=ECF_HOME + "/files",  # where to find script template
            ECF_INCLUDE=ECF_INCLUDE,  # where to find head.h tail.h
            SLEEP=2,
        ),
        Family("f3").add(
            Task("t1").add(Label("info", "none"), Meter("step", -1, 240)))))

SCRIPT_TEMPLATE = """#!/bin/bash
%include <head.h>
STEP=0
while [[ $STEP -le 240 ]] ; do
  sleep %SLEEP:1%; ecflow_client --meter step $STEP  # share progress
  msg="The date is now $(date)"
  ecflow_client --label info "$msg"
  (( STEP = STEP + 1))
done
ecflow_client --label info "job's done"
    v = "2.7"
    v = "3.5"
    INST = "/usr/local/apps/ecflow/lib/python%s/site-packages/ecflow:" % v
    sys.path.append(INST)
    INST = "/usr/local/lib/python%s/site-packages/ecflow" % v
    sys.path.append(INST)
    from ecf import (Defs, Suite, Defstatus, Edit, Task)
    if 0:
        raise Exception(  #ERR: could not import ecf. Does the following line help?"+
            "\nexport PYTHONPATH=$PYTHONPATH:%s" % INST)

print("Creating suite definition")
ECF_HOME = os.path.join(os.getenv("HOME"), "ecflow_server")
NAME = os.getenv("SUITE", "elearning")
DEFS = Defs()
DEFS.add(
    Suite(NAME).add(  # simplest suite definition
        Defstatus("suspended"),  # so that jobs do not start immediately
        Edit(
            ECF_HOME=ECF_HOME,  # where to find jobs + output files
            ECF_FILES=ECF_HOME + "/files",  # script template .ecf
            ECF_INCLUDE=ECF_HOME + "/include"),  # include files .h
        Task("t1")))  # a first task

if __name__ == '__main__':
    print(DEFS)
    NAME = ECF_HOME + NAME
    print("Saving definition to file '%s.def'" % NAME)
    os.system("mkdir -p %s" % NAME)
    DEFS.save_as_defs("%s.def" % NAME)  # an external client can use it
Beispiel #15
0
from __future__ import print_function
import os
import ecf as ecflow
from ecf import (Defstatus, Suite, Task, Variables)
ECF_HOME = os.path.join(os.getenv("HOME"), "ecflow_server")
ECF_FILES = ECF_HOME + "/files"
NAME = os.getenv("SUITE", "elearning")
DEFS = ecflow.Defs()
DEFS.add(  # suite definition
    Suite(NAME).add(
        Defstatus("suspended"),  # so that jobs do not start immediately
        Variables(  # we can add multiple variables at once
            ECF_HOME=ECF_HOME,  # where job files are created by ecflow
            ECF_FILES=ECF_FILES,  # where to find script templates .ecf
            ECF_INCLUDE=ECF_HOME + "/include",  # where to find head.h tail.h
            SLEEP=1,  # user variable to be inherited by task/families below
        ),
        Task("t1").add(
            Variables(SLEEP=5),  # overwriting with value 5 for task t1
        ),
        Task("t2").add(
            Variables(SLEEP=7),  # overwriting with value 7 for task t2
        )))

SCRIPT_TEMPLATE = """#!/bin/bash
%include <head.h>
echo "I will now sleep for %SLEEP:1% seconds"
sleep %SLEEP:1%
%include <tail.h>
"""

if __name__ == '__main__':
Beispiel #16
0
 Suite(NAME).add(
     Defstatus("suspended"),  # so that jobs do not start immediately
     Repeat(kind="day", step=1),
     Variables(ECF_HOME=HOME,
               ECF_INCLUDE=HOME + "/include",
               ECF_FILES=HOME + "/files"),
     [
         Family(str(cycle)).add(
             Variables(CYCLE=cycle, LAST_STEP=LAST_STEP[cycle]),
             cycle_trigger(cycle),
             Family("analysis").add(
                 Task("get_observations"),
                 Task("run_analysis").add(Trigger([
                     "get_observations",
                 ])),
                 Task("post_processing").add(Trigger([
                     "run_analysis",
                 ]))),
             Family("forecast").add(
                 Trigger("analysis == complete"),
                 Task("get_input_data").add(Label("info", "")),
                 Task("run_forecast").add(
                     Trigger([
                         "get_input_data",
                     ]), Meter("step", 0, LAST_STEP[cycle]))),
             Family("archive").add(
                 Family("analysis").add(
                     Variables(TYPE="analysis", STEP=0),
                     Trigger([
                         "../analysis/run_analysis",
                     ]), Task("save"), [
                         Family("step_%02d" % i).add(
                             Variables(TYPE="forecast", STEP=i),
                             Trigger(
                                 "../../forecast/run_forecast:step ge %d" %
                                 i), Task("save"))
                         for i in range(6, LAST_STEP[cycle] + 1, 6)
                     ]))) for cycle in ["00", "12"]
     ]))