Пример #1
0
 def task(cls):
   main = Process(name="main", cmdline="date && echo hello world")
   finalizer = Process(name="finalizer", cmdline="date", final=True)
   task = Task(name="task_with_finalizer",
               processes=[main, finalizer(name='finalizer1'), finalizer(name='finalizer2')],
               constraints=[Constraint(order=['finalizer1', 'finalizer2'])])
   return task.interpolate()[0]
Пример #2
0
 def task(cls):
   main = Process(name="main", cmdline="echo hello world")
   finalizer = Process(cmdline="date", final=True)
   task = Task(name="task_with_finalizer",
               processes=[main,
                          finalizer(name='finalizer1', cmdline='exit 1', max_failures=2),
                          finalizer(name='finalizer2')])
   return task.interpolate()[0]
Пример #3
0
 def task(cls):
   task = Task(
     name = "task_with_ephemeral",
     processes = [
       Process(name = "ephemeral_sleepy", ephemeral=True, cmdline="sleep 10"),
       Process(name = "sleepy", cmdline="sleep 1")
     ])
   return task.interpolate()[0]
Пример #4
0
 def task(cls):
     task = Task(name="task_with_ephemeral",
                 processes=[
                     Process(name="ephemeral_sleepy",
                             ephemeral=True,
                             cmdline="sleep 10"),
                     Process(name="sleepy", cmdline="sleep 1")
                 ])
     return task.interpolate()[0]
Пример #5
0
 def task(cls):
   base = Process(max_failures=2, min_duration=1)
   ex = base(cmdline="exit 1")
   hw = base(cmdline="echo hello world")
   task = Task(
     name = "failing_task",
     resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
     max_failures = 0,
     processes = [ex(name='f1'), ex(name='f2'), ex(name='f3'),
                  hw(name='s1'), hw(name='s2'), hw(name='s3')])
   return task.interpolate()[0]
Пример #6
0
 def task(cls):
     main = Process(name="main", cmdline="date && echo hello world")
     finalizer = Process(name="finalizer", cmdline="date", final=True)
     task = Task(
         name="task_with_finalizer",
         processes=[
             main,
             finalizer(name='finalizer1', cmdline='exit 1'),
             finalizer(name='finalizer2')
         ],
         constraints=[Constraint(order=['finalizer1', 'finalizer2'])])
     return task.interpolate()[0]
Пример #7
0
def flaky_task():
  task = Task(
    name = "failing_task",
    max_failures = 2,
    processes = [
      Process(name = "a", max_failures=1, min_duration=1, cmdline="echo hello world"),
      Process(name = "b", max_failures=2, min_duration=1, cmdline="exit 1"),
      Process(name = "c", max_failures=1, min_duration=1, final=True, cmdline="echo hello world")
    ],
    constraints = [{'order': ['a', 'b']}]
  )
  return task.interpolate()[0]
Пример #8
0
 def task(cls):
     main = Process(name="main", cmdline="echo hello world")
     finalizer = Process(cmdline="date", final=True)
     task = Task(name="task_with_finalizer",
                 processes=[
                     main,
                     finalizer(name='finalizer1',
                               cmdline='exit 1',
                               max_failures=2),
                     finalizer(name='finalizer2')
                 ])
     return task.interpolate()[0]
Пример #9
0
 def task(cls):
   task = Task(
     name = "failing_task",
     resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
     max_failures = 2,
     processes = [
       Process(name = "a", max_failures=1, min_duration=1, cmdline="echo hello world"),
       Process(name = "b", max_failures=1, min_duration=1, cmdline="exit 1"),
       Process(name = "c", max_failures=1, min_duration=1, cmdline="echo hello world")
     ],
     constraints = [{'order': ['a', 'b', 'c']}]
   )
   return task.interpolate()[0]
Пример #10
0
 def from_file(filename, **kw):
   try:
     with safe_open(filename) as fp:
       task = Task.json_load(fp)
     return ThermosTaskWrapper(task, **kw)
   except Exception as e:
     return None
Пример #11
0
 def from_file(filename, **kw):
     try:
         with safe_open(filename) as fp:
             task = Task.json_load(fp)
         return ThermosTaskWrapper(task, **kw)
     except Exception as e:
         return None
Пример #12
0
 def task(cls):
     hello_template = Process(cmdline="echo 1")
     t1 = hello_template(
         name="t1", cmdline="echo 1 port {{thermos.ports[named_port]}}")
     t2 = hello_template(name="t2")
     t3 = hello_template(name="t3")
     t4 = hello_template(name="t4")
     t5 = hello_template(name="t5")
     t6 = hello_template(name="t6")
     tsk = Task(name="complex", processes=[t1, t2, t3, t4, t5, t6])
     # three ways of tasks: t1 t2, t3 t4, t5 t6
     tsk = tsk(constraints=[{
         'order': ['t1', 't3']
     }, {
         'order': ['t1', 't4']
     }, {
         'order': ['t2', 't3']
     }, {
         'order': ['t2', 't4']
     }, {
         'order': ['t3', 't5']
     }, {
         'order': ['t3', 't6']
     }, {
         'order': ['t4', 't5']
     }, {
         'order': ['t4', 't6']
     }])
     return tsk
Пример #13
0
def test_combine_tasks():
  p1 = Process(name='p1')
  p2 = Process(name='p2')
  p3 = Process(name='p3')
  p4 = Process(name='p4')
  r100 = Resources(cpu=1, ram=0, disk=0)
  r010 = Resources(cpu=0, ram=1, disk=0)
  r001 = Resources(cpu=0, ram=0, disk=1)
  r111 = Units.resources_sum(r100, r010, r001)

  t1 = Task(name="p1p2", processes=[p1, p2], constraints=order(p1, p2),
            resources=Units.resources_sum(r100, r010))
  t2 = Task(name="p3p4", processes=[p3, p4], constraints=order(p3, p4),
            resources=r001)

  assert combine_tasks() == Task()
  assert combine_tasks(t1) == t1
  assert combine_tasks(t2) == t2

  t3 = combine_tasks(t1, t2)
  assert t3.name() == t2.name()
  assert t3.resources() == r111
  assert set(t3.processes()) == set([p1, p2, p3, p4])
  assert set(t3.constraints()) == set(order(p1, p2) + order(p3, p4))

  t4 = concat_tasks(t1, t2)
  assert t4.name() == t2.name()
  assert t4.resources() == r111
  assert set(t4.processes()) == set([p1, p2, p3, p4])
  assert set(t4.constraints()) == set(
      order(p1, p2) + order(p3, p4) + order(p1, p3) + order(p1, p4) +
      order(p2, p3) + order(p2, p4))
Пример #14
0
def flaky_task():
    task = Task(name="failing_task",
                max_failures=2,
                processes=[
                    Process(name="a",
                            max_failures=1,
                            min_duration=1,
                            cmdline="echo hello world"),
                    Process(name="b",
                            max_failures=2,
                            min_duration=1,
                            cmdline="exit 1"),
                    Process(name="c",
                            max_failures=1,
                            min_duration=1,
                            final=True,
                            cmdline="echo hello world")
                ],
                constraints=[{
                    'order': ['a', 'b']
                }])
    return task.interpolate()[0]
Пример #15
0
 def task(cls):
     hello_template = Process(cmdline="sleep 1")
     tsk = Task(name="complex",
                processes=[
                    hello_template(name="process1"),
                    hello_template(name="process2"),
                    hello_template(name="process3")
                ],
                resources=Resources(cpu=1.0,
                                    ram=16 * 1024 * 1024,
                                    disk=16 * 1024),
                max_concurrency=1)
     return tsk
Пример #16
0
 def task(cls):
   ping_template = Process(
     name="{{process_name}}",
     min_duration=1,
     max_failures=5,
     cmdline = "echo {{process_name}} pinging;                                "
               "echo ping >> {{process_name}};                                "
               "echo current count $(cat {{process_name}} | wc -l);           "
               "if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
               "  exit 0;                                             "
               "else                                                  "
               "  exit 1;                                             "
               "fi                                                    ")
   tsk = Task(
     name = "pingping",
     resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
     processes = [
       ping_template.bind(process_name = "p1", num_runs = 1),
       ping_template.bind(process_name = "p2", num_runs = 2),
       ping_template.bind(process_name = "p3", num_runs = 3),
     ]
   )
   return tsk.interpolate()[0]
Пример #17
0
 def task(cls):
     ping_template = Process(
         name="{{process_name}}",
         min_duration=1,
         max_failures=5,
         cmdline=
         "echo {{process_name}} pinging;                                "
         "echo ping >> {{process_name}};                                "
         "echo current count $(cat {{process_name}} | wc -l);           "
         "if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
         "  exit 0;                                             "
         "else                                                  "
         "  exit 1;                                             "
         "fi                                                    ")
     tsk = Task(name="pingping",
                resources=Resources(cpu=1.0,
                                    ram=16 * 1024 * 1024,
                                    disk=16 * 1024),
                processes=[
                    ping_template.bind(process_name="p1", num_runs=1),
                    ping_template.bind(process_name="p2", num_runs=2),
                    ping_template.bind(process_name="p3", num_runs=3),
                ])
     return tsk.interpolate()[0]
Пример #18
0
def test_tasklets():
  install_thermosrc = Process(name='install_thermosrc')
  setup_py3k = Process(name='setup_py3k')
  setup_ruby19 = Process(name='setup_ruby19')
  setup_php = Process(name='setup_php')
  recipe_py3k = SequentialTask(processes=[install_thermosrc, setup_py3k])
  recipe_ruby19 = SequentialTask(processes=[install_thermosrc, setup_ruby19])
  recipe_php = SequentialTask(processes=[install_thermosrc, setup_php])
  all_recipes = Tasks.combine(recipe_py3k, recipe_ruby19, recipe_php)
  my_task = Task(processes = [Process(name='my_process')])
  my_new_task = Tasks.concat(all_recipes, my_task)(name = 'my_task')

  # TODO(wickman) Probably should have Tasks.combine/concat do constraint
  # minimization since many constraints are redundant.
  for p in (install_thermosrc, setup_py3k, setup_ruby19, setup_php):
    assert p in my_new_task.processes()
Пример #19
0
    def task(cls):
        setup_bashrc = Process(name="setup_bashrc",
                               cmdline=dedent("""
    mkdir -p .profile.d
    cat <<EOF > .thermos_profile
    for i in .profile.d/*.sh ; do
      if [ -r "\\$i" ]; then
        . \\$i
      fi
    done
    EOF
    """))

        setup_foo = Process(name="setup_foo",
                            cmdline=dedent("""
    cat <<EOF > .profile.d/setup_foo.sh
    export FOO=1
    EOF
    """))

        setup_bar = Process(name="setup_bar",
                            cmdline=dedent("""
    cat <<EOF > .profile.d/setup_bar.sh
    export BAR=2
    EOF
    """))

        foo_recipe = SequentialTask(processes=[setup_bashrc, setup_foo])
        bar_recipe = SequentialTask(processes=[setup_bashrc, setup_bar])
        all_recipes = Tasks.combine(foo_recipe, bar_recipe)

        run = Process(name="run",
                      cmdline=dedent("""
    echo $FOO $BAR > expected_output.txt
    """))

        my_task = Task(processes=[run],
                       resources=Resources(cpu=1.0,
                                           ram=16 * 1024 * 1024,
                                           disk=16 * 1024))
        return Tasks.concat(all_recipes, my_task, name="my_task")
Пример #20
0
def simplerun(args, options):
    """Run a simple command line as a thermos task.

    Usage: thermos simplerun [options] [--] commandline
    Options:
      --user=USER		   run as this user.  if not $USER, must have setuid privilege.
      --name=STRING		   the name to give this task. ('simple' by default)
      --task_id=STRING		   the id to which this task should be bound, synthesized from the
                                   task name if none provided.
      -P/--port=NAME:PORT	   bind the named port NAME to port number PORT (may be specified
                                   multiple times to bind multiple names.)
      -E/--environment=NAME=VALUE  bind the configuration environment variable NAME to
                                   VALUE.
      --daemon			   Fork and daemonize the task.
  """
    try:
        cutoff = args.index('--')
        cmdline = ' '.join(args[cutoff + 1:])
    except ValueError:
        cmdline = ' '.join(args)

    print("Running command: '%s'" % cmdline)

    thermos_task = ThermosTaskWrapper(
        Task(name=options.name,
             resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0),
             processes=[Process(name=options.name, cmdline=cmdline)]))

    _really_run(thermos_task,
                options.root,
                None,
                task_id=options.task_id,
                user=options.user,
                prebound_ports=options.prebound_ports,
                chroot=False,
                daemon=options.daemon)
Пример #21
0
 def task(cls):
     task = Task(name="task", finalization_wait=3, processes=[ignorant_process(name="ignorant_process")])
     return task.interpolate()[0]
Пример #22
0
 def task(cls):
     task = Task(name="task", processes=[sleepy_process(name="process")])
     return task.interpolate()[0]
Пример #23
0
 def task(cls):
   main = Process(name="main", cmdline="date && echo hello world")
   finalizer = Process(name="finalizer", cmdline="date", final=True)
   task = Task(name="task_with_finalizer", processes = [main, finalizer])
   return task.interpolate()[0]
Пример #24
0
 def task(cls):
     main = Process(name="main", cmdline="date && echo hello world")
     finalizer = Process(name="finalizer", cmdline="date", final=True)
     task = Task(name="task_with_finalizer", processes=[main, finalizer])
     return task.interpolate()[0]
Пример #25
0
 def task(cls):
     task = Task(name="task", processes=[Process(name="process", cmdline=SIMPLEFORK_SCRIPT)])
     return task.interpolate()[0]
Пример #26
0
 def task(cls):
     task = Task(name="task", processes=[sleepy_process(name="process")])
     return task.interpolate()[0]
Пример #27
0
 def task(cls):
     task = Task(
         name="task",
         processes=[Process(name="process", cmdline=SIMPLEFORK_SCRIPT)])
     return task.interpolate()[0]
Пример #28
0
 def task(cls):
     task = Task(name="task",
                 finalization_wait=3,
                 processes=[ignorant_process(name="ignorant_process")])
     return task.interpolate()[0]
Пример #29
0
    Identity,
)

from pystachio import Map, String
from pystachio.naming import frozendict
import pytest

HELLO_WORLD = Job(name='hello_world',
                  role='john_doe',
                  environment='staging66',
                  cluster='smf1-test',
                  task=Task(
                      name='main',
                      processes=[
                          Process(name='hello_world',
                                  cmdline='echo {{mesos.instance}}')
                      ],
                      resources=Resources(cpu=0.1,
                                          ram=64 * 1048576,
                                          disk=64 * 1048576),
                  ))


def test_simple_config():
    job = convert_pystachio_to_thrift(HELLO_WORLD)
    assert job.instanceCount == 1
    tti = job.taskConfig
    assert job.key == JobKey(role=HELLO_WORLD.role().get(),
                             environment=HELLO_WORLD.environment().get(),
                             name=HELLO_WORLD.name().get())
    assert job.owner == Identity(role=HELLO_WORLD.role().get(),
                                 user=getpass.getuser())