def task(cls):
main = Process(name="main", cmdline="date && echo hello world")
finalizer = Process(name="finalizer", cmdline="date", final=True)
task = Task(name="task_with_finalizer",
processes=[main, finalizer(name='finalizer1'), finalizer(name='finalizer2')],
constraints=[Constraint(order=['finalizer1', 'finalizer2'])])
return task.interpolate()[0]
def task(cls):
main = Process(name="main", cmdline="echo hello world")
finalizer = Process(cmdline="date", final=True)
task = Task(name="task_with_finalizer",
processes=[main,
finalizer(name='finalizer1', cmdline='exit 1', max_failures=2),
finalizer(name='finalizer2')])
return task.interpolate()[0]
def task(cls):
task = Task(
name = "task_with_ephemeral",
processes = [
Process(name = "ephemeral_sleepy", ephemeral=True, cmdline="sleep 10"),
Process(name = "sleepy", cmdline="sleep 1")
])
return task.interpolate()[0]
def task(cls):
task = Task(name="task_with_ephemeral",
processes=[
Process(name="ephemeral_sleepy",
ephemeral=True,
cmdline="sleep 10"),
Process(name="sleepy", cmdline="sleep 1")
])
return task.interpolate()[0]
def task(cls):
base = Process(max_failures=2, min_duration=1)
ex = base(cmdline="exit 1")
hw = base(cmdline="echo hello world")
task = Task(
name = "failing_task",
resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
max_failures = 0,
processes = [ex(name='f1'), ex(name='f2'), ex(name='f3'),
hw(name='s1'), hw(name='s2'), hw(name='s3')])
return task.interpolate()[0]
def task(cls):
main = Process(name="main", cmdline="date && echo hello world")
finalizer = Process(name="finalizer", cmdline="date", final=True)
task = Task(
name="task_with_finalizer",
processes=[
main,
finalizer(name='finalizer1', cmdline='exit 1'),
finalizer(name='finalizer2')
],
constraints=[Constraint(order=['finalizer1', 'finalizer2'])])
return task.interpolate()[0]
def flaky_task():
task = Task(
name = "failing_task",
max_failures = 2,
processes = [
Process(name = "a", max_failures=1, min_duration=1, cmdline="echo hello world"),
Process(name = "b", max_failures=2, min_duration=1, cmdline="exit 1"),
Process(name = "c", max_failures=1, min_duration=1, final=True, cmdline="echo hello world")
],
constraints = [{'order': ['a', 'b']}]
)
return task.interpolate()[0]
def task(cls):
main = Process(name="main", cmdline="echo hello world")
finalizer = Process(cmdline="date", final=True)
task = Task(name="task_with_finalizer",
processes=[
main,
finalizer(name='finalizer1',
cmdline='exit 1',
max_failures=2),
finalizer(name='finalizer2')
])
return task.interpolate()[0]
def task(cls):
task = Task(
name = "failing_task",
resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
max_failures = 2,
processes = [
Process(name = "a", max_failures=1, min_duration=1, cmdline="echo hello world"),
Process(name = "b", max_failures=1, min_duration=1, cmdline="exit 1"),
Process(name = "c", max_failures=1, min_duration=1, cmdline="echo hello world")
],
constraints = [{'order': ['a', 'b', 'c']}]
)
return task.interpolate()[0]
def from_file(filename, **kw):
try:
with safe_open(filename) as fp:
task = Task.json_load(fp)
return ThermosTaskWrapper(task, **kw)
except Exception as e:
return None
def from_file(filename, **kw):
try:
with safe_open(filename) as fp:
task = Task.json_load(fp)
return ThermosTaskWrapper(task, **kw)
except Exception as e:
return None
def task(cls):
hello_template = Process(cmdline="echo 1")
t1 = hello_template(
name="t1", cmdline="echo 1 port {{thermos.ports[named_port]}}")
t2 = hello_template(name="t2")
t3 = hello_template(name="t3")
t4 = hello_template(name="t4")
t5 = hello_template(name="t5")
t6 = hello_template(name="t6")
tsk = Task(name="complex", processes=[t1, t2, t3, t4, t5, t6])
# three ways of tasks: t1 t2, t3 t4, t5 t6
tsk = tsk(constraints=[{
'order': ['t1', 't3']
}, {
'order': ['t1', 't4']
}, {
'order': ['t2', 't3']
}, {
'order': ['t2', 't4']
}, {
'order': ['t3', 't5']
}, {
'order': ['t3', 't6']
}, {
'order': ['t4', 't5']
}, {
'order': ['t4', 't6']
}])
return tsk
def test_combine_tasks():
p1 = Process(name='p1')
p2 = Process(name='p2')
p3 = Process(name='p3')
p4 = Process(name='p4')
r100 = Resources(cpu=1, ram=0, disk=0)
r010 = Resources(cpu=0, ram=1, disk=0)
r001 = Resources(cpu=0, ram=0, disk=1)
r111 = Units.resources_sum(r100, r010, r001)
t1 = Task(name="p1p2", processes=[p1, p2], constraints=order(p1, p2),
resources=Units.resources_sum(r100, r010))
t2 = Task(name="p3p4", processes=[p3, p4], constraints=order(p3, p4),
resources=r001)
assert combine_tasks() == Task()
assert combine_tasks(t1) == t1
assert combine_tasks(t2) == t2
t3 = combine_tasks(t1, t2)
assert t3.name() == t2.name()
assert t3.resources() == r111
assert set(t3.processes()) == set([p1, p2, p3, p4])
assert set(t3.constraints()) == set(order(p1, p2) + order(p3, p4))
t4 = concat_tasks(t1, t2)
assert t4.name() == t2.name()
assert t4.resources() == r111
assert set(t4.processes()) == set([p1, p2, p3, p4])
assert set(t4.constraints()) == set(
order(p1, p2) + order(p3, p4) + order(p1, p3) + order(p1, p4) +
order(p2, p3) + order(p2, p4))
def flaky_task():
task = Task(name="failing_task",
max_failures=2,
processes=[
Process(name="a",
max_failures=1,
min_duration=1,
cmdline="echo hello world"),
Process(name="b",
max_failures=2,
min_duration=1,
cmdline="exit 1"),
Process(name="c",
max_failures=1,
min_duration=1,
final=True,
cmdline="echo hello world")
],
constraints=[{
'order': ['a', 'b']
}])
return task.interpolate()[0]
def task(cls):
hello_template = Process(cmdline="sleep 1")
tsk = Task(name="complex",
processes=[
hello_template(name="process1"),
hello_template(name="process2"),
hello_template(name="process3")
],
resources=Resources(cpu=1.0,
ram=16 * 1024 * 1024,
disk=16 * 1024),
max_concurrency=1)
return tsk
def task(cls):
ping_template = Process(
name="{{process_name}}",
min_duration=1,
max_failures=5,
cmdline = "echo {{process_name}} pinging; "
"echo ping >> {{process_name}}; "
"echo current count $(cat {{process_name}} | wc -l); "
"if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
" exit 0; "
"else "
" exit 1; "
"fi ")
tsk = Task(
name = "pingping",
resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
processes = [
ping_template.bind(process_name = "p1", num_runs = 1),
ping_template.bind(process_name = "p2", num_runs = 2),
ping_template.bind(process_name = "p3", num_runs = 3),
]
)
return tsk.interpolate()[0]
def task(cls):
ping_template = Process(
name="{{process_name}}",
min_duration=1,
max_failures=5,
cmdline=
"echo {{process_name}} pinging; "
"echo ping >> {{process_name}}; "
"echo current count $(cat {{process_name}} | wc -l); "
"if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
" exit 0; "
"else "
" exit 1; "
"fi ")
tsk = Task(name="pingping",
resources=Resources(cpu=1.0,
ram=16 * 1024 * 1024,
disk=16 * 1024),
processes=[
ping_template.bind(process_name="p1", num_runs=1),
ping_template.bind(process_name="p2", num_runs=2),
ping_template.bind(process_name="p3", num_runs=3),
])
return tsk.interpolate()[0]
def test_tasklets():
install_thermosrc = Process(name='install_thermosrc')
setup_py3k = Process(name='setup_py3k')
setup_ruby19 = Process(name='setup_ruby19')
setup_php = Process(name='setup_php')
recipe_py3k = SequentialTask(processes=[install_thermosrc, setup_py3k])
recipe_ruby19 = SequentialTask(processes=[install_thermosrc, setup_ruby19])
recipe_php = SequentialTask(processes=[install_thermosrc, setup_php])
all_recipes = Tasks.combine(recipe_py3k, recipe_ruby19, recipe_php)
my_task = Task(processes = [Process(name='my_process')])
my_new_task = Tasks.concat(all_recipes, my_task)(name = 'my_task')
# TODO(wickman) Probably should have Tasks.combine/concat do constraint
# minimization since many constraints are redundant.
for p in (install_thermosrc, setup_py3k, setup_ruby19, setup_php):
assert p in my_new_task.processes()
def task(cls):
setup_bashrc = Process(name="setup_bashrc",
cmdline=dedent("""
mkdir -p .profile.d
cat <<EOF > .thermos_profile
for i in .profile.d/*.sh ; do
if [ -r "\\$i" ]; then
. \\$i
fi
done
EOF
"""))
setup_foo = Process(name="setup_foo",
cmdline=dedent("""
cat <<EOF > .profile.d/setup_foo.sh
export FOO=1
EOF
"""))
setup_bar = Process(name="setup_bar",
cmdline=dedent("""
cat <<EOF > .profile.d/setup_bar.sh
export BAR=2
EOF
"""))
foo_recipe = SequentialTask(processes=[setup_bashrc, setup_foo])
bar_recipe = SequentialTask(processes=[setup_bashrc, setup_bar])
all_recipes = Tasks.combine(foo_recipe, bar_recipe)
run = Process(name="run",
cmdline=dedent("""
echo $FOO $BAR > expected_output.txt
"""))
my_task = Task(processes=[run],
resources=Resources(cpu=1.0,
ram=16 * 1024 * 1024,
disk=16 * 1024))
return Tasks.concat(all_recipes, my_task, name="my_task")
def simplerun(args, options):
"""Run a simple command line as a thermos task.
Usage: thermos simplerun [options] [--] commandline
Options:
--user=USER run as this user. if not $USER, must have setuid privilege.
--name=STRING the name to give this task. ('simple' by default)
--task_id=STRING the id to which this task should be bound, synthesized from the
task name if none provided.
-P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified
multiple times to bind multiple names.)
-E/--environment=NAME=VALUE bind the configuration environment variable NAME to
VALUE.
--daemon Fork and daemonize the task.
"""
try:
cutoff = args.index('--')
cmdline = ' '.join(args[cutoff + 1:])
except ValueError:
cmdline = ' '.join(args)
print("Running command: '%s'" % cmdline)
thermos_task = ThermosTaskWrapper(
Task(name=options.name,
resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0),
processes=[Process(name=options.name, cmdline=cmdline)]))
_really_run(thermos_task,
options.root,
None,
task_id=options.task_id,
user=options.user,
prebound_ports=options.prebound_ports,
chroot=False,
daemon=options.daemon)
def task(cls):
task = Task(name="task", finalization_wait=3, processes=[ignorant_process(name="ignorant_process")])
return task.interpolate()[0]
def task(cls):
task = Task(name="task", processes=[sleepy_process(name="process")])
return task.interpolate()[0]
def task(cls): main = Process(name="main", cmdline="date && echo hello world") finalizer = Process(name="finalizer", cmdline="date", final=True) task = Task(name="task_with_finalizer", processes = [main, finalizer]) return task.interpolate()[0]
def task(cls):
main = Process(name="main", cmdline="date && echo hello world")
finalizer = Process(name="finalizer", cmdline="date", final=True)
task = Task(name="task_with_finalizer", processes=[main, finalizer])
return task.interpolate()[0]
def task(cls):
task = Task(name="task", processes=[Process(name="process", cmdline=SIMPLEFORK_SCRIPT)])
return task.interpolate()[0]
def task(cls):
task = Task(name="task", processes=[sleepy_process(name="process")])
return task.interpolate()[0]
def task(cls):
task = Task(
name="task",
processes=[Process(name="process", cmdline=SIMPLEFORK_SCRIPT)])
return task.interpolate()[0]
def task(cls):
task = Task(name="task",
finalization_wait=3,
processes=[ignorant_process(name="ignorant_process")])
return task.interpolate()[0]
Identity,
)
from pystachio import Map, String
from pystachio.naming import frozendict
import pytest
HELLO_WORLD = Job(name='hello_world',
role='john_doe',
environment='staging66',
cluster='smf1-test',
task=Task(
name='main',
processes=[
Process(name='hello_world',
cmdline='echo {{mesos.instance}}')
],
resources=Resources(cpu=0.1,
ram=64 * 1048576,
disk=64 * 1048576),
))
def test_simple_config():
job = convert_pystachio_to_thrift(HELLO_WORLD)
assert job.instanceCount == 1
tti = job.taskConfig
assert job.key == JobKey(role=HELLO_WORLD.role().get(),
environment=HELLO_WORLD.environment().get(),
name=HELLO_WORLD.name().get())
assert job.owner == Identity(role=HELLO_WORLD.role().get(),
user=getpass.getuser())
