def test_combine_tasks():
p1 = Process(name='p1')
p2 = Process(name='p2')
p3 = Process(name='p3')
p4 = Process(name='p4')
r100 = Resources(cpu=1, ram=0, disk=0)
r010 = Resources(cpu=0, ram=1, disk=0)
r001 = Resources(cpu=0, ram=0, disk=1)
r111 = Units.resources_sum(r100, r010, r001)
t1 = Task(name="p1p2", processes=[p1, p2], constraints=order(p1, p2),
resources=Units.resources_sum(r100, r010))
t2 = Task(name="p3p4", processes=[p3, p4], constraints=order(p3, p4),
resources=r001)
assert combine_tasks() == Task()
assert combine_tasks(t1) == t1
assert combine_tasks(t2) == t2
t3 = combine_tasks(t1, t2)
assert t3.name() == t2.name()
assert t3.resources() == r111
assert set(t3.processes()) == set([p1, p2, p3, p4])
assert set(t3.constraints()) == set(order(p1, p2) + order(p3, p4))
t4 = concat_tasks(t1, t2)
assert t4.name() == t2.name()
assert t4.resources() == r111
assert set(t4.processes()) == set([p1, p2, p3, p4])
assert set(t4.constraints()) == set(
order(p1, p2) + order(p3, p4) + order(p1, p3) + order(p1, p4) +
order(p2, p3) + order(p2, p4))
def task(cls):
task = Task(name="task_with_ephemeral",
processes=[
Process(name="ephemeral_sleepy",
ephemeral=True,
cmdline="sleep 10"),
Process(name="sleepy", cmdline="sleep 1")
])
return task.interpolate()[0]
def task(cls):
main = Process(name="main", cmdline="date && echo hello world")
finalizer = Process(name="finalizer", cmdline="date", final=True)
task = Task(
name="task_with_finalizer",
processes=[
main,
finalizer(name='finalizer1', cmdline='exit 1'),
finalizer(name='finalizer2')
],
constraints=[Constraint(order=['finalizer1', 'finalizer2'])])
return task.interpolate()[0]
def task(cls):
main = Process(name="main", cmdline="echo hello world")
finalizer = Process(cmdline="date", final=True)
task = Task(name="task_with_finalizer",
processes=[
main,
finalizer(name='finalizer1',
cmdline='exit 1',
max_failures=2),
finalizer(name='finalizer2')
])
return task.interpolate()[0]
def task(cls):
task = Task(
name = "failing_task",
resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
max_failures = 2,
processes = [
Process(name = "a", max_failures=1, min_duration=1, cmdline="echo hello world"),
Process(name = "b", max_failures=1, min_duration=1, cmdline="exit 1"),
Process(name = "c", max_failures=1, min_duration=1, cmdline="echo hello world")
],
constraints = [{'order': ['a', 'b', 'c']}]
)
return task.interpolate()[0]
def test_order():
p1 = Process(name='p1')
p2 = Process(name='p2')
p1p2 = [Constraint(order=['p1', 'p2'])]
assert order(p1, p2) == p1p2
assert order('p1', p2) == p1p2
assert order(p1, 'p2') == p1p2
assert order('p1', 'p2') == p1p2
assert order(u'p1', 'p2') == p1p2
with pytest.raises(ValueError):
order([p1])
with pytest.raises(ValueError):
order(None)
def test_tasklets():
install_thermosrc = Process(name='install_thermosrc')
setup_py3k = Process(name='setup_py3k')
setup_ruby19 = Process(name='setup_ruby19')
setup_php = Process(name='setup_php')
recipe_py3k = SequentialTask(processes=[install_thermosrc, setup_py3k])
recipe_ruby19 = SequentialTask(processes=[install_thermosrc, setup_ruby19])
recipe_php = SequentialTask(processes=[install_thermosrc, setup_php])
all_recipes = Tasks.combine(recipe_py3k, recipe_ruby19, recipe_php)
my_task = Task(processes = [Process(name='my_process')])
my_new_task = Tasks.concat(all_recipes, my_task)(name = 'my_task')
# TODO(wickman) Probably should have Tasks.combine/concat do constraint
# minimization since many constraints are redundant.
for p in (install_thermosrc, setup_py3k, setup_ruby19, setup_php):
assert p in my_new_task.processes()
def task(cls):
hello_template = Process(cmdline="echo 1")
t1 = hello_template(
name="t1", cmdline="echo 1 port {{thermos.ports[named_port]}}")
t2 = hello_template(name="t2")
t3 = hello_template(name="t3")
t4 = hello_template(name="t4")
t5 = hello_template(name="t5")
t6 = hello_template(name="t6")
tsk = Task(name="complex", processes=[t1, t2, t3, t4, t5, t6])
# three ways of tasks: t1 t2, t3 t4, t5 t6
tsk = tsk(constraints=[{
'order': ['t1', 't3']
}, {
'order': ['t1', 't4']
}, {
'order': ['t2', 't3']
}, {
'order': ['t2', 't4']
}, {
'order': ['t3', 't5']
}, {
'order': ['t3', 't6']
}, {
'order': ['t4', 't5']
}, {
'order': ['t4', 't6']
}])
return tsk
def test_config_with_ports():
hwc = HELLO_WORLD(task=HELLO_WORLD.task()(processes=[
Process(
name='hello_world',
cmdline='echo {{thermos.ports[http]}} {{thermos.ports[admin]}}')
]))
config = AuroraConfig(hwc)
job = config.job()
assert job.taskConfig.requestedPorts == set(['http', 'admin'])
def task(cls):
base = Process(max_failures=2, min_duration=1)
ex = base(cmdline="exit 1")
hw = base(cmdline="echo hello world")
task = Task(
name = "failing_task",
resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
max_failures = 0,
processes = [ex(name='f1'), ex(name='f2'), ex(name='f3'),
hw(name='s1'), hw(name='s2'), hw(name='s3')])
return task.interpolate()[0]
def task(cls):
setup_bashrc = Process(name="setup_bashrc",
cmdline=dedent("""
mkdir -p .profile.d
cat <<EOF > .thermos_profile
for i in .profile.d/*.sh ; do
if [ -r "\\$i" ]; then
. \\$i
fi
done
EOF
"""))
setup_foo = Process(name="setup_foo",
cmdline=dedent("""
cat <<EOF > .profile.d/setup_foo.sh
export FOO=1
EOF
"""))
setup_bar = Process(name="setup_bar",
cmdline=dedent("""
cat <<EOF > .profile.d/setup_bar.sh
export BAR=2
EOF
"""))
foo_recipe = SequentialTask(processes=[setup_bashrc, setup_foo])
bar_recipe = SequentialTask(processes=[setup_bashrc, setup_bar])
all_recipes = Tasks.combine(foo_recipe, bar_recipe)
run = Process(name="run",
cmdline=dedent("""
echo $FOO $BAR > expected_output.txt
"""))
my_task = Task(processes=[run],
resources=Resources(cpu=1.0,
ram=16 * 1024 * 1024,
disk=16 * 1024))
return Tasks.concat(all_recipes, my_task, name="my_task")
def flaky_task():
task = Task(name="failing_task",
max_failures=2,
processes=[
Process(name="a",
max_failures=1,
min_duration=1,
cmdline="echo hello world"),
Process(name="b",
max_failures=2,
min_duration=1,
cmdline="exit 1"),
Process(name="c",
max_failures=1,
min_duration=1,
final=True,
cmdline="echo hello world")
],
constraints=[{
'order': ['a', 'b']
}])
return task.interpolate()[0]
def task(cls):
hello_template = Process(cmdline="sleep 1")
tsk = Task(name="complex",
processes=[
hello_template(name="process1"),
hello_template(name="process2"),
hello_template(name="process3")
],
resources=Resources(cpu=1.0,
ram=16 * 1024 * 1024,
disk=16 * 1024),
max_concurrency=1)
return tsk
def task(cls):
ping_template = Process(
name="{{process_name}}",
min_duration=1,
max_failures=5,
cmdline = "echo {{process_name}} pinging; "
"echo ping >> {{process_name}}; "
"echo current count $(cat {{process_name}} | wc -l); "
"if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
" exit 0; "
"else "
" exit 1; "
"fi ")
tsk = Task(
name = "pingping",
resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
processes = [
ping_template.bind(process_name = "p1", num_runs = 1),
ping_template.bind(process_name = "p2", num_runs = 2),
ping_template.bind(process_name = "p3", num_runs = 3),
]
)
return tsk.interpolate()[0]
def task(cls):
ping_template = Process(
name="{{process_name}}",
min_duration=1,
max_failures=5,
cmdline=
"echo {{process_name}} pinging; "
"echo ping >> {{process_name}}; "
"echo current count $(cat {{process_name}} | wc -l); "
"if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
" exit 0; "
"else "
" exit 1; "
"fi ")
tsk = Task(name="pingping",
resources=Resources(cpu=1.0,
ram=16 * 1024 * 1024,
disk=16 * 1024),
processes=[
ping_template.bind(process_name="p1", num_runs=1),
ping_template.bind(process_name="p2", num_runs=2),
ping_template.bind(process_name="p3", num_runs=3),
])
return tsk.interpolate()[0]
def simplerun(args, options):
"""Run a simple command line as a thermos task.
Usage: thermos simplerun [options] [--] commandline
Options:
--user=USER run as this user. if not $USER, must have setuid privilege.
--name=STRING the name to give this task. ('simple' by default)
--task_id=STRING the id to which this task should be bound, synthesized from the
task name if none provided.
-P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified
multiple times to bind multiple names.)
-E/--environment=NAME=VALUE bind the configuration environment variable NAME to
VALUE.
--daemon Fork and daemonize the task.
"""
try:
cutoff = args.index('--')
cmdline = ' '.join(args[cutoff + 1:])
except ValueError:
cmdline = ' '.join(args)
print("Running command: '%s'" % cmdline)
thermos_task = ThermosTaskWrapper(
Task(name=options.name,
resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0),
processes=[Process(name=options.name, cmdline=cmdline)]))
_really_run(thermos_task,
options.root,
None,
task_id=options.task_id,
user=options.user,
prebound_ports=options.prebound_ports,
chroot=False,
daemon=options.daemon)
JobKey,
Identity,
)
from pystachio import Map, String
from pystachio.naming import frozendict
import pytest
HELLO_WORLD = Job(name='hello_world',
role='john_doe',
environment='staging66',
cluster='smf1-test',
task=Task(
name='main',
processes=[
Process(name='hello_world',
cmdline='echo {{mesos.instance}}')
],
resources=Resources(cpu=0.1,
ram=64 * 1048576,
disk=64 * 1048576),
))
def test_simple_config():
job = convert_pystachio_to_thrift(HELLO_WORLD)
assert job.instanceCount == 1
tti = job.taskConfig
assert job.key == JobKey(role=HELLO_WORLD.role().get(),
environment=HELLO_WORLD.environment().get(),
name=HELLO_WORLD.name().get())
assert job.owner == Identity(role=HELLO_WORLD.role().get(),
import sys
import threading
import time
from twitter.common.process import ProcessProviderFactory
from twitter.common.quantity import Amount, Time
from twitter.thermos.config.schema import (Task, Resources, Process)
from twitter.thermos.core.runner import TaskRunner
from twitter.thermos.monitoring.monitor import TaskMonitor
from twitter.thermos.testing.runner import Runner
from gen.twitter.thermos.ttypes import (TaskState, ProcessState)
import pytest
sleepy_process = Process(name="sleepy", cmdline="sleep 3", min_duration=1)
ignore_script = [
"import time, signal", "signal.signal(signal.SIGTERM, signal.SIG_IGN)",
"time.sleep(1000)"
]
ignorant_process = Process(name="ignorant",
cmdline="%s -c '%s'" %
(sys.executable, ';'.join(ignore_script)),
min_duration=1)
class RunnerBase(object):
@classmethod
def task(cls):
def task(cls):
task = Task(
name="task",
processes=[Process(name="process", cmdline=SIMPLEFORK_SCRIPT)])
return task.interpolate()[0]
def task(cls):
main = Process(name="main", cmdline="date && echo hello world")
finalizer = Process(name="finalizer", cmdline="date", final=True)
task = Task(name="task_with_finalizer", processes=[main, finalizer])
return task.interpolate()[0]
