def test_combine_tasks():
p1 = Process(name='p1')
p2 = Process(name='p2')
p3 = Process(name='p3')
p4 = Process(name='p4')
r100 = Resources(cpu=1, ram=0, disk=0)
r010 = Resources(cpu=0, ram=1, disk=0)
r001 = Resources(cpu=0, ram=0, disk=1)
r111 = Units.resources_sum(r100, r010, r001)
t1 = Task(name="p1p2", processes=[p1, p2], constraints=order(p1, p2),
resources=Units.resources_sum(r100, r010), finalization_wait=60)
t2 = Task(name="p3p4", processes=[p3, p4], constraints=order(p3, p4),
resources=r001, finalization_wait=45)
assert combine_tasks() == Task()
assert combine_tasks(t1) == t1
assert combine_tasks(t2) == t2
t3 = combine_tasks(t1, t2)
assert t3.name() == t2.name()
assert t3.resources() == r111
assert set(t3.processes()) == set([p1, p2, p3, p4])
assert set(t3.constraints()) == set(order(p1, p2) + order(p3, p4))
assert t3.finalization_wait().get() == t1.finalization_wait().get()
t4 = concat_tasks(t1, t2)
assert t4.name() == t2.name()
assert t4.resources() == r111
assert set(t4.processes()) == set([p1, p2, p3, p4])
assert set(t4.constraints()) == set(
order(p1, p2) + order(p3, p4) + order(p1, p3) + order(p1, p4) +
order(p2, p3) + order(p2, p4))
assert t4.finalization_wait().get() == t1.finalization_wait().get() + t2.finalization_wait().get()
def task(cls):
task = Task(name="task_with_ephemeral",
processes=[
Process(name="ephemeral_sleepy",
ephemeral=True,
cmdline="sleep 10"),
Process(name="sleepy", cmdline="sleep 1")
])
return task.interpolate()[0]
def task(cls):
main = Process(name="main", cmdline="echo hello world")
finalizer = Process(cmdline="date", final=True)
task = Task(name="task_with_finalizer",
processes=[
main,
finalizer(name='finalizer1',
cmdline='exit 1',
max_failures=2),
finalizer(name='finalizer2')
])
return task.interpolate()[0]
def task(cls):
main = Process(name="main", cmdline="date && echo hello world")
finalizer = Process(name="finalizer", cmdline="date", final=True)
task = Task(
name="task_with_finalizer",
processes=[
main,
finalizer(name='finalizer1', cmdline='exit 1'),
finalizer(name='finalizer2')
],
constraints=[Constraint(order=['finalizer1', 'finalizer2'])])
return task.interpolate()[0]
def simplerun(args, options):
"""Run a simple command line as a thermos task.
Usage: thermos simplerun [options] [--] commandline
"""
try:
cutoff = args.index('--')
cmdline = ' '.join(args[cutoff + 1:])
except ValueError:
cmdline = ' '.join(args)
print("Running command: '%s'" % cmdline)
thermos_task = ThermosTaskWrapper(Task(
name=options.name,
resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0),
processes=[Process(name=options.name, cmdline=cmdline)]))
really_run(thermos_task,
options.root,
tempfile.mkdtemp(),
task_id=options.task_id,
user=options.user,
prebound_ports=options.prebound_ports,
chroot=False,
daemon=options.daemon)
def test_order():
p1 = Process(name='p1')
p2 = Process(name='p2')
p1p2 = [Constraint(order=['p1', 'p2'])]
assert order(p1, p2) == p1p2
assert order('p1', p2) == p1p2
assert order(p1, 'p2') == p1p2
assert order('p1', 'p2') == p1p2
assert order(u'p1', 'p2') == p1p2
with pytest.raises(ValueError):
order([p1])
with pytest.raises(ValueError):
order(None)
def test_tasklets():
install_thermosrc = Process(name='install_thermosrc')
setup_py3k = Process(name='setup_py3k')
setup_ruby19 = Process(name='setup_ruby19')
setup_php = Process(name='setup_php')
recipe_py3k = SequentialTask(processes=[install_thermosrc, setup_py3k])
recipe_ruby19 = SequentialTask(processes=[install_thermosrc, setup_ruby19])
recipe_php = SequentialTask(processes=[install_thermosrc, setup_php])
all_recipes = Tasks.combine(recipe_py3k, recipe_ruby19, recipe_php)
my_task = Task(processes=[Process(name='my_process')])
my_new_task = Tasks.concat(all_recipes, my_task)(name='my_task')
# TODO(wickman) Probably should have Tasks.combine/concat do constraint
# minimization since many constraints are redundant.
for p in (install_thermosrc, setup_py3k, setup_ruby19, setup_php):
assert p in my_new_task.processes()
def test_config_with_ports():
hwc = HELLO_WORLD(task=HELLO_WORLD.task()(processes=[
Process(
name='hello_world',
cmdline='echo {{thermos.ports[http]}} {{thermos.ports[admin]}}')
]))
config = AuroraConfig(hwc)
job = config.job()
assert job.taskConfig.requestedPorts == set(['http', 'admin'])
def test_config_with_ports():
hwc = HELLO_WORLD(task=HELLO_WORLD.task()(processes=[
Process(
name='hello_world',
cmdline='echo {{thermos.ports[http]}} {{thermos.ports[admin]}}')
]))
config = AuroraConfig(hwc)
job = config.job()
assert Resource(namedPort='http') in list(job.taskConfig.resources)
assert Resource(namedPort='admin') in list(job.taskConfig.resources)
def task(cls):
hello_template = Process(cmdline="sleep 1")
tsk = Task(
name="complex",
processes=[hello_template(name="process1"),
hello_template(name="process2"),
hello_template(name="process3")],
resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024),
max_concurrency=1)
return tsk
def task(cls):
setup_bashrc = Process(name="setup_bashrc", cmdline=dedent(
"""
mkdir -p .profile.d
cat <<EOF > .thermos_profile
for i in .profile.d/*.sh ; do
if [ -r "\\$i" ]; then
. \\$i
fi
done
EOF
"""))
setup_foo = Process(name="setup_foo", cmdline=dedent(
"""
cat <<EOF > .profile.d/setup_foo.sh
export FOO=1
EOF
"""))
setup_bar = Process(name="setup_bar", cmdline=dedent(
"""
cat <<EOF > .profile.d/setup_bar.sh
export BAR=2
EOF
"""))
foo_recipe = SequentialTask(processes=[setup_bashrc, setup_foo])
bar_recipe = SequentialTask(processes=[setup_bashrc, setup_bar])
all_recipes = Tasks.combine(foo_recipe, bar_recipe)
run = Process(name="run", cmdline=dedent(
"""
echo $FOO $BAR > expected_output.txt
"""))
my_task = Task(processes=[run],
resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024))
return Tasks.concat(all_recipes, my_task, name="my_task")
def task(cls):
ping_template = Process(
name="{{process_name}}",
min_duration=1,
max_failures=5,
cmdline = "echo {{process_name}} pinging; "
"echo ping >> {{process_name}}; "
"echo current count $(cat {{process_name}} | wc -l); "
"if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
" exit 0; "
"else "
" exit 1; "
"fi ")
tsk = Task(
name = "pingping",
resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024),
processes = [
ping_template.bind(process_name = "p1", num_runs = 1),
ping_template.bind(process_name = "p2", num_runs = 2),
ping_template.bind(process_name = "p3", num_runs = 3),
]
)
return tsk.interpolate()[0]
def task(cls):
ping_template = Process(
name="{{process_name}}",
min_duration=1,
max_failures=5,
cmdline=
"echo {{process_name}} pinging; "
"echo ping >> {{process_name}}; "
"echo current count $(cat {{process_name}} | wc -l); "
"if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
" exit 0; "
"else "
" exit 1; "
"fi ")
tsk = Task(name="pingping",
resources=Resources(cpu=1.0,
ram=16 * 1024 * 1024,
disk=16 * 1024),
processes=[
ping_template.bind(process_name="p1", num_runs=1),
ping_template.bind(process_name="p2", num_runs=2),
ping_template.bind(process_name="p3", num_runs=3),
])
return tsk.interpolate()[0]
def task(cls):
hello_template = Process(cmdline="echo 1")
t1 = hello_template(name="t1", cmdline="echo 1 port {{thermos.ports[named_port]}}")
t2 = hello_template(name="t2")
t3 = hello_template(name="t3")
t4 = hello_template(name="t4")
t5 = hello_template(name="t5")
t6 = hello_template(name="t6")
tsk = Task(name="complex", processes=[t1, t2, t3, t4, t5, t6])
# three ways of tasks: t1 t2, t3 t4, t5 t6
tsk = tsk(constraints=[
{'order': ['t1', 't3']},
{'order': ['t1', 't4']},
{'order': ['t2', 't3']},
{'order': ['t2', 't4']},
{'order': ['t3', 't5']},
{'order': ['t3', 't6']},
{'order': ['t4', 't5']},
{'order': ['t4', 't6']}])
return tsk
def simplerun(args, options):
"""Run a simple command line as a thermos task.
Usage: thermos simplerun [options] [--] commandline
Options:
--user=USER run as this user. if not $USER, must have setuid privilege.
--name=STRING the name to give this task. ('simple' by default)
--task_id=STRING the id to which this task should be bound, synthesized from the
task name if none provided.
-P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified
multiple times to bind multiple names.)
-E/--environment=NAME=VALUE bind the configuration environment variable NAME to
VALUE.
--daemon Fork and daemonize the task.
"""
try:
cutoff = args.index('--')
cmdline = ' '.join(args[cutoff + 1:])
except ValueError:
cmdline = ' '.join(args)
print("Running command: '%s'" % cmdline)
thermos_task = ThermosTaskWrapper(Task(
name=options.name,
resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0),
processes=[Process(name=options.name, cmdline=cmdline)]))
_really_run(thermos_task,
options.root,
None,
task_id=options.task_id,
user=options.user,
prebound_ports=options.prebound_ports,
chroot=False,
daemon=options.daemon)
from apache.aurora.config.thrift import convert as convert_pystachio_to_thrift
from apache.aurora.config.thrift import InvalidConfig, task_instance_from_job
from apache.thermos.config.schema import Process, Resources, Task
from gen.apache.aurora.api.constants import GOOD_IDENTIFIER_PATTERN_PYTHON
from gen.apache.aurora.api.ttypes import CronCollisionPolicy, Identity, JobKey
from gen.apache.aurora.test.constants import INVALID_IDENTIFIERS, VALID_IDENTIFIERS
HELLO_WORLD = Job(
name='hello_world',
role='john_doe',
environment='staging66',
cluster='smf1-test',
task=Task(
name='main',
processes=[Process(name='hello_world', cmdline='echo {{mesos.instance}}')],
resources=Resources(cpu=0.1, ram=64 * 1048576, disk=64 * 1048576),
)
)
def test_simple_config():
job = convert_pystachio_to_thrift(HELLO_WORLD)
expected_key = JobKey(
role=HELLO_WORLD.role().get(),
environment=HELLO_WORLD.environment().get(),
name=HELLO_WORLD.name().get())
assert job.instanceCount == 1
tti = job.taskConfig
assert job.key == expected_key
assert job.owner == Identity(role=HELLO_WORLD.role().get(), user=getpass.getuser())
def task(cls):
task = Task(
name="task",
processes=[Process(name="process", cmdline=SIMPLEFORK_SCRIPT)])
return task.interpolate()[0]
import sys
import threading
import time
import pytest
from twitter.common.process import ProcessProviderFactory
from twitter.common.quantity import Amount, Time
from apache.thermos.config.schema import Process, Task
from apache.thermos.core.runner import TaskRunner
from apache.thermos.monitoring.monitor import TaskMonitor
from apache.thermos.testing.runner import Runner
from gen.apache.thermos.ttypes import ProcessState, TaskState
sleepy_process = Process(name="sleepy", cmdline="sleep 3", min_duration=1)
ignore_script = [
"import time, signal", "signal.signal(signal.SIGTERM, signal.SIG_IGN)",
"time.sleep(1000)"
]
ignorant_process = Process(name="ignorant",
cmdline="%s -c '%s'" %
(sys.executable, ';'.join(ignore_script)),
min_duration=1)
class RunnerBase(object):
@classmethod
def task(cls):
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from apache.thermos.common.planner import TaskPlanner
from apache.thermos.config.schema import Constraint, Process, Task
p1 = Process(name="p1", cmdline="")
p2 = Process(name="p2", cmdline="")
p3 = Process(name="p3", cmdline="")
unordered_task = Task(name="unordered", processes=[p1, p2, p3])
ordered_task = unordered_task(constraints=[{'order': ['p1', 'p2', 'p3']}])
empty_task = Task(name="empty", processes=[])
def _(*processes): # noqa
return set(processes)
empty = set()
def task(cls):
main = Process(name="main", cmdline="date && echo hello world")
finalizer = Process(name="finalizer", cmdline="date", final=True)
task = Task(name="task_with_finalizer", processes=[main, finalizer])
return task.interpolate()[0]
