def test_combine_tasks():
p1 = Process(name='p1')
p2 = Process(name='p2')
p3 = Process(name='p3')
p4 = Process(name='p4')
r100 = Resources(cpu=1, ram=0, disk=0)
r010 = Resources(cpu=0, ram=1, disk=0)
r001 = Resources(cpu=0, ram=0, disk=1)
r111 = Units.resources_sum(r100, r010, r001)
t1 = Task(name="p1p2", processes=[p1, p2], constraints=order(p1, p2),
resources=Units.resources_sum(r100, r010), finalization_wait=60)
t2 = Task(name="p3p4", processes=[p3, p4], constraints=order(p3, p4),
resources=r001, finalization_wait=45)
assert combine_tasks() == Task()
assert combine_tasks(t1) == t1
assert combine_tasks(t2) == t2
t3 = combine_tasks(t1, t2)
assert t3.name() == t2.name()
assert t3.resources() == r111
assert set(t3.processes()) == set([p1, p2, p3, p4])
assert set(t3.constraints()) == set(order(p1, p2) + order(p3, p4))
assert t3.finalization_wait().get() == t1.finalization_wait().get()
t4 = concat_tasks(t1, t2)
assert t4.name() == t2.name()
assert t4.resources() == r111
assert set(t4.processes()) == set([p1, p2, p3, p4])
assert set(t4.constraints()) == set(
order(p1, p2) + order(p3, p4) + order(p1, p3) + order(p1, p4) +
order(p2, p3) + order(p2, p4))
assert t4.finalization_wait().get() == t1.finalization_wait().get() + t2.finalization_wait().get()
def test_resource_manager_disk_exceeded():
resources = Resources(cpu=1, ram=128, disk=256)
resource_manager = ResourceManager(
resources,
_mock_resource_monitor(disk_usage=resources.disk().get() * 2))
result = resource_manager.status
assert result is not None
assert result.reason.startswith('Disk limit exceeded')
assert result.status == mesos_pb2.TASK_FAILED
def test_add_resources(): assert Units.resources_sum(Resources(), Resources()) == Resources(cpu=0, ram=0, disk=0) r100 = Resources(cpu=1, ram=0, disk=0) r010 = Resources(cpu=0, ram=1, disk=0) r001 = Resources(cpu=0, ram=0, disk=1) r111 = Resources(cpu=1, ram=1, disk=1) r222 = Resources(cpu=2, ram=2, disk=2) assert reduce(Units.resources_sum, [r100, r010, r001]) == r111 assert Units.resources_sum(r111, r111) == r222 assert r222 == Units.resources_sum(r100, r010, r001, r111, Resources())
def simplerun(args, options):
"""Run a simple command line as a thermos task.
Usage: thermos simplerun [options] [--] commandline
"""
try:
cutoff = args.index('--')
cmdline = ' '.join(args[cutoff + 1:])
except ValueError:
cmdline = ' '.join(args)
print("Running command: '%s'" % cmdline)
thermos_task = ThermosTaskWrapper(Task(
name=options.name,
resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0),
processes=[Process(name=options.name, cmdline=cmdline)]))
really_run(thermos_task,
options.root,
tempfile.mkdtemp(),
task_id=options.task_id,
user=options.user,
prebound_ports=options.prebound_ports,
chroot=False,
daemon=options.daemon)
def task(cls):
hello_template = Process(cmdline="sleep 1")
tsk = Task(
name="complex",
processes=[hello_template(name="process1"),
hello_template(name="process2"),
hello_template(name="process3")],
resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024),
max_concurrency=1)
return tsk
def test_simple_task():
name, command = 'simple_thing', 'echo hello'
st = SimpleTask(name, command)
assert isinstance(st, Task)
assert st.name().get() == name
assert len(st.constraints().get()) == 0
assert len(st.processes().get()) == 1
assert st.processes().get()[0]['cmdline'] == command
assert st.resources() == Resources(cpu=Tasks.SIMPLE_CPU,
ram=Tasks.SIMPLE_RAM,
disk=Tasks.SIMPLE_DISK)
def task(cls):
setup_bashrc = Process(name="setup_bashrc", cmdline=dedent(
"""
mkdir -p .profile.d
cat <<EOF > .thermos_profile
for i in .profile.d/*.sh ; do
if [ -r "\\$i" ]; then
. \\$i
fi
done
EOF
"""))
setup_foo = Process(name="setup_foo", cmdline=dedent(
"""
cat <<EOF > .profile.d/setup_foo.sh
export FOO=1
EOF
"""))
setup_bar = Process(name="setup_bar", cmdline=dedent(
"""
cat <<EOF > .profile.d/setup_bar.sh
export BAR=2
EOF
"""))
foo_recipe = SequentialTask(processes=[setup_bashrc, setup_foo])
bar_recipe = SequentialTask(processes=[setup_bashrc, setup_bar])
all_recipes = Tasks.combine(foo_recipe, bar_recipe)
run = Process(name="run", cmdline=dedent(
"""
echo $FOO $BAR > expected_output.txt
"""))
my_task = Task(processes=[run],
resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024))
return Tasks.concat(all_recipes, my_task, name="my_task")
def simplerun(args, options):
"""Run a simple command line as a thermos task.
Usage: thermos simplerun [options] [--] commandline
Options:
--user=USER run as this user. if not $USER, must have setuid privilege.
--name=STRING the name to give this task. ('simple' by default)
--task_id=STRING the id to which this task should be bound, synthesized from the
task name if none provided.
-P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified
multiple times to bind multiple names.)
-E/--environment=NAME=VALUE bind the configuration environment variable NAME to
VALUE.
--daemon Fork and daemonize the task.
"""
try:
cutoff = args.index('--')
cmdline = ' '.join(args[cutoff + 1:])
except ValueError:
cmdline = ' '.join(args)
print("Running command: '%s'" % cmdline)
thermos_task = ThermosTaskWrapper(Task(
name=options.name,
resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0),
processes=[Process(name=options.name, cmdline=cmdline)]))
_really_run(thermos_task,
options.root,
None,
task_id=options.task_id,
user=options.user,
prebound_ports=options.prebound_ports,
chroot=False,
daemon=options.daemon)
def task(cls):
ping_template = Process(
name="{{process_name}}",
min_duration=1,
max_failures=5,
cmdline=
"echo {{process_name}} pinging; "
"echo ping >> {{process_name}}; "
"echo current count $(cat {{process_name}} | wc -l); "
"if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then "
" exit 0; "
"else "
" exit 1; "
"fi ")
tsk = Task(name="pingping",
resources=Resources(cpu=1.0,
ram=16 * 1024 * 1024,
disk=16 * 1024),
processes=[
ping_template.bind(process_name="p1", num_runs=1),
ping_template.bind(process_name="p2", num_runs=2),
ping_template.bind(process_name="p3", num_runs=3),
])
return tsk.interpolate()[0]
def test_config_with_bad_resources():
MB = 1048576
hwtask = HELLO_WORLD.task()
convert_pystachio_to_thrift(HELLO_WORLD)
good_resources = [
Resources(cpu=1.0, ram=1 * MB, disk=1 * MB)
]
bad_resources = [
Resources(cpu=0, ram=1 * MB, disk=1 * MB),
Resources(cpu=1, ram=0 * MB, disk=1 * MB),
Resources(cpu=1, ram=1 * MB, disk=0 * MB),
Resources(cpu=1, ram=1 * MB - 1, disk=1 * MB),
Resources(cpu=1, ram=1 * MB, disk=1 * MB - 1)
]
for resource in good_resources:
convert_pystachio_to_thrift(HELLO_WORLD(task=hwtask(resources=resource)))
for resource in bad_resources:
with pytest.raises(ValueError):
convert_pystachio_to_thrift(HELLO_WORLD(task=hwtask(resources=resource)))
from apache.aurora.config.thrift import InvalidConfig, task_instance_from_job
from apache.thermos.config.schema import Process, Resources, Task
from gen.apache.aurora.api.constants import GOOD_IDENTIFIER_PATTERN_PYTHON
from gen.apache.aurora.api.ttypes import CronCollisionPolicy, Identity, JobKey
from gen.apache.aurora.test.constants import INVALID_IDENTIFIERS, VALID_IDENTIFIERS
HELLO_WORLD = Job(
name='hello_world',
role='john_doe',
environment='staging66',
cluster='smf1-test',
task=Task(
name='main',
processes=[Process(name='hello_world', cmdline='echo {{mesos.instance}}')],
resources=Resources(cpu=0.1, ram=64 * 1048576, disk=64 * 1048576),
)
)
def test_simple_config():
job = convert_pystachio_to_thrift(HELLO_WORLD)
expected_key = JobKey(
role=HELLO_WORLD.role().get(),
environment=HELLO_WORLD.environment().get(),
name=HELLO_WORLD.name().get())
assert job.instanceCount == 1
tti = job.taskConfig
assert job.key == expected_key
assert job.owner == Identity(role=HELLO_WORLD.role().get(), user=getpass.getuser())
assert job.cronSchedule is None
def test_resource_manager():
resource_manager = ResourceManager(
Resources(cpu=1, ram=128, disk=256),
_mock_resource_monitor())
assert resource_manager.status is None
def test_max_resources():
assert Resources(cpu=1, ram=2, disk=3, gpu=4) == Units.resources_max([
Resources(cpu=0, ram=2, disk=1, gpu=4),
Resources(cpu=1, ram=1, disk=2, gpu=0),
Resources(cpu=0, ram=1, disk=3, gpu=1)
])
def test_add_resources():
assert Units.resources_sum(Resources(), Resources()) == Resources(cpu=0,
ram=0,
disk=0,
gpu=0)
r1000 = Resources(cpu=1, ram=0, disk=0, gpu=0)
r1001 = Resources(cpu=1, ram=0, disk=0, gpu=1)
r0100 = Resources(cpu=0, ram=1, disk=0, gpu=0)
r0010 = Resources(cpu=0, ram=0, disk=1, gpu=0)
r1110 = Resources(cpu=1, ram=1, disk=1, gpu=0)
r1101 = Resources(cpu=1, ram=1, disk=0, gpu=1)
r2220 = Resources(cpu=2, ram=2, disk=2, gpu=0)
assert reduce(Units.resources_sum, [r1000, r0100, r0010]) == r1110
assert Units.resources_sum(r1110, r1110) == r2220
assert r2220 == Units.resources_sum(r1000, r0100, r0010, r1110,
Resources())
assert Units.resources_sum(r1001, r0100) == r1101