def test_combine_tasks(): p1 = Process(name='p1') p2 = Process(name='p2') p3 = Process(name='p3') p4 = Process(name='p4') r100 = Resources(cpu=1, ram=0, disk=0) r010 = Resources(cpu=0, ram=1, disk=0) r001 = Resources(cpu=0, ram=0, disk=1) r111 = Units.resources_sum(r100, r010, r001) t1 = Task(name="p1p2", processes=[p1, p2], constraints=order(p1, p2), resources=Units.resources_sum(r100, r010), finalization_wait=60) t2 = Task(name="p3p4", processes=[p3, p4], constraints=order(p3, p4), resources=r001, finalization_wait=45) assert combine_tasks() == Task() assert combine_tasks(t1) == t1 assert combine_tasks(t2) == t2 t3 = combine_tasks(t1, t2) assert t3.name() == t2.name() assert t3.resources() == r111 assert set(t3.processes()) == set([p1, p2, p3, p4]) assert set(t3.constraints()) == set(order(p1, p2) + order(p3, p4)) assert t3.finalization_wait().get() == t1.finalization_wait().get() t4 = concat_tasks(t1, t2) assert t4.name() == t2.name() assert t4.resources() == r111 assert set(t4.processes()) == set([p1, p2, p3, p4]) assert set(t4.constraints()) == set( order(p1, p2) + order(p3, p4) + order(p1, p3) + order(p1, p4) + order(p2, p3) + order(p2, p4)) assert t4.finalization_wait().get() == t1.finalization_wait().get() + t2.finalization_wait().get()
def task(cls): task = Task(name="task_with_ephemeral", processes=[ Process(name="ephemeral_sleepy", ephemeral=True, cmdline="sleep 10"), Process(name="sleepy", cmdline="sleep 1") ]) return task.interpolate()[0]
def task(cls): main = Process(name="main", cmdline="echo hello world") finalizer = Process(cmdline="date", final=True) task = Task(name="task_with_finalizer", processes=[ main, finalizer(name='finalizer1', cmdline='exit 1', max_failures=2), finalizer(name='finalizer2') ]) return task.interpolate()[0]
def task(cls): main = Process(name="main", cmdline="date && echo hello world") finalizer = Process(name="finalizer", cmdline="date", final=True) task = Task( name="task_with_finalizer", processes=[ main, finalizer(name='finalizer1', cmdline='exit 1'), finalizer(name='finalizer2') ], constraints=[Constraint(order=['finalizer1', 'finalizer2'])]) return task.interpolate()[0]
def simplerun(args, options): """Run a simple command line as a thermos task. Usage: thermos simplerun [options] [--] commandline """ try: cutoff = args.index('--') cmdline = ' '.join(args[cutoff + 1:]) except ValueError: cmdline = ' '.join(args) print("Running command: '%s'" % cmdline) thermos_task = ThermosTaskWrapper(Task( name=options.name, resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0), processes=[Process(name=options.name, cmdline=cmdline)])) really_run(thermos_task, options.root, tempfile.mkdtemp(), task_id=options.task_id, user=options.user, prebound_ports=options.prebound_ports, chroot=False, daemon=options.daemon)
def test_order(): p1 = Process(name='p1') p2 = Process(name='p2') p1p2 = [Constraint(order=['p1', 'p2'])] assert order(p1, p2) == p1p2 assert order('p1', p2) == p1p2 assert order(p1, 'p2') == p1p2 assert order('p1', 'p2') == p1p2 assert order(u'p1', 'p2') == p1p2 with pytest.raises(ValueError): order([p1]) with pytest.raises(ValueError): order(None)
def test_tasklets(): install_thermosrc = Process(name='install_thermosrc') setup_py3k = Process(name='setup_py3k') setup_ruby19 = Process(name='setup_ruby19') setup_php = Process(name='setup_php') recipe_py3k = SequentialTask(processes=[install_thermosrc, setup_py3k]) recipe_ruby19 = SequentialTask(processes=[install_thermosrc, setup_ruby19]) recipe_php = SequentialTask(processes=[install_thermosrc, setup_php]) all_recipes = Tasks.combine(recipe_py3k, recipe_ruby19, recipe_php) my_task = Task(processes=[Process(name='my_process')]) my_new_task = Tasks.concat(all_recipes, my_task)(name='my_task') # TODO(wickman) Probably should have Tasks.combine/concat do constraint # minimization since many constraints are redundant. for p in (install_thermosrc, setup_py3k, setup_ruby19, setup_php): assert p in my_new_task.processes()
def test_config_with_ports(): hwc = HELLO_WORLD(task=HELLO_WORLD.task()(processes=[ Process( name='hello_world', cmdline='echo {{thermos.ports[http]}} {{thermos.ports[admin]}}') ])) config = AuroraConfig(hwc) job = config.job() assert job.taskConfig.requestedPorts == set(['http', 'admin'])
def test_config_with_ports(): hwc = HELLO_WORLD(task=HELLO_WORLD.task()(processes=[ Process( name='hello_world', cmdline='echo {{thermos.ports[http]}} {{thermos.ports[admin]}}') ])) config = AuroraConfig(hwc) job = config.job() assert Resource(namedPort='http') in list(job.taskConfig.resources) assert Resource(namedPort='admin') in list(job.taskConfig.resources)
def task(cls): hello_template = Process(cmdline="sleep 1") tsk = Task( name="complex", processes=[hello_template(name="process1"), hello_template(name="process2"), hello_template(name="process3")], resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024), max_concurrency=1) return tsk
def task(cls): setup_bashrc = Process(name="setup_bashrc", cmdline=dedent( """ mkdir -p .profile.d cat <<EOF > .thermos_profile for i in .profile.d/*.sh ; do if [ -r "\\$i" ]; then . \\$i fi done EOF """)) setup_foo = Process(name="setup_foo", cmdline=dedent( """ cat <<EOF > .profile.d/setup_foo.sh export FOO=1 EOF """)) setup_bar = Process(name="setup_bar", cmdline=dedent( """ cat <<EOF > .profile.d/setup_bar.sh export BAR=2 EOF """)) foo_recipe = SequentialTask(processes=[setup_bashrc, setup_foo]) bar_recipe = SequentialTask(processes=[setup_bashrc, setup_bar]) all_recipes = Tasks.combine(foo_recipe, bar_recipe) run = Process(name="run", cmdline=dedent( """ echo $FOO $BAR > expected_output.txt """)) my_task = Task(processes=[run], resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024)) return Tasks.concat(all_recipes, my_task, name="my_task")
def task(cls): ping_template = Process( name="{{process_name}}", min_duration=1, max_failures=5, cmdline = "echo {{process_name}} pinging; " "echo ping >> {{process_name}}; " "echo current count $(cat {{process_name}} | wc -l); " "if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then " " exit 0; " "else " " exit 1; " "fi ") tsk = Task( name = "pingping", resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024), processes = [ ping_template.bind(process_name = "p1", num_runs = 1), ping_template.bind(process_name = "p2", num_runs = 2), ping_template.bind(process_name = "p3", num_runs = 3), ] ) return tsk.interpolate()[0]
def task(cls): ping_template = Process( name="{{process_name}}", min_duration=1, max_failures=5, cmdline= "echo {{process_name}} pinging; " "echo ping >> {{process_name}}; " "echo current count $(cat {{process_name}} | wc -l); " "if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then " " exit 0; " "else " " exit 1; " "fi ") tsk = Task(name="pingping", resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024), processes=[ ping_template.bind(process_name="p1", num_runs=1), ping_template.bind(process_name="p2", num_runs=2), ping_template.bind(process_name="p3", num_runs=3), ]) return tsk.interpolate()[0]
def task(cls): hello_template = Process(cmdline="echo 1") t1 = hello_template(name="t1", cmdline="echo 1 port {{thermos.ports[named_port]}}") t2 = hello_template(name="t2") t3 = hello_template(name="t3") t4 = hello_template(name="t4") t5 = hello_template(name="t5") t6 = hello_template(name="t6") tsk = Task(name="complex", processes=[t1, t2, t3, t4, t5, t6]) # three ways of tasks: t1 t2, t3 t4, t5 t6 tsk = tsk(constraints=[ {'order': ['t1', 't3']}, {'order': ['t1', 't4']}, {'order': ['t2', 't3']}, {'order': ['t2', 't4']}, {'order': ['t3', 't5']}, {'order': ['t3', 't6']}, {'order': ['t4', 't5']}, {'order': ['t4', 't6']}]) return tsk
def simplerun(args, options): """Run a simple command line as a thermos task. Usage: thermos simplerun [options] [--] commandline Options: --user=USER run as this user. if not $USER, must have setuid privilege. --name=STRING the name to give this task. ('simple' by default) --task_id=STRING the id to which this task should be bound, synthesized from the task name if none provided. -P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified multiple times to bind multiple names.) -E/--environment=NAME=VALUE bind the configuration environment variable NAME to VALUE. --daemon Fork and daemonize the task. """ try: cutoff = args.index('--') cmdline = ' '.join(args[cutoff + 1:]) except ValueError: cmdline = ' '.join(args) print("Running command: '%s'" % cmdline) thermos_task = ThermosTaskWrapper(Task( name=options.name, resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0), processes=[Process(name=options.name, cmdline=cmdline)])) _really_run(thermos_task, options.root, None, task_id=options.task_id, user=options.user, prebound_ports=options.prebound_ports, chroot=False, daemon=options.daemon)
from apache.aurora.config.thrift import convert as convert_pystachio_to_thrift from apache.aurora.config.thrift import InvalidConfig, task_instance_from_job from apache.thermos.config.schema import Process, Resources, Task from gen.apache.aurora.api.constants import GOOD_IDENTIFIER_PATTERN_PYTHON from gen.apache.aurora.api.ttypes import CronCollisionPolicy, Identity, JobKey from gen.apache.aurora.test.constants import INVALID_IDENTIFIERS, VALID_IDENTIFIERS HELLO_WORLD = Job( name='hello_world', role='john_doe', environment='staging66', cluster='smf1-test', task=Task( name='main', processes=[Process(name='hello_world', cmdline='echo {{mesos.instance}}')], resources=Resources(cpu=0.1, ram=64 * 1048576, disk=64 * 1048576), ) ) def test_simple_config(): job = convert_pystachio_to_thrift(HELLO_WORLD) expected_key = JobKey( role=HELLO_WORLD.role().get(), environment=HELLO_WORLD.environment().get(), name=HELLO_WORLD.name().get()) assert job.instanceCount == 1 tti = job.taskConfig assert job.key == expected_key assert job.owner == Identity(role=HELLO_WORLD.role().get(), user=getpass.getuser())
def task(cls): task = Task( name="task", processes=[Process(name="process", cmdline=SIMPLEFORK_SCRIPT)]) return task.interpolate()[0]
import sys import threading import time import pytest from twitter.common.process import ProcessProviderFactory from twitter.common.quantity import Amount, Time from apache.thermos.config.schema import Process, Task from apache.thermos.core.runner import TaskRunner from apache.thermos.monitoring.monitor import TaskMonitor from apache.thermos.testing.runner import Runner from gen.apache.thermos.ttypes import ProcessState, TaskState sleepy_process = Process(name="sleepy", cmdline="sleep 3", min_duration=1) ignore_script = [ "import time, signal", "signal.signal(signal.SIGTERM, signal.SIG_IGN)", "time.sleep(1000)" ] ignorant_process = Process(name="ignorant", cmdline="%s -c '%s'" % (sys.executable, ';'.join(ignore_script)), min_duration=1) class RunnerBase(object): @classmethod def task(cls):
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import pytest from apache.thermos.common.planner import TaskPlanner from apache.thermos.config.schema import Constraint, Process, Task p1 = Process(name="p1", cmdline="") p2 = Process(name="p2", cmdline="") p3 = Process(name="p3", cmdline="") unordered_task = Task(name="unordered", processes=[p1, p2, p3]) ordered_task = unordered_task(constraints=[{'order': ['p1', 'p2', 'p3']}]) empty_task = Task(name="empty", processes=[]) def _(*processes): # noqa return set(processes) empty = set()
def task(cls): main = Process(name="main", cmdline="date && echo hello world") finalizer = Process(name="finalizer", cmdline="date", final=True) task = Task(name="task_with_finalizer", processes=[main, finalizer]) return task.interpolate()[0]