def task(cls): main = Process(name="main", cmdline="date && echo hello world") finalizer = Process(name="finalizer", cmdline="date", final=True) task = Task(name="task_with_finalizer", processes=[main, finalizer(name='finalizer1'), finalizer(name='finalizer2')], constraints=[Constraint(order=['finalizer1', 'finalizer2'])]) return task.interpolate()[0]
def test_combine_tasks(): p1 = Process(name='p1') p2 = Process(name='p2') p3 = Process(name='p3') p4 = Process(name='p4') r100 = Resources(cpu=1, ram=0, disk=0) r010 = Resources(cpu=0, ram=1, disk=0) r001 = Resources(cpu=0, ram=0, disk=1) r111 = Units.resources_sum(r100, r010, r001) t1 = Task(name="p1p2", processes=[p1, p2], constraints=order(p1, p2), resources=Units.resources_sum(r100, r010)) t2 = Task(name="p3p4", processes=[p3, p4], constraints=order(p3, p4), resources=r001) assert combine_tasks() == Task() assert combine_tasks(t1) == t1 assert combine_tasks(t2) == t2 t3 = combine_tasks(t1, t2) assert t3.name() == t2.name() assert t3.resources() == r111 assert set(t3.processes()) == set([p1, p2, p3, p4]) assert set(t3.constraints()) == set(order(p1, p2) + order(p3, p4)) t4 = concat_tasks(t1, t2) assert t4.name() == t2.name() assert t4.resources() == r111 assert set(t4.processes()) == set([p1, p2, p3, p4]) assert set(t4.constraints()) == set( order(p1, p2) + order(p3, p4) + order(p1, p3) + order(p1, p4) + order(p2, p3) + order(p2, p4))
def test_combine_tasks(): p1 = Process(name='p1') p2 = Process(name='p2') p3 = Process(name='p3') p4 = Process(name='p4') r100 = Resources(cpu=1, ram=0, disk=0) r010 = Resources(cpu=0, ram=1, disk=0) r001 = Resources(cpu=0, ram=0, disk=1) r111 = Units.resources_sum(r100, r010, r001) t1 = Task(name="p1p2", processes=[p1, p2], constraints=order(p1, p2), resources=Units.resources_sum(r100, r010), finalization_wait=60) t2 = Task(name="p3p4", processes=[p3, p4], constraints=order(p3, p4), resources=r001, finalization_wait=45) assert combine_tasks() == Task() assert combine_tasks(t1) == t1 assert combine_tasks(t2) == t2 t3 = combine_tasks(t1, t2) assert t3.name() == t2.name() assert t3.resources() == r111 assert set(t3.processes()) == set([p1, p2, p3, p4]) assert set(t3.constraints()) == set(order(p1, p2) + order(p3, p4)) assert t3.finalization_wait().get() == t1.finalization_wait().get() t4 = concat_tasks(t1, t2) assert t4.name() == t2.name() assert t4.resources() == r111 assert set(t4.processes()) == set([p1, p2, p3, p4]) assert set(t4.constraints()) == set( order(p1, p2) + order(p3, p4) + order(p1, p3) + order(p1, p4) + order(p2, p3) + order(p2, p4)) assert t4.finalization_wait().get() == t1.finalization_wait().get() + t2.finalization_wait().get()
def task(cls): task = Task( name="task_with_ephemeral", processes=[ Process(name="ephemeral_sleepy", ephemeral=True, cmdline="sleep 10"), Process(name="sleepy", cmdline="sleep 1") ]) return task.interpolate()[0]
def task(cls): main = Process(name="main", cmdline="echo hello world") finalizer = Process(cmdline="date", final=True) task = Task(name="task_with_finalizer", processes=[main, finalizer(name='finalizer1', cmdline='exit 1', max_failures=2), finalizer(name='finalizer2')]) return task.interpolate()[0]
def task(cls): task = Task(name="task_with_ephemeral", processes=[ Process(name="ephemeral_sleepy", ephemeral=True, cmdline="sleep 10"), Process(name="sleepy", cmdline="sleep 1") ]) return task.interpolate()[0]
def task(cls): base = Process(max_failures=2, min_duration=1) ex = base(cmdline="exit 1") hw = base(cmdline="echo hello world") task = Task( name = "failing_task", resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024), max_failures = 0, processes = [ex(name='f1'), ex(name='f2'), ex(name='f3'), hw(name='s1'), hw(name='s2'), hw(name='s3')]) return task.interpolate()[0]
def task(cls): main = Process(name="main", cmdline="echo hello world") finalizer = Process(cmdline="date", final=True) task = Task(name="task_with_finalizer", processes=[ main, finalizer(name='finalizer1', cmdline='exit 1', max_failures=2), finalizer(name='finalizer2') ]) return task.interpolate()[0]
def task(cls): main = Process(name="main", cmdline="date && echo hello world") finalizer = Process(name="finalizer", cmdline="date", final=True) task = Task( name="task_with_finalizer", processes=[ main, finalizer(name='finalizer1', cmdline='exit 1'), finalizer(name='finalizer2') ], constraints=[Constraint(order=['finalizer1', 'finalizer2'])]) return task.interpolate()[0]
def flaky_task(): task = Task( name = "failing_task", max_failures = 2, processes = [ Process(name = "a", max_failures=1, min_duration=1, cmdline="echo hello world"), Process(name = "b", max_failures=2, min_duration=1, cmdline="exit 1"), Process(name = "c", max_failures=1, min_duration=1, final=True, cmdline="echo hello world") ], constraints = [{'order': ['a', 'b']}] ) return task.interpolate()[0]
def task(cls): task = Task( name = "failing_task", resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024), max_failures = 2, processes = [ Process(name = "a", max_failures=1, min_duration=1, cmdline="echo hello world"), Process(name = "b", max_failures=1, min_duration=1, cmdline="exit 1"), Process(name = "c", max_failures=1, min_duration=1, cmdline="echo hello world") ], constraints = [{'order': ['a', 'b', 'c']}] ) return task.interpolate()[0]
def simplerun(args, options): """Run a simple command line as a thermos task. Usage: thermos simplerun [options] [--] commandline """ try: cutoff = args.index('--') cmdline = ' '.join(args[cutoff + 1:]) except ValueError: cmdline = ' '.join(args) print("Running command: '%s'" % cmdline) thermos_task = ThermosTaskWrapper(Task( name=options.name, resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0), processes=[Process(name=options.name, cmdline=cmdline)])) really_run(thermos_task, options.root, tempfile.mkdtemp(), task_id=options.task_id, user=options.user, prebound_ports=options.prebound_ports, chroot=False, daemon=options.daemon)
def from_file(cls, filename, **kw): try: with safe_open(filename) as fp: task = Task.json_load(fp) return cls(task, **kw) except Exception: return None
def from_file(filename, **kw): try: with safe_open(filename) as fp: task = Task.json_load(fp) return ThermosTaskWrapper(task, **kw) except Exception as e: return None
def task(cls): hello_template = Process(cmdline="sleep 1") tsk = Task( name="complex", processes=[hello_template(name="process1"), hello_template(name="process2"), hello_template(name="process3")], resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024), max_concurrency=1) return tsk
def task(cls): ping_template = Process( name="{{process_name}}", min_duration=1, max_failures=5, cmdline = "echo {{process_name}} pinging; " "echo ping >> {{process_name}}; " "echo current count $(cat {{process_name}} | wc -l); " "if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then " " exit 0; " "else " " exit 1; " "fi ") tsk = Task( name = "pingping", resources = Resources(cpu = 1.0, ram = 16*1024*1024, disk = 16*1024), processes = [ ping_template.bind(process_name = "p1", num_runs = 1), ping_template.bind(process_name = "p2", num_runs = 2), ping_template.bind(process_name = "p3", num_runs = 3), ] ) return tsk.interpolate()[0]
def task(cls): ping_template = Process( name="{{process_name}}", min_duration=1, max_failures=5, cmdline= "echo {{process_name}} pinging; " "echo ping >> {{process_name}}; " "echo current count $(cat {{process_name}} | wc -l); " "if [ $(cat {{process_name}} | wc -l) -eq {{num_runs}} ]; then " " exit 0; " "else " " exit 1; " "fi ") tsk = Task(name="pingping", resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024), processes=[ ping_template.bind(process_name="p1", num_runs=1), ping_template.bind(process_name="p2", num_runs=2), ping_template.bind(process_name="p3", num_runs=3), ]) return tsk.interpolate()[0]
def test_tasklets(): install_thermosrc = Process(name='install_thermosrc') setup_py3k = Process(name='setup_py3k') setup_ruby19 = Process(name='setup_ruby19') setup_php = Process(name='setup_php') recipe_py3k = SequentialTask(processes=[install_thermosrc, setup_py3k]) recipe_ruby19 = SequentialTask(processes=[install_thermosrc, setup_ruby19]) recipe_php = SequentialTask(processes=[install_thermosrc, setup_php]) all_recipes = Tasks.combine(recipe_py3k, recipe_ruby19, recipe_php) my_task = Task(processes=[Process(name='my_process')]) my_new_task = Tasks.concat(all_recipes, my_task)(name='my_task') # TODO(wickman) Probably should have Tasks.combine/concat do constraint # minimization since many constraints are redundant. for p in (install_thermosrc, setup_py3k, setup_ruby19, setup_php): assert p in my_new_task.processes()
def task(cls): hello_template = Process(cmdline="echo 1") t1 = hello_template(name="t1", cmdline="echo 1 port {{thermos.ports[named_port]}}") t2 = hello_template(name="t2") t3 = hello_template(name="t3") t4 = hello_template(name="t4") t5 = hello_template(name="t5") t6 = hello_template(name="t6") tsk = Task(name="complex", processes=[t1, t2, t3, t4, t5, t6]) # three ways of tasks: t1 t2, t3 t4, t5 t6 tsk = tsk(constraints=[ {'order': ['t1', 't3']}, {'order': ['t1', 't4']}, {'order': ['t2', 't3']}, {'order': ['t2', 't4']}, {'order': ['t3', 't5']}, {'order': ['t3', 't6']}, {'order': ['t4', 't5']}, {'order': ['t4', 't6']}]) return tsk
def task(cls): setup_bashrc = Process(name="setup_bashrc", cmdline=dedent( """ mkdir -p .profile.d cat <<EOF > .thermos_profile for i in .profile.d/*.sh ; do if [ -r "\\$i" ]; then . \\$i fi done EOF """)) setup_foo = Process(name="setup_foo", cmdline=dedent( """ cat <<EOF > .profile.d/setup_foo.sh export FOO=1 EOF """)) setup_bar = Process(name="setup_bar", cmdline=dedent( """ cat <<EOF > .profile.d/setup_bar.sh export BAR=2 EOF """)) foo_recipe = SequentialTask(processes=[setup_bashrc, setup_foo]) bar_recipe = SequentialTask(processes=[setup_bashrc, setup_bar]) all_recipes = Tasks.combine(foo_recipe, bar_recipe) run = Process(name="run", cmdline=dedent( """ echo $FOO $BAR > expected_output.txt """)) my_task = Task(processes=[run], resources=Resources(cpu=1.0, ram=16 * 1024 * 1024, disk=16 * 1024)) return Tasks.concat(all_recipes, my_task, name="my_task")
def simplerun(args, options): """Run a simple command line as a thermos task. Usage: thermos simplerun [options] [--] commandline Options: --user=USER run as this user. if not $USER, must have setuid privilege. --name=STRING the name to give this task. ('simple' by default) --task_id=STRING the id to which this task should be bound, synthesized from the task name if none provided. -P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified multiple times to bind multiple names.) -E/--environment=NAME=VALUE bind the configuration environment variable NAME to VALUE. --daemon Fork and daemonize the task. """ try: cutoff = args.index('--') cmdline = ' '.join(args[cutoff + 1:]) except ValueError: cmdline = ' '.join(args) print("Running command: '%s'" % cmdline) thermos_task = ThermosTaskWrapper(Task( name=options.name, resources=Resources(cpu=1.0, ram=256 * 1024 * 1024, disk=0), processes=[Process(name=options.name, cmdline=cmdline)])) _really_run(thermos_task, options.root, None, task_id=options.task_id, user=options.user, prebound_ports=options.prebound_ports, chroot=False, daemon=options.daemon)
def task(cls): task = Task(name="task", processes=[sleepy_process(name="process")]) return task.interpolate()[0]
def task(cls): task = Task(name="task", finalization_wait=3, processes=[ignorant_process(name="ignorant_process")]) return task.interpolate()[0]
def test_mesos_hostname_in_task(): hw = HELLO_WORLD(task=Task(name="{{mesos.hostname}}")) instance = task_instance_from_job(hw, 0, 'test_host') assert str(instance.task().name()) == 'test_host'
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import pytest from apache.thermos.common.planner import TaskPlanner from apache.thermos.config.schema import Constraint, Process, Task p1 = Process(name="p1", cmdline="") p2 = Process(name="p2", cmdline="") p3 = Process(name="p3", cmdline="") unordered_task = Task(name="unordered", processes=[p1, p2, p3]) ordered_task = unordered_task(constraints=[{'order': ['p1', 'p2', 'p3']}]) empty_task = Task(name="empty", processes=[]) def _(*processes): # noqa return set(processes) empty = set() def approx_equal(a, b): return abs(a - b) < 0.001
def task(cls): main = Process(name="main", cmdline="date && echo hello world") finalizer = Process(name="finalizer", cmdline="date", final=True) task = Task(name="task_with_finalizer", processes = [main, finalizer]) return task.interpolate()[0]
def task(cls): main = Process(name="main", cmdline="date && echo hello world") finalizer = Process(name="finalizer", cmdline="date", final=True) task = Task(name="task_with_finalizer", processes=[main, finalizer]) return task.interpolate()[0]
def task(cls): task = Task( name="task", processes=[Process(name="process", cmdline=SIMPLEFORK_SCRIPT)]) return task.interpolate()[0]
from apache.aurora.config.thrift import convert as convert_pystachio_to_thrift from apache.aurora.config.thrift import InvalidConfig, task_instance_from_job from apache.thermos.config.schema import Process, Resources, Task from gen.apache.aurora.api.constants import GOOD_IDENTIFIER_PATTERN_PYTHON from gen.apache.aurora.api.ttypes import CronCollisionPolicy, Identity, JobKey from gen.apache.aurora.test.constants import INVALID_IDENTIFIERS, VALID_IDENTIFIERS HELLO_WORLD = Job( name='hello_world', role='john_doe', environment='staging66', cluster='smf1-test', task=Task( name='main', processes=[Process(name='hello_world', cmdline='echo {{mesos.instance}}')], resources=Resources(cpu=0.1, ram=64 * 1048576, disk=64 * 1048576), ) ) def test_simple_config(): job = convert_pystachio_to_thrift(HELLO_WORLD) expected_key = JobKey( role=HELLO_WORLD.role().get(), environment=HELLO_WORLD.environment().get(), name=HELLO_WORLD.name().get()) assert job.instanceCount == 1 tti = job.taskConfig assert job.key == expected_key assert job.owner == Identity(role=HELLO_WORLD.role().get(), user=getpass.getuser())
def task(cls): task = Task(name="task", processes=[Process(name="process", cmdline=SIMPLEFORK_SCRIPT)]) return task.interpolate()[0]