def test__create_a_stateless_job_with_3_tasks_on_3_different_hosts():
label_key = "job.name"
label_value = "peloton_stateless_job"
job = Job(
job_file="test_stateless_job.yaml",
config=IntegrationTestConfig(
max_retry_attempts=100,
pool_file='test_stateless_respool.yaml',
),
options=[
with_labels({label_key: label_value}),
with_constraint(_label_constraint(label_key, label_value)),
with_instance_count(3),
],
)
job.create()
job.wait_for_state(goal_state="RUNNING")
# Determine if tasks run on different hosts
hosts = set()
for _, task in job.get_tasks().iteritems():
task_info = task.get_info()
hosts = hosts.union({task_info.runtime.host})
kill_jobs([job])
# Ensure that the tasks run on 3 different hosts
assert len(hosts) == 3
def test_placement_exclusive_job(exclusive_host, peloton_client):
excl_constraint = task_pb2.Constraint(
type=1, # Label constraint
labelConstraint=task_pb2.LabelConstraint(
kind=2, # Host
condition=2, # Equal
requirement=1,
label=peloton_pb2.Label(
key="peloton/exclusive", value="exclusive-test-label"
),
),
)
# Set number of instances to be a few more than what can run on
# a single exclusive host
job = Job(
client=peloton_client,
job_file="long_running_job.yaml",
config=IntegrationTestConfig(max_retry_attempts=100, sleep_time_sec=2),
options=[with_constraint(excl_constraint), with_instance_count(6)],
)
job.job_config.defaultConfig.command.value = "sleep 10"
job.create()
job.wait_for_state()
# check that all of them ran on exclusive host
task_infos = job.list_tasks().value
for instance_id, task_info in task_infos.items():
assert "exclusive" in task_info.runtime.host
def test_placement_strategy_spread():
job = Job(
job_file="test_task.yaml",
options=[with_instance_count(3)])
job.job_config.placementStrategy = job_pb2.PLACEMENT_STRATEGY_SPREAD_JOB
job.create()
job.wait_for_state()
# check all of them ran on different hosts
hosts = set()
task_infos = job.list_tasks().value
for instance_id, task_info in task_infos.items():
assert task_info.runtime.host not in hosts
hosts.add(task_info.runtime.host)
def test_placement_non_exclusive_job(exclusive_host):
# Set number of instances to be a few more than what can run on
# 2 (non-exclusive) hosts
job = Job(job_file='long_running_job.yaml',
config=IntegrationTestConfig(max_retry_attempts=100,
sleep_time_sec=2),
options=[with_instance_count(12)])
job.job_config.defaultConfig.command.value = "sleep 10"
job.create()
job.wait_for_state()
# check that none of them ran on exclusive host
task_infos = job.list_tasks().value
for instance_id, task_info in task_infos.items():
assert "exclusive" not in task_info.runtime.host
def test_placement_strategy_pack():
job = Job(
job_file="test_task.yaml",
options=[with_instance_count(5)])
job.job_config.placementStrategy = job_pb2.PLACEMENT_STRATEGY_PACK_HOST
job.create()
job.wait_for_state()
# check all of them ran on same host
the_host = ""
task_infos = job.list_tasks().value
for instance_id, task_info in task_infos.items():
if the_host:
assert task_info.runtime.host == the_host
the_host = task_info.runtime.host
def test_placement_strategy_pack():
job = Job(job_file="test_task.yaml", options=[with_instance_count(5)])
"""
TODO Uncomment next line after peloton-client changes
#job.job_config.placementStrategy = "PLACEMENT_STRATEGY_PACK_HOST"
"""
job.create()
job.wait_for_state()
# check all of them ran on same host
the_host = ""
task_infos = job.list_tasks().value
for instance_id, task_info in task_infos.items():
if the_host:
assert task_info.runtime.host == the_host
the_host = task_info.runtime.host
def test_task_killed_in_ready_succeeds_when_re_enqueued(placement_engines):
# Tests that a if task is deleted which is in READY state in resource
# manager and if is re-enqueued succeeds.
# stop the placement engines to keep the tasks in READY state
placement_engines.stop()
# decorate the client to add peloton private API stubs
c = with_private_stubs(Client())
# create long running job with 2 instances
long_running_job = Job(
job_file='long_running_job.yaml',
options=[
with_instance_count(2),
],
client=c,
)
long_running_job.create()
long_running_job.wait_for_state(goal_state='PENDING')
task = long_running_job.get_task(0)
# wait for task to reach READY
task.wait_for_pending_state(goal_state='READY')
# kill the task
task.stop()
# re-enqueue the task
task.start()
# gentlemen, start your (placement) engines
placement_engines.start()
def wait_for_instance_to_run():
return long_running_job.get_task(0).state_str == 'RUNNING'
long_running_job.wait_for_condition(wait_for_instance_to_run)
def test__create_2_stateless_jobs_with_task_to_task_anti_affinity_between_jobs(
): # noqa
label_key = "job.name"
jobs = []
for i in range(2):
job = Job(
job_file="test_stateless_job.yaml",
config=IntegrationTestConfig(
max_retry_attempts=100,
pool_file='test_stateless_respool.yaml',
),
options=[
with_labels({label_key: "peloton_stateless_job%s" % i}),
with_job_name("TestPelotonDockerJob_Stateless" + repr(i)),
with_instance_count(1),
],
)
job.job_config.defaultConfig.constraint.CopyFrom(
task_pb2.Constraint(
type=2,
andConstraint=task_pb2.AndConstraint(constraints=[
task_pb2.Constraint(
type=1,
labelConstraint=task_pb2.LabelConstraint(
kind=1,
condition=2,
requirement=0,
label=peloton_pb2.Label(
# Tasks of my own job
key="job.name",
value="peloton_stateless_job%s" % i,
),
),
),
task_pb2.Constraint(
type=1,
labelConstraint=task_pb2.LabelConstraint(
kind=1,
condition=2,
requirement=0,
label=peloton_pb2.Label(
# Avoid tasks of the other job
key="job.name",
value="peloton_stateless_job%s" %
((i + 1) % 2),
),
),
),
]),
))
jobs.append(job)
for job in jobs:
job.create()
time.sleep(1)
# Determine if tasks run on different hosts
hosts = set()
for job in jobs:
job.wait_for_state(goal_state="RUNNING")
for _, task in job.get_tasks().iteritems():
task_info = task.get_info()
hosts = hosts.union(set({task_info.runtime.host}))
kill_jobs(jobs)
# Ensure that the tasks run on 2 different hosts
assert len(hosts) == 2
def test__tasks_reserve_execution(hostreservepool, peloton_client):
p_job_median = Job(
client=peloton_client,
job_file='test_hostreservation_job_median.yaml',
pool=hostreservepool,
config=IntegrationTestConfig(
max_retry_attempts=100,
sleep_time_sec=1),
)
p_job_median.create()
p_job_median.wait_for_state(goal_state='RUNNING')
# we should have all 3 tasks in running state
def all_running():
return all(t.state == task.RUNNING for t in p_job_median.get_tasks().values())
p_job_median.wait_for_condition(all_running)
# decorate the client to add peloton private API stubs
client = with_private_stubs(peloton_client)
p_job_large = Job(
job_file='test_hostreservation_job_large.yaml',
pool=hostreservepool,
config=IntegrationTestConfig(
sleep_time_sec=1,
max_retry_attempts=300),
options=[with_instance_count(1)],
client=client,
)
p_job_large.create()
p_job_large.wait_for_state(goal_state='PENDING')
request = hostmgr.GetHostsByQueryRequest()
# task should get into reserved state and RUNNING state
t1 = p_job_large.get_task(0)
t1.wait_for_pending_state(goal_state="RESERVED")
# the task is running on reserved host
def get_reserved_host():
resp = client.hostmgr_svc.GetHostsByQuery(
request,
metadata=p_job_large.client.hostmgr_metadata,
timeout=p_job_large.config.rpc_timeout_sec,)
for h in resp.hosts:
if h.status == 'reserved':
return h.hostname
return ''
def is_reserved():
return get_reserved_host() != ''
p_job_large.wait_for_condition(is_reserved)
reserved_host = get_reserved_host()
t1.wait_for_pending_state(goal_state="RUNNING")
assert reserved_host == t1.get_info().runtime.host
# p_job_large should succeed
p_job_large.wait_for_state()
# no host is in reserved state
response = client.hostmgr_svc.GetHostsByQuery(
request,
metadata=p_job_large.client.hostmgr_metadata,
timeout=p_job_large.config.rpc_timeout_sec,)
for host in response.hosts:
assert host.status != 'reserved'
kill_jobs([p_job_median, p_job_large])
