def test__create_a_stateless_job_with_3_tasks_on_3_different_hosts(): label_key = "job.name" label_value = "peloton_stateless_job" job = Job( job_file="test_stateless_job.yaml", config=IntegrationTestConfig( max_retry_attempts=100, pool_file='test_stateless_respool.yaml', ), options=[ with_labels({label_key: label_value}), with_constraint(_label_constraint(label_key, label_value)), with_instance_count(3), ], ) job.create() job.wait_for_state(goal_state="RUNNING") # Determine if tasks run on different hosts hosts = set() for _, task in job.get_tasks().iteritems(): task_info = task.get_info() hosts = hosts.union({task_info.runtime.host}) kill_jobs([job]) # Ensure that the tasks run on 3 different hosts assert len(hosts) == 3
def test_placement_exclusive_job(exclusive_host, peloton_client): excl_constraint = task_pb2.Constraint( type=1, # Label constraint labelConstraint=task_pb2.LabelConstraint( kind=2, # Host condition=2, # Equal requirement=1, label=peloton_pb2.Label( key="peloton/exclusive", value="exclusive-test-label" ), ), ) # Set number of instances to be a few more than what can run on # a single exclusive host job = Job( client=peloton_client, job_file="long_running_job.yaml", config=IntegrationTestConfig(max_retry_attempts=100, sleep_time_sec=2), options=[with_constraint(excl_constraint), with_instance_count(6)], ) job.job_config.defaultConfig.command.value = "sleep 10" job.create() job.wait_for_state() # check that all of them ran on exclusive host task_infos = job.list_tasks().value for instance_id, task_info in task_infos.items(): assert "exclusive" in task_info.runtime.host
def test_placement_strategy_spread(): job = Job( job_file="test_task.yaml", options=[with_instance_count(3)]) job.job_config.placementStrategy = job_pb2.PLACEMENT_STRATEGY_SPREAD_JOB job.create() job.wait_for_state() # check all of them ran on different hosts hosts = set() task_infos = job.list_tasks().value for instance_id, task_info in task_infos.items(): assert task_info.runtime.host not in hosts hosts.add(task_info.runtime.host)
def test_placement_non_exclusive_job(exclusive_host): # Set number of instances to be a few more than what can run on # 2 (non-exclusive) hosts job = Job(job_file='long_running_job.yaml', config=IntegrationTestConfig(max_retry_attempts=100, sleep_time_sec=2), options=[with_instance_count(12)]) job.job_config.defaultConfig.command.value = "sleep 10" job.create() job.wait_for_state() # check that none of them ran on exclusive host task_infos = job.list_tasks().value for instance_id, task_info in task_infos.items(): assert "exclusive" not in task_info.runtime.host
def test_placement_strategy_pack(): job = Job( job_file="test_task.yaml", options=[with_instance_count(5)]) job.job_config.placementStrategy = job_pb2.PLACEMENT_STRATEGY_PACK_HOST job.create() job.wait_for_state() # check all of them ran on same host the_host = "" task_infos = job.list_tasks().value for instance_id, task_info in task_infos.items(): if the_host: assert task_info.runtime.host == the_host the_host = task_info.runtime.host
def test_placement_strategy_pack(): job = Job(job_file="test_task.yaml", options=[with_instance_count(5)]) """ TODO Uncomment next line after peloton-client changes #job.job_config.placementStrategy = "PLACEMENT_STRATEGY_PACK_HOST" """ job.create() job.wait_for_state() # check all of them ran on same host the_host = "" task_infos = job.list_tasks().value for instance_id, task_info in task_infos.items(): if the_host: assert task_info.runtime.host == the_host the_host = task_info.runtime.host
def test_task_killed_in_ready_succeeds_when_re_enqueued(placement_engines): # Tests that a if task is deleted which is in READY state in resource # manager and if is re-enqueued succeeds. # stop the placement engines to keep the tasks in READY state placement_engines.stop() # decorate the client to add peloton private API stubs c = with_private_stubs(Client()) # create long running job with 2 instances long_running_job = Job( job_file='long_running_job.yaml', options=[ with_instance_count(2), ], client=c, ) long_running_job.create() long_running_job.wait_for_state(goal_state='PENDING') task = long_running_job.get_task(0) # wait for task to reach READY task.wait_for_pending_state(goal_state='READY') # kill the task task.stop() # re-enqueue the task task.start() # gentlemen, start your (placement) engines placement_engines.start() def wait_for_instance_to_run(): return long_running_job.get_task(0).state_str == 'RUNNING' long_running_job.wait_for_condition(wait_for_instance_to_run)
def test__create_2_stateless_jobs_with_task_to_task_anti_affinity_between_jobs( ): # noqa label_key = "job.name" jobs = [] for i in range(2): job = Job( job_file="test_stateless_job.yaml", config=IntegrationTestConfig( max_retry_attempts=100, pool_file='test_stateless_respool.yaml', ), options=[ with_labels({label_key: "peloton_stateless_job%s" % i}), with_job_name("TestPelotonDockerJob_Stateless" + repr(i)), with_instance_count(1), ], ) job.job_config.defaultConfig.constraint.CopyFrom( task_pb2.Constraint( type=2, andConstraint=task_pb2.AndConstraint(constraints=[ task_pb2.Constraint( type=1, labelConstraint=task_pb2.LabelConstraint( kind=1, condition=2, requirement=0, label=peloton_pb2.Label( # Tasks of my own job key="job.name", value="peloton_stateless_job%s" % i, ), ), ), task_pb2.Constraint( type=1, labelConstraint=task_pb2.LabelConstraint( kind=1, condition=2, requirement=0, label=peloton_pb2.Label( # Avoid tasks of the other job key="job.name", value="peloton_stateless_job%s" % ((i + 1) % 2), ), ), ), ]), )) jobs.append(job) for job in jobs: job.create() time.sleep(1) # Determine if tasks run on different hosts hosts = set() for job in jobs: job.wait_for_state(goal_state="RUNNING") for _, task in job.get_tasks().iteritems(): task_info = task.get_info() hosts = hosts.union(set({task_info.runtime.host})) kill_jobs(jobs) # Ensure that the tasks run on 2 different hosts assert len(hosts) == 2
def test__tasks_reserve_execution(hostreservepool, peloton_client): p_job_median = Job( client=peloton_client, job_file='test_hostreservation_job_median.yaml', pool=hostreservepool, config=IntegrationTestConfig( max_retry_attempts=100, sleep_time_sec=1), ) p_job_median.create() p_job_median.wait_for_state(goal_state='RUNNING') # we should have all 3 tasks in running state def all_running(): return all(t.state == task.RUNNING for t in p_job_median.get_tasks().values()) p_job_median.wait_for_condition(all_running) # decorate the client to add peloton private API stubs client = with_private_stubs(peloton_client) p_job_large = Job( job_file='test_hostreservation_job_large.yaml', pool=hostreservepool, config=IntegrationTestConfig( sleep_time_sec=1, max_retry_attempts=300), options=[with_instance_count(1)], client=client, ) p_job_large.create() p_job_large.wait_for_state(goal_state='PENDING') request = hostmgr.GetHostsByQueryRequest() # task should get into reserved state and RUNNING state t1 = p_job_large.get_task(0) t1.wait_for_pending_state(goal_state="RESERVED") # the task is running on reserved host def get_reserved_host(): resp = client.hostmgr_svc.GetHostsByQuery( request, metadata=p_job_large.client.hostmgr_metadata, timeout=p_job_large.config.rpc_timeout_sec,) for h in resp.hosts: if h.status == 'reserved': return h.hostname return '' def is_reserved(): return get_reserved_host() != '' p_job_large.wait_for_condition(is_reserved) reserved_host = get_reserved_host() t1.wait_for_pending_state(goal_state="RUNNING") assert reserved_host == t1.get_info().runtime.host # p_job_large should succeed p_job_large.wait_for_state() # no host is in reserved state response = client.hostmgr_svc.GetHostsByQuery( request, metadata=p_job_large.client.hostmgr_metadata, timeout=p_job_large.config.rpc_timeout_sec,) for host in response.hosts: assert host.status != 'reserved' kill_jobs([p_job_median, p_job_large])