def test__get_jobs__get_job_summary(client): # Verify no jobs are returned before jobs are created. res = client.get_job_summary("") assert len(res.summaries) == 0 res = client.get_jobs("") assert len(res.configs) == 0 # Create two jobs under same role. test_dc_labrat_key = start_job_update(client, "test_dc_labrat_read.yaml", "start job update test/dc/labrat") test_dc_labrat_0_key = start_job_update( client, "test_dc_labrat0.yaml", "start job update test/dc/labrat0") # Different role should not show up. start_job_update(client, "test2_dc2_labrat2.yaml", "start job update test2/dc2/labrat2") # Add some wait time for lucene index to build time.sleep(10) # reduce instance count by 1 for test/dc/labrat0 client.kill_tasks( test_dc_labrat_0_key, {0}, "killing instance 0 for task test/dc/labrat0", ) wait_for_killed(client, test_dc_labrat_0_key, {0}) # Ensure get_job_summary returns both jobs under role=test. res = client.get_job_summary(test_dc_labrat_key.role) assert len(res.summaries) == 2, "{jobs}".format( jobs=[s.job.key for s in res.summaries]) assert_keys_equal( [s.job.key for s in res.summaries], [test_dc_labrat_key, test_dc_labrat_0_key], ) for s in res.summaries: if s.job.key == test_dc_labrat_0_key: assert s.stats.activeTaskCount == 1 else: assert s.stats.activeTaskCount == 2 assert s.job.instanceCount == 2 # Ensure get_jobs returns both jobs under role=test. res = client.get_jobs(test_dc_labrat_key.role) assert len(res.configs) == 2 assert_keys_equal( [c.taskConfig.job for c in res.configs], [test_dc_labrat_key, test_dc_labrat_0_key], ) for c in res.configs: if c.key == test_dc_labrat_0_key: assert c.instanceCount == 1 else: assert c.instanceCount == 2
def test__start_job_update_revocable_job(client): """ Given 12 non-revocable cpus, and 12 revocable cpus Create a non-revocable of 3 instance, with 3 CPU per instance Create a revocable job of 1 instance, with 4 CPU per instance """ non_revocable_job = start_job_update( client, "test_dc_labrat_cpus_large.yaml", "start job update test/dc/labrat_large", ) revocable_job = start_job_update( client, "test_dc_labrat_revocable.yaml", "start job update test/dc/labrat_revocable", ) # Add some wait time for lucene index to build time.sleep(10) # validate 1 revocable tasks are running res = client.get_tasks_without_configs( api.TaskQuery(jobKeys={revocable_job}, statuses={api.ScheduleStatus.RUNNING})) assert len(res.tasks) == 1 # validate 3 non-revocable tasks are running res = client.get_tasks_without_configs( api.TaskQuery(jobKeys={non_revocable_job}, statuses={api.ScheduleStatus.RUNNING})) assert len(res.tasks) == 3
def test__get_job_update_details__deleted_job(client): """ test JobMgr's private API - QueryJobCache (used by getJobUpdateDetails) won't crash if the job is deleted. """ # start first update req1 = get_job_update_request("test_dc_labrat_large_job.yaml") req1.settings.updateGroupSize = 10 job_key = start_job_update(client, req1, "start job update test/dc/labrat_large_job") # force delete job delete_jobs() # start second update req2 = get_job_update_request("test_dc_labrat_large_job_diff_labels.yaml") req2.settings.updateGroupSize = 10 job_key = start_job_update(client, req2, "start job update test/dc/labrat_large_job") # verify getJobUpdateDetails response res = client.get_job_update_details(None, api.JobUpdateQuery(role=job_key.role)) assert len(res.detailsList) == 1
def test__simple_auto_rolled_back(client): """ Create a job, then issue a bad config update and validate job is rolled back to previous version """ start_job_update(client, 'test_dc_labrat.yaml', 'start job update test/dc/labrat') # Add some wait time for lucene index to build time.sleep(10) res = client.start_job_update( get_job_update_request('test_dc_labrat_bad_config.yaml'), 'rollout bad config') wait_for_rolled_back(client, res.key) # validate job is rolled back to previous config res = client.get_tasks_without_configs( api.TaskQuery(jobKeys={res.key.job}, statuses={api.ScheduleStatus.RUNNING})) tasks = res.tasks assert len(tasks) == 3 for t in tasks: for r in t.assignedTask.task.resources: if r.numCpus > 0: assert r.numCpus == 0.25 elif r.ramMb > 0: assert r.ramMb == 128 elif r.diskMb > 0: assert r.diskMb == 128 else: assert False, 'unexpected resource {}'.format(r)
def test__abort_auto_rollback_and_update(client): """ 1. Create a job 2. Start a bad update, wait for auto-rollback to kick-in 3. Once auto-rollback starts, abort an update. 4. Do a new good update and all the instances should converge to the new config. """ start_job_update( client, "test_dc_labrat_large_job.yaml", "start job update test/dc/labrat_large_job", ) # Add some wait time for lucene index to build time.sleep(10) res = client.start_job_update( get_job_update_request("test_dc_labrat_large_job_bad_config.yaml"), "rollout bad config", ) # wait for auto-rollback to kick-in wait_for_auto_rolling_back(client, res.key) client.abort_job_update(res.key, "abort update") wait_for_update_status( client, res.key, {api.JobUpdateStatus.ROLLING_BACK}, api.JobUpdateStatus.ABORTED, ) new_config = get_job_update_request( "test_dc_labrat_large_job_new_config.yaml" ) res = client.start_job_update(new_config, "rollout good config") # Sleep for a while so that update gets triggered. time.sleep(5) wait_for_rolled_forward(client, res.key) res = client.get_tasks_without_configs( api.TaskQuery( jobKeys={res.key.job}, statuses={api.ScheduleStatus.RUNNING} ) ) assert len(res.tasks) == 10 for t in res.tasks: assert len(t.assignedTask.task.metadata) == 1 assert ( list(t.assignedTask.task.metadata)[0].key == list(new_config.taskConfig.metadata)[0].key ) assert ( list(t.assignedTask.task.metadata)[0].value == list(new_config.taskConfig.metadata)[0].value ) assert t.ancestorId
def test__host_limit_1(client, hostmgr): """ - Create a job with host limit 1 constraint and validate each pod is running on different host. - Update a job, wait for it to complete and verify host limit 1 constraint. - Update a job, restart host manager, then wait for update to complete and lastly verify host limit 1 constraint. """ # Create job. job_key = start_job_update(client, "test_dc_labrat.yaml", "start job update test/dc/labrat") # Add some wait time for lucene index to build time.sleep(10) res = client.get_tasks_without_configs(api.TaskQuery(jobKeys={job_key})) assert len(res.tasks) == 3 verify_host_limit_1(res.tasks) # Start a update with host limit 1 constraint job_key = start_job_update(client, "test_dc_labrat_1.yaml", "start job update test/dc/labrat_1") res = client.get_tasks_without_configs( api.TaskQuery(jobKeys={job_key}, statuses={api.ScheduleStatus.RUNNING})) assert len(res.tasks) == 3 verify_host_limit_1(res.tasks) # Start an update, and restart hostmanager before update completes. res = client.start_job_update( get_job_update_request("test_dc_labrat.yaml"), "start job update test/dc/labrat", ) # restart host manager hostmgr.restart() wait_for_rolled_forward(client, res.key) res = client.get_tasks_without_configs( api.TaskQuery(jobKeys={job_key}, statuses={api.ScheduleStatus.RUNNING})) assert len(res.tasks) == 3 verify_host_limit_1(res.tasks)
def test__start_job_update_with_get_jobs(client, aurorabridge): """ Check GetJobs returns the correct results when queried by role, and after a new job is created. This is used to verify that StartJobUpdate would correctly invalidate role to job id cache. Meanwhile, the test will restart aurorabridge container at the end, and verify the cache will be populated correctly as well. """ labrat0 = start_job_update( client, "test_dc_labrat0.yaml", "start job update test/dc/labrat0", ) res = client.get_jobs(labrat0.role) assert len(res.configs) == 1 assert list(res.configs)[0].taskConfig.job == labrat0 # Verify that start a new job invalidates the job role to id cache, # and returns the correct result. labrat1 = start_job_update( client, "test_dc_labrat1.yaml", "start job update test/dc/labrat1", ) res = client.get_jobs(labrat1.role) assert len(res.configs) == 2 assert_keys_equal( [c.taskConfig.job for c in res.configs], [labrat0, labrat1], ) # Verify that after aurorabridge container restart, the cache will still # be populated correctly and return the correct result. aurorabridge.restart() time.sleep(10) res = client.get_jobs(labrat1.role) assert len(res.configs) == 2 assert_keys_equal( [c.taskConfig.job for c in res.configs], [labrat0, labrat1], )
def test__mesos_task_label(client): # verify aurora metadata is correctly populated in mesos task level start_job_update( client, 'test_dc_labrat_uns.yaml', 'start job update test/dc/labrat_uns') state = get_mesos_maser_state() assert len(state['frameworks']) == 1 assert state['frameworks'][0]['name'] == 'Peloton' framework = state['frameworks'][0] assert len(framework['tasks']) == 1 task = framework['tasks'][0] assert len(task['labels']) > 0 for l in task['labels']: if l['key'] == 'org.apache.aurora.metadata.uns': break else: assert False, 'expected label not found'
def test__get_jobs__get_job_summary(client): # Verify no jobs are returned before jobs are created. res = client.get_job_summary('') assert len(res.summaries) == 0 res = client.get_jobs('') assert len(res.configs) == 0 # Create two jobs under same role. test_dc_labrat_key = start_job_update(client, 'test_dc_labrat_read.yaml', 'start job update test/dc/labrat') test_dc_labrat_0_key = start_job_update( client, 'test_dc_labrat0.yaml', 'start job update test/dc/labrat0') # Different role should not show up. start_job_update(client, 'test2_dc2_labrat2.yaml', 'start job update test2/dc2/labrat2') # Ensure get_job_summary returns both jobs under role=test. res = client.get_job_summary(test_dc_labrat_key.role) assert len(res.summaries) == 2 assert_keys_equal([s.job.key for s in res.summaries], [test_dc_labrat_key, test_dc_labrat_0_key]) for s in res.summaries: assert s.stats.activeTaskCount == 2 assert s.job.instanceCount == 2 # Ensure get_jobs returns both jobs under role=test. res = client.get_jobs(test_dc_labrat_key.role) assert len(res.configs) == 2 assert_keys_equal([c.taskConfig.job for c in res.configs], [test_dc_labrat_key, test_dc_labrat_0_key]) for c in res.configs: assert c.instanceCount == 2
def test__mesos_task_label(client): # verify aurora metadata is correctly populated in mesos task level start_job_update( client, "test_dc_labrat_uns.yaml", "start job update test/dc/labrat_uns", ) state = get_mesos_maser_state() assert len(state["frameworks"]) == 1 assert state["frameworks"][0]["name"] == "Peloton" framework = state["frameworks"][0] assert len(framework["tasks"]) == 1 task = framework["tasks"][0] assert len(task["labels"]) > 0 for l in task["labels"]: if l["key"] == "org.apache.aurora.metadata.uns": break else: assert False, "expected label not found"
def test__get_job_update_details__filter_non_update_workflow(client): """ test getJobUpdateDetails endpoint for filtering non-update workflows """ req1 = get_job_update_request("test_dc_labrat_large_job.yaml") req1.settings.updateGroupSize = 10 req2 = get_job_update_request("test_dc_labrat_large_job_diff_labels.yaml") req2.settings.updateGroupSize = 10 # start a regular update job_key = start_job_update(client, req1, "start job update test/dc/labrat_large_job") # trigger an unexpected restart through peloton api jobs = list_jobs() assert len(jobs) == 1 job = StatelessJob(job_id=jobs[0].job_id.value) job.restart(batch_size=10) job.wait_for_workflow_state(goal_state="SUCCEEDED") # wait for restart # start a new update start_job_update(client, req2, "start job update test/dc/labrat_large_job") # verify getJobUpdateDetails response res = client.get_job_update_details(None, api.JobUpdateQuery(role=job_key.role)) assert len(res.detailsList) == 2 for i, detail in enumerate(res.detailsList): if i == 0: assert len(detail.update.instructions.initialState) > 0 for initial in detail.update.instructions.initialState: assert initial.task.metadata, 'Expect metadata to be present' else: assert len(detail.update.instructions.initialState) == 0
def test__start_job_update_with_msg(client): update_msg = 'update msg 1' job_key = start_job_update(client, 'test_dc_labrat.yaml', update_msg) res = client.get_job_update_details(None, api.JobUpdateQuery(jobKey=job_key)) assert len(res.detailsList) == 1 # verify events are sorted ascending assert len(res.detailsList[0].updateEvents) > 0 update_events_ts = [e.timestampMs for e in res.detailsList[0].updateEvents] assert update_events_ts == sorted(update_events_ts) assert len(res.detailsList[0].instanceEvents) > 0 instance_events_ts = [e.timestampMs for e in res.detailsList[0].instanceEvents] assert instance_events_ts == sorted(instance_events_ts) assert res.detailsList[0].updateEvents[0].status == \ api.JobUpdateStatus.ROLLING_FORWARD assert res.detailsList[0].updateEvents[0].message == update_msg assert res.detailsList[0].updateEvents[-1].status == \ api.JobUpdateStatus.ROLLED_FORWARD
def test__manual_rollback_abort(client): """ - Create Job - Start an update - Perform manual rollback on rolling_forward update - Abort rolling_back update - Stateless job will actually have two updates, but bridge will dedupe last two updates (as manual rollback was done) - Validate that task config for each instance """ # Create a job and wait for it to complete start_job_update( client, "test_dc_labrat_large_job.yaml", "start job update test/dc/labrat_large_job", ) # Do update on previously created job res = client.start_job_update( get_job_update_request("test_dc_labrat_large_job_diff_labels.yaml"), "start job update test/dc/labrat_large_job_diff_labels", ) job_update_key = res.key job_key = res.key.job # wait for few instances running time.sleep(10) # rollback update client.rollback_job_update(job_update_key) # wait for sometime to trigger manual rollback time.sleep(5) # abort the manual rollback client.abort_job_update(job_update_key, "abort update") # 2 updates must be present for this job res = client.get_job_update_details( None, api.JobUpdateQuery(jobKey=job_key) ) assert len(res.detailsList) == 2 # first event is rolling forward assert ( res.detailsList[0].updateEvents[0].status == api.JobUpdateStatus.ROLLING_FORWARD ) # second last element is rolling back, after manual rollback is triggered assert ( res.detailsList[0].updateEvents[-2].status == api.JobUpdateStatus.ROLLING_BACK ) # most recent event is aborted, once manual rollback is aborted assert ( res.detailsList[0].updateEvents[-1].status == api.JobUpdateStatus.ABORTED ) # wait for all tasks to be running after invoking abort count = 0 while count < 6: res = client.get_tasks_without_configs( api.TaskQuery(jobKeys={job_key}, statuses={ api.ScheduleStatus.RUNNING}) ) if len(res.tasks) == 10: break count = count + 1 time.sleep(10) # run-id == 1: Job Create # run-id == 2: Job Update with diff labels # run-id == 3: Update rollback to previous version for t in res.tasks: _, _, run_id = t.assignedTask.taskId.rsplit("-", 2) if run_id == "1" or run_id == "3": for m in t.assignedTask.task.metadata: if m.key == "test_key_1": assert m.value == "test_value_1" elif m.key == "test_key_2": assert m.value == "test_value_2" else: assert False, ( "unexpected metadata %s for affected instances" % m ) elif run_id == "2": # Preference to update instances has changed. # Previously behavior, start to update instance from instance_0. # run_1 (config_1) -> on_update run_2 (config_2) -> rollback run_3 (config_1) # # Current behavior, start with instance which is not available or killed, # thereby instances which were on-going update will be rolled back first. # If an instance was in KILLED state (run-id=1) when it was picked up for rollback, # the run-id will be bumped up to 2 and move the config to the original one. # An instance with run-id=2 can either be on config_1 or config_2 (due to first update) # run_1 (config_1) -> (instance is in KILLED state, rollback runs first on it) rollback run_2 (config_1) continue else: assert False, ( "unexpected run id %s" % t.assignedTask.taskId )
def test__simple_manual_rollback(client): """ Start a job update which will create a job. Do another update on the job and half-way to a manual rollback """ job_key = start_job_update( client, "test_dc_labrat_large_job.yaml", "start job update test/dc/labrat_large_job", ) res = client.get_job_update_details(None, api.JobUpdateQuery(jobKey=job_key)) verify_events_sorted(res.detailsList[0].updateEvents) verify_events_sorted(res.detailsList[0].instanceEvents) verify_first_and_last_job_update_status( res.detailsList[0].updateEvents, api.JobUpdateStatus.ROLLING_FORWARD, api.JobUpdateStatus.ROLLED_FORWARD, ) verify_task_config( client, job_key, { "test_key_1": "test_value_1", "test_key_2": "test_value_2" }, ) # Do update on previously created job res = client.start_job_update( get_job_update_request("test_dc_labrat_large_job_diff_labels.yaml"), "start job update test/dc/labrat_large_job_diff_labels", ) job_update_key = res.key job_key = res.key.job # wait for few instances running time.sleep(5) res = client.get_job_update_details(None, api.JobUpdateQuery(jobKey=job_key)) verify_events_sorted(res.detailsList[0].updateEvents) verify_events_sorted(res.detailsList[0].instanceEvents) verify_first_and_last_job_update_status( res.detailsList[0].updateEvents, api.JobUpdateStatus.ROLLING_FORWARD, api.JobUpdateStatus.ROLLING_FORWARD, ) verify_task_config( client, job_key, { "test_key_1": "test_value_1", "test_key_2": "test_value_2", "test_key_11": "test_value_11", "test_key_22": "test_value_22", }, ) # rollback update client.rollback_job_update(job_update_key) wait_for_rolled_back(client, job_update_key) # verify events are sorted ascending, and last update event is ROLLED_BACK res = client.get_job_update_details(None, api.JobUpdateQuery(jobKey=job_key)) verify_events_sorted(res.detailsList[0].updateEvents) verify_events_sorted(res.detailsList[0].instanceEvents) verify_first_and_last_job_update_status( res.detailsList[0].updateEvents, api.JobUpdateStatus.ROLLING_FORWARD, api.JobUpdateStatus.ROLLED_BACK, ) verify_task_config( client, job_key, { "test_key_1": "test_value_1", "test_key_2": "test_value_2" }, ) # rolled back to previous task config
def test__get_tasks_without_configs(client): # Create job. job_key = start_job_update(client, "test_dc_labrat_read.yaml", "start job update test/dc/labrat") # Add some wait time for lucene index to build time.sleep(10) res = client.get_tasks_without_configs(api.TaskQuery(jobKeys={job_key})) assert len(res.tasks) == 2 host_counts = defaultdict(int) for t in res.tasks: # ScheduledTask assert api.ScheduleStatus.RUNNING == t.status assert t.ancestorId is None # ScheduledTask.TaskEvent assert api.ScheduleStatus.RUNNING == t.taskEvents[-1].status assert "peloton" == t.taskEvents[-1].scheduler # ScheduledTask.AssignedTask assert t.assignedTask.taskId is not None assert t.assignedTask.slaveId is not None assert t.assignedTask.slaveHost is not None assert t.assignedTask.instanceId in (0, 1) # ScheduledTask.AssignedTask.TaskConfig assert "test" == t.assignedTask.task.job.role assert "dc" == t.assignedTask.task.job.environment assert "labrat" == t.assignedTask.task.job.name assert "testuser" == t.assignedTask.task.owner.user assert t.assignedTask.task.isService assert 5 == t.assignedTask.task.priority assert "preemptible" == t.assignedTask.task.tier assert 2 == len(t.assignedTask.task.metadata) for m in t.assignedTask.task.metadata: if "test_key_1" == m.key: assert "test_value_1" == m.value elif "test_key_2" == m.key: assert "test_value_2" == m.value else: assert False, "unexpected metadata {}".format(m) assert 3 == len(t.assignedTask.task.resources) for r in t.assignedTask.task.resources: if r.numCpus > 0: assert 0.25 == r.numCpus elif r.ramMb > 0: assert 128 == r.ramMb elif r.diskMb > 0: assert 128 == r.diskMb else: assert False, "unexpected resource {}".format(r) assert 1 == len(t.assignedTask.task.constraints) assert "host" == list(t.assignedTask.task.constraints)[0].name assert (1 == list( t.assignedTask.task.constraints)[0].constraint.limit.limit) host_counts[t.assignedTask.slaveHost] += 1 # Ensure the host limit is enforced. for host, count in host_counts.iteritems(): assert count == 1, "{host} has more than 1 task".format(host=host)
def test__manual_rollback_abort(client): """ - Create Job - Start an update - Perform manual rollback on rolling_forward update - Abort rolling_back update - Stateless job will actually have two updates, but bridge will dedupe last two updates (as manual rollback was done) - Validate that task config for each instance """ # Create a job and wait for it to complete start_job_update( client, "test_dc_labrat_large_job.yaml", "start job update test/dc/labrat_large_job", ) # Do update on previously created job res = client.start_job_update( get_job_update_request("test_dc_labrat_large_job_diff_labels.yaml"), "start job update test/dc/labrat_large_job_diff_labels", ) job_update_key = res.key job_key = res.key.job # wait for few instances running time.sleep(10) # rollback update client.rollback_job_update(job_update_key) # wait for sometime to trigger manual rollback time.sleep(5) # abort the manual rollback client.abort_job_update(job_update_key, "abort update") # 2 updates must be present for this job res = client.get_job_update_details(None, api.JobUpdateQuery(jobKey=job_key)) assert len(res.detailsList) == 2 # first event is rolling forward assert (res.detailsList[0].updateEvents[0].status == api.JobUpdateStatus.ROLLING_FORWARD) # second last element is rolling back, after manual rollback is triggered assert (res.detailsList[0].updateEvents[-2].status == api.JobUpdateStatus.ROLLING_BACK) # most recent event is aborted, once manual rollback is aborted assert (res.detailsList[0].updateEvents[-1].status == api.JobUpdateStatus.ABORTED) # wait for all tasks to be running after invoking abort count = 0 while count < 6: res = client.get_tasks_without_configs( api.TaskQuery(jobKeys={job_key}, statuses={api.ScheduleStatus.RUNNING})) if len(res.tasks) == 10: break count = count + 1 time.sleep(10) # run-id == 1: Job Create # run-id == 2: Job Update with diff labels # run-id == 3: Update rollback to previous version for t in res.tasks: _, _, run_id = t.assignedTask.taskId.rsplit("-", 2) if run_id == "1" or run_id == "3": for m in t.assignedTask.task.metadata: if m.key == "test_key_1": assert m.value == "test_value_1" elif m.key == "test_key_2": assert m.value == "test_value_2" else: assert False, ( "unexpected metadata %s for affected instances" % m) elif run_id == "2": for m in t.assignedTask.task.metadata: if m.key == "test_key_11": assert m.value == "test_value_11" elif m.key == "test_key_22": assert m.value == "test_value_22" else: assert False, ( "unexpected metadata %s for affected instances" % m) else: assert False, ("unexpected run id %s" % t.assignedTask.taskId)
def test__get_tasks_without_configs__previous_run(client): """ test getTasksWithoutConfigs endpoint for tasks from previous runs: 1. start a regular update (version 1) on all instances 2. start a another update (version 2) on all instances """ req1 = get_job_update_request("test_dc_labrat_large_job.yaml") req1.settings.updateGroupSize = 10 req2 = get_job_update_request("test_dc_labrat_large_job_diff_labels.yaml") req2.settings.updateGroupSize = 10 # start a regular update job_key = start_job_update(client, req1, "start job update test/dc/labrat_large_job") res = client.get_tasks_without_configs(api.TaskQuery(jobKeys={job_key})) assert len(res.tasks) == 10 for t in res.tasks: _, _, run_id = t.assignedTask.taskId.rsplit("-", 2) assert run_id == "1" assert len(t.assignedTask.task.metadata) == 2 for m in t.assignedTask.task.metadata: if m.key == "test_key_1": assert m.value == "test_value_1" elif m.key == "test_key_2": assert m.value == "test_value_2" else: assert False, "unexpected metadata %s" % m # start 6 new updates (assuming pod_runs_depth is 6), expect run id 1 # to be excluded start_job_update(client, req2, "start job update test/dc/labrat_large_job") start_job_update(client, req1, "start job update test/dc/labrat_large_job") start_job_update(client, req2, "start job update test/dc/labrat_large_job") start_job_update(client, req1, "start job update test/dc/labrat_large_job") start_job_update(client, req2, "start job update test/dc/labrat_large_job") start_job_update(client, req1, "start job update test/dc/labrat_large_job") res = client.get_tasks_without_configs(api.TaskQuery(jobKeys={job_key})) assert len(res.tasks) == 10 * 6 for t in res.tasks: _, _, run_id = t.assignedTask.taskId.rsplit("-", 2) assert len(t.assignedTask.task.metadata) == 2 if run_id in ("7"): assert t.status == api.ScheduleStatus.RUNNING for m in t.assignedTask.task.metadata: if m.key == "test_key_1": assert m.value == "test_value_1" elif m.key == "test_key_2": assert m.value == "test_value_2" else: assert False, "unexpected metadata %s" % m elif run_id in ("6", "4", "2"): assert t.status == api.ScheduleStatus.KILLED for m in t.assignedTask.task.metadata: if m.key == "test_key_11": assert m.value == "test_value_11" elif m.key == "test_key_22": assert m.value == "test_value_22" else: assert False, "unexpected metadata %s" % m elif run_id in ("5", "3"): assert t.status == api.ScheduleStatus.KILLED for m in t.assignedTask.task.metadata: if m.key == "test_key_1": assert m.value == "test_value_1" elif m.key == "test_key_2": assert m.value == "test_value_2" else: assert False, "unexpected metadata %s" % m else: assert False, "unexpected run id: %d" % run_id
def test__get_tasks_without_configs_task_queries(client): # Verify no tasks are returned before creating. res = client.get_tasks_without_configs(api.TaskQuery()) assert len(res.tasks) == 0 # Create jobs. test_dc_labrat_key = start_job_update(client, 'test_dc_labrat_read.yaml', 'start job update test/dc/labrat') test_dc_labrat_0_key = start_job_update( client, 'test_dc_labrat0.yaml', 'start job update test/dc/labrat0') test_dc_0_labrat_1_key = start_job_update( client, 'test_dc0_labrat1.yaml', 'start job update test/dc0/labrat1') test_dc_labrat_1_key = start_job_update( client, 'test_dc_labrat1.yaml', 'start job update test/dc/labrat1') test2_dc2_labrat2_key = start_job_update( client, 'test2_dc2_labrat2.yaml', 'start job update test2/dc2/labrat2') # Kill one of the jobs. client.kill_tasks(test_dc_labrat_1_key, {0, 1}, 'killing all tasks test/dc/labrat1') wait_for_killed(client, test_dc_labrat_1_key) for message, query, expected_job_keys in [ ( 'query job keys', api.TaskQuery(jobKeys={ test_dc_labrat_key, test_dc_labrat_0_key, test2_dc2_labrat2_key, }), [ test_dc_labrat_key, test_dc_labrat_0_key, test2_dc2_labrat2_key, ], ), ( 'query role + env + name', api.TaskQuery( role=test_dc_labrat_key.role, environment=test_dc_labrat_key.environment, jobName=test_dc_labrat_key.name, ), [test_dc_labrat_key], ), ( 'query role + env', api.TaskQuery( role=test_dc_labrat_key.role, environment=test_dc_labrat_key.environment, ), [ test_dc_labrat_key, test_dc_labrat_0_key, test_dc_labrat_1_key, ], ), ( 'query role', api.TaskQuery(role=test_dc_labrat_key.role), [ test_dc_labrat_key, test_dc_labrat_0_key, test_dc_labrat_1_key, test_dc_0_labrat_1_key, ], ), ( 'query role + statuses', api.TaskQuery( role=test_dc_labrat_key.role, statuses={api.ScheduleStatus.RUNNING}, ), [ test_dc_labrat_key, test_dc_labrat_0_key, test_dc_0_labrat_1_key, ], ) ]: res = client.get_tasks_without_configs(query) # Expect 3 tasks per job key. assert len(res.tasks) == len(expected_job_keys) * 2, message assert_keys_equal(remove_duplicate_keys(t.assignedTask.task.job for t in res.tasks), expected_job_keys, message=message)
def test__get_tasks_without_configs_task_queries(client): # Verify no tasks are returned before creating. res = client.get_tasks_without_configs(api.TaskQuery()) assert len(res.tasks) == 0 # Create jobs. test_dc_labrat_key = start_job_update(client, "test_dc_labrat_read.yaml", "start job update test/dc/labrat") test_dc_labrat_0_key = start_job_update( client, "test_dc_labrat0.yaml", "start job update test/dc/labrat0") test_dc_0_labrat_1_key = start_job_update( client, "test_dc0_labrat1.yaml", "start job update test/dc0/labrat1") test_dc_labrat_1_key = start_job_update( client, "test_dc_labrat1.yaml", "start job update test/dc/labrat1") test2_dc2_labrat2_key = start_job_update( client, "test2_dc2_labrat2.yaml", "start job update test2/dc2/labrat2") # Add some wait time for lucene index to build time.sleep(10) # Kill one of the jobs. client.kill_tasks(test_dc_labrat_1_key, None, "killing all tasks test/dc/labrat1") wait_for_killed(client, test_dc_labrat_1_key) for message, query, expected_job_keys in [ ( "query job keys", api.TaskQuery(jobKeys={ test_dc_labrat_key, test_dc_labrat_0_key, test2_dc2_labrat2_key, }), [test_dc_labrat_key, test_dc_labrat_0_key, test2_dc2_labrat2_key], ), ( "query role + env + name", api.TaskQuery( role=test_dc_labrat_key.role, environment=test_dc_labrat_key.environment, jobName=test_dc_labrat_key.name, ), [test_dc_labrat_key], ), ( "query role + env", api.TaskQuery( role=test_dc_labrat_key.role, environment=test_dc_labrat_key.environment, ), [test_dc_labrat_key, test_dc_labrat_0_key, test_dc_labrat_1_key], ), ( "query role", api.TaskQuery(role=test_dc_labrat_key.role), [ test_dc_labrat_key, test_dc_labrat_0_key, test_dc_labrat_1_key, test_dc_0_labrat_1_key, ], ), ( "query role + statuses", api.TaskQuery( role=test_dc_labrat_key.role, statuses={api.ScheduleStatus.RUNNING}, ), [test_dc_labrat_key, test_dc_labrat_0_key, test_dc_0_labrat_1_key], ), ]: res = client.get_tasks_without_configs(query) # Expect 3 tasks per job key. assert len(res.tasks) == len(expected_job_keys) * 2, message assert_keys_equal( remove_duplicate_keys(t.assignedTask.task.job for t in res.tasks), expected_job_keys, message=message, )
def test__get_config_summary__with_pinned_instances(client): """ test pinned instance update which divides instances to two sets of configs, and verify getConfigSummary endpoint returns the correct result. """ all_instances = set(range(10)) # start a regular update job_key = start_job_update( client, "test_dc_labrat_large_job.yaml", "start job update test/dc/labrat_large_job", ) tasks = get_running_tasks(client, job_key) assert len(tasks) == 10 # start a update with updateOnlyTheseInstances parameter update_instances = set([0, 2, 3, 7, 9]) pinned_req = get_job_update_request( "test_dc_labrat_large_job_diff_labels.yaml") pinned_req.settings.updateOnlyTheseInstances = set( [api.Range(first=i, last=i) for i in update_instances]) res = client.start_job_update( pinned_req, "start job update test/dc/labrat_large_job with pinned instances", ) wait_for_rolled_forward(client, res.key) job_key = res.key.job tasks = get_running_tasks(client, job_key) assert len(tasks) == 10 res = client.get_config_summary(job_key) assert len(res.summary.groups) == 2 for group in res.summary.groups: instances = set(expand_instance_range(group.instances)) if instances == update_instances: # instances updated in the second update assert len(group.config.metadata) == 2 for m in group.config.metadata: if m.key == "test_key_11": assert m.value == "test_value_11" elif m.key == "test_key_22": assert m.value == "test_value_22" else: assert False, "unexpected metadata %s" % m elif instances == all_instances - update_instances: # instances updated from the first update assert len(group.config.metadata) == 2 for m in group.config.metadata: if m.key == "test_key_1": assert m.value == "test_value_1" elif m.key == "test_key_2": assert m.value == "test_value_2" else: assert False, "unexpected metadata %s" % m else: assert False, "unexpected instance range: %s" % group.instances
def test__get_tasks_without_configs(client): # Create job. job_key = start_job_update(client, 'test_dc_labrat.yaml', 'start job update test/dc/labrat') res = client.get_tasks_without_configs(api.TaskQuery(jobKeys={job_key})) assert len(res.tasks) == 3 host_counts = defaultdict(int) for t in res.tasks: # ScheduledTask assert api.ScheduleStatus.RUNNING == t.status assert t.ancestorId is None # ScheduledTask.TaskEvent assert api.ScheduleStatus.RUNNING == t.taskEvents[-1].status assert "peloton" == t.taskEvents[-1].scheduler # ScheduledTask.AssignedTask assert t.assignedTask.taskId is not None assert t.assignedTask.slaveId is not None assert t.assignedTask.slaveHost is not None assert t.assignedTask.instanceId in (0, 1, 2) # ScheduledTask.AssignedTask.TaskConfig assert 'test' == t.assignedTask.task.job.role assert 'dc' == t.assignedTask.task.job.environment assert 'labrat' == t.assignedTask.task.job.name assert 'testuser' == t.assignedTask.task.owner.user assert t.assignedTask.task.isService assert 5 == t.assignedTask.task.priority assert 'preemptible' == t.assignedTask.task.tier assert 2 == len(t.assignedTask.task.metadata) for m in t.assignedTask.task.metadata: if 'test_key_1' == m.key: assert 'test_value_1' == m.value elif 'test_key_2' == m.key: assert 'test_value_2' == m.value else: assert False, 'unexpected metadata {}'.format(m) assert 3 == len(t.assignedTask.task.resources) for r in t.assignedTask.task.resources: if r.numCpus > 0: assert 0.25 == r.numCpus elif r.ramMb > 0: assert 32 == r.ramMb elif r.diskMb > 0: assert 128 == r.diskMb else: assert False, 'unexpected resource {}'.format(r) assert 1 == len(t.assignedTask.task.constraints) assert 'host' == list(t.assignedTask.task.constraints)[0].name assert 1 == list( t.assignedTask.task.constraints)[0].constraint.limit.limit host_counts[t.assignedTask.slaveHost] += 1 # Ensure the host limit is enforced. for host, count in host_counts.iteritems(): assert count == 1, '{host} has more than 1 task'.format(host=host)