def test_resource_offers_launch_tasks_failed(ef, fake_task, fake_offer,
fake_driver, mock_get_metric,
mock_time):
ef.driver = fake_driver
ef.driver.launchTasks = mock.Mock(side_effect=socket.timeout)
ef._last_offer_time = None
mock_time.return_value = 2.0
ef.suppress_after = 0.0
ef.offer_matches_pool = mock.Mock(return_value=True)
task_id = fake_task.task_id
docker_task = Dict(task_id=Dict(value=task_id))
task_metadata = ef_mdl.TaskMetadata(
task_config=fake_task,
task_state='fake_state',
task_state_history=m(fake_state=time.time()))
ef.get_tasks_to_launch = mock.Mock(return_value=[docker_task])
ef.task_queue.put(fake_task)
ef.task_metadata = ef.task_metadata.set(task_id, task_metadata)
ef.resourceOffers(ef.driver, [fake_offer])
assert fake_driver.suppressOffers.call_count == 0
assert not ef.are_offers_suppressed
assert fake_driver.declineOffer.call_count == 0
assert fake_driver.launchTasks.call_count == 1
assert mock_get_metric.call_count == 1
assert ef.task_metadata[task_id].task_state == 'UNKNOWN'
def test_resource_offers_launch(ef, fake_task, fake_offer, fake_driver,
mock_get_metric, mock_time):
ef.driver = fake_driver
ef._last_offer_time = 1.0
mock_time.return_value = 2.0
ef.suppress_after = 0.0
ef.offer_matches_pool = mock.Mock(return_value=True)
task_id = fake_task.task_id
docker_task = Dict(task_id=Dict(value=task_id))
task_metadata = ef_mdl.TaskMetadata(
task_config=fake_task,
task_state='fake_state',
task_state_history=m(fake_state=time.time()))
ef.get_tasks_to_launch = mock.Mock(return_value=[docker_task])
ef.task_queue.put(fake_task)
ef.task_metadata = ef.task_metadata.set(task_id, task_metadata)
ef.resourceOffers(ef.driver, [fake_offer])
assert fake_driver.suppressOffers.call_count == 0
assert not ef.are_offers_suppressed
assert fake_driver.declineOffer.call_count == 0
assert fake_driver.launchTasks.call_count == 1
assert mock_get_metric.call_count == 2
mock_get_metric.assert_any_call(ef_mdl.OFFER_DELAY_TIMER)
mock_get_metric.assert_any_call(ef_mdl.TASK_LAUNCHED_COUNT)
assert mock_get_metric.return_value.record.call_count == 1
assert mock_get_metric.return_value.record.call_args == mock.call(1.0)
assert mock_get_metric.return_value.count.call_count == 1
assert mock_get_metric.return_value.count.call_args == mock.call(1)
def test_create_new_docker_task(
ef,
fake_offer,
fake_task,
gpus_count,
containerizer,
container,
):
available_ports = [31200]
task_id = fake_task.task_id
task_metadata = ef_mdl.TaskMetadata(
task_config=fake_task,
task_state='fake_state',
task_state_history=m(fake_state=time.time()))
fake_task = fake_task.set(
volumes=v(
Dict(mode='RO',
container_path='fake_container_path',
host_path='fake_host_path')),
gpus=gpus_count,
containerizer=containerizer,
)
ef.task_metadata = ef.task_metadata.set(task_id, task_metadata)
docker_task = ef.create_new_docker_task(fake_offer, fake_task,
available_ports)
new_docker_task = Dict(
task_id=Dict(value=task_id),
agent_id=Dict(value='fake_agent_id'),
name='executor-{id}'.format(id=task_id),
resources=[
Dict(name='cpus',
type='SCALAR',
role='fake_role',
scalar=Dict(value=10.0)),
Dict(name='mem',
type='SCALAR',
role='fake_role',
scalar=Dict(value=1024.0)),
Dict(name='disk',
type='SCALAR',
role='fake_role',
scalar=Dict(value=1000.0)),
Dict(name='gpus',
type='SCALAR',
role='fake_role',
scalar=Dict(value=gpus_count)),
Dict(name='ports',
type='RANGES',
role='fake_role',
ranges=Dict(range=[Dict(begin=31200, end=31200)]))
],
command=Dict(value='echo "fake"',
uris=[],
environment=Dict(variables=[])),
container=container,
)
assert ef.task_metadata[task_id].agent_id == 'fake_agent_id'
assert docker_task == new_docker_task
def test_reenqueue_tasks_stuck_in_unknown_state(ef, fake_task, mock_sleep,
mock_get_metric):
task_id = fake_task.task_id
task_metadata = ef_mdl.TaskMetadata(
agent_id='fake_agent_id',
task_config=fake_task,
task_state='UNKNOWN',
task_state_history=m(UNKNOWN=0.0),
)
ef.task_staging_timeout_s = 0
ef.kill_task = mock.Mock()
ef.blacklist_slave = mock.Mock()
ef.enqueue_task = mock.Mock()
ef.task_metadata = ef.task_metadata.set(task_id, task_metadata)
ef._background_check()
assert ef.enqueue_task.call_count == 1
assert ef.enqueue_task.call_args == mock.call(
ef.task_metadata[task_id].task_config)
assert mock_get_metric.call_count == 1
assert mock_get_metric.call_args == mock.call(
ef_mdl.TASK_FAILED_TO_LAUNCH_COUNT)
assert mock_get_metric.return_value.count.call_count == 1
assert mock_get_metric.return_value.count.call_args == mock.call(1)
def status_update_test_prep(state, reason=''):
task = me_mdl.MesosTaskConfig(cmd='/bin/true',
name='fake_name',
image='fake_image')
task_id = task.task_id
update = Dict(task_id=Dict(value=task_id), state=state, reason=reason)
task_metadata = ef_mdl.TaskMetadata(
task_config=task,
task_state='TASK_INITED',
task_state_history=m(TASK_INITED=time.time()),
)
return update, task_id, task_metadata
def test_get_tasks_to_launch_ports_available(ef, fake_offer, fake_task,
fake_driver, mock_get_metric):
ef.create_new_docker_task = mock.Mock()
ef.get_available_ports = mock.Mock(return_value=[30000])
ef.task_queue.put(fake_task)
task_metadata = ef_mdl.TaskMetadata(
task_config=fake_task,
task_state='TASK_INITED',
task_state_history=m(TASK_INITED=time.time()))
ef.task_metadata = ef.task_metadata.set(fake_task.task_id, task_metadata)
tasks = ef.get_tasks_to_launch(fake_offer)
assert len(tasks) == 1
assert ef.task_queue.qsize() == 0
assert ef.create_new_docker_task.call_count == 1
def test_get_tasks_to_launch_sufficient_offer(ef, fake_task, fake_offer,
mock_get_metric, mock_time):
task_metadata = ef_mdl.TaskMetadata(task_config=fake_task,
task_state='TASK_INITED',
task_state_history=m(TASK_INITED=1.0))
ef.create_new_docker_task = mock.Mock()
mock_time.return_value = 2.0
ef.task_queue.put(fake_task)
ef.task_metadata = ef.task_metadata.set(fake_task.task_id, task_metadata)
tasks_to_launch = ef.get_tasks_to_launch(fake_offer)
assert ef.create_new_docker_task.return_value in tasks_to_launch
assert ef.task_queue.qsize() == 0
assert mock_get_metric.call_count == 1
assert mock_get_metric.call_args == mock.call(
ef_mdl.TASK_QUEUED_TIME_TIMER)
assert mock_get_metric.return_value.record.call_count == 1
assert mock_get_metric.return_value.record.call_args == mock.call(1.0)
def test_ef_kills_stuck_tasks(ef, fake_task, mock_sleep, mock_get_metric):
task_id = fake_task.task_id
task_metadata = ef_mdl.TaskMetadata(
agent_id='fake_agent_id',
task_config=fake_task,
task_state='TASK_STAGING',
task_state_history=m(TASK_STAGING=0.0),
)
ef.task_staging_timeout_s = 0
ef.kill_task = mock.Mock()
ef.blacklist_slave = mock.Mock()
ef.task_metadata = ef.task_metadata.set(task_id, task_metadata)
ef._background_check()
assert ef.kill_task.call_count == 1
assert ef.kill_task.call_args == mock.call(task_id)
assert ef.blacklist_slave.call_count == 1
assert ef.blacklist_slave.call_args == mock.call(agent_id='fake_agent_id',
timeout=900)
assert mock_get_metric.call_count == 1
assert mock_get_metric.call_args == mock.call(ef_mdl.TASK_STUCK_COUNT)
assert mock_get_metric.return_value.count.call_count == 1
assert mock_get_metric.return_value.count.call_args == mock.call(1)