def test_not_scale_down_if_there_are_pending_jobs_and_running_jobs():
submit_datetime = datetime(2018, 12, 21, 11, 00, 00)
jobs = [
GridEngineJob(
id=1,
name='name1',
user='******',
state=GridEngineJobState.RUNNING,
datetime=submit_datetime,
host=MASTER_HOST
),
GridEngineJob(
id=2,
name='name2',
user='******',
state=GridEngineJobState.PENDING,
datetime=submit_datetime + timedelta(seconds=scale_down_timeout),
)
]
grid_engine.get_jobs = MagicMock(return_value=jobs)
clock.now = MagicMock(return_value=submit_datetime + timedelta(seconds=scale_down_timeout))
autoscaler.scale()
autoscaler.scale_down.assert_not_called()
def test_not_scale_up_if_number_of_additional_workers_is_already_equals_to_the_limit_but_there_pending_array_jobs(
):
submit_datetime = datetime(2018, 12, 21, 11, 00, 00)
jobs = [
GridEngineJob(id=1,
name='name1',
user='******',
state=GridEngineJobState.RUNNING,
datetime=submit_datetime,
host=MASTER_HOST),
GridEngineJob(id=2,
name='name2',
user='******',
state=GridEngineJobState.PENDING,
datetime=submit_datetime,
array=range(5, 10))
]
for run_id in range(0, autoscaler.max_additional_hosts):
autoscaler.host_storage.add_host(str(run_id))
grid_engine.get_jobs = MagicMock(return_value=jobs)
clock.now = MagicMock(return_value=submit_datetime +
timedelta(seconds=scale_up_timeout))
autoscaler.scale()
autoscaler.scale_up.assert_not_called()
def test_kill_jobs():
jobs = [
GridEngineJob(id=1, name='', user='', state='', datetime=''),
GridEngineJob(id=2, name='', user='', state='', datetime='')
]
grid_engine.kill_jobs(jobs)
assert_first_argument_contained(executor.execute, 'qdel ')
assert_first_argument_contained(executor.execute, ' 1 2')
assert_first_argument_not_contained(executor.execute, '-f')
def test_scale_down_if_there_are_no_pending_and_running_jobs_for_more_than_scale_down_timeout():
submit_datetime = datetime(2018, 12, 21, 11, 00, 00)
jobs = [
GridEngineJob(
id=1,
name='name1',
user='******',
state=GridEngineJobState.RUNNING,
datetime=submit_datetime,
host=MASTER_HOST
)
]
grid_engine.get_jobs = MagicMock(return_value=jobs)
clock.now = MagicMock(return_value=submit_datetime)
autoscaler.scale()
autoscaler.scale_down.assert_not_called()
hosts = autoscaler.host_storage.load_hosts()
assert len(hosts) == 1
assert ADDITIONAL_HOST in hosts
grid_engine.get_jobs = MagicMock(return_value=[])
clock.now = MagicMock(return_value=submit_datetime + timedelta(seconds=scale_down_timeout))
autoscaler.scale()
autoscaler.scale_down.assert_called_with(ADDITIONAL_HOST)
assert len(autoscaler.host_storage.load_hosts()) == 0
def test_that_scale_down_only_stops_inactive_additional_hosts():
submit_datetime = datetime(2018, 12, 21, 11, 00, 00)
jobs = [
GridEngineJob(
id=1,
name='name1',
user='******',
state=GridEngineJobState.RUNNING,
datetime=submit_datetime,
host=ADDITIONAL_HOST
)
]
grid_engine.get_jobs = MagicMock(return_value=jobs)
clock.now = MagicMock(return_value=submit_datetime + timedelta(seconds=scale_down_timeout))
inactive_hosts = ['inactive-host-1', 'inactive-host-2']
for inactive_host in inactive_hosts:
autoscaler.host_storage.add_host(inactive_host)
for _ in range(0, len(inactive_hosts) * 2):
autoscaler.scale()
for inactive_host in inactive_hosts:
autoscaler.scale_down.assert_any_call(inactive_host)
hosts = autoscaler.host_storage.load_hosts()
assert len(hosts) == 1
assert ADDITIONAL_HOST in hosts
def test_force_killing_invalid_host_jobs():
jobs = [
GridEngineJob(id=1,
name='',
user='',
state='',
datetime='',
host=HOST2)
]
grid_engine.get_jobs = MagicMock(return_value=jobs)
worker_validator.validate_hosts()
grid_engine.kill_jobs.assert_called_with(jobs, force=True)
def test_not_scale_up_if_none_of_the_jobs_are_in_queue_for_more_than_scale_up_timeout(
):
submit_datetime = datetime(2018, 12, 21, 11, 00, 00)
jobs = [
GridEngineJob(id=1,
name='name1',
user='******',
state=GridEngineJobState.RUNNING,
datetime=submit_datetime,
host=MASTER_HOST),
GridEngineJob(id=2,
name='name2',
user='******',
state=GridEngineJobState.PENDING,
datetime=submit_datetime)
]
grid_engine.get_jobs = MagicMock(return_value=jobs)
clock.now = MagicMock(return_value=submit_datetime +
timedelta(seconds=scale_up_timeout - 1))
autoscaler.scale()
autoscaler.scale_up.assert_not_called()
def test_that_scale_up_will_not_launch_more_additional_workers_than_limit():
submit_datetime = datetime(2018, 12, 21, 11, 00, 00)
jobs = [
GridEngineJob(id=1,
name='name1',
user='******',
state=GridEngineJobState.RUNNING,
datetime=submit_datetime,
host=MASTER_HOST),
GridEngineJob(id=2,
name='name2',
user='******',
state=GridEngineJobState.PENDING,
datetime=submit_datetime)
]
grid_engine.get_jobs = MagicMock(return_value=jobs)
clock.now = MagicMock(return_value=submit_datetime +
timedelta(seconds=scale_up_timeout))
for _ in range(0, max_additional_hosts * 2):
autoscaler.scale()
assert autoscaler.scale_up.call_count == 2
assert len(autoscaler.host_storage.load_hosts()) == max_additional_hosts
def test_not_scale_down_if_there_are_pending_jobs_and_no_running_jobs_yet():
submit_datetime = datetime(2018, 12, 21, 11, 00, 00)
jobs = [
GridEngineJob(
id=1,
name='name1',
user='******',
state=GridEngineJobState.PENDING,
datetime=submit_datetime
)
]
grid_engine.get_jobs = MagicMock(return_value=jobs)
clock.now = MagicMock(return_value=submit_datetime)
autoscaler.scale()
autoscaler.scale_down.assert_not_called()
def test_scaling_down_if_host_has_no_running_jobs():
submit_datetime = datetime(2018, 12, 29, 11, 00, 00)
jobs = [
GridEngineJob(id=1,
name='name1',
user='******',
state=GridEngineJobState.RUNNING,
datetime=submit_datetime,
host=ANOTHER_HOSTNAME)
]
grid_engine.get_jobs = MagicMock(return_value=jobs)
assert scale_down_handler.scale_down(HOSTNAME)
grid_engine.disable_host.assert_called()
grid_engine.enable_host.assert_not_called()
grid_engine.delete_host.assert_called()
assert_first_argument_contained(cmd_executor.execute, HOSTNAME)
def test_not_scale_down_if_all_jobs_are_running_for_more_than_scale_down_timeout_and_additional_host_is_active():
submit_datetime = datetime(2018, 12, 21, 11, 00, 00)
jobs = [
GridEngineJob(
id=1,
name='name1',
user='******',
state=GridEngineJobState.RUNNING,
datetime=submit_datetime,
host=ADDITIONAL_HOST
)
]
grid_engine.get_jobs = MagicMock(return_value=jobs)
clock.now = MagicMock(return_value=submit_datetime + timedelta(seconds=scale_down_timeout))
autoscaler.scale()
autoscaler.scale_down.assert_not_called()
def test_host_is_not_removed_from_storage_if_scaling_down_is_aborted():
submit_datetime = datetime(2018, 12, 21, 11, 00, 00)
jobs = [
GridEngineJob(
id=1,
name='name1',
user='******',
state=GridEngineJobState.RUNNING,
datetime=submit_datetime,
host=MASTER_HOST
)
]
grid_engine.get_jobs = MagicMock(return_value=jobs)
clock.now = MagicMock(return_value=submit_datetime + timedelta(seconds=scale_down_timeout))
autoscaler.scale_down = MagicMock(return_value=False)
autoscaler.scale()
autoscaler.scale_down.assert_called()
assert ADDITIONAL_HOST in autoscaler.host_storage.load_hosts()
