def run(self):
# todo: 1) do all ansible calls through subprocess
# 2) move to Python 3 and asyncIO all in one thread + executors
# ... -> eliminate multiprocessing here,
# ... possible to use simple logging, with redis handler
self.log.info("Creating VM Spawner, HealthChecker, Terminator")
self.spawner = Spawner(self.opts)
self.checker = HealthChecker(self.opts)
self.terminator = Terminator(self.opts)
self.vm_manager = VmManager(
opts=self.opts, logger=self.log,
)
self.vm_manager.post_init()
self.log.info("Starting up VM EventHandler")
self.event_handler = EventHandler(self.opts,
vmm=self.vm_manager,
terminator=self.terminator)
self.event_handler.post_init()
self.event_handler.start()
self.log.info("Starting up VM Master")
self.vm_master = VmMaster(self.opts,
vmm=self.vm_manager,
spawner=self.spawner,
checker=self.checker)
self.vm_master.start()
setproctitle("Copr VMM base process")
def get_managed_vms_names():
result = []
if VmManager:
opts = BackendConfigReader().read()
vmm = VmManager(opts, log)
result.extend(vmd.vm_name.lower() for vmd in vmm.get_all_vm())
return result
class VmmRunner(object):
def __init__(self, opts):
self.opts = opts
self.log = get_redis_logger(self.opts, "vmm.main", "vmm")
def run(self):
# todo: 1) do all ansible calls through subprocess
# 2) move to Python 3 and asyncIO all in one thread + executors
# ... -> eliminate multiprocessing here,
# ... possible to use simple logging, with redis handler
self.log.info("Creating VM Spawner, HealthChecker, Terminator")
self.spawner = Spawner(self.opts)
self.checker = HealthChecker(self.opts)
self.terminator = Terminator(self.opts)
self.vm_manager = VmManager(
opts=self.opts, logger=self.log,
)
self.vm_manager.post_init()
self.log.info("Starting up VM EventHandler")
self.event_handler = EventHandler(self.opts,
vmm=self.vm_manager,
terminator=self.terminator)
self.event_handler.post_init()
self.event_handler.start()
self.log.info("Starting up VM Master")
self.vm_master = VmMaster(self.opts,
vmm=self.vm_manager,
spawner=self.spawner,
checker=self.checker)
self.vm_master.start()
setproctitle("Copr VMM base process")
def get_managed_vms():
result = {}
if VmManager:
opts = BackendConfigReader().read()
vmm = VmManager(opts, log)
for vmd in vmm.get_all_vm():
result[vmd.vm_name.lower()] = {
'unused': vmd.state == VmStates.READY,
}
return result
def setup_method(self, method):
self.opts = Munch(
redis_db=9,
redis_port=7777,
ssh=Munch(transport="ssh"),
build_groups_count=1,
build_groups={0: {"name": "base", "archs": ["i386", "x86_64"], "max_vm_per_user": 3}},
fedmsg_enabled=False,
sleeptime=0.1,
do_sign=True,
timeout=1800,
# destdir=self.tmp_dir_path,
results_baseurl="/tmp",
)
self.queue = Queue()
self.vm_ip = "127.0.0.1"
self.vm_name = "localhost"
self.group = 0
self.username = "******"
self.rc = get_redis_connection(self.opts)
self.ps = None
self.log_msg_list = []
self.callback = TestCallback()
self.queue = Queue()
self.vmm = VmManager(self.opts)
self.vmm.post_init()
self.vmm.log = MagicMock()
self.pid = 12345
def setup_method(self, method):
self.opts = Munch(
redis_db=9,
redis_port=7777,
ssh=Munch(transport="ssh"),
build_groups_count=2,
build_groups={
GID1: {
"name": "base",
"archs": ["i386", "x86_64"],
"max_vm_per_user": 3,
},
GID2: {
"name": "arm",
"archs": [
"armV7",
]
}
},
fedmsg_enabled=False,
sleeptime=0.1,
do_sign=True,
timeout=1800,
# destdir=self.tmp_dir_path,
results_baseurl="/tmp",
)
self.vm_ip = "127.0.0.1"
self.vm_name = "localhost"
self.vm2_ip = "127.0.0.2"
self.vm2_name = "localhost2"
self.ownername = "bob"
self.rc = get_redis_connection(self.opts)
self.ps = None
self.log_msg_list = []
self.vmm = VmManager(self.opts)
self.vmm.log = MagicMock()
self.pid = 12345
def setup_method(self, method):
self.vm_spawn_min_interval = 30
self.opts = Munch(
redis_host="127.0.0.1",
redis_db=9,
redis_port=7777,
ssh=Munch(
transport="ssh"
),
build_groups_count=2,
build_groups={
0: {
"name": "base",
"archs": ["i386", "x86_64"],
"max_vm_total": 5,
"max_spawn_processes": 3,
"vm_spawn_min_interval": self.vm_spawn_min_interval,
"vm_dirty_terminating_timeout": 120,
"vm_health_check_period": 10,
"vm_health_check_max_time": 60,
"vm_terminating_timeout": 300,
},
1: {
"name": "arm",
"archs": ["armV7"],
"vm_spawn_min_interval": self.vm_spawn_min_interval,
"vm_dirty_terminating_timeout": 120,
"vm_health_check_period": 10,
"vm_health_check_max_time": 60,
"vm_terminating_timeout": 300,
}
},
fedmsg_enabled=False,
sleeptime=0.1,
vm_cycle_timeout=10,
)
self.queue = Queue()
self.vm_ip = "127.0.0.1"
self.vm_name = "localhost"
self.group = 0
self.username = "******"
self.rc = get_redis_connection(self.opts)
self.ps = None
self.log_msg_list = []
self.callback = TestCallback()
# checker = HealthChecker(self.opts, self.callback)
self.checker = MagicMock()
self.spawner = MagicMock()
self.terminator = MagicMock()
self.mc_logger = MagicMock()
self.vmm = VmManager(self.opts, logger=self.mc_logger)
self.event_handler = MagicMock()
self.vm_master = VmMaster(
self.opts,
self.vmm,
self.spawner,
self.checker,
)
self.vm_master.event_handler = MagicMock()
self.pid = 12345
self.vm_ip = "127.0.0.1"
self.vm_name = "build 12345"
class TestVmMaster(object):
def setup_method(self, method):
self.vm_spawn_min_interval = 30
self.opts = Munch(
redis_host="127.0.0.1",
redis_db=9,
redis_port=7777,
ssh=Munch(
transport="ssh"
),
build_groups_count=2,
build_groups={
0: {
"name": "base",
"archs": ["i386", "x86_64"],
"max_vm_total": 5,
"max_spawn_processes": 3,
"vm_spawn_min_interval": self.vm_spawn_min_interval,
"vm_dirty_terminating_timeout": 120,
"vm_health_check_period": 10,
"vm_health_check_max_time": 60,
"vm_terminating_timeout": 300,
},
1: {
"name": "arm",
"archs": ["armV7"],
"vm_spawn_min_interval": self.vm_spawn_min_interval,
"vm_dirty_terminating_timeout": 120,
"vm_health_check_period": 10,
"vm_health_check_max_time": 60,
"vm_terminating_timeout": 300,
}
},
fedmsg_enabled=False,
sleeptime=0.1,
vm_cycle_timeout=10,
)
self.queue = Queue()
self.vm_ip = "127.0.0.1"
self.vm_name = "localhost"
self.group = 0
self.username = "******"
self.rc = get_redis_connection(self.opts)
self.ps = None
self.log_msg_list = []
self.callback = TestCallback()
# checker = HealthChecker(self.opts, self.callback)
self.checker = MagicMock()
self.spawner = MagicMock()
self.terminator = MagicMock()
self.mc_logger = MagicMock()
self.vmm = VmManager(self.opts, logger=self.mc_logger)
self.event_handler = MagicMock()
self.vm_master = VmMaster(
self.opts,
self.vmm,
self.spawner,
self.checker,
)
self.vm_master.event_handler = MagicMock()
self.pid = 12345
self.vm_ip = "127.0.0.1"
self.vm_name = "build 12345"
def clean_redis(self):
keys = self.vmm.rc.keys("*")
if keys:
self.vmm.rc.delete(*keys)
def teardown_method(self, method):
self.clean_redis()
@pytest.fixture
def add_vmd(self):
self.vmd_a1 = self.vmm.add_vm_to_pool("127.0.0.1", "a1", 0)
self.vmd_a2 = self.vmm.add_vm_to_pool("127.0.0.2", "a2", 0)
self.vmd_a3 = self.vmm.add_vm_to_pool("127.0.0.3", "a3", 0)
self.vmd_b1 = self.vmm.add_vm_to_pool("127.0.0.4", "b1", 1)
self.vmd_b2 = self.vmm.add_vm_to_pool("127.0.0.5", "b2", 1)
self.vmd_b3 = self.vmm.add_vm_to_pool("127.0.0.6", "b3", 1)
def rcv_from_ps_message_bus(self):
# don't forget to subscribe self.ps
rcv_msg_list = []
for i in range(10):
msg = self.ps.get_message()
if msg:
rcv_msg_list.append(msg)
time.sleep(0.01)
return rcv_msg_list
def test_pass(self, add_vmd):
pass
def test_remove_old_dirty_vms(self, mc_time, add_vmd):
# pass
self.vmm.start_vm_termination = types.MethodType(MagicMock(), self.vmm)
# VM in ready state, with not empty bount_to_user and (NOW - last_release) > threshold
# should be terminated
for vmd in [self.vmd_a1, self.vmd_a2, self.vmd_b1, self.vmd_b2]:
vmd.store_field(self.rc, "state", VmStates.READY)
for vmd in [self.vmd_a1, self.vmd_a2, self.vmd_b1]:
vmd.store_field(self.rc, "last_release", 0)
for vmd in [self.vmd_a1, self.vmd_b1, self.vmd_b2]:
vmd.store_field(self.rc, "bound_to_user", "user")
for vmd in [self.vmd_a3, self.vmd_b3]:
vmd.store_field(self.rc, "state", VmStates.IN_USE)
mc_time.time.return_value = 1
# no vm terminated
self.vm_master.remove_old_dirty_vms()
assert not self.vmm.start_vm_termination.called
mc_time.time.return_value = self.opts.build_groups[0]["vm_dirty_terminating_timeout"] + 1
# only "a1" and "b1" should be terminated
self.vm_master.remove_old_dirty_vms()
assert self.vmm.start_vm_termination.called
terminated_names = set([call[0][1] for call
in self.vmm.start_vm_termination.call_args_list])
assert set(["a1", "b1"]) == terminated_names
def disabled_test_remove_vm_with_dead_builder(self, mc_time, add_vmd, mc_psutil):
# todo: re-enable after psutil.Process.cmdline will be in use
mc_time.time.return_value = time.time()
self.vm_master.log = MagicMock()
self.vmm.start_vm_termination = MagicMock()
self.vmm.start_vm_termination.return_value = "OK"
for idx, vmd in enumerate([self.vmd_a1, self.vmd_a2,
self.vmd_b1, self.vmd_b2, self.vmd_b3]):
vmd.store_field(self.rc, "state", VmStates.IN_USE)
vmd.store_field(self.rc, "chroot", "fedora-20-x86_64")
vmd.store_field(self.rc, "task_id", "{}-fedora-20-x86_64".format(idx + 1))
vmd.store_field(self.rc, "build_id", idx + 1)
vmd.store_field(self.rc, "in_use_since", 0)
self.rc.hdel(self.vmd_b3.vm_key, "chroot")
for idx, vmd in enumerate([self.vmd_a1, self.vmd_a2, self.vmd_b2, self.vmd_b3]):
vmd.store_field(self.rc, "used_by_pid", idx + 1)
for vmd in [self.vmd_a3, self.vmd_a3]:
vmd.store_field(self.rc, "state", VmStates.READY)
def mc_psutil_process(pid):
p = MagicMock()
mapping = {
"1": "a1",
"2": "a2",
"3": "b1",
"4": "None",
"5": "None",
}
p.cmdline = ["builder vm_name={} suffix".format(mapping.get(str(pid))),]
return p
def mc_psutil_pid_exists(pid):
if str(pid) in ["1", "4"]:
return True
return False
mc_psutil.Process.side_effect = mc_psutil_process
mc_psutil.pid_exists.side_effect = mc_psutil_pid_exists
self.ps = self.vmm.rc.pubsub(ignore_subscribe_messages=True)
self.ps.subscribe(JOB_GRAB_TASK_END_PUBSUB)
self.vm_master.remove_vm_with_dead_builder()
msg_list = self.rcv_from_ps_message_bus()
assert self.vmm.start_vm_termination.call_args_list == [
mock.call('a2', allowed_pre_state='in_use'),
mock.call('b2', allowed_pre_state='in_use'),
mock.call('b3', allowed_pre_state='in_use')
]
# changed logic for the moment
# assert set(["2", "4"]) == set([json.loads(m["data"])["build_id"] for m in msg_list])
def test_check_vms_health(self, mc_time, add_vmd):
self.vm_master.start_vm_check = types.MethodType(MagicMock(), self.vmm)
for vmd in [self.vmd_a1, self.vmd_a2, self.vmd_a3, self.vmd_b1, self.vmd_b2, self.vmd_b3]:
vmd.store_field(self.rc, "last_health_check", 0)
self.vmd_a1.store_field(self.rc, "state", VmStates.IN_USE)
self.vmd_a2.store_field(self.rc, "state", VmStates.CHECK_HEALTH)
self.vmd_a3.store_field(self.rc, "state", VmStates.CHECK_HEALTH_FAILED)
self.vmd_b1.store_field(self.rc, "state", VmStates.GOT_IP)
self.vmd_b2.store_field(self.rc, "state", VmStates.READY)
self.vmd_b3.store_field(self.rc, "state", VmStates.TERMINATING)
mc_time.time.return_value = 1
self.vm_master.check_vms_health()
assert not self.vm_master.start_vm_check.called
mc_time.time.return_value = 1 + self.opts.build_groups[0]["vm_health_check_period"]
self.vm_master.check_vms_health()
to_check = set(call[0][1] for call in self.vm_master.start_vm_check.call_args_list)
assert set(['a1', 'a3', 'b1', 'b2']) == to_check
self.vm_master.start_vm_check.reset_mock()
for vmd in [self.vmd_a1, self.vmd_a2, self.vmd_a3, self.vmd_b1, self.vmd_b2, self.vmd_b3]:
self.rc.hdel(vmd.vm_key, "last_health_check")
self.vm_master.check_vms_health()
to_check = set(call[0][1] for call in self.vm_master.start_vm_check.call_args_list)
assert set(['a1', 'a3', 'b1', 'b2']) == to_check
def test_finalize_long_health_checks(self, mc_time, add_vmd):
mc_time.time.return_value = 0
self.vmd_a1.store_field(self.rc, "state", VmStates.IN_USE)
self.vmd_a2.store_field(self.rc, "state", VmStates.CHECK_HEALTH)
self.vmd_a3.store_field(self.rc, "state", VmStates.CHECK_HEALTH)
self.vmd_a2.store_field(self.rc, "last_health_check", 0)
self.vmd_a3.store_field(self.rc, "last_health_check",
self.opts.build_groups[0]["vm_health_check_max_time"] + 10)
mc_time.time.return_value = self.opts.build_groups[0]["vm_health_check_max_time"] + 11
self.vmm.mark_vm_check_failed = MagicMock()
self.vm_master.finalize_long_health_checks()
assert self.vmm.mark_vm_check_failed.called_once
assert self.vmm.mark_vm_check_failed.call_args[0][0] == "a2"
def test_terminate_again(self, mc_time, add_vmd):
mc_time.time.return_value = 0
self.vmd_a1.store_field(self.rc, "state", VmStates.IN_USE)
self.vmd_a2.store_field(self.rc, "state", VmStates.CHECK_HEALTH)
self.vmd_a3.store_field(self.rc, "state", VmStates.READY)
mc_time.time.return_value = 1
self.vmm.remove_vm_from_pool = MagicMock()
self.vmm.start_vm_termination = MagicMock()
# case 1 no VM in terminating states =>
# no start_vm_termination, no remove_vm_from_pool
# import ipdb; ipdb.set_trace()
self.vm_master.terminate_again()
assert not self.vmm.remove_vm_from_pool.called
assert not self.vmm.start_vm_termination.called
# case 2: one VM in terminating state with unique ip, time_elapsed < threshold
# no start_vm_termination, no remove_vm_from_pool
self.vmd_a1.store_field(self.rc, "state", VmStates.TERMINATING)
self.vmd_a1.store_field(self.rc, "terminating_since", 0)
self.vm_master.terminate_again()
assert not self.vmm.remove_vm_from_pool.called
assert not self.vmm.start_vm_termination.called
# case 3: one VM in terminating state with unique ip, time_elapsed > threshold
# start_vm_termination called, no remove_vm_from_pool
mc_time.time.return_value = 1 + self.opts.build_groups[0]["vm_terminating_timeout"]
self.vm_master.terminate_again()
assert not self.vmm.remove_vm_from_pool.called
assert self.vmm.start_vm_termination.called
assert self.vmm.start_vm_termination.call_args[0][0] == self.vmd_a1.vm_name
self.vmm.start_vm_termination.reset_mock()
# case 4: two VM with the same IP, one in terminating states, , time_elapsed < threshold
# no start_vm_termination, no remove_vm_from_pool
mc_time.time.return_value = 1
self.vmd_a2.store_field(self.rc, "vm_ip", self.vmd_a1.vm_ip)
self.vm_master.terminate_again()
assert not self.vmm.remove_vm_from_pool.called
assert not self.vmm.start_vm_termination.called
# case 4: two VM with the same IP, one in terminating states, , time_elapsed > threshold
# no start_vm_termination, remove_vm_from_pool
mc_time.time.return_value = 1 + self.opts.build_groups[0]["vm_terminating_timeout"]
self.vm_master.terminate_again()
assert self.vmm.remove_vm_from_pool.called
assert self.vmm.remove_vm_from_pool.call_args[0][0] == self.vmd_a1.vm_name
assert not self.vmm.start_vm_termination.called
# def test_run_undefined_helpers(self, mc_setproctitle):
# for target in ["spawner", "terminator", "checker"]:
# setattr(self.vmm, target, None)
# with pytest.raises(RuntimeError):
# self.vm_master.run()
#
# setattr(self.vmm, target, MagicMock())
#
# assert not mc_setproctitle.called
def test_dummy_run(self, mc_time, mc_setproctitle):
mc_do_cycle = MagicMock()
mc_do_cycle.side_effect = [
VmError("FooBar"),
None
]
self.vm_master.do_cycle = types.MethodType(mc_do_cycle, self.vm_master)
self.vmm.mark_server_start = MagicMock()
self.stage = 0
def on_sleep(*args, **kwargs):
self.stage += 1
if self.stage == 1:
pass
elif self.stage >= 2:
self.vm_master.kill_received = True
mc_time.sleep.side_effect = on_sleep
self.vm_master.run()
def test_dummy_terminate(self):
self.vm_master.terminate()
assert self.vm_master.kill_received
assert self.vm_master.checker.terminate.called
assert self.vm_master.spawner.terminate.called
def test_dummy_do_cycle(self):
self.vm_master.remove_old_dirty_vms = types.MethodType(MagicMock(), self.vm_master)
self.vm_master.check_vms_health = types.MethodType(MagicMock(), self.vm_master)
self.vm_master.start_spawn_if_required = types.MethodType(MagicMock(), self.vm_master)
# self.vm_master.remove_old_dirty_vms = types(MagicMock, self.vm_master)
self.vm_master.do_cycle()
assert self.vm_master.remove_old_dirty_vms.called
assert self.vm_master.check_vms_health.called
assert self.vm_master.start_spawn_if_required.called
assert self.vm_master.spawner.recycle.called
def test_dummy_start_spawn_if_required(self):
self.vm_master.try_spawn_one = MagicMock()
self.vm_master.start_spawn_if_required()
assert self.vm_master.try_spawn_one.call_args_list == [
mock.call(group) for group in range(self.opts.build_groups_count)
]
def test__check_total_running_vm_limit_raises(self):
self.vm_master.log = MagicMock()
active_vm_states = [VmStates.GOT_IP, VmStates.READY, VmStates.IN_USE, VmStates.CHECK_HEALTH]
cases = [
# active_vms_number , spawn_procs_number
(x, 11 - x) for x in range(12)
]
self.opts.build_groups[0]["max_vm_total"] = 11
for active_vms_number, spawn_procs_number in cases:
vmd_list = [
self.vmm.add_vm_to_pool("127.0.0.{}".format(idx + 1), "a{}".format(idx), 0)
for idx in range(active_vms_number)
]
for idx in range(active_vms_number):
state = choice(active_vm_states)
vmd_list[idx].store_field(self.rc, "state", state)
self.vm_master.spawner.get_proc_num_per_group.return_value = spawn_procs_number
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_total_running_vm_limit(0)
self.vm_master.log.reset_mock()
# teardown
self.clean_redis()
def test__check_total_running_vm_limit_ok(self):
self.vm_master.log = MagicMock()
active_vm_states = [VmStates.GOT_IP, VmStates.READY, VmStates.IN_USE, VmStates.CHECK_HEALTH]
cases = [
# active_vms_number , spawn_procs_number
(x, 11 - x) for x in range(12)
]
self.opts.build_groups[0]["max_vm_total"] = 12
for active_vms_number, spawn_procs_number in cases:
vmd_list = [
self.vmm.add_vm_to_pool("127.0.0.{}".format(idx + 1), "a{}".format(idx), 0)
for idx in range(active_vms_number)
]
for idx in range(active_vms_number):
state = choice(active_vm_states)
vmd_list[idx].store_field(self.rc, "state", state)
# self.vmm.spawner.children_number = spawn_procs_number
self.vm_master.spawner.get_proc_num_per_group.return_value = spawn_procs_number
# doesn't raise exception
self.vm_master._check_total_running_vm_limit(0)
self.vm_master.log.reset_mock()
# teardown
self.clean_redis()
def test__check_elapsed_time_after_spawn(self, mc_time):
# don't start new spawn if last_spawn_time was in less self.vm_spawn_min_interval ago
mc_time.time.return_value = 0
self.vm_master._check_elapsed_time_after_spawn(0)
self.vm_master.vmm.write_vm_pool_info(0, "last_vm_spawn_start", 0)
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_elapsed_time_after_spawn(0)
mc_time.time.return_value = -1 + self.vm_spawn_min_interval
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_elapsed_time_after_spawn(0)
mc_time.time.return_value = 1 + self.vm_spawn_min_interval
self.vm_master._check_elapsed_time_after_spawn(0)
# we don't care about other group
self.vm_master.vmm.write_vm_pool_info(1, "last_vm_spawn_start", mc_time.time.return_value)
self.vm_master._check_elapsed_time_after_spawn(0)
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_elapsed_time_after_spawn(1)
def test__check_number_of_running_spawn_processes(self):
for i in range(self.opts.build_groups[0]["max_spawn_processes"]):
self.vm_master.spawner.get_proc_num_per_group.return_value = i
self.vm_master._check_number_of_running_spawn_processes(0)
for i in [0, 1, 2, 5, 100]:
self.vm_master.spawner.get_proc_num_per_group.return_value = \
self.opts.build_groups[0]["max_spawn_processes"] + i
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_number_of_running_spawn_processes(0)
def test__check_total_vm_limit(self):
self.vm_master.vmm = MagicMock()
for i in range(2 * self.opts.build_groups[0]["max_vm_total"]):
self.vm_master.vmm.get_all_vm_in_group.return_value = [1 for _ in range(i)]
self.vm_master._check_total_vm_limit(0)
for i in range(2 * self.opts.build_groups[0]["max_vm_total"],
2 * self.opts.build_groups[0]["max_vm_total"] + 10):
self.vm_master.vmm.get_all_vm_in_group.return_value = [1 for _ in range(i)]
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_total_vm_limit(0)
def test_try_spawn_error_handling(self, mc_time):
mc_time.time.return_value = 0
self.vm_master.log = MagicMock()
self.vm_master.spawner.start_spawn.side_effect = IOError()
self.vm_master.try_spawn_one(0)
assert self.vm_master.spawner.start_spawn.called
def test_try_spawn_exit_on_check_fail(self):
check_mocks = []
for check_name in [
"_check_total_running_vm_limit",
"_check_elapsed_time_after_spawn",
"_check_number_of_running_spawn_processes",
"_check_total_vm_limit",
]:
mc_check_func = MagicMock()
check_mocks.append(mc_check_func)
setattr(self.vm_master, check_name, mc_check_func)
self.vm_master.vmm = MagicMock()
for idx in range(len(check_mocks)):
for cm in check_mocks:
cm.side_effect = None
check_mocks[idx].side_effect = VmSpawnLimitReached("test")
self.vm_master.try_spawn_one(0)
assert not self.vm_master.vmm.write_vm_pool_info.called
self.vm_master.vmm.write_vm_pool_info.reset_mock()
for cm in check_mocks:
cm.side_effect = None
self.vm_master.try_spawn_one(0)
assert self.vm_master.vmm.write_vm_pool_info.called
assert self.vm_master.spawner.start_spawn.called
def test_start_vm_check_ok_ok(self):
self.vmm.start_vm_termination = types.MethodType(MagicMock(), self.vmm)
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd = self.vmm.get_vm_by_name(self.vm_name)
# can start, no problem to start
# > can start IN_USE, don't change status
vmd.store_field(self.rc, "state", VmStates.IN_USE)
self.vm_master.start_vm_check(vm_name=self.vm_name)
assert self.checker.run_check_health.called
self.checker.run_check_health.reset_mock()
assert vmd.get_field(self.rc, "state") == VmStates.IN_USE
# > changes status to HEALTH_CHECK
states = [VmStates.GOT_IP, VmStates.CHECK_HEALTH_FAILED, VmStates.READY]
for state in states:
vmd.store_field(self.rc, "state", state)
self.vm_master.start_vm_check(vm_name=self.vm_name)
assert self.checker.run_check_health.called
self.checker.run_check_health.reset_mock()
assert vmd.get_field(self.rc, "state") == VmStates.CHECK_HEALTH
def test_start_vm_check_wrong_old_state(self):
self.vmm.start_vm_termination = types.MethodType(MagicMock(), self.vmm)
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd = self.vmm.get_vm_by_name(self.vm_name)
states = [VmStates.TERMINATING, VmStates.CHECK_HEALTH]
for state in states:
vmd.store_field(self.rc, "state", state)
assert not self.vm_master.start_vm_check(vm_name=self.vm_name)
assert not self.checker.run_check_health.called
assert vmd.get_field(self.rc, "state") == state
def test_start_vm_check_lua_ok_check_spawn_failed(self):
self.vmm.start_vm_termination = types.MethodType(MagicMock(), self.vmm)
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd = self.vmm.get_vm_by_name(self.vm_name)
self.vm_master.checker.run_check_health.side_effect = RuntimeError()
# restore orig state
states = [VmStates.GOT_IP, VmStates.CHECK_HEALTH_FAILED, VmStates.READY, VmStates.IN_USE]
for state in states:
vmd.store_field(self.rc, "state", state)
self.vm_master.start_vm_check(vm_name=self.vm_name)
assert self.checker.run_check_health.called
self.checker.run_check_health.reset_mock()
assert vmd.get_field(self.rc, "state") == state
def test_manager_setup(self):
vmm = VmManager(self.opts)
assert GID1 in vmm.vm_groups
assert GID2 in vmm.vm_groups
assert len(vmm.vm_groups) == 2
class TestManager(object):
def setup_method(self, method):
self.opts = Munch(
redis_db=9,
redis_port=7777,
ssh=Munch(transport="ssh"),
build_groups_count=2,
build_groups={
GID1: {
"name": "base",
"archs": ["i386", "x86_64"],
"max_vm_per_user": 3,
},
GID2: {
"name": "arm",
"archs": [
"armV7",
]
}
},
fedmsg_enabled=False,
sleeptime=0.1,
do_sign=True,
timeout=1800,
# destdir=self.tmp_dir_path,
results_baseurl="/tmp",
)
self.vm_ip = "127.0.0.1"
self.vm_name = "localhost"
self.vm2_ip = "127.0.0.2"
self.vm2_name = "localhost2"
self.ownername = "bob"
self.rc = get_redis_connection(self.opts)
self.ps = None
self.log_msg_list = []
self.vmm = VmManager(self.opts)
self.vmm.log = MagicMock()
self.pid = 12345
def teardown_method(self, method):
keys = self.vmm.rc.keys("*")
if keys:
self.vmm.rc.delete(*keys)
def test_manager_setup(self):
vmm = VmManager(self.opts)
assert GID1 in vmm.vm_groups
assert GID2 in vmm.vm_groups
assert len(vmm.vm_groups) == 2
def test_add_vm_to_pool(self):
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
with pytest.raises(VmError):
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vm_list = self.vmm.get_all_vm_in_group(GID1)
vm = self.vmm.get_vm_by_name(self.vm_name)
assert len(vm_list) == 1
assert vm_list[0].__dict__ == vm.__dict__
assert vm.vm_ip == self.vm_ip
assert vm.vm_name == self.vm_name
assert vm.group == GID1
def test_mark_vm_check_failed(self, mc_time):
self.vmm.start_vm_termination = types.MethodType(MagicMock(), self.vmm)
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vmd = self.vmm.get_vm_by_name(self.vm_name)
vmd.store_field(self.rc, "state", VmStates.CHECK_HEALTH)
vmd.store_field(self.rc, "last_health_check", 12345)
self.vmm.mark_vm_check_failed(self.vm_name)
assert vmd.get_field(self.rc, "state") == VmStates.CHECK_HEALTH_FAILED
states = [
VmStates.GOT_IP, VmStates.IN_USE, VmStates.READY,
VmStates.TERMINATING
]
for state in states:
vmd.store_field(self.rc, "state", state)
self.vmm.mark_vm_check_failed(self.vm_name)
assert vmd.get_field(self.rc, "state") == state
def test_acquire_vm_no_vm_after_server_restart(self, mc_time):
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vmd.store_field(self.rc, "state", VmStates.READY)
# undefined both last_health_check and server_start_timestamp
mc_time.time.return_value = 0.1
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm([GID1], self.ownername, 42, SANDBOX)
# only server start timestamp is defined
mc_time.time.return_value = 1
self.vmm.mark_server_start()
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm([GID1], self.ownername, 42, SANDBOX)
# only last_health_check defined
self.rc.delete(KEY_SERVER_INFO)
vmd.store_field(self.rc, "last_health_check", 0)
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm([GID1], self.ownername, 42, SANDBOX)
# both defined but last_health_check < server_start_time
self.vmm.mark_server_start()
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm([GID1], self.ownername, 42, SANDBOX)
# and finally last_health_check > server_start_time
vmd.store_field(self.rc, "last_health_check", 2)
vmd_res = self.vmm.acquire_vm([GID1], self.ownername, 42, SANDBOX)
assert vmd.vm_name == vmd_res.vm_name
def test_acquire_vm_extra_kwargs(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vmd.store_field(self.rc, "state", VmStates.READY)
vmd.store_field(self.rc, "last_health_check", 2)
kwargs = {
"task_id": "20-fedora-20-x86_64",
"build_id": "20",
"chroot": "fedora-20-x86_64"
}
vmd_got = self.vmm.acquire_vm([GID1], self.ownername, self.pid,
SANDBOX, **kwargs)
for k, v in kwargs.items():
assert vmd_got.get_field(self.rc, k) == v
def test_another_owner_cannot_acquire_vm(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vmd.store_field(self.rc, "state", VmStates.READY)
vmd.store_field(self.rc, "last_health_check", 2)
vmd.store_field(self.vmm.rc, "bound_to_user", "foo")
vmd.store_field(self.vmm.rc, "sandbox", SANDBOX)
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(groups=[GID1],
ownername=self.ownername,
pid=self.pid,
sandbox=SANDBOX)
vm = self.vmm.acquire_vm(groups=[GID1],
ownername="foo",
pid=self.pid,
sandbox=SANDBOX)
assert vm.vm_name == self.vm_name
def test_different_sandbox_cannot_acquire_vm(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vmd.store_field(self.rc, "state", VmStates.READY)
vmd.store_field(self.rc, "last_health_check", 2)
vmd.store_field(self.vmm.rc, "bound_to_user", "foo")
vmd.store_field(self.vmm.rc, "sandbox", "sandboxA")
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(groups=[GID1],
ownername="foo",
pid=self.pid,
sandbox="sandboxB")
vm = self.vmm.acquire_vm(groups=[GID1],
ownername="foo",
pid=self.pid,
sandbox="sandboxA")
assert vm.vm_name == self.vm_name
def test_acquire_vm(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vmd_alt = self.vmm.add_vm_to_pool(self.vm_ip, "vm_alt", GID1)
vmd2 = self.vmm.add_vm_to_pool(self.vm2_ip, self.vm2_name, GID2)
vmd.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.rc, "state", VmStates.READY)
vmd2.store_field(self.rc, "state", VmStates.READY)
vmd.store_field(self.rc, "last_health_check", 2)
vmd_alt.store_field(self.rc, "last_health_check", 2)
vmd2.store_field(self.rc, "last_health_check", 2)
vmd_alt.store_field(self.vmm.rc, "bound_to_user", self.ownername)
vmd_alt.store_field(self.vmm.rc, "sandbox", SANDBOX)
vmd_got_first = self.vmm.acquire_vm([GID1, GID2],
ownername=self.ownername,
pid=self.pid,
sandbox=SANDBOX)
assert vmd_got_first.vm_name == "vm_alt"
vmd_got_second = self.vmm.acquire_vm([GID1, GID2],
ownername=self.ownername,
pid=self.pid,
sandbox=SANDBOX)
assert vmd_got_second.vm_name == self.vm_name
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(groups=[GID1],
ownername=self.ownername,
pid=self.pid,
sandbox=SANDBOX)
vmd_got_third = self.vmm.acquire_vm(groups=[GID1, GID2],
ownername=self.ownername,
pid=self.pid,
sandbox=SANDBOX)
assert vmd_got_third.vm_name == self.vm2_name
def test_acquire_vm_per_user_limit(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
max_vm_per_user = self.opts.build_groups[GID1]["max_vm_per_user"]
vmd_list = []
for idx in range(max_vm_per_user + 1):
vmd = self.vmm.add_vm_to_pool("127.0.{}.1".format(idx),
"vm_{}".format(idx), GID1)
vmd.store_field(self.rc, "state", VmStates.READY)
vmd.store_field(self.rc, "last_health_check", 2)
vmd_list.append(vmd)
for idx in range(max_vm_per_user):
self.vmm.acquire_vm([GID1], self.ownername, idx, SANDBOX)
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm([GID1], self.ownername, 42, SANDBOX)
def test_acquire_only_ready_state(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
vmd_main = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vmd_main.store_field(self.rc, "last_health_check", 2)
for state in [
VmStates.IN_USE, VmStates.GOT_IP, VmStates.CHECK_HEALTH,
VmStates.TERMINATING, VmStates.CHECK_HEALTH_FAILED
]:
vmd_main.store_field(self.rc, "state", state)
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(groups=[GID1],
ownername=self.ownername,
pid=self.pid,
sandbox=SANDBOX)
def test_acquire_and_release_vm(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
vmd_main = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vmd_alt = self.vmm.add_vm_to_pool(self.vm_ip, "vm_alt", GID1)
vmd_main.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.vmm.rc, "bound_to_user", self.ownername)
vmd_alt.store_field(self.vmm.rc, "sandbox", SANDBOX)
vmd_main.store_field(self.rc, "last_health_check", 2)
vmd_alt.store_field(self.rc, "last_health_check", 2)
vmd_got_first = self.vmm.acquire_vm(groups=[GID1],
ownername=self.ownername,
pid=self.pid,
sandbox=SANDBOX)
assert vmd_got_first.vm_name == "vm_alt"
self.vmm.release_vm("vm_alt")
vmd_got_again = self.vmm.acquire_vm(groups=[GID1],
ownername=self.ownername,
pid=self.pid,
sandbox=SANDBOX)
assert vmd_got_again.vm_name == "vm_alt"
vmd_got_another = self.vmm.acquire_vm(groups=[GID1],
ownername=self.ownername,
pid=self.pid,
sandbox=SANDBOX)
assert vmd_got_another.vm_name == self.vm_name
def test_release_only_in_use(self):
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
for state in [
VmStates.READY, VmStates.GOT_IP, VmStates.CHECK_HEALTH,
VmStates.TERMINATING, VmStates.CHECK_HEALTH_FAILED
]:
vmd.store_field(self.rc, "state", state)
assert not self.vmm.release_vm(self.vm_name)
def rcv_from_ps_message_bus(self):
# don't forget to subscribe self.ps
rcv_msg_list = []
for i in range(10):
msg = self.ps.get_message()
if msg:
rcv_msg_list.append(msg)
time.sleep(0.01)
return rcv_msg_list
def test_start_vm_termination(self):
self.ps = self.vmm.rc.pubsub(ignore_subscribe_messages=True)
self.ps.subscribe(PUBSUB_MB)
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
self.vmm.start_vm_termination(self.vm_name)
rcv_msg_list = self.rcv_from_ps_message_bus()
# print(rcv_msg_list)
assert len(rcv_msg_list) == 1
msg = rcv_msg_list[0]
assert msg["type"] == "message"
data = json.loads(msg["data"])
assert data["topic"] == EventTopics.VM_TERMINATION_REQUEST
assert data["vm_name"] == self.vm_name
def test_start_vm_termination_2(self):
self.ps = self.vmm.rc.pubsub(ignore_subscribe_messages=True)
self.ps.subscribe(PUBSUB_MB)
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vmd.store_field(self.rc, "state", VmStates.TERMINATING)
self.vmm.start_vm_termination(self.vm_name,
allowed_pre_state=VmStates.TERMINATING)
rcv_msg_list = self.rcv_from_ps_message_bus()
# print(rcv_msg_list)
assert len(rcv_msg_list) == 1
msg = rcv_msg_list[0]
assert msg["type"] == "message"
data = json.loads(msg["data"])
assert data["topic"] == EventTopics.VM_TERMINATION_REQUEST
assert data["vm_name"] == self.vm_name
def test_start_vm_termination_fail(self):
self.ps = self.vmm.rc.pubsub(ignore_subscribe_messages=True)
self.ps.subscribe(PUBSUB_MB)
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
vmd.store_field(self.rc, "state", VmStates.TERMINATING)
self.vmm.start_vm_termination(self.vm_name)
rcv_msg_list = self.rcv_from_ps_message_bus()
assert len(rcv_msg_list) == 0
vmd.store_field(self.rc, "state", VmStates.READY)
self.vmm.start_vm_termination(self.vm_name,
allowed_pre_state=VmStates.IN_USE)
rcv_msg_list = self.rcv_from_ps_message_bus()
assert len(rcv_msg_list) == 0
assert vmd.get_field(self.rc, "state") == VmStates.READY
vmd.store_field(self.rc, "state", VmStates.TERMINATING)
self.vmm.start_vm_termination(self.vm_name)
rcv_msg_list = self.rcv_from_ps_message_bus()
assert len(rcv_msg_list) == 0
assert vmd.get_field(self.rc, "state") == VmStates.TERMINATING
def test_remove_vm_from_pool_only_terminated(self):
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, GID1)
for state in [
VmStates.IN_USE, VmStates.GOT_IP, VmStates.CHECK_HEALTH,
VmStates.READY, VmStates.CHECK_HEALTH_FAILED
]:
vmd.store_field(self.vmm.rc, "state", state)
with pytest.raises(VmError):
self.vmm.remove_vm_from_pool(self.vm_name)
vmd.store_field(self.vmm.rc, "state", VmStates.TERMINATING)
self.vmm.remove_vm_from_pool(self.vm_name)
assert self.vmm.rc.scard(KEY_VM_POOL.format(group=GID1)) == 0
def test_get_vms(self, capsys):
vmd_1 = self.vmm.add_vm_to_pool(self.vm_ip, "a1", GID1)
vmd_2 = self.vmm.add_vm_to_pool(self.vm_ip, "a2", GID1)
vmd_3 = self.vmm.add_vm_to_pool(self.vm_ip, "b1", 1)
vmd_4 = self.vmm.add_vm_to_pool(self.vm_ip, "b2", 1)
vmd_5 = self.vmm.add_vm_to_pool(self.vm_ip, "b3", 1)
assert set(v.vm_name for v in self.vmm.get_all_vm_in_group(0)) == set(
["a1", "a2"])
assert set(v.vm_name for v in self.vmm.get_all_vm_in_group(1)) == set(
["b1", "b2", "b3"])
assert set(v.vm_name for v in self.vmm.get_all_vm()) == set(
["a1", "a2", "b1", "b2", "b3"])
vmd_1.store_field(self.rc, "state", VmStates.GOT_IP)
vmd_2.store_field(self.rc, "state", VmStates.GOT_IP)
vmd_3.store_field(self.rc, "state", VmStates.GOT_IP)
vmd_4.store_field(self.rc, "state", VmStates.READY)
vmd_5.store_field(self.rc, "state", VmStates.IN_USE)
vmd_list = self.vmm.get_vm_by_group_and_state_list(
group=None, state_list=[VmStates.GOT_IP, VmStates.IN_USE])
assert set(v.vm_name
for v in vmd_list) == set(["a1", "a2", "b1", "b3"])
vmd_list = self.vmm.get_vm_by_group_and_state_list(
group=1, state_list=[VmStates.READY])
assert set(v.vm_name for v in vmd_list) == set(["b2"])
self.vmm.info()
def test_look_up_vms_by_ip(self, capsys):
self.vmm.add_vm_to_pool(self.vm_ip, "a1", GID1)
r1 = self.vmm.lookup_vms_by_ip(self.vm_ip)
assert len(r1) == 1
assert r1[0].vm_name == "a1"
self.vmm.add_vm_to_pool(self.vm_ip, "a2", GID1)
r2 = self.vmm.lookup_vms_by_ip(self.vm_ip)
assert len(r2) == 2
r2 = sorted(r2, key=lambda vmd: vmd.vm_name)
assert r2[0].vm_name == "a1"
assert r2[1].vm_name == "a2"
self.vmm.add_vm_to_pool("127.1.1.111", "b1", 1)
r3 = self.vmm.lookup_vms_by_ip(self.vm_ip)
assert len(r3) == 2
r3 = sorted(r3, key=lambda vmd: vmd.vm_name)
assert r3[0].vm_name == "a1"
assert r3[1].vm_name == "a2"
def test_mark_server_start(self, mc_time):
assert self.rc.hget(KEY_SERVER_INFO, "server_start_timestamp") is None
for i in range(100):
val = 100 * i + 0.12345
mc_time.time.return_value = val
self.vmm.mark_server_start()
assert self.rc.hget(KEY_SERVER_INFO,
"server_start_timestamp") == "{}".format(val)
class TestManager(object):
def setup_method(self, method):
self.opts = Munch(
redis_db=9,
redis_port=7777,
ssh=Munch(
transport="ssh"
),
build_groups_count=1,
build_groups={
0: {
"name": "base",
"archs": ["i386", "x86_64"],
"max_vm_per_user": 3,
}
},
fedmsg_enabled=False,
sleeptime=0.1,
do_sign=True,
timeout=1800,
# destdir=self.tmp_dir_path,
results_baseurl="/tmp",
)
self.queue = Queue()
self.vm_ip = "127.0.0.1"
self.vm_name = "localhost"
self.group = 0
self.username = "******"
self.rc = get_redis_connection(self.opts)
self.ps = None
self.log_msg_list = []
self.callback = TestCallback()
self.queue = Queue()
self.vmm = VmManager(self.opts)
self.vmm.log = MagicMock()
self.pid = 12345
def teardown_method(self, method):
keys = self.vmm.rc.keys("*")
if keys:
self.vmm.rc.delete(*keys)
@pytest.fixture
def f_second_group(self):
self.opts.build_groups_count = 2
self.opts.build_groups[1] = {
"name": "arm",
"archs": ["armV7",]
}
def test_add_vm_to_pool(self):
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vm_list = self.vmm.get_all_vm_in_group(self.group)
vm = self.vmm.get_vm_by_name(self.vm_name)
assert len(vm_list) == 1
assert vm_list[0].__dict__ == vm.__dict__
assert self.group in self.vmm.vm_groups
assert len(self.vmm.vm_groups) == 1
with pytest.raises(VmError):
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
def test_mark_vm_check_failed(self, mc_time):
self.vmm.start_vm_termination = types.MethodType(MagicMock(), self.vmm)
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd = self.vmm.get_vm_by_name(self.vm_name)
vmd.store_field(self.rc, "state", VmStates.CHECK_HEALTH)
vmd.store_field(self.rc, "last_health_check", 12345)
self.vmm.mark_vm_check_failed(self.vm_name)
assert vmd.get_field(self.rc, "state") == VmStates.CHECK_HEALTH_FAILED
states = [VmStates.GOT_IP, VmStates.IN_USE, VmStates.READY, VmStates.TERMINATING]
for state in states:
vmd.store_field(self.rc, "state", state)
self.vmm.mark_vm_check_failed(self.vm_name)
assert vmd.get_field(self.rc, "state") == state
def test_acquire_vm_no_vm_after_server_restart(self, mc_time):
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd.store_field(self.rc, "state", VmStates.READY)
# undefined both last_health_check and server_start_timestamp
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(0, self.username, 42)
# only server start timestamp is defined
mc_time.time.return_value = 1
self.vmm.mark_server_start()
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(0, self.username, 42)
# only last_health_check defined
self.rc.delete(KEY_SERVER_INFO)
vmd.store_field(self.rc, "last_health_check", 0)
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(0, self.username, 42)
# both defined but last_health_check < server_start_time
self.vmm.mark_server_start()
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(0, self.username, 42)
# and finally last_health_check > server_start_time
vmd.store_field(self.rc, "last_health_check", 2)
vmd_res = self.vmm.acquire_vm(0, self.username, 42)
assert vmd.vm_name == vmd_res.vm_name
def test_acquire_vm_extra_kwargs(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd.store_field(self.rc, "state", VmStates.READY)
vmd.store_field(self.rc, "last_health_check", 2)
kwargs = {
"task_id": "20-fedora-20-x86_64",
"build_id": "20",
"chroot": "fedora-20-x86_64"
}
vmd_got = self.vmm.acquire_vm(self.group, self.username, self.pid, **kwargs)
for k, v in kwargs.items():
assert vmd_got.get_field(self.rc, k) == v
def test_acquire_vm(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
vmd_main = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd_alt = self.vmm.add_vm_to_pool(self.vm_ip, "alternative", self.group)
vmd_main.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.vmm.rc, "bound_to_user", self.username)
vmd_main.store_field(self.rc, "last_health_check", 2)
vmd_alt.store_field(self.rc, "last_health_check", 2)
vmd_got_first = self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
assert vmd_got_first.vm_name == "alternative"
vmd_got_second = self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
assert vmd_got_second.vm_name == self.vm_name
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
def test_acquire_vm_per_user_limit(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
max_vm_per_user = self.opts.build_groups[0]["max_vm_per_user"]
acquired_vmd_list = []
for idx in range(max_vm_per_user + 1):
vmd = self.vmm.add_vm_to_pool("127.0.{}.1".format(idx), "vm_{}".format(idx), self.group)
vmd.store_field(self.rc, "state", VmStates.READY)
vmd.store_field(self.rc, "last_health_check", 2)
acquired_vmd_list.append(vmd)
for idx in range(max_vm_per_user):
vmd = self.vmm.acquire_vm(0, self.username, idx)
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(0, self.username, 42)
acquired_vmd_list[-1].store_field(self.rc, "state", VmStates.READY)
self.vmm.acquire_vm(0, self.username, 42)
def test_acquire_only_ready_state(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
vmd_main = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd_main.store_field(self.rc, "last_health_check", 2)
for state in [VmStates.IN_USE, VmStates.GOT_IP, VmStates.CHECK_HEALTH,
VmStates.TERMINATING, VmStates.CHECK_HEALTH_FAILED]:
vmd_main.store_field(self.rc, "state", state)
with pytest.raises(NoVmAvailable):
self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
def test_acquire_and_release_vm(self, mc_time):
mc_time.time.return_value = 0
self.vmm.mark_server_start()
vmd_main = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd_alt = self.vmm.add_vm_to_pool(self.vm_ip, "alternative", self.group)
vmd_main.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.rc, "state", VmStates.READY)
vmd_alt.store_field(self.vmm.rc, "bound_to_user", self.username)
vmd_main.store_field(self.rc, "last_health_check", 2)
vmd_alt.store_field(self.rc, "last_health_check", 2)
vmd_got_first = self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
assert vmd_got_first.vm_name == "alternative"
self.vmm.release_vm("alternative")
vmd_got_again = self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
assert vmd_got_again.vm_name == "alternative"
vmd_got_another = self.vmm.acquire_vm(group=self.group, username=self.username, pid=self.pid)
assert vmd_got_another.vm_name == self.vm_name
def test_release_only_in_use(self):
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
for state in [VmStates.READY, VmStates.GOT_IP, VmStates.CHECK_HEALTH,
VmStates.TERMINATING, VmStates.CHECK_HEALTH_FAILED]:
vmd.store_field(self.rc, "state", state)
assert not self.vmm.release_vm(self.vm_name)
def rcv_from_ps_message_bus(self):
# don't forget to subscribe self.ps
rcv_msg_list = []
for i in range(10):
msg = self.ps.get_message()
if msg:
rcv_msg_list.append(msg)
time.sleep(0.01)
return rcv_msg_list
def test_start_vm_termination(self):
self.ps = self.vmm.rc.pubsub(ignore_subscribe_messages=True)
self.ps.subscribe(PUBSUB_MB)
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
self.vmm.start_vm_termination(self.vm_name)
rcv_msg_list = self.rcv_from_ps_message_bus()
# print(rcv_msg_list)
assert len(rcv_msg_list) == 1
msg = rcv_msg_list[0]
assert msg["type"] == "message"
data = json.loads(msg["data"])
assert data["topic"] == EventTopics.VM_TERMINATION_REQUEST
assert data["vm_name"] == self.vm_name
def test_start_vm_termination_2(self):
self.ps = self.vmm.rc.pubsub(ignore_subscribe_messages=True)
self.ps.subscribe(PUBSUB_MB)
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd.store_field(self.rc, "state", VmStates.TERMINATING)
self.vmm.start_vm_termination(self.vm_name, allowed_pre_state=VmStates.TERMINATING)
rcv_msg_list = self.rcv_from_ps_message_bus()
# print(rcv_msg_list)
assert len(rcv_msg_list) == 1
msg = rcv_msg_list[0]
assert msg["type"] == "message"
data = json.loads(msg["data"])
assert data["topic"] == EventTopics.VM_TERMINATION_REQUEST
assert data["vm_name"] == self.vm_name
def test_start_vm_termination_fail(self):
self.ps = self.vmm.rc.pubsub(ignore_subscribe_messages=True)
self.ps.subscribe(PUBSUB_MB)
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd.store_field(self.rc, "state", VmStates.TERMINATING)
self.vmm.start_vm_termination(self.vm_name)
rcv_msg_list = self.rcv_from_ps_message_bus()
assert len(rcv_msg_list) == 0
vmd.store_field(self.rc, "state", VmStates.READY)
self.vmm.start_vm_termination(self.vm_name, allowed_pre_state=VmStates.IN_USE)
rcv_msg_list = self.rcv_from_ps_message_bus()
assert len(rcv_msg_list) == 0
assert vmd.get_field(self.rc, "state") == VmStates.READY
vmd.store_field(self.rc, "state", VmStates.TERMINATING)
self.vmm.start_vm_termination(self.vm_name)
rcv_msg_list = self.rcv_from_ps_message_bus()
assert len(rcv_msg_list) == 0
assert vmd.get_field(self.rc, "state") == VmStates.TERMINATING
def test_remove_vm_from_pool_only_terminated(self):
vmd = self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
for state in [VmStates.IN_USE, VmStates.GOT_IP, VmStates.CHECK_HEALTH,
VmStates.READY, VmStates.CHECK_HEALTH_FAILED]:
vmd.store_field(self.vmm.rc, "state", state)
with pytest.raises(VmError):
self.vmm.remove_vm_from_pool(self.vm_name)
vmd.store_field(self.vmm.rc, "state", VmStates.TERMINATING)
self.vmm.remove_vm_from_pool(self.vm_name)
assert self.vmm.rc.scard(KEY_VM_POOL.format(group=self.group)) == 0
def test_get_vms(self, f_second_group, capsys):
vmd_1 = self.vmm.add_vm_to_pool(self.vm_ip, "a1", self.group)
vmd_2 = self.vmm.add_vm_to_pool(self.vm_ip, "a2", self.group)
vmd_3 = self.vmm.add_vm_to_pool(self.vm_ip, "b1", 1)
vmd_4 = self.vmm.add_vm_to_pool(self.vm_ip, "b2", 1)
vmd_5 = self.vmm.add_vm_to_pool(self.vm_ip, "b3", 1)
assert set(v.vm_name for v in self.vmm.get_all_vm_in_group(0)) == set(["a1", "a2"])
assert set(v.vm_name for v in self.vmm.get_all_vm_in_group(1)) == set(["b1", "b2", "b3"])
assert set(v.vm_name for v in self.vmm.get_all_vm()) == set(["a1", "a2", "b1", "b2", "b3"])
vmd_1.store_field(self.rc, "state", VmStates.GOT_IP)
vmd_2.store_field(self.rc, "state", VmStates.GOT_IP)
vmd_3.store_field(self.rc, "state", VmStates.GOT_IP)
vmd_4.store_field(self.rc, "state", VmStates.READY)
vmd_5.store_field(self.rc, "state", VmStates.IN_USE)
vmd_list = self.vmm.get_vm_by_group_and_state_list(group=None, state_list=[VmStates.GOT_IP, VmStates.IN_USE])
assert set(v.vm_name for v in vmd_list) == set(["a1", "a2", "b1", "b3"])
vmd_list = self.vmm.get_vm_by_group_and_state_list(group=1, state_list=[VmStates.READY])
assert set(v.vm_name for v in vmd_list) == set(["b2"])
self.vmm.info()
def test_look_up_vms_by_ip(self, f_second_group, capsys):
self.vmm.add_vm_to_pool(self.vm_ip, "a1", self.group)
r1 = self.vmm.lookup_vms_by_ip(self.vm_ip)
assert len(r1) == 1
assert r1[0].vm_name == "a1"
self.vmm.add_vm_to_pool(self.vm_ip, "a2", self.group)
r2 = self.vmm.lookup_vms_by_ip(self.vm_ip)
assert len(r2) == 2
r2 = sorted(r2, key=lambda vmd: vmd.vm_name)
assert r2[0].vm_name == "a1"
assert r2[1].vm_name == "a2"
self.vmm.add_vm_to_pool("127.1.1.111", "b1", 1)
r3 = self.vmm.lookup_vms_by_ip(self.vm_ip)
assert len(r3) == 2
r3 = sorted(r3, key=lambda vmd: vmd.vm_name)
assert r3[0].vm_name == "a1"
assert r3[1].vm_name == "a2"
def test_mark_server_start(self, mc_time):
assert self.rc.hget(KEY_SERVER_INFO, "server_start_timestamp") is None
for i in range(100):
val = 100 * i + 0.12345
mc_time.time.return_value = val
self.vmm.mark_server_start()
assert self.rc.hget(KEY_SERVER_INFO, "server_start_timestamp") == "{}".format(val)
def main():
opts = BackendConfigReader().read()
vmm = VmManager(opts, None)
vmm.post_init()
print(vmm.info())
def setup_method(self, method):
self.vm_spawn_min_interval = 30
self.opts = Munch(
redis_host="127.0.0.1",
redis_db=9,
redis_port=7777,
ssh=Munch(transport="ssh"),
build_groups_count=2,
build_groups={
0: {
"name": "base",
"archs": ["i386", "x86_64"],
"max_vm_total": 5,
"max_spawn_processes": 3,
"vm_spawn_min_interval": self.vm_spawn_min_interval,
"vm_dirty_terminating_timeout": 120,
"vm_health_check_period": 10,
"vm_health_check_max_time": 60,
"vm_terminating_timeout": 300,
},
1: {
"name": "arm",
"archs": ["armV7"],
"vm_spawn_min_interval": self.vm_spawn_min_interval,
"vm_dirty_terminating_timeout": 120,
"vm_health_check_period": 10,
"vm_health_check_max_time": 60,
"vm_terminating_timeout": 300,
}
},
fedmsg_enabled=False,
sleeptime=0.1,
vm_cycle_timeout=10,
)
self.queue = Queue()
self.vm_ip = "127.0.0.1"
self.vm_name = "localhost"
self.group = 0
self.username = "******"
self.rc = get_redis_connection(self.opts)
self.ps = None
self.log_msg_list = []
self.callback = TestCallback()
# checker = HealthChecker(self.opts, self.callback)
self.checker = MagicMock()
self.spawner = MagicMock()
self.terminator = MagicMock()
self.mc_logger = MagicMock()
self.vmm = VmManager(self.opts, logger=self.mc_logger)
self.vmm.post_init()
self.event_handler = MagicMock()
self.vm_master = VmMaster(
self.opts,
self.vmm,
self.spawner,
self.checker,
)
self.vm_master.event_handler = MagicMock()
self.pid = 12345
self.vm_ip = "127.0.0.1"
self.vm_name = "build 12345"
class TestVmMaster(object):
def setup_method(self, method):
self.vm_spawn_min_interval = 30
self.opts = Munch(
redis_host="127.0.0.1",
redis_db=9,
redis_port=7777,
ssh=Munch(transport="ssh"),
build_groups_count=2,
build_groups={
0: {
"name": "base",
"archs": ["i386", "x86_64"],
"max_vm_total": 5,
"max_spawn_processes": 3,
"vm_spawn_min_interval": self.vm_spawn_min_interval,
"vm_dirty_terminating_timeout": 120,
"vm_health_check_period": 10,
"vm_health_check_max_time": 60,
"vm_terminating_timeout": 300,
},
1: {
"name": "arm",
"archs": ["armV7"],
"vm_spawn_min_interval": self.vm_spawn_min_interval,
"vm_dirty_terminating_timeout": 120,
"vm_health_check_period": 10,
"vm_health_check_max_time": 60,
"vm_terminating_timeout": 300,
}
},
fedmsg_enabled=False,
sleeptime=0.1,
vm_cycle_timeout=10,
)
self.queue = Queue()
self.vm_ip = "127.0.0.1"
self.vm_name = "localhost"
self.group = 0
self.username = "******"
self.rc = get_redis_connection(self.opts)
self.ps = None
self.log_msg_list = []
self.callback = TestCallback()
# checker = HealthChecker(self.opts, self.callback)
self.checker = MagicMock()
self.spawner = MagicMock()
self.terminator = MagicMock()
self.mc_logger = MagicMock()
self.vmm = VmManager(self.opts, logger=self.mc_logger)
self.vmm.post_init()
self.event_handler = MagicMock()
self.vm_master = VmMaster(
self.opts,
self.vmm,
self.spawner,
self.checker,
)
self.vm_master.event_handler = MagicMock()
self.pid = 12345
self.vm_ip = "127.0.0.1"
self.vm_name = "build 12345"
def clean_redis(self):
keys = self.vmm.rc.keys("*")
if keys:
self.vmm.rc.delete(*keys)
def teardown_method(self, method):
self.clean_redis()
@pytest.fixture
def add_vmd(self):
self.vmd_a1 = self.vmm.add_vm_to_pool("127.0.0.1", "a1", 0)
self.vmd_a2 = self.vmm.add_vm_to_pool("127.0.0.2", "a2", 0)
self.vmd_a3 = self.vmm.add_vm_to_pool("127.0.0.3", "a3", 0)
self.vmd_b1 = self.vmm.add_vm_to_pool("127.0.0.4", "b1", 1)
self.vmd_b2 = self.vmm.add_vm_to_pool("127.0.0.5", "b2", 1)
self.vmd_b3 = self.vmm.add_vm_to_pool("127.0.0.6", "b3", 1)
def rcv_from_ps_message_bus(self):
# don't forget to subscribe self.ps
rcv_msg_list = []
for i in range(10):
msg = self.ps.get_message()
if msg:
rcv_msg_list.append(msg)
time.sleep(0.01)
return rcv_msg_list
def test_pass(self, add_vmd):
pass
def test_remove_old_dirty_vms(self, mc_time, add_vmd):
# pass
self.vmm.start_vm_termination = types.MethodType(MagicMock(), self.vmm)
# VM in ready state, with not empty bount_to_user and (NOW - last_release) > threshold
# should be terminated
for vmd in [self.vmd_a1, self.vmd_a2, self.vmd_b1, self.vmd_b2]:
vmd.store_field(self.rc, "state", VmStates.READY)
for vmd in [self.vmd_a1, self.vmd_a2, self.vmd_b1]:
vmd.store_field(self.rc, "last_release", 0)
for vmd in [self.vmd_a1, self.vmd_b1, self.vmd_b2]:
vmd.store_field(self.rc, "bound_to_user", "user")
for vmd in [self.vmd_a3, self.vmd_b3]:
vmd.store_field(self.rc, "state", VmStates.IN_USE)
mc_time.time.return_value = 1
# no vm terminated
self.vm_master.remove_old_dirty_vms()
assert not self.vmm.start_vm_termination.called
mc_time.time.return_value = self.opts.build_groups[0][
"vm_dirty_terminating_timeout"] + 1
# only "a1" and "b1" should be terminated
self.vm_master.remove_old_dirty_vms()
assert self.vmm.start_vm_termination.called
terminated_names = set([
call[0][1] for call in self.vmm.start_vm_termination.call_args_list
])
assert set(["a1", "b1"]) == terminated_names
def disabled_test_remove_vm_with_dead_builder(self, mc_time, add_vmd,
mc_psutil):
# todo: re-enable after psutil.Process.cmdline will be in use
mc_time.time.return_value = time.time()
self.vm_master.log = MagicMock()
self.vmm.start_vm_termination = MagicMock()
self.vmm.start_vm_termination.return_value = "OK"
for idx, vmd in enumerate(
[self.vmd_a1, self.vmd_a2, self.vmd_b1, self.vmd_b2, self.vmd_b3]):
vmd.store_field(self.rc, "state", VmStates.IN_USE)
vmd.store_field(self.rc, "chroot", "fedora-20-x86_64")
vmd.store_field(self.rc, "task_id",
"{}-fedora-20-x86_64".format(idx + 1))
vmd.store_field(self.rc, "build_id", idx + 1)
vmd.store_field(self.rc, "in_use_since", 0)
self.rc.hdel(self.vmd_b3.vm_key, "chroot")
for idx, vmd in enumerate(
[self.vmd_a1, self.vmd_a2, self.vmd_b2, self.vmd_b3]):
vmd.store_field(self.rc, "used_by_pid", idx + 1)
for vmd in [self.vmd_a3, self.vmd_a3]:
vmd.store_field(self.rc, "state", VmStates.READY)
def mc_psutil_process(pid):
p = MagicMock()
mapping = {
"1": "a1",
"2": "a2",
"3": "b1",
"4": "None",
"5": "None",
}
p.cmdline = [
"builder vm_name={} suffix".format(mapping.get(str(pid))),
]
return p
def mc_psutil_pid_exists(pid):
if str(pid) in ["1", "4"]:
return True
return False
mc_psutil.Process.side_effect = mc_psutil_process
mc_psutil.pid_exists.side_effect = mc_psutil_pid_exists
self.ps = self.vmm.rc.pubsub(ignore_subscribe_messages=True)
self.ps.subscribe(JOB_GRAB_TASK_END_PUBSUB)
self.vm_master.remove_vm_with_dead_builder()
msg_list = self.rcv_from_ps_message_bus()
assert self.vmm.start_vm_termination.call_args_list == [
mock.call('a2', allowed_pre_state='in_use'),
mock.call('b2', allowed_pre_state='in_use'),
mock.call('b3', allowed_pre_state='in_use')
]
# changed logic for the moment
# assert set(["2", "4"]) == set([json.loads(m["data"])["build_id"] for m in msg_list])
def test_check_vms_health(self, mc_time, add_vmd):
self.vm_master.start_vm_check = types.MethodType(MagicMock(), self.vmm)
for vmd in [
self.vmd_a1, self.vmd_a2, self.vmd_a3, self.vmd_b1,
self.vmd_b2, self.vmd_b3
]:
vmd.store_field(self.rc, "last_health_check", 0)
self.vmd_a1.store_field(self.rc, "state", VmStates.IN_USE)
self.vmd_a2.store_field(self.rc, "state", VmStates.CHECK_HEALTH)
self.vmd_a3.store_field(self.rc, "state", VmStates.CHECK_HEALTH_FAILED)
self.vmd_b1.store_field(self.rc, "state", VmStates.GOT_IP)
self.vmd_b2.store_field(self.rc, "state", VmStates.READY)
self.vmd_b3.store_field(self.rc, "state", VmStates.TERMINATING)
mc_time.time.return_value = 1
self.vm_master.check_vms_health()
assert not self.vm_master.start_vm_check.called
mc_time.time.return_value = 1 + self.opts.build_groups[0][
"vm_health_check_period"]
self.vm_master.check_vms_health()
to_check = set(
call[0][1]
for call in self.vm_master.start_vm_check.call_args_list)
assert set(['a1', 'a3', 'b1', 'b2']) == to_check
self.vm_master.start_vm_check.reset_mock()
for vmd in [
self.vmd_a1, self.vmd_a2, self.vmd_a3, self.vmd_b1,
self.vmd_b2, self.vmd_b3
]:
self.rc.hdel(vmd.vm_key, "last_health_check")
self.vm_master.check_vms_health()
to_check = set(
call[0][1]
for call in self.vm_master.start_vm_check.call_args_list)
assert set(['a1', 'a3', 'b1', 'b2']) == to_check
def test_finalize_long_health_checks(self, mc_time, add_vmd):
mc_time.time.return_value = 0
self.vmd_a1.store_field(self.rc, "state", VmStates.IN_USE)
self.vmd_a2.store_field(self.rc, "state", VmStates.CHECK_HEALTH)
self.vmd_a3.store_field(self.rc, "state", VmStates.CHECK_HEALTH)
self.vmd_a2.store_field(self.rc, "last_health_check", 0)
self.vmd_a3.store_field(
self.rc, "last_health_check",
self.opts.build_groups[0]["vm_health_check_max_time"] + 10)
mc_time.time.return_value = self.opts.build_groups[0][
"vm_health_check_max_time"] + 11
self.vmm.mark_vm_check_failed = MagicMock()
self.vm_master.finalize_long_health_checks()
assert self.vmm.mark_vm_check_failed.called_once
assert self.vmm.mark_vm_check_failed.call_args[0][0] == "a2"
def test_terminate_again(self, mc_time, add_vmd):
mc_time.time.return_value = 0
self.vmd_a1.store_field(self.rc, "state", VmStates.IN_USE)
self.vmd_a2.store_field(self.rc, "state", VmStates.CHECK_HEALTH)
self.vmd_a3.store_field(self.rc, "state", VmStates.READY)
mc_time.time.return_value = 1
self.vmm.remove_vm_from_pool = MagicMock()
self.vmm.start_vm_termination = MagicMock()
# case 1 no VM in terminating states =>
# no start_vm_termination, no remove_vm_from_pool
# import ipdb; ipdb.set_trace()
self.vm_master.terminate_again()
assert not self.vmm.remove_vm_from_pool.called
assert not self.vmm.start_vm_termination.called
# case 2: one VM in terminating state with unique ip, time_elapsed < threshold
# no start_vm_termination, no remove_vm_from_pool
self.vmd_a1.store_field(self.rc, "state", VmStates.TERMINATING)
self.vmd_a1.store_field(self.rc, "terminating_since", 0)
self.vm_master.terminate_again()
assert not self.vmm.remove_vm_from_pool.called
assert not self.vmm.start_vm_termination.called
# case 3: one VM in terminating state with unique ip, time_elapsed > threshold
# start_vm_termination called, no remove_vm_from_pool
mc_time.time.return_value = 1 + self.opts.build_groups[0][
"vm_terminating_timeout"]
self.vm_master.terminate_again()
assert not self.vmm.remove_vm_from_pool.called
assert self.vmm.start_vm_termination.called
assert self.vmm.start_vm_termination.call_args[0][
0] == self.vmd_a1.vm_name
self.vmm.start_vm_termination.reset_mock()
# case 4: two VM with the same IP, one in terminating states, , time_elapsed < threshold
# no start_vm_termination, no remove_vm_from_pool
mc_time.time.return_value = 1
self.vmd_a2.store_field(self.rc, "vm_ip", self.vmd_a1.vm_ip)
self.vm_master.terminate_again()
assert not self.vmm.remove_vm_from_pool.called
assert not self.vmm.start_vm_termination.called
# case 4: two VM with the same IP, one in terminating states, , time_elapsed > threshold
# no start_vm_termination, remove_vm_from_pool
mc_time.time.return_value = 1 + self.opts.build_groups[0][
"vm_terminating_timeout"]
self.vm_master.terminate_again()
assert self.vmm.remove_vm_from_pool.called
assert self.vmm.remove_vm_from_pool.call_args[0][
0] == self.vmd_a1.vm_name
assert not self.vmm.start_vm_termination.called
# def test_run_undefined_helpers(self, mc_setproctitle):
# for target in ["spawner", "terminator", "checker"]:
# setattr(self.vmm, target, None)
# with pytest.raises(RuntimeError):
# self.vm_master.run()
#
# setattr(self.vmm, target, MagicMock())
#
# assert not mc_setproctitle.called
def test_dummy_run(self, mc_time, mc_setproctitle):
mc_do_cycle = MagicMock()
mc_do_cycle.side_effect = [VmError("FooBar"), None]
self.vm_master.do_cycle = types.MethodType(mc_do_cycle, self.vm_master)
self.vmm.mark_server_start = MagicMock()
self.stage = 0
def on_sleep(*args, **kwargs):
self.stage += 1
if self.stage == 1:
pass
elif self.stage >= 2:
self.vm_master.kill_received = True
mc_time.sleep.side_effect = on_sleep
self.vm_master.run()
def test_dummy_terminate(self):
self.vm_master.terminate()
assert self.vm_master.kill_received
assert self.vm_master.checker.terminate.called
assert self.vm_master.spawner.terminate.called
def test_dummy_do_cycle(self):
self.vm_master.remove_old_dirty_vms = types.MethodType(
MagicMock(), self.vm_master)
self.vm_master.check_vms_health = types.MethodType(
MagicMock(), self.vm_master)
self.vm_master.start_spawn_if_required = types.MethodType(
MagicMock(), self.vm_master)
# self.vm_master.remove_old_dirty_vms = types(MagicMock, self.vm_master)
self.vm_master.do_cycle()
assert self.vm_master.remove_old_dirty_vms.called
assert self.vm_master.check_vms_health.called
assert self.vm_master.start_spawn_if_required.called
assert self.vm_master.spawner.recycle.called
def test_dummy_start_spawn_if_required(self):
self.vm_master.try_spawn_one = MagicMock()
self.vm_master.start_spawn_if_required()
assert self.vm_master.try_spawn_one.call_args_list == [
mock.call(group) for group in range(self.opts.build_groups_count)
]
def test__check_total_running_vm_limit_raises(self):
self.vm_master.log = MagicMock()
active_vm_states = [
VmStates.GOT_IP, VmStates.READY, VmStates.IN_USE,
VmStates.CHECK_HEALTH
]
cases = [
# active_vms_number , spawn_procs_number
(x, 11 - x) for x in range(12)
]
self.opts.build_groups[0]["max_vm_total"] = 11
for active_vms_number, spawn_procs_number in cases:
vmd_list = [
self.vmm.add_vm_to_pool("127.0.0.{}".format(idx + 1),
"a{}".format(idx), 0)
for idx in range(active_vms_number)
]
for idx in range(active_vms_number):
state = choice(active_vm_states)
vmd_list[idx].store_field(self.rc, "state", state)
self.vm_master.spawner.get_proc_num_per_group.return_value = spawn_procs_number
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_total_running_vm_limit(0)
self.vm_master.log.reset_mock()
# teardown
self.clean_redis()
def test__check_total_running_vm_limit_ok(self):
self.vm_master.log = MagicMock()
active_vm_states = [
VmStates.GOT_IP, VmStates.READY, VmStates.IN_USE,
VmStates.CHECK_HEALTH
]
cases = [
# active_vms_number , spawn_procs_number
(x, 11 - x) for x in range(12)
]
self.opts.build_groups[0]["max_vm_total"] = 12
for active_vms_number, spawn_procs_number in cases:
vmd_list = [
self.vmm.add_vm_to_pool("127.0.0.{}".format(idx + 1),
"a{}".format(idx), 0)
for idx in range(active_vms_number)
]
for idx in range(active_vms_number):
state = choice(active_vm_states)
vmd_list[idx].store_field(self.rc, "state", state)
# self.vmm.spawner.children_number = spawn_procs_number
self.vm_master.spawner.get_proc_num_per_group.return_value = spawn_procs_number
# doesn't raise exception
self.vm_master._check_total_running_vm_limit(0)
self.vm_master.log.reset_mock()
# teardown
self.clean_redis()
def test__check_elapsed_time_after_spawn(self, mc_time):
# don't start new spawn if last_spawn_time was in less self.vm_spawn_min_interval ago
mc_time.time.return_value = 0
self.vm_master._check_elapsed_time_after_spawn(0)
self.vm_master.vmm.write_vm_pool_info(0, "last_vm_spawn_start", 0)
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_elapsed_time_after_spawn(0)
mc_time.time.return_value = -1 + self.vm_spawn_min_interval
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_elapsed_time_after_spawn(0)
mc_time.time.return_value = 1 + self.vm_spawn_min_interval
self.vm_master._check_elapsed_time_after_spawn(0)
# we don't care about other group
self.vm_master.vmm.write_vm_pool_info(1, "last_vm_spawn_start",
mc_time.time.return_value)
self.vm_master._check_elapsed_time_after_spawn(0)
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_elapsed_time_after_spawn(1)
def test__check_number_of_running_spawn_processes(self):
for i in range(self.opts.build_groups[0]["max_spawn_processes"]):
self.vm_master.spawner.get_proc_num_per_group.return_value = i
self.vm_master._check_number_of_running_spawn_processes(0)
for i in [0, 1, 2, 5, 100]:
self.vm_master.spawner.get_proc_num_per_group.return_value = \
self.opts.build_groups[0]["max_spawn_processes"] + i
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_number_of_running_spawn_processes(0)
def test__check_total_vm_limit(self):
self.vm_master.vmm = MagicMock()
for i in range(2 * self.opts.build_groups[0]["max_vm_total"]):
self.vm_master.vmm.get_all_vm_in_group.return_value = [
1 for _ in range(i)
]
self.vm_master._check_total_vm_limit(0)
for i in range(2 * self.opts.build_groups[0]["max_vm_total"],
2 * self.opts.build_groups[0]["max_vm_total"] + 10):
self.vm_master.vmm.get_all_vm_in_group.return_value = [
1 for _ in range(i)
]
with pytest.raises(VmSpawnLimitReached):
self.vm_master._check_total_vm_limit(0)
def test_try_spawn_error_handling(self, mc_time):
mc_time.time.return_value = 0
self.vm_master.log = MagicMock()
self.vm_master.spawner.start_spawn.side_effect = IOError()
self.vm_master.try_spawn_one(0)
assert self.vm_master.spawner.start_spawn.called
def test_try_spawn_exit_on_check_fail(self):
check_mocks = []
for check_name in [
"_check_total_running_vm_limit",
"_check_elapsed_time_after_spawn",
"_check_number_of_running_spawn_processes",
"_check_total_vm_limit",
]:
mc_check_func = MagicMock()
check_mocks.append(mc_check_func)
setattr(self.vm_master, check_name, mc_check_func)
self.vm_master.vmm = MagicMock()
for idx in range(len(check_mocks)):
for cm in check_mocks:
cm.side_effect = None
check_mocks[idx].side_effect = VmSpawnLimitReached("test")
self.vm_master.try_spawn_one(0)
assert not self.vm_master.vmm.write_vm_pool_info.called
self.vm_master.vmm.write_vm_pool_info.reset_mock()
for cm in check_mocks:
cm.side_effect = None
self.vm_master.try_spawn_one(0)
assert self.vm_master.vmm.write_vm_pool_info.called
assert self.vm_master.spawner.start_spawn.called
def test_start_vm_check_ok_ok(self):
self.vmm.start_vm_termination = types.MethodType(MagicMock(), self.vmm)
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd = self.vmm.get_vm_by_name(self.vm_name)
# can start, no problem to start
# > can start IN_USE, don't change status
vmd.store_field(self.rc, "state", VmStates.IN_USE)
self.vm_master.start_vm_check(vm_name=self.vm_name)
assert self.checker.run_check_health.called
self.checker.run_check_health.reset_mock()
assert vmd.get_field(self.rc, "state") == VmStates.IN_USE
# > changes status to HEALTH_CHECK
states = [
VmStates.GOT_IP, VmStates.CHECK_HEALTH_FAILED, VmStates.READY
]
for state in states:
vmd.store_field(self.rc, "state", state)
self.vm_master.start_vm_check(vm_name=self.vm_name)
assert self.checker.run_check_health.called
self.checker.run_check_health.reset_mock()
assert vmd.get_field(self.rc, "state") == VmStates.CHECK_HEALTH
def test_start_vm_check_wrong_old_state(self):
self.vmm.start_vm_termination = types.MethodType(MagicMock(), self.vmm)
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd = self.vmm.get_vm_by_name(self.vm_name)
states = [VmStates.TERMINATING, VmStates.CHECK_HEALTH]
for state in states:
vmd.store_field(self.rc, "state", state)
assert not self.vm_master.start_vm_check(vm_name=self.vm_name)
assert not self.checker.run_check_health.called
assert vmd.get_field(self.rc, "state") == state
def test_start_vm_check_lua_ok_check_spawn_failed(self):
self.vmm.start_vm_termination = types.MethodType(MagicMock(), self.vmm)
self.vmm.add_vm_to_pool(self.vm_ip, self.vm_name, self.group)
vmd = self.vmm.get_vm_by_name(self.vm_name)
self.vm_master.checker.run_check_health.side_effect = RuntimeError()
# restore orig state
states = [
VmStates.GOT_IP, VmStates.CHECK_HEALTH_FAILED, VmStates.READY,
VmStates.IN_USE
]
for state in states:
vmd.store_field(self.rc, "state", state)
self.vm_master.start_vm_check(vm_name=self.vm_name)
assert self.checker.run_check_health.called
self.checker.run_check_health.reset_mock()
assert vmd.get_field(self.rc, "state") == state
def main():
opts = BackendConfigReader().read()
vmm = VmManager(opts, None)
print(vmm.info())