def test_remove_slave_by_slave_instance_removes_slave_from_both_dicts(
self):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave2 = Slave('leonardo.turtles.gov', 1)
slave_registry.add_slave(slave1)
slave_registry.add_slave(slave2)
self.assertEqual(
2, len(slave_registry.get_all_slaves_by_id()),
'Exactly two slaves should be in the all_slaves_by_id dict.')
self.assertEqual(
2, len(slave_registry.get_all_slaves_by_url()),
'Exactly two slaves should be in the all_slaves_by_url dict.')
slave_registry.remove_slave(slave=slave1)
self.assertEqual(
1, len(slave_registry.get_all_slaves_by_id()),
'Exactly one slave should be in the all_slaves_by_id dict after removing one slave.'
)
self.assertEqual(
1, len(slave_registry.get_all_slaves_by_url()),
'Exactly one slave should be in the all_slaves_by_url dict after removing one slave.'
)
def test_teardown_called_on_slave_when_no_subjobs_remain(self):
build = Build(BuildRequest({}))
slave = Slave('', 1)
slave.teardown = MagicMock()
slave.free_executor = MagicMock(return_value=0)
build._unstarted_subjobs = Queue()
build.execute_next_subjob_on_slave(slave)
slave.teardown.assert_called_with()
def test_get_slave_raises_exception_on_slave_not_found(self, get_slave_kwargs):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave2 = Slave('leonardo.turtles.gov', 1)
slave_registry.add_slave(slave1)
slave_registry.add_slave(slave2)
with self.assertRaises(ItemNotFoundError):
slave_registry.get_slave(**get_slave_kwargs)
def test_add_idle_slave_should_not_add_slave_to_queue_if_slave_is_shutdown(self):
mock_slave = Slave('', 10)
mock_slave.kill = Mock(return_value=None)
mock_slave.set_shutdown_mode()
slave_allocator = self._create_slave_allocator()
slave_allocator._idle_slaves.put = Mock()
slave_allocator.add_idle_slave(mock_slave)
self.assertFalse(slave_allocator._idle_slaves.put.called)
def test_teardown_called_on_slave_when_no_subjobs_remain(self):
build = Build(BuildRequest({}))
slave = Slave('', 1)
slave.teardown = MagicMock()
slave.free_executor = MagicMock(return_value=0)
build._unstarted_subjobs = Queue()
build._slaves_allocated = [slave]
build.execute_next_subjob_on_slave(slave)
slave.teardown.assert_called_with()
def test_get_slave_returns_valid_slave(self):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave2 = Slave('leonardo.turtles.gov', 1)
slave_registry.add_slave(slave1)
slave_registry.add_slave(slave2)
self.assertEquals(slave_registry.get_slave(slave_url=slave1.url), slave1,
'Get slave with url should return valid slave.')
self.assertEquals(slave_registry.get_slave(slave_id=slave2.id), slave2,
'Get slave with id should return valid slave.')
def test_add_slave_adds_slave_in_both_dicts(self):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave2 = Slave('leonardo.turtles.gov', 1)
slave_registry.add_slave(slave1)
slave_registry.add_slave(slave2)
self.assertEqual(2, len(slave_registry.get_all_slaves_by_id()),
'Exactly two slaves should be in the all_slaves_by_id dict.')
self.assertEqual(2, len(slave_registry.get_all_slaves_by_url()),
'Exactly two slaves should be in the all_slaves_by_url dict.')
def test_add_idle_slave_should_not_add_slave_to_queue_if_slave_is_shutdown(
self):
mock_slave = Slave('', 10)
mock_slave.kill = Mock(return_value=None)
mock_slave.set_shutdown_mode()
slave_allocator = self._create_slave_allocator()
slave_allocator._idle_slaves.put = Mock()
slave_allocator.add_idle_slave(mock_slave)
self.assertFalse(slave_allocator._idle_slaves.put.called)
def test_teardown_called_on_slave_when_slave_in_shutdown_mode(self):
build = Build(BuildRequest({}))
slave = Slave('', 1)
slave.teardown = MagicMock()
slave._is_in_shutdown_mode = True
slave.free_executor = MagicMock(return_value=0)
build._unstarted_subjobs = Queue()
build._unstarted_subjobs.put(Mock(spec=Subjob))
build._slaves_allocated = [slave]
build.execute_next_subjob_or_teardown_slave(slave)
slave.teardown.assert_called_with()
def test_teardown_called_on_slave_when_slave_in_shutdown_mode(self):
build = Build(BuildRequest({}))
slave = Slave('', 1)
slave.teardown = MagicMock()
slave._is_in_shutdown_mode = True
slave.free_executor = MagicMock(return_value=0)
build._unstarted_subjobs = Queue()
build._unstarted_subjobs.put(Mock(spec=Subjob))
build._slaves_allocated = [slave]
build.execute_next_subjob_or_teardown_slave(slave)
slave.teardown.assert_called_with()
def test_get_slave_raises_exception_on_invalid_arguments(self, get_slave_kwargs):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave_registry.add_slave(slave1)
with self.assertRaises(ValueError):
slave_registry.get_slave(**get_slave_kwargs)
def _create_slave(self, **kwargs) -> Slave:
"""
Create a slave for testing.
:param kwargs: Any constructor parameters for the slave; if none are specified, test defaults will be used.
"""
kwargs.setdefault('slave_url', self._FAKE_SLAVE_URL)
kwargs.setdefault('num_executors', self._FAKE_NUM_EXECUTORS)
return Slave(**kwargs)
def test_remove_slave_raises_exception_on_invalid_arguments(self):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave_registry.add_slave(slave1)
with self.assertRaises(ValueError):
# Both arguments specified
slave_registry.remove_slave(slave=slave1, slave_url=slave1.url)
# No arguments specified
slave_registry.remove_slave(slave=None, slave_url=None)
def test_updating_slave_to_idle_state_does_not_mark_build_finished_when_slaves_not_done(
self):
master = ClusterMaster()
slave1 = Slave('', 1)
slave2 = Slave('', 1)
slave3 = Slave('', 1)
slave1.current_build_id = 1
slave2.current_build_id = None
slave3.current_build_id = 1
build1 = Build(BuildRequest({}))
master._all_slaves_by_url = {'1': slave1, '2': slave2, '3': slave3}
master._all_builds_by_id = {1: build1}
build1._build_id = 1
build1.finish = MagicMock()
master.handle_slave_state_update(slave1, SlaveState.IDLE)
self.assertFalse(build1.finish.called)
def _create_mock_slave(self, num_executors=5):
"""
:type num_executors: int
:rtype: Slave | MagicMock
"""
slave_spec = Slave('', 0) # constructor values don't matter since this is just a spec object
mock_slave = MagicMock(spec_set=slave_spec, url=self._FAKE_SLAVE_URL, num_executors=num_executors)
counter = Counter()
mock_slave.claim_executor.side_effect = counter.increment
mock_slave.free_executor.side_effect = counter.decrement
return mock_slave
def connect_slave(self, slave_url, num_executors, slave_session_id=None):
"""
Connect a slave to this master.
:type slave_url: str
:type num_executors: int
:type slave_session_id: str | None
:return: The response with the slave id of the slave.
:rtype: dict[str, str]
"""
# todo: Validate arg types for this and other methods called via API.
# If a slave had previously been connected, and is now being reconnected, the cleanest way to resolve this
# bookkeeping is for the master to forget about the previous slave instance and start with a fresh instance.
try:
old_slave = self._slave_registry.get_slave(slave_url=slave_url)
except ItemNotFoundError:
pass
else:
self._logger.warning(
'Slave requested to connect to master, even though previously connected as {}. '
+
'Removing existing slave instance from the master\'s bookkeeping.',
old_slave)
# If a slave has requested to reconnect, we have to assume that whatever build the dead slave was
# working on no longer has valid results.
if old_slave.current_build_id is not None:
self._logger.info(
'{} has build [{}] running on it. Attempting to cancel build.',
old_slave, old_slave.current_build_id)
try:
build = self.get_build(old_slave.current_build_id)
build.cancel()
self._logger.info(
'Cancelled build {} due to dead slave {}',
old_slave.current_build_id, old_slave)
except ItemNotFoundError:
self._logger.info(
'Failed to find build {} that was running on {}',
old_slave.current_build_id, old_slave)
# Remove old slave from registry
self._slave_registry.remove_slave(slave=old_slave)
slave = Slave(slave_url, num_executors, slave_session_id)
self._slave_registry.add_slave(slave)
self._slave_allocator.add_idle_slave(slave)
self._logger.info(
'Slave on {} connected to master with {} executors. (id: {})',
slave_url, num_executors, slave.id)
return {'slave_id': str(slave.id)}
def connect_new_slave(self, slave_url, num_executors):
"""
Add a new slave to this master.
:type slave_url: str
:type num_executors: int
:return: The slave id of the new slave
:rtype: int
"""
slave = Slave(slave_url, num_executors)
self._all_slaves_by_url[slave_url] = slave
self._add_idle_slave(slave)
self._logger.info('Slave on {} connected to master with {} executors. (id: {})',
slave_url, num_executors, slave.id)
return {'slave_id': str(slave.id)}
def allocate_slave(self, slave: Slave) -> bool:
"""
Allocate a slave to this build. This tells the slave to execute setup commands for this build.
:param slave: The slave to allocate
:return: Whether slave allocation was successful; this can fail if the slave is unresponsive
"""
if not self._build_started:
self._build_started = True
self._build.mark_started()
# Increment executors before triggering setup. This helps make sure the build won't take down
# every slave in the cluster if setup calls fail because of a problem with the build.
next_executor_index = self._num_executors_allocated
self._num_executors_allocated += min(slave.num_executors, self._max_executors_per_slave)
analytics.record_event(analytics.BUILD_SETUP_START, build_id=self._build.build_id(), slave_id=slave.id)
self._slaves_allocated.append(slave)
return slave.setup(self._build, executor_start_index=next_executor_index)
def test_add_idle_slave_does_not_mark_build_finished_when_slaves_not_done(self):
master = ClusterMaster()
slave1 = Slave('', 1)
slave2 = Slave('', 1)
slave3 = Slave('', 1)
slave1.current_build_id = 1
slave2.current_build_id = None
slave3.current_build_id = 1
build1 = Build(BuildRequest({}))
master._all_slaves_by_url = {'1': slave1, '2': slave2, '3': slave3}
master._all_builds_by_id = {1: build1}
build1._build_id = 1
build1.finish = MagicMock()
master.add_idle_slave(slave1)
self.assertFalse(build1.finish.called)
def _create_mock_slave(self, num_executors=5):
mock = MagicMock(spec_set=Slave(slave_url=self._FAKE_SLAVE_URL,
num_executors=num_executors))
mock.url = self._FAKE_SLAVE_URL
mock.num_executors = num_executors
return mock
