def test_get_slave_raises_exception_on_invalid_arguments(self, get_slave_kwargs):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave_registry.add_slave(slave1)
with self.assertRaises(ValueError):
slave_registry.get_slave(**get_slave_kwargs)
def post(self):
shutdown_all = self.decoded_body.get('shutdown_all')
if shutdown_all:
slaves_to_shutdown = SlaveRegistry.singleton().get_all_slaves_by_id().keys()
else:
slaves_to_shutdown = [int(slave_id) for slave_id in self.decoded_body.get('slaves')]
self._cluster_master.set_shutdown_mode_on_slaves(slaves_to_shutdown)
def put(self, slave_id):
new_slave_state = self.decoded_body.get('slave', {}).get('state')
slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
self._cluster_master.handle_slave_state_update(slave, new_slave_state)
self._cluster_master.update_slave_last_heartbeat_time(slave)
self._write_status({
'slave': slave.api_representation()
})
def test_get_slave_raises_exception_on_slave_not_found(self, get_slave_kwargs):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave2 = Slave('leonardo.turtles.gov', 1)
slave_registry.add_slave(slave1)
slave_registry.add_slave(slave2)
with self.assertRaises(ItemNotFoundError):
slave_registry.get_slave(**get_slave_kwargs)
def test_updating_slave_to_nonexistent_state_should_raise_bad_request_error(self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
slave_url = 'raphael.turtles.gov'
master.connect_slave(slave_url, 10)
slave = slave_registry.get_slave(slave_url=slave_url)
with self.assertRaises(BadRequestError):
master.handle_slave_state_update(slave, 'NONEXISTENT_STATE')
def test_connect_slave_adds_new_slave_if_slave_never_connected_before(self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
master.connect_slave('never-before-seen.turtles.gov', 10)
self.assertEqual(1, len(slave_registry.get_all_slaves_by_id()),
'Exactly one slave should be registered with the master.')
self.assertIsNotNone(slave_registry.get_slave(slave_id=None, slave_url='never-before-seen.turtles.gov'),
'Registered slave does not have the expected url.')
def test_remove_slave_raises_exception_on_invalid_arguments(self):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave_registry.add_slave(slave1)
with self.assertRaises(ValueError):
# Both arguments specified
slave_registry.remove_slave(slave=slave1, slave_url=slave1.url)
# No arguments specified
slave_registry.remove_slave(slave=None, slave_url=None)
def test_updating_slave_to_disconnected_state_should_reset_slave_current_build_id(self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
slave_url = 'raphael.turtles.gov'
master.connect_slave(slave_url, num_executors=10)
slave = slave_registry.get_slave(slave_url=slave_url)
slave.current_build_id = 4
master.handle_slave_state_update(slave, SlaveState.DISCONNECTED)
self.assertIsNone(slave.current_build_id)
def test_get_slave_returns_valid_slave(self):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave2 = Slave('leonardo.turtles.gov', 1)
slave_registry.add_slave(slave1)
slave_registry.add_slave(slave2)
self.assertEquals(slave_registry.get_slave(slave_url=slave1.url), slave1,
'Get slave with url should return valid slave.')
self.assertEquals(slave_registry.get_slave(slave_id=slave2.id), slave2,
'Get slave with id should return valid slave.')
def post(self):
shutdown_all = self.decoded_body.get('shutdown_all')
if shutdown_all:
slaves_to_shutdown = SlaveRegistry.singleton(
).get_all_slaves_by_id().keys()
else:
slaves_to_shutdown = [
int(slave_id) for slave_id in self.decoded_body.get('slaves')
]
self._cluster_master.set_shutdown_mode_on_slaves(slaves_to_shutdown)
def test_add_slave_adds_slave_in_both_dicts(self):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave2 = Slave('leonardo.turtles.gov', 1)
slave_registry.add_slave(slave1)
slave_registry.add_slave(slave2)
self.assertEqual(2, len(slave_registry.get_all_slaves_by_id()),
'Exactly two slaves should be in the all_slaves_by_id dict.')
self.assertEqual(2, len(slave_registry.get_all_slaves_by_url()),
'Exactly two slaves should be in the all_slaves_by_url dict.')
def test_updating_slave_to_disconnected_state_should_mark_slave_as_dead(self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
slave_url = 'raphael.turtles.gov'
master.connect_slave(slave_url, num_executors=10)
slave = slave_registry.get_slave(slave_url=slave_url)
self.assertTrue(slave.is_alive())
master.handle_slave_state_update(slave, SlaveState.DISCONNECTED)
self.assertFalse(slave.is_alive())
def test_updating_slave_to_shutdown_should_call_slave_set_shutdown_mode(self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
slave_url = 'raphael.turtles.gov'
master.connect_slave(slave_url, 10)
slave = slave_registry.get_slave(slave_url=slave_url)
slave.set_shutdown_mode = Mock()
master.handle_slave_state_update(slave, SlaveState.SHUTDOWN)
slave.set_shutdown_mode.assert_called_once_with()
def test_connect_slave_with_existing_dead_slave_removes_old_slave_entry_from_registry(self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
master.connect_slave('existing-slave.turtles.gov', 10)
old_existing_slave = slave_registry.get_slave(slave_id=None, slave_url='existing-slave.turtles.gov')
old_existing_slave_id = old_existing_slave.id
connect_response = master.connect_slave('existing-slave.turtles.gov', 10)
with self.assertRaises(ItemNotFoundError):
slave_registry.get_slave(slave_id=old_existing_slave_id)
def test_get_slave_returns_valid_slave(self):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave2 = Slave('leonardo.turtles.gov', 1)
slave_registry.add_slave(slave1)
slave_registry.add_slave(slave2)
self.assertEquals(slave_registry.get_slave(slave_url=slave1.url),
slave1,
'Get slave with url should return valid slave.')
self.assertEquals(slave_registry.get_slave(slave_id=slave2.id), slave2,
'Get slave with id should return valid slave.')
def test_updating_slave_to_shutdown_should_call_slave_set_shutdown_mode(
self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
slave_url = 'raphael.turtles.gov'
master.connect_slave(slave_url, 10)
slave = slave_registry.get_slave(slave_url=slave_url)
slave.set_shutdown_mode = Mock()
master.handle_slave_state_update(slave, SlaveState.SHUTDOWN)
slave.set_shutdown_mode.assert_called_once_with()
def test_updating_slave_to_disconnected_state_should_reset_slave_current_build_id(
self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
slave_url = 'raphael.turtles.gov'
master.connect_slave(slave_url, num_executors=10)
slave = slave_registry.get_slave(slave_url=slave_url)
slave.current_build_id = 4
master.handle_slave_state_update(slave, SlaveState.DISCONNECTED)
self.assertIsNone(slave.current_build_id)
def test_updating_slave_to_disconnected_state_should_mark_slave_as_dead(
self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
slave_url = 'raphael.turtles.gov'
master.connect_slave(slave_url, num_executors=10)
slave = slave_registry.get_slave(slave_url=slave_url)
self.assertTrue(slave.is_alive())
master.handle_slave_state_update(slave, SlaveState.DISCONNECTED)
self.assertFalse(slave.is_alive())
def test_add_slave_adds_slave_in_both_dicts(self):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave2 = Slave('leonardo.turtles.gov', 1)
slave_registry.add_slave(slave1)
slave_registry.add_slave(slave2)
self.assertEqual(
2, len(slave_registry.get_all_slaves_by_id()),
'Exactly two slaves should be in the all_slaves_by_id dict.')
self.assertEqual(
2, len(slave_registry.get_all_slaves_by_url()),
'Exactly two slaves should be in the all_slaves_by_url dict.')
def test_connect_slave_with_existing_slave_running_build_cancels_build(self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
master.connect_slave('running-slave.turtles.gov', 10)
build_mock = MagicMock(spec_set=Build)
BuildStore._all_builds_by_id[1] = build_mock
existing_slave = slave_registry.get_slave(slave_id=None, slave_url='running-slave.turtles.gov')
existing_slave.current_build_id = 1
master.connect_slave('running-slave.turtles.gov', 10)
self.assertTrue(build_mock.cancel.called, 'The build was not cancelled.')
def post(self, slave_id):
slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
# If the slave has been marked dead, but still sends heartbeat, the master does not update the last
# heartbeat time and the method returns false. Additionally, master responds to the slave with slave
# status. The slave will treat a is_alive=false response as a heartbeat failure, and die.
#
# The reason master returns the status to the slave instead of simply marking the slave as alive is
# because the master or slave do not maintain an explicit state about when and why the slave was marked
# dead. It is a lot cleaner for the heartbeat functionality to indicate an heartbeat failure and let the
# slave make a decision based on that.
is_alive = self._cluster_master.update_slave_last_heartbeat_time(slave)
self.write({'is_alive': is_alive})
def post(self, build_id, subjob_id):
slave_url = self.decoded_body.get('slave')
slave = SlaveRegistry.singleton().get_slave(slave_url=slave_url)
file_payload = self.request.files.get('file')
if not file_payload:
raise RuntimeError('Result file not provided')
slave_executor_id = self.decoded_body.get('metric_data', {}).get('executor_id')
analytics.record_event(analytics.MASTER_RECEIVED_RESULT, executor_id=slave_executor_id, build_id=int(build_id),
subjob_id=int(subjob_id), slave_id=slave.id)
self._cluster_master.handle_result_reported_from_slave(
slave_url, int(build_id), int(subjob_id), file_payload[0])
self._write_status()
def post(self, slave_id):
slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
# If the slave has been marked dead, but still sends heartbeat, the master does not update the last
# heartbeat time and the method returns false. Additionally, master responds to the slave with slave
# status. The slave will treat a is_alive=false response as a heartbeat failure, and die.
#
# The reason master returns the status to the slave instead of simply marking the slave as alive is
# because the master or slave do not maintain an explicit state about when and why the slave was marked
# dead. It is a lot cleaner for the heartbeat functionality to indicate an heartbeat failure and let the
# slave make a decision based on that.
is_alive = self._cluster_master.update_slave_last_heartbeat_time(slave)
self.write({'is_alive': is_alive})
def test_connect_slave_adds_new_slave_if_slave_never_connected_before(
self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
master.connect_slave('never-before-seen.turtles.gov', 10)
self.assertEqual(
1, len(slave_registry.get_all_slaves_by_id()),
'Exactly one slave should be registered with the master.')
self.assertIsNotNone(
slave_registry.get_slave(
slave_id=None, slave_url='never-before-seen.turtles.gov'),
'Registered slave does not have the expected url.')
def test_exception_raised_during_complete_subjob_does_not_prevent_slave_teardown(self):
slave_url = 'raphael.turtles.gov'
mock_build = Mock(spec_set=Build, build_id=lambda: 777, is_finished=False)
mock_build.complete_subjob.side_effect = [RuntimeError('Write failed')]
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
BuildStore._all_builds_by_id[mock_build.build_id()] = mock_build
slave_registry._all_slaves_by_url[slave_url] = Mock()
mock_scheduler = self.mock_scheduler_pool.get(mock_build)
with self.assertRaisesRegex(RuntimeError, 'Write failed'):
master.handle_result_reported_from_slave(slave_url, mock_build.build_id(), subjob_id=888)
self.assertEqual(mock_scheduler.execute_next_subjob_or_free_executor.call_count, 1)
def test_remove_slave_by_slave_url_removes_slave_from_both_dicts(self):
slave_registry = SlaveRegistry.singleton()
slave1 = Slave('raphael.turtles.gov', 1)
slave2 = Slave('leonardo.turtles.gov', 1)
slave_registry.add_slave(slave1)
slave_registry.add_slave(slave2)
self.assertEqual(2, len(slave_registry.get_all_slaves_by_id()),
'Exactly two slaves should be in the all_slaves_by_id dict.')
self.assertEqual(2, len(slave_registry.get_all_slaves_by_url()),
'Exactly two slaves should be in the all_slaves_by_url dict.')
slave_registry.remove_slave(slave_url=slave1.url)
self.assertEqual(1, len(slave_registry.get_all_slaves_by_id()),
'Exactly one slave should be in the all_slaves_by_id dict after removing one slave.')
self.assertEqual(1, len(slave_registry.get_all_slaves_by_url()),
'Exactly one slave should be in the all_slaves_by_url dict after removing one slave.')
def test_connect_slave_with_existing_dead_slave_creates_new_alive_instance(self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
master.connect_slave('existing-slave.turtles.gov', 10)
existing_slave = slave_registry.get_slave(slave_id=None, slave_url='existing-slave.turtles.gov')
existing_slave.set_is_alive(False)
existing_slave_id = existing_slave.id
connect_response = master.connect_slave('existing-slave.turtles.gov', 10)
new_slave = slave_registry.get_slave(slave_url='existing-slave.turtles.gov')
self.assertNotEqual(str(existing_slave_id), connect_response['slave_id'],
'The re-connected slave should have generated a new slave id.')
self.assertTrue(new_slave.is_alive(use_cached=True),
'The new slave should have been marked as alive once instantiated.')
self.assertEquals(2, self.mock_slave_allocator.add_idle_slave.call_count,
'Expected slave to be added to the idle slaves list.')
def post(self, build_id, subjob_id):
slave_url = self.decoded_body.get('slave')
slave = SlaveRegistry.singleton().get_slave(slave_url=slave_url)
file_payload = self.request.files.get('file')
if not file_payload:
raise RuntimeError('Result file not provided')
slave_executor_id = self.decoded_body.get('metric_data',
{}).get('executor_id')
analytics.record_event(analytics.MASTER_RECEIVED_RESULT,
executor_id=slave_executor_id,
build_id=int(build_id),
subjob_id=int(subjob_id),
slave_id=slave.id)
self._cluster_master.handle_result_reported_from_slave(
slave_url, int(build_id), int(subjob_id), file_payload[0])
self._write_status()
def test_updating_slave_to_setup_completed_state_should_tell_build_to_begin_subjob_execution(self):
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
fake_build = MagicMock(spec_set=Build)
master.get_build = MagicMock(return_value=fake_build)
slave_url = 'raphael.turtles.gov'
master.connect_slave(slave_url, 10)
slave = slave_registry.get_slave(slave_url=slave_url)
mock_scheduler = self.mock_scheduler_pool.get(fake_build)
scheduler_begin_event = Event()
mock_scheduler.begin_subjob_executions_on_slave.side_effect = lambda **_: scheduler_begin_event.set()
master.handle_slave_state_update(slave, SlaveState.SETUP_COMPLETED)
was_called = scheduler_begin_event.wait(timeout=5)
self.assertTrue(was_called, 'scheduler.begin_subjob_executions_on_slave should be called in response '
'to slave setup completing.')
_, call_kwargs = mock_scheduler.begin_subjob_executions_on_slave.call_args
self.assertEqual(call_kwargs.get('slave'), slave)
def test_handle_result_reported_from_slave_when_build_is_canceled(self):
build_id = 1
slave_url = "url"
build = Build(BuildRequest({}))
self.patch('app.master.build.util')
build.generate_project_type()
build.cancel()
self.patch_object(build, '_handle_subjob_payload')
self.patch_object(build, '_mark_subjob_complete')
master = ClusterMaster()
slave_registry = SlaveRegistry.singleton()
BuildStore._all_builds_by_id[build_id] = build
slave_registry._all_slaves_by_url[slave_url] = Mock()
mock_scheduler = self.mock_scheduler_pool.get(build)
master.handle_result_reported_from_slave(slave_url, build_id, 1)
self.assertEqual(build._handle_subjob_payload.call_count, 1, "Canceled builds should "
"handle payload")
self.assertEqual(build._mark_subjob_complete.call_count, 1, "Canceled builds should mark "
"their subjobs complete")
self.assertTrue(mock_scheduler.execute_next_subjob_or_free_executor.called)
def get(self, slave_id):
slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
response = {'slave': slave.api_representation()}
self.write(response)
def post(self, slave_id):
slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
self._cluster_master.update_slave_last_heartbeat_time(slave)
def get(self):
response = {
'slaves': [slave.api_representation() for slave in SlaveRegistry.singleton().get_all_slaves_by_id().values()]
}
self.write(response)
def get(self, slave_id):
slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
response = {
'slave': slave.api_representation()
}
self.write(response)