def test_build_doesnt_use_more_than_max_executors(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [ self._create_mock_slave(num_executors=5) for _ in range(3) ] expected_num_executors = 12 # We expect the build to use 12 out of 15 available executors. build = Build(BuildRequest({'setup': fake_setup_command})) build._project_type = mock_project_type build.execute_next_subjob_or_teardown_slave = MagicMock() build.prepare( subjobs, self._create_job_config(max_executors=expected_num_executors)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] [ build.begin_subjob_executions_on_slave(mock_slave) for mock_slave in mock_slaves ] self.assertEqual( build.execute_next_subjob_or_teardown_slave.call_count, expected_num_executors, 'Build should start executing as many subjobs as its max_executors setting.' )
def test_build_status_returns_finished_after_all_subjobs_complete_and_slaves_finished( self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=3) postbuild_tasks_complete_event = Event() build = Build(BuildRequest({})) build._project_type = mock_project_type build._create_build_artifact = MagicMock() self._on_async_postbuild_tasks_completed( build, postbuild_tasks_complete_event.set) build.prepare(subjobs, self._create_job_config()) build.allocate_slave( mock_slave) # all three subjobs are now "in progress" for subjob in subjobs: build.complete_subjob(subjob.subjob_id()) # Wait for the async thread to complete executing postbuild tasks. self.assertTrue( postbuild_tasks_complete_event.wait(timeout=2), 'Postbuild tasks should complete within a few' 'seconds.') # Verify build artifacts was called after subjobs completed build._create_build_artifact.assert_called_once_with() self.assertTrue(build._subjobs_are_finished) self.assertEqual(build._status(), BuildStatus.FINISHED)
def test_build_doesnt_use_more_than_max_executors_per_slave(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [self._create_mock_slave(num_executors=5) for _ in range(3)] max_executors_per_slave = 2 expected_total_num_executors_used = 6 # We expect the build to use 2 executors on each of the 3 slaves. build = Build(BuildRequest({'setup': fake_setup_command})) build._project_type = mock_project_type build.execute_next_subjob_or_teardown_slave = MagicMock() build.prepare(subjobs, self._create_job_config(max_executors_per_slave=max_executors_per_slave)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] expected_current_num_executors_used = 0 for i in range(len(mock_slaves)): build.begin_subjob_executions_on_slave(mock_slaves[i]) expected_current_num_executors_used += max_executors_per_slave self.assertEqual( build.execute_next_subjob_or_teardown_slave.call_count, expected_current_num_executors_used, 'After allocating {} slaves, build with max_executors_per_slave set to {} should only be using {} ' 'executors.'.format(i + 1, max_executors_per_slave, expected_current_num_executors_used)) self.assertEqual( build.execute_next_subjob_or_teardown_slave.call_count, expected_total_num_executors_used, 'Build should start executing as many subjobs per slave as its max_executors_per_slave setting.')
def test_build_cannot_be_prepared_more_than_once(self): build = Build(BuildRequest({})) subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) with self.assertRaises(Exception): build.prepare(subjobs, mock_project_type, self._create_job_config())
def test_need_more_slaves_returns_true_if_max_processes_is_not_reached(self): subjobs = self._create_subjobs(count=8) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=5) build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config(max_executors=8)) build.allocate_slave(mock_slave) self.assertTrue(build.needs_more_slaves(), "if max_processes is not reached, we should need more slaves")
def test_exception_is_raised_if_problem_occurs_writing_subjob(self): Configuration['results_directory'] = abspath(join('some', 'temp', 'directory')) build = Build(BuildRequest({})) build._project_type = self._create_mock_project_type() subjob = self._create_subjobs(count=1, build_id=build.build_id())[0] build.prepare([subjob], self._create_job_config()) self.mock_fs.write_file.side_effect = FileExistsError with self.assertRaises(Exception): payload = {'filename': 'turtles.txt', 'body': 'Heroes in a half shell.'} build.complete_subjob(subjob.subjob_id(), payload=payload)
def test_allocate_slave_calls_slave_setup(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave() build = Build(Mock(spec_set=BuildRequest)) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) mock_slave.setup.assert_called_once_with(build)
def test_build_status_returns_queued_after_build_preparation(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) status = build._status() self.assertEqual(status, BuildStatus.QUEUED, 'Build status should be QUEUED after build has been prepared.')
def test_complete_subjob_writes_and_extracts_payload_to_correct_directory(self): Configuration['results_directory'] = '/tmp/results' build = Build(BuildRequest({})) build._project_type = self._create_mock_project_type() subjob = self._create_subjobs(count=1, build_id=build.build_id())[0] build.prepare([subjob], self._create_job_config()) payload = {'filename': 'turtles.txt', 'body': 'Heroes in a half shell.'} build.complete_subjob(subjob.subjob_id(), payload=payload) self.mock_fs.write_file.assert_called_once_with('Heroes in a half shell.', '/tmp/results/1/turtles.txt') self.mock_fs.extract_tar.assert_called_once_with('/tmp/results/1/turtles.txt', delete=True)
def test_need_more_slaves_returns_false_if_max_processes_is_reached(self): subjobs = self._create_subjobs(count=5) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=1) build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config(max_executors=1)) build.allocate_slave(mock_slave) self.assertFalse( build.needs_more_slaves(), "if max processes is reached, we shouldn't need more slaves")
def test_build_status_returns_queued_after_build_preparation(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) status = build._status() self.assertEqual( status, BuildStatus.QUEUED, 'Build status should be QUEUED after build has been prepared.')
def test_build_status_returns_building_after_setup_has_started(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave() build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) self.assertEqual(build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after setup has started on slaves.')
def test_build_status_returns_building_after_setup_is_complete_and_subjobs_are_executing(self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=2) build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) build.begin_subjob_executions_on_slave(mock_slave) # two out of three subjobs are now in progress self.assertEqual(build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after subjobs have started executing on slaves.')
def test_build_status_returns_building_after_setup_has_started(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave() build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) self.assertEqual( build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after setup has started on slaves.' )
def test_exception_is_raised_if_problem_occurs_writing_subjob(self): Configuration['results_directory'] = '/tmp/results' build = Build(BuildRequest({})) build._project_type = self._create_mock_project_type() subjob = self._create_subjobs(count=1, build_id=build.build_id())[0] build.prepare([subjob], self._create_job_config()) self.mock_fs.write_file.side_effect = FileExistsError with self.assertRaises(Exception): payload = { 'filename': 'turtles.txt', 'body': 'Heroes in a half shell.' } build.complete_subjob(subjob.subjob_id(), payload=payload)
def test_build_status_returns_building_after_setup_is_complete_and_subjobs_are_executing( self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=2) build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) build.begin_subjob_executions_on_slave( mock_slave) # two out of three subjobs are now in progress self.assertEqual( build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after subjobs have started executing on slaves.' )
def test_build_doesnt_use_more_than_max_executors(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [self._create_mock_slave(num_executors=5) for _ in range(3)] expected_num_executors = 12 # We expect the build to use 12 out of 15 available executors. build = Build(BuildRequest({'setup': fake_setup_command})) build._project_type = mock_project_type build.execute_next_subjob_or_teardown_slave = MagicMock() build.prepare(subjobs, self._create_job_config(max_executors=expected_num_executors)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] [build.begin_subjob_executions_on_slave(mock_slave) for mock_slave in mock_slaves] self.assertEqual(build.execute_next_subjob_or_teardown_slave.call_count, expected_num_executors, 'Build should start executing as many subjobs as its max_executors setting.')
def test_complete_subjob_writes_and_extracts_payload_to_correct_directory( self): Configuration['results_directory'] = '/tmp/results' build = Build(BuildRequest({})) build._project_type = self._create_mock_project_type() subjob = self._create_subjobs(count=1, build_id=build.build_id())[0] build.prepare([subjob], self._create_job_config()) payload = { 'filename': 'turtles.txt', 'body': 'Heroes in a half shell.' } build.complete_subjob(subjob.subjob_id(), payload=payload) self.mock_fs.write_file.assert_called_once_with( 'Heroes in a half shell.', '/tmp/results/1/turtles.txt') self.mock_fs.extract_tar.assert_called_once_with( '/tmp/results/1/turtles.txt', delete=True)
def test_build_status_returns_finished_after_all_subjobs_complete_and_slaves_finished(self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=3) postbuild_tasks_complete_event = Event() build = Build(BuildRequest({})) build._project_type = mock_project_type build._create_build_artifact = MagicMock() self._on_async_postbuild_tasks_completed(build, postbuild_tasks_complete_event.set) build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) # all three subjobs are now "in progress" for subjob in subjobs: build.complete_subjob(subjob.subjob_id()) # Wait for the async thread to complete executing postbuild tasks. self.assertTrue(postbuild_tasks_complete_event.wait(timeout=2), 'Postbuild tasks should complete within a few' 'seconds.') # Verify build artifacts was called after subjobs completed build._create_build_artifact.assert_called_once_with() self.assertTrue(build._subjobs_are_finished) self.assertEqual(build._status(), BuildStatus.FINISHED)
def test_build_doesnt_use_more_than_max_executors_per_slave(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [ self._create_mock_slave(num_executors=5) for _ in range(3) ] max_executors_per_slave = 2 expected_total_num_executors_used = 6 # We expect the build to use 2 executors on each of the 3 slaves. build = Build(BuildRequest({'setup': fake_setup_command})) build._project_type = mock_project_type build.execute_next_subjob_or_teardown_slave = MagicMock() build.prepare( subjobs, self._create_job_config( max_executors_per_slave=max_executors_per_slave)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] expected_current_num_executors_used = 0 for i in range(len(mock_slaves)): build.begin_subjob_executions_on_slave(mock_slaves[i]) expected_current_num_executors_used += max_executors_per_slave self.assertEqual( build.execute_next_subjob_or_teardown_slave.call_count, expected_current_num_executors_used, 'After allocating {} slaves, build with max_executors_per_slave set to {} should only be using {} ' 'executors.'.format(i + 1, max_executors_per_slave, expected_current_num_executors_used)) self.assertEqual( build.execute_next_subjob_or_teardown_slave.call_count, expected_total_num_executors_used, 'Build should start executing as many subjobs per slave as its max_executors_per_slave setting.' )