def test_build_status_returns_finished_after_all_subjobs_complete_and_slaves_finished( self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=3) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config()) build.allocate_slave( mock_slave) # all three subjobs are now "in progress" # Mock out call to create build artifacts after subjobs complete build._create_build_artifact = MagicMock() for subjob in subjobs: build.mark_subjob_complete(subjob.subjob_id()) # Note: this was never a unit test! We have to wait for a thread to complete post build # actions here. TODO: Fix this poll.wait_for(lambda: build._postbuild_tasks_are_finished, 5) # Verify build artifacts was called after subjobs completed build._create_build_artifact.assert_called_once_with() build.finish() status = build._status() self.assertTrue(build._subjobs_are_finished) self.assertTrue(build._postbuild_tasks_are_finished) self.assertTrue(build._teardowns_finished) self.assertEqual(status, BuildStatus.FINISHED)
def test_build_status_returns_finished_after_all_subjobs_complete_and_slaves_finished(self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=3) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config(self._FAKE_MAX_EXECUTORS)) build.allocate_slave(mock_slave) # all three subjobs are now "in progress" # Mock out call to create build artifacts after subjobs complete build._create_build_artifact = MagicMock() for subjob in subjobs: build.mark_subjob_complete(subjob.subjob_id()) # Note: this was never a unit test! We have to wait for a thread to complete post build # actions here. TODO: Fix this poll.wait_for(lambda: build._postbuild_tasks_are_finished, 5) # Verify build artifacts was called after subjobs completed build._create_build_artifact.assert_called_once_with() build.finish() status = build._status() self.assertTrue(build._subjobs_are_finished) self.assertTrue(build._postbuild_tasks_are_finished) self.assertTrue(build._teardowns_finished) self.assertEqual(status, BuildStatus.FINISHED)
def test_build_doesnt_use_more_than_max_executors_per_slave(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [self._create_mock_slave(num_executors=5) for _ in range(3)] max_executors_per_slave = 2 expected_total_num_executors_used = 6 # We expect the build to use 2 executors on each of the 3 slaves. build = Build(BuildRequest({'setup': fake_setup_command})) build._project_type = mock_project_type build.execute_next_subjob_or_teardown_slave = MagicMock() build.prepare(subjobs, self._create_job_config(max_executors_per_slave=max_executors_per_slave)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] expected_current_num_executors_used = 0 for i in range(len(mock_slaves)): build.begin_subjob_executions_on_slave(mock_slaves[i]) expected_current_num_executors_used += max_executors_per_slave self.assertEqual( build.execute_next_subjob_or_teardown_slave.call_count, expected_current_num_executors_used, 'After allocating {} slaves, build with max_executors_per_slave set to {} should only be using {} ' 'executors.'.format(i + 1, max_executors_per_slave, expected_current_num_executors_used)) self.assertEqual( build.execute_next_subjob_or_teardown_slave.call_count, expected_total_num_executors_used, 'Build should start executing as many subjobs per slave as its max_executors_per_slave setting.')
def test_build_doesnt_use_more_than_max_executors(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [ self._create_mock_slave(num_executors=5) for _ in range(3) ] expected_num_executors = 12 # We expect the build to use 12 out of 15 available executors. build = Build(BuildRequest({'setup': fake_setup_command})) build.execute_next_subjob_on_slave = MagicMock() build.prepare( subjobs, mock_project_type, self._create_job_config(max_executors=expected_num_executors)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] [ build.begin_subjob_executions_on_slave(mock_slave) for mock_slave in mock_slaves ] self.assertEqual( build.execute_next_subjob_on_slave.call_count, expected_num_executors, 'Build should start executing as many subjobs as its max_executors setting.' )
def test_build_status_returns_finished_after_all_subjobs_complete_and_slaves_finished( self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=3) postbuild_tasks_complete_event = Event() build = Build(BuildRequest({})) build._project_type = mock_project_type build._create_build_artifact = MagicMock() self._on_async_postbuild_tasks_completed( build, postbuild_tasks_complete_event.set) build.prepare(subjobs, self._create_job_config()) build.allocate_slave( mock_slave) # all three subjobs are now "in progress" for subjob in subjobs: build.complete_subjob(subjob.subjob_id()) # Wait for the async thread to complete executing postbuild tasks. self.assertTrue( postbuild_tasks_complete_event.wait(timeout=2), 'Postbuild tasks should complete within a few' 'seconds.') # Verify build artifacts was called after subjobs completed build._create_build_artifact.assert_called_once_with() self.assertTrue(build._subjobs_are_finished) self.assertEqual(build._status(), BuildStatus.FINISHED)
def test_build_doesnt_use_more_than_max_executors_per_slave(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [ self._create_mock_slave(num_executors=5) for _ in range(3) ] max_executors_per_slave = 2 expected_total_num_executors_used = 6 # We expect the build to use 2 executors on each of the 3 slaves. build = Build(BuildRequest({'setup': fake_setup_command})) build.execute_next_subjob_on_slave = MagicMock() build.prepare( subjobs, mock_project_type, self._create_job_config( max_executors_per_slave=max_executors_per_slave)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] expected_current_num_executors_used = 0 for i in range(len(mock_slaves)): build.begin_subjob_executions_on_slave(mock_slaves[i]) expected_current_num_executors_used += max_executors_per_slave self.assertEqual( build.execute_next_subjob_on_slave.call_count, expected_current_num_executors_used, 'After allocating {} slaves, build with max_executors_per_slave set to {} should only be using {} ' 'executors.'.format(i + 1, max_executors_per_slave, expected_current_num_executors_used)) self.assertEqual( build.execute_next_subjob_on_slave.call_count, expected_total_num_executors_used, 'Build should start executing as many subjobs per slave as its max_executors_per_slave setting.' )
def test_need_more_slaves_returns_true_if_max_processes_is_not_reached(self): subjobs = self._create_subjobs(count=8) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=5) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config(max_executors=8)) build.allocate_slave(mock_slave) self.assertTrue(build.needs_more_slaves(), "if max_processes is not reached, we should need more slaves")
def test_build_cannot_be_prepared_more_than_once(self): build = Build(BuildRequest({})) subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() build.prepare(subjobs, mock_project_type, self._create_job_config(max_executors=self._FAKE_MAX_EXECUTORS)) with self.assertRaises(Exception): build.prepare(subjobs, mock_project_type, self._create_job_config(max_executors=self._FAKE_MAX_EXECUTORS))
def test_build_status_returns_queued_after_build_preparation(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config(self._FAKE_MAX_EXECUTORS)) status = build._status() self.assertEqual(status, BuildStatus.QUEUED)
def test_need_more_slaves_returns_false_if_max_processes_is_reached(self): subjobs = self._create_subjobs(count=5) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=1) build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config(max_executors=1)) build.allocate_slave(mock_slave) self.assertFalse(build.needs_more_slaves(), "if max processes is reached, we shouldn't need more slaves")
def test_build_cannot_be_prepared_more_than_once(self): build = Build(BuildRequest({})) subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() build.prepare(subjobs, mock_project_type, self._create_job_config()) with self.assertRaises(Exception): build.prepare(subjobs, mock_project_type, self._create_job_config())
def test_build_status_returns_queued_after_build_preparation(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) status = build._status() self.assertEqual(status, BuildStatus.QUEUED, 'Build status should be QUEUED after build has been prepared.')
def test_allocate_slave_calls_slave_setup(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave() build = Build(Mock(spec_set=BuildRequest)) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) mock_slave.setup.assert_called_once_with(build)
def test_exception_is_raised_if_problem_occurs_writing_subjob(self): Configuration['results_directory'] = abspath(join('some', 'temp', 'directory')) build = Build(BuildRequest({})) build._project_type = self._create_mock_project_type() subjob = self._create_subjobs(count=1, build_id=build.build_id())[0] build.prepare([subjob], self._create_job_config()) self.mock_fs.write_file.side_effect = FileExistsError with self.assertRaises(Exception): payload = {'filename': 'turtles.txt', 'body': 'Heroes in a half shell.'} build.complete_subjob(subjob.subjob_id(), payload=payload)
def test_build_status_returns_building_after_some_subjobs_are_executing(self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=2) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config(self._FAKE_MAX_EXECUTORS)) build.allocate_slave(mock_slave) # two out of three subjobs are now "in progress" status = build._status() self.assertEqual(status, BuildStatus.BUILDING)
def test_build_status_returns_queued_after_build_preparation(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config()) status = build._status() self.assertEqual( status, BuildStatus.QUEUED, 'Build status should be QUEUED after build has been prepared.')
def test_complete_subjob_writes_and_extracts_payload_to_correct_directory(self): Configuration['results_directory'] = '/tmp/results' build = Build(BuildRequest({})) build._project_type = self._create_mock_project_type() subjob = self._create_subjobs(count=1, build_id=build.build_id())[0] build.prepare([subjob], self._create_job_config()) payload = {'filename': 'turtles.txt', 'body': 'Heroes in a half shell.'} build.complete_subjob(subjob.subjob_id(), payload=payload) self.mock_fs.write_file.assert_called_once_with('Heroes in a half shell.', '/tmp/results/1/turtles.txt') self.mock_fs.extract_tar.assert_called_once_with('/tmp/results/1/turtles.txt', delete=True)
def test_allocate_slave_doesnt_use_more_than_max_executors(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slave = self._create_mock_slave() build = Build(BuildRequest({'setup': fake_setup_command})) build.prepare(subjobs, mock_project_type, self._create_job_config(1)) build.allocate_slave(mock_slave) self.assertEqual(build._num_allocated_executors, build._max_executors)
def test_need_more_slaves_returns_false_if_max_processes_is_reached(self): subjobs = self._create_subjobs(count=5) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=1) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config(max_executors=1)) build.allocate_slave(mock_slave) self.assertFalse( build.needs_more_slaves(), "if max processes is reached, we shouldn't need more slaves")
def test_build_status_returns_building_after_setup_has_started(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave() build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) self.assertEqual(build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after setup has started on slaves.')
def test_need_more_slaves_returns_true_if_max_processes_is_not_reached( self): subjobs = self._create_subjobs(count=8) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=5) build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config(max_executors=8)) build.allocate_slave(mock_slave) self.assertTrue( build.needs_more_slaves(), "if max_processes is not reached, we should need more slaves")
def test_build_status_returns_building_after_setup_is_complete_and_subjobs_are_executing(self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=2) build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) build.begin_subjob_executions_on_slave(mock_slave) # two out of three subjobs are now in progress self.assertEqual(build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after subjobs have started executing on slaves.')
def test_build_status_returns_building_after_setup_has_started(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave() build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config()) build.allocate_slave(mock_slave) self.assertEqual( build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after setup has started on slaves.' )
def test_exception_is_raised_if_problem_occurs_writing_subjob(self): Configuration['results_directory'] = '/tmp/results' build = Build(BuildRequest({})) build._project_type = self._create_mock_project_type() subjob = self._create_subjobs(count=1, build_id=build.build_id())[0] build.prepare([subjob], self._create_job_config()) self.mock_fs.write_file.side_effect = FileExistsError with self.assertRaises(Exception): payload = { 'filename': 'turtles.txt', 'body': 'Heroes in a half shell.' } build.complete_subjob(subjob.subjob_id(), payload=payload)
def test_allocate_slave_calls_slave_setup(self): # arrange subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'docker pull my:leg' mock_slave = self._create_mock_slave() # act build = Build(BuildRequest({'setup': fake_setup_command})) build.prepare(subjobs, mock_project_type, self._create_job_config(self._FAKE_MAX_EXECUTORS)) build.allocate_slave(mock_slave) # assert mock_slave.setup.assert_called_once_with(build.build_id(), project_type_params={'setup': fake_setup_command})
def test_build_status_returns_building_after_setup_is_complete_and_subjobs_are_executing( self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=2) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config()) build.allocate_slave(mock_slave) build.begin_subjob_executions_on_slave( mock_slave) # two out of three subjobs are now in progress self.assertEqual( build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after subjobs have started executing on slaves.' )
def test_allocate_slave_calls_slave_setup(self): # arrange subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'docker pull my:leg' mock_slave = self._create_mock_slave() # act build = Build(BuildRequest({'setup': fake_setup_command})) build.prepare(subjobs, mock_project_type, self._create_job_config()) build.allocate_slave(mock_slave) # assert mock_slave.setup.assert_called_once_with( build.build_id(), project_type_params={'setup': fake_setup_command})
def test_build_doesnt_use_more_than_max_executors(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [self._create_mock_slave(num_executors=5) for _ in range(3)] expected_num_executors = 12 # We expect the build to use 12 out of 15 available executors. build = Build(BuildRequest({'setup': fake_setup_command})) build._project_type = mock_project_type build.execute_next_subjob_or_teardown_slave = MagicMock() build.prepare(subjobs, self._create_job_config(max_executors=expected_num_executors)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] [build.begin_subjob_executions_on_slave(mock_slave) for mock_slave in mock_slaves] self.assertEqual(build.execute_next_subjob_or_teardown_slave.call_count, expected_num_executors, 'Build should start executing as many subjobs as its max_executors setting.')
def test_complete_subjob_writes_and_extracts_payload_to_correct_directory( self): Configuration['results_directory'] = '/tmp/results' build = Build(BuildRequest({})) build._project_type = self._create_mock_project_type() subjob = self._create_subjobs(count=1, build_id=build.build_id())[0] build.prepare([subjob], self._create_job_config()) payload = { 'filename': 'turtles.txt', 'body': 'Heroes in a half shell.' } build.complete_subjob(subjob.subjob_id(), payload=payload) self.mock_fs.write_file.assert_called_once_with( 'Heroes in a half shell.', '/tmp/results/1/turtles.txt') self.mock_fs.extract_tar.assert_called_once_with( '/tmp/results/1/turtles.txt', delete=True)
def test_build_status_returns_finished_after_all_subjobs_complete_and_slaves_finished(self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=3) postbuild_tasks_complete_event = Event() build = Build(BuildRequest({})) build._project_type = mock_project_type build._create_build_artifact = MagicMock() self._on_async_postbuild_tasks_completed(build, postbuild_tasks_complete_event.set) build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) # all three subjobs are now "in progress" for subjob in subjobs: build.complete_subjob(subjob.subjob_id()) # Wait for the async thread to complete executing postbuild tasks. self.assertTrue(postbuild_tasks_complete_event.wait(timeout=2), 'Postbuild tasks should complete within a few' 'seconds.') # Verify build artifacts was called after subjobs completed build._create_build_artifact.assert_called_once_with() self.assertTrue(build._subjobs_are_finished) self.assertEqual(build._status(), BuildStatus.FINISHED)
def _create_test_build( self, build_status=None, job_config=None, num_subjobs=3, num_atoms_per_subjob=3, slaves=None, ): """ Create a Build instance for testing purposes. The instance will be created and brought to the specified state similarly to how it would reach that state in actual app execution. Build instances have a huge amount of internal state with complicated interactions, so this helper method helps us write tests that are much more consistent and closer to reality. It also helps us avoid modifying a build's private members directly. :type build_status: BuildStatus :rtype: Build """ build = Build(BuildRequest(build_parameters={})) if build_status is None: return build # QUEUED: Instantiate a mock project_type instance for the build. mock_project_type = self._create_mock_project_type() self.patch('app.master.build.util.create_project_type' ).return_value = mock_project_type build.generate_project_type() if build_status is BuildStatus.QUEUED: return build # PREPARED: Create a fake job config and subjobs and hand them off to the build. job_config = job_config or self._create_job_config() mock_project_type.job_config.return_value = job_config subjobs = self._create_subjobs(count=num_subjobs, num_atoms_each=num_atoms_per_subjob, job_config=job_config) subjob_calculator = self._create_mock_subjob_calc(subjobs) build.prepare(subjob_calculator) if build_status is BuildStatus.PREPARED: return build # BUILDING: Allocate a slave and begin subjob executions on that slave. slaves = slaves or [self._create_mock_slave()] scheduler = self.scheduler_pool.get(build) for slave in slaves: scheduler.allocate_slave(slave=slave) scheduler.begin_subjob_executions_on_slave(slave=slave) if build_status is BuildStatus.BUILDING: return build # ERROR: Mark the in-progress build as failed. if build_status is BuildStatus.ERROR: build.mark_failed( failure_reason='Test build was intentionally marked failed.') return build # CANCELED: Cancel the in-progress build. if build_status is BuildStatus.CANCELED: build.cancel() return build # FINISHED: Complete all subjobs and allow all postbuild tasks to execute. self._finish_test_build(build) if build_status is BuildStatus.FINISHED: return build raise ValueError( 'Unsupported value for build_status: "{}".'.format(build_status))
def _create_test_build( self, build_status=None, job_config=None, num_subjobs=3, num_atoms_per_subjob=3, slaves=None, ): """ Create a Build instance for testing purposes. The instance will be created and brought to the specified state similarly to how it would reach that state in actual app execution. Build instances have a huge amount of internal state with complicated interactions, so this helper method helps us write tests that are much more consistent and closer to reality. It also helps us avoid modifying a build's private members directly. :type build_status: BuildStatus :rtype: Build """ build = Build(BuildRequest(build_parameters={})) if build_status is None: return build # QUEUED: Instantiate a mock project_type instance for the build. mock_project_type = self._create_mock_project_type() self.patch('app.master.build.util.create_project_type').return_value = mock_project_type build.generate_project_type() if build_status is BuildStatus.QUEUED: return build # PREPARED: Create a fake job config and subjobs and hand them off to the build. job_config = job_config or self._create_job_config() mock_project_type.job_config.return_value = job_config subjobs = self._create_subjobs(count=num_subjobs, num_atoms_each=num_atoms_per_subjob, job_config=job_config) subjob_calculator = self._create_mock_subjob_calc(subjobs) build.prepare(subjob_calculator) if build_status is BuildStatus.PREPARED: return build # BUILDING: Allocate a slave and begin subjob executions on that slave. slaves = slaves or [self._create_mock_slave()] scheduler = self.scheduler_pool.get(build) for slave in slaves: scheduler.allocate_slave(slave=slave) scheduler.begin_subjob_executions_on_slave(slave=slave) if build_status is BuildStatus.BUILDING: return build # ERROR: Mark the in-progress build as failed. if build_status is BuildStatus.ERROR: build.mark_failed(failure_reason='Test build was intentionally marked failed.') return build # CANCELED: Cancel the in-progress build. if build_status is BuildStatus.CANCELED: build.cancel() return build # FINISHED: Complete all subjobs and allow all postbuild tasks to execute. self._finish_test_build(build) if build_status is BuildStatus.FINISHED: return build raise ValueError('Unsupported value for build_status: "{}".'.format(build_status))