def test_get_workunit_no_prefetch(self): self.prefetch_quantity = 0 self.workunit_provider = Mock(spec=WorkUnitProvider) self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider, self.prefetch_quantity) prefetch_workunit_mock = self.mock_prefetch_workunit( bypass_threading=True) workunit1 = self.create_workunit() workunit2 = self.create_workunit() workunit3 = self.create_workunit() self.set_workunit_provider_return_values( [workunit1, workunit2, workunit3, NoAvailableWorkException()]) assert_that(self.undertest.get_workunit(), equal_to(workunit1)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) assert_that(self.undertest.get_workunit(), equal_to(workunit2)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) assert_that(self.undertest.get_workunit(), equal_to(workunit3)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit)
def test_get_workunit_no_prefetch(self): self.prefetch_quantity = 0 self.workunit_provider = Mock(spec=WorkUnitProvider) self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider, self.prefetch_quantity) prefetch_workunit_mock = self.mock_prefetch_workunit(bypass_threading=True) workunit1 = self.create_workunit() workunit2 = self.create_workunit() workunit3 = self.create_workunit() self.set_workunit_provider_return_values( [workunit1, workunit2, workunit3, NoAvailableWorkException()]) assert_that(self.undertest.get_workunit(), equal_to(workunit1)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) assert_that(self.undertest.get_workunit(), equal_to(workunit2)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) assert_that(self.undertest.get_workunit(), equal_to(workunit3)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit)
def setUp(self): self.prefetch_quantity = 2 self.workunit_provider = CopyingMock(spec=WorkUnitProvider) self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider, self.prefetch_quantity) self._workunit_number = 0
class PreFetchingWorkUnitProviderTest(unittest.TestCase): def setUp(self): self.prefetch_quantity = 2 self.workunit_provider = CopyingMock(spec=WorkUnitProvider) self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider, self.prefetch_quantity) self._workunit_number = 0 def create_workunit(self, num=None): if num is None: num = self._workunit_number workunit = Mock(spec=WorkUnit) workunit.get_filename.return_value = "Workunit%d" % num self._workunit_number += 1 return workunit def set_workunit_provider_return_values(self, return_values): def side_effect(*args, **kwargs): result = return_values.pop(0) if isinstance(result, Exception): raise result return result self.workunit_provider.get_workunit.side_effect = side_effect def mock_prefetch_workunit(self, bypass_threading=False): prefetch_workunit_mock = Mock() if bypass_threading: def do_prefetch(): self.undertest._do_prefetch_workunit() prefetch_workunit_mock.side_effect = do_prefetch else: def generate_mocked_workunit(): self.undertest.workunits.append(self.create_workunit()) prefetch_workunit_mock.side_effect = generate_mocked_workunit self.undertest.prefetch_workunit = prefetch_workunit_mock return prefetch_workunit_mock def test_get_first_workunit_also_prefetches_configured_quantity(self): prefetch_workunit_mock = self.mock_prefetch_workunit() self.undertest.get_workunit() assert_that(prefetch_workunit_mock.call_count, equal_to(self.prefetch_quantity)) def test_get_second_workunit_prefetches_one_more(self): prefetch_workunit_mock = self.mock_prefetch_workunit() self.undertest.get_workunit() self.undertest.get_workunit() assert_that(prefetch_workunit_mock.call_count, equal_to(self.prefetch_quantity + 1)) def test_all_prefetched_doesnt_raise_no_available_work_until_all_retrieved( self): prefetch_workunit_mock = self.mock_prefetch_workunit( bypass_threading=True) workunit1 = self.create_workunit() workunit2 = self.create_workunit() workunit3 = self.create_workunit() self.set_workunit_provider_return_values( [workunit1, workunit2, workunit3, NoAvailableWorkException()]) assert_that(self.undertest.get_workunit(), equal_to(workunit1)) assert_that(prefetch_workunit_mock.call_count, equal_to(self.prefetch_quantity)) assert_that(self.undertest.get_workunit(), equal_to(workunit2)) # This call to fetch will raise a NoAvailableWorkException assert_that(prefetch_workunit_mock.call_count, equal_to(self.prefetch_quantity + 1)) assert_that(self.undertest.get_workunit(), equal_to(workunit3)) # Note we don't try to fetch anymore assert_that(prefetch_workunit_mock.call_count, equal_to(self.prefetch_quantity + 1)) self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit) def test_prefetch_excludes_already_fetched(self): prefetch_workunit_mock = self.mock_prefetch_workunit( bypass_threading=True) workunit1 = self.create_workunit() workunit2 = self.create_workunit() workunit3 = self.create_workunit() self.set_workunit_provider_return_values( [workunit1, workunit2, workunit3, NoAvailableWorkException()]) self.undertest.get_workunit() self.undertest.get_workunit() self.undertest.get_workunit() self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit) expected_calls = [ call(ignore_list=[]), call(ignore_list=[workunit1.get_filename()]), call(ignore_list=[ workunit1.get_filename(), workunit2.get_filename() ]), call(ignore_list=[ workunit1.get_filename(), workunit2.get_filename(), workunit3.get_filename() ]) ] self.workunit_provider.get_workunit.assert_has_calls(expected_calls) def test_get_workunit_no_prefetch(self): self.prefetch_quantity = 0 self.workunit_provider = Mock(spec=WorkUnitProvider) self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider, self.prefetch_quantity) prefetch_workunit_mock = self.mock_prefetch_workunit( bypass_threading=True) workunit1 = self.create_workunit() workunit2 = self.create_workunit() workunit3 = self.create_workunit() self.set_workunit_provider_return_values( [workunit1, workunit2, workunit3, NoAvailableWorkException()]) assert_that(self.undertest.get_workunit(), equal_to(workunit1)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) assert_that(self.undertest.get_workunit(), equal_to(workunit2)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) assert_that(self.undertest.get_workunit(), equal_to(workunit3)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit) def test_duplicate_workunit_ignored_(self): # This tests the case when 2 or more threads created back to back # end up retrieving the same workunit. Only want to return it once # though. prefetch_workunit_mock = self.mock_prefetch_workunit( bypass_threading=True) workunit1 = self.create_workunit(1) workunit2 = self.create_workunit(2) workunit3 = self.create_workunit(2) self.set_workunit_provider_return_values( [workunit1, workunit2, workunit3, NoAvailableWorkException()]) assert_that(self.undertest.get_workunit(), equal_to(workunit1)) assert_that(self.undertest.get_workunit(), equal_to(workunit2)) self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit)
def __init__(self, working_directory, output_directory, dry_run=False, debug=False, name_filter=None, user_id=None): self.dry_run = dry_run self.user_id = user_id logger.info("Input directory set to: %s" % working_directory) logger.info("Output directory set to: %s" % output_directory) working_context = context.get_context(working_directory, userid=self.user_id) output_context = context.get_context(output_directory, userid=self.user_id) if dry_run and working_context.is_remote(): sys.stdout.write("A dry run can only be done on local files.\n") sys.exit(0) if output_context.is_remote(): sys.stdout.write("The output directory must be local.\n") sys.exit(0) image_manager = self._create_image_manager() progress_manager = working_context.get_progress_manager() builder = self._create_workunit_builder(working_context, output_context, progress_manager) workunit_provider = WorkUnitProvider( self.input_suffix, working_context, progress_manager, builder, randomize=self.should_randomize_workunits, name_filter=name_filter) prefetching_workunit_provider = PreFetchingWorkUnitProvider( workunit_provider, config.read("PREFETCH.NUMBER"), image_manager) if working_context.is_remote(): synchronization_manager = SynchronizationManager(working_context, sync_enabled=True) else: synchronization_manager = None model = TransAckValidationModel(prefetching_workunit_provider, image_manager, synchronization_manager) logger.debug("Created model.") view = self._create_view(model, debug=debug) logger.debug("Created view.") model.start_work() self.model = model self.view = view self.controller = view.controller self.controller.display_current_image() if not synchronization_manager: self.view.disable_sync_menu() self.view.show()
class PreFetchingWorkUnitProviderTest(unittest.TestCase): def setUp(self): self.prefetch_quantity = 2 self.workunit_provider = CopyingMock(spec=WorkUnitProvider) self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider, self.prefetch_quantity) self._workunit_number = 0 def create_workunit(self, num=None): if num is None: num = self._workunit_number workunit = Mock(spec=WorkUnit) workunit.get_filename.return_value = "Workunit%d" % num self._workunit_number += 1 return workunit def set_workunit_provider_return_values(self, return_values): def side_effect(*args, **kwargs): result = return_values.pop(0) if isinstance(result, Exception): raise result return result self.workunit_provider.get_workunit.side_effect = side_effect def mock_prefetch_workunit(self, bypass_threading=False): prefetch_workunit_mock = Mock() if bypass_threading: def do_prefetch(): self.undertest._do_prefetch_workunit() prefetch_workunit_mock.side_effect = do_prefetch else: def generate_mocked_workunit(): self.undertest.workunits.append(self.create_workunit()) prefetch_workunit_mock.side_effect = generate_mocked_workunit self.undertest.prefetch_workunit = prefetch_workunit_mock return prefetch_workunit_mock def test_get_first_workunit_also_prefetches_configured_quantity(self): prefetch_workunit_mock = self.mock_prefetch_workunit() self.undertest.get_workunit() assert_that(prefetch_workunit_mock.call_count, equal_to(self.prefetch_quantity)) def test_get_second_workunit_prefetches_one_more(self): prefetch_workunit_mock = self.mock_prefetch_workunit() self.undertest.get_workunit() self.undertest.get_workunit() assert_that(prefetch_workunit_mock.call_count, equal_to(self.prefetch_quantity + 1)) def test_all_prefetched_doesnt_raise_no_available_work_until_all_retrieved(self): prefetch_workunit_mock = self.mock_prefetch_workunit(bypass_threading=True) workunit1 = self.create_workunit() workunit2 = self.create_workunit() workunit3 = self.create_workunit() self.set_workunit_provider_return_values( [workunit1, workunit2, workunit3, NoAvailableWorkException()]) assert_that(self.undertest.get_workunit(), equal_to(workunit1)) assert_that(prefetch_workunit_mock.call_count, equal_to(self.prefetch_quantity)) assert_that(self.undertest.get_workunit(), equal_to(workunit2)) # This call to fetch will raise a NoAvailableWorkException assert_that(prefetch_workunit_mock.call_count, equal_to(self.prefetch_quantity + 1)) assert_that(self.undertest.get_workunit(), equal_to(workunit3)) # Note we don't try to fetch anymore assert_that(prefetch_workunit_mock.call_count, equal_to(self.prefetch_quantity + 1)) self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit) def test_prefetch_excludes_already_fetched(self): prefetch_workunit_mock = self.mock_prefetch_workunit(bypass_threading=True) workunit1 = self.create_workunit() workunit2 = self.create_workunit() workunit3 = self.create_workunit() self.set_workunit_provider_return_values( [workunit1, workunit2, workunit3, NoAvailableWorkException()]) self.undertest.get_workunit() self.undertest.get_workunit() self.undertest.get_workunit() self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit) expected_calls = [ call(ignore_list=[]), call(ignore_list=[workunit1.get_filename()]), call(ignore_list=[workunit1.get_filename(), workunit2.get_filename()]), call(ignore_list=[workunit1.get_filename(), workunit2.get_filename(), workunit3.get_filename()]) ] self.workunit_provider.get_workunit.assert_has_calls(expected_calls) def test_get_workunit_no_prefetch(self): self.prefetch_quantity = 0 self.workunit_provider = Mock(spec=WorkUnitProvider) self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider, self.prefetch_quantity) prefetch_workunit_mock = self.mock_prefetch_workunit(bypass_threading=True) workunit1 = self.create_workunit() workunit2 = self.create_workunit() workunit3 = self.create_workunit() self.set_workunit_provider_return_values( [workunit1, workunit2, workunit3, NoAvailableWorkException()]) assert_that(self.undertest.get_workunit(), equal_to(workunit1)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) assert_that(self.undertest.get_workunit(), equal_to(workunit2)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) assert_that(self.undertest.get_workunit(), equal_to(workunit3)) assert_that(prefetch_workunit_mock.call_count, equal_to(0)) self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit) def test_duplicate_workunit_ignored_(self): # This tests the case when 2 or more threads created back to back # end up retrieving the same workunit. Only want to return it once # though. prefetch_workunit_mock = self.mock_prefetch_workunit(bypass_threading=True) workunit1 = self.create_workunit(1) workunit2 = self.create_workunit(2) workunit3 = self.create_workunit(2) self.set_workunit_provider_return_values( [workunit1, workunit2, workunit3, NoAvailableWorkException()]) assert_that(self.undertest.get_workunit(), equal_to(workunit1)) assert_that(self.undertest.get_workunit(), equal_to(workunit2)) self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit)