def test_get_workunit_no_prefetch(self):
self.prefetch_quantity = 0
self.workunit_provider = Mock(spec=WorkUnitProvider)
self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider,
self.prefetch_quantity)
prefetch_workunit_mock = self.mock_prefetch_workunit(
bypass_threading=True)
workunit1 = self.create_workunit()
workunit2 = self.create_workunit()
workunit3 = self.create_workunit()
self.set_workunit_provider_return_values(
[workunit1, workunit2, workunit3,
NoAvailableWorkException()])
assert_that(self.undertest.get_workunit(), equal_to(workunit1))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
assert_that(self.undertest.get_workunit(), equal_to(workunit2))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
assert_that(self.undertest.get_workunit(), equal_to(workunit3))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
self.assertRaises(NoAvailableWorkException,
self.undertest.get_workunit)
def test_get_workunit_no_prefetch(self):
self.prefetch_quantity = 0
self.workunit_provider = Mock(spec=WorkUnitProvider)
self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider,
self.prefetch_quantity)
prefetch_workunit_mock = self.mock_prefetch_workunit(bypass_threading=True)
workunit1 = self.create_workunit()
workunit2 = self.create_workunit()
workunit3 = self.create_workunit()
self.set_workunit_provider_return_values(
[workunit1, workunit2, workunit3, NoAvailableWorkException()])
assert_that(self.undertest.get_workunit(), equal_to(workunit1))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
assert_that(self.undertest.get_workunit(), equal_to(workunit2))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
assert_that(self.undertest.get_workunit(), equal_to(workunit3))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit)
def setUp(self):
self.prefetch_quantity = 2
self.workunit_provider = CopyingMock(spec=WorkUnitProvider)
self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider,
self.prefetch_quantity)
self._workunit_number = 0
class PreFetchingWorkUnitProviderTest(unittest.TestCase):
def setUp(self):
self.prefetch_quantity = 2
self.workunit_provider = CopyingMock(spec=WorkUnitProvider)
self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider,
self.prefetch_quantity)
self._workunit_number = 0
def create_workunit(self, num=None):
if num is None:
num = self._workunit_number
workunit = Mock(spec=WorkUnit)
workunit.get_filename.return_value = "Workunit%d" % num
self._workunit_number += 1
return workunit
def set_workunit_provider_return_values(self, return_values):
def side_effect(*args, **kwargs):
result = return_values.pop(0)
if isinstance(result, Exception):
raise result
return result
self.workunit_provider.get_workunit.side_effect = side_effect
def mock_prefetch_workunit(self, bypass_threading=False):
prefetch_workunit_mock = Mock()
if bypass_threading:
def do_prefetch():
self.undertest._do_prefetch_workunit()
prefetch_workunit_mock.side_effect = do_prefetch
else:
def generate_mocked_workunit():
self.undertest.workunits.append(self.create_workunit())
prefetch_workunit_mock.side_effect = generate_mocked_workunit
self.undertest.prefetch_workunit = prefetch_workunit_mock
return prefetch_workunit_mock
def test_get_first_workunit_also_prefetches_configured_quantity(self):
prefetch_workunit_mock = self.mock_prefetch_workunit()
self.undertest.get_workunit()
assert_that(prefetch_workunit_mock.call_count,
equal_to(self.prefetch_quantity))
def test_get_second_workunit_prefetches_one_more(self):
prefetch_workunit_mock = self.mock_prefetch_workunit()
self.undertest.get_workunit()
self.undertest.get_workunit()
assert_that(prefetch_workunit_mock.call_count,
equal_to(self.prefetch_quantity + 1))
def test_all_prefetched_doesnt_raise_no_available_work_until_all_retrieved(
self):
prefetch_workunit_mock = self.mock_prefetch_workunit(
bypass_threading=True)
workunit1 = self.create_workunit()
workunit2 = self.create_workunit()
workunit3 = self.create_workunit()
self.set_workunit_provider_return_values(
[workunit1, workunit2, workunit3,
NoAvailableWorkException()])
assert_that(self.undertest.get_workunit(), equal_to(workunit1))
assert_that(prefetch_workunit_mock.call_count,
equal_to(self.prefetch_quantity))
assert_that(self.undertest.get_workunit(), equal_to(workunit2))
# This call to fetch will raise a NoAvailableWorkException
assert_that(prefetch_workunit_mock.call_count,
equal_to(self.prefetch_quantity + 1))
assert_that(self.undertest.get_workunit(), equal_to(workunit3))
# Note we don't try to fetch anymore
assert_that(prefetch_workunit_mock.call_count,
equal_to(self.prefetch_quantity + 1))
self.assertRaises(NoAvailableWorkException,
self.undertest.get_workunit)
def test_prefetch_excludes_already_fetched(self):
prefetch_workunit_mock = self.mock_prefetch_workunit(
bypass_threading=True)
workunit1 = self.create_workunit()
workunit2 = self.create_workunit()
workunit3 = self.create_workunit()
self.set_workunit_provider_return_values(
[workunit1, workunit2, workunit3,
NoAvailableWorkException()])
self.undertest.get_workunit()
self.undertest.get_workunit()
self.undertest.get_workunit()
self.assertRaises(NoAvailableWorkException,
self.undertest.get_workunit)
expected_calls = [
call(ignore_list=[]),
call(ignore_list=[workunit1.get_filename()]),
call(ignore_list=[
workunit1.get_filename(),
workunit2.get_filename()
]),
call(ignore_list=[
workunit1.get_filename(),
workunit2.get_filename(),
workunit3.get_filename()
])
]
self.workunit_provider.get_workunit.assert_has_calls(expected_calls)
def test_get_workunit_no_prefetch(self):
self.prefetch_quantity = 0
self.workunit_provider = Mock(spec=WorkUnitProvider)
self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider,
self.prefetch_quantity)
prefetch_workunit_mock = self.mock_prefetch_workunit(
bypass_threading=True)
workunit1 = self.create_workunit()
workunit2 = self.create_workunit()
workunit3 = self.create_workunit()
self.set_workunit_provider_return_values(
[workunit1, workunit2, workunit3,
NoAvailableWorkException()])
assert_that(self.undertest.get_workunit(), equal_to(workunit1))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
assert_that(self.undertest.get_workunit(), equal_to(workunit2))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
assert_that(self.undertest.get_workunit(), equal_to(workunit3))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
self.assertRaises(NoAvailableWorkException,
self.undertest.get_workunit)
def test_duplicate_workunit_ignored_(self):
# This tests the case when 2 or more threads created back to back
# end up retrieving the same workunit. Only want to return it once
# though.
prefetch_workunit_mock = self.mock_prefetch_workunit(
bypass_threading=True)
workunit1 = self.create_workunit(1)
workunit2 = self.create_workunit(2)
workunit3 = self.create_workunit(2)
self.set_workunit_provider_return_values(
[workunit1, workunit2, workunit3,
NoAvailableWorkException()])
assert_that(self.undertest.get_workunit(), equal_to(workunit1))
assert_that(self.undertest.get_workunit(), equal_to(workunit2))
self.assertRaises(NoAvailableWorkException,
self.undertest.get_workunit)
def __init__(self,
working_directory,
output_directory,
dry_run=False,
debug=False,
name_filter=None,
user_id=None):
self.dry_run = dry_run
self.user_id = user_id
logger.info("Input directory set to: %s" % working_directory)
logger.info("Output directory set to: %s" % output_directory)
working_context = context.get_context(working_directory,
userid=self.user_id)
output_context = context.get_context(output_directory,
userid=self.user_id)
if dry_run and working_context.is_remote():
sys.stdout.write("A dry run can only be done on local files.\n")
sys.exit(0)
if output_context.is_remote():
sys.stdout.write("The output directory must be local.\n")
sys.exit(0)
image_manager = self._create_image_manager()
progress_manager = working_context.get_progress_manager()
builder = self._create_workunit_builder(working_context,
output_context,
progress_manager)
workunit_provider = WorkUnitProvider(
self.input_suffix,
working_context,
progress_manager,
builder,
randomize=self.should_randomize_workunits,
name_filter=name_filter)
prefetching_workunit_provider = PreFetchingWorkUnitProvider(
workunit_provider, config.read("PREFETCH.NUMBER"), image_manager)
if working_context.is_remote():
synchronization_manager = SynchronizationManager(working_context,
sync_enabled=True)
else:
synchronization_manager = None
model = TransAckValidationModel(prefetching_workunit_provider,
image_manager, synchronization_manager)
logger.debug("Created model.")
view = self._create_view(model, debug=debug)
logger.debug("Created view.")
model.start_work()
self.model = model
self.view = view
self.controller = view.controller
self.controller.display_current_image()
if not synchronization_manager:
self.view.disable_sync_menu()
self.view.show()
def setUp(self):
self.prefetch_quantity = 2
self.workunit_provider = CopyingMock(spec=WorkUnitProvider)
self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider,
self.prefetch_quantity)
self._workunit_number = 0
class PreFetchingWorkUnitProviderTest(unittest.TestCase):
def setUp(self):
self.prefetch_quantity = 2
self.workunit_provider = CopyingMock(spec=WorkUnitProvider)
self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider,
self.prefetch_quantity)
self._workunit_number = 0
def create_workunit(self, num=None):
if num is None:
num = self._workunit_number
workunit = Mock(spec=WorkUnit)
workunit.get_filename.return_value = "Workunit%d" % num
self._workunit_number += 1
return workunit
def set_workunit_provider_return_values(self, return_values):
def side_effect(*args, **kwargs):
result = return_values.pop(0)
if isinstance(result, Exception):
raise result
return result
self.workunit_provider.get_workunit.side_effect = side_effect
def mock_prefetch_workunit(self, bypass_threading=False):
prefetch_workunit_mock = Mock()
if bypass_threading:
def do_prefetch():
self.undertest._do_prefetch_workunit()
prefetch_workunit_mock.side_effect = do_prefetch
else:
def generate_mocked_workunit():
self.undertest.workunits.append(self.create_workunit())
prefetch_workunit_mock.side_effect = generate_mocked_workunit
self.undertest.prefetch_workunit = prefetch_workunit_mock
return prefetch_workunit_mock
def test_get_first_workunit_also_prefetches_configured_quantity(self):
prefetch_workunit_mock = self.mock_prefetch_workunit()
self.undertest.get_workunit()
assert_that(prefetch_workunit_mock.call_count,
equal_to(self.prefetch_quantity))
def test_get_second_workunit_prefetches_one_more(self):
prefetch_workunit_mock = self.mock_prefetch_workunit()
self.undertest.get_workunit()
self.undertest.get_workunit()
assert_that(prefetch_workunit_mock.call_count,
equal_to(self.prefetch_quantity + 1))
def test_all_prefetched_doesnt_raise_no_available_work_until_all_retrieved(self):
prefetch_workunit_mock = self.mock_prefetch_workunit(bypass_threading=True)
workunit1 = self.create_workunit()
workunit2 = self.create_workunit()
workunit3 = self.create_workunit()
self.set_workunit_provider_return_values(
[workunit1, workunit2, workunit3, NoAvailableWorkException()])
assert_that(self.undertest.get_workunit(), equal_to(workunit1))
assert_that(prefetch_workunit_mock.call_count,
equal_to(self.prefetch_quantity))
assert_that(self.undertest.get_workunit(), equal_to(workunit2))
# This call to fetch will raise a NoAvailableWorkException
assert_that(prefetch_workunit_mock.call_count,
equal_to(self.prefetch_quantity + 1))
assert_that(self.undertest.get_workunit(), equal_to(workunit3))
# Note we don't try to fetch anymore
assert_that(prefetch_workunit_mock.call_count,
equal_to(self.prefetch_quantity + 1))
self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit)
def test_prefetch_excludes_already_fetched(self):
prefetch_workunit_mock = self.mock_prefetch_workunit(bypass_threading=True)
workunit1 = self.create_workunit()
workunit2 = self.create_workunit()
workunit3 = self.create_workunit()
self.set_workunit_provider_return_values(
[workunit1, workunit2, workunit3, NoAvailableWorkException()])
self.undertest.get_workunit()
self.undertest.get_workunit()
self.undertest.get_workunit()
self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit)
expected_calls = [
call(ignore_list=[]),
call(ignore_list=[workunit1.get_filename()]),
call(ignore_list=[workunit1.get_filename(), workunit2.get_filename()]),
call(ignore_list=[workunit1.get_filename(), workunit2.get_filename(),
workunit3.get_filename()])
]
self.workunit_provider.get_workunit.assert_has_calls(expected_calls)
def test_get_workunit_no_prefetch(self):
self.prefetch_quantity = 0
self.workunit_provider = Mock(spec=WorkUnitProvider)
self.undertest = PreFetchingWorkUnitProvider(self.workunit_provider,
self.prefetch_quantity)
prefetch_workunit_mock = self.mock_prefetch_workunit(bypass_threading=True)
workunit1 = self.create_workunit()
workunit2 = self.create_workunit()
workunit3 = self.create_workunit()
self.set_workunit_provider_return_values(
[workunit1, workunit2, workunit3, NoAvailableWorkException()])
assert_that(self.undertest.get_workunit(), equal_to(workunit1))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
assert_that(self.undertest.get_workunit(), equal_to(workunit2))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
assert_that(self.undertest.get_workunit(), equal_to(workunit3))
assert_that(prefetch_workunit_mock.call_count, equal_to(0))
self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit)
def test_duplicate_workunit_ignored_(self):
# This tests the case when 2 or more threads created back to back
# end up retrieving the same workunit. Only want to return it once
# though.
prefetch_workunit_mock = self.mock_prefetch_workunit(bypass_threading=True)
workunit1 = self.create_workunit(1)
workunit2 = self.create_workunit(2)
workunit3 = self.create_workunit(2)
self.set_workunit_provider_return_values(
[workunit1, workunit2, workunit3, NoAvailableWorkException()])
assert_that(self.undertest.get_workunit(), equal_to(workunit1))
assert_that(self.undertest.get_workunit(), equal_to(workunit2))
self.assertRaises(NoAvailableWorkException, self.undertest.get_workunit)
