def setUp(self):
self.main_user_id = "main_user"
self.working_directory = self.get_abs_path(WD_NO_LOG)
directory_manager = LocalDirectoryWorkingContext(
self.working_directory)
self.progress_manager = LocalProgressManager(directory_manager,
userid=self.main_user_id)
def setUp(self):
super(RealsModelPersistenceTest, self).setUp()
concurrent_directory_manager = LocalDirectoryWorkingContext(
self._get_working_dir())
self.concurrent_progress_manager = LocalProgressManager(
concurrent_directory_manager)
class ProgressManagerLoadingTest(FileReadingTestCase):
def setUp(self):
self.working_directory = self.get_abs_path(WD_HAS_PROGRESS)
directory_manager = LocalDirectoryWorkingContext(self.working_directory)
self.progress_manager = LocalProgressManager(directory_manager)
def tearDown(self):
self.progress_manager.clean(suffixes=[LOCK_SUFFIX])
def test_load_progress(self):
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
contains_inanyorder("xxx1.cands.astrom", "xxx3.cands.astrom"))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
contains_inanyorder("xxx3.reals.astrom"))
class ProgressManagerLoadingTest(FileReadingTestCase):
def setUp(self):
self.working_directory = self.get_abs_path(WD_HAS_PROGRESS)
directory_manager = LocalDirectoryWorkingContext(
self.working_directory)
self.progress_manager = LocalProgressManager(directory_manager)
def tearDown(self):
self.progress_manager.clean(suffixes=[LOCK_SUFFIX])
def test_load_progress(self):
assert_that(
self.progress_manager.get_done(tasks.CANDS_TASK),
contains_inanyorder("xxx1.cands.astrom", "xxx3.cands.astrom"))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
contains_inanyorder("xxx3.reals.astrom"))
def setUp(self):
events.unsub_all()
parser = AstromParser()
directory_manager = LocalDirectoryWorkingContext(
self._get_working_dir())
self.context = directory_manager
progress_manager = LocalProgressManager(directory_manager)
workunit_provider = WorkUnitProvider(
tasks.get_suffix(self._get_task()), directory_manager,
progress_manager,
self._get_workunit_builder(parser, progress_manager))
self.workunit_provider = workunit_provider
self.progress_manager = progress_manager
self.singlet_download_manager = Mock(spec=AsynchronousDownloadManager)
self.triplet_download_manager = Mock(spec=AsynchronousDownloadManager)
self.image_manager = ImageManager(self.singlet_download_manager,
self.triplet_download_manager)
self.model = ValidationModel(self.workunit_provider,
self.image_manager, None)
self.custom_setup()
self.model.start_work()
def setUp(self):
WxWidgetTestCase.setUp(self)
parser = AstromParser()
context = LocalDirectoryWorkingContext(
self.get_abs_path("data/controller_testdir"))
progress_manager = LocalProgressManager(context)
workunit_provider = WorkUnitProvider(tasks.get_suffix(tasks.REALS_TASK),
context, progress_manager,
RealsWorkUnitBuilder(
parser,
context,
context,
progress_manager))
image_manager = Mock(spec=ImageManager)
self.model = ValidationModel(workunit_provider, image_manager, None)
self.model.start_work()
self.model.get_writer = Mock(return_value=Mock(spec=mpc.MPCWriter))
# We don't actually have any images loaded, so mock this out
source_cutout = Mock(spec=SourceCutout)
source_cutout.pixel_x = 11
source_cutout.pixel_y = 50
self.model.get_current_cutout = Mock(return_value=source_cutout)
x_cen = 10
y_cen = 50
mag = 24.0
self.model.get_current_source_observed_magnitude = Mock(return_value=(x_cen, y_cen, mag))
self.model.is_current_source_adjusted = Mock(return_value=False)
self.model.get_current_fits_header = Mock()
self.name_generator = Mock(spec=ProvisionalNameGenerator)
self.name_generator.generate_name.return_value = TEST_PROVISIONAL_NAME
class TestFactory(object):
def __init__(self, model, name_generator):
self.model = model
self.name_generator = name_generator
def create_controller(self, view):
return ProcessRealsController(self.model, view,
self.name_generator)
self.view = ApplicationView(
TestFactory(self.model, self.name_generator))
self.controller = self.view.controller
self.controller.display_current_image = Mock()
def setUp(self):
self.main_user_id = "main_user"
self.working_directory = self.get_abs_path(WD_NO_LOG)
directory_manager = LocalDirectoryWorkingContext(self.working_directory)
self.progress_manager = LocalProgressManager(directory_manager,
userid=self.main_user_id)
class RealsModelPersistenceTest(GeneralModelTest):
def setUp(self):
super(RealsModelPersistenceTest, self).setUp()
concurrent_directory_manager = LocalDirectoryWorkingContext(
self._get_working_dir())
self.concurrent_progress_manager = LocalProgressManager(
concurrent_directory_manager)
def _get_task(self):
return tasks.REALS_TASK
def _get_workunit_builder(self, parser, progress_manager):
return RealsWorkUnitBuilder(parser, self.context, self.context,
progress_manager)
def _get_working_dir(self):
return self.get_abs_path(FRESH_TEST_DIR)
def get_directory_to_clean(self):
return self._get_working_dir()
def get_files_to_keep(self):
return TEST_FILES
def test_record_progress_only_after_source_finished(self):
first_file = self.model.get_current_filename()
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
has_length(0))
self.model.set_current_source_name(TEST_OBJECT_NAME)
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
has_length(0))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
has_length(0))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0))
def test_record_progress_multiple_sources(self):
first_file = self.model.get_current_filename()
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
has_length(0))
self.model.set_current_source_name(TEST_OBJECT_NAME)
# source 1 of 3, reading 1 of 3
self.model.accept_current_item()
self.model.next_item()
# source 1 of 3 reading 2 of 3
self.model.accept_current_item()
self.model.next_item()
# source 1 of 3 reading 3 of 3
self.model.accept_current_item()
self.model.next_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0))
self.model.set_current_source_name(TEST_OBJECT_NAME)
# source 2 of 3 reading 1 of 3
self.model.accept_current_item()
self.model.next_item()
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0))
# source 2 of 3 reading 2 of 3
self.model.accept_current_item()
self.model.next_item()
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0))
# source 2 of 3 reading 3 of 3
self.model.accept_current_item()
self.model.next_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1))
self.model.set_current_source_name(TEST_OBJECT_NAME)
# source 3 of 3 reading 1 of 3
self.model.next_item()
self.model.accept_current_item()
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1))
# source 3 of 3 reading 2 of 3
self.model.next_item()
self.model.accept_current_item()
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1))
# source 3 of 3 reading 3 of 3
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(True))
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1, 2))
def test_file_processed_event(self):
observer = Mock()
events.subscribe(events.FINISHED_WORKUNIT, observer.on_file_processed)
workunit = self.model.get_current_workunit()
accepts_before_next_file = 9
while accepts_before_next_file > 1:
self.model.set_current_source_name(TEST_OBJECT_NAME)
self.model.accept_current_item()
self.model.next_item()
assert_that(observer.on_file_processed.call_count, equal_to(0))
accepts_before_next_file -= 1
self.model.accept_current_item()
assert_that(observer.on_file_processed.call_count, equal_to(1))
# Make sure it was triggered with the right data
args = observer.on_file_processed.call_args[0]
assert_that(args, has_length(1))
msg = args[0]
assert_that(msg.topic, equal_to(events.FINISHED_WORKUNIT))
assert_that(msg.data, equal_to(workunit.get_results_file_paths()))
def test_unlock_on_exit(self):
current_file = self.model.get_current_filename()
assert_that(self.progress_manager.owns_lock(current_file),
equal_to(True))
self.model.exit()
assert_that(self.progress_manager.owns_lock(current_file),
equal_to(False))
class CandidatesModelPersistenceTest(GeneralModelTest):
def setUp(self):
super(CandidatesModelPersistenceTest, self).setUp()
concurrent_directory_manager = LocalDirectoryWorkingContext(
self._get_working_dir())
self.concurrent_progress_manager = LocalProgressManager(
concurrent_directory_manager)
def _get_task(self):
return tasks.CANDS_TASK
def _get_workunit_builder(self, parser, progress_manager):
return CandidatesWorkUnitBuilder(parser, self.context, self.context,
progress_manager)
def _get_working_dir(self):
return self.get_abs_path(FRESH_TEST_DIR)
def get_directory_to_clean(self):
return self._get_working_dir()
def get_files_to_keep(self):
return TEST_FILES
def test_record_progress_cands_multiple_files(self):
first_file = self.model.get_current_filename()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(True))
assert_that(
self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1, 2))
self.model.next_item()
second_file = self.model.get_current_filename()
assert_that(second_file, is_not(equal_to(first_file)))
assert_that(
self.concurrent_progress_manager.get_processed_indices(
second_file), has_length(0))
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(second_file),
equal_to(False))
assert_that(
self.concurrent_progress_manager.get_processed_indices(
second_file), contains_inanyorder(0))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(second_file),
equal_to(False))
assert_that(
self.concurrent_progress_manager.get_processed_indices(
second_file), contains_inanyorder(0, 1))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(second_file),
equal_to(True))
assert_that(
self.concurrent_progress_manager.get_processed_indices(
second_file), contains_inanyorder(0, 1, 2))
def setUp(self):
super(CandidatesModelPersistenceTest, self).setUp()
concurrent_directory_manager = LocalDirectoryWorkingContext(self._get_working_dir())
self.concurrent_progress_manager = LocalProgressManager(concurrent_directory_manager)
class CandidatesModelPersistenceTest(GeneralModelTest):
def setUp(self):
super(CandidatesModelPersistenceTest, self).setUp()
concurrent_directory_manager = LocalDirectoryWorkingContext(self._get_working_dir())
self.concurrent_progress_manager = LocalProgressManager(concurrent_directory_manager)
def _get_task(self):
return tasks.CANDS_TASK
def _get_workunit_builder(self, parser, progress_manager):
return CandidatesWorkUnitBuilder(parser, self.context, self.context,
progress_manager)
def _get_working_dir(self):
return self.get_abs_path(FRESH_TEST_DIR)
def get_directory_to_clean(self):
return self._get_working_dir()
def get_files_to_keep(self):
return TEST_FILES
def test_record_progress_cands_multiple_files(self):
first_file = self.model.get_current_filename()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(True))
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1, 2))
self.model.next_item()
second_file = self.model.get_current_filename()
assert_that(second_file, is_not(equal_to(first_file)))
assert_that(self.concurrent_progress_manager.get_processed_indices(second_file),
has_length(0))
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(second_file),
equal_to(False))
assert_that(self.concurrent_progress_manager.get_processed_indices(second_file),
contains_inanyorder(0))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(second_file),
equal_to(False))
assert_that(self.concurrent_progress_manager.get_processed_indices(second_file),
contains_inanyorder(0, 1))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(second_file),
equal_to(True))
assert_that(self.concurrent_progress_manager.get_processed_indices(second_file),
contains_inanyorder(0, 1, 2))
class RealsModelPersistenceTest(GeneralModelTest):
def setUp(self):
super(RealsModelPersistenceTest, self).setUp()
concurrent_directory_manager = LocalDirectoryWorkingContext(self._get_working_dir())
self.concurrent_progress_manager = LocalProgressManager(concurrent_directory_manager)
def _get_task(self):
return tasks.REALS_TASK
def _get_workunit_builder(self, parser, progress_manager):
return RealsWorkUnitBuilder(parser, self.context, self.context,
progress_manager)
def _get_working_dir(self):
return self.get_abs_path(FRESH_TEST_DIR)
def get_directory_to_clean(self):
return self._get_working_dir()
def get_files_to_keep(self):
return TEST_FILES
def test_record_progress_only_after_source_finished(self):
first_file = self.model.get_current_filename()
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
has_length(0))
self.model.set_current_source_name(TEST_OBJECT_NAME)
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
has_length(0))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
has_length(0))
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0))
def test_record_progress_multiple_sources(self):
first_file = self.model.get_current_filename()
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
has_length(0))
self.model.set_current_source_name(TEST_OBJECT_NAME)
# source 1 of 3, reading 1 of 3
self.model.accept_current_item()
self.model.next_item()
# source 1 of 3 reading 2 of 3
self.model.accept_current_item()
self.model.next_item()
# source 1 of 3 reading 3 of 3
self.model.accept_current_item()
self.model.next_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0))
self.model.set_current_source_name(TEST_OBJECT_NAME)
# source 2 of 3 reading 1 of 3
self.model.accept_current_item()
self.model.next_item()
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0))
# source 2 of 3 reading 2 of 3
self.model.accept_current_item()
self.model.next_item()
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0))
# source 2 of 3 reading 3 of 3
self.model.accept_current_item()
self.model.next_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(False))
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1))
self.model.set_current_source_name(TEST_OBJECT_NAME)
# source 3 of 3 reading 1 of 3
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1))
# source 3 of 3 reading 2 of 3
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1))
# source 3 of 3 reading 3 of 3
self.model.next_item()
self.model.accept_current_item()
assert_that(self.concurrent_progress_manager.is_done(first_file),
equal_to(True))
assert_that(self.concurrent_progress_manager.get_processed_indices(first_file),
contains_inanyorder(0, 1, 2))
def test_file_processed_event(self):
observer = Mock()
events.subscribe(events.FINISHED_WORKUNIT, observer.on_file_processed)
workunit = self.model.get_current_workunit()
accepts_before_next_file = 9
while accepts_before_next_file > 1:
self.model.set_current_source_name(TEST_OBJECT_NAME)
self.model.accept_current_item()
self.model.next_item()
assert_that(observer.on_file_processed.call_count, equal_to(0))
accepts_before_next_file -= 1
self.model.accept_current_item()
assert_that(observer.on_file_processed.call_count, equal_to(1))
# Make sure it was triggered with the right data
args = observer.on_file_processed.call_args[0]
assert_that(args, has_length(1))
msg = args[0]
assert_that(msg.topic, equal_to(events.FINISHED_WORKUNIT))
assert_that(msg.data, equal_to(workunit.get_results_file_paths()))
def test_unlock_on_exit(self):
current_file = self.model.get_current_filename()
assert_that(self.progress_manager.owns_lock(current_file), equal_to(True))
self.model.exit()
assert_that(self.progress_manager.owns_lock(current_file), equal_to(False))
def setUp(self):
self.working_directory = self.get_abs_path(WD_HAS_PROGRESS)
directory_manager = LocalDirectoryWorkingContext(
self.working_directory)
self.progress_manager = LocalProgressManager(directory_manager)
class ProgressManagerFreshDirectoryTest(FileReadingTestCase):
def setUp(self):
self.main_user_id = "main_user"
self.working_directory = self.get_abs_path(WD_NO_LOG)
directory_manager = LocalDirectoryWorkingContext(self.working_directory)
self.progress_manager = LocalProgressManager(directory_manager,
userid=self.main_user_id)
def create_concurrent_progress_manager(self, userid="test_user"):
directory_manager = LocalDirectoryWorkingContext(self.working_directory)
return LocalProgressManager(directory_manager, userid=userid)
def tearDown(self):
# Get rid of generated files so we don't interfere with other tests
self.progress_manager.clean()
def test_load_progress_no_logs(self):
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
has_length(0))
def test_record_done_requires_lock(self):
self.assertRaises(
RequiresLockException,
self.progress_manager.record_done, "xxx2.reals.astrom")
def test_write_progress(self):
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
has_length(0))
processed1 = "xxx2.reals.astrom"
self.progress_manager.lock(processed1)
self.progress_manager.record_done(processed1)
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
contains(processed1))
assert_that(self.progress_manager.is_done(processed1), equal_to(True))
# Create a second persistence manager and make sure it sees the changes
manager2 = self.create_concurrent_progress_manager()
assert_that(manager2.get_done(tasks.CANDS_TASK), has_length(0))
assert_that(manager2.get_done(tasks.REALS_TASK), contains(processed1))
assert_that(manager2.is_done(processed1), equal_to(True))
def test_write_progress_two_simultaneous_managers(self):
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
has_length(0))
processed1 = "xxx2.reals.astrom"
self.progress_manager.lock(processed1)
self.progress_manager.record_done(processed1)
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
contains(processed1))
# Create a second simultaneous manager
manager2 = self.create_concurrent_progress_manager()
processed2 = "xxx3.reals.astrom"
manager2.lock(processed2)
manager2.record_done(processed2)
# Make sure second manager sees both entries
assert_that(manager2.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(manager2.get_done(tasks.REALS_TASK),
contains_inanyorder(processed1, processed2))
# Make sure original manager sees both entries
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
contains_inanyorder(processed1, processed2))
def test_lock_file(self):
file1 = "xxx1.cands.astrom"
self.progress_manager.lock(file1)
# No-one else should be able to acquire the lock...
lock_requesting_user = "******"
manager2 = self.create_concurrent_progress_manager(lock_requesting_user)
self.assertRaises(FileLockedException, manager2.lock, file1)
# ... until we unlock it
self.progress_manager.unlock(file1)
manager2.lock(file1)
def test_lock_has_locker_id(self):
lock_holding_user = self.main_user_id
lock_requesting_user = "******"
file1 = "xxx1.cands.astrom"
self.progress_manager.lock(file1)
manager2 = self.create_concurrent_progress_manager(lock_requesting_user)
try:
manager2.lock(file1)
self.fail("Should have thrown FileLockedExcecption")
except FileLockedException as ex:
assert_that(ex.filename, equal_to(file1))
assert_that(ex.locker, equal_to(lock_holding_user))
def test_record_index_requires_lock(self):
self.assertRaises(RequiresLockException,
self.progress_manager.record_index,
"xxx1.cands.astrom", 0)
def test_record_index(self):
file1 = "xxx1.cands.astrom"
self.progress_manager.lock(file1)
self.progress_manager.record_index(file1, 1)
self.progress_manager.record_index(file1, 3)
self.progress_manager.record_index(file1, 0)
assert_that(self.progress_manager.get_processed_indices(file1),
contains_inanyorder(1, 3, 0))
# Check they are still recorded after we release lock
self.progress_manager.unlock(file1)
assert_that(self.progress_manager.get_processed_indices(file1),
contains_inanyorder(1, 3, 0))
# Check other clients can read them
manager2 = self.create_concurrent_progress_manager()
assert_that(manager2.get_processed_indices(file1),
contains_inanyorder(1, 3, 0))
def test_unlock_after_record_done_no_error(self):
file1 = "xxx1.cands.astrom"
self.progress_manager.lock(file1)
self.progress_manager.record_done(file1)
self.progress_manager.unlock(file1)
def test_record_done_does_not_unlock_all(self):
file1 = "xxx1.cands.astrom"
file2 = "xxx2.cands.astrom"
manager2 = self.create_concurrent_progress_manager()
self.progress_manager.lock(file1)
manager2.lock(file2)
self.progress_manager.record_done(file1)
assert_that(manager2.owns_lock(file2), equal_to(True))
manager2.unlock(file2)
assert_that(manager2.owns_lock(file2), equal_to(False))
def test_get_processed_indices_empty_should_not_cause_error(self):
assert_that(self.progress_manager.get_processed_indices("xxx1.cands.astrom"),
has_length(0))
def test_get_processed_indices_after_done(self):
filename = "xxx1.cands.astrom"
self.progress_manager.lock(filename)
self.progress_manager.record_index(filename, 0)
self.progress_manager.record_index(filename, 1)
self.progress_manager.record_index(filename, 2)
self.progress_manager.unlock(filename)
assert_that(self.progress_manager.get_processed_indices(filename),
contains_inanyorder(0, 1, 2))
self.progress_manager.lock(filename)
self.progress_manager.record_done(filename)
self.progress_manager.unlock(filename)
assert_that(self.progress_manager.get_processed_indices(filename),
contains_inanyorder(0, 1, 2))
# Double check with a second manager
assert_that(self.create_concurrent_progress_manager().get_processed_indices(filename),
contains_inanyorder(0, 1, 2))
def setUp(self):
self.working_directory = self.get_abs_path(WD_HAS_PROGRESS)
directory_manager = LocalDirectoryWorkingContext(self.working_directory)
self.progress_manager = LocalProgressManager(directory_manager)
class ProgressManagerFreshDirectoryTest(FileReadingTestCase):
def setUp(self):
self.main_user_id = "main_user"
self.working_directory = self.get_abs_path(WD_NO_LOG)
directory_manager = LocalDirectoryWorkingContext(
self.working_directory)
self.progress_manager = LocalProgressManager(directory_manager,
userid=self.main_user_id)
def create_concurrent_progress_manager(self, userid="test_user"):
directory_manager = LocalDirectoryWorkingContext(
self.working_directory)
return LocalProgressManager(directory_manager, userid=userid)
def tearDown(self):
# Get rid of generated files so we don't interfere with other tests
self.progress_manager.clean()
def test_load_progress_no_logs(self):
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
has_length(0))
def test_record_done_requires_lock(self):
self.assertRaises(RequiresLockException,
self.progress_manager.record_done,
"xxx2.reals.astrom")
def test_write_progress(self):
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
has_length(0))
processed1 = "xxx2.reals.astrom"
self.progress_manager.lock(processed1)
self.progress_manager.record_done(processed1)
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
contains(processed1))
assert_that(self.progress_manager.is_done(processed1), equal_to(True))
# Create a second persistence manager and make sure it sees the changes
manager2 = self.create_concurrent_progress_manager()
assert_that(manager2.get_done(tasks.CANDS_TASK), has_length(0))
assert_that(manager2.get_done(tasks.REALS_TASK), contains(processed1))
assert_that(manager2.is_done(processed1), equal_to(True))
def test_write_progress_two_simultaneous_managers(self):
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
has_length(0))
processed1 = "xxx2.reals.astrom"
self.progress_manager.lock(processed1)
self.progress_manager.record_done(processed1)
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
contains(processed1))
# Create a second simultaneous manager
manager2 = self.create_concurrent_progress_manager()
processed2 = "xxx3.reals.astrom"
manager2.lock(processed2)
manager2.record_done(processed2)
# Make sure second manager sees both entries
assert_that(manager2.get_done(tasks.CANDS_TASK), has_length(0))
assert_that(manager2.get_done(tasks.REALS_TASK),
contains_inanyorder(processed1, processed2))
# Make sure original manager sees both entries
assert_that(self.progress_manager.get_done(tasks.CANDS_TASK),
has_length(0))
assert_that(self.progress_manager.get_done(tasks.REALS_TASK),
contains_inanyorder(processed1, processed2))
def test_lock_file(self):
file1 = "xxx1.cands.astrom"
self.progress_manager.lock(file1)
# No-one else should be able to acquire the lock...
lock_requesting_user = "******"
manager2 = self.create_concurrent_progress_manager(
lock_requesting_user)
self.assertRaises(FileLockedException, manager2.lock, file1)
# ... until we unlock it
self.progress_manager.unlock(file1)
manager2.lock(file1)
def test_lock_has_locker_id(self):
lock_holding_user = self.main_user_id
lock_requesting_user = "******"
file1 = "xxx1.cands.astrom"
self.progress_manager.lock(file1)
manager2 = self.create_concurrent_progress_manager(
lock_requesting_user)
try:
manager2.lock(file1)
self.fail("Should have thrown FileLockedExcecption")
except FileLockedException as ex:
assert_that(ex.filename, equal_to(file1))
assert_that(ex.locker, equal_to(lock_holding_user))
def test_record_index_requires_lock(self):
self.assertRaises(RequiresLockException,
self.progress_manager.record_index,
"xxx1.cands.astrom", 0)
def test_record_index(self):
file1 = "xxx1.cands.astrom"
self.progress_manager.lock(file1)
self.progress_manager.record_index(file1, 1)
self.progress_manager.record_index(file1, 3)
self.progress_manager.record_index(file1, 0)
assert_that(self.progress_manager.get_processed_indices(file1),
contains_inanyorder(1, 3, 0))
# Check they are still recorded after we release lock
self.progress_manager.unlock(file1)
assert_that(self.progress_manager.get_processed_indices(file1),
contains_inanyorder(1, 3, 0))
# Check other clients can read them
manager2 = self.create_concurrent_progress_manager()
assert_that(manager2.get_processed_indices(file1),
contains_inanyorder(1, 3, 0))
def test_unlock_after_record_done_no_error(self):
file1 = "xxx1.cands.astrom"
self.progress_manager.lock(file1)
self.progress_manager.record_done(file1)
self.progress_manager.unlock(file1)
def test_record_done_does_not_unlock_all(self):
file1 = "xxx1.cands.astrom"
file2 = "xxx2.cands.astrom"
manager2 = self.create_concurrent_progress_manager()
self.progress_manager.lock(file1)
manager2.lock(file2)
self.progress_manager.record_done(file1)
assert_that(manager2.owns_lock(file2), equal_to(True))
manager2.unlock(file2)
assert_that(manager2.owns_lock(file2), equal_to(False))
def test_get_processed_indices_empty_should_not_cause_error(self):
assert_that(
self.progress_manager.get_processed_indices("xxx1.cands.astrom"),
has_length(0))
def test_get_processed_indices_after_done(self):
filename = "xxx1.cands.astrom"
self.progress_manager.lock(filename)
self.progress_manager.record_index(filename, 0)
self.progress_manager.record_index(filename, 1)
self.progress_manager.record_index(filename, 2)
self.progress_manager.unlock(filename)
assert_that(self.progress_manager.get_processed_indices(filename),
contains_inanyorder(0, 1, 2))
self.progress_manager.lock(filename)
self.progress_manager.record_done(filename)
self.progress_manager.unlock(filename)
assert_that(self.progress_manager.get_processed_indices(filename),
contains_inanyorder(0, 1, 2))
# Double check with a second manager
assert_that(
self.create_concurrent_progress_manager().get_processed_indices(
filename), contains_inanyorder(0, 1, 2))
def get_progress_manager(self):
return LocalProgressManager(self, userid=self.userid)
def create_concurrent_progress_manager(self, userid="test_user"):
directory_manager = LocalDirectoryWorkingContext(
self.working_directory)
return LocalProgressManager(directory_manager, userid=userid)