def setUp(self):
self.maxDiff = None
self.data = [i for i in range(30)]
self.temp_directory = write_data_to_files_in_temp_directory(
self.data, 10, file_prefix=TestSynchronisedFilesDataSource._FILE_PREFIX)
def extract_adapter(file_path: str) -> Any:
return extract_data_from_file(file_path, parser=lambda data: int(data), separator='\n')
def is_test_file(file_path: str) -> bool:
return TestSynchronisedFilesDataSource._FILE_PREFIX in file_path
self.source = StubSynchronisedInFileDataSource(self.temp_directory)
self.source.is_data_file = MagicMock(side_effect=is_test_file)
self.source.extract_data_from_file = MagicMock(side_effect=extract_adapter)
class TestSynchronisedFilesDataSource(unittest.TestCase):
"""
Tests for `SynchronisedFilesDataSource`.
"""
_FILE_PREFIX = "test_file"
def setUp(self):
self.maxDiff = None
self.data = [i for i in range(30)]
self.temp_directory = write_data_to_files_in_temp_directory(
self.data, 10, file_prefix=TestSynchronisedFilesDataSource._FILE_PREFIX)
def extract_adapter(file_path: str) -> Any:
return extract_data_from_file(file_path, parser=lambda data: int(data), separator='\n')
def is_test_file(file_path: str) -> bool:
return TestSynchronisedFilesDataSource._FILE_PREFIX in file_path
self.source = StubSynchronisedInFileDataSource(self.temp_directory)
self.source.is_data_file = MagicMock(side_effect=is_test_file)
self.source.extract_data_from_file = MagicMock(side_effect=extract_adapter)
def test_start_if_started(self):
self.source.start()
self.assertRaises(RuntimeError, self.source.start)
def test_start_after_stop(self):
self.source.start()
self.source.stop()
self.source.start()
def test_get_all_when_never_started(self):
self.assertRaises(RuntimeError, self.source.get_all)
def test_get_all_when_stopped(self):
self.source.start()
self.source.stop()
self.assertRaises(RuntimeError, self.source.get_all)
def test_get_all_when_changed_on_restart(self):
self.source.start()
self.assertCountEqual(self.source.get_all(), self.data)
self.source.stop()
for file_path in glob.iglob("%s/*" % self.temp_directory):
os.remove(file_path)
self.source.start()
self.assertEqual(len(self.source.get_all()), 0)
def test_get_all_when_file_created(self):
self.source.start()
block_until_synchronised_files_data_source_started(self.source)
change_trigger = Semaphore(0)
def on_change(change: FileSystemChange):
if change == FileSystemChange.CREATE:
change_trigger.release()
self.source.add_listener(on_change)
more_data = [i for i in range(50)]
write_data_to_files_in_temp_directory(more_data, 10, dir=self.temp_directory,
file_prefix=TestSynchronisedFilesDataSource._FILE_PREFIX)
even_more_data = self._add_more_data_in_nested_directory(10)[1]
triggers = 0
while triggers != 20:
change_trigger.acquire()
triggers += 1
logging.debug(self.source._origin_mapped_data)
self.assertCountEqual(self.source.get_all(), self.data + more_data + even_more_data)
def test_get_all_when_file_deleted(self):
self.source.start()
block_until_synchronised_files_data_source_started(self.source)
change_lock = Lock()
change_lock.acquire()
def on_change(change: FileSystemChange):
if change == FileSystemChange.DELETE:
change_lock.release()
self.source.add_listener(on_change)
to_delete_file_path = glob.glob("%s/*" % self.temp_directory)[0]
deleted_data = extract_data_from_file(to_delete_file_path, parser=lambda data: int(data), separator='\n')
os.remove(to_delete_file_path)
change_lock.acquire()
self.assertCountEqual(self.source.get_all(), [x for x in self.data if x not in deleted_data])
def test_get_all_when_folder_containing_files_is_deleted(self):
nested_directory_path = self._add_more_data_in_nested_directory()[0]
self.source.start()
block_until_synchronised_files_data_source_started(self.source)
change_lock = Lock()
change_lock.acquire()
def on_change(change: FileSystemChange):
if change == FileSystemChange.DELETE:
change_lock.release()
self.source.add_listener(on_change)
shutil.rmtree(nested_directory_path)
change_lock.acquire()
self.assertCountEqual(self.source.get_all(), self.data)
def test_get_all_when_file_modified(self):
self.source.start()
block_until_synchronised_files_data_source_started(self.source)
change_lock = Lock()
change_lock.acquire()
def on_change(change: FileSystemChange):
if change == FileSystemChange.MODIFY:
change_lock.release()
self.source.add_listener(on_change)
to_modify_file_path = glob.glob("%s/*" % self.temp_directory)[0]
to_modify = extract_data_from_file(to_modify_file_path, parser=lambda data: int(data), separator='\n')
modified = to_modify[0: -1]
with open(to_modify_file_path, 'w') as file:
file.write('\n'.join([str(x) for x in modified]))
change_lock.acquire()
self.assertCountEqual(self.source.get_all(), [x for x in self.data if x not in to_modify] + modified)
def test_get_all_when_file_moved(self):
self.source.start()
block_until_synchronised_files_data_source_started(self.source)
move_semaphore = Semaphore(0)
deleted = False
def on_change(change: FileSystemChange):
nonlocal deleted
if change == FileSystemChange.DELETE:
move_semaphore.release()
deleted = True
if deleted and change == FileSystemChange.CREATE:
move_semaphore.release()
self.source.add_listener(on_change)
to_move_file_path = glob.glob("%s/*" % self.temp_directory)[0]
move_to = "%s_moved" % to_move_file_path
shutil.move(to_move_file_path, move_to)
move_semaphore.acquire()
move_semaphore.acquire()
self.assertCountEqual(self.source.get_all(), self.data)
def _add_more_data_in_nested_directory(self, number_of_extra_files: int=1) -> Tuple[str, List[int]]:
"""
Adds more data in a directory nested inside the temp directory.
:param number_of_extra_files: (optional) the number of files to put the new data in inside the nested directory
:return: a tuple where the first value is the path to the new nested directory and the second is the new data
"""
nested_directory_path = os.path.join(self.temp_directory, "nested")
os.makedirs(nested_directory_path)
more_data = [i for i in range(50)]
write_data_to_files_in_temp_directory(more_data, number_of_extra_files, dir=nested_directory_path,
file_prefix=TestSynchronisedFilesDataSource._FILE_PREFIX)
return (nested_directory_path, more_data)
def tearDown(self):
self.source.stop()
shutil.rmtree(self.temp_directory)
