def test_downloaded_file_with_changed_remote_size_is_queued(self): # Disable auto-extract self.context.config.autoqueue.auto_extract = False persist = AutoQueuePersist() persist.add_pattern(AutoQueuePattern(pattern="File.One")) # noinspection PyTypeChecker auto_queue = AutoQueue(self.context, persist, self.controller) file_one = ModelFile("File.One", True) file_one.remote_size = 100 file_one.local_size = 100 file_one.state = ModelFile.State.DOWNLOADED self.model_listener.file_added(file_one) auto_queue.process() self.controller.queue_command.assert_not_called() file_one_updated = ModelFile("File.One", True) file_one_updated.remote_size = 200 file_one_updated.local_size = 100 file_one_updated.state = ModelFile.State.DEFAULT self.model_listener.file_updated(file_one, file_one_updated) auto_queue.process() self.controller.queue_command.assert_called_once_with( unittest.mock.ANY) command = self.controller.queue_command.call_args[0][0] self.assertEqual(Controller.Command.Action.QUEUE, command.action) self.assertEqual("File.One", command.filename)
def test_partial_file_is_auto_queued_after_remote_discovery(self): # Test that a partial local file is auto-queued when discovered on remote some time later persist = AutoQueuePersist() persist.add_pattern(AutoQueuePattern(pattern="File.One")) # noinspection PyTypeChecker auto_queue = AutoQueue(self.context, persist, self.controller) # Local discovery file_one = ModelFile("File.One", True) file_one.local_size = 100 self.model_listener.file_added(file_one) auto_queue.process() self.controller.queue_command.assert_not_called() # Remote discovery file_one_new = ModelFile("File.One", True) file_one_new.local_size = 100 file_one_new.remote_size = 200 self.model_listener.file_updated(file_one, file_one_new) auto_queue.process() self.controller.queue_command.assert_called_once_with( unittest.mock.ANY) command = self.controller.queue_command.call_args[0][0] self.assertEqual(Controller.Command.Action.QUEUE, command.action) self.assertEqual("File.One", command.filename)
def test_non_extractable_files_are_not_extracted(self): persist = AutoQueuePersist() file_one = ModelFile("File.One", True) file_one.local_size = 100 file_one.state = ModelFile.State.DOWNLOADED file_one.is_extractable = True file_two = ModelFile("File.Two", True) file_two.local_size = 200 file_two.state = ModelFile.State.DOWNLOADED file_two.is_extractable = False self.initial_model = [file_one, file_two] # noinspection PyTypeChecker auto_queue = AutoQueue(self.context, persist, self.controller) auto_queue.process() self.controller.queue_command.assert_not_called() persist.add_pattern(AutoQueuePattern(pattern="File.One")) auto_queue.process() self.controller.queue_command.assert_called_once_with( unittest.mock.ANY) command = self.controller.queue_command.call_args[0][0] self.assertEqual(Controller.Command.Action.EXTRACT, command.action) self.assertEqual("File.One", command.filename) self.controller.queue_command.reset_mock() persist.add_pattern(AutoQueuePattern(pattern="File.Two")) auto_queue.process() self.controller.queue_command.assert_not_called()
def test_extract_skips_remaining_on_shutdown(self): # Send two extract commands # Call shutdown after first one runs # Check that second command did not run self.mock_is_archive.return_value = True self.call_stop = False def _extract_archive(**kwargs): print(kwargs) self.call_stop = True time.sleep(0.5) # wait a bit so shutdown is called self.mock_extract_archive.side_effect = _extract_archive mf1 = ModelFile("aaa", False) mf1.local_size = 100 mf2 = ModelFile("bbb", False) mf2.local_size = 100 self.dispatch.add_listener(self.listener) self.dispatch.extract(mf1) self.dispatch.extract(mf2) while not self.call_stop: pass self.dispatch.stop() while self.mock_extract_archive.call_count < 1 \ or self.listener.extract_completed.call_count < 1: pass self.assertEqual(1, self.mock_extract_archive.call_count) self.listener.extract_completed.assert_called_once_with("aaa", False) self.listener.extract_failed.assert_not_called()
def test_extract_maintains_order(self): self.mock_is_archive.return_value = True mf1 = ModelFile("aaa", False) mf1.local_size = 100 mf2 = ModelFile("bbb", False) mf2.local_size = 100 mf3 = ModelFile("ccc", False) mf3.local_size = 100 self.dispatch.extract(mf1) self.dispatch.extract(mf2) self.dispatch.extract(mf3) while self.mock_extract_archive.call_count < 3: pass self.assertEqual(3, self.mock_extract_archive.call_count) args_list = self.mock_extract_archive.call_args_list self.assertEqual(args_list, [ call(archive_path=os.path.join(self.local_path, "aaa"), out_dir_path=self.out_dir_path), call(archive_path=os.path.join(self.local_path, "bbb"), out_dir_path=self.out_dir_path), call(archive_path=os.path.join(self.local_path, "ccc"), out_dir_path=self.out_dir_path) ])
def test_update_event_files(self): serialize = SerializeModel() a1 = ModelFile("a", False) a1.local_size = 100 a2 = ModelFile("a", False) a2.local_size = 200 out = parse_stream( serialize.update_event( SerializeModel.UpdateEvent( SerializeModel.UpdateEvent.Change.UPDATED, a1, a2))) data = json.loads(out["data"]) self.assertEqual("a", data["old_file"]["name"]) self.assertEqual(100, data["old_file"]["local_size"]) self.assertEqual("a", data["new_file"]["name"]) self.assertEqual(200, data["new_file"]["local_size"]) out = parse_stream( serialize.update_event( SerializeModel.UpdateEvent( SerializeModel.UpdateEvent.Change.ADDED, None, a1))) data = json.loads(out["data"]) self.assertEqual(None, data["old_file"]) self.assertEqual("a", data["new_file"]["name"]) self.assertEqual(100, data["new_file"]["local_size"]) out = parse_stream( serialize.update_event( SerializeModel.UpdateEvent( SerializeModel.UpdateEvent.Change.ADDED, a2, None))) data = json.loads(out["data"]) self.assertEqual("a", data["old_file"]["name"]) self.assertEqual(200, data["old_file"]["local_size"]) self.assertEqual(None, data["new_file"])
def test_downloaded_file_is_NOT_re_extracted_after_modified(self): persist = AutoQueuePersist() persist.add_pattern(AutoQueuePattern(pattern="File.One")) # noinspection PyTypeChecker auto_queue = AutoQueue(self.context, persist, self.controller) # File is auto-extracted file_one = ModelFile("File.One", True) file_one.local_size = 100 file_one.state = ModelFile.State.DOWNLOADED file_one.is_extractable = True self.model_listener.file_added(file_one) auto_queue.process() self.controller.queue_command.assert_called_once_with( unittest.mock.ANY) command = self.controller.queue_command.call_args[0][0] self.assertEqual(Controller.Command.Action.EXTRACT, command.action) self.assertEqual("File.One", command.filename) self.controller.queue_command.reset_mock() # File is modified file_one_new = ModelFile("File.One", True) file_one_new.local_size = 101 file_one_new.state = ModelFile.State.DOWNLOADED file_one_new.is_extractable = True self.model_listener.file_updated(file_one, file_one_new) auto_queue.process() self.controller.queue_command.assert_not_called()
def test_updated(self): model_before = Model() model_after = Model() a1 = ModelFile("a", False) a1.local_size = 100 a2 = ModelFile("a", False) a2.local_size = 200 model_before.add_file(a1) model_after.add_file(a2) diff = ModelDiffUtil.diff_models(model_before, model_after) self.assertEqual([ModelDiff(ModelDiff.Change.UPDATED, a1, a2)], diff)
def test_local_size(self): file = ModelFile("test", False) file.local_size = 100 self.assertEqual(100, file.local_size) file.local_size = None self.assertEqual(None, file.local_size) with self.assertRaises(TypeError): file.local_size = "BadValue" with self.assertRaises(ValueError): file.local_size = -100
def test_matching_new_files_are_extracted(self): persist = AutoQueuePersist() persist.add_pattern(AutoQueuePattern(pattern="File.One")) persist.add_pattern(AutoQueuePattern(pattern="File.Two")) persist.add_pattern(AutoQueuePattern(pattern="File.Three")) # noinspection PyTypeChecker auto_queue = AutoQueue(self.context, persist, self.controller) file_one = ModelFile("File.One", True) file_one.state = ModelFile.State.DOWNLOADED file_one.local_size = 100 file_one.is_extractable = True file_two = ModelFile("File.Two", True) file_two.state = ModelFile.State.DOWNLOADED file_two.local_size = 200 file_two.is_extractable = True file_three = ModelFile("File.Three", True) file_three.state = ModelFile.State.DOWNLOADED file_three.local_size = 300 file_three.is_extractable = True self.model_listener.file_added(file_one) auto_queue.process() command = self.controller.queue_command.call_args[0][0] self.assertEqual(Controller.Command.Action.EXTRACT, command.action) self.assertEqual("File.One", command.filename) self.model_listener.file_added(file_two) auto_queue.process() command = self.controller.queue_command.call_args[0][0] self.assertEqual(Controller.Command.Action.EXTRACT, command.action) self.assertEqual("File.Two", command.filename) self.model_listener.file_added(file_three) auto_queue.process() command = self.controller.queue_command.call_args[0][0] self.assertEqual(Controller.Command.Action.EXTRACT, command.action) self.assertEqual("File.Three", command.filename) # All at once self.model_listener.file_added(file_one) self.model_listener.file_added(file_two) self.model_listener.file_added(file_three) auto_queue.process() calls = self.controller.queue_command.call_args_list[-3:] commands = [calls[i][0][0] for i in range(3)] self.assertEqual(set([Controller.Command.Action.EXTRACT] * 3), {c.action for c in commands}) self.assertEqual({"File.One", "File.Two", "File.Three"}, {c.filename for c in commands})
def test_extract_dir(self): self.mock_is_archive.return_value = True self.actual_calls = set() def _extract(archive_path: str, out_dir_path: str): self.actual_calls.add((archive_path, out_dir_path)) self.mock_extract_archive.side_effect = _extract a = ModelFile("a", True) a.local_size = 500 aa = ModelFile("aa", True) aa.local_size = 300 a.add_child(aa) aaa = ModelFile("aaa", False) aaa.local_size = 100 aa.add_child(aaa) aab = ModelFile("aab", False) aab.local_size = 100 aa.add_child(aab) aac = ModelFile("aac", True) aac.local_size = 100 aa.add_child(aac) aaca = ModelFile("aaca", False) aaca.local_size = 100 aac.add_child(aaca) ab = ModelFile("ab", True) ab.local_size = 100 a.add_child(ab) aba = ModelFile("aba", False) aba.local_size = 100 ab.add_child(aba) ac = ModelFile("ac", False) ac.local_size = 100 a.add_child(ac) self.dispatch.add_listener(self.listener) self.dispatch.extract(a) while self.listener.extract_completed.call_count < 1: pass self.listener.extract_completed.assert_called_once_with("a", True) golden_calls = { (os.path.join(self.local_path, "a", "aa", "aaa"), os.path.join(self.out_dir_path, "a", "aa")), (os.path.join(self.local_path, "a", "aa", "aab"), os.path.join(self.out_dir_path, "a", "aa")), (os.path.join(self.local_path, "a", "aa", "aac", "aaca"), os.path.join(self.out_dir_path, "a", "aa", "aac")), (os.path.join(self.local_path, "a", "ab", "aba"), os.path.join(self.out_dir_path, "a", "ab")), (os.path.join(self.local_path, "a", "ac"), os.path.join(self.out_dir_path, "a")), } self.assertEqual(5, self.mock_extract_archive.call_count) self.assertEqual(golden_calls, self.actual_calls)
def test_extract_single_raises_error_on_remote_only_file(self): mf = ModelFile("aaa", False) mf.local_size = None with self.assertRaises(ExtractDispatchError) as ctx: self.dispatch.extract(mf) self.assertTrue( str(ctx.exception).startswith("File does not exist locally")) mf = ModelFile("aaa", False) mf.local_size = 0 with self.assertRaises(ExtractDispatchError) as ctx: self.dispatch.extract(mf) self.assertTrue( str(ctx.exception).startswith("File does not exist locally"))
def test_equality_operator(self): # check that timestamp does not affect equality now = datetime.now() file1 = ModelFile("test", False) file1.local_size = 100 file1.update_timestamp = now file2 = ModelFile("test", False) file2.local_size = 200 file2.update_timestamp = now self.assertFalse(file1 == file2) file2.local_size = 100 file2.update_timestamp = datetime.now() self.assertTrue(file1 == file2)
def test_listener_file_updated(self): listener = DummyModelListener() self.model.add_listener(listener) listener.file_updated = MagicMock() old_file = ModelFile("test", False) old_file.local_size = 100 self.model.add_file(old_file) new_file = ModelFile("test", False) new_file.local_size = 200 self.model.update_file(new_file) # noinspection PyUnresolvedReferences listener.file_updated.assert_called_once_with(old_file, new_file)
def test_local_size(self): serialize = SerializeModel() a = ModelFile("a", True) a.local_size = None b = ModelFile("b", False) b.local_size = 0 c = ModelFile("c", True) c.local_size = 100 files = [a, b, c] out = parse_stream(serialize.model(files)) data = json.loads(out["data"]) self.assertEqual(3, len(data)) self.assertEqual(None, data[0]["local_size"]) self.assertEqual(0, data[1]["local_size"]) self.assertEqual(100, data[2]["local_size"])
def test_extract_ignores_duplicate_calls(self): # Send two extract commands to same file # Expect that only one extract operation is performed self.mock_is_archive.return_value = True self.barrier = False def _extract_archive(**kwargs): print(kwargs) while not self.barrier: pass self.mock_extract_archive.side_effect = _extract_archive a = ModelFile("a", False) a.local_size = 200 self.dispatch.add_listener(self.listener) self.dispatch.extract(a) self.dispatch.extract(a) time.sleep(0.1) self.barrier = True time.sleep(0.1) while self.mock_extract_archive.call_count < 1 and \ self.listener.extract_completed.call_count < 1: pass time.sleep(0.1) self.listener.extract_completed.assert_called_once_with("a", False) self.listener.extract_failed.assert_not_called() self.assertEqual(1, self.mock_extract_archive.call_count)
def test_new_file(self): new_file = ModelFile("a", False) new_file.local_size = 100 diff = ModelDiff(ModelDiff.Change.ADDED, None, new_file) self.assertEqual(new_file, diff.new_file) diff = ModelDiff(ModelDiff.Change.ADDED, None, None) self.assertEqual(None, diff.new_file)
def test_old_file(self): old_file = ModelFile("a", False) old_file.local_size = 100 diff = ModelDiff(ModelDiff.Change.ADDED, old_file, None) self.assertEqual(old_file, diff.old_file) diff = ModelDiff(ModelDiff.Change.ADDED, None, None) self.assertEqual(None, diff.old_file)
def test_all_files_are_extracted_when_patterns_only_disabled(self): self.context.config.autoqueue.patterns_only = False persist = AutoQueuePersist() persist.add_pattern(AutoQueuePattern(pattern="File.One")) persist.add_pattern(AutoQueuePattern(pattern="File.Two")) persist.add_pattern(AutoQueuePattern(pattern="File.Three")) file_one = ModelFile("File.One", True) file_one.local_size = 100 file_one.state = ModelFile.State.DOWNLOADED file_one.is_extractable = True file_two = ModelFile("File.Two", True) file_two.local_size = 200 file_two.state = ModelFile.State.DOWNLOADED file_two.is_extractable = True file_three = ModelFile("File.Three", True) file_three.local_size = 300 file_three.state = ModelFile.State.DOWNLOADED file_three.is_extractable = True file_four = ModelFile("File.Four", True) file_four.local_size = 400 file_four.state = ModelFile.State.DOWNLOADED file_four.is_extractable = True file_five = ModelFile("File.Five", True) file_five.local_size = 500 file_five.state = ModelFile.State.DOWNLOADED file_five.is_extractable = True self.initial_model = [ file_one, file_two, file_three, file_four, file_five ] # noinspection PyTypeChecker auto_queue = AutoQueue(self.context, persist, self.controller) auto_queue.process() calls = self.controller.queue_command.call_args_list self.assertEqual(5, len(calls)) commands = [calls[i][0][0] for i in range(5)] self.assertEqual(set([Controller.Command.Action.EXTRACT] * 5), {c.action for c in commands}) self.assertEqual( {"File.One", "File.Two", "File.Three", "File.Four", "File.Five"}, {c.filename for c in commands})
def test_added(self): model_before = Model() model_after = Model() a = ModelFile("a", False) a.local_size = 100 model_after.add_file(a) diff = ModelDiffUtil.diff_models(model_before, model_after) self.assertEqual([ModelDiff(ModelDiff.Change.ADDED, None, a)], diff)
def test_removed(self): model_before = Model() model_after = Model() a = ModelFile("a", False) a.local_size = 100 model_before.add_file(a) diff = ModelDiffUtil.diff_models(model_before, model_after) self.assertEqual([ModelDiff(ModelDiff.Change.REMOVED, a, None)], diff)
def test_extract_dir_raises_error_on_no_archives(self): self.mock_is_archive.return_value = False a = ModelFile("a", True) a.local_size = 100 aa = ModelFile("aa", False) aa.local_size = 50 a.add_child(aa) ab = ModelFile("ab", False) ab.local_size = 50 a.add_child(ab) with self.assertRaises(ExtractDispatchError) as ctx: self.dispatch.extract(a) self.assertTrue( str(ctx.exception).startswith( "Directory does not contain any archives"))
def test_matching_downloading_files_are_not_queued(self): persist = AutoQueuePersist() persist.add_pattern(AutoQueuePattern(pattern="File.One")) # noinspection PyTypeChecker auto_queue = AutoQueue(self.context, persist, self.controller) file_one = ModelFile("File.One", True) file_one.remote_size = 100 file_one.local_size = 0 file_one.state = ModelFile.State.DOWNLOADING self.model_listener.file_added(file_one) auto_queue.process() self.controller.queue_command.assert_not_called() file_one_new = ModelFile("File.One", True) file_one_new.remote_size = 100 file_one_new.local_size = 50 file_one_new.state = ModelFile.State.DOWNLOADING self.model_listener.file_updated(file_one, file_one_new) auto_queue.process() self.controller.queue_command.assert_not_called()
def test_update_file(self): file = ModelFile("test", False) file.local_size = 100 self.model.add_file(file) recv_file = self.model.get_file("test") self.assertEqual(100, recv_file.local_size) recv_file.local_size = 200 self.model.update_file(recv_file) recv_file = self.model.get_file("test") self.assertEqual(200, recv_file.local_size)
def test_extract_calls_listeners_in_correct_sequence(self): self.mock_is_archive.return_value = True self.count = 0 # noinspection PyUnusedLocal def _extract_archive(**kwargs): # raise error for first and third extractions self.count += 1 if self.count in (1, 3): raise ExtractError() self.mock_extract_archive.side_effect = _extract_archive mf1 = ModelFile("aaa", False) mf1.local_size = 100 mf2 = ModelFile("bbb", False) mf2.local_size = 100 mf3 = ModelFile("ccc", False) mf3.local_size = 100 listener_calls = [] def _completed(name, is_dir): listener_calls.append((True, name, is_dir)) def _failed(name, is_dir): listener_calls.append((False, name, is_dir)) self.listener.extract_completed.side_effect = _completed self.listener.extract_failed.side_effect = _failed self.dispatch.add_listener(self.listener) self.dispatch.extract(mf1) self.dispatch.extract(mf2) self.dispatch.extract(mf3) while self.mock_extract_archive.call_count < 3 \ or self.listener.extract_failed.call_count < 2 \ or self.listener.extract_completed.call_count < 1: pass self.assertEqual(3, self.mock_extract_archive.call_count) self.assertEqual([(False, "aaa", False), (True, "bbb", False), (False, "ccc", False)], listener_calls)
def test_no_files_are_extracted_when_auto_extract_disabled(self): self.context.config.autoqueue.enabled = True self.context.config.autoqueue.auto_extract = False persist = AutoQueuePersist() persist.add_pattern(AutoQueuePattern(pattern="File.One")) persist.add_pattern(AutoQueuePattern(pattern="File.Two")) persist.add_pattern(AutoQueuePattern(pattern="File.Three")) file_one = ModelFile("File.One", True) file_one.local_size = 100 file_one.state = ModelFile.State.DOWNLOADED file_two = ModelFile("File.Two", True) file_two.local_size = 200 file_two.state = ModelFile.State.DOWNLOADED file_three = ModelFile("File.Three", True) file_three.local_size = 300 file_three.state = ModelFile.State.DOWNLOADED file_four = ModelFile("File.Four", True) file_four.local_size = 400 file_four.state = ModelFile.State.DOWNLOADED file_five = ModelFile("File.Five", True) file_five.local_size = 500 file_five.state = ModelFile.State.DOWNLOADED self.initial_model = [ file_one, file_two, file_three, file_four, file_five ] # First with patterns_only ON self.context.config.autoqueue.patterns_only = True # noinspection PyTypeChecker auto_queue = AutoQueue(self.context, persist, self.controller) auto_queue.process() self.controller.queue_command.assert_not_called() # Second with patterns_only OFF self.context.config.autoqueue.patterns_only = False # noinspection PyTypeChecker auto_queue = AutoQueue(self.context, persist, self.controller) auto_queue.process() self.controller.queue_command.assert_not_called()
def __fill_model_file( _model_file: ModelFile, _remote: Optional[SystemFile], _local: Optional[SystemFile], _transfer_state: Optional[LftpJobStatus.TransferState]): # set local and remote sizes if _remote: _model_file.remote_size = _remote.size if _local: _model_file.local_size = _local.size # Note: no longer use lftp's file sizes # they represent remaining size for resumed downloads # set the downloading speed and eta if _transfer_state: _model_file.downloading_speed = _transfer_state.speed _model_file.eta = _transfer_state.eta # set the transferred size (only if file or dir exists on both ends) if _local and _remote: if _model_file.is_dir: # dir transferred size is updated by child files _model_file.transferred_size = 0 else: _model_file.transferred_size = min( _local.size, _remote.size) # also update all parent directories _parent_file = _model_file.parent while _parent_file is not None: _parent_file.transferred_size += _model_file.transferred_size _parent_file = _parent_file.parent # set the is_extractable flag if not _model_file.is_dir and Extract.is_archive_fast( _model_file.name): _model_file.is_extractable = True # Also set the flag for all of its parents _parent_file = _model_file.parent while _parent_file is not None: _parent_file.is_extractable = True _parent_file = _parent_file.parent # set the timestamps if _local: if _local.timestamp_created: _model_file.local_created_timestamp = _local.timestamp_created if _local.timestamp_modified: _model_file.local_modified_timestamp = _local.timestamp_modified if _remote: if _remote.timestamp_created: _model_file.remote_created_timestamp = _remote.timestamp_created if _remote.timestamp_modified: _model_file.remote_modified_timestamp = _remote.timestamp_modified
def test_matching_local_files_are_not_queued(self): persist = AutoQueuePersist() persist.add_pattern(AutoQueuePattern(pattern="File.One")) # noinspection PyTypeChecker auto_queue = AutoQueue(self.context, persist, self.controller) file_one = ModelFile("File.One", True) file_one.remote_size = None file_one.local_size = 100 self.model_listener.file_added(file_one) auto_queue.process() self.controller.queue_command.assert_not_called()
def test_extract_single_raises_error_on_bad_archive(self): self.mock_is_archive.return_value = False mf = ModelFile("aaa", False) mf.local_size = 100 with self.assertRaises(ExtractDispatchError) as ctx: self.dispatch.extract(mf) self.assertTrue( str(ctx.exception).startswith("File is not an archive")) self.mock_is_archive.assert_called_once_with( os.path.join(self.local_path, mf.name))
def test_extract_single(self): self.mock_is_archive.return_value = True mf = ModelFile("aaa", False) mf.local_size = 100 self.dispatch.extract(mf) while self.mock_extract_archive.call_count < 1: pass self.mock_extract_archive.assert_called_once_with( archive_path=os.path.join(self.local_path, "aaa"), out_dir_path=self.out_dir_path)