def test_non_extractable_files_are_not_extracted(self):
persist = AutoQueuePersist()
file_one = ModelFile("File.One", True)
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
file_one.is_extractable = True
file_two = ModelFile("File.Two", True)
file_two.local_size = 200
file_two.state = ModelFile.State.DOWNLOADED
file_two.is_extractable = False
self.initial_model = [file_one, file_two]
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
self.controller.queue_command.assert_not_called()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.One", command.filename)
self.controller.queue_command.reset_mock()
persist.add_pattern(AutoQueuePattern(pattern="File.Two"))
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_downloaded_file_is_NOT_re_extracted_after_modified(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
# File is auto-extracted
file_one = ModelFile("File.One", True)
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
file_one.is_extractable = True
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.One", command.filename)
self.controller.queue_command.reset_mock()
# File is modified
file_one_new = ModelFile("File.One", True)
file_one_new.local_size = 101
file_one_new.state = ModelFile.State.DOWNLOADED
file_one_new.is_extractable = True
self.model_listener.file_updated(file_one, file_one_new)
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_file_is_extractable(self):
serialize = SerializeModel()
a = ModelFile("a", True)
a.is_extractable = False
b = ModelFile("b", False)
b.is_extractable = True
files = [a, b]
out = parse_stream(serialize.model(files))
data = json.loads(out["data"])
self.assertEqual(2, len(data))
self.assertEqual(False, data[0]["is_extractable"])
self.assertEqual(True, data[1]["is_extractable"])
def test_matching_new_files_are_extracted(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
persist.add_pattern(AutoQueuePattern(pattern="File.Two"))
persist.add_pattern(AutoQueuePattern(pattern="File.Three"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.state = ModelFile.State.DOWNLOADED
file_one.local_size = 100
file_one.is_extractable = True
file_two = ModelFile("File.Two", True)
file_two.state = ModelFile.State.DOWNLOADED
file_two.local_size = 200
file_two.is_extractable = True
file_three = ModelFile("File.Three", True)
file_three.state = ModelFile.State.DOWNLOADED
file_three.local_size = 300
file_three.is_extractable = True
self.model_listener.file_added(file_one)
auto_queue.process()
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.One", command.filename)
self.model_listener.file_added(file_two)
auto_queue.process()
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.Two", command.filename)
self.model_listener.file_added(file_three)
auto_queue.process()
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.Three", command.filename)
# All at once
self.model_listener.file_added(file_one)
self.model_listener.file_added(file_two)
self.model_listener.file_added(file_three)
auto_queue.process()
calls = self.controller.queue_command.call_args_list[-3:]
commands = [calls[i][0][0] for i in range(3)]
self.assertEqual(set([Controller.Command.Action.EXTRACT] * 3),
{c.action
for c in commands})
self.assertEqual({"File.One", "File.Two", "File.Three"},
{c.filename
for c in commands})
def test_all_files_are_extracted_when_patterns_only_disabled(self):
self.context.config.autoqueue.patterns_only = False
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
persist.add_pattern(AutoQueuePattern(pattern="File.Two"))
persist.add_pattern(AutoQueuePattern(pattern="File.Three"))
file_one = ModelFile("File.One", True)
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
file_one.is_extractable = True
file_two = ModelFile("File.Two", True)
file_two.local_size = 200
file_two.state = ModelFile.State.DOWNLOADED
file_two.is_extractable = True
file_three = ModelFile("File.Three", True)
file_three.local_size = 300
file_three.state = ModelFile.State.DOWNLOADED
file_three.is_extractable = True
file_four = ModelFile("File.Four", True)
file_four.local_size = 400
file_four.state = ModelFile.State.DOWNLOADED
file_four.is_extractable = True
file_five = ModelFile("File.Five", True)
file_five.local_size = 500
file_five.state = ModelFile.State.DOWNLOADED
file_five.is_extractable = True
self.initial_model = [
file_one, file_two, file_three, file_four, file_five
]
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
calls = self.controller.queue_command.call_args_list
self.assertEqual(5, len(calls))
commands = [calls[i][0][0] for i in range(5)]
self.assertEqual(set([Controller.Command.Action.EXTRACT] * 5),
{c.action
for c in commands})
self.assertEqual(
{"File.One", "File.Two", "File.Three", "File.Four", "File.Five"},
{c.filename
for c in commands})
def __fill_model_file(
_model_file: ModelFile, _remote: Optional[SystemFile],
_local: Optional[SystemFile],
_transfer_state: Optional[LftpJobStatus.TransferState]):
# set local and remote sizes
if _remote:
_model_file.remote_size = _remote.size
if _local:
_model_file.local_size = _local.size
# Note: no longer use lftp's file sizes
# they represent remaining size for resumed downloads
# set the downloading speed and eta
if _transfer_state:
_model_file.downloading_speed = _transfer_state.speed
_model_file.eta = _transfer_state.eta
# set the transferred size (only if file or dir exists on both ends)
if _local and _remote:
if _model_file.is_dir:
# dir transferred size is updated by child files
_model_file.transferred_size = 0
else:
_model_file.transferred_size = min(
_local.size, _remote.size)
# also update all parent directories
_parent_file = _model_file.parent
while _parent_file is not None:
_parent_file.transferred_size += _model_file.transferred_size
_parent_file = _parent_file.parent
# set the is_extractable flag
if not _model_file.is_dir and Extract.is_archive_fast(
_model_file.name):
_model_file.is_extractable = True
# Also set the flag for all of its parents
_parent_file = _model_file.parent
while _parent_file is not None:
_parent_file.is_extractable = True
_parent_file = _parent_file.parent
# set the timestamps
if _local:
if _local.timestamp_created:
_model_file.local_created_timestamp = _local.timestamp_created
if _local.timestamp_modified:
_model_file.local_modified_timestamp = _local.timestamp_modified
if _remote:
if _remote.timestamp_created:
_model_file.remote_created_timestamp = _remote.timestamp_created
if _remote.timestamp_modified:
_model_file.remote_modified_timestamp = _remote.timestamp_modified
def test_is_extractable(self):
file = ModelFile("test", True)
file.is_extractable = True
self.assertTrue(file.is_extractable)
file.is_extractable = False
self.assertFalse(file.is_extractable)
