def test_downloaded_file_is_NOT_re_extracted_after_modified(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
# File is auto-extracted
file_one = ModelFile("File.One", True)
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
file_one.is_extractable = True
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.One", command.filename)
self.controller.queue_command.reset_mock()
# File is modified
file_one_new = ModelFile("File.One", True)
file_one_new.local_size = 101
file_one_new.state = ModelFile.State.DOWNLOADED
file_one_new.is_extractable = True
self.model_listener.file_updated(file_one, file_one_new)
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_non_extractable_files_are_not_extracted(self):
persist = AutoQueuePersist()
file_one = ModelFile("File.One", True)
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
file_one.is_extractable = True
file_two = ModelFile("File.Two", True)
file_two.local_size = 200
file_two.state = ModelFile.State.DOWNLOADED
file_two.is_extractable = False
self.initial_model = [file_one, file_two]
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
self.controller.queue_command.assert_not_called()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.One", command.filename)
self.controller.queue_command.reset_mock()
persist.add_pattern(AutoQueuePattern(pattern="File.Two"))
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_old_file(self):
old_file = ModelFile("a", False)
old_file.local_size = 100
diff = ModelDiff(ModelDiff.Change.ADDED, old_file, None)
self.assertEqual(old_file, diff.old_file)
diff = ModelDiff(ModelDiff.Change.ADDED, None, None)
self.assertEqual(None, diff.old_file)
def test_extract_ignores_duplicate_calls(self):
# Send two extract commands to same file
# Expect that only one extract operation is performed
self.mock_is_archive.return_value = True
self.barrier = False
def _extract_archive(**kwargs):
print(kwargs)
while not self.barrier:
pass
self.mock_extract_archive.side_effect = _extract_archive
a = ModelFile("a", False)
a.local_size = 200
self.dispatch.add_listener(self.listener)
self.dispatch.extract(a)
self.dispatch.extract(a)
time.sleep(0.1)
self.barrier = True
time.sleep(0.1)
while self.mock_extract_archive.call_count < 1 and \
self.listener.extract_completed.call_count < 1:
pass
time.sleep(0.1)
self.listener.extract_completed.assert_called_once_with("a", False)
self.listener.extract_failed.assert_not_called()
self.assertEqual(1, self.mock_extract_archive.call_count)
def test_new_file(self):
new_file = ModelFile("a", False)
new_file.local_size = 100
diff = ModelDiff(ModelDiff.Change.ADDED, None, new_file)
self.assertEqual(new_file, diff.new_file)
diff = ModelDiff(ModelDiff.Change.ADDED, None, None)
self.assertEqual(None, diff.new_file)
def test_downloaded_file_with_changed_remote_size_is_queued(self):
# Disable auto-extract
self.context.config.autoqueue.auto_extract = False
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_not_called()
file_one_updated = ModelFile("File.One", True)
file_one_updated.remote_size = 200
file_one_updated.local_size = 100
file_one_updated.state = ModelFile.State.DEFAULT
self.model_listener.file_updated(file_one, file_one_updated)
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.QUEUE, command.action)
self.assertEqual("File.One", command.filename)
def _extract(file: ModelFile):
print(file.name)
if self.extract_counter.value == 0:
self.assertEqual("a", file.name)
self.assertEqual(True, file.is_dir)
self.assertEqual(100, file.local_size)
children = file.get_children()
self.assertEqual(2, len(children))
self.assertEqual("aa", children[0].name)
self.assertEqual(False, children[0].is_dir)
self.assertEqual(60, children[0].local_size)
self.assertEqual("ab", children[1].name)
self.assertEqual(False, children[0].is_dir)
self.assertEqual(40, children[1].local_size)
elif self.extract_counter.value == 1:
self.assertEqual("b", file.name)
self.assertEqual(True, file.is_dir)
self.assertEqual(10, file.local_size)
self.assertEqual(1, len(file.get_children()))
child = file.get_children()[0]
self.assertEqual("ba", child.name)
self.assertEqual(True, child.is_dir)
self.assertEqual(10, child.local_size)
self.assertEqual(1, len(child.get_children()))
subchild = child.get_children()[0]
self.assertEqual("baa", subchild.name)
self.assertEqual(False, subchild.is_dir)
self.assertEqual(10, subchild.local_size)
elif self.extract_counter.value == 2:
self.assertEqual("c", file.name)
self.assertEqual(False, file.is_dir)
self.assertEqual(1234, file.local_size)
self.extract_counter.value += 1
def test_file_name(self):
serialize = SerializeModel()
files = [ModelFile("a", True), ModelFile("b", False)]
out = parse_stream(serialize.model(files))
data = json.loads(out["data"])
self.assertEqual(2, len(data))
self.assertEqual("a", data[0]["name"])
self.assertEqual("b", data[1]["name"])
def test_added(self):
model_before = Model()
model_after = Model()
a = ModelFile("a", False)
a.local_size = 100
model_after.add_file(a)
diff = ModelDiffUtil.diff_models(model_before, model_after)
self.assertEqual([ModelDiff(ModelDiff.Change.ADDED, None, a)], diff)
def test_removed(self):
model_before = Model()
model_after = Model()
a = ModelFile("a", False)
a.local_size = 100
model_before.add_file(a)
diff = ModelDiffUtil.diff_models(model_before, model_after)
self.assertEqual([ModelDiff(ModelDiff.Change.REMOVED, a, None)], diff)
def test_state(self):
file = ModelFile("test", False)
file.state = ModelFile.State.DOWNLOADING
self.assertEqual(ModelFile.State.DOWNLOADING, file.state)
with self.assertRaises(TypeError):
file.state = "BadState"
def test_fail_add_child_to_nondir(self):
file_parent = ModelFile("parent", False)
file_child1 = ModelFile("child1", True)
with self.assertRaises(TypeError) as context:
file_parent.add_child(file_child1)
self.assertTrue(
str(context.exception).startswith(
"Cannot add child to a non-directory"))
def test_update_timestamp(self):
file = ModelFile("test", False)
from datetime import datetime
now = datetime.now()
file.update_timestamp = now
self.assertEqual(now, file.update_timestamp)
with self.assertRaises(TypeError):
file.update_timestamp = 100
def test_update_file(self):
file = ModelFile("test", False)
file.local_size = 100
self.model.add_file(file)
recv_file = self.model.get_file("test")
self.assertEqual(100, recv_file.local_size)
recv_file.local_size = 200
self.model.update_file(recv_file)
recv_file = self.model.get_file("test")
self.assertEqual(200, recv_file.local_size)
def test_extract_maintains_order(self):
self.mock_is_archive.return_value = True
mf1 = ModelFile("aaa", False)
mf1.local_size = 100
mf2 = ModelFile("bbb", False)
mf2.local_size = 100
mf3 = ModelFile("ccc", False)
mf3.local_size = 100
self.dispatch.extract(mf1)
self.dispatch.extract(mf2)
self.dispatch.extract(mf3)
while self.mock_extract_archive.call_count < 3:
pass
self.assertEqual(3, self.mock_extract_archive.call_count)
args_list = self.mock_extract_archive.call_args_list
self.assertEqual(args_list, [
call(archive_path=os.path.join(self.local_path, "aaa"),
out_dir_path=self.out_dir_path),
call(archive_path=os.path.join(self.local_path, "bbb"),
out_dir_path=self.out_dir_path),
call(archive_path=os.path.join(self.local_path, "ccc"),
out_dir_path=self.out_dir_path)
])
def test_matching_queued_files_are_not_queued(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
file_one.state = ModelFile.State.QUEUED
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_remote_modified_timestamp(self):
serialize = SerializeModel()
a = ModelFile("a", True)
b = ModelFile("b", False)
b.remote_modified_timestamp = datetime(2018, 11, 9, 21, 40, 18, tzinfo=timezone('UTC'))
files = [a, b]
out = parse_stream(serialize.model(files))
data = json.loads(out["data"])
self.assertEqual(2, len(data))
self.assertEqual(None, data[0]["remote_modified_timestamp"])
self.assertEqual(str(1541799618.0), data[1]["remote_modified_timestamp"])
def test_remove_listener(self):
listener = DummyModelListener()
listener.file_added = MagicMock()
self.model.add_listener(listener)
file = ModelFile("test", False)
self.model.add_file(file)
listener.file_added.assert_called_once_with(file)
self.model.remove_listener(listener)
self.model.add_file(ModelFile("test2", False))
listener.file_added.assert_called_once_with(file)
def test_get_file_names(self):
self.assertEqual(set(), self.model.get_file_names())
self.model.add_file(ModelFile("a", False))
self.assertEqual({"a"}, self.model.get_file_names())
self.model.add_file(ModelFile("b", False))
self.assertEqual({"a", "b"}, self.model.get_file_names())
self.model.add_file(ModelFile("c", False))
self.assertEqual({"a", "b", "c"}, self.model.get_file_names())
self.model.remove_file("b")
self.assertEqual({"a", "c"}, self.model.get_file_names())
self.model.add_file(ModelFile("d", False))
self.assertEqual({"a", "c", "d"}, self.model.get_file_names())
def test_extract_single_raises_error_on_bad_archive(self):
self.mock_is_archive.return_value = False
mf = ModelFile("aaa", False)
mf.local_size = 100
with self.assertRaises(ExtractDispatchError) as ctx:
self.dispatch.extract(mf)
self.assertTrue(
str(ctx.exception).startswith("File is not an archive"))
self.mock_is_archive.assert_called_once_with(
os.path.join(self.local_path, mf.name))
def test_model_is_a_list(self):
serialize = SerializeModel()
files = []
out = parse_stream(serialize.model(files))
data = json.loads(out["data"])
self.assertEqual(list, type(data))
files = [ModelFile("a", True), ModelFile("b", False)]
out = parse_stream(serialize.model(files))
data = json.loads(out["data"])
self.assertEqual(list, type(data))
self.assertEqual(2, len(data))
def test_partial_matches(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
self.model_listener.file_added(file_one)
auto_queue.process()
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.QUEUE, command.action)
self.assertEqual("File.One", command.filename)
def test_parent(self):
a = ModelFile("a", True)
aa = ModelFile("aa", True)
a.add_child(aa)
aaa = ModelFile("aaa", False)
aa.add_child(aaa)
self.assertIsNone(a.parent)
self.assertEqual(a, aa.parent)
self.assertEqual(aa, aaa.parent)
def test_extract_single(self):
self.mock_is_archive.return_value = True
mf = ModelFile("aaa", False)
mf.local_size = 100
self.dispatch.extract(mf)
while self.mock_extract_archive.call_count < 1:
pass
self.mock_extract_archive.assert_called_once_with(
archive_path=os.path.join(self.local_path, "aaa"),
out_dir_path=self.out_dir_path)
def test_stream_model_serializes_initial_model(self, mock_serialize_model_cls):
# Schedule server stop
Timer(0.5, self.web_app.stop).start()
# Setup mock serialize instance
mock_serialize = mock_serialize_model_cls.return_value
mock_serialize.model.return_value = "\n"
# Initial model
self.model_files = [ModelFile("a", True), ModelFile("b", False)]
self.test_app.get("/server/stream")
mock_serialize.model.assert_called_once_with([ModelFile("a", True), ModelFile("b", False)])
def test_equality_operator(self):
# check that timestamp does not affect equality
now = datetime.now()
file1 = ModelFile("test", False)
file1.local_size = 100
file1.update_timestamp = now
file2 = ModelFile("test", False)
file2.local_size = 200
file2.update_timestamp = now
self.assertFalse(file1 == file2)
file2.local_size = 100
file2.update_timestamp = datetime.now()
self.assertTrue(file1 == file2)
def test_matching_downloaded_files_are_not_queued(self):
# Disable auto-extract
self.context.config.autoqueue.auto_extract = False
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
file_one.state = ModelFile.State.DOWNLOADED
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_extract_calls_listener_on_completed(self):
self.mock_is_archive.return_value = True
mf1 = ModelFile("aaa", False)
mf1.local_size = 100
self.dispatch.add_listener(self.listener)
self.dispatch.extract(mf1)
while self.mock_extract_archive.call_count < 1 \
or self.listener.extract_completed.call_count < 1:
pass
self.assertEqual(1, self.mock_extract_archive.call_count)
self.listener.extract_completed.assert_called_once_with("aaa", False)
self.listener.extract_failed.assert_not_called()
def test_extract_dir_raises_error_on_empty_dir(self):
mf = ModelFile("aaa", True)
with self.assertRaises(ExtractDispatchError) as ctx:
self.dispatch.extract(mf)
self.assertTrue(
str(ctx.exception).startswith(
"Directory does not contain any archives"))
def test_new_matching_pattern_doesnt_queue_local_file(self):
persist = AutoQueuePersist()
file_one = ModelFile("File.One", True)
file_one.local_size = 100
self.initial_model = [file_one]
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
self.controller.queue_command.assert_not_called()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
auto_queue.process()
self.controller.queue_command.assert_not_called()
