def test_full_path(self):
serialize = SerializeModel()
a = ModelFile("a", True)
b = ModelFile("b", True)
b.add_child(ModelFile("ba", False))
b.add_child(ModelFile("bb", True))
c = ModelFile("c", True)
ca = ModelFile("ca", True)
ca.add_child(ModelFile("caa", False))
ca.add_child(ModelFile("cab", False))
c.add_child(ca)
cb = ModelFile("cb", False)
c.add_child(cb)
c.eta = 100
files = [a, b, c]
out = parse_stream(serialize.model(files))
data = json.loads(out["data"])
self.assertEqual(3, len(data))
self.assertEqual("a", data[0]["full_path"])
self.assertEqual("b", data[1]["full_path"])
self.assertEqual("b/ba", data[1]["children"][0]["full_path"])
self.assertEqual("b/bb", data[1]["children"][1]["full_path"])
self.assertEqual("c", data[2]["full_path"])
self.assertEqual("c/ca", data[2]["children"][0]["full_path"])
self.assertEqual("c/ca/caa", data[2]["children"][0]["children"][0]["full_path"])
self.assertEqual("c/ca/cab", data[2]["children"][0]["children"][1]["full_path"])
self.assertEqual("c/cb", data[2]["children"][1]["full_path"])
def test_fail_add_child_to_nondir(self):
file_parent = ModelFile("parent", False)
file_child1 = ModelFile("child1", True)
with self.assertRaises(TypeError) as context:
file_parent.add_child(file_child1)
self.assertTrue(
str(context.exception).startswith(
"Cannot add child to a non-directory"))
def test_parent(self):
a = ModelFile("a", True)
aa = ModelFile("aa", True)
a.add_child(aa)
aaa = ModelFile("aaa", False)
aa.add_child(aaa)
self.assertIsNone(a.parent)
self.assertEqual(a, aa.parent)
self.assertEqual(aa, aaa.parent)
def test_child(self):
file_parent = ModelFile("parent", True)
file_child1 = ModelFile("child1", True)
file_child2 = ModelFile("child2", False)
self.assertEqual(0, len(file_parent.get_children()))
file_parent.add_child(file_child1)
self.assertEqual([file_child1], file_parent.get_children())
file_parent.add_child(file_child2)
self.assertEqual([file_child1, file_child2],
file_parent.get_children())
def test_updated_children(self):
model_before = Model()
model_after = Model()
a1 = ModelFile("a", True)
aa1 = ModelFile("aa", False)
aa1.local_size = 100
a1.add_child(aa1)
a2 = ModelFile("a", True)
aa2 = ModelFile("aa", False)
aa2.local_size = 200
a2.add_child(aa2)
model_before.add_file(a1)
model_after.add_file(a2)
diff = ModelDiffUtil.diff_models(model_before, model_after)
self.assertEqual([ModelDiff(ModelDiff.Change.UPDATED, a1, a2)], diff)
def test_extract_dir_raises_error_on_no_local_files(self):
self.mock_is_archive.return_value = True
a = ModelFile("a", True)
a.remote_size = 100
aa = ModelFile("aa", False)
aa.remote_size = 50
a.add_child(aa)
ab = ModelFile("ab", False)
ab.remote_size = 50
a.add_child(ab)
with self.assertRaises(ExtractDispatchError) as ctx:
self.dispatch.extract(a)
self.assertTrue(
str(ctx.exception).startswith(
"Directory does not contain any archives"))
def test_extract_dir_exits_command_early_on_shutdown(self):
# Send extract dir command with two archives
# Call shutdown after first extract but before second
# Verify second extract is not called
self.mock_is_archive.return_value = True
self.call_stop = False
def _extract_archive(**kwargs):
print(kwargs)
self.call_stop = True
time.sleep(0.5) # wait a bit so shutdown is called
self.mock_extract_archive.side_effect = _extract_archive
a = ModelFile("a", True)
a.local_size = 200
aa = ModelFile("aa", False)
aa.local_size = 100
a.add_child(aa)
ab = ModelFile("ab", False)
ab.local_size = 100
a.add_child(ab)
self.dispatch.add_listener(self.listener)
self.dispatch.extract(a)
while not self.call_stop:
pass
self.dispatch.stop()
while self.mock_extract_archive.call_count < 1 \
or self.listener.extract_failed.call_count < 1:
pass
self.listener.extract_completed.assert_not_called()
self.listener.extract_failed.assert_called_once_with("a", True)
self.assertEqual(1, self.mock_extract_archive.call_count)
def test_full_path(self):
file_a = ModelFile("a", True)
file_aa = ModelFile("aa", True)
file_a.add_child(file_aa)
file_aaa = ModelFile("aaa", True)
file_aa.add_child(file_aaa)
file_ab = ModelFile("ab", True)
file_a.add_child(file_ab)
self.assertEqual("a", file_a.full_path)
self.assertEqual("a/aa", file_aa.full_path)
self.assertEqual("a/aa/aaa", file_aaa.full_path)
self.assertEqual("a/ab", file_ab.full_path)
def test_fail_add_child_twice(self):
file_parent = ModelFile("parent", True)
file_parent.add_child(ModelFile("child1", True))
file_parent.add_child(ModelFile("child2", True))
with self.assertRaises(ValueError) as context:
file_parent.add_child(ModelFile("child1", True))
self.assertTrue(
str(context.exception).startswith(
"Cannot add child more than once"))
with self.assertRaises(ValueError) as context:
file_parent.add_child(ModelFile("child2", True))
self.assertTrue(
str(context.exception).startswith(
"Cannot add child more than once"))
def test_status(self):
self.mock_is_archive.return_value = True
self.send_count = 0
self.rx_count = 0
# noinspection PyUnusedLocal
def _extract(**kwargs):
# barrier implementation
while self.send_count <= self.rx_count:
pass
self.rx_count += 1
self.mock_extract_archive.side_effect = _extract
a = ModelFile("a", True)
a.local_size = 200
aa = ModelFile("aa", False)
aa.local_size = 100
a.add_child(aa)
ab = ModelFile("ab", False)
ab.local_size = 100
a.add_child(ab)
b = ModelFile("b", True)
b.local_size = 100
ba = ModelFile("ba", False)
ba.local_size = 100
b.add_child(ba)
c = ModelFile("c", False)
c.local_size = 100
# Initial status should be empty
status = self.dispatch.status()
self.assertEqual(0, len(status))
self.dispatch.add_listener(self.listener)
self.dispatch.extract(a)
self.dispatch.extract(b)
self.dispatch.extract(c)
status = self.dispatch.status()
self.assertEqual(3, len(status))
self.assertEqual("a", status[0].name)
self.assertEqual(True, status[0].is_dir)
self.assertEqual(ExtractStatus.State.EXTRACTING, status[0].state)
self.assertEqual("b", status[1].name)
self.assertEqual(True, status[1].is_dir)
self.assertEqual(ExtractStatus.State.EXTRACTING, status[1].state)
self.assertEqual("c", status[2].name)
self.assertEqual(False, status[2].is_dir)
self.assertEqual(ExtractStatus.State.EXTRACTING, status[2].state)
# Wait for first dir to start extracting
self.send_count = 1
while self.rx_count < self.send_count:
pass
status = self.dispatch.status()
self.assertEqual(3, len(status))
self.assertEqual("a", status[0].name)
self.assertEqual(True, status[0].is_dir)
self.assertEqual(ExtractStatus.State.EXTRACTING, status[0].state)
self.assertEqual("b", status[1].name)
self.assertEqual(True, status[1].is_dir)
self.assertEqual(ExtractStatus.State.EXTRACTING, status[1].state)
self.assertEqual("c", status[2].name)
self.assertEqual(False, status[2].is_dir)
self.assertEqual(ExtractStatus.State.EXTRACTING, status[2].state)
# After first directory finishes
self.send_count = 2
while self.listener.extract_completed.call_count < 1:
pass
self.listener.extract_completed.assert_called_with("a", True)
status = self.dispatch.status()
self.assertEqual(2, len(status))
self.assertEqual("b", status[0].name)
self.assertEqual(True, status[0].is_dir)
self.assertEqual(ExtractStatus.State.EXTRACTING, status[0].state)
self.assertEqual("c", status[1].name)
self.assertEqual(False, status[1].is_dir)
self.assertEqual(ExtractStatus.State.EXTRACTING, status[1].state)
# After second directory finishes
self.send_count = 3
while self.listener.extract_completed.call_count < 2:
pass
self.listener.extract_completed.assert_called_with("b", True)
status = self.dispatch.status()
self.assertEqual(1, len(status))
self.assertEqual("c", status[0].name)
self.assertEqual(False, status[0].is_dir)
self.assertEqual(ExtractStatus.State.EXTRACTING, status[0].state)
# After third/last file finishes
self.send_count = 4
while self.listener.extract_completed.call_count < 3:
pass
self.listener.extract_completed.assert_called_with("c", False)
status = self.dispatch.status()
self.assertEqual(0, len(status))
def test_extract_dir_does_not_extract_split_rar_files(self):
self.mock_is_archive.return_value = True
self.actual_calls = set()
def _extract(archive_path: str, out_dir_path: str):
self.actual_calls.add((archive_path, out_dir_path))
self.mock_extract_archive.side_effect = _extract
a = ModelFile("a", True)
a.local_size = 80
aa = ModelFile("aa.rar", False)
aa.local_size = 10
a.add_child(aa)
aa0 = ModelFile("aa.r00", False)
aa0.local_size = 10
a.add_child(aa0)
aa1 = ModelFile("aa.r01", False)
aa1.local_size = 10
a.add_child(aa1)
aa2 = ModelFile("aa.r02", False)
aa2.local_size = 10
a.add_child(aa2)
aa15 = ModelFile("aa.r15", False)
aa15.local_size = 10
a.add_child(aa15)
ab = ModelFile("ab.rar", False)
ab.local_size = 10
a.add_child(ab)
ab0 = ModelFile("ab.r000", False)
ab0.local_size = 10
a.add_child(ab0)
ab1 = ModelFile("ab.r001", False)
ab1.local_size = 10
a.add_child(ab1)
ac = ModelFile("ac", True)
ac.local_size = 20
a.add_child(ac)
aca = ModelFile("aca", True)
aca.local_size = 20
ac.add_child(aca)
acaa = ModelFile("acaa.rar", False)
acaa.local_size = 10
aca.add_child(acaa)
acaa0 = ModelFile("acaa.r00", False)
acaa0.local_size = 10
aca.add_child(acaa0)
self.dispatch.add_listener(self.listener)
self.dispatch.extract(a)
while self.listener.extract_completed.call_count < 1:
pass
self.listener.extract_completed.assert_called_once_with("a", True)
golden_calls = {
(os.path.join(self.local_path, "a",
"aa.rar"), os.path.join(self.out_dir_path, "a")),
(os.path.join(self.local_path, "a",
"ab.rar"), os.path.join(self.out_dir_path, "a")),
(os.path.join(self.local_path, "a", "ac", "aca", "acaa.rar"),
os.path.join(self.out_dir_path, "a", "ac", "aca")),
}
self.assertEqual(3, self.mock_extract_archive.call_count)
self.assertEqual(golden_calls, self.actual_calls)
def test_extract_dir_skips_non_archive_files(self):
# noinspection SpellCheckingInspection
def _is_archive(archive_path: str):
return archive_path in (os.path.join(self.local_path, "a", "aa",
"aaa"),
os.path.join(self.local_path, "a", "aa",
"aac", "aaca"),
os.path.join(self.local_path, "a", "ab",
"aba"))
self.mock_is_archive.side_effect = _is_archive
self.actual_calls = set()
def _extract(archive_path: str, out_dir_path: str):
self.actual_calls.add((archive_path, out_dir_path))
self.mock_extract_archive.side_effect = _extract
a = ModelFile("a", True)
a.local_size = 500
aa = ModelFile("aa", True)
aa.local_size = 300
a.add_child(aa)
aaa = ModelFile("aaa", False)
aaa.local_size = 100
aa.add_child(aaa)
aab = ModelFile("aab", False)
aab.local_size = 100
aa.add_child(aab)
aac = ModelFile("aac", True)
aac.local_size = 100
aa.add_child(aac)
aaca = ModelFile("aaca", False)
aaca.local_size = 100
aac.add_child(aaca)
ab = ModelFile("ab", True)
ab.local_size = 100
a.add_child(ab)
aba = ModelFile("aba", False)
aba.local_size = 100
ab.add_child(aba)
ac = ModelFile("ac", False)
ac.local_size = 100
a.add_child(ac)
self.dispatch.add_listener(self.listener)
self.dispatch.extract(a)
while self.listener.extract_completed.call_count < 1:
pass
self.listener.extract_completed.assert_called_once_with("a", True)
golden_calls = {
(os.path.join(self.local_path, "a", "aa",
"aaa"), os.path.join(self.out_dir_path, "a", "aa")),
(os.path.join(self.local_path, "a", "aa", "aac", "aaca"),
os.path.join(self.out_dir_path, "a", "aa", "aac")),
(os.path.join(self.local_path, "a", "ab",
"aba"), os.path.join(self.out_dir_path, "a", "ab")),
}
self.assertEqual(3, self.mock_extract_archive.call_count)
self.assertEqual(golden_calls, self.actual_calls)
def test_child_equality(self):
l_a = ModelFile("a", True)
l_a.remote_size = 3 + 1 + 2
l_aa = ModelFile("aa", True)
l_aa.remote_size = 3 + 1
l_a.add_child(l_aa)
l_aaa = ModelFile("aaa", False)
l_aaa.remote_size = 1
l_aa.add_child(l_aaa)
l_aab = ModelFile("aab", False)
l_aab.remote_size = 3
l_aa.add_child(l_aab)
l_ab = ModelFile("ab", False)
l_ab.remote_size = 2
l_a.add_child(l_ab)
r_a = ModelFile("a", True)
r_a.remote_size = 3 + 1 + 2
r_aa = ModelFile("aa", True)
r_aa.remote_size = 3 + 1
r_a.add_child(r_aa)
r_aaa = ModelFile("aaa", False)
r_aaa.remote_size = 1
r_aa.add_child(r_aaa)
r_aab = ModelFile("aab", False)
r_aab.remote_size = 3
r_aa.add_child(r_aab)
r_ab = ModelFile("ab", False)
r_ab.remote_size = 2
r_a.add_child(r_ab)
self.assertEqual(l_a, r_a)
r_aaa.remote_size = 2
self.assertNotEqual(l_a, r_a)
def test_forwards_extract_commands(self):
a = ModelFile("a", True)
a.local_size = 100
aa = ModelFile("aa", False)
aa.local_size = 60
a.add_child(aa)
ab = ModelFile("ab", False)
ab.local_size = 40
a.add_child(ab)
b = ModelFile("b", True)
b.local_size = 10
ba = ModelFile("ba", True)
ba.local_size = 10
b.add_child(ba)
baa = ModelFile("baa", False)
baa.local_size = 10
ba.add_child(baa)
c = ModelFile("c", False)
c.local_size = 1234
self.extract_counter = multiprocessing.Value('i', 0)
def _extract(file: ModelFile):
print(file.name)
if self.extract_counter.value == 0:
self.assertEqual("a", file.name)
self.assertEqual(True, file.is_dir)
self.assertEqual(100, file.local_size)
children = file.get_children()
self.assertEqual(2, len(children))
self.assertEqual("aa", children[0].name)
self.assertEqual(False, children[0].is_dir)
self.assertEqual(60, children[0].local_size)
self.assertEqual("ab", children[1].name)
self.assertEqual(False, children[0].is_dir)
self.assertEqual(40, children[1].local_size)
elif self.extract_counter.value == 1:
self.assertEqual("b", file.name)
self.assertEqual(True, file.is_dir)
self.assertEqual(10, file.local_size)
self.assertEqual(1, len(file.get_children()))
child = file.get_children()[0]
self.assertEqual("ba", child.name)
self.assertEqual(True, child.is_dir)
self.assertEqual(10, child.local_size)
self.assertEqual(1, len(child.get_children()))
subchild = child.get_children()[0]
self.assertEqual("baa", subchild.name)
self.assertEqual(False, subchild.is_dir)
self.assertEqual(10, subchild.local_size)
elif self.extract_counter.value == 2:
self.assertEqual("c", file.name)
self.assertEqual(False, file.is_dir)
self.assertEqual(1234, file.local_size)
self.extract_counter.value += 1
self.mock_dispatch.extract.side_effect = _extract
self.process = ExtractProcess(out_dir_path="", local_path="")
self.process.start()
self.process.extract(a)
time.sleep(1)
self.process.extract(b)
self.process.extract(c)
while self.extract_counter.value < 3:
pass