def test_scan_single_non_existing_path_fails(self):
self.setup_default_tree()
scanner = SystemScanner(
path_to_scan=os.path.join(TestSystemScanner.temp_dir))
with self.assertRaises(SystemScannerError) as ex:
scanner.scan_single("nonexisting")
self.assertTrue(str(ex.exception).startswith("Path does not exist"))
def test_scan_single_lftp_partial_file(self):
# Scan a single partial file
tempdir = tempfile.mkdtemp(prefix="test_system_scanner")
# Create a partial file
path = os.path.join(tempdir, "partial.mkv")
with open(path, 'wb') as f:
f.write(bytearray([0xff] * 24588))
# Write the lftp status out
path = os.path.join(tempdir, "partial.mkv.lftp-pget-status")
with open(path, "w") as f:
f.write("""
size=24588
0.pos=3157
0.limit=6147
1.pos=11578
1.limit=12294
2.pos=12295
2.limit=18441
3.pos=20000
3.limit=24588
""")
scanner = SystemScanner(tempdir)
partial_mkv = scanner.scan_single("partial.mkv")
self.assertEqual("partial.mkv", partial_mkv.name)
self.assertEqual(10148, partial_mkv.size)
# Cleanup
shutil.rmtree(tempdir)
def test_scan_single_file(self):
scanner = SystemScanner(TestSystemScanner.temp_dir)
c = scanner.scan_single("c")
self.assertEqual("c", c.name)
self.assertFalse(c.is_dir)
self.assertEqual(1234, c.size)
def test_scan_single_dir(self):
scanner = SystemScanner(TestSystemScanner.temp_dir)
a = scanner.scan_single("a")
self.assertEqual("a", a.name)
self.assertTrue(a.is_dir)
self.assertEqual(2, len(a.children))
aa, ab = tuple(a.children)
self.assertEqual("aa", aa.name)
self.assertTrue(aa.is_dir)
self.assertEqual(2, len(aa.children))
aaa, aab = tuple(aa.children)
self.assertEqual(".aaa", aaa.name)
self.assertTrue(aaa.is_dir)
self.assertEqual(".aab", aab.name)
self.assertFalse(aab.is_dir)
self.assertEqual("ab", ab.name)
self.assertFalse(ab.is_dir)
self.assertEqual(12 * 1024 + 4 + 512, a.size)
self.assertEqual(512, aa.size)
self.assertEqual(0, aaa.size)
self.assertEqual(512, aab.size)
self.assertEqual(12 * 1024 + 4, ab.size)
class ActiveScanner(IScanner):
"""
Scanner implementation to scan the active files only
A caller sets the names of the active files that need to be scanned.
A multiprocessing.Queue is used to store the names because the set and scan
methods are called by different processes.
"""
def __init__(self, local_path: str):
self.__scanner = SystemScanner(local_path)
self.__active_files_queue = multiprocessing.Queue()
self.__active_files = [] # latest state
self.logger = logging.getLogger(self.__class__.__name__)
@overrides(IScanner)
def set_base_logger(self, base_logger: logging.Logger):
self.logger = base_logger.getChild(self.__class__.__name__)
def set_active_files(self, file_names: List[str]):
"""
Set the list of active file names. Only these files will be scanned.
:param file_names:
:return:
"""
self.__active_files_queue.put(file_names)
@overrides(IScanner)
def scan(self) -> List[SystemFile]:
# Grab the latest list of active files, if any
try:
while True:
self.__active_files = self.__active_files_queue.get(
block=False)
except queue.Empty:
pass
# Do the scan
# self.logger.debug("Scanning files: {}".format(str(self.__active_files)))
result = []
for file_name in self.__active_files:
try:
result.append(self.__scanner.scan_single(file_name))
except SystemScannerError as ex:
# Ignore errors here, file may have been deleted
self.logger.warning(str(ex))
return result
def test_scan_single_lftp_temp_file(self):
tempdir = tempfile.mkdtemp(prefix="test_system_scanner")
# Create:
# temp file
# non-temp file and
# non-temp directory with temp name
# non-temp directory with non-temp name
temp1 = os.path.join(tempdir, "a.mkv.lftp")
with open(temp1, 'wb') as f:
f.write(bytearray([0xff] * 100))
nontemp1 = os.path.join(tempdir, "b.rar")
with open(nontemp1, 'wb') as f:
f.write(bytearray([0xff] * 300))
nontemp2 = os.path.join(tempdir, "c.lftp")
os.mkdir(nontemp2)
temp2 = os.path.join(nontemp2, "c.txt.lftp")
with open(temp2, 'wb') as f:
f.write(bytearray([0xff] * 500))
nontemp3 = os.path.join(tempdir, "d")
os.mkdir(nontemp3)
temp3 = os.path.join(nontemp3, "d.avi.lftp")
with open(temp3, 'wb') as f:
f.write(bytearray([0xff] * 600))
scanner = SystemScanner(tempdir)
# No temp suffix set, must include temp suffix in name param
file = scanner.scan_single("a.mkv.lftp")
self.assertEqual("a.mkv.lftp", file.name)
self.assertEqual(100, file.size)
self.assertEqual(False, file.is_dir)
file = scanner.scan_single("b.rar")
self.assertEqual("b.rar", file.name)
self.assertEqual(300, file.size)
self.assertEqual(False, file.is_dir)
file = scanner.scan_single("c.lftp")
self.assertEqual("c.lftp", file.name)
self.assertEqual(500, file.size)
self.assertEqual(True, file.is_dir)
self.assertEqual(1, len(file.children))
child = file.children[0]
self.assertEqual("c.txt.lftp", child.name)
self.assertEqual(500, child.size)
self.assertEqual(False, child.is_dir)
file = scanner.scan_single("d")
self.assertEqual("d", file.name)
self.assertEqual(600, file.size)
self.assertEqual(True, file.is_dir)
child = file.children[0]
self.assertEqual("d.avi.lftp", child.name)
self.assertEqual(600, child.size)
self.assertEqual(False, child.is_dir)
# Temp suffix set, must NOT include temp suffix in name param
scanner.set_lftp_temp_suffix(".lftp")
file = scanner.scan_single("a.mkv")
self.assertEqual("a.mkv", file.name)
self.assertEqual(100, file.size)
self.assertEqual(False, file.is_dir)
file = scanner.scan_single("b.rar")
self.assertEqual("b.rar", file.name)
self.assertEqual(300, file.size)
self.assertEqual(False, file.is_dir)
file = scanner.scan_single("c.lftp")
self.assertEqual("c.lftp", file.name)
self.assertEqual(500, file.size)
self.assertEqual(True, file.is_dir)
self.assertEqual(1, len(file.children))
child = file.children[0]
self.assertEqual("c.txt", child.name)
self.assertEqual(500, child.size)
self.assertEqual(False, child.is_dir)
# also, shouldn't look for directories with temp suffix
with self.assertRaises(SystemScannerError) as ctx:
scanner.scan_single("c")
self.assertTrue("Path does not exist" in str(ctx.exception))
file = scanner.scan_single("d")
self.assertEqual("d", file.name)
self.assertEqual(600, file.size)
self.assertEqual(True, file.is_dir)
child = file.children[0]
self.assertEqual("d.avi", child.name)
self.assertEqual(600, child.size)
self.assertEqual(False, child.is_dir)
# No file and no temp file
with self.assertRaises(SystemScannerError) as ctx:
scanner.scan_single("blah")
self.assertTrue("Path does not exist" in str(ctx.exception))
# Cleanup
shutil.rmtree(tempdir)