def test_scan_tree_excluded_prefix(self): self.setup_default_tree() scanner = SystemScanner(TestSystemScanner.temp_dir) scanner.add_exclude_prefix(".") files = scanner.scan() self.assertEqual(3, len(files)) a, b, c = tuple(files) aa, ab = tuple(a.children) ba, bb = tuple(b.children) bba, bbb, bbc = tuple(bb.children) bbca = bbc.children[0] self.assertEqual(0, len(aa.children)) self.assertEqual(0, len(bbca.children)) scanner.add_exclude_prefix("ab") files = scanner.scan() self.assertEqual(3, len(files)) a, b, c = tuple(files) self.assertEqual(1, len(a.children)) aa = a.children[0] ba, bb = tuple(b.children) bba, bbb, bbc = tuple(bb.children) bbca = bbc.children[0] self.assertEqual("aa", aa.name) self.assertEqual(0, len(bbca.children))
def test_scan_non_existing_dir_fails(self): self.setup_default_tree() scanner = SystemScanner(path_to_scan=os.path.join( TestSystemScanner.temp_dir, "nonexisting")) with self.assertRaises(SystemScannerError) as ex: scanner.scan() self.assertTrue(str(ex.exception).startswith("Path does not exist"))
def test_scan_size_excluded_prefix(self): scanner = SystemScanner(TestSystemScanner.temp_dir) scanner.add_exclude_prefix(".") files = scanner.scan() self.assertEqual(3, len(files)) a, b, c = tuple(files) aa, ab = tuple(a.children) ba, bb = tuple(b.children) bba, bbb, bbc = tuple(bb.children) bbca = bbc.children[0] self.assertEqual(12 * 1024 + 4, a.size) self.assertEqual(0, aa.size) self.assertEqual(24 * 1024 * 1024 + 24 + 0, bb.size) self.assertEqual(0, bbc.size) self.assertEqual(0, bbca.size) scanner.add_exclude_prefix("ab") files = scanner.scan() self.assertEqual(3, len(files)) a, b, c = tuple(files) self.assertEqual(1, len(a.children)) aa = a.children[0] self.assertEqual("aa", aa.name) self.assertEqual(0, a.size) self.assertEqual(0, aa.size)
def test_scan_file_fails(self): scanner = SystemScanner( path_to_scan=os.path.join(TestSystemScanner.temp_dir, "c")) with self.assertRaises(SystemScannerError) as ex: scanner.scan() self.assertTrue( str(ex.exception).startswith("Path is not a directory"))
def test_scan_size(self): scanner = SystemScanner(TestSystemScanner.temp_dir) files = scanner.scan() self.assertEqual(3, len(files)) a, b, c = tuple(files) aa, ab = tuple(a.children) aaa, aab = tuple(aa.children) ba, bb = tuple(b.children) baa = ba.children[0] bba, bbb, bbc = tuple(bb.children) bbca = bbc.children[0] bbcaa = bbca.children[0] self.assertEqual(12 * 1024 + 4 + 512, a.size) self.assertEqual(512, aa.size) self.assertEqual(0, aaa.size) self.assertEqual(512, aab.size) self.assertEqual(12 * 1024 + 4, ab.size) self.assertEqual(512 + 7 + 24 * 1024 * 1024 + 24 + 1, b.size) self.assertEqual(512 + 7, ba.size) self.assertEqual(512 + 7, baa.size) self.assertEqual(24 * 1024 * 1024 + 24 + 1, bb.size) self.assertEqual(0, bba.size) self.assertEqual(24 * 1024 * 1024 + 24, bbb.size) self.assertEqual(1, bbc.size) self.assertEqual(1, bbca.size) self.assertEqual(1, bbcaa.size) self.assertEqual(1234, c.size)
def test_scan_lftp_partial_file(self): tempdir = tempfile.mkdtemp(prefix="test_system_scanner") # Create a partial file os.mkdir(os.path.join(tempdir, "t")) path = os.path.join(tempdir, "t", "partial.mkv") with open(path, 'wb') as f: f.write(bytearray([0xff] * 24588)) # Write the lftp status out path = os.path.join(tempdir, "t", "partial.mkv.lftp-pget-status") with open(path, "w") as f: f.write(""" size=24588 0.pos=3157 0.limit=6147 1.pos=11578 1.limit=12294 2.pos=12295 2.limit=18441 3.pos=20000 3.limit=24588 """) scanner = SystemScanner(tempdir) files = scanner.scan() self.assertEqual(1, len(files)) t = files[0] self.assertEqual("t", t.name) self.assertEqual(10148, t.size) self.assertEqual(1, len(t.children)) partial_mkv = t.children[0] self.assertEqual("partial.mkv", partial_mkv.name) self.assertEqual(10148, partial_mkv.size) # Cleanup shutil.rmtree(tempdir)
def test_files_deleted_while_scanning(self): scanner = SystemScanner(TestSystemScanner.temp_dir) stop = False # Make and delete files while test runs def monkey_with_files(): orig = os.path.join(TestSystemScanner.temp_dir, "b") dest = os.path.join(TestSystemScanner.temp_dir, "b_copy") while not stop: shutil.copytree(orig, dest) shutil.rmtree(dest) thread = Thread(target=monkey_with_files) thread.start() try: # Scan a bunch of times for i in range(0, 2000): files = scanner.scan() # Must have at least the untouched files self.assertGreaterEqual(len(files), 3) names = set([f.name for f in files]) self.assertIn("a", names) self.assertIn("b", names) self.assertIn("c", names) finally: stop = True thread.join()
def test_scan_tree(self): self.setup_default_tree() scanner = SystemScanner(TestSystemScanner.temp_dir) files = scanner.scan() self.assertEqual(3, len(files)) a, b, c = tuple(files) self.assertEqual("a", a.name) self.assertTrue(a.is_dir) self.assertEqual("b", b.name) self.assertTrue(b.is_dir) self.assertEqual("c", c.name) self.assertFalse(c.is_dir) self.assertEqual(2, len(a.children)) aa, ab = tuple(a.children) self.assertEqual("aa", aa.name) self.assertTrue(aa.is_dir) self.assertEqual(2, len(aa.children)) aaa, aab = tuple(aa.children) self.assertEqual(".aaa", aaa.name) self.assertTrue(aaa.is_dir) self.assertEqual(".aab", aab.name) self.assertFalse(aab.is_dir) self.assertEqual("ab", ab.name) self.assertFalse(ab.is_dir) self.assertEqual(2, len(b.children)) ba, bb = tuple(b.children) self.assertEqual("ba", ba.name) self.assertTrue(ba.is_dir) self.assertEqual(1, len(ba.children)) baa = ba.children[0] self.assertEqual("baa", baa.name) self.assertFalse(baa.is_dir) self.assertEqual("bb", bb.name) self.assertTrue(bb.is_dir) self.assertEqual(3, len(bb.children)) bba, bbb, bbc = tuple(bb.children) self.assertEqual("bba", bba.name) self.assertTrue(bba.is_dir) self.assertEqual("bbb", bbb.name) self.assertFalse(bbb.is_dir) self.assertEqual("bbc", bbc.name) self.assertTrue(bbc.is_dir) self.assertEqual(1, len(bbc.children)) bbca = bbc.children[0] self.assertEqual("bbca", bbca.name) self.assertTrue(bbca.is_dir) self.assertEqual(1, len(bbca.children)) bbcaa = bbca.children[0] self.assertEqual(".bbcaa", bbcaa.name) self.assertFalse(bbcaa.is_dir)
def test_scan_file_with_unicode_chars(self): tempdir = TestSystemScanner.temp_dir # déģķ [dir] # dőÀ× [file, 128 bytes] my_mkdir("déģķ") my_touch(128, "dőÀ") scanner = SystemScanner(tempdir) files = scanner.scan() self.assertEqual(2, len(files)) folder, file = tuple(files) self.assertEqual(0, len(folder.children)) self.assertEqual("déģķ", folder.name) self.assertEqual("dőÀ", file.name) self.assertEqual(128, file.size)
def test_scan_file_with_latin_chars(self): tempdir = TestSystemScanner.temp_dir # a\xe9b [dir] # c\xe9d [file, 128 bytes] my_mkdir_latin(b"dir\xe9dir") my_touch_latin(128, b"file\xd9file") scanner = SystemScanner(tempdir) files = scanner.scan() self.assertEqual(2, len(files)) folder, file = tuple(files) self.assertEqual(0, len(folder.children)) self.assertEqual("dir�dir", folder.name) self.assertEqual("file�file", file.name) self.assertEqual(128, file.size)
def test_scan_size_excluded_suffix(self): scanner = SystemScanner(TestSystemScanner.temp_dir) scanner.add_exclude_suffix("ab") scanner.add_exclude_suffix("bb") files = scanner.scan() a, b, c = tuple(files) aa = a.children[0] aaa = aa.children[0] ba = b.children[0] self.assertEqual(0, a.size) self.assertEqual(0, aa.size) self.assertEqual(0, aaa.size) self.assertEqual(512 + 7, b.size) self.assertEqual(512 + 7, ba.size) self.assertEqual(1234, c.size)
def test_scan_tree_excluded_suffix(self): scanner = SystemScanner(TestSystemScanner.temp_dir) scanner.add_exclude_suffix("ab") scanner.add_exclude_suffix("bb") files = scanner.scan() self.assertEqual(3, len(files)) a, b, c = tuple(files) self.assertEqual(1, len(a.children)) aa = a.children[0] self.assertEqual("aa", aa.name) self.assertEqual(1, len(aa.children)) aaa = aa.children[0] self.assertEqual(".aaa", aaa.name) self.assertEqual(1, len(b.children)) ba = b.children[0] self.assertEqual("ba", ba.name)
def test_scan_modified_time(self): # directory os.utime(os.path.join(TestSystemScanner.temp_dir, "a"), (datetime.now().timestamp(), datetime(2018, 11, 9, 21, 40, 18).timestamp())) # file os.utime(os.path.join(TestSystemScanner.temp_dir, "c"), (datetime.now().timestamp(), datetime(2018, 11, 9, 21, 40, 17).timestamp())) scanner = SystemScanner(TestSystemScanner.temp_dir) files = scanner.scan() self.assertEqual(3, len(files)) a, b, c = tuple(files) self.assertEqual(datetime(2018, 11, 9, 21, 40, 18), a.timestamp_modified) self.assertEqual(datetime(2018, 11, 9, 21, 40, 17), c.timestamp_modified)
class LocalScanner(IScanner): """ Scanner implementation to scan the local filesystem """ def __init__(self, local_path: str): self.__scanner = SystemScanner(local_path) self.logger = logging.getLogger("LocalScanner") @overrides(IScanner) def set_base_logger(self, base_logger: logging.Logger): self.logger = base_logger.getChild("LocalScanner") @overrides(IScanner) def scan(self) -> List[SystemFile]: try: result = self.__scanner.scan() except SystemScannerError: self.logger.exception("Caught SystemScannerError") raise AppError(Localization.Error.LOCAL_SERVER_SCAN) return result
class LocalScanner(IScanner): """ Scanner implementation to scan the local filesystem """ def __init__(self, local_path: str, use_temp_file: bool): self.__scanner = SystemScanner(local_path) if use_temp_file: self.__scanner.set_lftp_temp_suffix( Constants.LFTP_TEMP_FILE_SUFFIX) self.logger = logging.getLogger("LocalScanner") @overrides(IScanner) def set_base_logger(self, base_logger: logging.Logger): self.logger = base_logger.getChild("LocalScanner") @overrides(IScanner) def scan(self) -> List[SystemFile]: try: result = self.__scanner.scan() except SystemScannerError: self.logger.exception("Caught SystemScannerError") raise ScannerError(Localization.Error.LOCAL_SERVER_SCAN, recoverable=False) return result
if sys.hexversion < 0x03050000: sys.exit("Python 3.5 or newer is required to run this program.") parser = argparse.ArgumentParser(description="File size scanner") parser.add_argument("path", help="Path of the root directory to scan") parser.add_argument("-e", "--exclude-hidden", action="store_true", default=False, help="Exclude hidden files") parser.add_argument("-H", "--human-readable", action="store_true", default=False, help="Human readable output") args = parser.parse_args() scanner = SystemScanner(args.path) if args.exclude_hidden: scanner.add_exclude_prefix(".") try: root_files = scanner.scan() except SystemScannerError as e: sys.exit("SystemScannerError: {}".format(str(e))) if args.human_readable: def print_file(file: SystemFile, level: int): sys.stdout.write(" "*level) sys.stdout.write("{} {} {}\n".format( file.name, "d" if file.is_dir else "f", file.size )) for child in file.children: print_file(child, level+1) for root_file in root_files: print_file(root_file, 0) else:
def test_scan_lftp_temp_file(self): tempdir = tempfile.mkdtemp(prefix="test_system_scanner") # Create some temp and non-temp files temp1 = os.path.join(tempdir, "a.mkv.lftp") with open(temp1, 'wb') as f: f.write(bytearray([0xff] * 100)) temp2 = os.path.join(tempdir, "b.rar.lftp") with open(temp2, 'wb') as f: f.write(bytearray([0xff] * 200)) nontemp1 = os.path.join(tempdir, "c.rar") with open(nontemp1, 'wb') as f: f.write(bytearray([0xff] * 300)) nontemp2 = os.path.join(tempdir, "d.lftp.avi") with open(nontemp2, 'wb') as f: f.write(bytearray([0xff] * 400)) nontemp3 = os.path.join(tempdir, "e") os.mkdir(nontemp3) temp3 = os.path.join(nontemp3, "ea.txt.lftp") with open(temp3, 'wb') as f: f.write(bytearray([0xff] * 500)) nontemp4 = os.path.join(tempdir, "f.lftp") os.mkdir(nontemp4) scanner = SystemScanner(tempdir) # No temp suffix set files = scanner.scan() self.assertEqual(6, len(files)) a, b, c, d, e, f = tuple(files) self.assertEqual("a.mkv.lftp", a.name) self.assertEqual(100, a.size) self.assertEqual(False, a.is_dir) self.assertEqual("b.rar.lftp", b.name) self.assertEqual(200, b.size) self.assertEqual(False, b.is_dir) self.assertEqual("c.rar", c.name) self.assertEqual(300, c.size) self.assertEqual(False, c.is_dir) self.assertEqual("d.lftp.avi", d.name) self.assertEqual(400, d.size) self.assertEqual(False, d.is_dir) self.assertEqual("e", e.name) self.assertEqual(500, e.size) self.assertEqual(True, e.is_dir) self.assertEqual(1, len(e.children)) ea = e.children[0] self.assertEqual("ea.txt.lftp", ea.name) self.assertEqual(500, ea.size) self.assertEqual(False, ea.is_dir) self.assertEqual("f.lftp", f.name) self.assertEqual(0, f.size) self.assertEqual(True, f.is_dir) # Temp suffix set scanner.set_lftp_temp_suffix(".lftp") files = scanner.scan() self.assertEqual(6, len(files)) a, b, c, d, e, f = tuple(files) self.assertEqual("a.mkv", a.name) self.assertEqual(100, a.size) self.assertEqual(False, a.is_dir) self.assertEqual("b.rar", b.name) self.assertEqual(200, b.size) self.assertEqual(False, b.is_dir) self.assertEqual("c.rar", c.name) self.assertEqual(300, c.size) self.assertEqual(False, c.is_dir) self.assertEqual("d.lftp.avi", d.name) self.assertEqual(400, d.size) self.assertEqual(False, d.is_dir) self.assertEqual("e", e.name) self.assertEqual(500, e.size) self.assertEqual(True, e.is_dir) self.assertEqual(1, len(e.children)) ea = e.children[0] self.assertEqual("ea.txt", ea.name) self.assertEqual(500, ea.size) self.assertEqual(False, ea.is_dir) self.assertEqual("f.lftp", f.name) self.assertEqual(0, f.size) self.assertEqual(True, f.is_dir) # Cleanup shutil.rmtree(tempdir)