def test_scan_single_non_existing_path_fails(self): self.setup_default_tree() scanner = SystemScanner( path_to_scan=os.path.join(TestSystemScanner.temp_dir)) with self.assertRaises(SystemScannerError) as ex: scanner.scan_single("nonexisting") self.assertTrue(str(ex.exception).startswith("Path does not exist"))
def test_scan_single_lftp_partial_file(self): # Scan a single partial file tempdir = tempfile.mkdtemp(prefix="test_system_scanner") # Create a partial file path = os.path.join(tempdir, "partial.mkv") with open(path, 'wb') as f: f.write(bytearray([0xff] * 24588)) # Write the lftp status out path = os.path.join(tempdir, "partial.mkv.lftp-pget-status") with open(path, "w") as f: f.write(""" size=24588 0.pos=3157 0.limit=6147 1.pos=11578 1.limit=12294 2.pos=12295 2.limit=18441 3.pos=20000 3.limit=24588 """) scanner = SystemScanner(tempdir) partial_mkv = scanner.scan_single("partial.mkv") self.assertEqual("partial.mkv", partial_mkv.name) self.assertEqual(10148, partial_mkv.size) # Cleanup shutil.rmtree(tempdir)
def test_scan_single_file(self): scanner = SystemScanner(TestSystemScanner.temp_dir) c = scanner.scan_single("c") self.assertEqual("c", c.name) self.assertFalse(c.is_dir) self.assertEqual(1234, c.size)
def test_scan_single_dir(self): scanner = SystemScanner(TestSystemScanner.temp_dir) a = scanner.scan_single("a") self.assertEqual("a", a.name) self.assertTrue(a.is_dir) self.assertEqual(2, len(a.children)) aa, ab = tuple(a.children) self.assertEqual("aa", aa.name) self.assertTrue(aa.is_dir) self.assertEqual(2, len(aa.children)) aaa, aab = tuple(aa.children) self.assertEqual(".aaa", aaa.name) self.assertTrue(aaa.is_dir) self.assertEqual(".aab", aab.name) self.assertFalse(aab.is_dir) self.assertEqual("ab", ab.name) self.assertFalse(ab.is_dir) self.assertEqual(12 * 1024 + 4 + 512, a.size) self.assertEqual(512, aa.size) self.assertEqual(0, aaa.size) self.assertEqual(512, aab.size) self.assertEqual(12 * 1024 + 4, ab.size)
class ActiveScanner(IScanner): """ Scanner implementation to scan the active files only A caller sets the names of the active files that need to be scanned. A multiprocessing.Queue is used to store the names because the set and scan methods are called by different processes. """ def __init__(self, local_path: str): self.__scanner = SystemScanner(local_path) self.__active_files_queue = multiprocessing.Queue() self.__active_files = [] # latest state self.logger = logging.getLogger(self.__class__.__name__) @overrides(IScanner) def set_base_logger(self, base_logger: logging.Logger): self.logger = base_logger.getChild(self.__class__.__name__) def set_active_files(self, file_names: List[str]): """ Set the list of active file names. Only these files will be scanned. :param file_names: :return: """ self.__active_files_queue.put(file_names) @overrides(IScanner) def scan(self) -> List[SystemFile]: # Grab the latest list of active files, if any try: while True: self.__active_files = self.__active_files_queue.get( block=False) except queue.Empty: pass # Do the scan # self.logger.debug("Scanning files: {}".format(str(self.__active_files))) result = [] for file_name in self.__active_files: try: result.append(self.__scanner.scan_single(file_name)) except SystemScannerError as ex: # Ignore errors here, file may have been deleted self.logger.warning(str(ex)) return result
def test_scan_single_lftp_temp_file(self): tempdir = tempfile.mkdtemp(prefix="test_system_scanner") # Create: # temp file # non-temp file and # non-temp directory with temp name # non-temp directory with non-temp name temp1 = os.path.join(tempdir, "a.mkv.lftp") with open(temp1, 'wb') as f: f.write(bytearray([0xff] * 100)) nontemp1 = os.path.join(tempdir, "b.rar") with open(nontemp1, 'wb') as f: f.write(bytearray([0xff] * 300)) nontemp2 = os.path.join(tempdir, "c.lftp") os.mkdir(nontemp2) temp2 = os.path.join(nontemp2, "c.txt.lftp") with open(temp2, 'wb') as f: f.write(bytearray([0xff] * 500)) nontemp3 = os.path.join(tempdir, "d") os.mkdir(nontemp3) temp3 = os.path.join(nontemp3, "d.avi.lftp") with open(temp3, 'wb') as f: f.write(bytearray([0xff] * 600)) scanner = SystemScanner(tempdir) # No temp suffix set, must include temp suffix in name param file = scanner.scan_single("a.mkv.lftp") self.assertEqual("a.mkv.lftp", file.name) self.assertEqual(100, file.size) self.assertEqual(False, file.is_dir) file = scanner.scan_single("b.rar") self.assertEqual("b.rar", file.name) self.assertEqual(300, file.size) self.assertEqual(False, file.is_dir) file = scanner.scan_single("c.lftp") self.assertEqual("c.lftp", file.name) self.assertEqual(500, file.size) self.assertEqual(True, file.is_dir) self.assertEqual(1, len(file.children)) child = file.children[0] self.assertEqual("c.txt.lftp", child.name) self.assertEqual(500, child.size) self.assertEqual(False, child.is_dir) file = scanner.scan_single("d") self.assertEqual("d", file.name) self.assertEqual(600, file.size) self.assertEqual(True, file.is_dir) child = file.children[0] self.assertEqual("d.avi.lftp", child.name) self.assertEqual(600, child.size) self.assertEqual(False, child.is_dir) # Temp suffix set, must NOT include temp suffix in name param scanner.set_lftp_temp_suffix(".lftp") file = scanner.scan_single("a.mkv") self.assertEqual("a.mkv", file.name) self.assertEqual(100, file.size) self.assertEqual(False, file.is_dir) file = scanner.scan_single("b.rar") self.assertEqual("b.rar", file.name) self.assertEqual(300, file.size) self.assertEqual(False, file.is_dir) file = scanner.scan_single("c.lftp") self.assertEqual("c.lftp", file.name) self.assertEqual(500, file.size) self.assertEqual(True, file.is_dir) self.assertEqual(1, len(file.children)) child = file.children[0] self.assertEqual("c.txt", child.name) self.assertEqual(500, child.size) self.assertEqual(False, child.is_dir) # also, shouldn't look for directories with temp suffix with self.assertRaises(SystemScannerError) as ctx: scanner.scan_single("c") self.assertTrue("Path does not exist" in str(ctx.exception)) file = scanner.scan_single("d") self.assertEqual("d", file.name) self.assertEqual(600, file.size) self.assertEqual(True, file.is_dir) child = file.children[0] self.assertEqual("d.avi", child.name) self.assertEqual(600, child.size) self.assertEqual(False, child.is_dir) # No file and no temp file with self.assertRaises(SystemScannerError) as ctx: scanner.scan_single("blah") self.assertTrue("Path does not exist" in str(ctx.exception)) # Cleanup shutil.rmtree(tempdir)