class PersistTestCase(unittest.TestCase): """ Proves that the scan results can be persisted to a storage form, and re-constituted with the same data. """ def setUp(self): self.p = PersistentScanningState("test.sqlite", echo_sql=False) def tearDown(self): del self.p os.remove("test.sqlite") def test_scan_is_stored(self): t = DirectoryScanner() t.addPathsForScanning([good_app_path()]) # since its an iterable, force its execution for x in self.p.storeFilesystemSnapshot(t): pass # and the dir count contains something values = self.p.session.query(FileSystemSnapshot).all() self.assertTrue(len(values) > 0) self.assertTrue(DocumentStorage.deleteDocumentNamed('test.sqlite')) def test_scan_paths_being_scanned(self): self.p.storePathsBeingScanned(['one_path', 'two_path']) # open up the DB and check... paths = [ p.abs_path for p in self.p.pathsBeingScanned() ] logger.info("paths are: {0}".format(paths)) self.assertTrue( "one_path" in paths ) self.assertTrue( "two_path" in paths )
class PersistTestCase(unittest.TestCase): """ Proves that the scan results can be persisted to a storage form, and re-constituted with the same data. """ def setUp(self): self.p = PersistentScanningState("test.sqlite", echo_sql=False) def tearDown(self): del self.p os.remove("test.sqlite") def test_scan_is_stored(self): t = DirectoryScanner() t.addPathsForScanning([good_app_path()]) # since its an iterable, force its execution for x in self.p.storeFilesystemSnapshot(t): pass # and the dir count contains something values = self.p.session.query(FileSystemSnapshot).all() self.assertTrue(len(values) > 0) self.assertTrue(DocumentStorage.deleteDocumentNamed('test.sqlite')) def test_scan_paths_being_scanned(self): self.p.storePathsBeingScanned(['one_path', 'two_path']) # open up the DB and check... paths = [p.abs_path for p in self.p.pathsBeingScanned()] logger.info("paths are: {0}".format(paths)) self.assertTrue("one_path" in paths) self.assertTrue("two_path" in paths)
class ScanningController(QObject): scanStarted = pyqtSignal(str) scanProgress = pyqtSignal(int, str) scanStateChanged = pyqtSignal(int) scanFinished = pyqtSignal(int) mergeCompleted = pyqtSignal() def __init__(self): super(QObject, self).__init__() self.document = None self.isScanning = True self.storage = None def stopScanning(self): self.isScanning = False if self.storage is not None: self.storage.isScanning = False def beginInitialScan(self, scan_paths, doc_name = None): # construct an initial scan, based on the selected items within the model... # workflow: scan to produce a persistent model, monitor/rescan, re-integrate changes into initial model (changed, deleted, added etc) scanner = DirectoryScanner() scanner.addPathsForScanning(scan_paths) # we want to store the results too - so build a storage device, its all being run via iterables # so its easy to chain these together. doc_path = DocumentStorage.getDocumentPathForName(doc_name, scan_paths) self.storage = PersistentScanningState(doc_path) self.storage.storePathsBeingScanned(scanner.paths_to_scan) self.storage.scanningStateChanged.connect(lambda x: self.scanStateChanged.emit(x)) self.isScanning = True self.scanStarted.emit(doc_path) # this one line performs a recursive disk scan on multiple folders, obtains file/dir info, persists # this to the DB and then finally exposes [idx, f] so the UI can display progress. Long live iterables... (and C++, you can die in a fire) total_found = 0 for idx, f in enumerate(self.storage.storeFilesystemSnapshot(scanner)): total_found = idx if not (idx % 100): self.scanProgress.emit(idx, f.abs_path) if not self.isScanning: break registryScanner = RegistryScanner() totalRegistry = 0 for idx ,r in enumerate(self.storage.storeRegistryEntrySnapshot(registryScanner)): totalRegistry = idx if not (idx % 100): self.scanProgress.emit(idx, r.key_name) if not self.isScanning: break self.stopScanning() # complete the scan... self.scanFinished.emit(total_found) self.storage = None def beginChangesScan(self, doc_name): """ Find the selected item, and use the directories inside to perform a scan, the merge the results of that with the existing document - forming a set of objects that have been added/changed/removed """ # now fetch the data model for that one to get the paths that need to be scanned self.storage = PersistentScanningState(DocumentStorage.documentFullPath(doc_name)) #return self.__beginChangesScanWithDocument() self.__beginChangesScanWithDocument() def __beginChangesScanWithDocument(self): scan_paths = [ p.abs_path for p in self.storage.pathsBeingScanned() ] # kick off another scan scan = DirectoryScanner() scan.addPathsForScanning(scan_paths) self.scanStarted.emit(self.storage.filename) # now re-scan, we'll use a 'merge' facility from the persistent scanning state, it automatically # creates the required set of (added, modified, deleted) files. total_found = 0 merge = self.storage.storeSecondScan(scan) for idx, f in enumerate(merge): total_found = idx #yield idx, f if not (idx % 100): self.scanProgress.emit(idx, ensure_unicode(f.abs_path)) if not self.isScanning: break total_found = 0 registryScanner = RegistryScanner() for idx, r in enumerate(self.storage.storeSecondRegistryScan(registryScanner)): total_found = idx if not (idx % 100): self.scanProgress.emit(idx, ensure_unicode(r.key)) if not self.isScanning: break self.scanFinished.emit(total_found) self.mergeCompleted.emit() self.stopScanning() self.storage = None
class ScanningController(QObject): scanStarted = pyqtSignal(str) scanProgress = pyqtSignal(int, str) scanStateChanged = pyqtSignal(int) scanFinished = pyqtSignal(int) mergeCompleted = pyqtSignal() def __init__(self): super(QObject, self).__init__() self.document = None self.isScanning = True self.storage = None def stopScanning(self): self.isScanning = False if self.storage is not None: self.storage.isScanning = False def beginInitialScan(self, scan_paths, doc_name=None): # construct an initial scan, based on the selected items within the model... # workflow: scan to produce a persistent model, monitor/rescan, re-integrate changes into initial model (changed, deleted, added etc) scanner = DirectoryScanner() scanner.addPathsForScanning(scan_paths) # we want to store the results too - so build a storage device, its all being run via iterables # so its easy to chain these together. doc_path = DocumentStorage.getDocumentPathForName(doc_name, scan_paths) self.storage = PersistentScanningState(doc_path) self.storage.storePathsBeingScanned(scanner.paths_to_scan) self.storage.scanningStateChanged.connect( lambda x: self.scanStateChanged.emit(x)) self.isScanning = True self.scanStarted.emit(doc_path) # this one line performs a recursive disk scan on multiple folders, obtains file/dir info, persists # this to the DB and then finally exposes [idx, f] so the UI can display progress. Long live iterables... (and C++, you can die in a fire) total_found = 0 for idx, f in enumerate(self.storage.storeFilesystemSnapshot(scanner)): total_found = idx if not (idx % 100): self.scanProgress.emit(idx, f.abs_path) if not self.isScanning: break registryScanner = RegistryScanner() totalRegistry = 0 for idx, r in enumerate( self.storage.storeRegistryEntrySnapshot(registryScanner)): totalRegistry = idx if not (idx % 100): self.scanProgress.emit(idx, r.key_name) if not self.isScanning: break self.stopScanning() # complete the scan... self.scanFinished.emit(total_found) self.storage = None def beginChangesScan(self, doc_name): """ Find the selected item, and use the directories inside to perform a scan, the merge the results of that with the existing document - forming a set of objects that have been added/changed/removed """ # now fetch the data model for that one to get the paths that need to be scanned self.storage = PersistentScanningState( DocumentStorage.documentFullPath(doc_name)) #return self.__beginChangesScanWithDocument() self.__beginChangesScanWithDocument() def __beginChangesScanWithDocument(self): scan_paths = [p.abs_path for p in self.storage.pathsBeingScanned()] # kick off another scan scan = DirectoryScanner() scan.addPathsForScanning(scan_paths) self.scanStarted.emit(self.storage.filename) # now re-scan, we'll use a 'merge' facility from the persistent scanning state, it automatically # creates the required set of (added, modified, deleted) files. total_found = 0 merge = self.storage.storeSecondScan(scan) for idx, f in enumerate(merge): total_found = idx #yield idx, f if not (idx % 100): self.scanProgress.emit(idx, ensure_unicode(f.abs_path)) if not self.isScanning: break total_found = 0 registryScanner = RegistryScanner() for idx, r in enumerate( self.storage.storeSecondRegistryScan(registryScanner)): total_found = idx if not (idx % 100): self.scanProgress.emit(idx, ensure_unicode(r.key)) if not self.isScanning: break self.scanFinished.emit(total_found) self.mergeCompleted.emit() self.stopScanning() self.storage = None