def __init__(self, owner=None): super(Widget, self).__init__(owner) # make up a doc self.t = DirectoryScanner() if Platform.isMac: self.t.addPathsForScanning(['/Applications']) else: self.t.addPathsForScanning(['C:\\Program Files']) self.p = PersistentScanningState("demo.sqlite", echo_sql=False) self.initialScan() self.mergeScan() print "Info Is:", self.p.scanInfo() self.mapper = MergeScanMapper(self.p) self.ui = Ui_CustomTreeWidget() self.ui.setupUi(self) self.ui.treeView.setModel( MergeScanTreeModel(self.p, self.mapper, self.p.roots(), self)) self.ui.treeView.expandToDepth(1) model = self.ui.treeView.model() header = self.ui.treeView.header() header.setResizeMode(MergeScanTreeModel.COL_CHECKED, QHeaderView.ResizeToContents) header.setResizeMode(MergeScanTreeModel.COL_PERMISSIONS, QHeaderView.ResizeToContents) self.ui.treeView.setAttribute(Qt.WA_MacShowFocusRect, False)
def test_scan_is_stored(self): t = DirectoryScanner() t.addPathsForScanning([good_app_path()]) # since its an iterable, force its execution for x in self.p.storeFilesystemSnapshot(t): pass # and the dir count contains something values = self.p.session.query(FileSystemSnapshot).all() self.assertTrue(len(values) > 0) self.assertTrue(DocumentStorage.deleteDocumentNamed('test.sqlite'))
def __init__(self, owner = None): super(Widget, self).__init__(owner) # make up a doc self.t = DirectoryScanner() if Platform.isMac: self.t.addPathsForScanning(['/Applications']) else: self.t.addPathsForScanning(['C:\\Program Files']) self.p = PersistentScanningState("demo.sqlite", echo_sql=False) self.initialScan() self.mergeScan() print "Info Is:", self.p.scanInfo() self.mapper = MergeScanMapper(self.p) self.ui = Ui_CustomTreeWidget() self.ui.setupUi(self) self.ui.treeView.setModel(MergeScanTreeModel(self.p, self.mapper, self.p.roots(), self)) self.ui.treeView.expandToDepth(1) model = self.ui.treeView.model() header = self.ui.treeView.header() header.setResizeMode(MergeScanTreeModel.COL_CHECKED, QHeaderView.ResizeToContents) header.setResizeMode(MergeScanTreeModel.COL_PERMISSIONS, QHeaderView.ResizeToContents) self.ui.treeView.setAttribute(Qt.WA_MacShowFocusRect, False)
def setUp(self): self.t = DirectoryScanner() self.builder = DirectoryTreeBuilder() # construct a known directory structure, suitable for testing - it includes files, directories, ACL's, etc self.contents_path = self.builder.make_dir("BB/TextEdit.app/Contents", 0755) self.builder.make_dir("BB/TextEdit.app/Contents/Resources", 0755) self.builder.make_dir("BB/TextEdit.app/Contents/Frameworks", 0755) self.builder.create_file("BB/TextEdit.app/Contents/Stupid.txt", 0755, 425) # produce a single scan of the fake file-system entries self.p = PersistentScanningState("tree-tests.sqlite") self.t.addPathsForScanning([self.builder.rootDir]) self.initialScan() self.mergeScan()
def beginInitialScan(self, scan_paths, doc_name=None): # construct an initial scan, based on the selected items within the model... # workflow: scan to produce a persistent model, monitor/rescan, re-integrate changes into initial model (changed, deleted, added etc) scanner = DirectoryScanner() scanner.addPathsForScanning(scan_paths) # we want to store the results too - so build a storage device, its all being run via iterables # so its easy to chain these together. doc_path = DocumentStorage.getDocumentPathForName(doc_name, scan_paths) self.storage = PersistentScanningState(doc_path) self.storage.storePathsBeingScanned(scanner.paths_to_scan) self.storage.scanningStateChanged.connect( lambda x: self.scanStateChanged.emit(x)) self.isScanning = True self.scanStarted.emit(doc_path) # this one line performs a recursive disk scan on multiple folders, obtains file/dir info, persists # this to the DB and then finally exposes [idx, f] so the UI can display progress. Long live iterables... (and C++, you can die in a fire) total_found = 0 for idx, f in enumerate(self.storage.storeFilesystemSnapshot(scanner)): total_found = idx if not (idx % 100): self.scanProgress.emit(idx, f.abs_path) if not self.isScanning: break registryScanner = RegistryScanner() totalRegistry = 0 for idx, r in enumerate( self.storage.storeRegistryEntrySnapshot(registryScanner)): totalRegistry = idx if not (idx % 100): self.scanProgress.emit(idx, r.key_name) if not self.isScanning: break self.stopScanning() # complete the scan... self.scanFinished.emit(total_found) self.storage = None
def beginInitialScan(self, scan_paths, doc_name = None): # construct an initial scan, based on the selected items within the model... # workflow: scan to produce a persistent model, monitor/rescan, re-integrate changes into initial model (changed, deleted, added etc) scanner = DirectoryScanner() scanner.addPathsForScanning(scan_paths) # we want to store the results too - so build a storage device, its all being run via iterables # so its easy to chain these together. doc_path = DocumentStorage.getDocumentPathForName(doc_name, scan_paths) self.storage = PersistentScanningState(doc_path) self.storage.storePathsBeingScanned(scanner.paths_to_scan) self.storage.scanningStateChanged.connect(lambda x: self.scanStateChanged.emit(x)) self.isScanning = True self.scanStarted.emit(doc_path) # this one line performs a recursive disk scan on multiple folders, obtains file/dir info, persists # this to the DB and then finally exposes [idx, f] so the UI can display progress. Long live iterables... (and C++, you can die in a fire) total_found = 0 for idx, f in enumerate(self.storage.storeFilesystemSnapshot(scanner)): total_found = idx if not (idx % 100): self.scanProgress.emit(idx, f.abs_path) if not self.isScanning: break registryScanner = RegistryScanner() totalRegistry = 0 for idx ,r in enumerate(self.storage.storeRegistryEntrySnapshot(registryScanner)): totalRegistry = idx if not (idx % 100): self.scanProgress.emit(idx, r.key_name) if not self.isScanning: break self.stopScanning() # complete the scan... self.scanFinished.emit(total_found) self.storage = None
def setUp(self): self.t = DirectoryScanner() self.builder = DirectoryTreeBuilder() # construct a known directory structure, suitable for testing - it includes files, directories, ACL's, etc self.contents_path = self.builder.make_dir("AA/TextEdit.app/Contents", 0755) self.builder.make_dir("AA/TextEdit.app/Contents/Resources", 0755) self.builder.make_dir("AA/TextEdit.app/Contents/Frameworks", 0755) self.dbName = "find-diffs.sqlite" self.removeDB() # produce a single scan of the fake file-system entries self.p = PersistentScanningState(self.dbName) self.t.addPathsForScanning([self.builder.rootDir]) self.initialScan()
def test_scan_with_exclusion_of_a_file(self): # scan the directory and store the number of files found there count_of_all = count_of(DirectoryScanner().performScan( utils.good_app_path())) self.assertTrue(count_of_all > 0) if Platform.isMac: self.t.file_excludes.append( r'.*/TextEdit.app/Contents/Info.plist$') else: self.t.file_excludes.append(r'.*python.exe$') count = count_of(self.t.performScan(utils.good_app_path())) self.assertEqual(count, count_of_all - 1)
def __beginChangesScanWithDocument(self): scan_paths = [p.abs_path for p in self.storage.pathsBeingScanned()] # kick off another scan scan = DirectoryScanner() scan.addPathsForScanning(scan_paths) self.scanStarted.emit(self.storage.filename) # now re-scan, we'll use a 'merge' facility from the persistent scanning state, it automatically # creates the required set of (added, modified, deleted) files. total_found = 0 merge = self.storage.storeSecondScan(scan) for idx, f in enumerate(merge): total_found = idx #yield idx, f if not (idx % 100): self.scanProgress.emit(idx, ensure_unicode(f.abs_path)) if not self.isScanning: break total_found = 0 registryScanner = RegistryScanner() for idx, r in enumerate( self.storage.storeSecondRegistryScan(registryScanner)): total_found = idx if not (idx % 100): self.scanProgress.emit(idx, ensure_unicode(r.key)) if not self.isScanning: break self.scanFinished.emit(total_found) self.mergeCompleted.emit() self.stopScanning() self.storage = None
def __beginChangesScanWithDocument(self): scan_paths = [ p.abs_path for p in self.storage.pathsBeingScanned() ] # kick off another scan scan = DirectoryScanner() scan.addPathsForScanning(scan_paths) self.scanStarted.emit(self.storage.filename) # now re-scan, we'll use a 'merge' facility from the persistent scanning state, it automatically # creates the required set of (added, modified, deleted) files. total_found = 0 merge = self.storage.storeSecondScan(scan) for idx, f in enumerate(merge): total_found = idx #yield idx, f if not (idx % 100): self.scanProgress.emit(idx, ensure_unicode(f.abs_path)) if not self.isScanning: break total_found = 0 registryScanner = RegistryScanner() for idx, r in enumerate(self.storage.storeSecondRegistryScan(registryScanner)): total_found = idx if not (idx % 100): self.scanProgress.emit(idx, ensure_unicode(r.key)) if not self.isScanning: break self.scanFinished.emit(total_found) self.mergeCompleted.emit() self.stopScanning() self.storage = None
def test_can_store_changes_to_dir_exclusions(self): new_rule = ".* dir rule.*" # adds a new rule rules = self.r.dirExcludes() rules.append(new_rule) self.r.setDirExcludes(rules) self.assertEqual("FileWave", self.r.settings.organizationName()) self.assertEqual("UnitTest", self.r.settings.applicationName()) # fetch and test (the self.r rules are stored/read in default QSettings, unit test main sets this up) p = DirectoryScanner() self.assertTrue(new_rule not in p.file_excludes) self.assertTrue(new_rule in p.dir_excludes)
class ScanningTestCase(TestCase): def setUp(self): self.t = DirectoryScanner() def test_scanner_can_cope_with_scanning_a_bad_path(self): results = self.t.performScan(utils.bad_app_path()) self.assertEqual(0, count_of(results)) def test_scanner_with_crap_dir_expressions(self): self.t.dir_excludes += "this won't \\(compile - no way" self.assertRaises(re.error, lambda: self.t.performScan(utils.good_app_path())) def test_scanner_with_crap_file_expressions(self): self.t.file_excludes += "this won't \\(compile - no way" self.assertRaises(re.error, lambda: self.t.performScan(utils.good_app_path())) def test_scan_can_run_on_an_existing_path_and_find_some_files(self): count = count_of(self.t.performScan(utils.good_app_path())) logger.info("text edit.app contains {0} items".format(count)) self.assertNotEqual(count, 0, "should not be None, text edit exists right?") def test_scan_with_exclusion_of_everything(self): self.t.dir_excludes.append(r'.*') self.t.file_excludes.append(r'.*') count = count_of(self.t.performScan(utils.good_app_path())) self.assertEqual(count, 0) @unittest.skipIf( Platform.isWindows, "testing exclusion of .DS_Store items makes no sense on Win32") def test_exclusion_of_dsstore_items(self): results = self.t.performScan(utils.good_app_path()) logger.info("scanning for .DS_Store... hope we dont find it") self.assertFalse('.DS_Store' in [file.basename for file in results]) def test_scan_with_exclusion_of_a_file(self): # scan the directory and store the number of files found there count_of_all = count_of(DirectoryScanner().performScan( utils.good_app_path())) self.assertTrue(count_of_all > 0) if Platform.isMac: self.t.file_excludes.append( r'.*/TextEdit.app/Contents/Info.plist$') else: self.t.file_excludes.append(r'.*python.exe$') count = count_of(self.t.performScan(utils.good_app_path())) self.assertEqual(count, count_of_all - 1)
class MacTestBuildTreeModel(unittest.TestCase): """ Tests how a tree model is built in different path/scan situations, and with different input data - esp. between the mac and windows implementations. """ def setUp(self): self.t = DirectoryScanner() self.builder = DirectoryTreeBuilder() # construct a known directory structure, suitable for testing - it includes files, directories, ACL's, etc self.contents_path = self.builder.make_dir("BB/TextEdit.app/Contents", 0755) self.builder.make_dir("BB/TextEdit.app/Contents/Resources", 0755) self.builder.make_dir("BB/TextEdit.app/Contents/Frameworks", 0755) self.builder.create_file("BB/TextEdit.app/Contents/Stupid.txt", 0755, 425) # produce a single scan of the fake file-system entries self.p = PersistentScanningState("tree-tests.sqlite") self.t.addPathsForScanning([self.builder.rootDir]) self.initialScan() self.mergeScan() def initialScan(self): for value in self.p.storeFilesystemSnapshot(self.t.performScan()): pass self.assertTrue(self.p.numberOfScannedFiles() > 0) def mergeScan(self): for value in self.p.storeSecondScan(self.t.performScan()): pass self.assertTrue(self.p.numberOfMergedFiles() > 0) def test_simple_tree_model(self): # grab a fake scan of something we know the content of, e.g. one directory containing one file builder = FileSystemTreeModelBuilder(self.p) builder.buildModel() topLevel = builder.itemRootedAtPath(self.builder.rootDir) itemBB = builder.childOfItem(topLevel, "BB") self.assertTrue(itemBB is not None) self.assertEqual(itemBB.data().toString(), "BB") self.assertEqual(itemBB.rowCount(), 1) # fetch something that doesn't exist - should bring me back None self.assertEqual(None, builder.childOfItem(itemBB, "Not Here")) self.assertEqual(None, builder.itemRootedAtPath(os.path.join(self.builder.rootDir, "WOOT"))) itemTextEdit = builder.childOfItem(itemBB, "TextEdit.app") self.assertTrue(itemTextEdit is not None) self.assertEqual(itemTextEdit.data().toString(), "TextEdit.app") self.assertEqual(itemTextEdit.rowCount(), 1) itemContents = builder.childOfItem(itemTextEdit, "Contents") self.assertTrue(itemContents is not None) self.assertEqual(itemContents.data().toString(), "Contents") self.assertEqual(itemContents.rowCount(), 2) # go grab both - they should be the frameworks and resources items = builder.childrenOfItem(itemContents, ["Resources", "Frameworks"]) self.assertEqual(items["Resources"].data().toString(), "Resources") self.assertEqual(items["Frameworks"].data().toString(), "Frameworks")
def setUp(self): self.t = DirectoryScanner()
class FindDifferencesTestCase(TestCase): def setUp(self): self.t = DirectoryScanner() self.builder = DirectoryTreeBuilder() # construct a known directory structure, suitable for testing - it includes files, directories, ACL's, etc self.contents_path = self.builder.make_dir("AA/TextEdit.app/Contents", 0755) self.builder.make_dir("AA/TextEdit.app/Contents/Resources", 0755) self.builder.make_dir("AA/TextEdit.app/Contents/Frameworks", 0755) self.dbName = "find-diffs.sqlite" self.removeDB() # produce a single scan of the fake file-system entries self.p = PersistentScanningState(self.dbName) self.t.addPathsForScanning([self.builder.rootDir]) self.initialScan() def tearDown(self): self.p = None self.t = None self.builder = None self.removeDB() def removeDB(self): try: if os.path.exists(self.dbName): os.unlink(self.dbName) except Exception: logger.critical("all bets are off - could not delete {0}".format( self.dbName)) def initialScan(self): for value in self.p.storeFilesystemSnapshot(self.t): pass self.assertTrue(self.p.scanInfo().files_scanned > 0) self.assertEqual(0, self.p.scanInfo().files_merged) def mergeScan(self): for value in self.p.storeSecondScan(self.t): pass self.assertTrue(self.p.scanInfo().files_merged > 0) def test_parent_child_relationships_in_file_system(self): # fetch the root FileSystemSnapshot object, grab its children path = FileSystemHelper.convertedPath('%AA/TextEdit.app/Contents') root = self.p.session.query(FileSystemSnapshot).filter( FileSystemSnapshot.abs_path.like(path)).one() self.assertTrue(root is not None) children = root.children for child in children: logger.info("{0}".format(child)) def test_find_new_directory(self): self.builder.make_dir("AA/NewDirectory", 0777) self.mergeScan() ms = self.p.session.query(FileSystemMerge).filter( FileSystemMerge.abs_path.like('%NewDirectory')).one() logger.info("row, after add: {0}".format(ms.abs_path)) self.assertEqual(Qt.Checked, ms.checked) self.assertEqual(PersistentScanningState.ITEM_ADDED, ms.flags) # now find AA, it should be partial because NewDirectory is well, new, and other stuff is marked as Qt.Unchecked fs_parent = self.p.session.query(FileSystemSnapshot).get( ms.parent.abs_path) self.assertTrue(fs_parent is not None) has_parent_changed = fs_parent.path_info != ms.parent.path_info if has_parent_changed: self.assertEqual(Qt.PartiallyChecked, ms.parent.checked) self.assertEqual(PersistentScanningState.ITEM_MODIFIED, ms.parent.flags) else: self.assertEqual(PersistentScanningState.ITEM_UNCHANGED, ms.parent.flags) def test_find_deleted_directory(self): self.builder.del_dir("AA/TextEdit.app/Contents/Frameworks") self.mergeScan() ms = self.p.session.query(FileSystemMerge).filter( FileSystemMerge.abs_path.like('%Frameworks')).one() self.assertTrue( ms.abs_path.endswith( FileSystemHelper.convertedPath( "AA/TextEdit.app/Contents/Frameworks"))) self.assertEqual(PersistentScanningState.ITEM_DELETED, ms.flags) self.assertEqual(Qt.Unchecked, ms.checked) @skipIf(Platform.isWindows, "POSIX tests cannot run on Windows") def test_modified_directory_owner(self): self.builder.change_posix("AA/TextEdit.app/Contents", 0500) self.mergeScan() query = self.p.session.query("ct", "flags", "checked").from_statement( "SELECT count(*) as ct, flags, checked FROM file_system_merge WHERE abs_path LIKE '%Contents'" ).one() count = query.ct flags = query.flags checked = query.checked # even though the two folders UNDER this one are unchanged, the fact that this folder is modified takes precedence self.assertEqual(count, 1) self.assertEqual(flags, PersistentScanningState.ITEM_MODIFIED) self.assertEqual(checked, Qt.Checked) def test_root_scan_paths_are_in_persistent_document(self): paths = self.p.pathsBeingScanned() self.assertEqual(paths[0].abs_path, self.builder.rootDir) # run a merge scan to produce content in the FileSystemMerge table self.mergeScan() # and that this path is explicitly available from the FileSystemMerge model... root_path = self.p.session.query(FileSystemMerge).filter( FileSystemMerge.abs_path == self.builder.rootDir).one() self.assertTrue(root_path is not None) self.assertEqual(root_path.abs_path, self.builder.rootDir) def test_find_modified_directories_and_files(self): pass