def __init__(self, owner=None):
super(Widget, self).__init__(owner)
# make up a doc
self.t = DirectoryScanner()
if Platform.isMac:
self.t.addPathsForScanning(['/Applications'])
else:
self.t.addPathsForScanning(['C:\\Program Files'])
self.p = PersistentScanningState("demo.sqlite", echo_sql=False)
self.initialScan()
self.mergeScan()
print "Info Is:", self.p.scanInfo()
self.mapper = MergeScanMapper(self.p)
self.ui = Ui_CustomTreeWidget()
self.ui.setupUi(self)
self.ui.treeView.setModel(
MergeScanTreeModel(self.p, self.mapper, self.p.roots(), self))
self.ui.treeView.expandToDepth(1)
model = self.ui.treeView.model()
header = self.ui.treeView.header()
header.setResizeMode(MergeScanTreeModel.COL_CHECKED,
QHeaderView.ResizeToContents)
header.setResizeMode(MergeScanTreeModel.COL_PERMISSIONS,
QHeaderView.ResizeToContents)
self.ui.treeView.setAttribute(Qt.WA_MacShowFocusRect, False)
def test_scan_is_stored(self):
t = DirectoryScanner()
t.addPathsForScanning([good_app_path()])
# since its an iterable, force its execution
for x in self.p.storeFilesystemSnapshot(t):
pass
# and the dir count contains something
values = self.p.session.query(FileSystemSnapshot).all()
self.assertTrue(len(values) > 0)
self.assertTrue(DocumentStorage.deleteDocumentNamed('test.sqlite'))
def test_scan_is_stored(self):
t = DirectoryScanner()
t.addPathsForScanning([good_app_path()])
# since its an iterable, force its execution
for x in self.p.storeFilesystemSnapshot(t):
pass
# and the dir count contains something
values = self.p.session.query(FileSystemSnapshot).all()
self.assertTrue(len(values) > 0)
self.assertTrue(DocumentStorage.deleteDocumentNamed('test.sqlite'))
def __init__(self, owner = None):
super(Widget, self).__init__(owner)
# make up a doc
self.t = DirectoryScanner()
if Platform.isMac:
self.t.addPathsForScanning(['/Applications'])
else:
self.t.addPathsForScanning(['C:\\Program Files'])
self.p = PersistentScanningState("demo.sqlite", echo_sql=False)
self.initialScan()
self.mergeScan()
print "Info Is:", self.p.scanInfo()
self.mapper = MergeScanMapper(self.p)
self.ui = Ui_CustomTreeWidget()
self.ui.setupUi(self)
self.ui.treeView.setModel(MergeScanTreeModel(self.p, self.mapper, self.p.roots(), self))
self.ui.treeView.expandToDepth(1)
model = self.ui.treeView.model()
header = self.ui.treeView.header()
header.setResizeMode(MergeScanTreeModel.COL_CHECKED, QHeaderView.ResizeToContents)
header.setResizeMode(MergeScanTreeModel.COL_PERMISSIONS, QHeaderView.ResizeToContents)
self.ui.treeView.setAttribute(Qt.WA_MacShowFocusRect, False)
def setUp(self):
self.t = DirectoryScanner()
self.builder = DirectoryTreeBuilder()
# construct a known directory structure, suitable for testing - it includes files, directories, ACL's, etc
self.contents_path = self.builder.make_dir("BB/TextEdit.app/Contents", 0755)
self.builder.make_dir("BB/TextEdit.app/Contents/Resources", 0755)
self.builder.make_dir("BB/TextEdit.app/Contents/Frameworks", 0755)
self.builder.create_file("BB/TextEdit.app/Contents/Stupid.txt", 0755, 425)
# produce a single scan of the fake file-system entries
self.p = PersistentScanningState("tree-tests.sqlite")
self.t.addPathsForScanning([self.builder.rootDir])
self.initialScan()
self.mergeScan()
def beginInitialScan(self, scan_paths, doc_name=None):
# construct an initial scan, based on the selected items within the model...
# workflow: scan to produce a persistent model, monitor/rescan, re-integrate changes into initial model (changed, deleted, added etc)
scanner = DirectoryScanner()
scanner.addPathsForScanning(scan_paths)
# we want to store the results too - so build a storage device, its all being run via iterables
# so its easy to chain these together.
doc_path = DocumentStorage.getDocumentPathForName(doc_name, scan_paths)
self.storage = PersistentScanningState(doc_path)
self.storage.storePathsBeingScanned(scanner.paths_to_scan)
self.storage.scanningStateChanged.connect(
lambda x: self.scanStateChanged.emit(x))
self.isScanning = True
self.scanStarted.emit(doc_path)
# this one line performs a recursive disk scan on multiple folders, obtains file/dir info, persists
# this to the DB and then finally exposes [idx, f] so the UI can display progress. Long live iterables... (and C++, you can die in a fire)
total_found = 0
for idx, f in enumerate(self.storage.storeFilesystemSnapshot(scanner)):
total_found = idx
if not (idx % 100):
self.scanProgress.emit(idx, f.abs_path)
if not self.isScanning:
break
registryScanner = RegistryScanner()
totalRegistry = 0
for idx, r in enumerate(
self.storage.storeRegistryEntrySnapshot(registryScanner)):
totalRegistry = idx
if not (idx % 100):
self.scanProgress.emit(idx, r.key_name)
if not self.isScanning:
break
self.stopScanning()
# complete the scan...
self.scanFinished.emit(total_found)
self.storage = None
def beginInitialScan(self, scan_paths, doc_name = None):
# construct an initial scan, based on the selected items within the model...
# workflow: scan to produce a persistent model, monitor/rescan, re-integrate changes into initial model (changed, deleted, added etc)
scanner = DirectoryScanner()
scanner.addPathsForScanning(scan_paths)
# we want to store the results too - so build a storage device, its all being run via iterables
# so its easy to chain these together.
doc_path = DocumentStorage.getDocumentPathForName(doc_name, scan_paths)
self.storage = PersistentScanningState(doc_path)
self.storage.storePathsBeingScanned(scanner.paths_to_scan)
self.storage.scanningStateChanged.connect(lambda x: self.scanStateChanged.emit(x))
self.isScanning = True
self.scanStarted.emit(doc_path)
# this one line performs a recursive disk scan on multiple folders, obtains file/dir info, persists
# this to the DB and then finally exposes [idx, f] so the UI can display progress. Long live iterables... (and C++, you can die in a fire)
total_found = 0
for idx, f in enumerate(self.storage.storeFilesystemSnapshot(scanner)):
total_found = idx
if not (idx % 100):
self.scanProgress.emit(idx, f.abs_path)
if not self.isScanning:
break
registryScanner = RegistryScanner()
totalRegistry = 0
for idx ,r in enumerate(self.storage.storeRegistryEntrySnapshot(registryScanner)):
totalRegistry = idx
if not (idx % 100):
self.scanProgress.emit(idx, r.key_name)
if not self.isScanning:
break
self.stopScanning()
# complete the scan...
self.scanFinished.emit(total_found)
self.storage = None
def setUp(self):
self.t = DirectoryScanner()
self.builder = DirectoryTreeBuilder()
# construct a known directory structure, suitable for testing - it includes files, directories, ACL's, etc
self.contents_path = self.builder.make_dir("AA/TextEdit.app/Contents",
0755)
self.builder.make_dir("AA/TextEdit.app/Contents/Resources", 0755)
self.builder.make_dir("AA/TextEdit.app/Contents/Frameworks", 0755)
self.dbName = "find-diffs.sqlite"
self.removeDB()
# produce a single scan of the fake file-system entries
self.p = PersistentScanningState(self.dbName)
self.t.addPathsForScanning([self.builder.rootDir])
self.initialScan()
def test_scan_with_exclusion_of_a_file(self):
# scan the directory and store the number of files found there
count_of_all = count_of(DirectoryScanner().performScan(
utils.good_app_path()))
self.assertTrue(count_of_all > 0)
if Platform.isMac:
self.t.file_excludes.append(
r'.*/TextEdit.app/Contents/Info.plist$')
else:
self.t.file_excludes.append(r'.*python.exe$')
count = count_of(self.t.performScan(utils.good_app_path()))
self.assertEqual(count, count_of_all - 1)
def __beginChangesScanWithDocument(self):
scan_paths = [p.abs_path for p in self.storage.pathsBeingScanned()]
# kick off another scan
scan = DirectoryScanner()
scan.addPathsForScanning(scan_paths)
self.scanStarted.emit(self.storage.filename)
# now re-scan, we'll use a 'merge' facility from the persistent scanning state, it automatically
# creates the required set of (added, modified, deleted) files.
total_found = 0
merge = self.storage.storeSecondScan(scan)
for idx, f in enumerate(merge):
total_found = idx
#yield idx, f
if not (idx % 100):
self.scanProgress.emit(idx, ensure_unicode(f.abs_path))
if not self.isScanning:
break
total_found = 0
registryScanner = RegistryScanner()
for idx, r in enumerate(
self.storage.storeSecondRegistryScan(registryScanner)):
total_found = idx
if not (idx % 100):
self.scanProgress.emit(idx, ensure_unicode(r.key))
if not self.isScanning:
break
self.scanFinished.emit(total_found)
self.mergeCompleted.emit()
self.stopScanning()
self.storage = None
def __beginChangesScanWithDocument(self):
scan_paths = [ p.abs_path for p in self.storage.pathsBeingScanned() ]
# kick off another scan
scan = DirectoryScanner()
scan.addPathsForScanning(scan_paths)
self.scanStarted.emit(self.storage.filename)
# now re-scan, we'll use a 'merge' facility from the persistent scanning state, it automatically
# creates the required set of (added, modified, deleted) files.
total_found = 0
merge = self.storage.storeSecondScan(scan)
for idx, f in enumerate(merge):
total_found = idx
#yield idx, f
if not (idx % 100):
self.scanProgress.emit(idx, ensure_unicode(f.abs_path))
if not self.isScanning:
break
total_found = 0
registryScanner = RegistryScanner()
for idx, r in enumerate(self.storage.storeSecondRegistryScan(registryScanner)):
total_found = idx
if not (idx % 100):
self.scanProgress.emit(idx, ensure_unicode(r.key))
if not self.isScanning:
break
self.scanFinished.emit(total_found)
self.mergeCompleted.emit()
self.stopScanning()
self.storage = None
def test_can_store_changes_to_dir_exclusions(self):
new_rule = ".* dir rule.*"
# adds a new rule
rules = self.r.dirExcludes()
rules.append(new_rule)
self.r.setDirExcludes(rules)
self.assertEqual("FileWave", self.r.settings.organizationName())
self.assertEqual("UnitTest", self.r.settings.applicationName())
# fetch and test (the self.r rules are stored/read in default QSettings, unit test main sets this up)
p = DirectoryScanner()
self.assertTrue(new_rule not in p.file_excludes)
self.assertTrue(new_rule in p.dir_excludes)
class ScanningTestCase(TestCase):
def setUp(self):
self.t = DirectoryScanner()
def test_scanner_can_cope_with_scanning_a_bad_path(self):
results = self.t.performScan(utils.bad_app_path())
self.assertEqual(0, count_of(results))
def test_scanner_with_crap_dir_expressions(self):
self.t.dir_excludes += "this won't \\(compile - no way"
self.assertRaises(re.error,
lambda: self.t.performScan(utils.good_app_path()))
def test_scanner_with_crap_file_expressions(self):
self.t.file_excludes += "this won't \\(compile - no way"
self.assertRaises(re.error,
lambda: self.t.performScan(utils.good_app_path()))
def test_scan_can_run_on_an_existing_path_and_find_some_files(self):
count = count_of(self.t.performScan(utils.good_app_path()))
logger.info("text edit.app contains {0} items".format(count))
self.assertNotEqual(count, 0,
"should not be None, text edit exists right?")
def test_scan_with_exclusion_of_everything(self):
self.t.dir_excludes.append(r'.*')
self.t.file_excludes.append(r'.*')
count = count_of(self.t.performScan(utils.good_app_path()))
self.assertEqual(count, 0)
@unittest.skipIf(
Platform.isWindows,
"testing exclusion of .DS_Store items makes no sense on Win32")
def test_exclusion_of_dsstore_items(self):
results = self.t.performScan(utils.good_app_path())
logger.info("scanning for .DS_Store... hope we dont find it")
self.assertFalse('.DS_Store' in [file.basename for file in results])
def test_scan_with_exclusion_of_a_file(self):
# scan the directory and store the number of files found there
count_of_all = count_of(DirectoryScanner().performScan(
utils.good_app_path()))
self.assertTrue(count_of_all > 0)
if Platform.isMac:
self.t.file_excludes.append(
r'.*/TextEdit.app/Contents/Info.plist$')
else:
self.t.file_excludes.append(r'.*python.exe$')
count = count_of(self.t.performScan(utils.good_app_path()))
self.assertEqual(count, count_of_all - 1)
class MacTestBuildTreeModel(unittest.TestCase):
"""
Tests how a tree model is built in different path/scan situations, and with different input data - esp. between
the mac and windows implementations.
"""
def setUp(self):
self.t = DirectoryScanner()
self.builder = DirectoryTreeBuilder()
# construct a known directory structure, suitable for testing - it includes files, directories, ACL's, etc
self.contents_path = self.builder.make_dir("BB/TextEdit.app/Contents", 0755)
self.builder.make_dir("BB/TextEdit.app/Contents/Resources", 0755)
self.builder.make_dir("BB/TextEdit.app/Contents/Frameworks", 0755)
self.builder.create_file("BB/TextEdit.app/Contents/Stupid.txt", 0755, 425)
# produce a single scan of the fake file-system entries
self.p = PersistentScanningState("tree-tests.sqlite")
self.t.addPathsForScanning([self.builder.rootDir])
self.initialScan()
self.mergeScan()
def initialScan(self):
for value in self.p.storeFilesystemSnapshot(self.t.performScan()):
pass
self.assertTrue(self.p.numberOfScannedFiles() > 0)
def mergeScan(self):
for value in self.p.storeSecondScan(self.t.performScan()):
pass
self.assertTrue(self.p.numberOfMergedFiles() > 0)
def test_simple_tree_model(self):
# grab a fake scan of something we know the content of, e.g. one directory containing one file
builder = FileSystemTreeModelBuilder(self.p)
builder.buildModel()
topLevel = builder.itemRootedAtPath(self.builder.rootDir)
itemBB = builder.childOfItem(topLevel, "BB")
self.assertTrue(itemBB is not None)
self.assertEqual(itemBB.data().toString(), "BB")
self.assertEqual(itemBB.rowCount(), 1)
# fetch something that doesn't exist - should bring me back None
self.assertEqual(None, builder.childOfItem(itemBB, "Not Here"))
self.assertEqual(None, builder.itemRootedAtPath(os.path.join(self.builder.rootDir, "WOOT")))
itemTextEdit = builder.childOfItem(itemBB, "TextEdit.app")
self.assertTrue(itemTextEdit is not None)
self.assertEqual(itemTextEdit.data().toString(), "TextEdit.app")
self.assertEqual(itemTextEdit.rowCount(), 1)
itemContents = builder.childOfItem(itemTextEdit, "Contents")
self.assertTrue(itemContents is not None)
self.assertEqual(itemContents.data().toString(), "Contents")
self.assertEqual(itemContents.rowCount(), 2)
# go grab both - they should be the frameworks and resources
items = builder.childrenOfItem(itemContents, ["Resources", "Frameworks"])
self.assertEqual(items["Resources"].data().toString(), "Resources")
self.assertEqual(items["Frameworks"].data().toString(), "Frameworks")
def setUp(self):
self.t = DirectoryScanner()
class FindDifferencesTestCase(TestCase):
def setUp(self):
self.t = DirectoryScanner()
self.builder = DirectoryTreeBuilder()
# construct a known directory structure, suitable for testing - it includes files, directories, ACL's, etc
self.contents_path = self.builder.make_dir("AA/TextEdit.app/Contents",
0755)
self.builder.make_dir("AA/TextEdit.app/Contents/Resources", 0755)
self.builder.make_dir("AA/TextEdit.app/Contents/Frameworks", 0755)
self.dbName = "find-diffs.sqlite"
self.removeDB()
# produce a single scan of the fake file-system entries
self.p = PersistentScanningState(self.dbName)
self.t.addPathsForScanning([self.builder.rootDir])
self.initialScan()
def tearDown(self):
self.p = None
self.t = None
self.builder = None
self.removeDB()
def removeDB(self):
try:
if os.path.exists(self.dbName):
os.unlink(self.dbName)
except Exception:
logger.critical("all bets are off - could not delete {0}".format(
self.dbName))
def initialScan(self):
for value in self.p.storeFilesystemSnapshot(self.t):
pass
self.assertTrue(self.p.scanInfo().files_scanned > 0)
self.assertEqual(0, self.p.scanInfo().files_merged)
def mergeScan(self):
for value in self.p.storeSecondScan(self.t):
pass
self.assertTrue(self.p.scanInfo().files_merged > 0)
def test_parent_child_relationships_in_file_system(self):
# fetch the root FileSystemSnapshot object, grab its children
path = FileSystemHelper.convertedPath('%AA/TextEdit.app/Contents')
root = self.p.session.query(FileSystemSnapshot).filter(
FileSystemSnapshot.abs_path.like(path)).one()
self.assertTrue(root is not None)
children = root.children
for child in children:
logger.info("{0}".format(child))
def test_find_new_directory(self):
self.builder.make_dir("AA/NewDirectory", 0777)
self.mergeScan()
ms = self.p.session.query(FileSystemMerge).filter(
FileSystemMerge.abs_path.like('%NewDirectory')).one()
logger.info("row, after add: {0}".format(ms.abs_path))
self.assertEqual(Qt.Checked, ms.checked)
self.assertEqual(PersistentScanningState.ITEM_ADDED, ms.flags)
# now find AA, it should be partial because NewDirectory is well, new, and other stuff is marked as Qt.Unchecked
fs_parent = self.p.session.query(FileSystemSnapshot).get(
ms.parent.abs_path)
self.assertTrue(fs_parent is not None)
has_parent_changed = fs_parent.path_info != ms.parent.path_info
if has_parent_changed:
self.assertEqual(Qt.PartiallyChecked, ms.parent.checked)
self.assertEqual(PersistentScanningState.ITEM_MODIFIED,
ms.parent.flags)
else:
self.assertEqual(PersistentScanningState.ITEM_UNCHANGED,
ms.parent.flags)
def test_find_deleted_directory(self):
self.builder.del_dir("AA/TextEdit.app/Contents/Frameworks")
self.mergeScan()
ms = self.p.session.query(FileSystemMerge).filter(
FileSystemMerge.abs_path.like('%Frameworks')).one()
self.assertTrue(
ms.abs_path.endswith(
FileSystemHelper.convertedPath(
"AA/TextEdit.app/Contents/Frameworks")))
self.assertEqual(PersistentScanningState.ITEM_DELETED, ms.flags)
self.assertEqual(Qt.Unchecked, ms.checked)
@skipIf(Platform.isWindows, "POSIX tests cannot run on Windows")
def test_modified_directory_owner(self):
self.builder.change_posix("AA/TextEdit.app/Contents", 0500)
self.mergeScan()
query = self.p.session.query("ct", "flags", "checked").from_statement(
"SELECT count(*) as ct, flags, checked FROM file_system_merge WHERE abs_path LIKE '%Contents'"
).one()
count = query.ct
flags = query.flags
checked = query.checked
# even though the two folders UNDER this one are unchanged, the fact that this folder is modified takes precedence
self.assertEqual(count, 1)
self.assertEqual(flags, PersistentScanningState.ITEM_MODIFIED)
self.assertEqual(checked, Qt.Checked)
def test_root_scan_paths_are_in_persistent_document(self):
paths = self.p.pathsBeingScanned()
self.assertEqual(paths[0].abs_path, self.builder.rootDir)
# run a merge scan to produce content in the FileSystemMerge table
self.mergeScan()
# and that this path is explicitly available from the FileSystemMerge model...
root_path = self.p.session.query(FileSystemMerge).filter(
FileSystemMerge.abs_path == self.builder.rootDir).one()
self.assertTrue(root_path is not None)
self.assertEqual(root_path.abs_path, self.builder.rootDir)
def test_find_modified_directories_and_files(self):
pass
