def cleanupScan(self):
if self.scanner is not None:
self.scanner.stopScanning()
if self.first_scan:
DocumentStorage.deleteDocumentNamed(self.document_name)
logger.debug("Scanner stopped and was reset")
self.scanner = None
def test_model_directory_can_be_created(self):
# ensure a fake file is listed inside the data model directory
f = open(DocumentStorage.documentFullPath('test.sqlite'), "w")
f.write("something")
f.close()
all_docs = DocumentStorage.documents()
for a_doc in all_docs:
if 'test.sqlite' in a_doc:
return
raise SystemError("Failure - expected test.sqlite to be in these docs somewhere")
def __deleteSelectedItems(self):
toDelete = self.deleteSelectedTreeViewItems(self.ui.treeView, "Remove Scan Files?",
"Are you sure you want to delete the selected scan files - this action cannot be undone")
for rowItems in toDelete:
for item in rowItems:
name = item.data(Qt.DisplayRole).toString()
logger.debug("going to delete: {0}".format(name))
DocumentStorage.deleteDocumentNamed(name)
self.scanFileDeleted.emit(name)
break
self.completeChanged.emit()
def __deleteSelectedItems(self):
toDelete = self.deleteSelectedTreeViewItems(
self.ui.treeView, "Remove Scan Files?",
"Are you sure you want to delete the selected scan files - this action cannot be undone"
)
for rowItems in toDelete:
for item in rowItems:
name = item.data(Qt.DisplayRole).toString()
logger.debug("going to delete: {0}".format(name))
DocumentStorage.deleteDocumentNamed(name)
self.scanFileDeleted.emit(name)
break
self.completeChanged.emit()
def beginChangesScan(self, doc_name):
"""
Find the selected item, and use the directories inside to perform a scan, the merge the results of that
with the existing document - forming a set of objects that have been added/changed/removed
"""
# now fetch the data model for that one to get the paths that need to be scanned
self.storage = PersistentScanningState(DocumentStorage.documentFullPath(doc_name))
#return self.__beginChangesScanWithDocument()
self.__beginChangesScanWithDocument()
def __onListSelectionChanged(self):
numSelected = len(self.ui.listWidget.selectedItems())
if not numSelected:
self.ui.status.setText("")
else:
self.ui.status.setText("%d path%s selected" % (numSelected, "" if numSelected <= 1 else "s"))
self.ui.lineEditDocumentName.setText(DocumentStorage.getNewDocumentName(self.__scanPaths()))
self.completeChanged.emit()
def __onListSelectionChanged(self):
numSelected = len(self.ui.listWidget.selectedItems())
if not numSelected:
self.ui.status.setText("")
else:
self.ui.status.setText("%d path%s selected" % (numSelected, '' if numSelected <= 1 else 's'))
self.ui.lineEditDocumentName.setText(DocumentStorage.getNewDocumentName(self.__scanPaths()))
self.completeChanged.emit()
def beginChangesScan(self, doc_name):
"""
Find the selected item, and use the directories inside to perform a scan, the merge the results of that
with the existing document - forming a set of objects that have been added/changed/removed
"""
# now fetch the data model for that one to get the paths that need to be scanned
self.storage = PersistentScanningState(
DocumentStorage.documentFullPath(doc_name))
#return self.__beginChangesScanWithDocument()
self.__beginChangesScanWithDocument()
def initializePage(self): name_of_file = self.wizard().documentName() self.__refreshResultsUsingDocument(PersistentScanningState(DocumentStorage.documentFullPath(name_of_file))) self.wizard().removePage(WizardPage.CREATE_NEW) self.wizard().removePage(WizardPage.FIRST_SCAN_PROGRESS) self.wizard().removePage(WizardPage.SECOND_SCAN_PROGRESS) self.wizard().removePage(WizardPage.SCANNING_COMPLETE_INSTALL_NOW) self.wizard().reinsertCreateScanPage()
def isComplete(self):
done = self.document_name is not None
if done:
# if the second scan completed OK, we can skip the second scan
fullpath_doc_name = DocumentStorage.documentFullPath(self.document_name)
logger.debug("going with full path of document: {0}".format(fullpath_doc_name))
persist_doc = PersistentScanningState(fullpath_doc_name)
self.setField("secondScanComplete", persist_doc.info.merge_complete)
self.wizard().setDocumentName(self.document_name)
self.wizard().setWindowFilePath(fullpath_doc_name)
return done
def test_scan_is_stored(self):
t = DirectoryScanner()
t.addPathsForScanning([good_app_path()])
# since its an iterable, force its execution
for x in self.p.storeFilesystemSnapshot(t):
pass
# and the dir count contains something
values = self.p.session.query(FileSystemSnapshot).all()
self.assertTrue(len(values) > 0)
self.assertTrue(DocumentStorage.deleteDocumentNamed('test.sqlite'))
def test_scan_is_stored(self):
t = DirectoryScanner()
t.addPathsForScanning([good_app_path()])
# since its an iterable, force its execution
for x in self.p.storeFilesystemSnapshot(t):
pass
# and the dir count contains something
values = self.p.session.query(FileSystemSnapshot).all()
self.assertTrue(len(values) > 0)
self.assertTrue(DocumentStorage.deleteDocumentNamed('test.sqlite'))
def initializePage(self):
name_of_file = self.wizard().documentName()
self.__refreshResultsUsingDocument(
PersistentScanningState(
DocumentStorage.documentFullPath(name_of_file)))
self.wizard().removePage(WizardPage.CREATE_NEW)
self.wizard().removePage(WizardPage.FIRST_SCAN_PROGRESS)
self.wizard().removePage(WizardPage.SECOND_SCAN_PROGRESS)
self.wizard().removePage(WizardPage.SCANNING_COMPLETE_INSTALL_NOW)
self.wizard().reinsertCreateScanPage()
def initializePage(self):
model = QStandardItemModel()
# using a document model, we can obtain all the documents to list inside this view...
all_docs = DocumentStorage.documents()
for doc_filename in all_docs:
# find out the last scan date, number of files, etc
document = PersistentScanningState(doc_filename)
logger.debug("found info: {0}".format(document.info))
creation_date_str = str(document.info.date_created)
last_date = str(document.info.date_last_scanned)
num_scanned = "{0:,}".format(document.info.files_scanned)
num_merged = "{0:,}".format(document.info.files_merged)
is_merged = document.info.merge_complete
if not is_merged:
last_date_str = ""
else:
last_date_str = last_date
doc = os.path.split(doc_filename)[1][:-len('.sqlite')]
items = [
QStandardItem(doc),
QStandardItem(str(num_scanned)),
QStandardItem(creation_date_str),
QStandardItem(str(num_merged)),
QStandardItem(last_date_str),
]
items[1].setData(Qt.AlignRight, Qt.TextAlignmentRole)
items[3].setData(Qt.AlignRight, Qt.TextAlignmentRole)
model.invisibleRootItem().appendRow(items)
self.ui.treeView.setModel(model)
header = self.ui.treeView.header()
model.setHeaderData(0, Qt.Horizontal, "Name")
model.setHeaderData(1, Qt.Horizontal, "# First Scan")
model.setHeaderData(2, Qt.Horizontal, "Date Created")
model.setHeaderData(3, Qt.Horizontal, "# Second Scan")
model.setHeaderData(4, Qt.Horizontal, "Date Scanned")
header.setResizeMode(0, QHeaderView.ResizeToContents)
header.setResizeMode(1, QHeaderView.ResizeToContents)
header.setResizeMode(2, QHeaderView.ResizeToContents)
header.setResizeMode(3, QHeaderView.ResizeToContents)
header.setResizeMode(4, QHeaderView.ResizeToContents)
self.ui.treeView.selectionModel().currentRowChanged.connect(
self.__onSelectionChanged)
def initializePage(self):
model = QStandardItemModel()
# using a document model, we can obtain all the documents to list inside this view...
all_docs = DocumentStorage.documents()
for doc_filename in all_docs:
# find out the last scan date, number of files, etc
document = PersistentScanningState(doc_filename)
logger.debug("found info: {0}".format(document.info))
creation_date_str = str(document.info.date_created)
last_date = str(document.info.date_last_scanned)
num_scanned = "{0:,}".format(document.info.files_scanned)
num_merged = "{0:,}".format(document.info.files_merged)
is_merged = document.info.merge_complete
if not is_merged:
last_date_str = ""
else:
last_date_str = last_date
doc = os.path.split(doc_filename)[1][:-len('.sqlite')]
items = [
QStandardItem(doc),
QStandardItem(str(num_scanned)),
QStandardItem(creation_date_str),
QStandardItem(str(num_merged)),
QStandardItem(last_date_str),
]
items[1].setData(Qt.AlignRight, Qt.TextAlignmentRole)
items[3].setData(Qt.AlignRight, Qt.TextAlignmentRole)
model.invisibleRootItem().appendRow(items)
self.ui.treeView.setModel(model)
header = self.ui.treeView.header()
model.setHeaderData(0, Qt.Horizontal, "Name")
model.setHeaderData(1, Qt.Horizontal, "# First Scan")
model.setHeaderData(2, Qt.Horizontal, "Date Created")
model.setHeaderData(3, Qt.Horizontal, "# Second Scan")
model.setHeaderData(4, Qt.Horizontal, "Date Scanned")
header.setResizeMode(0, QHeaderView.ResizeToContents)
header.setResizeMode(1, QHeaderView.ResizeToContents)
header.setResizeMode(2, QHeaderView.ResizeToContents)
header.setResizeMode(3, QHeaderView.ResizeToContents)
header.setResizeMode(4, QHeaderView.ResizeToContents)
self.ui.treeView.selectionModel().currentRowChanged.connect(self.__onSelectionChanged)
def isComplete(self):
done = self.document_name is not None
if done:
# if the second scan completed OK, we can skip the second scan
fullpath_doc_name = DocumentStorage.documentFullPath(
self.document_name)
logger.debug("going with full path of document: {0}".format(
fullpath_doc_name))
persist_doc = PersistentScanningState(fullpath_doc_name)
self.setField("secondScanComplete",
persist_doc.info.merge_complete)
self.wizard().setDocumentName(self.document_name)
self.wizard().setWindowFilePath(fullpath_doc_name)
return done
def beginInitialScan(self, scan_paths, doc_name=None):
# construct an initial scan, based on the selected items within the model...
# workflow: scan to produce a persistent model, monitor/rescan, re-integrate changes into initial model (changed, deleted, added etc)
scanner = DirectoryScanner()
scanner.addPathsForScanning(scan_paths)
# we want to store the results too - so build a storage device, its all being run via iterables
# so its easy to chain these together.
doc_path = DocumentStorage.getDocumentPathForName(doc_name, scan_paths)
self.storage = PersistentScanningState(doc_path)
self.storage.storePathsBeingScanned(scanner.paths_to_scan)
self.storage.scanningStateChanged.connect(
lambda x: self.scanStateChanged.emit(x))
self.isScanning = True
self.scanStarted.emit(doc_path)
# this one line performs a recursive disk scan on multiple folders, obtains file/dir info, persists
# this to the DB and then finally exposes [idx, f] so the UI can display progress. Long live iterables... (and C++, you can die in a fire)
total_found = 0
for idx, f in enumerate(self.storage.storeFilesystemSnapshot(scanner)):
total_found = idx
if not (idx % 100):
self.scanProgress.emit(idx, f.abs_path)
if not self.isScanning:
break
registryScanner = RegistryScanner()
totalRegistry = 0
for idx, r in enumerate(
self.storage.storeRegistryEntrySnapshot(registryScanner)):
totalRegistry = idx
if not (idx % 100):
self.scanProgress.emit(idx, r.key_name)
if not self.isScanning:
break
self.stopScanning()
# complete the scan...
self.scanFinished.emit(total_found)
self.storage = None
def beginInitialScan(self, scan_paths, doc_name = None):
# construct an initial scan, based on the selected items within the model...
# workflow: scan to produce a persistent model, monitor/rescan, re-integrate changes into initial model (changed, deleted, added etc)
scanner = DirectoryScanner()
scanner.addPathsForScanning(scan_paths)
# we want to store the results too - so build a storage device, its all being run via iterables
# so its easy to chain these together.
doc_path = DocumentStorage.getDocumentPathForName(doc_name, scan_paths)
self.storage = PersistentScanningState(doc_path)
self.storage.storePathsBeingScanned(scanner.paths_to_scan)
self.storage.scanningStateChanged.connect(lambda x: self.scanStateChanged.emit(x))
self.isScanning = True
self.scanStarted.emit(doc_path)
# this one line performs a recursive disk scan on multiple folders, obtains file/dir info, persists
# this to the DB and then finally exposes [idx, f] so the UI can display progress. Long live iterables... (and C++, you can die in a fire)
total_found = 0
for idx, f in enumerate(self.storage.storeFilesystemSnapshot(scanner)):
total_found = idx
if not (idx % 100):
self.scanProgress.emit(idx, f.abs_path)
if not self.isScanning:
break
registryScanner = RegistryScanner()
totalRegistry = 0
for idx ,r in enumerate(self.storage.storeRegistryEntrySnapshot(registryScanner)):
totalRegistry = idx
if not (idx % 100):
self.scanProgress.emit(idx, r.key_name)
if not self.isScanning:
break
self.stopScanning()
# complete the scan...
self.scanFinished.emit(total_found)
self.storage = None
def initializePage(self):
"""
Find and load the persistent document, create the export UI and insert it into the 'container'. This
method assumes that the wizard has a documentName set.
"""
self.container = QWidget(self)
if self.plugin is not None:
self.plugin.ui.setParent(None)
self.gridLayout.removeWidget(self.plugin.ui)
self.plugin.ui = None
self.plugin = None
name_of_file = self.wizard().documentName()
document = PersistentScanningState(DocumentStorage.documentFullPath(name_of_file))
self.plugin = self.__createPluginNamed(self.field("exportType").toString())
ui = self.__createExportUI(document, self.container)
self.gridLayout.addWidget(ui)
self.changed()
def initializePage(self):
"""
Find and load the persistent document, create the export UI and insert it into the 'container'. This
method assumes that the wizard has a documentName set.
"""
self.container = QWidget(self)
if self.plugin is not None:
self.plugin.ui.setParent(None)
self.gridLayout.removeWidget(self.plugin.ui)
self.plugin.ui = None
self.plugin = None
name_of_file = self.wizard().documentName()
document = PersistentScanningState(
DocumentStorage.documentFullPath(name_of_file))
self.plugin = self.__createPluginNamed(
self.field("exportType").toString())
ui = self.__createExportUI(document, self.container)
self.gridLayout.addWidget(ui)
self.changed()
def test_document_gives_specific_filename(self):
new_name = DocumentStorage.getDocumentPathForName("something", None)
self.assertNotEqual(None, new_name)
self.assertTrue(new_name.endswith("something.sqlite"))
def test_document_gives_new_filenames(self):
new_name = DocumentStorage.getNewDocumentName(None)
self.assertTrue('New Scan' in new_name)
def test_creating_name_by_using_scanpaths(self):
name = DocumentStorage.getDocumentPathForName(None, ['/Something'])
base, filename = os.path.split(name)
self.assertTrue(filename.startswith("Something -"))
self.assertTrue(filename.endswith(".sqlite"))
def test_cdrive_path_creates_sane_document_name(self):
path = "c:\\"
name = DocumentStorage.getNewDocumentName([path])
self.assertTrue("c__" in name)
