def clipboardNodes(self):
progress = ProgressBar("Copy to clipboard", self.mainWindow)
indexes = self.mainWindow.tree.selectionModel().selectedRows()
progress.setMaximum(len(indexes))
output = io.StringIO()
try:
writer = csv.writer(output, delimiter='\t', quotechar='"', quoting=csv.QUOTE_ALL, doublequote=True,
lineterminator='\r\n')
#headers
row = [str(val) for val in self.mainWindow.tree.treemodel.getRowHeader()]
writer.writerow(row)
#rows
for no in range(len(indexes)):
if progress.wasCanceled:
break
row = [str(val) for val in self.mainWindow.tree.treemodel.getRowData(indexes[no])]
writer.writerow(row)
progress.step()
clipboard = QApplication.clipboard()
clipboard.setText(output.getvalue())
finally:
output.close()
progress.close()
def exportSelectedNodes(self, output):
progress = ProgressBar("Exporting data...", self.mainWindow)
#indexes = self.mainWindow.tree.selectionModel().selectedRows()
#if child nodes should be exported as well, uncomment this line an comment the previous one
indexes = self.mainWindow.tree.selectedIndexesAndChildren()
indexes = list(indexes)
progress.setMaximum(len(indexes))
try:
delimiter = self.optionSeparator.currentText()
delimiter = delimiter.encode('utf-8').decode('unicode_escape')
writer = csv.writer(output,
delimiter=delimiter,
quotechar='"',
quoting=csv.QUOTE_ALL,
doublequote=True,
lineterminator='\r\n')
#headers
row = [
str(val)
for val in self.mainWindow.tree.treemodel.getRowHeader()
]
if self.optionLinebreaks.isChecked():
row = [
val.replace('\n', ' ').replace('\r', ' ') for val in row
]
row = ['path'] + row
writer.writerow(row)
#rows
path = []
for index in indexes:
if progress.wasCanceled:
break
# data
rowdata = self.mainWindow.tree.treemodel.getRowData(index)
# path of parents (#2=level;#3=object ID)
while rowdata[2] < len(path):
path.pop()
path.append(rowdata[3])
# values
row = [str(val) for val in rowdata]
row = ["/".join(path)] + row
if self.optionLinebreaks.isChecked():
row = [
val.replace('\n', ' ').replace('\r', ' ')
for val in row
]
writer.writerow(row)
progress.step()
finally:
progress.close()
def exportAllNodes(self,output):
progress = ProgressBar("Exporting data...", self.mainWindow)
progress.setMaximum(Node.query.count())
try:
delimiter = self.optionSeparator.currentText()
delimiter = delimiter.encode('utf-8').decode('unicode_escape')
writer = csv.writer(output, delimiter=delimiter, quotechar='"',
quoting=csv.QUOTE_ALL, doublequote=True,
lineterminator='\r\n')
# Headers
row = ["level", "id", "parent_id", "object_id", "object_type","object_key",
"query_status", "query_time", "query_type"]
for key in extractNames(self.mainWindow.tree.treemodel.customcolumns):
row.append(key)
if self.optionLinebreaks.isChecked():
row = [val.replace('\n', ' ').replace('\r',' ') for val in row]
writer.writerow(row)
# Rows
page = 0
while not progress.wasCanceled:
allnodes = Node.query.offset(page * 5000).limit(5000)
if allnodes.count() == 0:
break
for node in allnodes:
if progress.wasCanceled:
break
row = [node.level, node.id, node.parent_id, node.objectid,
node.objecttype,getDictValue(node.queryparams,'nodedata'),
node.querystatus, node.querytime, node.querytype]
for key in self.mainWindow.tree.treemodel.customcolumns:
row.append(node.getResponseValue(key)[1])
if self.optionLinebreaks.isChecked():
row = [str(val).replace('\n', ' ').replace('\r',' ') for val in row]
writer.writerow(row)
# Step the bar
progress.step()
page += 1
finally:
progress.close()
def addAllColumns(self):
progress = ProgressBar("Analyzing data...", self.mainWindow)
columns = self.mainWindow.fieldList.toPlainText().splitlines()
try:
indexes = self.mainWindow.tree.selectedIndexesAndChildren()
indexes = list(indexes)
progress.setMaximum(len(indexes))
for no in range(len(indexes)):
progress.step()
item = indexes[no].internalPointer()
columns.extend([key for key in recursiveIterKeys(item.data['response']) if not key in columns])
if progress.wasCanceled:
break
finally:
self.mainWindow.fieldList.setPlainText("\n".join(columns))
self.mainWindow.tree.treemodel.setCustomColumns(columns)
progress.close()
def deleteNodes(self):
reply = QMessageBox.question(self.mainWindow, 'Delete Nodes', "Are you sure to delete all selected nodes?",
QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
if reply != QMessageBox.Yes:
return
progress = ProgressBar("Deleting data...", self.mainWindow)
self.mainWindow.tree.setUpdatesEnabled(False)
try:
todo = self.mainWindow.tree.selectedIndexesAndChildren({'persistent': True})
todo = list(todo)
progress.setMaximum(len(todo))
for index in todo:
progress.step()
self.mainWindow.tree.treemodel.deleteNode(index, delaycommit=True)
if progress.wasCanceled:
break
finally:
# commit the operation on the db-layer afterwards (delaycommit is True)
self.mainWindow.tree.treemodel.commitNewNodes()
self.mainWindow.tree.setUpdatesEnabled(True)
progress.close()
class PresetWindow(QDialog):
logmessage = Signal(str)
progressStart = Signal()
progressShow = Signal()
progressMax = Signal(int)
progressStep = Signal()
progressStop = Signal()
def __init__(self, parent=None):
super(PresetWindow, self).__init__(parent)
self.mainWindow = parent
self.setWindowTitle("Presets")
self.setMinimumWidth(800)
self.setMinimumHeight(600)
#layout
layout = QVBoxLayout(self)
self.setLayout(layout)
#loading indicator
self.loadingLock = threading.Lock()
self.loadingIndicator = QLabel('Loading...please wait a second.')
self.loadingIndicator.hide()
layout.addWidget(self.loadingIndicator)
#Middle
central = QSplitter(self)
layout.addWidget(central, 1)
#list view
self.presetList = QTreeWidget(self)
self.presetList.setHeaderHidden(True)
self.presetList.setColumnCount(1)
self.presetList.setIndentation(15)
self.presetList.itemSelectionChanged.connect(self.currentChanged)
central.addWidget(self.presetList)
central.setStretchFactor(0, 0)
# category / pipeline
self.categoryWidget = QWidget()
p = self.categoryWidget.palette()
p.setColor(self.categoryWidget.backgroundRole(), Qt.white)
self.categoryWidget.setPalette(p)
self.categoryWidget.setAutoFillBackground(True)
self.categoryLayout = QVBoxLayout()
#self.categoryLayout.setContentsMargins(0, 0, 0, 0)
self.categoryWidget.setLayout(self.categoryLayout)
self.categoryView = QScrollArea()
self.categoryView.setWidgetResizable(True)
self.categoryView.setWidget(self.categoryWidget)
central.addWidget(self.categoryView)
central.setStretchFactor(1, 2)
# Pipeline header
self.pipelineName = QLabel('')
self.pipelineName.setWordWrap(True)
self.pipelineName.setStyleSheet("QLabel {font-size:15pt;}")
self.categoryLayout.addWidget(self.pipelineName)
# Pipeline items
# self.pipelineWidget = QTreeWidget()
# self.pipelineWidget.setIndentation(0)
# self.pipelineWidget.setUniformRowHeights(True)
# self.pipelineWidget.setColumnCount(4)
# self.pipelineWidget.setHeaderLabels(['Name','Module','Basepath','Resource'])
# self.categoryLayout.addWidget(self.pipelineWidget)
# preset widget
self.presetWidget = QWidget()
p = self.presetWidget.palette()
p.setColor(self.presetWidget.backgroundRole(), Qt.white)
self.presetWidget.setPalette(p)
self.presetWidget.setAutoFillBackground(True)
#self.presetWidget.setStyleSheet("background-color: rgb(255,255,255);")
self.presetView = QScrollArea()
self.presetView.setWidgetResizable(True)
self.presetView.setWidget(self.presetWidget)
central.addWidget(self.presetView)
central.setStretchFactor(2, 2)
#self.detailView.setFrameStyle(QFrame.Box)
self.presetLayout = QVBoxLayout()
self.presetWidget.setLayout(self.presetLayout)
self.detailName = QLabel('')
self.detailName.setWordWrap(True)
self.detailName.setStyleSheet("QLabel {font-size:15pt;}")
self.presetLayout.addWidget(self.detailName)
self.detailDescription = TextViewer()
self.presetLayout.addWidget(self.detailDescription)
self.presetForm = QFormLayout()
self.presetForm.setRowWrapPolicy(QFormLayout.DontWrapRows)
self.presetForm.setFieldGrowthPolicy(QFormLayout.AllNonFixedFieldsGrow)
self.presetForm.setFormAlignment(Qt.AlignLeft | Qt.AlignTop)
self.presetForm.setLabelAlignment(Qt.AlignLeft)
self.presetLayout.addLayout(self.presetForm, 1)
# Module
self.detailModule = QLabel('')
self.presetForm.addRow('<b>Module</b>', self.detailModule)
#Options
self.detailOptionsLabel = QLabel('<b>Options</b>')
self.detailOptionsLabel.setStyleSheet("QLabel {height:25px;}")
self.detailOptions = TextViewer()
self.presetForm.addRow(self.detailOptionsLabel, self.detailOptions)
# Columns
self.detailColumnsLabel = QLabel('<b>Columns</b>')
self.detailColumnsLabel.setStyleSheet("QLabel {height:25px;}")
self.detailColumns = TextViewer()
self.presetForm.addRow(self.detailColumnsLabel, self.detailColumns)
# Speed
self.detailSpeed = QLabel('')
self.presetForm.addRow('<b>Speed</b>', self.detailSpeed)
# Timeout
self.detailTimeout = QLabel('')
self.presetForm.addRow('<b>Timeout</b>', self.detailTimeout)
# Headers
self.detailHeaders = QLabel('')
self.presetForm.addRow('<b>Header nodes</b>', self.detailHeaders)
# Buttons
buttons = QHBoxLayout() #QDialogButtonBox()
self.saveButton = QPushButton('New preset')
self.saveButton.clicked.connect(self.newPreset)
self.saveButton.setToolTip(
wraptip(
"Create a new preset using the current tab and parameters"))
#buttons.addButton(self.saveButton,QDialogButtonBox.ActionRole)
buttons.addWidget(self.saveButton)
self.overwriteButton = QPushButton('Edit preset')
self.overwriteButton.clicked.connect(self.overwritePreset)
self.overwriteButton.setToolTip(wraptip("Edit the selected preset."))
#buttons.addButton(self.overwriteButton,QDialogButtonBox.ActionRole)
buttons.addWidget(self.overwriteButton)
self.deleteButton = QPushButton('Delete preset')
self.deleteButton.clicked.connect(self.deletePreset)
self.deleteButton.setToolTip(
wraptip(
"Delete the selected preset. Default presets can not be deleted."
))
#buttons.addButton(self.deleteButton,QDialogButtonBox.ActionRole)
buttons.addWidget(self.deleteButton)
#layout.addWidget(buttons,1)
buttons.addStretch()
self.reloadButton = QPushButton('Reload')
self.reloadButton.clicked.connect(self.reloadPresets)
self.reloadButton.setToolTip(wraptip("Reload all preset files."))
buttons.addWidget(self.reloadButton)
self.rejectButton = QPushButton('Cancel')
self.rejectButton.clicked.connect(self.close)
self.rejectButton.setToolTip(wraptip("Close the preset dialog."))
buttons.addWidget(self.rejectButton)
self.applyButton = QPushButton('Apply')
self.applyButton.setDefault(True)
self.applyButton.clicked.connect(self.applyPreset)
self.applyButton.setToolTip(wraptip("Load the selected preset."))
buttons.addWidget(self.applyButton)
layout.addLayout(buttons)
#status bar
self.statusbar = QStatusBar()
#self.folderLabel = QLabel("")
self.folderButton = QPushButton("")
self.folderButton.setFlat(True)
self.folderButton.clicked.connect(self.statusBarClicked)
self.statusbar.insertWidget(0, self.folderButton)
layout.addWidget(self.statusbar)
#self.presetFolder = os.path.join(os.path.dirname(self.mainWindow.settings.fileName()),'presets')
self.presetFolder = os.path.join(os.path.expanduser("~"), 'Facepager',
'Presets')
self.presetFolderDefault = os.path.join(os.path.expanduser("~"),
'Facepager', 'DefaultPresets')
self.folderButton.setText(self.presetFolder)
self.presetsDownloaded = False
self.presetSuffix = ['.3_9.json', '.3_10.json', '.fp4.json']
self.lastSelected = None
# Progress bar (sync with download thread by signals
self.progress = ProgressBar(
"Downloading default presets from GitHub...", self, hidden=True)
self.progressStart.connect(self.setProgressStart)
self.progressShow.connect(self.setProgressShow)
self.progressMax.connect(self.setProgressMax)
self.progressStep.connect(self.setProgressStep)
self.progressStop.connect(self.setProgressStop)
# if getattr(sys, 'frozen', False):
# self.defaultPresetFolder = os.path.join(os.path.dirname(sys.executable),'presets')
# elif __file__:
# self.defaultPresetFolder = os.path.join(os.path.dirname(__file__),'presets')
# Sycn progress bar with download thread
@Slot()
def setProgressStart(self):
if self.progress is None:
self.progress = ProgressBar(
"Downloading default presets from GitHub...",
self,
hidden=True)
def setProgressShow(self):
if self.progress is not None:
self.progress.setModal(True)
self.progress.show()
QApplication.processEvents()
def setProgressMax(self, maximum):
if self.progress is not None:
self.progress.setMaximum(maximum)
def setProgressStep(self):
if self.progress is not None:
self.progress.step()
def setProgressStop(self):
self.progress.close()
self.progress = None
def statusBarClicked(self):
if not os.path.exists(self.presetFolder):
os.makedirs(self.presetFolder)
if platform.system() == "Windows":
webbrowser.open(self.presetFolder)
elif platform.system() == "Darwin":
webbrowser.open('file:///' + self.presetFolder)
else:
webbrowser.open('file:///' + self.presetFolder)
def currentChanged(self):
#hide
self.detailName.setText("")
self.detailModule.setText("")
self.detailDescription.setText("")
self.detailOptions.setText("")
self.detailColumns.setText("")
self.presetView.hide()
self.categoryView.hide()
current = self.presetList.currentItem()
if current and current.isSelected():
data = current.data(0, Qt.UserRole)
# Single preset
if not data.get('iscategory', False):
self.lastSelected = os.path.join(data.get('folder', ''),
data.get('filename', ''))
self.detailName.setText(data.get('name'))
self.detailModule.setText(data.get('module'))
self.detailDescription.setText(data.get('description') + "\n")
self.detailOptions.setHtml(formatdict(data.get('options', [])))
self.detailColumns.setText("\r\n".join(data.get('columns',
[])))
self.detailSpeed.setText(str(data.get('speed', '')))
self.detailTimeout.setText(str(data.get('timeout', '')))
#self.applyButton.setText("Apply")
self.presetView.show()
# Category
else:
# self.pipelineName.setText(str(data.get('category')))
# self.pipelineWidget.clear()
# for i in range(current.childCount()):
# presetitem = current.child(i)
# preset = presetitem.data(0, Qt.UserRole)
#
# treeitem = QTreeWidgetItem(self.pipelineWidget)
# treeitem.setText(0,preset.get('name'))
# treeitem.setText(1, preset.get('module'))
# treeitem.setText(2, getDictValue(preset,'options.basepath'))
# treeitem.setText(3, getDictValue(preset,'options.resource'))
# # treeitem.setText(4, preset.get('description'))
#
# self.pipelineWidget.addTopLevelItem(treeitem)
#self.applyButton.setText("Run pipeline")
self.categoryView.show()
def showPresets(self):
self.clear()
self.show()
QApplication.processEvents()
self.progressShow.emit()
self.initPresets()
self.raise_()
def addPresetItem(self, folder, filename, default=False, online=False):
try:
if online:
data = requests.get(folder + filename).json()
else:
with open(os.path.join(folder, filename),
'r',
encoding="utf-8") as input:
data = json.load(input)
data['filename'] = filename
data['folder'] = folder
data['default'] = default
data['online'] = online
if data.get('type', 'preset') == 'pipeline':
data['category'] = data.get('category', 'noname')
if not data['category'] in self.categoryNodes:
categoryItem = PresetWidgetItem()
categoryItem.setText(0, data['category'])
ft = categoryItem.font(0)
ft.setWeight(QFont.Bold)
categoryItem.setFont(0, ft)
self.presetList.addTopLevelItem(categoryItem)
else:
categoryItem = self.categoryNodes[data['category']]
data['iscategory'] = True
categoryItem.setData(0, Qt.UserRole, data)
else:
data['caption'] = data.get('name')
if default:
data['caption'] = data['caption'] + " *"
data['category'] = data.get('category', '')
if (data['category'] == ''):
if (data.get('module') in ['Generic', 'Files']):
try:
data['category'] = data.get(
'module') + " (" + urlparse(
data['options']['basepath']).netloc + ")"
except:
data['category'] = data.get('module')
else:
data['category'] = data.get('module')
if not data['category'] in self.categoryNodes:
categoryItem = PresetWidgetItem()
categoryItem.setText(0, data['category'])
ft = categoryItem.font(0)
ft.setWeight(QFont.Bold)
categoryItem.setFont(0, ft)
categoryItem.setData(
0, Qt.UserRole, {
'iscategory': True,
'name': data['module'],
'category': data['category']
})
self.presetList.addTopLevelItem(categoryItem)
self.categoryNodes[data['category']] = categoryItem
else:
categoryItem = self.categoryNodes[data['category']]
newItem = PresetWidgetItem()
newItem.setText(0, data['caption'])
newItem.setData(0, Qt.UserRole, data)
if default:
newItem.setForeground(0, QBrush(QColor("darkblue")))
categoryItem.addChild(newItem)
#self.presetList.setCurrentItem(newItem,0)
QApplication.processEvents()
return newItem
except Exception as e:
QMessageBox.information(self, "Facepager",
"Error loading preset:" + str(e))
return None
def clear(self):
self.presetList.clear()
self.presetView.hide()
self.categoryView.hide()
self.loadingIndicator.show()
def checkDefaultFiles(self):
if not os.path.exists(self.presetFolderDefault):
self.downloadDefaultFiles()
elif len(os.listdir(self.presetFolderDefault)) == 0:
self.downloadDefaultFiles()
def downloadDefaultFiles(self, silent=False):
with self.loadingLock:
if self.presetsDownloaded:
return False
# Progress
self.progressStart.emit()
if not silent:
self.progressShow.emit()
# Create temporary download folder
tmp = TemporaryDirectory(suffix='FacepagerDefaultPresets')
try:
#Download
files = requests.get(
"https://api.github.com/repos/strohne/Facepager/contents/presets"
).json()
files = [
f['path'] for f in files
if f['path'].endswith(tuple(self.presetSuffix))
]
self.progressMax.emit(len(files))
for filename in files:
response = requests.get(
"https://raw.githubusercontent.com/strohne/Facepager/master/"
+ filename)
if response.status_code != 200:
raise (
f"GitHub is not available (status code {response.status_code})"
)
with open(
os.path.join(tmp.name, os.path.basename(filename)),
'wb') as f:
f.write(response.content)
self.progressStep.emit()
#Create folder
if not os.path.exists(self.presetFolderDefault):
os.makedirs(self.presetFolderDefault)
#Clear folder
for filename in os.listdir(self.presetFolderDefault):
os.remove(os.path.join(self.presetFolderDefault, filename))
# Move files from tempfolder
for filename in os.listdir(tmp.name):
shutil.move(os.path.join(tmp.name, filename),
self.presetFolderDefault)
self.logmessage.emit("Default presets downloaded from GitHub.")
except Exception as e:
if not silent:
QMessageBox.information(
self, "Facepager",
"Error downloading default presets:" + str(e))
self.logmessage.emit("Error downloading default presets:" +
str(e))
return False
else:
self.presetsDownloaded = True
return True
finally:
tmp.cleanup()
self.progressStop.emit()
def reloadPresets(self):
self.presetsDownloaded = False
self.downloadDefaultFiles()
self.initPresets()
def initPresets(self):
self.loadingIndicator.show()
#self.defaultPresetFolder
self.categoryNodes = {}
self.presetList.clear()
self.presetView.hide()
self.categoryView.hide()
selectitem = None
while not self.presetsDownloaded:
QApplication.processEvents()
if os.path.exists(self.presetFolderDefault):
files = [
f for f in os.listdir(self.presetFolderDefault)
if f.endswith(tuple(self.presetSuffix))
]
for filename in files:
newitem = self.addPresetItem(self.presetFolderDefault,
filename, True)
if self.lastSelected is not None and (
self.lastSelected == os.path.join(
self.presetFolderDefault, filename)):
selectitem = newitem
if os.path.exists(self.presetFolder):
files = [
f for f in os.listdir(self.presetFolder)
if f.endswith(tuple(self.presetSuffix))
]
for filename in files:
newitem = self.addPresetItem(self.presetFolder, filename)
if self.lastSelected is not None and (self.lastSelected == str(
os.path.join(self.presetFolder, filename))):
selectitem = newitem
#self.presetList.expandAll()
self.presetList.setFocus()
self.presetList.sortItems(0, Qt.AscendingOrder)
selectitem = self.presetList.topLevelItem(
0) if selectitem is None else selectitem
self.presetList.setCurrentItem(selectitem)
self.applyButton.setDefault(True)
self.loadingIndicator.hide()
def getCategories(self):
categories = []
root = self.presetList.invisibleRootItem()
for i in range(root.childCount()):
item = root.child(i)
data = item.data(0, Qt.UserRole)
categories.append(data.get('category', ''))
return categories
def startPipeline(self):
if not self.presetList.currentItem():
return False
# Get category item
root_item = self.presetList.currentItem()
root_data = root_item.data(0, Qt.UserRole)
if not root_data.get('iscategory', False):
root_item = root_item.parent()
root_data = root_item.data(0, Qt.UserRole)
# Create pipeline
pipeline = []
for i in range(root_item.childCount()):
item = root_item.child(i)
preset = item.data(0, Qt.UserRole)
module = self.mainWindow.getModule(preset.get('module', None))
options = module.getOptions()
options.update(preset.get('options', {}))
preset['options'] = options.copy()
preset['item'] = item
pipeline.append(preset)
# Process pipeline
return self.mainWindow.apiActions.queryPipeline(pipeline)
#self.close()
def applyPreset(self):
if not self.presetList.currentItem():
return False
data = self.presetList.currentItem().data(0, Qt.UserRole)
if data.get('iscategory', False):
return False
#self.startPipeline()
else:
self.mainWindow.apiActions.applySettings(data)
self.close()
def uniqueFilename(self, name):
filename = re.sub('[^a-zA-Z0-9_-]+', '_', name) + self.presetSuffix[-1]
i = 1
while os.path.exists(os.path.join(self.presetFolder,
filename)) and i < 10000:
filename = re.sub('[^a-zA-Z0-9_-]+', '_',
name) + "-" + str(i) + self.presetSuffix[-1]
i += 1
if os.path.exists(os.path.join(self.presetFolder, filename)):
raise Exception('Could not find unique filename')
return filename
def deletePreset(self):
if not self.presetList.currentItem():
return False
data = self.presetList.currentItem().data(0, Qt.UserRole)
if data.get('default', False):
QMessageBox.information(self, "Facepager",
"Cannot delete default presets.")
return False
if data.get('iscategory', False):
return False
reply = QMessageBox.question(
self, 'Delete Preset',
"Are you sure to delete the preset \"{0}\"?".format(
data.get('name', '')), QMessageBox.Yes | QMessageBox.No,
QMessageBox.No)
if reply != QMessageBox.Yes:
return False
os.remove(os.path.join(self.presetFolder, data.get('filename')))
self.initPresets()
def editPreset(self, data=None):
dialog = QDialog(self.mainWindow)
self.currentData = data if data is not None else {}
self.currentFilename = self.currentData.get('filename', None)
if self.currentFilename is None:
dialog.setWindowTitle("New Preset")
else:
dialog.setWindowTitle("Edit selected preset")
layout = QVBoxLayout()
label = QLabel("<b>Name</b>")
layout.addWidget(label)
name = QLineEdit()
name.setText(self.currentData.get('name', ''))
layout.addWidget(name, 0)
label = QLabel("<b>Category</b>")
layout.addWidget(label)
category = QComboBox(self)
category.addItems(self.getCategories())
category.setEditable(True)
category.setCurrentText(self.currentData.get('category', ''))
layout.addWidget(category, 0)
label = QLabel("<b>Description</b>")
layout.addWidget(label)
description = QTextEdit()
description.setMinimumWidth(500)
description.acceptRichText = False
description.setPlainText(self.currentData.get('description', ''))
description.setFocus()
layout.addWidget(description, 1)
overwriteLayout = QHBoxLayout()
self.overwriteCheckbox = QCheckBox(self)
self.overwriteCheckbox.setCheckState(Qt.Unchecked)
overwriteLayout.addWidget(self.overwriteCheckbox)
label = QLabel("<b>Overwrite parameters with current settings</b>")
overwriteLayout.addWidget(label)
overwriteLayout.addStretch()
if self.currentFilename is not None:
layout.addLayout(overwriteLayout)
buttons = QDialogButtonBox(QDialogButtonBox.Ok
| QDialogButtonBox.Cancel)
layout.addWidget(buttons, 0)
dialog.setLayout(layout)
def save():
data_meta = {
'name': name.text(),
'category': category.currentText(),
'description': description.toPlainText()
}
data_settings = self.mainWindow.apiActions.getPresetOptions()
self.currentData.update(data_meta)
if self.currentFilename is None:
self.currentData.update(data_settings)
elif self.overwriteCheckbox.isChecked():
reply = QMessageBox.question(
self, 'Overwrite Preset',
"Are you sure to overwrite the selected preset \"{0}\" with the current settings?"
.format(data.get('name', '')),
QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
if reply != QMessageBox.Yes:
dialog.close()
self.currentFilename = None
return False
else:
self.currentData.update(data_settings)
# Sanitize and reorder
keys = [
'name', 'category', 'description', 'module', 'options',
'speed', 'saveheaders', 'timeout', 'columns'
]
self.currentData = {k: self.currentData.get(k, None) for k in keys}
# Create folder
if not os.path.exists(self.presetFolder):
os.makedirs(self.presetFolder)
# Remove old file
if self.currentFilename is not None:
filepath = os.path.join(self.presetFolder,
self.currentFilename)
if os.path.exists(filepath):
os.remove(filepath)
# Save new file
catname = category.currentText() if category.currentText(
) != "" else self.mainWindow.RequestTabs.currentWidget().name
self.currentFilename = self.uniqueFilename(catname + "-" +
name.text())
with open(os.path.join(self.presetFolder, self.currentFilename),
'w') as outfile:
json.dump(self.currentData,
outfile,
indent=2,
separators=(',', ': '))
dialog.close()
return True
def close():
dialog.close()
self.currentFilename = None
return False
#connect the nested functions above to the dialog-buttons
buttons.accepted.connect(save)
buttons.rejected.connect(close)
dialog.exec()
return self.currentFilename
def newPreset(self):
filename = self.editPreset()
if filename is not None:
newitem = self.addPresetItem(self.presetFolder, filename)
self.presetList.sortItems(0, Qt.AscendingOrder)
self.presetList.setCurrentItem(newitem, 0)
def overwritePreset(self):
if not self.presetList.currentItem():
return False
item = self.presetList.currentItem()
data = item.data(0, Qt.UserRole)
if data.get('default', False):
QMessageBox.information(self, "Facepager",
"Cannot edit default presets.")
return False
if data.get('iscategory', False):
return False
filename = self.editPreset(data)
if filename is not None:
item.parent().removeChild(item)
item = self.addPresetItem(self.presetFolder, filename)
self.presetList.sortItems(0, Qt.AscendingOrder)
self.presetList.setCurrentItem(item, 0)
class DataViewer(QDialog):
def __init__(self, parent=None):
super(DataViewer, self).__init__(parent)
# Main window
self.mainWindow = parent
self.setWindowTitle("Extract Data")
self.setMinimumWidth(400)
# layout
layout = QVBoxLayout(self)
self.setLayout(layout)
self.extractLayout = QFormLayout()
layout.addLayout(self.extractLayout)
# Extract key input
self.input_extract = QLineEdit()
self.input_extract.setFocus()
self.input_extract.textChanged.connect(self.delayPreview)
self.extractLayout.addRow("Key to extract:",self.input_extract)
# Object ID key input
self.input_id = QLineEdit()
self.input_id.textChanged.connect(self.delayPreview)
self.extractLayout.addRow("Key for Object ID:", self.input_id)
# Options
optionsLayout = QHBoxLayout()
self.extractLayout.addRow(optionsLayout)
self.allnodesCheckbox = QCheckBox("Select all nodes")
self.allnodesCheckbox.setToolTip(wraptip("Check if you want to extract data for all nodes."))
self.allnodesCheckbox.setChecked(False)
optionsLayout.addWidget(self.allnodesCheckbox)
self.levelEdit=QSpinBox()
self.levelEdit.setMinimum(1)
self.levelEdit.setToolTip(wraptip("Based on the selected nodes, only extract data for nodes and subnodes of the specified level (base level is 1)"))
self.levelEdit.valueChanged.connect(self.delayPreview)
optionsLayout.addWidget(QLabel("Node level"))
optionsLayout.addWidget(self.levelEdit)
self.objecttypeEdit = QLineEdit("offcut")
self.objecttypeEdit.setToolTip(wraptip("Skip nodes with these object types."))
self.objecttypeEdit.textChanged.connect(self.delayPreview)
optionsLayout.addWidget(QLabel("Exclude object types"))
optionsLayout.addWidget(self.objecttypeEdit)
#self.extractLayout.addRow("Exclude object types", self.objecttypeEdit)
# Preview toggle
previewLayout = QHBoxLayout()
layout.addLayout(previewLayout)
self.togglePreviewCheckbox = QCheckBox()
self.togglePreviewCheckbox .setCheckState(Qt.Checked)
self.togglePreviewCheckbox.setToolTip(wraptip("Check to see a dumped preview of the value"))
self.togglePreviewCheckbox.stateChanged.connect(self.showPreview)
previewLayout.addWidget(self.togglePreviewCheckbox)
self.previewTimer = QTimer()
self.previewTimer.timeout.connect(self.showPreview)
self.previewTimer.setSingleShot(True)
self.togglePreviewLabel = QLabel("Preview")
previewLayout.addWidget(self.togglePreviewLabel)
previewLayout.addStretch()
# Data
self.dataEdit = QTextEdit(self)
self.dataEdit.setReadOnly(True)
self.dataEdit.setMinimumHeight(400)
layout.addWidget(self.dataEdit)
# Buttons
buttons = QDialogButtonBox(QDialogButtonBox.Apply | QDialogButtonBox.Close)
buttons.button(QDialogButtonBox.Apply).clicked.connect(self.createNodes)
buttons.button(QDialogButtonBox.Close).clicked.connect(self.close)
layout.addWidget(buttons)
def showValue(self, key = ''):
self.input_extract.setText(key)
#self.input_id.setText(key)
self.allnodesCheckbox.setChecked(self.mainWindow.allnodesCheckbox.isChecked())
self.levelEdit.setValue(self.mainWindow.levelEdit.value())
self.objecttypeEdit.setText(self.mainWindow.typesEdit.text())
self.show()
self.raise_()
@Slot()
def delayPreview(self):
self.previewTimer.stop()
self.previewTimer.start(500)
def updateNode(self, current):
self.delayPreview()
if current.isValid():
level = current.model().getLevel(current) + 1
self.levelEdit.setValue(level)
@Slot()
def showPreview(self):
if self.togglePreviewCheckbox.isChecked():
try:
# Get nodes
key_nodes = self.input_extract.text()
subkey = key_nodes.split('|').pop(0).rsplit('.', 1)[0]
key_id = self.input_id.text()
#selected = self.mainWindow.tree.selectionModel().selectedRows()
objecttypes = self.objecttypeEdit.text().replace(' ', '').split(',')
level = self.levelEdit.value() - 1
conditions = {'filter': {'level': level, '!objecttype': objecttypes}}
selected = self.mainWindow.tree.selectedIndexesAndChildren(conditions)
nodes = []
for item in selected:
if not item.isValid():
continue
treenode = item.internalPointer()
dbnode = treenode.dbnode()
if dbnode is not None:
name, nodes = extractValue(dbnode.response, key_nodes, dump=False)
break
# Dump nodes
value = []
nodes = [nodes] if not (type(nodes) is list) else nodes
for n in nodes:
nodedata = json.dumps(n) if isinstance(n, Mapping) else n
n = n if isinstance(n, Mapping) else {subkey: n}
objectid = extractValue(n, key_id, default=None)[1] if key_id != '' else ''
value.append((str(objectid), str(nodedata)))
except Exception as e:
value = [('',str(e))]
value = ['<b>{}</b><p>{}</p><hr>'.format(html.escape(x),html.escape(y)) for x,y in value]
value = "\n\n".join(value)
self.dataEdit.setHtml(value)
self.dataEdit.setVisible(self.togglePreviewCheckbox.isChecked())
if not self.togglePreviewCheckbox.isChecked():
self.adjustSize()
self.show()
def initProgress(self):
self.progressBar = ProgressBar("Extracting data...", self.mainWindow)
self.progressTotal = None
self.progressLevel = None
self.progressUpdate = datetime.now()
def updateProgress(self,current, total, level=0):
if datetime.now() >= self.progressUpdate:
if (self.progressLevel is None) or (level < self.progressLevel):
self.progressLevel = level
self.progressTotal = total
if (level == self.progressLevel) or (total > self.progressTotal):
self.progressBar.setMaximum(total)
self.progressBar.setValue(current)
self.progressUpdate = datetime.now() + timedelta(milliseconds=50)
QApplication.processEvents()
return not self.progressBar.wasCanceled
def finishProgress(self):
self.progressBar.close()
@Slot()
def createNodes(self):
key_nodes = self.input_extract.text()
key_objectid = self.input_id.text()
if key_nodes == '':
return False
if key_objectid == '':
key_objectid = None
try:
self.initProgress()
objecttypes = self.objecttypeEdit.text().replace(' ', '').split(',')
level = self.levelEdit.value() - 1
allnodes = self.allnodesCheckbox.isChecked()
conditions = {'filter': {'level': level, '!objecttype': objecttypes},
'selectall': allnodes}
selected = self.mainWindow.tree.selectedIndexesAndChildren(conditions, self.updateProgress)
for item in selected:
if self.progressBar.wasCanceled:
break
if not item.isValid():
continue
treenode = item.internalPointer()
treenode.unpackList(key_nodes, key_objectid, delaycommit=True)
except Exception as e:
self.mainWindow.logmessage(e)
finally:
self.mainWindow.tree.treemodel.commitNewNodes()
self.finishProgress()
#self.close()
return True
class TransferNodes(QDialog):
def __init__(self, parent=None):
super(TransferNodes, self).__init__(parent)
# Main window
self.mainWindow = parent
self.setWindowTitle("Add selected nodes as seed nodes")
#self.setMinimumWidth(400)
# layout
layout = QVBoxLayout(self)
self.setLayout(layout)
self.extractLayout = QFormLayout()
layout.addLayout(self.extractLayout)
# Options
self.allnodesCheckbox = QCheckBox("Select all nodes")
self.allnodesCheckbox.setToolTip(wraptip("Check if you want to transfer all nodes."))
self.allnodesCheckbox.setChecked(False)
self.extractLayout.addRow("Nodes", self.allnodesCheckbox)
self.levelEdit=QSpinBox()
self.levelEdit.setMinimum(1)
self.levelEdit.setToolTip(wraptip("Based on the selected nodes, only subnodes of the specified level are processed (base level is 1)"))
self.extractLayout.addRow("Node level", self.levelEdit)
self.objecttypeEdit = QLineEdit("offcut")
self.objecttypeEdit.setToolTip(wraptip("Skip nodes with these object types."))
self.extractLayout.addRow("Exclude object types", self.objecttypeEdit)
# Buttons
buttons = QDialogButtonBox(QDialogButtonBox.Apply | QDialogButtonBox.Close)
buttons.button(QDialogButtonBox.Apply).clicked.connect(self.createNodes)
buttons.button(QDialogButtonBox.Close).clicked.connect(self.close)
layout.addWidget(buttons)
def show(self):
self.allnodesCheckbox.setChecked(self.mainWindow.allnodesCheckbox.isChecked())
self.levelEdit.setValue(self.mainWindow.levelEdit.value())
self.objecttypeEdit.setText(self.mainWindow.typesEdit.text())
super(TransferNodes, self).show()
self.raise_()
def updateNode(self, current):
if current.isValid():
level = current.model().getLevel(current) + 1
self.levelEdit.setValue(level)
def initProgress(self):
self.progressBar = ProgressBar("Transferring nodes...", self.mainWindow)
self.progressTotal = None
self.progressLevel = None
self.progressUpdate = datetime.now()
def updateProgress(self,current, total, level=0):
if datetime.now() >= self.progressUpdate:
if (self.progressLevel is None) or (level < self.progressLevel):
self.progressLevel = level
self.progressTotal = total
if (level == self.progressLevel) or (total > self.progressTotal):
self.progressBar.setMaximum(total)
self.progressBar.setValue(current)
self.progressUpdate = datetime.now() + timedelta(milliseconds=50)
QApplication.processEvents()
return not self.progressBar.wasCanceled
def finishProgress(self):
self.progressBar.close()
@Slot()
def createNodes(self):
try:
self.initProgress()
# Get list of seed nodes to avoid duplicates
seednodes = Node.query.filter(Node.parent_id == None).add_columns('objectid')
seednodes = [node.objectid for node in seednodes]
# Iterate nodes
objecttypes = self.objecttypeEdit.text().replace(' ', '').split(',')
level = self.levelEdit.value() - 1
allnodes = self.allnodesCheckbox.isChecked()
conditions = {'filter': {'level': level, '!objecttype': objecttypes},
'selectall': allnodes}
selected = self.mainWindow.tree.selectedIndexesAndChildren(conditions, self.updateProgress)
nodes_new = 0
nodes_dupl = 0
for item in selected:
if self.progressBar.wasCanceled:
break
if not item.isValid():
continue
treenode = item.internalPointer()
objectid = treenode.data.get("objectid")
# no duplicates
if not objectid in seednodes:
seednodes.append(objectid)
treenode.copyNode(delaycommit=True)
nodes_new += 1
else:
nodes_dupl += 1
self.mainWindow.tree.treemodel.commitNewNodes()
self.mainWindow.tree.selectLastRow()
self.mainWindow.logmessage(f"{nodes_new} nodes added as seed nodes. {nodes_dupl} duplicate nodes skipped.")
self.close()
except Exception as e:
self.mainWindow.logmessage(e)
finally:
self.mainWindow.tree.treemodel.commitNewNodes()
self.finishProgress()
#self.close()
return True
def fetchData(self, indexes=None, apimodule=False, options=None):
# Check seed nodes
if not (self.mainWindow.tree.selectedCount() or self.mainWindow.allnodesCheckbox.isChecked() or (indexes is not None)):
return False
# Get options
apimodule, options = self.getQueryOptions(apimodule, options)
if not apimodule.auth_userauthorized and apimodule.auth_preregistered:
msg = 'You are not authorized, login please!'
QMessageBox.critical(self.mainWindow, "Not authorized",msg,QMessageBox.StandardButton.Ok)
return False
#Show progress window
progress = ProgressBar("Fetching Data", parent=self.mainWindow)
try:
# Get seed nodes
indexes = self.getIndexes(options, indexes, progress)
# Update progress window
self.mainWindow.logmessage("Start fetching data.")
totalnodes = 0
hasindexes = True
progress.setMaximum(totalnodes)
self.mainWindow.tree.treemodel.nodecounter = 0
#Init status messages
statuscount = defaultdict(int)
errorcount = 0
ratelimitcount = 0
allowedstatus = ['fetched (200)','downloaded (200)','fetched (202)']
try:
#Spawn Threadpool
threadpool = ApiThreadPool(apimodule)
threadpool.spawnThreads(options.get("threads", 1))
#Process Logging/Input/Output Queue
while True:
try:
#Logging (sync logs in threads with main thread)
msg = threadpool.getLogMessage()
if msg is not None:
self.mainWindow.logmessage(msg)
# Jobs in: packages of 100 at a time
jobsin = 0
while hasindexes and (jobsin < 100):
index = next(indexes, False)
if index:
jobsin += 1
totalnodes += 1
if index.isValid():
job = self.prepareJob(index, options)
threadpool.addJob(job)
else:
threadpool.applyJobs()
progress.setRemaining(threadpool.getJobCount())
progress.resetRate()
hasindexes = False
progress.removeInfo('input')
self.mainWindow.logmessage("Added {} node(s) to queue.".format(totalnodes))
if jobsin > 0:
progress.setMaximum(totalnodes)
#Jobs out
job = threadpool.getJob()
#-Finished all nodes (sentinel)...
if job is None:
break
#-Finished one node...
elif 'progress' in job:
progresskey = 'nodeprogress' + str(job.get('threadnumber', ''))
# Update single progress
if 'current' in job:
percent = int((job.get('current',0) * 100.0 / job.get('total',1)))
progress.showInfo(progresskey, "{}% of current node processed.".format(percent))
elif 'page' in job:
if job.get('page', 0) > 1:
progress.showInfo(progresskey, "{} page(s) of current node processed.".format(job.get('page',0)))
# Update total progress
else:
progress.removeInfo(progresskey)
if not threadpool.suspended:
progress.step()
#-Add data...
elif 'data' in job and (not progress.wasCanceled):
if not job['nodeindex'].isValid():
continue
# Add data
treeindex = job['nodeindex']
treenode = treeindex.internalPointer()
newcount = treenode.appendNodes(job['data'], job['options'], True)
if options.get('expand',False):
self.mainWindow.tree.setExpanded(treeindex,True)
# Count status and errors
status = job['options'].get('querystatus', 'empty')
statuscount[status] += 1
errorcount += int(not status in allowedstatus)
# Detect rate limit
ratelimit = job['options'].get('ratelimit', False)
#ratelimit = ratelimit or (not newcount)
ratelimitcount += int(ratelimit)
autoretry = (ratelimitcount) or (status == "request error")
# Clear errors when everything is ok
if not threadpool.suspended and (status in allowedstatus) and (not ratelimit):
#threadpool.clearRetry()
errorcount = 0
ratelimitcount = 0
self.state = 'fetchdata'
# Suspend on error or ratelimit
elif (errorcount >= options['errors']) or (ratelimitcount > 0):
threadpool.suspendJobs()
self.state = 'ratelimit'
if ratelimit:
msg = "You reached the rate limit of the API."
else:
msg = "{} consecutive errors occurred.\nPlease check your settings.".format(errorcount)
timeout = 60 * 5 # 5 minutes
# Adjust progress
progress.showError(msg, timeout, autoretry)
self.mainWindow.tree.treemodel.commitNewNodes()
# Add job for retry
if not status in allowedstatus:
threadpool.addError(job)
# Show info
progress.showInfo(status,"{} response(s) with status: {}".format(statuscount[status],status))
progress.showInfo('newnodes',"{} new node(s) created".format(self.mainWindow.tree.treemodel.nodecounter))
progress.showInfo('threads',"{} active thread(s)".format(threadpool.getThreadCount()))
progress.setRemaining(threadpool.getJobCount())
# Custom info from modules
info = job['options'].get('info', {})
for name, value in info.items():
progress.showInfo(name, value)
# Abort
elif progress.wasCanceled:
progress.showInfo('cancel', "Disconnecting from stream, may take some time.")
threadpool.stopJobs()
# Retry
elif progress.wasResumed:
if progress.wasRetried:
threadpool.retryJobs()
else:
threadpool.clearRetry()
# errorcount = 0
# ratelimitcount = 0
threadpool.resumeJobs()
progress.setRemaining(threadpool.getJobCount())
progress.hideError()
# Continue
elif not threadpool.suspended:
threadpool.resumeJobs()
# Finished with pending errors
if not threadpool.hasJobs() and threadpool.hasErrorJobs():
msg = "All nodes finished but you have {} pending errors. Skip or retry?".format(threadpool.getErrorJobsCount())
autoretry = False
timeout = 60 * 5 # 5 minutes
progress.showError(msg, timeout, autoretry)
# Finished
if not threadpool.hasJobs():
progress.showInfo('cancel', "Work finished, shutting down threads.")
threadpool.stopJobs()
#-Waiting...
progress.computeRate()
time.sleep(1.0 / 1000.0)
finally:
QApplication.processEvents()
finally:
request_summary = [str(val)+" x "+key for key,val in statuscount.items()]
request_summary = ", ".join(request_summary)
request_end = "Fetching completed" if not progress.wasCanceled else 'Fetching cancelled by user'
self.mainWindow.logmessage("{}, {} new node(s) created. Summary of responses: {}.".format(request_end, self.mainWindow.tree.treemodel.nodecounter,request_summary))
self.mainWindow.tree.treemodel.commitNewNodes()
except Exception as e:
self.mainWindow.logmessage("Error in scheduler, fetching aborted: {}.".format(str(e)))
finally:
progress.close()
return not progress.wasCanceled
def downloadDefaultFiles(self, silent=False):
with self.loadingLock:
if self.filesDownloaded:
return False
# Progress
progress = ProgressBar(
"Downloading default API definitions from GitHub...",
self) if not silent else None
QApplication.processEvents()
# Create temporary download folder
tmp = TemporaryDirectory(suffix='FacepagerDefaultAPIs')
try:
#Download
files = requests.get(
"https://api.github.com/repos/strohne/Facepager/contents/apis"
).json()
files = [
f['path'] for f in files
if f['path'].endswith(tuple(self.filesSuffix))
]
if progress is not None:
progress.setMaximum(len(files))
for filename in files:
response = requests.get(
"https://raw.githubusercontent.com/strohne/Facepager/master/"
+ filename)
if response.status_code != 200:
raise Exception(
f"GitHub is not available (status code {response.status_code})"
)
with open(
os.path.join(tmp.name, os.path.basename(filename)),
'wb') as f:
f.write(response.content)
if progress is not None:
progress.step()
# Create folder
if not os.path.exists(self.folderDefault):
os.makedirs(self.folderDefault)
# Clear folder
for filename in os.listdir(self.folderDefault):
os.remove(os.path.join(self.folderDefault, filename))
# Move files from tempfolder
for filename in os.listdir(tmp.name):
shutil.move(os.path.join(tmp.name, filename),
self.folderDefault)
self.logmessage.emit(
"Default API definitions downloaded from GitHub.")
except Exception as e:
if not silent:
QMessageBox.information(
self, "Facepager",
"Error downloading default API definitions:" + str(e))
self.logmessage.emit(
"Error downloading default API definitions:" + str(e))
return False
else:
self.filesDownloaded = True
return True
finally:
tmp.cleanup()
if progress is not None:
progress.close()