def redo(self): keys = self.receiver.componentList.elements.keys() progress = QProgressDialog("Saving Component elements", QString(), 0, len(keys), self.receiver.componentList) progress.setWindowTitle("Save All Components") progress.setWindowModality(Qt.WindowModal) progress.show() for i in range(len(keys)): icp = keys[i] cp = self.receiver.componentList.elements[icp] if cp.instance is None: # self._cpEdit = FieldWg() cpEdit = Component(self.receiver.componentList) cpEdit.id = cp.id cpEdit.directory = self.receiver.componentList.directory cpEdit.name = \ self.receiver.componentList.elements[cp.id].name cpEdit.createGUI() cpEdit.addContextMenu(self.receiver.contextMenuActions) cpEdit.createHeader() cpEdit.dialog.setWindowTitle("%s [Component]" % cp.name) cp.instance = cpEdit if cp.instance is not None: cp.instance.merge(False) cp.instance.save() progress.setValue(i) progress.setValue(len(keys)) progress.close() logger.debug("EXEC componentSaveAll")
def loadFile(self): mnFileName = unicode(self.mnFileNameInput.text()) if not mnFileName: QMessageBox.warning(glb.mainWin, QCoreApplication.translate('SmileKit', 'No filename specified'), QCoreApplication.translate('SmileKit', 'Please enter a filename in the input box!')) return progressDlg = QProgressDialog(glb.mainWin) progressDlg.setRange(0, 100) progressDlg.setLabelText('Reading File...') progressDlg.show() a = glb.application def progFunc(p): progressDlg.setValue(p) if a.hasPendingEvents(): a.processEvents() r = readMnFile(mnFileName, progress=progFunc) self.data = r progressDlg.close() self.locationFrame.setEnabled(1) self.statisticsFrame.setEnabled(1) if ncOk: self.ncFileFrame.setEnabled(1) self.statisticsButton.setCheckState(Qt.Unchecked) self.locNameInput.setText(r['name']) c = unicode(QCoreApplication.translate('SmileKit', "created %s from %s by Meteo2Nc.py (v%d)")) self.locCommentInput.setText(c % (time.ctime(), mnFileName, version)) self.locLatitudeInput.setValue(r['latitude']) self.locLongitudeInput.setValue(r['longitude']) self.locHeightInput.setValue(r['height']) self.locTimeZoneInput.setValue(r['timezone']) path, tmp = os.path.split(mnFileName) ncFileName = os.path.join(path, makeFileName(r['name'])+'_weather.nc') self.ncFileNameInput.setText(ncFileName)
def redo(self): keys = self.receiver.sourceList.elements.keys() progress = QProgressDialog("Saving DataSource elements", QString(), 0, len(keys), self.receiver.sourceList) progress.setWindowTitle("Save All DataSources") progress.setWindowModality(Qt.WindowModal) progress.show() for i in range(len(keys)): ids = keys[i] ds = self.receiver.sourceList.elements[ids] if ds.instance is None: dsEdit = DataSource.DataSource(self.receiver.sourceList) dsEdit.id = ds.id dsEdit.directory = self.receiver.sourceList.directory dsEdit.name = \ self.receiver.sourceList.elements[ds.id].name ds.instance = dsEdit if ds.instance is not None: if ds.instance.save(): ds.savedName = ds.name progress.setValue(i) progress.setValue(len(keys)) progress.close() ds = self.receiver.sourceList.currentListElement() if hasattr(ds, "id"): self.receiver.sourceList.populateElements(ds.id) else: self.receiver.sourceList.populateElements() logger.debug("EXEC dsourceSaveAll")
def addMediaFile(self): path = '.' filename = self.labeltool.getCurrentFilename() if (filename is not None) and (len(filename) > 0): path = QFileInfo(filename).path() image_types = [ '*.jpg', '*.bmp', '*.png', '*.pgm', '*.ppm', '*.ppm', '*.tif', '*.gif' ] video_types = [ '*.mp4', '*.mpg', '*.mpeg', '*.avi', '*.mov', '*.vob' ] format_str = ' '.join(image_types + video_types) fnames = QFileDialog.getOpenFileNames(self, "%s - Add Media File" % APP_NAME, path, "Media files (%s)" % (format_str, )) item = None numFiles = len(fnames) progress_bar = QProgressDialog('Importing files...', 'Cancel import', 0, numFiles, self) for fname,c in zip(fnames, range(numFiles)): if len(str(fname)) == 0: continue fname = str(fname) if os.path.isabs(fname): fname = os.path.relpath(fname, str(path)) for pattern in image_types: if fnmatch.fnmatch(fname, pattern): item = self.labeltool.addImageFile(fname) progress_bar.setValue(c) if item is None: return self.labeltool.addVideoFile(fname) progress_bar.close() return item
def addMediaFile(self): path = '.' filename = self.labeltool.getCurrentFilename() if (filename is not None) and (len(filename) > 0): path = QFileInfo(filename).path() image_types = [ '*.jpg', '*.jpeg', '*.bmp', '*.png', '*.pgm', '*.ppm', '*.tiff', '*.tif', '*.gif' ] video_types = [ '*.mp4', '*.mpg', '*.mpeg', '*.avi', '*.mov', '*.vob' ] format_str = ' '.join(image_types + video_types) fnames = QFileDialog.getOpenFileNames(self, "%s - Add Media File" % APP_NAME, path, "Media files (%s)" % (format_str, )) item = None numFiles = len(fnames) progress_bar = QProgressDialog('Importing files...', 'Cancel import', 0, numFiles, self) for fname,c in zip(fnames, range(numFiles)): if len(str(fname)) == 0: continue fname = str(fname) if os.path.isabs(fname): fname = os.path.relpath(fname, str(path)) for pattern in image_types: if fnmatch.fnmatch(fname.lower(), pattern): item = self.labeltool.addImageFile(fname) progress_bar.setValue(c) if item is None: return self.labeltool.addVideoFile(fname) progress_bar.close() return item
def init_game_tree(self, mainAppWindow = None): # ugly workaround: # the next lines are just to ensure that # the "board cache" (see doc. of python-chess lib) # is initialized. The call to the last board of the main # line ensures recursive calls to the boards up to the root # # if a mainAppWindow is passed, a progress bar # will be displayed while the game tree is initialized # the app might "freeze" otherwise, as longer games # i.e. (> 70 moves) can take some time to initialize temp = self.current.root() end = self.current.end() mainline_nodes = [temp] while(not temp == end): temp = temp.variations[0] mainline_nodes.append(temp) cnt = len(mainline_nodes) if(not mainAppWindow == None): pDialog = QProgressDialog(mainAppWindow.trUtf8("Initializing Game Tree"),None,0,cnt,mainAppWindow) pDialog.setWindowModality(Qt.WindowModal) pDialog.show() QApplication.processEvents() for i,n in enumerate(mainline_nodes): QApplication.processEvents() pDialog.setValue(i) if(i > 0 and i % 25 == 0): _ = n.cache_board() pDialog.close() else: QApplication.processEvents() for i,n in enumerate(mainline_nodes): if(i > 0 and i % 25 == 0): _ = n.cache_board()
def download(self): rows = self.table.rowCount() progress = QProgressDialog("Downloading books...", "Abort download", 0, rows, self) progress.setWindowModality(Qt.WindowModal) progress.show() to_download = [self.table.item(row, 2).text() for row in range(rows) if self.table.item(row, 0).checkState() == Qt.Checked] for i, book in enumerate(to_download): progress.setValue(i) book_id = self.feed.download(str(book)) if not book_id: QMessageBox.critical(self, 'Error', 'Could not download the ' 'book') elif book_id != -1: book = Book(book_id) insert_library(book) if progress.wasCanceled(): break progress.setValue(rows) progress.close() self.parent().library.refresh()
def dump_script(self): to_dump = self.get_checked([self.ui.treeFileList.topLevelItem(i) for i in range(self.ui.treeFileList.topLevelItemCount())]) if not to_dump: QtGui.QMessageBox.warning(self, "No Selection", "No folders have beens selected to dump.") return out_file = get_save_file(self, self.last_file, "Text files (*.txt)") if out_file == "": return translated = not self.ui.chkUntranslated.isChecked() strip_clt = self.ui.chkStripClt.isChecked() only_voiced = self.ui.chkOnlyVoiced.isChecked() line_numbers = self.ui.chkLineNumbers.isChecked() progress = QProgressDialog("Dumping...", QtCore.QString(), 0, len(to_dump), self) progress.setWindowTitle("Dumping...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) # print out_file with open(out_file, "wb") as f: for dir in to_dump: progress.setLabelText("Dumping %s..." % dir) f.write(script_to_text(dir, translated, strip_clt, only_voiced, line_numbers).encode("UTF-8")) progress.setValue(progress.value() + 1) progress.close() self.last_file = out_file
def redo(self): if self.receiver.configServer: if self.receiver.configServer.connected: try: cc = DataSourceCreator(self.receiver.configServer, self.receiver) if cc.checkOnlineFile(self.receiver.onlineFile): self.receiver.onlineFile = cc.onlineFile cc.create() if cc.action: keys = cc.datasources.keys() progress = QProgressDialog( "Storing DataSource elements", QString(), 0, len(keys), self.receiver.sourceList) progress.setWindowTitle( "Store Created DataSources") progress.setWindowModality(Qt.WindowModal) progress.show() i = 0 dsid = None for ds, xml in cc.datasources.items(): dsid = self.__addDataSource(ds, xml, cc.action) progress.setValue(i) i += 1 progress.setValue(len(keys)) progress.close() if dsid: self.receiver.sourceList.populateElements(dsid) except Exception, e: QMessageBox.warning(self.receiver, "Error in creating Component", unicode(e))
def ProgressBar(n_steps, label="", allow_cancel=False, parent=None): """ Progressbar context manager for showing progress of workflow to user. Example:: with emzed.gui.ProgressBar(n_steps=100, allow_cancel=True) as handler: for i in range(100): # we simulate work of step i # we update progressbar handler.update(i, "step %03d" % i) # we can check if user pressed "Cancel" button and stop our "workflow": if handler.is_canceled(): break """ app = guidata.qapplication() dlg = QProgressDialog(parent) dlg.setLabelText(label) dlg.setAutoClose(False) dlg.setAutoReset(False) if allow_cancel: dlg.setCancelButtonText("Cancel") dlg.setMaximum(n_steps) class ProgressBarHandler(object): def __init__(self, n_steps, dlg): self._dlg = dlg self._n_steps = n_steps self._n = 0 self._canceled = False dlg.canceled.connect(self._set_canceled) dlg.setValue(0) def _set_canceled(self): self._canceled = True dlg.close() def update(self, n, message=None): app.processEvents() self._n = n dlg.setValue(n + 1) if message is not None: dlg.setLabelText(message) dlg.update() app.processEvents() def is_canceled(self): return self._canceled dlg.activateWindow() dlg.show() dlg.raise_() app.processEvents() handler = ProgressBarHandler(n_steps, dlg) yield handler dlg.close()
class IsoBuilder(): def __init__(self, parent=None): self.parent = parent self.process = None def __parse_output(self): if not self.process: return output = QString(self.process.readAll()) output = output.split("\n", QString.SkipEmptyParts) for line in output: line = common.qt_to_unicode(line) match = OUTPUT_RE.match(line) if match == None: continue percent = float(match.group(1)) self.progress.setValue(percent) def build_iso(self, directory, iso_file): if self.process: return directory = os.path.abspath(directory) iso_file = os.path.abspath(iso_file) self.progress = QProgressDialog("Building ISO...", QtCore.QString(), 0, 0, self.parent) self.progress.setWindowTitle("Building ISO") self.progress.setWindowModality(Qt.Qt.WindowModal) self.progress.setAutoClose(False) self.progress.setMinimumDuration(1000) self.progress.show() self.progress.setValue(0) self.progress.setMaximum(100) self.process = QProcess() self.process.finished.connect(self.__build_finished) self.process.setReadChannel(QProcess.StandardError) self.process.readyRead.connect(self.__parse_output) self.process.start("tools/mkisofs", [ "-sort", "data/file_order.txt", "-iso-level", "4", "-xa", "-A", "PSP GAME", "-V", "DANGANRONPA", "-sysid", "PSP GAME", "-volset", "DANGANRONPA", "-p", "SPIKE", "-publisher", "SPIKE", "-o", iso_file, directory ]) def __build_finished(self, code, status): self.progress.close() self.process = None
def on_actionOpenSession_triggered(self, filename=None): if not filename: filename = QFileDialog.getOpenFileName(self, 'Open Session') if filename: if self.session.messages: args = argv[:] if len(args) > 1: args[1] = filename else: args.append(abspath(str(filename))) try: pid = spawnvp(P_NOWAIT, args[0], args) except (NameError, ): Popen('"%s" "%s" "%s"' % (executable, args[0], args[1])) return if not self.warningOpenTabs(): return dlg = QProgressDialog(self) dlg.setLabelText('Reading session file.') dlg.setCancelButtonText('Abort') dlg.setWindowModality(Qt.WindowModal) dlg.setWindowTitle('Reading...') self.show() processEvents() dlg.show() processEvents() try: loadit = self.session.load(str(filename)) count = loadit.next() last = count - 1 except (StopIteration, ): msg = 'Warning session not loaded from "%s"' % filename dlg.close() else: dlg.setLabelText('Loading session messages.') dlg.setWindowTitle('Loading...') dlg.setMaximum(last) msgid = -1 for msgid in loadit: processEvents() dlg.setValue(msgid) if dlg.wasCanceled(): dlg.close() break if msgid == last: msg = 'Loaded all %s messages from file "%s".' msg %= (count, filename) self.setCurrentSession(filename) else: msg = 'Load aborted; loaded %s messages of %s.' msg %= (msgid+1, count) self.statusBar().showMessage(msg, 5000)
class IsoBuilder(): def __init__(self, parent = None): self.parent = parent self.process = None def __parse_output(self): if not self.process: return output = QString(self.process.readAll()) output = output.split("\n", QString.SkipEmptyParts) for line in output: line = common.qt_to_unicode(line) match = OUTPUT_RE.match(line) if match == None: continue percent = float(match.group(1)) self.progress.setValue(percent) def build_iso(self, directory, iso_file): if self.process: return directory = os.path.abspath(directory) iso_file = os.path.abspath(iso_file) self.progress = QProgressDialog("Building ISO...", QtCore.QString(), 0, 0, self.parent) self.progress.setWindowTitle("Building ISO") self.progress.setWindowModality(Qt.Qt.WindowModal) self.progress.setAutoClose(False) self.progress.setMinimumDuration(1000) self.progress.show() self.progress.setValue(0) self.progress.setMaximum(100) self.process = QProcess() self.process.finished.connect(self.__build_finished) self.process.setReadChannel(QProcess.StandardError) self.process.readyRead.connect(self.__parse_output) self.process.start("tools/mkisofs", ["-sort", "data/file_order.txt", "-iso-level", "4", "-xa", "-A", "PSP GAME", "-V", "DANGANRONPA", "-sysid", "PSP GAME", "-volset", "DANGANRONPA", "-p", "SPIKE", "-publisher", "SPIKE", "-o", iso_file, directory]) def __build_finished(self, code, status): self.progress.close() self.process = None
def init_from_pgn(self, mainWindow, msg): with open(self.filename) as pgn: size = os.path.getsize(self.filename) self.entries = [] pDialog = QProgressDialog(msg, None, 0, size, mainWindow) pDialog.show() pDialog.setWindowModality(PyQt4.QtCore.Qt.WindowModal) QApplication.processEvents() for offset, headers in chess.pgn.scan_headers(pgn): QApplication.processEvents() pDialog.setValue(offset) self.entries.append(Entry(offset, headers)) pDialog.close() self.checksum = crc32_from_file(self.filename)
def loadList(self, externalActions=None, itemActions=None): try: dirList = [l for l in os.listdir(self.directory) if (l.endswith(self.extention) and (not self.disextention or not l.endswith(self.disextention)) )] except: try: if os.path.exists(os.path.join(os.getcwd(), self.name)): self.directory = os.path.abspath( os.path.join(os.getcwd(), self.name)) else: self.directory = os.getcwd() dirList = [l for l in os.listdir(self.directory) if (l.endswith(self.extention) and (not self.disextention or not l.endswith(self.disextention)) )] except: return progress = QProgressDialog( "Loading %s elements" % self.clName, QString(), 0, len(dirList), self) progress.setWindowTitle("Load Elements") progress.setWindowModality(Qt.WindowModal) progress.forceShow() for i in range(len(dirList)): fname = dirList[i] name = self.nameFromFile(fname) dlg = self.createElement(name) dlg.load() if hasattr(dlg, "addContextMenu"): dlg.addContextMenu(itemActions) actions = externalActions if externalActions else {} if hasattr(dlg, "connectExternalActions"): dlg.connectExternalActions(**actions) el = LabeledObject(name, dlg) self.elements[id(el)] = el if el.instance is not None: el.instance.id = el.id logger.info("loading %s" % name) progress.setValue(i) progress.setValue(len(dirList)) progress.close()
def init_from_pgn(self, mainWindow, msg): print("loading from: "+self.filename) with open(self.filename) as pgn: size = os.path.getsize(self.filename) self.entries = [] pDialog = QProgressDialog(msg,None,0,size,mainWindow) pDialog.show() pDialog.setWindowModality(PyQt4.QtCore.Qt.WindowModal) QApplication.processEvents() for offset, headers in chess.pgn.scan_headers(pgn): print(headers) QApplication.processEvents() pDialog.setValue(offset) self.entries.append(Entry(offset,headers)) pDialog.close() self.checksum = crc32_from_file(self.filename)
def search_bar_old(self, dir, query): files = list_all_files(dir) progress = QProgressDialog("", "Abort", 0, len(files), self) progress.setWindowTitle("Searching...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) width = self.width() height = self.height() x = self.x() y = self.y() matches = [] if not self.ui.chkRegEx.isChecked(): query = re.escape(query) query_re = re.compile(query, re.IGNORECASE | re.DOTALL | re.UNICODE) for i, file in enumerate(files): if progress.wasCanceled(): break if i % 500 == 0: progress.setLabelText(file) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) if os.path.splitext(file)[1] == ".txt": text = load_text(file) if not query_re.search(text) == None: matches.append(file) progress.setValue(i + 1) progress.close() return matches
def generate_font(self): progress = QProgressDialog("", QtCore.QString(), 0, 0, self) progress.setWindowModality(Qt.Qt.WindowModal) progress.setWindowTitle("Generating font...") progress.setLabelText("Generating font...") progress.setMinimumDuration(0) progress.setAutoClose(False) # Thread this because it's slow as hell and we don't want to lock up the GUI. thread = threading.Thread(target = self.__generate_font__) thread.start() while thread.isAlive(): thread.join(THREAD_TIMEOUT) # It has to change by some amount or it won't update and the UI will lock up. progress.setValue(progress.value() - 1) progress.close()
def generate_font(self): progress = QProgressDialog("", QtCore.QString(), 0, 0, self) progress.setWindowModality(Qt.Qt.WindowModal) progress.setWindowTitle("Generating font...") progress.setLabelText("Generating font...") progress.setMinimumDuration(0) progress.setAutoClose(False) # Thread this because it's slow as hell and we don't want to lock up the GUI. thread = threading.Thread(target=self.__generate_font__) thread.start() while thread.isAlive(): thread.join(THREAD_TIMEOUT) # It has to change by some amount or it won't update and the UI will lock up. progress.setValue(progress.value() - 1) progress.close()
def dump_script(self): to_dump = self.get_checked([ self.ui.treeFileList.topLevelItem(i) for i in range(self.ui.treeFileList.topLevelItemCount()) ]) if not to_dump: QtGui.QMessageBox.warning( self, "No Selection", "No folders have beens selected to dump.") return out_file = get_save_file(self, self.last_file, "Text files (*.txt)") if out_file == "": return translated = not self.ui.chkUntranslated.isChecked() strip_clt = self.ui.chkStripClt.isChecked() only_voiced = self.ui.chkOnlyVoiced.isChecked() line_numbers = self.ui.chkLineNumbers.isChecked() progress = QProgressDialog("Dumping...", QtCore.QString(), 0, len(to_dump), self) progress.setWindowTitle("Dumping...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) # print out_file with open(out_file, "wb") as f: for dir in to_dump: progress.setLabelText("Dumping %s..." % dir) f.write( script_to_text(dir, translated, strip_clt, only_voiced, line_numbers).encode("UTF-8")) progress.setValue(progress.value() + 1) progress.close() self.last_file = out_file
def link_clicked(self, url): surl = str(url.toString()) if surl.endswith('.epub'): progress = QProgressDialog("Downloding the book...", "Abort", 0, -1, self) progress.show() book_id = FeedBooks().download(surl[:-5]) progress.close() if not book_id: QMessageBox.critical(self, 'Error', 'Could not download the ' 'book') elif book_id != -1: book = Book(book_id) insert_library(book) self.parent().library.refresh() else: book_id = surl[surl.rfind('/')+1:-5] if book_id: self.load_book(book_id) else: webbrowser.open_new_tab(surl)
def write_data_to_file(self, open_mode): progressDialog = QProgressDialog() progressDialog.setModal(True) progressDialog.setLabelText(self.tr('Guardando...')) progressDialog.setMaximum(8) progressDialog.setCancelButton(None) progressDialog.show() try: # File is closed automatically even on error with open(unicode(self.filename), open_mode) as file_obj: for count, elem_edit in enumerate(self.lineEdits_list, 1): file_obj.write(''.join([str(elem_edit[0].text()), '\n'])) file_obj.write(''.join([str(elem_edit[1].text()), '\n'])) file_obj.write(''.join([str(elem_edit[2].text()), '\n'])) file_obj.write(''.join([str(elem_edit[3].text()), '\n'])) file_obj.write(''.join([str(elem_edit[4].text()), '\n'])) file_obj.write(''.join([str(elem_edit[5].text()), '\n'])) file_obj.write(''.join([str(elem_edit[6].text()), '\n'])) file_obj.write(''.join([str(elem_edit[7].text()), '\n'])) file_obj.write(''.join([str(elem_edit[8].text()), '\n'])) file_obj.write(''.join([str(elem_edit[9].text()), '\n'])) file_obj.write(''.join([str(elem_edit[10].text()), '\n'])) file_obj.write(''.join([str(elem_edit[11].text()), '\n'])) progressDialog.setValue(count) except (IOError, OSError): progressDialog.close() messageBox = QMessageBox(self) messageBox.setStyleSheet( 'QMessageBox QLabel {font: bold 14pt "Cantarell";}') messageBox.setWindowTitle(self.tr('Error')) messageBox.setText(self.tr('Error al guardar')) messageBox.setStandardButtons(QMessageBox.Ok) messageBox.setIcon(QMessageBox.Critical) messageBox.exec_() else: self.statusBar1.showMessage(self.tr('Guardado'), 3000)
class AddonManagerDialog(QDialog): _packages = None def __init__(self, parent=None, **kwargs): super().__init__(parent, acceptDrops=True, **kwargs) self.setLayout(QVBoxLayout()) self.layout().setContentsMargins(0, 0, 0, 0) self.addonwidget = AddonManagerWidget() self.layout().addWidget(self.addonwidget) info_bar = QWidget() info_layout = QHBoxLayout() info_bar.setLayout(info_layout) self.layout().addWidget(info_bar) buttons = QDialogButtonBox( orientation=Qt.Horizontal, standardButtons=QDialogButtonBox.Ok | QDialogButtonBox.Cancel ) buttons.accepted.connect(self.__accepted) buttons.rejected.connect(self.reject) self.layout().addWidget(buttons) # No system access => install into user site-packages self.user_install = not os.access(sysconfig.get_path("purelib"), os.W_OK) self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) if AddonManagerDialog._packages is None: self._f_pypi_addons = self._executor.submit(list_pypi_addons) else: self._f_pypi_addons = concurrent.futures.Future() self._f_pypi_addons.set_result(AddonManagerDialog._packages) self._f_pypi_addons.add_done_callback( method_queued(self._set_packages, (object,)) ) self.__progress = QProgressDialog( self, Qt.Sheet, minimum=0, maximum=0, labelText=self.tr("Retrieving package list"), sizeGripEnabled=False, windowTitle="Progress" ) self.__progress.rejected.connect(self.reject) self.__thread = None self.__installer = None @Slot(object) def _set_packages(self, f): if self.__progress.isVisible(): self.__progress.close() try: packages = f.result() except (IOError, OSError) as err: message_warning( "Could not retrieve package list", title="Error", informative_text=str(err), parent=self ) packages = [] except Exception: raise else: AddonManagerDialog._packages = packages installed = list_installed_addons() dists = {dist.project_name: dist for dist in installed} packages = {pkg.name: pkg for pkg in packages} # For every pypi available distribution not listed by # list_installed_addons, check if it is actually already # installed. ws = pkg_resources.WorkingSet() for pkg_name in set(packages.keys()).difference(set(dists.keys())): try: d = ws.find(pkg_resources.Requirement.parse(pkg_name)) except pkg_resources.VersionConflict: pass except ValueError: # Requirements.parse error ? pass else: if d is not None: dists[d.project_name] = d project_names = unique( itertools.chain(packages.keys(), dists.keys()) ) items = [] for name in project_names: if name in dists and name in packages: item = Installed(packages[name], dists[name]) elif name in dists: item = Installed(None, dists[name]) elif name in packages: item = Available(packages[name]) else: assert False items.append(item) self.addonwidget.set_items(items) def showEvent(self, event): super().showEvent(event) if not self._f_pypi_addons.done(): QTimer.singleShot(0, self.__progress.show) def done(self, retcode): super().done(retcode) self._f_pypi_addons.cancel() self._executor.shutdown(wait=False) if self.__thread is not None: self.__thread.quit() self.__thread.wait(1000) def closeEvent(self, event): super().closeEvent(event) self._f_pypi_addons.cancel() self._executor.shutdown(wait=False) if self.__thread is not None: self.__thread.quit() self.__thread.wait(1000) ADDON_EXTENSIONS = ('.zip', '.whl', '.tar.gz') def dragEnterEvent(self, event): urls = event.mimeData().urls() if any((OSX_NSURL_toLocalFile(url) or url.toLocalFile()) .endswith(self.ADDON_EXTENSIONS) for url in urls): event.acceptProposedAction() def dropEvent(self, event): """Allow dropping add-ons (zip or wheel archives) on this dialog to install them""" packages = [] names = [] for url in event.mimeData().urls(): path = OSX_NSURL_toLocalFile(url) or url.toLocalFile() if path.endswith(self.ADDON_EXTENSIONS): name, vers, summary, descr = (get_meta_from_archive(path) or (os.path.basename(path), '', '', '')) names.append(name) packages.append( Installable(name, vers, summary, descr or summary, path, [path])) future = concurrent.futures.Future() future.set_result((AddonManagerDialog._packages or []) + packages) self._set_packages(future) self.addonwidget.set_install_projects(names) def __accepted(self): steps = self.addonwidget.item_state() if steps: # Move all uninstall steps to the front steps = sorted( steps, key=lambda step: 0 if step[0] == Uninstall else 1 ) self.__installer = Installer(steps=steps, user_install=self.user_install) self.__thread = QThread(self) self.__thread.start() self.__installer.moveToThread(self.__thread) self.__installer.finished.connect(self.__on_installer_finished) self.__installer.error.connect(self.__on_installer_error) self.__installer.installStatusChanged.connect( self.__progress.setLabelText) self.__progress.show() self.__progress.setLabelText("Installing") self.__installer.start() else: self.accept() def __on_installer_error(self, command, pkg, retcode, output): message_error( "An error occurred while running a subprocess", title="Error", informative_text="{} exited with non zero status.".format(command), details="".join(output), parent=self ) self.reject() def __on_installer_finished(self): message = ( ("Changes successfully applied in <i>{}</i>.<br>".format( USER_SITE) if self.user_install else '') + "Please restart Orange for changes to take effect.") message_information(message, parent=self) self.accept()
class CpkPacker(): def __init__(self, parent = None): self.parent = parent self.process = None def __pack_cpk(self, csv, cpk): self.progress.setValue(0) self.progress.setMaximum(1000) self.progress.setLabelText("Building %s" % cpk) process = QProcess() process.start("tools/cpkmakec", [csv, cpk, "-align=2048", "-mode=FILENAME"]) percent = 0 while not process.waitForFinished(100): output = QString(process.readAll()) output = output.split("\n", QString.SkipEmptyParts) for line in output: line = common.qt_to_unicode(line) match = OUTPUT_RE.search(line) if match == None: continue percent = float(match.group(1)) * 1000 self.progress.setValue(percent) percent += 1 def __cache_outdated(self, src_dir, cache_file): if not os.path.isfile(cache_file): return True cache_updated = os.path.getmtime(cache_file) for src_file in list_all_files(src_dir): if os.path.getmtime(src_file) > cache_updated: return True return False def create_archives(self): try: self.width = self.parent.width() self.height = self.parent.height() self.x = self.parent.x() self.y = self.parent.y() except: self.width = 1920 self.height = 1080 self.x = 0 self.y = 0 self.progress = QProgressDialog("Reading...", QtCore.QString(), 0, 7600, self.parent) self.progress.setWindowModality(Qt.Qt.WindowModal) self.progress.setValue(0) self.progress.setAutoClose(False) self.progress.setMinimumDuration(0) USRDIR = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "USRDIR") eboot_path = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN") eboot = BitStream(filename = eboot_path) eboot = eboot_patch.apply_eboot_patches(eboot) # So we can loop. :) ARCHIVE_INFO = [ { "dir": common.editor_config.data00_dir, "cpk": os.path.join(USRDIR, "data00.cpk"), "csv": os.path.join("data", "data00.csv" if not common.editor_config.quick_build else "data00-quick.csv"), "name": "data00.cpk", "pack": common.editor_config.pack_data00, }, { "dir": common.editor_config.data01_dir, "cpk": os.path.join(USRDIR, "data01.cpk"), "csv": os.path.join("data", "data01.csv" if not common.editor_config.quick_build else "data01-quick.csv"), "name": "data01.cpk", "pack": common.editor_config.pack_data01, }, ] # temp_dir = tempfile.mkdtemp(prefix = "sdse-") temp_dir = common.editor_config.build_cache for archive in ARCHIVE_INFO: if not archive["pack"]: continue self.progress.setWindowTitle("Building " + archive["name"]) toc_info = {} file_list = None if archive["toc"]: file_list = [] toc = get_toc(eboot, archive["toc"]) for entry in toc: filename = entry["filename"] pos_pos = entry["file_pos_pos"] len_pos = entry["file_len_pos"] toc_info[filename] = [pos_pos, len_pos] file_list.append(filename) # Causes memory issues if I use the original order, for whatever reason. file_list = None csv_template_f = open(archive["csv"], "rb") csv_template = csv.reader(csv_template_f) csv_out_path = os.path.join(temp_dir, "cpk.csv") csv_out_f = open(csv_out_path, "wb") csv_out = csv.writer(csv_out_f) for row in csv_template: if len(row) < 4: continue base_path = row[0] real_path = os.path.join(archive["dir"], base_path) out_path = os.path.join(temp_dir, archive["name"], base_path) self.progress.setValue(self.progress.value() + 1) self.progress.setLabelText("Reading...\n%s" % real_path) # All items in the CPK list should be files. # Therefore, if we have a directory, then it needs to be packed. if os.path.isdir(real_path): if self.__cache_outdated(real_path, out_path): out_dir = os.path.dirname(out_path) try: os.makedirs(out_dir) except: pass data = pack_dir(real_path) with open(out_path, "wb") as out_file: data.tofile(out_file) del data elif os.path.isfile(real_path): # If it's a file, though, we can just use it directly. out_path = real_path row[0] = out_path csv_out.writerow(row) csv_template_f.close() csv_out_f.close() self.__pack_cpk(csv_out_path, archive["cpk"]) # We're playing fast and loose with the file count anyway, so why not? self.file_count += 1 self.progress.setValue(self.file_count) self.progress.setLabelText("Saving " + archive["name"] + "...") if archive["toc"]: for entry in table_of_contents: if not entry in toc_info: _LOGGER.warning("%s missing from %s table of contents." % (entry, archive["name"])) continue file_pos = table_of_contents[entry]["pos"] file_size = table_of_contents[entry]["size"] eboot.overwrite(BitStream(uintle = file_pos, length = 32), toc_info[entry][0] * 8) eboot.overwrite(BitStream(uintle = file_size, length = 32), toc_info[entry][1] * 8) del table_of_contents self.progress.setWindowTitle("Building...") self.progress.setLabelText("Saving EBOOT.BIN...") self.progress.setValue(self.progress.maximum()) with open(eboot_path, "wb") as f: eboot.tofile(f) # Text replacement to_replace = eboot_text.get_eboot_text() for replacement in to_replace: orig = bytearray(replacement.orig, encoding = replacement.enc) # If they left something blank, write the original text back. if len(replacement.text) == 0: data = orig else: data = bytearray(replacement.text, encoding = replacement.enc) pos = replacement.pos.int + eboot_offset padding = len(orig) - len(data) if padding > 0: # Null bytes to fill the rest of the space the original took. data.extend(bytearray(padding)) data = ConstBitStream(bytes = data) eboot.overwrite(data, pos * 8) eboot_out = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN") with open(eboot_out, "wb") as f: eboot.tofile(f) self.progress.close()
def __init__(self, navdb): super(Gui, self).__init__([]) self.acdata = ACDataEvent() self.navdb = navdb self.radarwidget = [] self.command_history = [] self.cmdargs = [] self.history_pos = 0 self.command_mem = '' self.command_line = '' self.prev_cmdline = '' self.simevent_target = 0 self.mousepos = (0, 0) self.prevmousepos = (0, 0) self.panzoomchanged = False # Register our custom pan/zoom event for etype in [PanZoomEventType, ACDataEventType, SimInfoEventType, StackTextEventType, ShowDialogEventType, DisplayFlagEventType, RouteDataEventType, DisplayShapeEventType]: reg_etype = QEvent.registerEventType(etype) if reg_etype != etype: print('Warning: Registered event type differs from requested type id (%d != %d)' % (reg_etype, etype)) self.splash = Splash() self.splash.show() self.splash.showMessage('Constructing main window') self.processEvents() # Install error message handler handler = QErrorMessage.qtHandler() handler.setWindowFlags(Qt.WindowStaysOnTopHint) # Check and set OpenGL capabilities if not QGLFormat.hasOpenGL(): raise RuntimeError('No OpenGL support detected for this system!') else: f = QGLFormat() f.setVersion(3, 3) f.setProfile(QGLFormat.CoreProfile) f.setDoubleBuffer(True) QGLFormat.setDefaultFormat(f) print('QGLWidget initialized for OpenGL version %d.%d' % (f.majorVersion(), f.minorVersion())) # Create the main window and related widgets self.radarwidget = RadarWidget(navdb) self.win = MainWindow(self, self.radarwidget) self.nd = ND(shareWidget=self.radarwidget) # Enable HiDPI support (Qt5 only) if QT_VERSION == 5: self.setAttribute(Qt.AA_UseHighDpiPixmaps) timer = QTimer(self) timer.timeout.connect(self.radarwidget.updateGL) timer.timeout.connect(self.nd.updateGL) timer.start(50) # Load geo data if False: pb = QProgressDialog('Binary buffer file not found, or file out of date: Constructing vertex buffers from geo data.', 'Cancel', 0, 100) pb.setWindowFlags(Qt.WindowStaysOnTopHint) pb.show() for i in range(101): pb.setValue(i) self.processEvents() QThread.msleep(100) pb.close()
def export_umdimage2(src, dst, convert_gim = True, unique = False, parent = None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError("Cannot export %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Export Directory", "Exporting directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Proceed?", buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton = QtGui.QMessageBox.No ) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Exporting...", "Cancel", 0, 0, parent) progress.setWindowTitle("Exporting...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) if unique: tmp_dst = tempfile.mkdtemp(prefix = "sdse-") else: tmp_dst = dst seen_groups = [] for pak in glob.iglob(os.path.join(src, "bg_*.pak")): if progress.wasCanceled(): break pak_name = os.path.basename(pak) out_dir = os.path.join(tmp_dst, pak_name) progress.setLabelText("Extracting:\n" + pak) thread = threading.Thread(target = extract_model_pak, args = (pak, out_dir, convert_gim)) thread.start() while thread.isAlive(): thread.join(MIN_INTERVAL) progress.setValue(progress.value() ^ 1) if progress.wasCanceled(): progress.setLabelText("Canceling...") if progress.wasCanceled(): break if unique: for img in list_all_files(out_dir): img_base = img[len(tmp_dst) + 1:] dupe_name = os.path.splitext(img_base)[0] + ".gim" dupe_name = os.path.join("umdimage2", dupe_name) dupe_name = os.path.normpath(os.path.normcase(dupe_name)) group = _DUPE_DB.group_from_file(dupe_name) if group in seen_groups: continue if not group == None: seen_groups.append(group) dst_file = os.path.join(dst, img_base) dst_dir = os.path.dirname(dst_file) try: os.makedirs(dst_dir) except: pass shutil.copy2(img, dst_file) shutil.rmtree(out_dir) if unique: shutil.rmtree(tmp_dst) progress.close()
def onGenerate(self): """ Slot raised to initiate the certificate generation process. """ self._notifBar.clear() #Validate records records = self.personFKMapper.entities() if records == None: self._notifBar.insertErrorNotification(QApplication.translate("PersonDocumentGenerator", \ "Please select at least one person record")) return if self._docTemplatePath == "": self._notifBar.insertErrorNotification(QApplication.translate("PersonDocumentGenerator", \ "Please select a document template to use")) return documentNamingAttrs = self.lstDocNaming.selectedMappings() if self.chkUseOutputFolder.checkState() == Qt.Checked and len(documentNamingAttrs) == 0: self._notifBar.insertErrorNotification(QApplication.translate("PersonDocumentGenerator", \ "Please select at least one field for naming the output document.")) return #Set output file properties if self.rbExpImage.isChecked(): outputMode = DocumentGenerator.Image fileExtension = self.cboImageType.currentText() saveAsText = "Image File" else: outputMode = DocumentGenerator.PDF fileExtension = "pdf" saveAsText = "PDF File" #Show save file dialog if not using output folder if self.chkUseOutputFolder.checkState() == Qt.Unchecked: docDir = "" if self._outputFilePath != "": fileInfo = QFileInfo(self._outputFilePath) docDir = fileInfo.dir().path() self._outputFilePath = QFileDialog.getSaveFileName(self,QApplication.translate("PersonDocumentGenerator", \ "Save Document"), \ docDir, \ "{0} (*.{1})".format(QApplication.translate("PersonDocumentGenerator", \ saveAsText), \ fileExtension)) if self._outputFilePath == "": self._notifBar.insertErrorNotification(QApplication.translate("PersonDocumentGenerator", \ "Process aborted. No output file was specified.")) return #Include extension in file name self._outputFilePath = self._outputFilePath #+ "." + fileExtension else: #Multiple files to be generated. pass docGenerator = DocumentGenerator(self._iface,self) #Apply cell formatters for naming output files docGenerator.setAttrValueFormatters(self.personFKMapper.cellFormatters()) entityFieldName = "id" #Iterate through the selected records progressDlg = QProgressDialog(self) progressDlg.setMaximum(len(records)) try: for i,record in enumerate(records): progressDlg.setValue(i) if progressDlg.wasCanceled(): break #User-defined location if self.chkUseOutputFolder.checkState() == Qt.Unchecked: status,msg = docGenerator.run(self._docTemplatePath,entityFieldName,record.id,outputMode, \ filePath = self._outputFilePath) #Output folder location using custom naming else: status,msg = docGenerator.run(self._docTemplatePath,entityFieldName,record.id,outputMode, \ dataFields = documentNamingAttrs,fileExtension = fileExtension, \ dbmodel = self._dbModel) if not status: result = QMessageBox.warning(self, QApplication.translate("PersonDocumentGenerator","Document Generate Error"), msg, QMessageBox.Ignore|QMessageBox.Abort) if result == QMessageBox.Abort: progressDlg.close() return else: progressDlg.setValue(len(records)) QMessageBox.information(self, QApplication.translate("PersonDocumentGenerator","Document Generation Complete"), QApplication.translate("PersonDocumentGenerator","Document generation has successfully completed.") ) except Exception as ex: QMessageBox.information(self,"STDM",QApplication.translate("PersonDocumentGenerator","Error Generating documents %s"%(str(ex.message)))) return #Reset UI self.reset()
def export_umdimage(src, dst, convert_gim=True, unique=False, parent=None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError( "Cannot export %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Export Directory", "Exporting directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Proceed?", buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton=QtGui.QMessageBox.No) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Exporting...", "Cancel", 0, 0, parent) progress.setWindowTitle("Exporting...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) if parent: width = parent.width() height = parent.height() x = parent.x() y = parent.y() else: width = 1920 height = 1080 x = 0 y = 0 seen_groups = [] count = 0 last_update = time.time() progress.setMaximum(60000) for filename in list_all_files(src): if progress.wasCanceled(): break count += 1 if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Exporting...\n" + filename) progress.setValue(count) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) base_name = filename[len(src) + 1:] if unique: dupe_name = os.path.join("umdimage", base_name) dupe_name = os.path.normpath(os.path.normcase(dupe_name)) group = _DUPE_DB.group_from_file(dupe_name) if group in seen_groups: continue if not group == None: seen_groups.append(group) dst_file = os.path.join(dst, base_name) dst_dir = os.path.dirname(dst_file) ext = os.path.splitext(dst_file)[1].lower() try: os.makedirs(dst_dir) except: pass if ext == ".gim" and convert_gim: dst_file = os.path.splitext(dst_file)[0] + ".png" _CONV.gim_to_png(filename, dst_file) else: shutil.copy2(filename, dst_file) progress.close()
class AddonManagerDialog(QDialog): _packages = None def __init__(self, parent=None, **kwargs): super().__init__(parent, **kwargs) self.setLayout(QVBoxLayout()) self.layout().setContentsMargins(0, 0, 0, 0) self.addonwidget = AddonManagerWidget() self.layout().addWidget(self.addonwidget) buttons = QDialogButtonBox( orientation=Qt.Horizontal, standardButtons=QDialogButtonBox.Ok | QDialogButtonBox.Cancel ) buttons.accepted.connect(self.__accepted) buttons.rejected.connect(self.reject) self.layout().addWidget(buttons) self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) if AddonManagerDialog._packages is None: self._f_pypi_addons = self._executor.submit(list_pypi_addons) else: self._f_pypi_addons = concurrent.futures.Future() self._f_pypi_addons.set_result(AddonManagerDialog._packages) self._f_pypi_addons.add_done_callback( method_queued(self._set_packages, (object,)) ) self.__progress = QProgressDialog( self, Qt.Sheet, minimum=0, maximum=0, labelText=self.tr("Retrieving package list"), sizeGripEnabled=False, ) self.__progress.rejected.connect(self.reject) self.__thread = None self.__installer = None @Slot(object) def _set_packages(self, f): if self.__progress.isVisible(): self.__progress.close() try: packages = f.result() except (IOError, OSError) as err: message_warning( "Could not retrieve package list", title="Error", informative_text=str(err), parent=self ) packages = [] except Exception: raise else: AddonManagerDialog._packages = packages installed = list_installed_addons() dists = {dist.project_name: dist for dist in installed} packages = {pkg.name: pkg for pkg in packages} project_names = unique( itertools.chain(packages.keys(), dists.keys()) ) items = [] for name in project_names: if name in dists and name in packages: item = Installed(packages[name], dists[name]) elif name in dists: item = Installed(None, dists[name]) elif name in packages: item = Available(packages[name]) else: assert False items.append(item) self.addonwidget.set_items(items) def showEvent(self, event): super().showEvent(event) if not self._f_pypi_addons.done(): QTimer.singleShot(0, self.__progress.show) def done(self, retcode): super().done(retcode) self._f_pypi_addons.cancel() self._executor.shutdown(wait=False) if self.__thread is not None: self.__thread.quit() self.__thread.wait(1000) def closeEvent(self, event): super().closeEvent(event) self._f_pypi_addons.cancel() self._executor.shutdown(wait=False) if self.__thread is not None: self.__thread.quit() self.__thread.wait(1000) def __accepted(self): steps = self.addonwidget.item_state() if steps: # Move all uninstall steps to the front steps = sorted( steps, key=lambda step: 0 if step[0] == Uninstall else 1 ) self.__installer = Installer(steps=steps) self.__thread = QThread(self) self.__thread.start() self.__installer.moveToThread(self.__thread) self.__installer.finished.connect(self.__on_installer_finished) self.__installer.error.connect(self.__on_installer_error) self.__installer.installStatusChanged.connect( self.__progress.setLabelText) self.__progress.show() self.__progress.setLabelText("Installing") self.__installer.start() else: self.accept() def __on_installer_error(self, command, pkg, retcode, output): message_error( "An error occurred while running a subprocess", title="Error", informative_text="{} exited with non zero status.".format(command), details="".join(output), parent=self ) self.reject() def __on_installer_finished(self): message_information( "Please restart the application for changes to take effect.", parent=self) self.accept()
class DatPacker(): def __init__(self, parent=None): self.file_count = 0 self.parent = parent def create_archives(self): try: self.width = self.parent.width() self.height = self.parent.height() self.x = self.parent.x() self.y = self.parent.y() except: self.width = 1920 self.height = 1080 self.x = 0 self.y = 0 self.file_count = 0 self.progress = QProgressDialog("Reading...", QtCore.QString(), 0, 72000, self.parent) self.progress.setWindowModality(Qt.Qt.WindowModal) self.progress.setValue(0) self.progress.setAutoClose(False) self.progress.setMinimumDuration(0) # with open(common.editor_config.eboot_orig, "rb") as f: with open( os.path.join(common.editor_config.iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN"), "rb") as f: eboot = BitStream(bytes=f.read()) eboot, eboot_offset = eboot_patch.apply_eboot_patches(eboot) USRDIR = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "USRDIR") # So we can loop. :) ARCHIVE_INFO = [ { "toc": UMDIMAGES.umdimage, "dir": common.editor_config.umdimage_dir, "dat": os.path.join(USRDIR, "umdimage.dat"), "name": "umdimage.dat", "pack": common.editor_config.pack_umdimage, "eof": False, }, { "toc": UMDIMAGES.umdimage2, "dir": common.editor_config.umdimage2_dir, "dat": os.path.join(USRDIR, "umdimage2.dat"), "name": "umdimage2.dat", "pack": common.editor_config.pack_umdimage2, "eof": False, }, { "toc": None, "dir": common.editor_config.voice_dir, "dat": os.path.join(USRDIR, "voice.pak"), "name": "voice.pak", "pack": common.editor_config.pack_voice, "eof": True, }, { "toc": None, "dir": common.editor_config.bgm_dir, "dat": os.path.join(USRDIR, "bgm.pak"), "name": "bgm.pak", "pack": common.editor_config.pack_bgm, "eof": True, }, ] for archive in ARCHIVE_INFO: if not archive["pack"]: continue self.progress.setWindowTitle("Building " + archive["name"]) toc_info = {} file_list = None if archive["toc"]: file_list = [] toc = get_toc(eboot, archive["toc"]) for entry in toc: filename = entry["filename"] pos_pos = entry["file_pos_pos"] len_pos = entry["file_len_pos"] toc_info[filename] = [pos_pos, len_pos] file_list.append(filename) # Causes memory issues if I use the original order, for whatever reason. file_list = None with io.FileIO(archive["dat"], "w") as handler: table_of_contents = self.pack_dir(archive["dir"], handler, file_list=file_list, eof=archive["eof"]) # We're playing fast and loose with the file count anyway, so why not? self.file_count += 1 self.progress.setValue(self.file_count) self.progress.setLabelText("Saving " + archive["name"] + "...") if archive["toc"]: for entry in table_of_contents: if not entry in toc_info: _LOGGER.warning( "%s missing from %s table of contents." % (entry, archive["name"])) continue file_pos = table_of_contents[entry]["pos"] file_size = table_of_contents[entry]["size"] eboot.overwrite(BitStream(uintle=file_pos, length=32), toc_info[entry][0] * 8) eboot.overwrite(BitStream(uintle=file_size, length=32), toc_info[entry][1] * 8) del table_of_contents self.progress.setLabelText("Saving EBOOT.BIN...") self.progress.setValue(self.progress.maximum()) # Text replacement to_replace = eboot_text.get_eboot_text() for replacement in to_replace: orig = bytearray(replacement.orig, encoding=replacement.enc) # If they left something blank, write the original text back. if len(replacement.text) == 0: data = orig else: data = bytearray(replacement.text, encoding=replacement.enc) pos = replacement.pos.int + eboot_offset padding = len(orig) - len(data) if padding > 0: # Null bytes to fill the rest of the space the original took. data.extend(bytearray(padding)) data = ConstBitStream(bytes=data) eboot.overwrite(data, pos * 8) eboot_out = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN") with open(eboot_out, "wb") as f: eboot.tofile(f) self.progress.close() def pack_dir(self, dir, handler, file_list=None, align_toc=16, align_files=16, eof=False): table_of_contents = {} if file_list == None: file_list = sorted(os.listdir(dir)) num_files = len(file_list) toc_length = (num_files + 1) * 4 if eof: toc_length += 1 if toc_length % align_toc > 0: toc_length += align_toc - (toc_length % align_toc) handler.seek(0) handler.write(struct.pack("<I", num_files)) handler.write(bytearray(toc_length - 4)) for file_num, item in enumerate(file_list): full_path = os.path.join(dir, item) if os.path.isfile(full_path): basename = os.path.basename(item) basename, ext = os.path.splitext(basename) # Special handling for certain data types. if ext == ".txt": data = self.pack_txt(full_path) # anagram_81.dat is not a valid anagram file. <_> elif basename[: 8] == "anagram_" and ext == ".dat" and not basename == "anagram_81": anagram = AnagramFile(full_path) data = anagram.pack(for_game=True).bytes else: with open(full_path, "rb") as f: data = f.read() else: temp_align_toc = 16 temp_align_files = 4 if item in SPECIAL_ALIGN: temp_align_toc = SPECIAL_ALIGN[item][0] temp_align_files = SPECIAL_ALIGN[item][1] elif os.path.basename(dir) in SPECIAL_ALIGN and len( SPECIAL_ALIGN[os.path.basename(dir)]) == 4: temp_align_toc = SPECIAL_ALIGN[os.path.basename(dir)][2] temp_align_files = SPECIAL_ALIGN[os.path.basename(dir)][3] if os.path.splitext(full_path)[1].lower() == ".lin": data = self.pack_lin(full_path) else: data = io.BytesIO() with io.BufferedWriter(data) as fh: self.pack_dir(full_path, fh, align_toc=temp_align_toc, align_files=temp_align_files, eof=eof) fh.flush() data = data.getvalue() data = bytearray(data) file_size = len(data) padding = 0 if file_size % align_files > 0: padding = align_files - (file_size % align_files) data.extend(bytearray(padding)) handler.seek(0, io.SEEK_END) file_pos = handler.tell() handler.write(data) handler.seek((file_num + 1) * 4) handler.write(struct.pack("<I", file_pos)) del data self.file_count += 1 if self.file_count % 25 == 0: self.progress.setLabelText("Reading...\n" + full_path) self.progress.setValue(self.file_count) # Re-center the dialog. progress_w = self.progress.geometry().width() progress_h = self.progress.geometry().height() new_x = self.x + ((self.width - progress_w) / 2) new_y = self.y + ((self.height - progress_h) / 2) self.progress.move(new_x, new_y) table_of_contents[item] = {} table_of_contents[item]["size"] = file_size table_of_contents[item]["pos"] = file_pos if eof: handler.seek(0, io.SEEK_END) archive_len = handler.tell() handler.seek((num_files + 1) * 4) handler.write(struct.pack("<I", archive_len)) return table_of_contents def pack_txt(self, filename): if os.path.basename(os.path.dirname(filename)) in SCRIPT_NONSTOP: is_nonstop = True else: is_nonstop = False text = text_files.load_text(filename) text = RE_SCRIPT.sub(u"\g<1>", text) # Nonstop Debate lines need an extra newline at the end # so they show up in the backlog properly. if is_nonstop and not text[-1] == "\n": text += "\n" return SCRIPT_BOM.bytes + bytearray( text, encoding="UTF-16LE") + SCRIPT_NULL.bytes def pack_lin(self, dir): # Collect our files. file_list = sorted(list_all_files(dir)) txt = [ filename for filename in file_list if os.path.splitext(filename)[1].lower() == ".txt" ] wrd = [ filename for filename in file_list if os.path.splitext(filename)[1].lower() == ".wrd" ] py = [ filename for filename in file_list if os.path.splitext(filename)[1].lower() == ".py" ] # If there are more than one for whatever reason, just take the first. # We only have use for a single wrd or python file. wrd = wrd[0] if wrd else None py = py[0] if py else None # Prepare our temporary output directory. temp_dir = tempfile.mkdtemp(prefix="sdse-") # Where we're outputting our wrd file, regardless of whether it's a python # file or a raw binary data file. wrd_dst = os.path.join(temp_dir, "0.scp.wrd") if py: # _LOGGER.info("Compiling %s to binary." % py) try: wrd_file = WrdFile(py) except: _LOGGER.warning( "%s failed to compile. Parsing wrd file instead. Exception info:\n%s" % (py, traceback.format_exc())) shutil.copy(wrd, wrd_dst) else: # If we succeeded in loading the python file, compile it to binary. # wrd_file.save_bin(wrd) wrd_file.save_bin(wrd_dst) else: shutil.copy(wrd, wrd_dst) # Pack the text files in-place to save us a bunch of copying # and then move it to the tmp directory with the wrd file. if txt: with io.FileIO(os.path.join(temp_dir, "1.dat"), "w") as h: self.pack_dir(dir, h, file_list=txt) # Then pack it like normal. data = io.BytesIO() with io.BufferedWriter(data) as h: self.pack_dir(temp_dir, h) h.flush() data = data.getvalue() shutil.rmtree(temp_dir) return data
class BAONQtApplication(QApplication): BACKUP_DIALOG_CAPTION = 'Rename Plan Backup Detected' BACKUP_DIALOG_ERROR_CAPTION = 'Error' BACKUP_INTRO_TEXT = 'BAON has detected a backed up rename plan from a previous run of the '\ 'application. This suggests that the application crashed partway through executing a rename operation. The '\ 'files may have been left in an inconsistent state.' BACKUP_PROMPT_TEXT = 'What do you want to do?' REVERT_BACKUP_PROGRESS_TEXT = 'Reverting the rename operation' SUCCESS_DIALOG_CAPTION = 'Success' WARNING_DIALOG_CAPTION = 'Warning' BACKUP_DELETED_DIALOG_TEXT =\ 'The backed up plan has been deleted. Further runs of the application will proceed normally.' BACKUP_REVERTED_SUCCESS_DIALOG_TEXT = 'The rename operation has been reverted successfully. The directory state '\ 'should now have been completely restored.' BACKUP_REVERTED_WARNING_DIALOG_TEXT = 'There were inconsistencies while trying to revert the previous rename '\ 'operation. The directory state may not have been fully restored.' REVERT_BACKUP_BUTTON_TEXT = 'Revert the rename (recommended)' DELETE_BACKUP_BUTTON_TEXT = 'Delete the backup file' EXAMINE_BACKUP_BUTTON_TEXT = 'Examine the plan in a text editor' QUIT_BUTTON_TEXT = 'Quit BAON' request_revert_backup = pyqtSignal() request_delete_backup = pyqtSignal() _main_window = None _core = None _progress_dialog = None _core_thread = None def __init__(self, args): super().__init__(sys.argv) # Actually we do quit when the last window is closed, but we need to do this in a more controlled way self.setQuitOnLastWindowClosed(False) self._init_threads() self._init_main_objects(args) self._connect_main_objects() self._start_core() def event(self, evt): if isinstance(evt, QFileOpenEvent): path = evt.file() if not os.path.isdir(path): path, _ = os.path.split(path) self._main_window.set_base_path(path) return True return super().event(evt) def _init_threads(self): self._core_thread = QThread() self._core_thread.start() def _init_main_objects(self, args): self._main_window = MainWindow(args) self._core = BAONQtCore(args) self._core.moveToThread(self._core_thread) def _connect_main_objects(self): self.aboutToQuit.connect(self._on_quit) # Core vs. application self._core.request_backup_decision.connect(self.backup_decision_requested) self._core.reverted_backup.connect(self.notify_backup_reverted) self._core.revert_backup_error.connect(self.handle_backup_op_error) self._core.deleted_backup.connect(self.notify_backup_deleted) self._core.delete_backup_error.connect(self.handle_backup_op_error) self.request_revert_backup.connect(self._core.revert_backup) self.request_delete_backup.connect(self._core.delete_backup) # Core vs. main window self._core.prologue_finished.connect(self._main_window.show_first_time) self._core.status_changed.connect(self._main_window.report_status) self._core.scanned_files_updated.connect(self._main_window.update_scanned_files) self._core.renamed_files_updated.connect(self._main_window.update_renamed_files) self._core.has_shutdown.connect(self.quit) self._main_window.base_path_edited.connect(self._core.update_base_path) self._main_window.scan_recursive_changed.connect(self._core.update_scan_recursive) self._main_window.rules_text_changed.connect(self._core.update_rules_text) self._main_window.use_path_changed.connect(self._core.update_use_path) self._main_window.use_extension_changed.connect(self._core.update_use_extension) self._main_window.request_add_override.connect(self._core.add_override) self._main_window.request_remove_override.connect(self._core.remove_override) self._main_window.request_do_rename.connect(self._core.do_rename) self._main_window.request_rescan.connect(self._core.rescan) self._main_window.rejected.connect(self._core.shutdown) def _start_core(self): QMetaObject.invokeMethod(self._core, 'start', Qt.QueuedConnection) def _on_quit(self): self._core_thread.quit() self._core_thread.wait() @pyqtSlot() def backup_decision_requested(self): self._show_backup_decision() @pyqtSlot(Exception) def handle_backup_op_error(self, error): self._close_progress_dialog() self._show_backup_decision(error=error) @pyqtSlot() def notify_backup_deleted(self): QMessageBox.information( None, self.SUCCESS_DIALOG_CAPTION, self.BACKUP_DELETED_DIALOG_TEXT, ) @pyqtSlot(bool) def notify_backup_reverted(self, complete_success): self._close_progress_dialog() if complete_success: QMessageBox.information( None, self.SUCCESS_DIALOG_CAPTION, self.BACKUP_REVERTED_SUCCESS_DIALOG_TEXT, ) else: QMessageBox.warning( None, self.WARNING_DIALOG_CAPTION, self.BACKUP_REVERTED_WARNING_DIALOG_TEXT, ) def _show_backup_decision(self, error=None): text = '<p>{0}</p><p>{1}</p>'.format( self.BACKUP_INTRO_TEXT if error is None else error, self.BACKUP_PROMPT_TEXT, ) dialog = QMessageBox( QMessageBox.Question if error is None else QMessageBox.Critical, self.BACKUP_DIALOG_CAPTION if error is None else self.BACKUP_DIALOG_ERROR_CAPTION, text, ) revert_button = dialog.addButton(self.REVERT_BACKUP_BUTTON_TEXT, QMessageBox.AcceptRole) delete_button = dialog.addButton(self.DELETE_BACKUP_BUTTON_TEXT, QMessageBox.DestructiveRole) examine_button = dialog.addButton(self.EXAMINE_BACKUP_BUTTON_TEXT, QMessageBox.ActionRole) dialog.addButton(self.QUIT_BUTTON_TEXT, QMessageBox.RejectRole) dialog.exec() clicked_button = dialog.clickedButton() if clicked_button == examine_button: QMetaObject.invokeMethod(self, '_examine_backup', Qt.QueuedConnection) elif clicked_button == revert_button: self._progress_dialog = QProgressDialog(None) self._progress_dialog.setLabelText(self.REVERT_BACKUP_PROGRESS_TEXT) self._progress_dialog.setCancelButton(None) self._progress_dialog.setRange(0, 0) self._progress_dialog.forceShow() self.request_revert_backup.emit() elif clicked_button == delete_button: self.request_delete_backup.emit() else: self.quit() @pyqtSlot() def _examine_backup(self): error = None try: filename = get_rename_plan_backup_filename() QDesktopServices.openUrl(QUrl.fromLocalFile(filename)) except Exception as err: error = err finally: self._show_backup_decision(error) def _close_progress_dialog(self): if self._progress_dialog is not None: self._progress_dialog.close() self._progress_dialog = None
class CpkPacker(): def __init__(self, parent = None): self.parent = parent self.process = None def __pack_cpk(self, csv, cpk): self.progress.setValue(0) self.progress.setMaximum(100000) self.progress.setLabelText("Building %s" % cpk) process = QProcess() process.start("tools/cpkmakec", [csv, cpk, "-align=2048", "-mode=FILENAME"]) percent = 0 while not process.waitForFinished(100): output = QString(process.readAll()) output = output.split("\n", QString.SkipEmptyParts) for line in output: line = common.qt_to_unicode(line) match = OUTPUT_RE.search(line) if match == None: continue percent = float(match.group(1)) * 1000 self.progress.setValue(percent) percent += 1 def __cache_outdated(self, src_dir, cache_file): if not os.path.isfile(cache_file): return True cache_updated = os.path.getmtime(cache_file) for src_file in list_all_files(src_dir): if os.path.getmtime(src_file) > cache_updated: return True return False def create_archives(self): try: self.width = self.parent.width() self.height = self.parent.height() self.x = self.parent.x() self.y = self.parent.y() except: self.width = 1920 self.height = 1080 self.x = 0 self.y = 0 self.progress = QProgressDialog("Reading...", QtCore.QString(), 0, 7600, self.parent) self.progress.setWindowModality(Qt.Qt.WindowModal) self.progress.setValue(0) self.progress.setAutoClose(False) self.progress.setMinimumDuration(0) USRDIR = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "USRDIR") eboot_path = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN") eboot = BitStream(filename = eboot_path) eboot = eboot_patch.apply_eboot_patches(eboot) # So we can loop. :) ARCHIVE_INFO = [ { "dir": common.editor_config.data00_dir, "cpk": os.path.join(USRDIR, "data00.cpk"), "csv": os.path.join("data", "data00.csv" if not common.editor_config.quick_build else "data00-quick.csv"), "name": "data00.cpk", "pack": common.editor_config.pack_data00, }, { "dir": common.editor_config.data01_dir, "cpk": os.path.join(USRDIR, "data01.cpk"), "csv": os.path.join("data", "data01.csv" if not common.editor_config.quick_build else "data01-quick.csv"), "name": "data01.cpk", "pack": common.editor_config.pack_data01, }, ] # temp_dir = tempfile.mkdtemp(prefix = "sdse-") temp_dir = common.editor_config.build_cache for archive in ARCHIVE_INFO: if not archive["pack"]: continue self.progress.setWindowTitle("Building " + archive["name"]) csv_template_f = open(archive["csv"], "rb") csv_template = csv.reader(csv_template_f) csv_out_path = os.path.join(temp_dir, "cpk.csv") csv_out_f = open(csv_out_path, "wb") csv_out = csv.writer(csv_out_f) for row in csv_template: if len(row) < 4: continue base_path = row[0] real_path = os.path.join(archive["dir"], base_path) out_path = os.path.join(temp_dir, archive["name"], base_path) self.progress.setValue(self.progress.value() + 1) self.progress.setLabelText("Reading...\n%s" % real_path) # All items in the CPK list should be files. # Therefore, if we have a directory, then it needs to be packed. if os.path.isdir(real_path): if self.__cache_outdated(real_path, out_path): out_dir = os.path.dirname(out_path) try: os.makedirs(out_dir) except: pass data = pack_dir(real_path) with open(out_path, "wb") as out_file: data.tofile(out_file) del data elif os.path.isfile(real_path): # If it's a file, though, we can just use it directly. out_path = real_path row[0] = out_path csv_out.writerow(row) csv_template_f.close() csv_out_f.close() self.__pack_cpk(csv_out_path, archive["cpk"]) self.progress.setWindowTitle("Building...") self.progress.setLabelText("Saving EBOOT.BIN...") self.progress.setValue(self.progress.maximum()) with open(eboot_path, "wb") as f: eboot.tofile(f) # self.progress.setLabelText("Deleting temporary files...") # shutil.rmtree(temp_dir) self.progress.close()
class CSVGenerator: def __init__(self, left, top, right, bottom, csv_filename, username, password, client_id, client_secret, days_to_query=60): self.left = left self.top = top self.right = right self.bottom = bottom self.csv_filename = csv_filename self.days_to_query = days_to_query self.begin_date = None self.end_date = None self.username = username self.password = password self.client_id = client_id self.client_secret = client_secret # throw up a progress dialog min_progress = 0.0 max_progress = ((self.right - self.left) / INCREMENTAL_INTERVAL) * \ ((self.top - self.bottom) / INCREMENTAL_INTERVAL) self.current_progress = min_progress self.progress_dialog = QProgressDialog("Building up CSV file", "Abort", int(min_progress), int(max_progress), None) self.progress_dialog.setCancelButton(None) self.progress_dialog.setWindowTitle("CSV Output") self.progress_dialog.setLabelText("Building up CSV file") self.progress_dialog.setMinimumDuration(0) self.progress_dialog.setValue(0) self.progress_dialog.show() self.csv_elements = [] self.csv_generator_object = CSVGeneratorObject(self) self.vector_header_dict = {} self.pool = QThreadPool() self.finished_submissions = False self.lock = Lock() def generate_csv(self): # dates now = datetime.now() self.end_date = now self.begin_date = now - timedelta(days=self.days_to_query) current_x = self.left current_y = self.bottom serial_no = 1 # get header dict vector_query = VectorQuery(username=self.username, password=self.password, client_id=self.client_id, client_secret=self.client_secret) vector_query.log_in() vector_params = InfoCubeVectorParams(top=self.top, bottom=self.bottom, left=self.left, right=self.right, time_begin=self.begin_date, time_end=self.end_date) header_result = vector_query.get_vector_result(vector_params) self.vector_header_dict = vector_query.get_vector_data(header_result) for next_x in drange(self.left + INCREMENTAL_INTERVAL, self.right, INCREMENTAL_INTERVAL): for next_y in drange(self.bottom + INCREMENTAL_INTERVAL, self.top, INCREMENTAL_INTERVAL): username = self.username password = self.password client_id = self.client_id client_secret = self.client_secret csv_runnable = CSVRunnable(username, password, client_id, client_secret, serial_no, next_y, current_x, next_x, current_y, self.begin_date, self.end_date, self.vector_header_dict) csv_runnable.csv_object.new_csv_element.connect(self.csv_generator_object.callback) self.pool.start(csv_runnable) serial_no += 1 current_y = next_y current_y = self.bottom current_x = next_x self.finished_submissions = True def on_completion(self): self.csv_elements.sort(key=lambda element: element.serial_no) log.info("Sort complete") # write file csv_file = open(self.csv_filename, 'w') # write the header header = CSVOutput.get_csv_header() if self.vector_header_dict: for term in self.vector_header_dict: header = header + str(term) + "," header = header[:-1] csv_file.write(header) csv_file.write("\n") for csv_element in self.csv_elements: csv_file.write(str(csv_element)) csv_file.write("\n") csv_file.close() log.info("Write complete") if self.progress_dialog: self.progress_dialog.close() self.csv_generator_object.message_complete.emit(self.csv_filename)
def import_umdimage(src, dst, convert_png=True, propogate=True, parent=None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError( "Cannot import %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Import Directory", "Importing directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Any affected files will be backed up. Proceed?", buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton=QtGui.QMessageBox.No) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Finding files...", "Cancel", 0, 1, parent) progress.setWindowTitle("Importing...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) if parent: width = parent.width() height = parent.height() x = parent.x() y = parent.y() else: width = 1920 height = 1080 x = 0 y = 0 progress.setMaximum(0) progress.setValue(0) # The raw list of files we're importing. files = [] # A list of lists, including all dupes of the files being imported, too. affected_files = [] file_count = 0 dupe_base = "umdimage" tmp = tempfile.mkdtemp(prefix="sdse-") seen_groups = [] count = 0 last_update = time.time() for file in list_all_files(src): if progress.wasCanceled(): break # Strip our base directory, so we have just a relative file list. file = os.path.normpath(os.path.normcase(file[len(src) + 1:])) files.append(file) count += 1 if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Finding files...\n" + file) # progress.setValue(count) progress.setValue(progress.value() ^ 1) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) affected_files.append([]) if os.path.splitext( file)[1] == ".png" and convert_png and file not in SKIP_CONV: file = os.path.splitext(file)[0] + ".gim" if propogate: file_group = _DUPE_DB.group_from_file(os.path.join( dupe_base, file)) else: file_group = None if file_group in seen_groups: continue # If there are no dupes, just add this file. if file_group == None: affected_files[-1].append(file) file_count += 1 continue seen_groups.append(file_group) for dupe in _DUPE_DB.files_in_group(file_group): # Minus the "umdimage" part dupe = dupe[len(dupe_base) + 1:] affected_files[-1].append(dupe) file_count += 1 progress.setValue(0) progress.setMaximum(file_count) # Make a backup first. backup_dir = None count = 0 for file_set in affected_files: if progress.wasCanceled(): break for file in file_set: if progress.wasCanceled(): break count += 1 if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Backing up...\n" + file) progress.setValue(count) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) # It's perfectly possible we want to import some files that # don't already exist. Such as when importing a directory # with added lines. if not os.path.isfile(os.path.join(dst, file)): continue backup_dir = backup_files(dst, [file], suffix="_IMPORT", backup_dir=backup_dir) progress.setValue(0) # And now do our importing. import_all_new = False skip_all_new = False count = 0 for index, src_file in enumerate(files): if progress.wasCanceled(): break if os.path.splitext(src_file)[ 1] == ".png" and convert_png and src_file not in SKIP_CONV: tmp_src_file = os.path.join(tmp, os.path.basename(src_file)) tmp_src_file = os.path.splitext(tmp_src_file)[0] + ".gim" quantize = QuantizeType.auto for regex, q in FORCE_QUANTIZE: if not regex.search(src_file) == None: quantize = q break _CONV.png_to_gim(os.path.join(src, src_file), tmp_src_file, quantize) src_file = tmp_src_file else: src_file = os.path.join(src, src_file) for file in affected_files[index]: if progress.wasCanceled(): break dst_file = os.path.join(dst, file) count += 1 # if count % 25 == 0: if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Importing...\n" + file) progress.setValue(count) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) # We may be allowed to import files that don't exist, but we're # going to ask them about it anyway. if not os.path.isfile(dst_file): if skip_all_new: continue if not import_all_new: answer = QtGui.QMessageBox.question( parent, "File Not Found", "File:\n\n" + file + "\n\n" + "does not exist in the target directory. Import anyway?", buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.YesToAll | QtGui.QMessageBox.No | QtGui.QMessageBox.NoToAll, defaultButton=QtGui.QMessageBox.No) if answer == QtGui.QMessageBox.YesToAll: import_all_new = True skip_all_new = False elif answer == QtGui.QMessageBox.NoToAll: skip_all_new = True import_all_new = False continue elif answer == QtGui.QMessageBox.No: continue basedir = os.path.dirname(dst_file) if not os.path.isdir(basedir): os.makedirs(basedir) shutil.copy2(src_file, dst_file) shutil.rmtree(tmp) progress.close()
class AddonManagerDialog(QDialog): _packages = None def __init__(self, parent=None, **kwargs): super().__init__(parent, acceptDrops=True, **kwargs) self.setLayout(QVBoxLayout()) self.layout().setContentsMargins(0, 0, 0, 0) self.addonwidget = AddonManagerWidget() self.layout().addWidget(self.addonwidget) info_bar = QWidget() info_layout = QHBoxLayout() info_bar.setLayout(info_layout) self.layout().addWidget(info_bar) buttons = QDialogButtonBox(orientation=Qt.Horizontal, standardButtons=QDialogButtonBox.Ok | QDialogButtonBox.Cancel) buttons.accepted.connect(self.__accepted) buttons.rejected.connect(self.reject) self.layout().addWidget(buttons) # No system access => install into user site-packages self.user_install = not os.access(sysconfig.get_path("purelib"), os.W_OK) self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) if AddonManagerDialog._packages is None: self._f_pypi_addons = self._executor.submit(list_pypi_addons) else: self._f_pypi_addons = concurrent.futures.Future() self._f_pypi_addons.set_result(AddonManagerDialog._packages) self._f_pypi_addons.add_done_callback( method_queued(self._set_packages, (object, ))) self.__progress = QProgressDialog( self, Qt.Sheet, minimum=0, maximum=0, labelText=self.tr("Retrieving package list"), sizeGripEnabled=False, windowTitle="Progress") self.__progress.rejected.connect(self.reject) self.__thread = None self.__installer = None @Slot(object) def _set_packages(self, f): if self.__progress.isVisible(): self.__progress.close() try: packages = f.result() except (IOError, OSError) as err: message_warning("Could not retrieve package list", title="Error", informative_text=str(err), parent=self) packages = [] except Exception: raise else: AddonManagerDialog._packages = packages installed = list_installed_addons() dists = {dist.project_name: dist for dist in installed} packages = {pkg.name: pkg for pkg in packages} # For every pypi available distribution not listed by # list_installed_addons, check if it is actually already # installed. ws = pkg_resources.WorkingSet() for pkg_name in set(packages.keys()).difference(set(dists.keys())): try: d = ws.find(pkg_resources.Requirement.parse(pkg_name)) except pkg_resources.VersionConflict: pass except ValueError: # Requirements.parse error ? pass else: if d is not None: dists[d.project_name] = d project_names = unique(itertools.chain(packages.keys(), dists.keys())) items = [] for name in project_names: if name in dists and name in packages: item = Installed(packages[name], dists[name]) elif name in dists: item = Installed(None, dists[name]) elif name in packages: item = Available(packages[name]) else: assert False items.append(item) self.addonwidget.set_items(items) def showEvent(self, event): super().showEvent(event) if not self._f_pypi_addons.done(): QTimer.singleShot(0, self.__progress.show) def done(self, retcode): super().done(retcode) self._f_pypi_addons.cancel() self._executor.shutdown(wait=False) if self.__thread is not None: self.__thread.quit() self.__thread.wait(1000) def closeEvent(self, event): super().closeEvent(event) self._f_pypi_addons.cancel() self._executor.shutdown(wait=False) if self.__thread is not None: self.__thread.quit() self.__thread.wait(1000) ADDON_EXTENSIONS = ('.zip', '.whl', '.tar.gz') def dragEnterEvent(self, event): urls = event.mimeData().urls() if any((OSX_NSURL_toLocalFile(url) or url.toLocalFile() ).endswith(self.ADDON_EXTENSIONS) for url in urls): event.acceptProposedAction() def dropEvent(self, event): """Allow dropping add-ons (zip or wheel archives) on this dialog to install them""" packages = [] names = [] for url in event.mimeData().urls(): path = OSX_NSURL_toLocalFile(url) or url.toLocalFile() if path.endswith(self.ADDON_EXTENSIONS): name, vers, summary, descr = (get_meta_from_archive(path) or (os.path.basename(path), '', '', '')) names.append(name) packages.append( Installable(name, vers, summary, descr or summary, path, [path])) future = concurrent.futures.Future() future.set_result((AddonManagerDialog._packages or []) + packages) self._set_packages(future) self.addonwidget.set_install_projects(names) def __accepted(self): steps = self.addonwidget.item_state() if steps: # Move all uninstall steps to the front steps = sorted(steps, key=lambda step: 0 if step[0] == Uninstall else 1) self.__installer = Installer(steps=steps, user_install=self.user_install) self.__thread = QThread(self) self.__thread.start() self.__installer.moveToThread(self.__thread) self.__installer.finished.connect(self.__on_installer_finished) self.__installer.error.connect(self.__on_installer_error) self.__installer.installStatusChanged.connect( self.__progress.setLabelText) self.__progress.show() self.__progress.setLabelText("Installing") self.__installer.start() else: self.accept() def __on_installer_error(self, command, pkg, retcode, output): message_error( "An error occurred while running a subprocess", title="Error", informative_text="{} exited with non zero status.".format(command), details="".join(output), parent=self) self.reject() def __on_installer_finished(self): message = ( ("Changes successfully applied in <i>{}</i>.<br>".format(USER_SITE) if self.user_install else '') + "Please restart Orange for changes to take effect.") message_information(message, parent=self) self.accept()
class ExportESUShapes: """ Class handling the exports of ESUs to lines sahpefiles """ progress = QtCore.pyqtSignal(float) def __init__(self, iface, db, unassigned, export_path): self.killed = False self.iface = iface self.db = db self.unassigned = unassigned self.export_path = export_path self.prepare_sql_queries() self.progresswin = QProgressDialog("Exporting Shapefile...", "Abort", 0, 100) self.progresswin.setWindowModality(Qt.WindowModal) self.progresswin.setWindowFlags(Qt.CustomizeWindowHint | Qt.WindowTitleHint) self.progresswin.setWindowTitle(" ") def kill_export(self): self.killed = True # @thread def export_esu_line(self): """ Export ESUs :return: """ canvas = self.iface.mapCanvas() clayer = canvas.currentLayer() # will return 0 if none selected count = clayer.selectedFeatureCount() feature_count = clayer.featureCount() # Get list of selected features selected_esu_ids = list() if count > 0: selectedfeats = clayer.selectedFeatures() for feat in selectedfeats: selected_esu_ids.append(int(feat.attribute('esu_id'))) feature_count = clayer.selectedFeatureCount() self.warn_about_selected_features(feature_count) # Prepare sql query if self.unassigned: nsgexportsql = self.sql_queries['export_all'] else: nsgexportsql = self.sql_queries['export_assigned_only'] # SQL to filter out selected records if count > 0: nsgexportsql += " WHERE esu.esu_id IN ({})".format(', '.join( map(str, selected_esu_ids))) # Setup database temporary tables for table in ['qryType12', 'qryType3', 'qryType4']: # Drop tables if left behind from last export args = {'table': table} query = self.run_sql('drop_table', args) query = self.run_sql('create_qryType12') query = self.run_sql('create_qryType3') query = self.run_sql('create_qryType4') # Run the main query if config.DEBUG_MODE: print(nsgexportsql) query = QSqlQuery(self.db) query.setForwardOnly(True) query.exec_(nsgexportsql) if query.isActive() is False: raise StandardError('Database query problem: {}'.format( query.lastError().text())) # create layer vlayer = QgsVectorLayer("multilinestring?crs=EPSG:27700", "temp", "memory") vlayer.setCrs( QgsCoordinateReferenceSystem( 27700, QgsCoordinateReferenceSystem.EpsgCrsId)) provider = vlayer.dataProvider() # add fields self.fields = [ QgsField("esu_id", QVariant.String), QgsField("USRN", QVariant.LongLong), QgsField("Rec_type", QVariant.Int), QgsField("DescTxt", QVariant.String), QgsField("Locality", QVariant.String), QgsField("Town", QVariant.String), QgsField("Entry_date", QVariant.Date), QgsField("Type_3_USRN", QVariant.LongLong), QgsField("Type_3_Desc", QVariant.String), QgsField("Type_4_USRN", QVariant.LongLong), QgsField("Type_4_Desc", QVariant.String) ] provider.addAttributes(self.fields) vlayer.updateFields() # Exit if output file path is invalid if len(str(self.export_path)) < 0: return False if self.check_if_export_file_in_use(): return False # Run through SQL results creating features from rows self.progresswin.show() i = 0 while query.next(): if self.progresswin.wasCanceled(): self.kill_export() break record = query.record() new_feature = self.create_feature_from_record(record) provider.addFeatures([new_feature]) # Update progress bar i += 1 diff = feature_count + ( i - feature_count) if i > feature_count else feature_count percent_complete = (i / float(diff)) * 100 self.progresswin.setValue(percent_complete) if self.killed: # Show message and exit if killed export_error_msg_box = QMessageBox( QMessageBox.Warning, " ", "An error occurred while exporting shapefile", QMessageBox.Ok, None) export_error_msg_box.setWindowFlags(Qt.CustomizeWindowHint | Qt.WindowTitleHint) export_error_msg_box.exec_() return False vlayer.updateExtents() result = QgsVectorFileWriter.writeAsVectorFormat( vlayer, self.export_path, "utf-8", None, "ESRI Shapefile") # checks for completed export if result == 0: self.progresswin.close() if config.DEBUG_MODE: print('DEBUG_MODE: {} features exported'.format( vlayer.featureCount())) return True def check_if_export_file_in_use(self): """ Attempts to write to export file, to check if in use. Warns if so. This check only works in Windows. :return: boolean """ if config.DEBUG_MODE: print('DEBUG_MODE: Checking if output file in use.') field_map = QgsFields() for field in self.fields: field_map.append(field) writer = QgsVectorFileWriter(str(self.export_path), "utf-8", field_map, QGis.WKBMultiLineString, None, "ESRI Shapefile") if writer.hasError() != QgsVectorFileWriter.NoError: file_open_msg_box = QMessageBox( QMessageBox.Warning, " ", "The file {} is already open " "(possibly in another application).\n" "Close the file and try again".format(str(self.export_path)), QMessageBox.Ok, None) file_open_msg_box.setWindowFlags(Qt.CustomizeWindowHint | Qt.WindowTitleHint) file_open_msg_box.exec_() return True return False def create_feature_from_record(self, record): """ Create feature in temporary layer from record returned from query. :param record: QSqlQuery record describing feature """ feature = QgsFeature() wkb = record.value('geom') if type(wkb) == QByteArray: geometry = QgsGeometry() geometry.fromWkb(wkb) feature.setGeometry(geometry) else: "NOT a byte" feature.setAttributes([ record.value('esu_id'), record.value('usrn'), record.value('rec_type'), record.value('desctxt'), record.value('locality'), record.value('town'), record.value('entry_date'), record.value('type_3_usrn'), record.value('type_3_desc'), record.value('type_4_usrn'), record.value('type_4_desc') ]) return feature def warn_about_selected_features(self, feature_count): """ Pop a message box if user has selected features. :param feature_count: Number of selected features for export """ selection_warning_message = QMessageBox( QMessageBox.Information, " ", "Exporting from the {} selected ESU features only".format( feature_count), QMessageBox.Ok, None) selection_warning_message.setWindowFlags(Qt.CustomizeWindowHint | Qt.WindowTitleHint) selection_warning_message.exec_() def run_sql(self, query, kwargs={}): """ Run SQL query (defined with key 'query' in self.sql_queries) on the database. return: QSqlQuery object to extract results """ query = self.sql_queries[query] sql = query.format(**kwargs) if config.DEBUG_MODE: print(sql) active_query = QSqlQuery(sql, self.db) if active_query.isActive() is False: raise StandardError('Database query problem: {}'.format( active_query.lastError().text())) return active_query def prepare_sql_queries(self): """ Store the sql queries used in the export. """ self.sql_queries = { 'drop_table': """ DROP TABLE IF EXISTS {table} ;""", 'create_qryType12': """ CREATE TEMP TABLE qryType12 AS SELECT lnkesu_street.esu_id, tblstreet.usrn, tblstreet.street_ref_type AS Rec_type, tblstreet.description AS DescTxt, tlkplocality.NAME AS Locality, tlkptown.NAME AS Town, tblstreet.entry_date FROM lnkesu_street INNER JOIN tblstreet ON lnkesu_street.usrn_version_no = tblstreet.version_no AND lnkesu_street.usrn = tblstreet.usrn INNER JOIN tlkplocality ON tblstreet.loc_ref = tlkplocality.loc_ref INNER JOIN tlkptown ON tblstreet.town_ref = tlkptown.town_ref WHERE tblstreet.street_ref_type IN (1, 2) AND lnkesu_street.currency_flag = 0 AND tblstreet.currency_flag = 0 ;""", 'create_qryType3': """ CREATE TEMP TABLE qryType3 AS SELECT lnkesu_street.esu_id, tblstreet.usrn AS Typ3USRN, tblstreet.description AS Typ3Desc FROM lnkesu_street INNER JOIN tblstreet ON lnkesu_street.usrn = tblstreet.usrn AND lnkesu_street.usrn_version_no = tblstreet.version_no WHERE lnkesu_street.currency_flag = 0 AND tblstreet.street_ref_type = 3 AND tblstreet.currency_flag = 0 ;""", 'create_qryType4': """ CREATE TEMP TABLE qryType4 AS SELECT lnkesu_street.esu_id, tblstreet.usrn AS Typ4USRN, tblstreet.description AS Typ4Desc FROM lnkesu_street INNER JOIN tblstreet ON lnkesu_street.usrn_version_no = tblstreet.version_no AND lnkesu_street.usrn = tblstreet.usrn WHERE lnkesu_street.currency_flag = 0 AND tblstreet.street_ref_type = 4 AND tblstreet.currency_flag = 0 ;""", 'export_all': """ SELECT Asbinary(esu.geometry) AS geom, qryType12.esu_id AS esu_id, qryType12.usrn AS usrn, qryType12.rec_type AS rec_type, qryType12.desctxt AS desctxt, qryType12.locality AS locality, qryType12.town AS town, qryType12.entry_date AS entry_date, qryType3.typ3usrn AS type_3_usrn, qryType3.typ3desc AS type_3_desc, qryType4.typ4usrn AS type_3_usrn, qryType4.typ4desc AS type_4_desc FROM esu LEFT JOIN qryType12 ON esu.esu_id = qryType12.esu_id LEFT JOIN qryType3 ON esu.esu_id = qryType3.esu_id LEFT JOIN qryType4 ON esu.esu_id = qryType4.esu_id """, 'export_assigned_only': """ SELECT Asbinary(esu.geometry) AS geom, qryType12.esu_id AS esu_id, qryType12.usrn AS usrn, qryType12.rec_type AS rec_type, qryType12.desctxt AS desctxt, qryType12.locality AS locality, qryType12.town AS town, qryType12.entry_date AS entry_date, qryType3.typ3usrn AS type_3_usrn, qryType3.typ3desc AS type_3_desc, qryType4.typ4usrn AS type_3_usrn, qryType4.typ4desc AS type_4_desc FROM qryType12 LEFT JOIN qryType3 ON qryType12.esu_id = qryType3.esu_id LEFT JOIN qryType4 ON qryType12.esu_id = qryType4.esu_id LEFT JOIN esu ON qryType12.esu_id = esu.esu_id """ }
def export_umdimage(src, dst, convert_gim = True, unique = False, parent = None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError("Cannot export %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Export Directory", "Exporting directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Proceed?", buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton = QtGui.QMessageBox.No ) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Exporting...", "Cancel", 0, 0, parent) progress.setWindowTitle("Exporting...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) if parent: width = parent.width() height = parent.height() x = parent.x() y = parent.y() else: width = 1920 height = 1080 x = 0 y = 0 seen_groups = [] count = 0 last_update = time.time() progress.setMaximum(60000) for filename in list_all_files(src): if progress.wasCanceled(): break count += 1 if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Exporting...\n" + filename) progress.setValue(count) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) base_name = filename[len(src) + 1:] if unique: dupe_name = os.path.join("umdimage", base_name) dupe_name = os.path.normpath(os.path.normcase(dupe_name)) group = _DUPE_DB.group_from_file(dupe_name) if group in seen_groups: continue if not group == None: seen_groups.append(group) dst_file = os.path.join(dst, base_name) dst_dir = os.path.dirname(dst_file) ext = os.path.splitext(dst_file)[1].lower() try: os.makedirs(dst_dir) except: pass if ext == ".gim" and convert_gim: dst_file = os.path.splitext(dst_file)[0] + ".png" _CONV.gim_to_png(filename, dst_file) else: shutil.copy2(filename, dst_file) progress.close()
class AddonManagerDialog(QDialog): _packages = None def __init__(self, parent=None, **kwargs): super().__init__(parent, **kwargs) self.setLayout(QVBoxLayout()) self.layout().setContentsMargins(0, 0, 0, 0) self.addonwidget = AddonManagerWidget() self.layout().addWidget(self.addonwidget) info_bar = QWidget() info_layout = QHBoxLayout() info_bar.setLayout(info_layout) info_icon = QLabel() info_text = QLabel() info_layout.addWidget(info_icon) info_layout.addWidget(info_text) self.layout().addWidget(info_bar) buttons = QDialogButtonBox( orientation=Qt.Horizontal, standardButtons=QDialogButtonBox.Ok | QDialogButtonBox.Cancel ) buttons.accepted.connect(self.__accepted) buttons.rejected.connect(self.reject) self.layout().addWidget(buttons) if not os.access(sysconfig.get_path("purelib"), os.W_OK): if sysconfig.get_platform().startswith("macosx"): info = "You must install Orange by dragging it into" \ " Applications before installing add-ons." else: info = "You do not have permissions to write into Orange " \ "directory.\nYou may need to contact an administrator " \ "for assistance." info_text.setText(info) style = QApplication.instance().style() info_icon.setPixmap(style.standardIcon( QStyle.SP_MessageBoxCritical).pixmap(14, 14)) buttons.button(QDialogButtonBox.Ok ).setEnabled(False) self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) if AddonManagerDialog._packages is None: self._f_pypi_addons = self._executor.submit(list_pypi_addons) else: self._f_pypi_addons = concurrent.futures.Future() self._f_pypi_addons.set_result(AddonManagerDialog._packages) self._f_pypi_addons.add_done_callback( method_queued(self._set_packages, (object,)) ) self.__progress = QProgressDialog( self, Qt.Sheet, minimum=0, maximum=0, labelText=self.tr("Retrieving package list"), sizeGripEnabled=False, windowTitle="Progress" ) self.__progress.rejected.connect(self.reject) self.__thread = None self.__installer = None @Slot(object) def _set_packages(self, f): if self.__progress.isVisible(): self.__progress.close() try: packages = f.result() except (IOError, OSError) as err: message_warning( "Could not retrieve package list", title="Error", informative_text=str(err), parent=self ) packages = [] except Exception: raise else: AddonManagerDialog._packages = packages installed = list_installed_addons() dists = {dist.project_name: dist for dist in installed} packages = {pkg.name: pkg for pkg in packages} # For every pypi available distribution not listed by # list_installed_addons, check if it is actually already # installed. ws = pkg_resources.WorkingSet() for pkg_name in set(packages.keys()).difference(set(dists.keys())): try: d = ws.find(pkg_resources.Requirement.parse(pkg_name)) except pkg_resources.VersionConflict: pass except ValueError: # Requirements.parse error ? pass else: if d is not None: dists[d.project_name] = d project_names = unique( itertools.chain(packages.keys(), dists.keys()) ) items = [] for name in project_names: if name in dists and name in packages: item = Installed(packages[name], dists[name]) elif name in dists: item = Installed(None, dists[name]) elif name in packages: item = Available(packages[name]) else: assert False items.append(item) self.addonwidget.set_items(items) def showEvent(self, event): super().showEvent(event) if not self._f_pypi_addons.done(): QTimer.singleShot(0, self.__progress.show) def done(self, retcode): super().done(retcode) self._f_pypi_addons.cancel() self._executor.shutdown(wait=False) if self.__thread is not None: self.__thread.quit() self.__thread.wait(1000) def closeEvent(self, event): super().closeEvent(event) self._f_pypi_addons.cancel() self._executor.shutdown(wait=False) if self.__thread is not None: self.__thread.quit() self.__thread.wait(1000) def __accepted(self): steps = self.addonwidget.item_state() if steps: # Move all uninstall steps to the front steps = sorted( steps, key=lambda step: 0 if step[0] == Uninstall else 1 ) self.__installer = Installer(steps=steps) self.__thread = QThread(self) self.__thread.start() self.__installer.moveToThread(self.__thread) self.__installer.finished.connect(self.__on_installer_finished) self.__installer.error.connect(self.__on_installer_error) self.__installer.installStatusChanged.connect( self.__progress.setLabelText) self.__progress.show() self.__progress.setLabelText("Installing") self.__installer.start() else: self.accept() def __on_installer_error(self, command, pkg, retcode, output): message_error( "An error occurred while running a subprocess", title="Error", informative_text="{} exited with non zero status.".format(command), details="".join(output), parent=self ) self.reject() def __on_installer_finished(self): message_information( "Please restart Orange for changes to take effect.", parent=self) self.accept()
def onGenerate(self): """ Slot raised to initiate the certificate generation process. """ self._notif_bar.clear() success_status = True config = self.current_config() self.last_data_source = config.data_source() if config is None: self._notif_bar.insertErrorNotification(QApplication.translate("DocumentGeneratorDialog", \ "The entity configuration could not be extracted.")) return #Get selected records and validate records = self.tabWidget.currentWidget().entities() if self.chk_template_datasource.isChecked(): records = self._dummy_template_records() if len(records) == 0: self._notif_bar.insertErrorNotification(QApplication.translate("DocumentGeneratorDialog", \ "Please load at least one entity record")) return if not self._docTemplatePath: self._notif_bar.insertErrorNotification(QApplication.translate("DocumentGeneratorDialog", \ "Please select a document template to use")) return documentNamingAttrs = self.lstDocNaming.selectedMappings() if self.chkUseOutputFolder.checkState() == Qt.Checked and len( documentNamingAttrs) == 0: self._notif_bar.insertErrorNotification(QApplication.translate("DocumentGeneratorDialog", \ "Please select at least one field for naming the output document")) return #Set output file properties if self.rbExpImage.isChecked(): outputMode = DocumentGenerator.Image fileExtension = self.cboImageType.currentText() saveAsText = "Image File" else: outputMode = DocumentGenerator.PDF fileExtension = "pdf" saveAsText = "PDF File" #Show save file dialog if not using output folder if self.chkUseOutputFolder.checkState() == Qt.Unchecked: docDir = source_document_location() if self._outputFilePath: fileInfo = QFileInfo(self._outputFilePath) docDir = fileInfo.dir().path() self._outputFilePath = QFileDialog.getSaveFileName( self, QApplication.translate("DocumentGeneratorDialog", "Save Document"), docDir, "{0} (*.{1})".format( QApplication.translate("DocumentGeneratorDialog", saveAsText), fileExtension)) if not self._outputFilePath: self._notif_bar.insertErrorNotification( QApplication.translate( "DocumentGeneratorDialog", "Process aborted. No output file was specified.")) return #Include extension in file name self._outputFilePath = self._outputFilePath #+ "." + fileExtension else: #Multiple files to be generated. pass self._doc_generator.set_link_field(config.link_field()) self._doc_generator.clear_attr_value_formatters() if not self.chk_template_datasource.isChecked(): #Apply cell formatters for naming output files self._doc_generator.set_attr_value_formatters(config.formatters()) entity_field_name = "id" #Iterate through the selected records progressDlg = QProgressDialog(self) progressDlg.setMaximum(len(records)) try: QApplication.setOverrideCursor(QCursor(Qt.WaitCursor)) for i, record in enumerate(records): progressDlg.setValue(i) if progressDlg.wasCanceled(): success_status = False break #User-defined location if self.chkUseOutputFolder.checkState() == Qt.Unchecked: status, msg = self._doc_generator.run( self._docTemplatePath, entity_field_name, record.id, outputMode, filePath=self._outputFilePath) self._doc_generator.clear_temporary_layers() #Output folder location using custom naming else: status, msg = self._doc_generator.run( self._docTemplatePath, entity_field_name, record.id, outputMode, dataFields=documentNamingAttrs, fileExtension=fileExtension, data_source=self.ds_entity.name) self._doc_generator.clear_temporary_layers() if not status: result = QMessageBox.warning( self, QApplication.translate("DocumentGeneratorDialog", "Document Generate Error"), msg, QMessageBox.Ignore | QMessageBox.Abort) if result == QMessageBox.Abort: progressDlg.close() success_status = False #Restore cursor QApplication.restoreOverrideCursor() return #If its the last record and user has selected to ignore if i + 1 == len(records): progressDlg.close() success_status = False #Restore cursor QApplication.restoreOverrideCursor() return else: progressDlg.setValue(len(records)) QApplication.restoreOverrideCursor() QMessageBox.information( self, QApplication.translate("DocumentGeneratorDialog", "Document Generation Complete"), QApplication.translate( "DocumentGeneratorDialog", "Document generation has successfully completed.")) except Exception as ex: LOGGER.debug(str(ex)) err_msg = sys.exc_info()[1] QApplication.restoreOverrideCursor() QMessageBox.critical( self, "STDM", QApplication.translate( "DocumentGeneratorDialog", "Error Generating documents - %s" % (err_msg))) success_status = False #Reset UI self.reset(success_status)
def search_bar(self, query): matches = [] progress = QProgressDialog("", "Abort", 0, 50000, self) progress.setWindowTitle("Searching...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setMinimumDuration(0) width = self.width() height = self.height() x = self.x() y = self.y() self.re_flags = re.UNICODE | re.MULTILINE if not self.ui.chkAdvRegex.isChecked(): query = re.escape(query) if not self.ui.chkAdvCase.isChecked(): self.re_flags |= re.IGNORECASE if self.ui.chkAdvNewline.isChecked(): self.re_flags |= re.DOTALL self.query_re = re.compile(query, self.re_flags) dir_filter = common.qt_to_unicode(self.ui.txtFilterRe.text()) if dir_filter == "": filter_re = script_analytics.DEFAULT_FILTER else: filter_re = re.compile(dir_filter, re.IGNORECASE | re.DOTALL | re.UNICODE) self.search_flags = 0 if self.ui.chkAdvTrans.isChecked(): self.search_flags |= script_analytics.SEARCH_TRANSLATED if self.ui.chkAdvOrig.isChecked(): self.search_flags |= script_analytics.SEARCH_ORIGINAL if self.ui.chkAdvComments.isChecked(): self.search_flags |= script_analytics.SEARCH_COMMENTS if self.ui.chkAdvNoTags.isChecked(): self.search_flags |= script_analytics.SEARCH_NOTAGS matches = [] for i, total, filename, partial_results in script_analytics.SA.search_gen(self.query_re, filter_re, self.search_flags): if progress.wasCanceled(): break progress.setValue(i) progress.setMaximum(total) progress.setLabelText(filename) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) matches.extend(partial_results) progress.close() return matches
def setup_workspace(self): umdimage = os.path.join(self.iso_dir, UMDIMAGE_DAT) umdimage2 = os.path.join(self.iso_dir, UMDIMAGE2_DAT) voice = os.path.join(self.iso_dir, VOICE_PAK) self.generate_directories() progress = QProgressDialog("", QtCore.QString(), 0, 11000, self) progress.setWindowTitle("Setting up workspace...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setMinimumDuration(0) progress.setValue(0) progress.setAutoClose(False) progress.setAutoReset(False) progress.setLabelText("Creating directories...") # Do the easy stuff first. if not os.path.isdir(self.changes_dir): os.makedirs(self.changes_dir) progress.setValue(progress.value() + 1) if not os.path.isdir(self.backup_dir): os.makedirs(self.backup_dir) progress.setValue(progress.value() + 1) if not os.path.isdir(self.editor_data_dir): os.makedirs(self.editor_data_dir) progress.setValue(progress.value() + 1) thread_fns = [ {"target": extract_umdimage, "kwargs": {"filename": umdimage, "out_dir": self.umdimage_dir, "eboot": self.eboot_path, "umdimage": UMDIMAGES.umdimage}}, {"target": extract_umdimage, "kwargs": {"filename": umdimage2, "out_dir": self.umdimage2_dir, "eboot": self.eboot_path, "umdimage": UMDIMAGES.umdimage2}}, {"target": extract_pak, "kwargs": {"filename": voice, "out_dir": self.voice_dir}}, ] # Going to capture stdout because I don't feel like # rewriting the extract functions to play nice with GUI. stdout = sys.stdout sys.stdout = cStringIO.StringIO() for thread_fn in thread_fns: thread = threading.Thread(**thread_fn) thread.start() while thread.isAlive(): thread.join(THREAD_TIMEOUT) output = [line for line in sys.stdout.getvalue().split('\n') if len(line) > 0] progress.setValue(progress.value() + len(output)) if len(output) > 0: progress.setLabelText("Extracting %s..." % output[-1]) sys.stdout = cStringIO.StringIO() sys.stdout = stdout # Give us an ISO directory for the editor to place modified files in. progress.setLabelText("Copying ISO files...") # ISO directory needs to not exist for copytree. if os.path.isdir(self.edited_iso_dir): shutil.rmtree(self.edited_iso_dir) # One more thing we want threaded so it doesn't lock up the GUI. thread = threading.Thread(target = shutil.copytree, kwargs = {"src": self.iso_dir, "dst": self.edited_iso_dir}) thread.start() while thread.isAlive(): thread.join(THREAD_TIMEOUT) progress.setLabelText("Copying ISO files...") # It has to increase by some amount or it won't update and the UI will lock up. progress.setValue(progress.value() + 1) # shutil.copytree(self.iso_dir, self.edited_iso_dir) progress.setValue(progress.value() + 1) # Files we want to make blank, because they're unnecessary. blank_files = [ os.path.join(self.edited_iso_dir, "PSP_GAME", "INSDIR", "UMDIMAGE.DAT"), os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "DATA.BIN"), os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "EBOOT.BIN"), os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "PARAM.SFO"), ] for blank in blank_files: with open(blank, "wb") as f: pass # Copy the decrypted EBOOT into the ISO folder and apply our hacks to it. progress.setLabelText("Hacking EBOOT...") progress.setValue(progress.value() + 1) hacked_eboot = BitStream(filename = self.eboot_path) hacked_eboot, offset = apply_eboot_patches(hacked_eboot) with open(os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN"), "wb") as f: hacked_eboot.tofile(f) # shutil.copy(self.eboot_path, os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN")) progress.setLabelText("Extracting editor data...") progress.setValue(progress.value() + 1) # Extract the editor data. editor_data = zipfile.ZipFile("data/editor_data.zip", "r") editor_data.extractall(self.editor_data_dir) editor_data.close() progress.setValue(progress.maximum()) progress.close() self.ui.grpStep4.setEnabled(False) self.ui.grpStep5.setEnabled(True)
def close(self): self._timer.stop() # For some weird reason, canceled() signal is sent upon close, whether the user canceled # or not. If we don't want a false cancellation, we have to disconnect it. self.canceled.disconnect() QProgressDialog.close(self)
def close(self): self._timer.stop() QProgressDialog.close(self)
def copy_gfx(self): gfx_dir = os.path.join(self.editor_data_dir, "gfx") if os.path.isdir(gfx_dir): shutil.rmtree(gfx_dir) os.makedirs(gfx_dir) progress = QProgressDialog("", "Abort", 0, 0, self) progress.setWindowTitle("Copying GFX...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setMinimumDuration(0) progress.setValue(0) progress.setAutoClose(False) progress.setLabelText("Setting up GFX dir.") progress.setMaximum(5) progress.setValue(0) # Extract the images we can't just take directly from the game's data. gfx_bin = zipfile.ZipFile("data/gfx_base.zip", "r") progress.setValue(1) progress.setValue(2) gfx_bin.extractall(gfx_dir) progress.setValue(5) gfx_bin.close() # We can mostly loop this. gfx_data = [ ("ammo", "kotodama_icn_???.gim"), ("bgd", "bgd_???.gim"), ("cutin", "cutin_icn_???.gim"), ("events", "gallery_icn_???.gim"), ("movies", "bin_movie_gallery_l.pak/0000/000[1789].gim"), ("movies", "bin_movie_gallery_l.pak/0000/00[123]?.gim"), ("movies", "gallery_ico_m_none.gim"), ("movies", "gallery_thumbnail_m_???.gim"), ("nametags", "tex_system.pak/00[12]?.gim"), ("nametags", "tex_system.pak/003[0123456].gim"), ("presents", "present_icn_???.gim"), ("sprites", "bustup_??_??.gim"), ("sprites", "stand_??_??.gmo"), ] for (dir, file_glob) in gfx_data: out_dir = os.path.join(gfx_dir, dir) files = glob.glob(os.path.join(self.data0_dir, file_glob)) progress.setLabelText("Copying %s." % dir) progress.setMaximum(len(files)) progress.setValue(0) if not os.path.isdir(out_dir): os.makedirs(out_dir) for i, image in enumerate(files): if i % 10 == 0: progress.setValue(i) if progress.wasCanceled(): return src = image dest = os.path.join(out_dir, os.path.basename(src)) shutil.copy(src, dest) progress.setValue(len(files)) progress.setLabelText("Copying font.") progress.setMaximum(4) progress.setValue(0) # The font we have to get from umdimage2. font_dir = os.path.join(gfx_dir, "font") if not os.path.isdir(font_dir): os.makedirs(font_dir) progress.setValue(1) # And convert to PNG with an alpha channel so our editor can use it. font1 = font_bmp_to_alpha(os.path.join(self.data01_dir, "jp", "font", "font.pak", "0000.bmp")) progress.setValue(2) font2 = font_bmp_to_alpha(os.path.join(self.data01_dir, "jp", "font", "font.pak", "0002.bmp")) progress.setValue(3) font1.save(os.path.join(font_dir, "Font01.png")) font2.save(os.path.join(font_dir, "Font02.png")) shutil.copy( os.path.join(self.data01_dir, "jp", "font", "font.pak", "0001.font"), os.path.join(font_dir, "Font01.font") ) shutil.copy( os.path.join(self.data01_dir, "jp", "font", "font.pak", "0003.font"), os.path.join(font_dir, "Font02.font") ) progress.setValue(4) # And then the flash files. This'll be fun. flash_dir = os.path.join(gfx_dir, "flash") if not os.path.isdir(flash_dir): os.makedirs(flash_dir) # Because there's so many in so many different places, I just stored a list # of the flash files we need in the gfx_base archive. So let's load that. with open(os.path.join(gfx_dir, "fla.txt"), "rb") as fla: fla_list = fla.readlines() progress.setLabelText("Copying flash.") progress.setMaximum(len(fla_list)) progress.setValue(0) for i, flash in enumerate(fla_list): if i % 10 == 0: progress.setValue(i) if progress.wasCanceled(): return flash = flash.strip() fla_name = flash[:7] # fla_### src = os.path.join(self.data01_dir, "all", "flash", flash) dest = os.path.join(flash_dir, "%s.gim" % fla_name) shutil.copy(src, dest) progress.setValue(len(fla_list)) # We have a couple sets of files that aren't named the way we want them to # be, just because of how they're stored in umdimage. progress.setLabelText("Renaming files.") to_rename = [("movies", "movie_%03d.gim", range(32)), ("nametags", "%02d.gim", range(23) + [24, 25, 30, 31])] for (folder, pattern, nums) in to_rename: folder = os.path.join(gfx_dir, folder) files = glob.glob(os.path.join(folder, "*.gim")) progress.setMaximum(len(files)) progress.setValue(0) for i, image in enumerate(files): if i % 10 == 0: progress.setValue(i) if progress.wasCanceled(): return src = image dest = os.path.join(folder, pattern % nums[i]) if os.path.isfile(dest): os.remove(dest) shutil.move(src, dest) sprite_dir = os.path.join(gfx_dir, "sprites") gmo_files = glob.glob(os.path.join(sprite_dir, "*.gmo")) progress.setLabelText("Extracting GMO files.") progress.setValue(0) progress.setMaximum(len(gmo_files)) for i, gmo_file in enumerate(gmo_files): if i % 10 == 0: progress.setValue(i) if progress.wasCanceled(): return name, ext = os.path.splitext(os.path.basename(gmo_file)) gim_file = os.path.join(sprite_dir, name + ".gim") gmo = GmoFile(filename=gmo_file) # Once we've loaded it, we're all done with it, so make it go away. os.remove(gmo_file) if gmo.gim_count() == 0: continue gim = gmo.get_gim(0) with open(gim_file, "wb") as f: gim.tofile(f) if self.ui.chkGimToPng.isChecked(): gim_files = glob.glob(os.path.join(gfx_dir, "*", "*.gim")) progress.setLabelText("Converting GIM to PNG.") progress.setValue(0) progress.setMaximum(len(gim_files)) converter = GimConverter() for i, gim_file in enumerate(gim_files): progress.setValue(i) if progress.wasCanceled(): return converter.gim_to_png(gim_file) os.remove(gim_file) progress.close() self.gfx_dir = gfx_dir self.ui.grpStep5.setEnabled(False) self.ui.grpStep6.setEnabled(True)
def import_umdimage2(src, dst, convert_png = True, propogate = True, parent = None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError("Cannot import %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Import Directory", "Importing directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Any affected files will be backed up. Proceed?", buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton = QtGui.QMessageBox.No ) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Importing...", "Cancel", 0, 0, parent) progress.setWindowTitle("Importing...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) tmp_dst = tempfile.mkdtemp(prefix = "sdse-") backup_dir = None for pak in glob.iglob(os.path.join(src, "bg_*.pak")): if progress.wasCanceled(): break pak_name = os.path.basename(pak) backup_dir = backup_files(dst, [pak_name], suffix = "_IMPORT", backup_dir = backup_dir) # If we have a regular file with the bg_*.pak name, then just drop it in. if os.path.isfile(pak): progress.setLabelText("Importing:\n" + pak_name) progress.setValue(progress.value() ^ 1) shutil.copy2(pak, os.path.join(dst, pak_name)) # Otherwise, if it's a directory, insert all the textures we find # into the target bg_*.pak file. elif os.path.isdir(pak): for image in list_all_files(pak): if progress.wasCanceled(): break ext = os.path.splitext(image)[1].lower() if ext == ".png" and not convert_png: continue base_name = image[len(src) + 1:] dst_files = [] if propogate: dupe_name = os.path.splitext(base_name)[0] + ".gim" dupe_name = os.path.join("umdimage2", dupe_name) dupe_name = os.path.normpath(os.path.normcase(dupe_name)) dupes = _DUPE_DB.files_in_same_group(dupe_name) if dupes == None: dupes = [dupe_name] for dupe in dupes: dst_file = dupe[10:] # chop off the "umdimage2/" dst_file = os.path.splitext(dst_file)[0] + ext # original extension dst_file = os.path.join(tmp_dst, dst_file) dst_files.append(dst_file) else: dst_files = [os.path.join(tmp_dst, base_name)] for dst_file in dst_files: try: os.makedirs(os.path.dirname(dst_file)) except: pass shutil.copy(image, dst_file) if progress.wasCanceled(): break progress.setLabelText("Inserting textures into:\n" + pak_name) progress.setValue(progress.value() ^ 1) pak_dir = os.path.join(tmp_dst, pak_name) pak_file = os.path.join(dst, pak_name) # If we didn't copy anything over, just move on. if not os.path.isdir(pak_dir): continue thread = threading.Thread(target = insert_textures, args = (pak_dir, pak_file)) thread.start() while thread.isAlive(): thread.join(MIN_INTERVAL) progress.setValue(progress.value() ^ 1) if progress.wasCanceled(): progress.setLabelText("Canceling...") shutil.rmtree(tmp_dst) progress.close()
stop = timeit.default_timer() total_time = stop - start print("option1 (segundos) ", total_time) """ print() start = timeit.default_timer() option1_1() stop = timeit.default_timer() total_time = stop - start print("option1_1 (segundos)", total_time) print() start = timeit.default_timer() option1_2() stop = timeit.default_timer() total_time = stop - start print("option1_2 (segundos)", total_time) """ print() start = timeit.default_timer() option2() stop = timeit.default_timer() total_time = stop - start print("option2 (segundos)", total_time) """ # option3([1,2]) bar.close()
def onGenerate(self): """ Slot raised to initiate the certificate generation process. """ self._notif_bar.clear() success_status = True config = self.current_config() self.last_data_source = config.data_source() if config is None: self._notif_bar.insertErrorNotification(QApplication.translate("DocumentGeneratorDialog", \ "The entity configuration could not be extracted.")) return #Get selected records and validate records = self.tabWidget.currentWidget().entities() if self.chk_template_datasource.isChecked(): records = self._dummy_template_records() if len(records) == 0: self._notif_bar.insertErrorNotification(QApplication.translate("DocumentGeneratorDialog", \ "Please load at least one entity record")) return if not self._docTemplatePath: self._notif_bar.insertErrorNotification(QApplication.translate("DocumentGeneratorDialog", \ "Please select a document template to use")) return documentNamingAttrs = self.lstDocNaming.selectedMappings() if self.chkUseOutputFolder.checkState() == Qt.Checked and len(documentNamingAttrs) == 0: self._notif_bar.insertErrorNotification(QApplication.translate("DocumentGeneratorDialog", \ "Please select at least one field for naming the output document")) return #Set output file properties if self.rbExpImage.isChecked(): outputMode = DocumentGenerator.Image fileExtension = self.cboImageType.currentText() saveAsText = "Image File" else: outputMode = DocumentGenerator.PDF fileExtension = "pdf" saveAsText = "PDF File" #Show save file dialog if not using output folder if self.chkUseOutputFolder.checkState() == Qt.Unchecked: docDir = source_document_location() if self._outputFilePath: fileInfo = QFileInfo(self._outputFilePath) docDir = fileInfo.dir().path() self._outputFilePath = QFileDialog.getSaveFileName(self, QApplication.translate("DocumentGeneratorDialog", "Save Document"), docDir, "{0} (*.{1})".format( QApplication.translate("DocumentGeneratorDialog", saveAsText), fileExtension)) if not self._outputFilePath: self._notif_bar.insertErrorNotification( QApplication.translate("DocumentGeneratorDialog", "Process aborted. No output file was specified.")) return #Include extension in file name self._outputFilePath = self._outputFilePath #+ "." + fileExtension else: #Multiple files to be generated. pass self._doc_generator.set_link_field(config.link_field()) self._doc_generator.clear_attr_value_formatters() if not self.chk_template_datasource.isChecked(): #Apply cell formatters for naming output files self._doc_generator.set_attr_value_formatters(config.formatters()) entity_field_name = "id" #Iterate through the selected records progressDlg = QProgressDialog(self) progressDlg.setMaximum(len(records)) try: QApplication.setOverrideCursor(QCursor(Qt.WaitCursor)) for i, record in enumerate(records): progressDlg.setValue(i) if progressDlg.wasCanceled(): success_status = False break #User-defined location if self.chkUseOutputFolder.checkState() == Qt.Unchecked: status,msg = self._doc_generator.run(self._docTemplatePath, entity_field_name, record.id, outputMode, filePath = self._outputFilePath) self._doc_generator.clear_temporary_layers() #Output folder location using custom naming else: status, msg = self._doc_generator.run(self._docTemplatePath, entity_field_name, record.id, outputMode, dataFields = documentNamingAttrs, fileExtension = fileExtension, data_source = self.ds_entity.name) self._doc_generator.clear_temporary_layers() if not status: result = QMessageBox.warning(self, QApplication.translate("DocumentGeneratorDialog", "Document Generate Error"), msg, QMessageBox.Ignore | QMessageBox.Abort) if result == QMessageBox.Abort: progressDlg.close() success_status = False #Restore cursor QApplication.restoreOverrideCursor() return #If its the last record and user has selected to ignore if i+1 == len(records): progressDlg.close() success_status = False #Restore cursor QApplication.restoreOverrideCursor() return else: progressDlg.setValue(len(records)) QApplication.restoreOverrideCursor() QMessageBox.information(self, QApplication.translate("DocumentGeneratorDialog", "Document Generation Complete"), QApplication.translate("DocumentGeneratorDialog", "Document generation has successfully completed.") ) except Exception as ex: LOGGER.debug(str(ex)) err_msg = sys.exc_info()[1] QApplication.restoreOverrideCursor() QMessageBox.critical( self, "STDM", QApplication.translate( "DocumentGeneratorDialog", "Error Generating documents - %s"%(err_msg) ) ) success_status = False #Reset UI self.reset(success_status)
def copy_gfx(self): gfx_dir = os.path.join(self.editor_data_dir, "gfx") if os.path.isdir(gfx_dir): shutil.rmtree(gfx_dir) os.makedirs(gfx_dir) progress = QProgressDialog("", "Abort", 0, 0, self) progress.setWindowTitle("Copying GFX...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setMinimumDuration(0) progress.setValue(0) progress.setAutoClose(False) progress.setLabelText("Setting up GFX dir.") progress.setMaximum(5) progress.setValue(0) # Extract the images we can't just take directly from the game's data. gfx_bin = zipfile.ZipFile("data/gfx_base.bin", "r") progress.setValue(1) gfx_enc = gfx_bin.open("gfx_base.bin") progress.setValue(2) gfx_dec = cStringIO.StringIO() base64.decode(gfx_enc, gfx_dec) progress.setValue(3) gfx_base = zipfile.ZipFile(gfx_dec, "r") progress.setValue(4) gfx_base.extractall(gfx_dir) progress.setValue(5) gfx_base.close() gfx_dec.close() gfx_enc.close() gfx_bin.close() # We can mostly loop this. gfx_data = [ ("ammo", "kotodama_icn_???.gim"), ("bgd", "bgd_???.gim"), ("cutin", "cutin_icn_???.gim"), ("events", "gallery_icn_???.gim"), ("movies", "bin_movie_gallery_l.pak/0000/000[1789].gim"), ("movies", "bin_movie_gallery_l.pak/0000/00[123]?.gim"), ("nametags", "tex_system.pak/00[12]?.gim"), ("nametags", "tex_system.pak/003[0123456].gim"), ("presents", "present_icn_???.gim"), ("sprites", "bustup_??_??.gim"), ("sprites", "stand_??_??.gmo"), ] for (dir, file_glob) in gfx_data: out_dir = os.path.join(gfx_dir, dir) files = glob.glob(os.path.join(self.umdimage_dir, file_glob)) progress.setLabelText("Copying %s." % dir) progress.setMaximum(len(files)) progress.setValue(0) if not os.path.isdir(out_dir): os.makedirs(out_dir) for i, image in enumerate(files): if i % 10 == 0: progress.setValue(i) if progress.wasCanceled(): return src = image dest = os.path.join(out_dir, os.path.basename(src)) shutil.copy(src, dest) progress.setValue(len(files)) progress.setLabelText("Copying font.") progress.setMaximum(4) progress.setValue(0) # The font we have to get from umdimage2. font_dir = os.path.join(gfx_dir, "font") if not os.path.isdir(font_dir): os.makedirs(font_dir) progress.setValue(1) # And convert to PNG with an alpha channel so our editor can use it. font1 = font_bmp_to_alpha(os.path.join(self.umdimage2_dir, "font.pak", "0000.bmp")) progress.setValue(2) font2 = font_bmp_to_alpha(os.path.join(self.umdimage2_dir, "font.pak", "0002.bmp")) progress.setValue(3) font1.save(os.path.join(font_dir, "Font01.png")) font2.save(os.path.join(font_dir, "Font02.png")) shutil.copy(os.path.join(self.umdimage2_dir, "font.pak", "0001.font"), os.path.join(font_dir, "Font01.font")) shutil.copy(os.path.join(self.umdimage2_dir, "font.pak", "0003.font"), os.path.join(font_dir, "Font02.font")) progress.setValue(4) # And then the flash files. This'll be fun. flash_dir = os.path.join(gfx_dir, "flash") if not os.path.isdir(flash_dir): os.makedirs(flash_dir) # Because there's so many in so many different places, I just stored a list # of the flash files we need in the gfx_base archive. So let's load that. with open(os.path.join(gfx_dir, "fla.txt"), "rb") as fla: fla_list = fla.readlines() progress.setLabelText("Copying flash.") progress.setMaximum(len(fla_list)) progress.setValue(0) for i, flash in enumerate(fla_list): if i % 10 == 0: progress.setValue(i) if progress.wasCanceled(): return flash = flash.strip() fla_name = flash[:7] # fla_### src = os.path.join(self.umdimage_dir, flash) dest = os.path.join(flash_dir, "%s.gim" % fla_name) shutil.copy(src, dest) progress.setValue(len(fla_list)) # We have a couple sets of files that aren't named the way we want them to # be, just because of how they're stored in umdimage. progress.setLabelText("Renaming files.") to_rename = [ ("movies", "movie_%03d.gim", range(32)), ("nametags", "%02d.gim", range(23) + [24, 25, 30, 31]), ] for (folder, pattern, nums) in to_rename: folder = os.path.join(gfx_dir, folder) files = glob.glob(os.path.join(folder, "*.gim")) progress.setMaximum(len(files)) progress.setValue(0) for i, image in enumerate(files): if i % 10 == 0: progress.setValue(i) if progress.wasCanceled(): return src = image dest = os.path.join(folder, pattern % nums[i]) if os.path.isfile(dest): os.remove(dest) shutil.move(src, dest) sprite_dir = os.path.join(gfx_dir, "sprites") gmo_files = glob.glob(os.path.join(sprite_dir, "*.gmo")) progress.setLabelText("Extracting GMO files.") progress.setValue(0) progress.setMaximum(len(gmo_files)) for i, gmo_file in enumerate(gmo_files): if i % 10 == 0: progress.setValue(i) if progress.wasCanceled(): return name, ext = os.path.splitext(os.path.basename(gmo_file)) gim_file = os.path.join(sprite_dir, name + ".gim") gmo = GmoFile(filename = gmo_file) # Once we've loaded it, we're all done with it, so make it go away. os.remove(gmo_file) if gmo.gim_count() == 0: continue gim = gmo.get_gim(0) with open(gim_file, "wb") as f: gim.tofile(f) if self.ui.chkGimToPng.isChecked(): gim_files = glob.glob(os.path.join(gfx_dir, "*", "*.gim")) progress.setLabelText("Converting GIM to PNG.") progress.setValue(0) progress.setMaximum(len(gim_files)) converter = GimConverter() for i, gim_file in enumerate(gim_files): progress.setValue(i) if progress.wasCanceled(): return converter.gim_to_png(gim_file) os.remove(gim_file) progress.close() self.gfx_dir = gfx_dir self.ui.grpStep5.setEnabled(False) self.ui.grpStep6.setEnabled(True)
def import_umdimage(src, dst, convert_png = True, propogate = True, parent = None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError("Cannot import %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Import Directory", "Importing directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Any affected files will be backed up. Proceed?", buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton = QtGui.QMessageBox.No ) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Finding files...", "Cancel", 0, 1, parent) progress.setWindowTitle("Importing...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) if parent: width = parent.width() height = parent.height() x = parent.x() y = parent.y() else: width = 1920 height = 1080 x = 0 y = 0 progress.setMaximum(0) progress.setValue(0) # The raw list of files we're importing. files = [] # A list of lists, including all dupes of the files being imported, too. affected_files = [] file_count = 0 dupe_base = "umdimage" tmp = tempfile.mkdtemp(prefix = "sdse-") seen_groups = [] count = 0 last_update = time.time() for file in list_all_files(src): if progress.wasCanceled(): break # Strip our base directory, so we have just a relative file list. file = os.path.normpath(os.path.normcase(file[len(src) + 1:])) files.append(file) count += 1 if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Finding files...\n" + file) # progress.setValue(count) progress.setValue(progress.value() ^ 1) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) affected_files.append([]) if os.path.splitext(file)[1] == ".png" and convert_png and file not in SKIP_CONV: file = os.path.splitext(file)[0] + ".gim" if propogate: file_group = _DUPE_DB.group_from_file(os.path.join(dupe_base, file)) else: file_group = None if file_group in seen_groups: continue # If there are no dupes, just add this file. if file_group == None: affected_files[-1].append(file) file_count += 1 continue seen_groups.append(file_group) for dupe in _DUPE_DB.files_in_group(file_group): # Minus the "umdimage" part dupe = dupe[len(dupe_base) + 1:] affected_files[-1].append(dupe) file_count += 1 progress.setValue(0) progress.setMaximum(file_count) # Make a backup first. backup_dir = None count = 0 for file_set in affected_files: if progress.wasCanceled(): break for file in file_set: if progress.wasCanceled(): break count += 1 if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Backing up...\n" + file) progress.setValue(count) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) # It's perfectly possible we want to import some files that # don't already exist. Such as when importing a directory # with added lines. if not os.path.isfile(os.path.join(dst, file)): continue backup_dir = backup_files(dst, [file], suffix = "_IMPORT", backup_dir = backup_dir) progress.setValue(0) # And now do our importing. import_all_new = False skip_all_new = False count = 0 for index, src_file in enumerate(files): if progress.wasCanceled(): break if os.path.splitext(src_file)[1] == ".png" and convert_png and src_file not in SKIP_CONV: tmp_src_file = os.path.join(tmp, os.path.basename(src_file)) tmp_src_file = os.path.splitext(tmp_src_file)[0] + ".gim" quantize = QuantizeType.auto for regex, q in FORCE_QUANTIZE: if not regex.search(src_file) == None: quantize = q break _CONV.png_to_gim(os.path.join(src, src_file), tmp_src_file, quantize) src_file = tmp_src_file else: src_file = os.path.join(src, src_file) for file in affected_files[index]: if progress.wasCanceled(): break dst_file = os.path.join(dst, file) count += 1 # if count % 25 == 0: if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Importing...\n" + file) progress.setValue(count) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) # We may be allowed to import files that don't exist, but we're # going to ask them about it anyway. if not os.path.isfile(dst_file): if skip_all_new: continue if not import_all_new: answer = QtGui.QMessageBox.question( parent, "File Not Found", "File:\n\n" + file + "\n\n" + "does not exist in the target directory. Import anyway?", buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.YesToAll | QtGui.QMessageBox.No | QtGui.QMessageBox.NoToAll, defaultButton = QtGui.QMessageBox.No ) if answer == QtGui.QMessageBox.YesToAll: import_all_new = True skip_all_new = False elif answer == QtGui.QMessageBox.NoToAll: skip_all_new = True import_all_new = False continue elif answer == QtGui.QMessageBox.No: continue basedir = os.path.dirname(dst_file) if not os.path.isdir(basedir): os.makedirs(basedir) shutil.copy2(src_file, dst_file) shutil.rmtree(tmp) progress.close()
def import_umdimage2(src, dst, convert_png=True, propogate=True, parent=None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError( "Cannot import %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Import Directory", "Importing directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Any affected files will be backed up. Proceed?", buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton=QtGui.QMessageBox.No) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Importing...", "Cancel", 0, 0, parent) progress.setWindowTitle("Importing...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) tmp_dst = tempfile.mkdtemp(prefix="sdse-") backup_dir = None for pak in glob.iglob(os.path.join(src, "bg_*.pak")): if progress.wasCanceled(): break pak_name = os.path.basename(pak) backup_dir = backup_files(dst, [pak_name], suffix="_IMPORT", backup_dir=backup_dir) # If we have a regular file with the bg_*.pak name, then just drop it in. if os.path.isfile(pak): progress.setLabelText("Importing:\n" + pak_name) progress.setValue(progress.value() ^ 1) shutil.copy2(pak, os.path.join(dst, pak_name)) # Otherwise, if it's a directory, insert all the textures we find # into the target bg_*.pak file. elif os.path.isdir(pak): for image in list_all_files(pak): if progress.wasCanceled(): break ext = os.path.splitext(image)[1].lower() if ext == ".png" and not convert_png: continue base_name = image[len(src) + 1:] dst_files = [] if propogate: dupe_name = os.path.splitext(base_name)[0] + ".gim" dupe_name = os.path.join("umdimage2", dupe_name) dupe_name = os.path.normpath(os.path.normcase(dupe_name)) dupes = _DUPE_DB.files_in_same_group(dupe_name) if dupes == None: dupes = [dupe_name] for dupe in dupes: dst_file = dupe[10:] # chop off the "umdimage2/" dst_file = os.path.splitext( dst_file)[0] + ext # original extension dst_file = os.path.join(tmp_dst, dst_file) dst_files.append(dst_file) else: dst_files = [os.path.join(tmp_dst, base_name)] for dst_file in dst_files: try: os.makedirs(os.path.dirname(dst_file)) except: pass shutil.copy(image, dst_file) if progress.wasCanceled(): break progress.setLabelText("Inserting textures into:\n" + pak_name) progress.setValue(progress.value() ^ 1) pak_dir = os.path.join(tmp_dst, pak_name) pak_file = os.path.join(dst, pak_name) # If we didn't copy anything over, just move on. if not os.path.isdir(pak_dir): continue thread = threading.Thread(target=insert_textures, args=(pak_dir, pak_file)) thread.start() while thread.isAlive(): thread.join(MIN_INTERVAL) progress.setValue(progress.value() ^ 1) if progress.wasCanceled(): progress.setLabelText("Canceling...") shutil.rmtree(tmp_dst) progress.close()
class ExportValidationReport: """ Control the overall process of exporting validation reports to text file from the validation report form on the admin menu """ def __init__(self, report_title, org_name, db, iface, file_path=None): self.report_title = report_title.upper() self.db = db self.iface = iface self.org_name = org_name.upper() self.user = "******" self.file_path = file_path self.report_file = None self.start_point = QgsPoint() self.end_point = QgsPoint() self.esu_layer = QgsMapLayerRegistry.instance().mapLayersByName( 'ESU Graphic')[0] self.poly_layer = QgsMapLayerRegistry.instance().mapLayersByName( 'Road Polygons')[0] self.filter = None self.queries = {} self.headers = {} self.headers_no_items = {} self.column_names = {} self.init_headers() self.init_queries() self.validation_dia = None self.progress_win = None self.progress_win = QProgressDialog("", None, 0, 13, self.validation_dia) self.progress_win.setFixedSize(380, 100) self.progress_win.setModal(True) self.progress_win.setWindowFlags(Qt.CustomizeWindowHint | Qt.WindowTitleHint) self.progress_win.setWindowTitle("Export Validation Report") def init_headers(self): # initialises the text of headers and column names to use in the report self.headers = {0: 'Duplicate street descriptions :', 1: 'Streets not linked to ESUs :', 2: ['ESUs linked to Public Type 1 or 2 Streets but not to Type 3 :', '(Including Footpaths)', 'NOTE : Some of these esus may be linked to private parts ' 'of streets that are part public and part private'], 3: ['Duplicate ESU References : ', 'The following ESUs have duplicate ESU references. ' 'It is important that these are unique values \n' \ 'These references are based on the esu midpoint and are used when exporting to DTF'], 4: 'ESUs not linked to Streets :', 5: 'ESUs incorrectly linked to Type 1 and Type 2 Streets :', 6: 'ESUs linked to Type 3 or 4 Streets but not to Type 1 or 2 :', 7: 'ESUs linked to Unofficial street types (ie. not 1,2,3 or 4) :', 8: 'Streets with Start / End coordinates more than {0}m from ESUs :', 9: 'IDs of tiny {0}s :', 10: 'IDs of emtpy geometries :', 11: 'Type 1/2 Streets not linked to Maintenance Records :', 12: 'Type 1/2 Streets not linked to Reinstatement Records :', 13: 'Maintenance Records With Invalid Start-End Co-ordinates', 14: 'Reinstatement Records With Invalid Start-End Co-ordinates', 15: 'Special Designation Records With Invalid Start-End Co-ordinates', 16: 'Maintenance Records not linked to any Polygons :', 17: 'Polygons not linked to a Maintenance Record :', 18: 'Polygons wrongly assigned to more than one Maintenance record :'} self.headers_no_items = { 0: 'No duplicate street descriptions.', 1: 'No Unreferenced Streets found.', 2: 'No ESUs linked to Public Type 1 or 2 Streets but not to Type 3.', 3: 'No Duplicate ESU References found.', 4: 'No Unreferenced ESUs found.', 5: 'No ESUs incorrectly linked to Type 1 & 2 Streets.', 6: 'No ESUs incorrectly linked to Type 3 or 4 Streets but not 1 or 2.', 7: 'No ESUs incorrectly linked to Unofficial street types.', 8: 'No problems found.', 9: 'No tiny {0}s found.', 10: 'No invalid geometries found.', 11: 'No Type 1/2 Streets Without Maintenance Records.', 12: 'No Type 1/2 Streets Without Reinstatement Records.', 13: 'No Maintenance Records With Invalid Start-End Co-ordinates.', 14: 'No Reinstatement Records With Invalid Start-End Co-ordinates.', 15: 'No Special Designation Records With Invalid Start-End Co-ordinates.', 16: 'No Maintenance Records without polygons found.', 17: 'No polygons without Maintenance Records found.', 18: 'No Polygons wrongly assigned to more than one Maintenance record.' } self.column_names = { 0: ['Description, Locality, Town'], 1: ['USRN, Version, Type, Description, Update Date, Updated By'], 2: ['ESU ID, Street Reference Type, USRN, Street Description'], 3: ['ESU ID, ESU Reference (External)'], 4: ['ESU ID'], 5: ['USRN, Start/End, Type, Description'], 6: ['USRN, Type, Description'], 7: ['USRN, ASD ID, Reference Number'], 8: ['USRN, Maintenance ID, Status'], 9: ['Polygon ID'], 10: ['Polygon ID, USRN, Maintenance ID'] } def write_content(self, query_id, header_id, header_no_content_id, columns_name_id, include_footpaths=True, include_subtitle=False): """ format the content of the data coming from the db either in text or dialog format :param query_id: int reference to the query dictionary :param header_id: int reference to the header dictionary :param header_no_content_id: int reference to the header no content dictionary :param columns_name_id: int reference to the list of columns of the required table :param include_footpaths: bool value to include footpaths in type 3 streets query :param include_subtitle: bool indicates if the header has a subtitle :return: void """ # build a query model object query_model = QSqlQueryModel() if not include_footpaths: filtered_query = self.queries[2].replace( "AND (lnkESU_STREET.Currency_flag = 0)", "AND (lnkESU_STREET.Currency_flag = 0) AND " "(tblStreet.Description not like '%Footpath%')") query_model.setQuery(filtered_query) else: query_model.setQuery(self.queries[query_id]) while query_model.canFetchMore(): query_model.fetchMore() parent_model_index = QModelIndex() # if the path is not specified sends data to function creating a list if self.file_path is None: assert isinstance(columns_name_id, object) items_list = self.content_to_screen( content_list=None, query_model=query_model, columns_name_id=columns_name_id, no_content_id=header_no_content_id) return items_list def start_report(self): self.progress_win.show() self.progress_win.setLabelText("Starting report") self.progress_win.setValue(1) def dup_street_desc(self): """ duplicate street description report section creation :return: void if the report is to text list[string] if the report is to screen """ # self.progress_win.show() self.progress_win.setLabelText( "Checking duplicate street descriptions...") items_list = self.write_content(0, 0, 0, 0) self.progress_win.setValue(1) return items_list def street_not_esu_desc(self): """ streets not linked to ESU report section creation :return: void if the report is to text list[string] if the report is to screen """ self.progress_win.setLabelText( "Checking streets not linked to ESUs...") items_list = self.write_content(1, 1, 1, 1) self.progress_win.setValue(2) return items_list def no_type3_desc(self, include_footpath): """ streets not connected to type 3 report section :param include_footpath: bool, include or not footpath in the query :return: void if the report is to text list[string] if the report is to screen """ if include_footpath: self.progress_win.setLabelText( "Checking ESUs linked to Public Type 1 or 2 Streets but not to Type 3 " "(Including Footpaths)...") items_list = self.write_content(2, 2, 2, 2) self.progress_win.setValue(3) return items_list else: self.progress_win.setLabelText( "Checking ESUs linked to Public Type 1 or 2 Streets but not to Type 3..." ) items_list = self.write_content(2, 2, 2, 2, False) self.progress_win.setValue(3) return items_list def dup_esu_ref(self): """ duplicate ESU references report section :param include_subtitle: include or not a header subtitle :return: void if the report is to text list[string] if the report is to screen, the subtitile is included by default """ self.progress_win.setLabelText("Checking Duplicate ESU References...") items_list = self.write_content(3, 3, 3, 3) self.progress_win.setValue(4) return items_list def no_link_esu_streets(self): """ section report on ESUs not linked to any street :return:void if the report is to text list[string] if the report is to screen, the subtitile is not included by default """ self.progress_win.setLabelText( "Checking ESUs not linked to Streets...") items_list = self.write_content(4, 4, 4, 4) self.progress_win.setValue(5) return items_list def invalid_cross_references(self): """ this function handles three checks that are grouped under the same checkbox in the form 'invalid cross references' first check: ESUs not linked to type 1 and 2 streets second check: ESUs linked to type 3 and 4 streets but not linked to type 1 or 2 third check: ESUs linked to unofficial street types (none of the previous type) :return void if the report is to text list[[string]x3] if the report is to screen """ self.progress_win.setLabelText("Checking Invalid Cross-References...") results_list = [] esu12_list = self.write_content(5, 5, 5, 4) esu34_list = self.write_content(6, 6, 6, 4) unoff_list = self.write_content(7, 7, 7, 4) results_list.append(esu12_list) results_list.append(esu34_list) results_list.append(unoff_list) self.progress_win.setValue(6) return results_list def check_start_end(self, tol): # set the feature counter to 0 count = 0 # initialises two virtual objects points (start and end point) start_point = self.start_point end_point = self.end_point # uses the qgis python api to access the ESU Graphic Layer esu_layer = self.esu_layer # runs the query number 8 to retrieve all streets from the Database streets_model = QSqlQueryModel() streets_model.setQuery(self.queries[8]) while streets_model.canFetchMore(): streets_model.fetchMore() n_columns = streets_model.columnCount() n_rows = streets_model.rowCount() i = 0 j = 0 # first loop start (for each street): start_end_content = [] while i <= n_rows - 1: self.progress_win.setLabelText("Checking start-end Coordinates...") # initialises the state of the checks booleans both to false start_ok = False end_ok = False col_info = [] while j <= n_columns - 1: model_index = streets_model.createIndex(i, j) if j == 0: data = model_index.data() col_info.append(data) if j == 1: data = model_index.data() col_info.append(data) if j == 2: data = model_index.data() col_info.append(data) if j >= 3: data = model_index.data() col_info.append(data) j += 1 usrn = col_info[0] ref_type = col_info[2] desc = col_info[1] start_point.set(float(col_info[3]), float(col_info[4])) end_point.set(float(col_info[5]), float(col_info[6])) # filter the layer "ESU Graphic" for the ESUs Ids returned from the list # deal just with the arcs part of multi arcs street esus_list = self.get_linked_esu_list(usrn) feat_filter = self.build_layer_filter(esus_list) feat_request = QgsFeatureRequest() feat_request.setFilterExpression(feat_filter) # second loop starts (for each arc (ESU) composing the street) # iterate through all filtered features and their proximity with the start and end of the street features = self.esu_layer.getFeatures(feat_request) features.rewind() # iterates through features for feat in features: # check start end points for each of the only if none of the start end points of # a ESU on each street is not already matched if (start_ok is not True) or (end_ok is not True): result = self.start_end_proximity(start_point, end_point, feat, tol) # both dist are ok if result == 3: start_ok = True end_ok = True # just end dist is ok elif result == 2: end_ok = True # just start dist is ok elif result == 1: start_ok = True else: break # in case of problems if not start_ok or not end_ok: count += 1 start_end_item = [str(col_info[0]) + ","] # handles the creation of the report on a text file if not start_ok and not end_ok: start_end_item.append("(both),") if not start_ok and end_ok: start_end_item.append("(start),") if start_ok and not end_ok: start_end_item.append("(end),") start_end_item.append(str(ref_type) + ",") start_end_item.append(str(desc) + "\n") start_end_content.append(start_end_item) j = 0 i += 1 if count == 0: self.progress_win.setValue(7) return else: start_end_content.insert(0, self.column_names[5]) self.progress_win.setValue(7) return self.content_to_screen(content_list=start_end_content, query_model=None, columns_name_id=None, no_content_id=8) def start_end_proximity(self, start_point_street, end_point_street, feature, tolerance): """ check distance of start and end point of the street with each ESU if it is greater or smaller than the set tolerance :param start_point: start point of each street to which the ESU is linked :param end_point: endpoint of each street to which the ESU is linked :param feature: the ESU to test :param tolerance: distance in metres expressed by the user, over which we flag up a problem :return int result: integer that expresses if the distance of the start end point of the ESU from the tart end point of the street is greater or lower than the set tolerance """ result = 0 geom = feature.geometry() multi_poly_list = geom.asMultiPolyline() # if the geometry is not empty check start end coords if len(multi_poly_list) > 0: multi_poly = geom.asMultiPolyline()[0] len_list = len(multi_poly) start_point_esu = multi_poly[0] end_point_esu = multi_poly[len_list - 1] # test 1: distance between start vertex of ESU and start point of street dist_start_1 = sqrt(start_point_esu.sqrDist(start_point_street)) # test 2: distance between start vertex of ESU and end point of street dist_end_1 = sqrt(start_point_esu.sqrDist(end_point_street)) # test 3: distance between end vertex of ESU and start point of street dist_start_2 = sqrt(end_point_esu.sqrDist(start_point_street)) # test 4: distance between end vertex of ESU and end point of street dist_end_2 = sqrt(end_point_esu.sqrDist(end_point_street)) # start tolerances evaluation if dist_start_1 < tolerance or dist_start_2 < tolerance: result = 1 # end tolerances evaluation if dist_end_1 < tolerance or dist_end_2 < tolerance: result += 2 return result def get_linked_esu_list(self, usrn): """ function that selects all esus for a determined street :param usrn: the unique identifier of a certain street :return: list[esu_ids] all esu ids linked to a certain street or void in case a street does not have any linked esu """ # executing the query esus_query_model = QSqlQueryModel() esus_query_model.setQuery(self.queries[9].format(usrn)) while esus_query_model.canFetchMore(): esus_query_model.fetchMore() n_rows = esus_query_model.rowCount() # skip if no esus are linked if n_rows == 0: return else: i = 0 esus_list = [] # creating a list of ESUs Ids that are linked to the street while i <= n_rows - 1: model_index = esus_query_model.createIndex(i, 0) esu = model_index.data() esus_list.append(esu) i += 1 return esus_list def build_layer_filter(self, esus_list): """ builds a qgis layer expression from a list of strings to return filtered features from a layer :param esus_list: list of strings containing all ESUs in a street :return: string filter to apply to feature request """ str_esu = "" i = 0 while i <= len(esus_list) - 1: if i == 0 or i == len(esus_list): str_esu += '"esu_id" = ' + str(esus_list[i]) + ' ' else: str_esu += 'OR "esu_id" = ' + str(esus_list[i]) + ' ' i += 1 return str_esu def check_tiny_esus(self, type, tolerance): """ function that checks for empty geometries and for features smaller than a set tolerance dimension expressed in metres :param type: string, indicates which layer is to check ("ESUS or polygons") :param tolerance: int, the tolerance :return: """ check_layer = None field_name = None check_geom = None check_geom_dim = None empty_geoms = [] tiny_shapes = [] if type == "esu": # self.progress_win.setLabelText("Checking tiny and empty ESUs geometries...") check_layer = self.esu_layer field_name = "esu_id" # self.progress_win.setValue(8) if type == "rd_poly": # self.progress_win.setLabelText("Checking tiny and empty polygons geometries...") check_layer = self.poly_layer field_name = "rd_pol_id" # self.progress_win.setValue(13) if not check_layer: no_layer_msg_box = QMessageBox( QMessageBox.Warning, " ", "Cannot retrieve {} Layer".format(type), QMessageBox.Ok, None) no_layer_msg_box.setWindowFlags(Qt.CustomizeWindowHint | Qt.WindowTitleHint) no_layer_msg_box.exec_() return else: # checks the field index exists fields = check_layer.pendingFields().toList() field_names_list = [] for field in fields: field_names_list.append(field.name()) if field_name not in field_names_list: no_field_msg_box = QMessageBox( QMessageBox.Warning, " ", "Cannot find field named {}".format(field_name), QMessageBox.Ok, None) no_field_msg_box.setWindowFlags(Qt.CustomizeWindowHint | Qt.WindowTitleHint) no_field_msg_box.exec_() return else: # loop through all features in the layer check_features = check_layer.getFeatures() check_features.rewind() # checks for empty geometries for check_feature in check_features: check_geom = check_feature.geometry() check_geom_id = check_feature[field_name] if check_geom is None: empty_geoms.append(check_geom_id) elif not check_geom.isGeosValid(): empty_geoms.append(check_geom_id) # check for feature dimensions smaller than tolerance else: if type == 'esu': check_geom_dim = check_geom.length() if type == 'rd_poly': check_geom_dim = check_geom.area() if check_geom_dim < tolerance: tiny_shapes.append(check_geom_id) tiny_content = [] empty_content = [] check_list = [] if type == "esu": tiny_content.append(self.column_names[4]) empty_content.append(self.column_names[4]) if type == "rd_poly": tiny_content.append(self.column_names[9]) empty_content.append(self.column_names[9]) # if there are any problems with tiny shapes if len(tiny_shapes) > 0: for shape in tiny_shapes: tiny_item = [str(shape)] tiny_content.append(tiny_item) check_list.append(tiny_content) if len(tiny_shapes) == 0: last_item = [] tiny_content.append(last_item) last_item.append(str(self.headers_no_items[9].format(type))) # return tiny_content check_list.append(tiny_content) # if there are any problems with empty geometries if len(empty_geoms) > 0: for empty in empty_geoms: empty_item = [str(empty)] empty_content.append(empty_item) check_list.append(empty_content) if len(empty_geoms) == 0: last_item = [] empty_content.append(last_item) last_item.append(str(self.headers_no_items[10].format(type))) check_list.append(empty_content) return check_list def check_maint_reinst(self): """ groups two checks for streets type 1 and 2, checks streets that are not linked to maintenance records and check streets that are not linked to reinstatement records :return: void if the report is to text list[[string]x2] if the report is to screen """ # self.progress_win.setLabelText("Checking maintenance records for streets...") if self.file_path is None: results_list = [ self.write_content(10, 11, 11, 6), self.write_content(11, 12, 12, 6) ] self.progress_win.setValue(9) return results_list def check_asd_coords(self): """ groups three checks on start and end coordinates for streets classified as maintenance, reinstatement and special designation :return: void if the report is to text list[[string]x3] if the report is to screen """ self.progress_win.setLabelText("Checking streets asd coords...") results_list = [ self.write_content(12, 13, 13, 7), self.write_content(13, 14, 14, 7), self.write_content(14, 15, 15, 7) ] self.progress_win.setValue(10) return results_list def maint_no_poly(self): # function that checks maintenance record that have no link to # road polygons self.progress_win.setLabelText( "Checking maintenance records for polygons...") result_list = self.write_content(15, 16, 16, 8) self.progress_win.setValue(11) return result_list def poly_no_maint(self): """ group of two checks on road polygons 1: polygons that have not link to maintenance records 2: polygons wrongly assigned to more than one maintenance record :return: void """ results_list = [ self.write_content(16, 17, 17, 9), self.write_content(17, 18, 18, 10) ] self.progress_win.setValue(12) return results_list def end_report(self, parent): self.progress_win.close() parent.close() def content_to_screen(self, content_list=None, query_model=None, columns_name_id=None, no_content_id=None): """ handles the creation of an on-screen validation report version, the function handles the case of a report created both from db records and mixed data from db and spatial features :param content_list: list[string] 'ready made' list of values to print on the screen if data comes from mixed sources (db + features) :param query_model: QtSqlQueryModel model of the query if all data comes from db :param columns_name_id: int index of the column names dictionary to print column names on tables """ if content_list is None: parent_model_index = QModelIndex() # get number of rows and columns n_rows = query_model.rowCount(parent_model_index) n_columns = query_model.columnCount(parent_model_index) # if the path is not specified build the list view # creates a list of values content_list = [] items_list = [] i = 0 j = 0 k = 0 while k <= len(self.column_names[columns_name_id]) - 1: content_list.append(self.column_names[columns_name_id]) k += 1 # if there are no problems just print a message # on the first item in the list if n_rows < 1: content_list.append([self.headers_no_items[no_content_id]]) else: # identify data in the model and write to txt file while i <= n_rows - 1: while j <= n_columns - 1: model_index = query_model.createIndex(i, j) data = str(model_index.data()) items_list.append(data) j += 1 content_list.append(items_list) j = 0 items_list = [] i += 1 return content_list def init_queries(self): """ database queries initialisation""" self.queries = { 0: "SELECT (ifnull(tblSTREET.Description, '') ||'|'|| ifnull(tlkpLOCALITY.Name, '')||'|'|| " "ifnull(tlkpTOWN.Name,'')) AS DuplicateRoads FROM (tblSTREET INNER JOIN tlkpLOCALITY " "ON tblSTREET.Loc_Ref = tlkpLOCALITY.Loc_Ref) INNER JOIN tlkpTOWN " "ON tblSTREET.Town_Ref = tlkpTOWN.Town_Ref GROUP BY tblSTREET.Currency_flag, " "tblSTREET.Description, tlkpLOCALITY.Name, tlkpTOWN.Name HAVING (((Count(tblSTREET.USRN))>1) " "AND ((tblSTREET.Currency_flag)=0))", 1: "SELECT (ifnull(tblSTREET.USRN, '') ||'|'|| ifnull(tblSTREET.Version_No, '') ||'|'|| " "ifnull(tblSTREET.Street_ref_type, '') ||'|'|| ifnull(tblSTREET.Description, '')||'|'|| " "ifnull(tblSTREET.Update_date, '') ||'|'|| ifnull(tblSTREET.Updated_by, '')) AS not_linked_usrn " "FROM tblSTREET LEFT JOIN (SELECT * from lnkESU_STREET where lnkESU_Street.Currency_Flag = 0) " "AS STREET_LINK ON (tblSTREET.Version_No = STREET_LINK.usrn_version_no) " "AND (tblSTREET.USRN = STREET_LINK.usrn) WHERE (((tblSTREET.Currency_flag) = 0) " "AND ((STREET_LINK.usrn) Is Null)) ORDER BY tblSTREET.USRN ", 2: "SELECT (ifnull(q12.esu_id,'') ||'|'|| ifnull(q12.street_ref_type,'') ||'|'|| " "ifnull(q12.USRN,'') ||'|'|| ifnull(q12.Description,'')) AS type3_not_linked " "FROM (SELECT DISTINCT lnkESU_STREET.esu_id, " "tblSTREET.Street_ref_type, tblStreet.Description,tblStreet.USRN " "FROM (lnkESU_STREET INNER JOIN tblSTREET ON (lnkESU_STREET.usrn_version_no = tblSTREET.Version_No) " "AND (lnkESU_STREET.usrn = tblSTREET.USRN)) INNER JOIN tblMAINT ON (tblSTREET.USRN = tblMAINT.USRN) " "WHERE (((tblSTREET.street_ref_type = 1) Or (tblSTREET.street_ref_type = 2)) AND " "(lnkESU_STREET.Currency_flag = 0) And (tblSTREET.Currency_flag = 0) AND " "(tblMAINT.Road_Status_Ref = 1)) ORDER BY lnkESU_STREET.esu_id) AS q12 LEFT JOIN " "(SELECT lnkESU_STREET.esu_id, tblSTREET.Street_ref_type FROM lnkESU_STREET INNER JOIN tblSTREET " "ON (lnkESU_STREET.usrn_version_no = tblSTREET.Version_No) AND (lnkESU_STREET.usrn = tblSTREET.USRN) " "WHERE ((tblSTREET.street_ref_type = 3) " "AND (lnkESU_STREET.Currency_flag = 0) And (tblSTREET.Currency_flag = 0)) " "ORDER BY lnkESU_STREET.esu_id) AS q3 ON q12.esu_id = q3.esu_id WHERE (q3.esu_id Is Null)", 3: "SELECT (ifnull(qryID.esu_id,'') ||'|'|| ifnull(qryID.esuxy,'')) as dup_esu_ref FROM " "(SELECT Count(tblESU.esu_id) AS CountOfesu_id, " "((substr('0000000'|| xref, -7,7) || substr('0000000'||yref, -7,7))) AS esuxy " " FROM tblESU WHERE tblESU.currency_flag=0 GROUP BY tblESU.currency_flag, " "(substr('0000000'|| xref, -7, 7) || substr('0000000'||yref, -7,7))) qryCntID " " INNER JOIN (SELECT tblESU.esu_id, (substr('0000000'|| xref, -7, 7) || substr('0000000'||yref, -7,7))" " AS esuxy From tblESU WHERE (tblESU.currency_flag=0)) qryID " "ON qryCntID.esuxy = qryID.esuxy WHERE (qryCntID.CountOfesu_id>1)", 4: " SELECT tblESU.esu_id AS esu_not_linked " "FROM tblESU LEFT JOIN " "((SELECT lnkESU_STREET.* From lnkESU_STREET WHERE (((lnkESU_STREET.currency_flag)=0)))) " "AS STREET_LINK ON (tblESU.version_no = STREET_LINK.esu_version_no) " "AND (tblESU.esu_id = STREET_LINK.esu_id) " "WHERE (((tblESU.currency_flag)=0) AND ((STREET_LINK.usrn) Is Null))", 5: "SELECT * FROM (SELECT DISTINCT esu_id, count(USRN) AS USRN_Count " "FROM (SELECT lnkESU_Street.esu_id,tblStreet.USRN " "FROM tblSTREET INNER JOIN lnkESU_STREET ON (tblSTREET.USRN = lnkESU_STREET.usrn) " "AND (tblSTREET.Version_No = lnkESU_STREET.usrn_version_no) " "WHERE (((tblSTREET.Street_ref_type)=1 Or (tblSTREET.Street_ref_type)=2) " "AND ((tblSTREET.Currency_flag)=0) AND ((lnkESU_STREET.currency_flag)=0))) AS sq " "GROUP BY esu_id) WHERE USRN_Count > 1", 6: "SELECT q34.esu_id as ESUID " "FROM (SELECT lnkESU_STREET.esu_id, tblSTREET.Street_ref_type " "FROM lnkESU_STREET INNER JOIN tblSTREET ON (lnkESU_STREET.usrn_version_no = tblSTREET.Version_No) " "AND (lnkESU_STREET.usrn = tblSTREET.USRN) " "WHERE (((tblSTREET.street_ref_type) = 3 OR (tblSTREET.street_ref_type) = 4) " "AND ((lnkESU_STREET.Currency_flag) = 0) And ((tblSTREET.Currency_flag) = 0)) " "ORDER BY lnkESU_STREET.esu_id) as q34 " "LEFT JOIN (SELECT lnkESU_STREET.esu_id, tblSTREET.Street_ref_type " "FROM lnkESU_STREET INNER JOIN tblSTREET ON (lnkESU_STREET.usrn_version_no = tblSTREET.Version_No) " "AND (lnkESU_STREET.usrn = tblSTREET.USRN) " "WHERE (((tblSTREET.street_ref_type) = 1 Or (tblSTREET.street_ref_type) = 2) " "AND ((lnkESU_STREET.Currency_flag) = 0) AND ((tblSTREET.Currency_flag) = 0)) " "ORDER BY lnkESU_STREET.esu_id) as q12 ON q34.esu_id = q12.esu_id WHERE (((q12.esu_id) Is Null))", 7: "SELECT lnkESU_STREET.esu_id as ESUID, tblSTREET.USRN as USRN, tblSTREET.Street_ref_type as REFTYPE " "FROM lnkESU_STREET INNER JOIN tblSTREET ON (lnkESU_STREET.usrn_version_no = tblSTREET.Version_No) " "AND (lnkESU_STREET.usrn = tblSTREET.USRN) " "WHERE (((tblSTREET.Street_ref_type) Not In (1,2,3,4)) " "AND ((lnkESU_STREET.currency_flag)=0) " "AND ((tblSTREET.Currency_flag)=0)) " "ORDER BY lnkESU_STREET.esu_id", 8: "SELECT DISTINCT tblStreet.USRN, tblStreet.Description, " "tblStreet.Street_ref_type, ifnull(tblStreet.Start_xref, 0), " "ifnull(tblStreet.Start_yref, 0), ifnull(tblStreet.End_xref, 0), ifnull(tblStreet.End_yref, 0) " "FROM tblStreet LEFT JOIN lnkESU_STREET ON (tblStreet.USRN = lnkESU_STREET.usrn) " "AND (tblStreet.Version_No = lnkESU_STREET.usrn_version_no) " "WHERE (((lnkESU_STREET.usrn) Is Not Null) AND ((tblStreet.Currency_flag)=0) " "AND ((lnkESU_STREET.currency_flag)=0)) " "ORDER BY tblStreet.USRN;", 9: "SELECT lnkESU_STREET.esu_id " "From lnkESU_STREET " " WHERE (((lnkESU_STREET.usrn)= {0}) " " AND ((lnkESU_STREET.currency_flag)=0))", 10: "SELECT (ifnull(tblSTREET.USRN,'') ||'|'|| ifnull(tblSTREET.Street_ref_type,'') ||'|'|| " "ifnull(tblSTREET.Description,'')) AS type12_maint " "FROM tblSTREET LEFT JOIN (SELECT tblMAINT.USRN, tblMAINT.Maint_id " "FROM tblMAINT WHERE (((tblMAINT.Currency_flag)=0))) as maint ON tblSTREET.USRN = maint.USRN " "WHERE (((tblSTREET.street_ref_type) = 1 Or (tblSTREET.street_ref_type) = 2) " "AND ((tblSTREET.Currency_flag) = 0) And ((maint.maint_id) Is Null)) " "ORDER BY tblSTREET.USRN", 11: "SELECT (ifnull(tblSTREET.USRN,'') ||'|'|| ifnull(tblSTREET.Street_ref_type,'') " "||'|'|| ifnull(tblSTREET.Description,'')) AS check_reins " "FROM tblSTREET LEFT JOIN (SELECT tblReins_Cat.USRN, tblReins_Cat.Reins_cat_id FROM tblReins_Cat " "WHERE (((tblReins_Cat.Currency_flag)=0))) as reins ON tblSTREET.USRN = reins.USRN " "WHERE (((tblSTREET.street_ref_type) = 1 Or (tblSTREET.street_ref_type) = 2) " "AND ((tblSTREET.Currency_flag) = 0) And ((reins.Reins_cat_id) Is Null)) " "AND tblStreet.USRN not in (Select distinct s.USRN " "FROM tblStreet s inner join tblMaint m on m.USRN = s.USRN WHERE s.Currency_flag = 0 " "AND m.currency_flag = 0 and m.road_status_ref = 4) " "ORDER BY tblSTREET.USRN", 12: "SELECT (ifnull(S.USRN,'') ||'|'||ifnull(ASD.Maint_ID,'') ||'|'|| ifnull(ASD.Reference_No,'')) " "as inv_maint FROM tblMaint ASD left join tblSTREET S ON S.USRN = ASD.USRN " "WHERE ASD.Currency_flag = 0 AND S.Currency_Flag = 0 AND ASD.Whole_Road = 0 AND ((ASD.start_xref = 0 " "or ASD.start_Yref = 0 or ASD.end_xref = 0 or ASD.end_yref = 0 ) or " "(ASD.start_xref = NULL or ASD.start_Yref = NULL or ASD.end_xref = NULL or ASD.end_yref = NULL)) " "ORDER BY S.USRN", 13: "SELECT (ifnull(S.USRN,'') ||'|'|| ifnull(ASD.Reins_Cat_Id,'') ||'|'|| ifnull(ASD.Reference_No,'')) " "AS inv_reins FROM tblReins_Cat ASD left join tblSTREET S ON S.USRN = ASD.USRN " "WHERE ASD.Currency_flag = 0 AND S.Currency_Flag = 0 AND ASD.Whole_Road = 0 " "AND ((ASD.start_xref = 0 or ASD.start_Yref = 0 or ASD.end_xref = 0 or ASD.end_yref = 0 ) or " "(ASD.start_xref = NULL or ASD.start_Yref = NULL or ASD.end_xref = NULL or ASD.end_yref = NULL)) " "ORDER BY S.USRN", 14: "SELECT (ifnull(S.USRN,'') ||'|'|| ifnull(ASD.Spec_Des_ID,'') ||'|'|| ifnull(ASD.Reference_No,'')) " "AS inv_spec_des FROM tblSpec_Des ASD left join tblSTREET S ON S.USRN = ASD.USRN " "WHERE ASD.Currency_flag = 0 AND S.Currency_Flag = 0 AND ASD.Whole_Road = 0 " "AND ((ASD.start_xref = 0 or ASD.start_Yref = 0 or ASD.end_xref = 0 or ASD.end_yref = 0 ) or " "(ASD.start_xref = NULL or ASD.start_Yref = NULL or ASD.end_xref = NULL or ASD.end_yref = NULL)) " "ORDER BY S.USRN", 15: "SELECT (ifnull(tblMAINT.USRN,'') ||'|'|| ifnull(tblMAINT.Maint_id, '') ||'|'|| " "ifnull(tlkpROAD_STATUS.Description, '')) as road_status " "FROM (tblMAINT LEFT JOIN lnkMAINT_RD_POL ON tblMAINT.Maint_id = lnkMAINT_RD_POL.maint_id) " "INNER JOIN tlkpROAD_STATUS ON tblMAINT.Road_status_ref = tlkpROAD_STATUS.Road_status_ref " "WHERE (((tblMAINT.Currency_flag) = 0) And ((lnkMAINT_RD_POL.rd_pol_id) Is Null)) " "ORDER BY tblMAINT.USRN", 16: "SELECT rdpoly.RD_POL_ID FROM rdpoly " "LEFT JOIN (select * from lnkMaint_RD_POL WHERE currency_flag = 0) " "AS mrp ON rdpoly.RD_POL_ID = mrp.rd_pol_id WHERE (((mrp.Maint_ID) Is Null)) ORDER BY rdpoly.RD_POL_ID", 17: "SELECT (ifnull(mp.rd_pol_id,'') ||'|'|| ifnull(tblMAINT.USRN,'') ||'|'|| " "ifnull(tblMAINT.Maint_id,'')) AS check_polys " "FROM (SELECT lnkMAINT_RD_POL.rd_pol_id, Count(lnkMAINT_RD_POL.maint_id) AS CountOfmaint_id " "From lnkMAINT_RD_POL " "GROUP BY lnkMAINT_RD_POL.rd_pol_id, lnkMAINT_RD_POL.currency_flag " "HAVING (((Count(lnkMAINT_RD_POL.maint_id))>1) AND ((lnkMAINT_RD_POL.currency_flag)=0))) as mp " "INNER JOIN (lnkMAINT_RD_POL INNER JOIN tblMAINT ON lnkMAINT_RD_POL.maint_id = tblMAINT.Maint_id) " "ON mp.rd_pol_id = lnkMAINT_RD_POL.rd_pol_id " "Where (((lnkMAINT_RD_POL.Currency_flag) = 0) And ((tblMAINT.Currency_flag) = 0)) " "ORDER BY mp.rd_pol_id, tblMAINT.USRN;" }
def export_umdimage2(src, dst, convert_gim=True, unique=False, parent=None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError( "Cannot export %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Export Directory", "Exporting directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Proceed?", buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton=QtGui.QMessageBox.No) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Exporting...", "Cancel", 0, 0, parent) progress.setWindowTitle("Exporting...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) if unique: tmp_dst = tempfile.mkdtemp(prefix="sdse-") else: tmp_dst = dst seen_groups = [] for pak in glob.iglob(os.path.join(src, "bg_*.pak")): if progress.wasCanceled(): break pak_name = os.path.basename(pak) out_dir = os.path.join(tmp_dst, pak_name) progress.setLabelText("Extracting:\n" + pak) thread = threading.Thread(target=extract_model_pak, args=(pak, out_dir, convert_gim)) thread.start() while thread.isAlive(): thread.join(MIN_INTERVAL) progress.setValue(progress.value() ^ 1) if progress.wasCanceled(): progress.setLabelText("Canceling...") if progress.wasCanceled(): break if unique: for img in list_all_files(out_dir): img_base = img[len(tmp_dst) + 1:] dupe_name = os.path.splitext(img_base)[0] + ".gim" dupe_name = os.path.join("umdimage2", dupe_name) dupe_name = os.path.normpath(os.path.normcase(dupe_name)) group = _DUPE_DB.group_from_file(dupe_name) if group in seen_groups: continue if not group == None: seen_groups.append(group) dst_file = os.path.join(dst, img_base) dst_dir = os.path.dirname(dst_file) try: os.makedirs(dst_dir) except: pass shutil.copy2(img, dst_file) shutil.rmtree(out_dir) if unique: shutil.rmtree(tmp_dst) progress.close()
def calculate_progress(parent, filter = script_analytics.DEFAULT_FILTER): start_time = time.time() progress = QProgressDialog("Calculating translation progress...", "Abort", 0, 72000, parent) progress.setWindowTitle("Translation Progress") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) # For our dupe database, we need "umdimage" instead of wherever the files # are really stored, so we strip that part off first. dir_start = len(common.editor_config.umdimage_dir) + 1 total_files = 0 unique_files = 0 translated_files = 0 translated_unique = 0 total_chars = 0 unique_chars = 0 translated_chars = 0 translated_unique_chars = 0 translated_words = 0 translated_unique_words = 0 total_bytes = 0 unique_bytes = 0 translated_bytes = 0 translated_unique_bytes = 0 groups_seen = set() files_seen = set() untranslated_lines = [] for i, total, filename, data in script_analytics.SA.get_data(filter): if progress.wasCanceled(): return progress.setValue(i) progress.setMaximum(total) if data == None: continue db_name = os.path.join("umdimage", filename) real_name = os.path.join(common.editor_config.umdimage_dir, filename) if db_name in files_seen: continue dupe_group = dupe_db.db.group_from_file(db_name) # Add the whole group to the translated files, but only one # to the unique translated. If there is no group, it's size 1. group_size = 1 if not dupe_group == None: if dupe_group in groups_seen: continue else: groups_seen.add(dupe_group) group_files = dupe_db.db.files_in_group(dupe_group) group_size = 0 for dupe_file in group_files: #if filter.search(dupe_file): group_size += 1 files_seen.update(group_files) total_files += group_size unique_files += 1 #file = script_for_counting(data) file = data # How many characters is the untranslated, non-tagged text? num_chars = len(file.original_notags) #num_bytes = len(bytearray(file.original_notags, encoding = "SJIS", errors = "replace")) total_chars += num_chars * group_size unique_chars += num_chars #total_bytes += num_bytes * group_size #unique_bytes += num_bytes if not file.translated_notags == "" or num_chars == 0: translated_files += group_size translated_unique += 1 translated_chars += num_chars * group_size translated_unique_chars += num_chars words = count_words(file.translated_notags) translated_words += words * group_size translated_unique_words += words #translated_bytes += num_bytes * group_size #translated_unique_bytes += num_bytes #elif file.translated_notags == "": #untranslated_lines.append(db_name) progress.close() #print "Took %s seconds." % (time.time() - start_time) files_percent = 100.0 if total_files == 0 else float(translated_files) / total_files * 100 unique_files_percent = 100.0 if unique_files == 0 else float(translated_unique) / unique_files * 100 chars_percent = 100.0 if total_chars == 0 else float(translated_chars) / total_chars * 100 unique_chars_percent = 100.0 if unique_chars == 0 else float(translated_unique_chars) / unique_chars * 100 bytes_percent = 100.0 if total_bytes == 0 else float(translated_bytes) / total_bytes * 100 unique_bytes_percent = 100.0 if unique_bytes == 0 else float(translated_unique_bytes) / unique_bytes * 100 QtGui.QMessageBox.information( parent, "Translation Progress", ("Files: %d / %d (%0.2f%%)\n" % (translated_files, total_files, files_percent)) + ("Unique Files: %d / %d (%0.2f%%)\n" % (translated_unique, unique_files, unique_files_percent)) + "\n" + ("Japanese Characters: %d / %d (%0.2f%%)\n" % (translated_chars, total_chars, chars_percent)) + ("Unique Characters: %d / %d (%0.2f%%)\n" % (translated_unique_chars, unique_chars, unique_chars_percent)) + #"\n" + #("Bytes: %d / %d (%0.2f%%)\n" % (translated_bytes, total_bytes, bytes_percent)) + #("Unique Bytes: %d / %d (%0.2f%%)\n" % (translated_unique_bytes, unique_bytes, unique_bytes_percent)) + "\n" + ("English Words: %d\n" % (translated_words)) + ("Unique Words: %d\n" % (translated_unique_words)) + "\n" + "NOTE: Unique X is lazy for \"X in all unique files\"\n" + "and not what it seems to imply." + "", buttons = QtGui.QMessageBox.Ok, defaultButton = QtGui.QMessageBox.Ok )
class CSVGenerator: def __init__(self, left, top, right, bottom, csv_filename, username, password, client_id, client_secret, days_to_query=60): self.left = left self.top = top self.right = right self.bottom = bottom self.csv_filename = csv_filename self.days_to_query = days_to_query self.begin_date = None self.end_date = None self.username = username self.password = password self.client_id = client_id self.client_secret = client_secret # throw up a progress dialog min_progress = 0.0 max_progress = ((self.right - self.left) / INCREMENTAL_INTERVAL) * \ ((self.top - self.bottom) / INCREMENTAL_INTERVAL) self.current_progress = min_progress self.progress_dialog = QProgressDialog("Building up CSV file", "Abort", int(min_progress), int(max_progress), None) self.progress_dialog.setCancelButton(None) self.progress_dialog.setWindowTitle("CSV Output") self.progress_dialog.setLabelText("Building up CSV file") self.progress_dialog.setMinimumDuration(0) self.progress_dialog.setValue(0) self.progress_dialog.show() self.csv_elements = [] self.csv_generator_object = CSVGeneratorObject(self) self.vector_header_dict = {} self.pool = QThreadPool() self.finished_submissions = False self.lock = Lock() def generate_csv(self): # dates now = datetime.now() self.end_date = now self.begin_date = now - timedelta(days=self.days_to_query) current_x = self.left current_y = self.bottom serial_no = 1 # get header dict vector_query = VectorQuery(username=self.username, password=self.password, client_id=self.client_id, client_secret=self.client_secret) vector_query.log_in() vector_params = InfoCubeVectorParams(top=self.top, bottom=self.bottom, left=self.left, right=self.right, time_begin=self.begin_date, time_end=self.end_date) header_result = vector_query.get_vector_result(vector_params) self.vector_header_dict = vector_query.get_vector_data(header_result) for next_x in drange(self.left + INCREMENTAL_INTERVAL, self.right, INCREMENTAL_INTERVAL): for next_y in drange(self.bottom + INCREMENTAL_INTERVAL, self.top, INCREMENTAL_INTERVAL): username = self.username password = self.password client_id = self.client_id client_secret = self.client_secret csv_runnable = CSVRunnable(username, password, client_id, client_secret, serial_no, next_y, current_x, next_x, current_y, self.begin_date, self.end_date, self.vector_header_dict) csv_runnable.csv_object.new_csv_element.connect( self.csv_generator_object.callback) self.pool.start(csv_runnable) serial_no += 1 current_y = next_y current_y = self.bottom current_x = next_x self.finished_submissions = True def on_completion(self): self.csv_elements.sort(key=lambda element: element.serial_no) log.info("Sort complete") # write file csv_file = open(self.csv_filename, 'w') # write the header header = CSVOutput.get_csv_header() if self.vector_header_dict: for term in self.vector_header_dict: header = header + str(term) + "," header = header[:-1] csv_file.write(header) csv_file.write("\n") for csv_element in self.csv_elements: csv_file.write(str(csv_element)) csv_file.write("\n") csv_file.close() log.info("Write complete") if self.progress_dialog: self.progress_dialog.close() self.csv_generator_object.message_complete.emit(self.csv_filename)
def setup_workspace(self): data0 = os.path.join(self.iso_dir, DATA0_CPK) self.generate_directories() progress = QProgressDialog("", QtCore.QString(), 0, 11000, self) progress.setWindowTitle("Setting up workspace...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setMinimumDuration(0) progress.setValue(0) progress.setAutoClose(False) progress.setAutoReset(False) progress.setLabelText("Creating directories...") # Do the easy stuff first. if not os.path.isdir(self.changes_dir): os.makedirs(self.changes_dir) progress.setValue(progress.value() + 1) if not os.path.isdir(self.backup_dir): os.makedirs(self.backup_dir) progress.setValue(progress.value() + 1) thread_fns = [{"target": extract_cpk, "kwargs": {"filename": data0, "out_dir": self.data0_dir}}] # Going to capture stdout because I don't feel like # rewriting the extract functions to play nice with GUI. stdout = sys.stdout sys.stdout = cStringIO.StringIO() for thread_fn in thread_fns: thread = threading.Thread(**thread_fn) thread.start() while thread.isAlive(): thread.join(THREAD_TIMEOUT) output = [line for line in sys.stdout.getvalue().split("\n") if len(line) > 0] progress.setValue(progress.value() + len(output)) if len(output) > 0: progress.setLabelText("Extracting %s..." % output[-1]) sys.stdout = cStringIO.StringIO() sys.stdout = stdout # Give us an ISO directory for the editor to place modified files in. progress.setLabelText("Copying ISO files...") # ISO directory needs to not exist for copytree. if os.path.isdir(self.edited_iso_dir): shutil.rmtree(self.edited_iso_dir) # One more thing we want threaded so it doesn't lock up the GUI. thread = threading.Thread(target=shutil.copytree, kwargs={"src": self.iso_dir, "dst": self.edited_iso_dir}) thread.start() while thread.isAlive(): thread.join(THREAD_TIMEOUT) progress.setLabelText("Copying ISO files...") # It has to increase by some amount or it won't update and the UI will lock up. progress.setValue(progress.value() + 1) # shutil.copytree(self.iso_dir, self.edited_iso_dir) progress.setValue(progress.value() + 1) # Files we want to make blank, because they're unnecessary. blank_files = [ os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "DATA.BIN"), os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "EBOOT.BIN"), os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "UPDATE", "PARAM.SFO"), ] for blank in blank_files: with open(blank, "wb") as f: pass # Copy the decrypted EBOOT into the ISO folder and apply our hacks to it. progress.setLabelText("Hacking EBOOT...") progress.setValue(progress.value() + 1) hacked_eboot = BitStream(filename=self.eboot_path) hacked_eboot = apply_eboot_patches(hacked_eboot) with open(os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN"), "wb") as f: hacked_eboot.tofile(f) # shutil.copy(self.eboot_path, os.path.join(self.edited_iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN")) progress.setLabelText("Extracting editor data...") progress.setValue(progress.value() + 1) # Extract the editor data. editor_data = zipfile.ZipFile("data/editor_data.zip", "r") editor_data.extractall(self.editor_data_dir) editor_data.close() progress.setValue(progress.maximum()) progress.close() self.ui.grpStep4.setEnabled(False) self.ui.grpStep5.setEnabled(True)
class AddonManagerDialog(QDialog): _packages = None def __init__(self, parent=None, **kwargs): super().__init__(parent, **kwargs) self.setLayout(QVBoxLayout()) self.layout().setContentsMargins(0, 0, 0, 0) self.addonwidget = AddonManagerWidget() self.layout().addWidget(self.addonwidget) buttons = QDialogButtonBox(orientation=Qt.Horizontal, standardButtons=QDialogButtonBox.Ok | QDialogButtonBox.Cancel) buttons.accepted.connect(self.__accepted) buttons.rejected.connect(self.reject) self.layout().addWidget(buttons) self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) if AddonManagerDialog._packages is None: self._f_pypi_addons = self._executor.submit(list_pypi_addons) else: self._f_pypi_addons = concurrent.futures.Future() self._f_pypi_addons.set_result(AddonManagerDialog._packages) self._f_pypi_addons.add_done_callback( method_queued(self._set_packages, (object, ))) self.__progress = QProgressDialog( self, Qt.Sheet, minimum=0, maximum=0, labelText=self.tr("Retrieving package list"), sizeGripEnabled=False, ) self.__progress.rejected.connect(self.reject) self.__thread = None self.__installer = None @Slot(object) def _set_packages(self, f): if self.__progress.isVisible(): self.__progress.close() try: packages = f.result() except (IOError, OSError) as err: message_warning("Could not retrieve package list", title="Error", informative_text=str(err), parent=self) packages = [] except Exception: raise else: AddonManagerDialog._packages = packages installed = list_installed_addons() dists = {dist.project_name: dist for dist in installed} packages = {pkg.name: pkg for pkg in packages} project_names = unique(itertools.chain(packages.keys(), dists.keys())) items = [] for name in project_names: if name in dists and name in packages: item = Installed(packages[name], dists[name]) elif name in dists: item = Installed(None, dists[name]) elif name in packages: item = Available(packages[name]) else: assert False items.append(item) self.addonwidget.set_items(items) def showEvent(self, event): super().showEvent(event) if not self._f_pypi_addons.done(): QTimer.singleShot(0, self.__progress.show) def done(self, retcode): super().done(retcode) self._f_pypi_addons.cancel() self._executor.shutdown(wait=False) if self.__thread is not None: self.__thread.quit() self.__thread.wait(1000) def closeEvent(self, event): super().closeEvent(event) self._f_pypi_addons.cancel() self._executor.shutdown(wait=False) if self.__thread is not None: self.__thread.quit() self.__thread.wait(1000) def __accepted(self): steps = self.addonwidget.item_state() if steps: # Move all uninstall steps to the front steps = sorted(steps, key=lambda step: 0 if step[0] == Uninstall else 1) self.__installer = Installer(steps=steps) self.__thread = QThread(self) self.__thread.start() self.__installer.moveToThread(self.__thread) self.__installer.finished.connect(self.__on_installer_finished) self.__installer.error.connect(self.__on_installer_error) self.__installer.installStatusChanged.connect( self.__progress.setLabelText) self.__progress.show() self.__progress.setLabelText("Installing") self.__installer.start() else: self.accept() def __on_installer_error(self, command, pkg, retcode, output): message_error( "An error occurred while running a subprocess", title="Error", informative_text="{} exited with non zero status.".format(command), details="".join(output), parent=self) self.reject() def __on_installer_finished(self): message_information( "Please restart the application for changes to take effect.", parent=self) self.accept()