def pip_list(self, name=None, prefix=None, abspath=True, emit=False): """ Get list of pip installed packages. """ if (name and prefix) or not (name or prefix): raise TypeError("conda pip: exactly one of 'name' ""or 'prefix' " "required.") if self._is_not_running: cmd_list = self._abspath(abspath) if name: cmd_list.extend(['list', '--name', name]) if prefix: cmd_list.extend(['list', '--prefix', prefix]) qprocess = QProcess() qprocess.start(cmd_list[0], cmd_list[1:]) qprocess.waitForFinished() output = qprocess.readAllStandardOutput() output = handle_qbytearray(output, CondaProcess.ENCODING) result = [] lines = output.split('\n') for line in lines: if '<pip>' in line: temp = line.split()[:-1] + ['pip'] result.append('-'.join(temp)) if emit: self.sig_finished.emit("pip", str(result), "") return result
def environment_exists(self, name=None, prefix=None, abspath=True, emit=False): """ Check if an environment exists by 'name' or by 'prefix'. If query is by 'name' only the default conda environments directory is searched. """ if name and prefix: raise TypeError("Exactly one of 'name' or 'prefix' is required.") qprocess = QProcess() cmd_list = self._abspath(abspath) cmd_list.extend(['list', '--json']) if name: cmd_list.extend(['--name', name]) else: cmd_list.extend(['--prefix', prefix]) qprocess.start(cmd_list[0], cmd_list[1:]) qprocess.waitForFinished() output = qprocess.readAllStandardOutput() output = handle_qbytearray(output, CondaProcess.ENCODING) info = json.loads(output) if emit: self.sig_finished.emit("info", unicode(info), "") return 'error' not in info
class Installer: def __init__(self, output_widget: QTextEdit = None): from ...plugins import plugin_manager # create install process self._output_widget = None self.process = QProcess() self.process.setProgram(sys.executable) self.process.setProcessChannelMode(QProcess.MergedChannels) self.process.readyReadStandardOutput.connect(self._on_stdout_ready) # setup process path env = QProcessEnvironment() combined_paths = os.pathsep.join([ user_site_packages(), env.systemEnvironment().value("PYTHONPATH") ]) env.insert("PYTHONPATH", combined_paths) # use path of parent process env.insert("PATH", QProcessEnvironment.systemEnvironment().value("PATH")) self.process.setProcessEnvironment(env) self.process.finished.connect(lambda: plugin_manager.discover()) self.process.finished.connect(lambda: plugin_manager.prune()) self.set_output_widget(output_widget) def set_output_widget(self, output_widget: QTextEdit): if output_widget: self._output_widget = output_widget self.process.setParent(output_widget) def _on_stdout_ready(self): if self._output_widget: text = self.process.readAllStandardOutput().data().decode() self._output_widget.append(text) def install(self, pkg_list: Sequence[str]): cmd = ['-m', 'pip', 'install', '--upgrade'] if running_as_bundled_app() and sys.platform.startswith('linux'): cmd += [ '--no-warn-script-location', '--prefix', user_plugin_dir(), ] self.process.setArguments(cmd + list(pkg_list)) if self._output_widget: self._output_widget.clear() self.process.start() def uninstall(self, pkg_list: Sequence[str]): args = ['-m', 'pip', 'uninstall', '-y'] self.process.setArguments(args + list(pkg_list)) if self._output_widget: self._output_widget.clear() self.process.start() for pkg in pkg_list: plugin_manager.unregister(pkg)
def info(self, abspath=True): """ Return a dictionary with configuration information. No guarantee is made about which keys exist. Therefore this function should only be used for testing and debugging. """ # return self._call_and_parse(['info', '--json'], abspath=abspath) qprocess = QProcess() cmd_list = self._abspath(abspath) cmd_list.extend(['info', '--json']) qprocess.start(cmd_list[0], cmd_list[1:]) qprocess.waitForFinished() output = qprocess.readAllStandardOutput() output = handle_qbytearray(output, CondaProcess.ENCODING) info = json.loads(output) return info
def info(self, abspath=True): """ Return a dictionary with configuration information. No guarantee is made about which keys exist. Therefore this function should only be used for testing and debugging. """ if self._is_not_running(): qprocess = QProcess() cmd_list = self._abspath(abspath) cmd_list.extend(["info", "--json"]) qprocess.start(cmd_list[0], cmd_list[1:]) qprocess.waitForFinished() output = qprocess.readAllStandardOutput() output = handle_qbytearray(output, CondaProcess.ENCODING) info = json.loads(output) self.sig_finished.emit("info", str(info), "") return info
class ProcessWorker(QObject): """Conda worker based on a QProcess for non blocking UI.""" sig_finished = Signal(object, object, object) sig_partial = Signal(object, object, object) def __init__(self, cmd_list, parse=False, pip=False, callback=None, extra_kwargs=None): """Conda worker based on a QProcess for non blocking UI. Parameters ---------- cmd_list : list of str Command line arguments to execute. parse : bool (optional) Parse json from output. pip : bool (optional) Define as a pip command. callback : func (optional) If the process has a callback to process output from comd_list. extra_kwargs : dict Arguments for the callback. """ super(ProcessWorker, self).__init__() self._result = None self._cmd_list = cmd_list self._parse = parse self._pip = pip self._conda = not pip self._callback = callback self._fired = False self._communicate_first = False self._partial_stdout = None self._extra_kwargs = extra_kwargs if extra_kwargs else {} self._timer = QTimer() self._process = QProcess() self._timer.setInterval(150) self._timer.timeout.connect(self._communicate) # self._process.finished.connect(self._communicate) self._process.readyReadStandardOutput.connect(self._partial) def _partial(self): """Callback for partial output.""" raw_stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(raw_stdout, _CondaAPI.UTF8) json_stdout = stdout.replace('\n\x00', '') try: json_stdout = json.loads(json_stdout) except Exception: json_stdout = stdout if self._partial_stdout is None: self._partial_stdout = stdout else: self._partial_stdout += stdout self.sig_partial.emit(self, json_stdout, None) def _communicate(self): """Callback for communicate.""" if (not self._communicate_first and self._process.state() == QProcess.NotRunning): self.communicate() elif self._fired: self._timer.stop() def communicate(self): """Retrieve information.""" self._communicate_first = True self._process.waitForFinished() if self._partial_stdout is None: raw_stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(raw_stdout, _CondaAPI.UTF8) else: stdout = self._partial_stdout raw_stderr = self._process.readAllStandardError() stderr = handle_qbytearray(raw_stderr, _CondaAPI.UTF8) result = [stdout.encode(_CondaAPI.UTF8), stderr.encode(_CondaAPI.UTF8)] # FIXME: Why does anaconda client print to stderr??? if PY2: stderr = stderr.decode() if 'using anaconda' not in stderr.lower(): if stderr.strip() and self._conda: logger.error('{0}:\nSTDERR:\n{1}\nEND'.format( ' '.join(self._cmd_list), stderr)) elif stderr.strip() and self._pip: logger.error("pip error: {}".format(self._cmd_list)) result[-1] = '' if self._parse and stdout: try: result = json.loads(stdout), result[-1] except Exception as error: result = stdout, str(error) if 'error' in result[0]: if not isinstance(result[0], dict): result = {'error': str(result[0])}, None error = '{0}: {1}'.format(" ".join(self._cmd_list), result[0]['error']) result = result[0], error if self._callback: result = self._callback(result[0], result[-1], **self._extra_kwargs), result[-1] self._result = result self.sig_finished.emit(self, result[0], result[-1]) if result[-1]: logger.error(str(('error', result[-1]))) self._fired = True return result def close(self): """Close the running process.""" self._process.close() def is_finished(self): """Return True if worker has finished processing.""" return self._process.state() == QProcess.NotRunning and self._fired def start(self): """Start process.""" logger.debug(str(' '.join(self._cmd_list))) if not self._fired: self._partial_ouput = None self._process.start(self._cmd_list[0], self._cmd_list[1:]) self._timer.start() else: raise CondaProcessWorker('A Conda ProcessWorker can only run once ' 'per method call.')
class ProcessWorker(QObject): """Process worker based on a QProcess for non blocking UI.""" sig_started = Signal(object) sig_finished = Signal(object, object, object) sig_partial = Signal(object, object, object) def __init__(self, cmd_list, environ=None): """ Process worker based on a QProcess for non blocking UI. Parameters ---------- cmd_list : list of str Command line arguments to execute. environ : dict Process environment, """ super(ProcessWorker, self).__init__() self._result = None self._cmd_list = cmd_list self._fired = False self._communicate_first = False self._partial_stdout = None self._started = False self._timer = QTimer() self._process = QProcess() self._set_environment(environ) self._timer.setInterval(150) self._timer.timeout.connect(self._communicate) self._process.readyReadStandardOutput.connect(self._partial) def _get_encoding(self): """Return the encoding/codepage to use.""" enco = 'utf-8' # Currently only cp1252 is allowed? if WIN: import ctypes codepage = to_text_string(ctypes.cdll.kernel32.GetACP()) # import locale # locale.getpreferredencoding() # Differences? enco = 'cp' + codepage return enco def _set_environment(self, environ): """Set the environment on the QProcess.""" if environ: q_environ = self._process.processEnvironment() for k, v in environ.items(): q_environ.insert(k, v) self._process.setProcessEnvironment(q_environ) def _partial(self): """Callback for partial output.""" raw_stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(raw_stdout, self._get_encoding()) if self._partial_stdout is None: self._partial_stdout = stdout else: self._partial_stdout += stdout self.sig_partial.emit(self, stdout, None) def _communicate(self): """Callback for communicate.""" if (not self._communicate_first and self._process.state() == QProcess.NotRunning): self.communicate() elif self._fired: self._timer.stop() def communicate(self): """Retrieve information.""" self._communicate_first = True self._process.waitForFinished() enco = self._get_encoding() if self._partial_stdout is None: raw_stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(raw_stdout, enco) else: stdout = self._partial_stdout raw_stderr = self._process.readAllStandardError() stderr = handle_qbytearray(raw_stderr, enco) result = [stdout.encode(enco), stderr.encode(enco)] if PY2: stderr = stderr.decode() result[-1] = '' self._result = result if not self._fired: self.sig_finished.emit(self, result[0], result[-1]) self._fired = True return result def close(self): """Close the running process.""" self._process.close() def is_finished(self): """Return True if worker has finished processing.""" return self._process.state() == QProcess.NotRunning and self._fired def _start(self): """Start process.""" if not self._fired: self._partial_ouput = None self._process.start(self._cmd_list[0], self._cmd_list[1:]) self._timer.start() def terminate(self): """Terminate running processes.""" if self._process.state() == QProcess.Running: try: self._process.terminate() except Exception: pass self._fired = True def start(self): """Start worker.""" if not self._started: self.sig_started.emit(self) self._started = True
class QJackCaptureMainWindow(QDialog): sample_formats = { "32-bit float": "FLOAT", "8-bit integer": "8", "16-bit integer": "16", "24-bit integer": "24", "32-bit integer": "32", } def __init__(self, parent, jack_client, jack_name=PROGRAM): QDialog.__init__(self, parent) self.ui = Ui_MainWindow() self.ui.setupUi(self) self.fFreewheel = False self.fLastTime = -1 self.fMaxTime = 180 self.fTimer = QTimer(self) self.fProcess = QProcess(self) self.fJackClient = jack_client self.fJackName = jack_name self.fBufferSize = self.fJackClient.get_buffer_size() self.fSampleRate = self.fJackClient.get_sample_rate() # Selected ports used as recording sources self.rec_sources = set() self.createUi() self.loadSettings() self.populatePortLists(init=True) # listen to changes to JACK ports self._refresh_timer = None self.fJackClient.ports_changed.connect(self.slot_refreshPortsLists) @Slot() def slot_refreshPortsLists(self, delay=200): if not self._refresh_timer or not self._refresh_timer.isActive(): log.debug("Scheduling port lists refresh in %i ms...", delay) self._refresh_timer = QTimer() self._refresh_timer.setSingleShot(True) self._refresh_timer.timeout.connect(self.populatePortLists) self._refresh_timer.start(delay) def populateFileFormats(self): # Get list of supported file formats self.fProcess.start(gJackCapturePath, ["-pf"]) self.fProcess.waitForFinished() formats = [] for fmt in str(self.fProcess.readAllStandardOutput(), encoding="utf-8").split(): fmt = fmt.strip() if fmt: formats.append(fmt) # Put all file formats in combo-box, select 'wav' option self.ui.cb_format.clear() for i, fmt in enumerate(sorted(formats)): self.ui.cb_format.addItem(fmt) if fmt == "wav": self.ui.cb_format.setCurrentIndex(i) def populateSampleFormats(self): # Put all sample formats in combo-box, select 'FLOAT' option self.ui.cb_depth.clear() for i, (label, fmt) in enumerate(self.sample_formats.items()): self.ui.cb_depth.addItem(label, fmt) if fmt == "FLOAT": self.ui.cb_depth.setCurrentIndex(i) def populatePortLists(self, init=False): log.debug("Populating port lists (init=%s)...", init) if init: self.outputs_model = QStandardItemModel(0, 1, self) self.inputs_model = QStandardItemModel(0, 1, self) else: self.outputs_model.clear() self.inputs_model.clear() output_ports = list(self.fJackClient.get_output_ports()) self.populatePortList(self.outputs_model, self.ui.tree_outputs, output_ports) input_ports = list(self.fJackClient.get_input_ports()) self.populatePortList(self.inputs_model, self.ui.tree_inputs, input_ports) # Remove ports, which are no longer present, from recording sources all_ports = set((p.client, p.name) for p in output_ports) all_ports |= set((p.client, p.name) for p in input_ports) self.rec_sources.intersection_update(all_ports) self.slot_toggleRecordingSource() def makePortTooltip(self, port): s = [] if port.pretty_name: s.append(f"<b>Pretty name:</b> <em>{port.pretty_name}</em><br>") s.append(f"<b>Port:</b> <tt>{port.client}:{port.name}</tt><br>") for i, alias in enumerate(port.aliases, 1): s.append(f"<b>Alias {i}:</b> <tt>{alias}</tt><br>") s.append(f"<b>UUID:</b> <tt>{port.uuid}</tt>") return "<small>{}</small>".format("\n".join(s)) def populatePortList(self, model, tv, ports): tv.setModel(model) root = model.invisibleRootItem() portsdict = {} for port in ports: if port.client not in portsdict: portsdict[port.client] = [] portsdict[port.client].append(port) for client in humansorted(portsdict): clientitem = QStandardItem(client) for port in humansorted(portsdict[client], key=attrgetter("group", "order", "name")): portspec = (port.client, port.name) if port.pretty_name: label = "%s (%s)" % (port.pretty_name, port.name) else: label = port.name portitem = QStandardItem(label) portitem.setData(portspec) portitem.setCheckable(True) portitem.setUserTristate(False) # Check box toggling is done in the treeview clicked handler "on_port_clicked" portitem.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled) portitem.setToolTip(self.makePortTooltip(port)) if portspec in self.rec_sources: portitem.setCheckState(2) clientitem.appendRow(portitem) root.appendRow(clientitem) tv.expandAll() def createUi(self): # ------------------------------------------------------------- # Set-up GUI stuff for i in range(self.ui.cb_buffer_size.count()): if int(self.ui.cb_buffer_size.itemText(i)) == self.fBufferSize: self.ui.cb_buffer_size.setCurrentIndex(i) break else: self.ui.cb_buffer_size.addItem(str(self.fBufferSize)) self.ui.cb_buffer_size.setCurrentIndex( self.ui.cb_buffer_size.count() - 1) self.populateFileFormats() self.populateSampleFormats() self.ui.rb_stereo.setChecked(True) self.ui.te_end.setTime(QTime(0, 3, 0)) self.ui.progressBar.setFormat("") self.ui.progressBar.setMinimum(0) self.ui.progressBar.setMaximum(1) self.ui.progressBar.setValue(0) self.ui.b_render.setIcon(get_icon("media-record")) self.ui.b_stop.setIcon(get_icon("media-playback-stop")) self.ui.b_close.setIcon(get_icon("window-close")) self.ui.b_open.setIcon(get_icon("document-open")) self.ui.b_stop.setVisible(False) self.ui.le_folder.setText(expanduser("~")) # ------------------------------------------------------------- # Set-up connections self.ui.b_render.clicked.connect(self.slot_renderStart) self.ui.b_stop.clicked.connect(self.slot_renderStop) self.ui.b_open.clicked.connect(self.slot_getAndSetPath) self.ui.b_now_start.clicked.connect(self.slot_setStartNow) self.ui.b_now_end.clicked.connect(self.slot_setEndNow) self.ui.te_start.timeChanged.connect(self.slot_updateStartTime) self.ui.te_end.timeChanged.connect(self.slot_updateEndTime) self.ui.group_time.clicked.connect(self.slot_transportChecked) self.ui.rb_source_default.toggled.connect( self.slot_toggleRecordingSource) self.ui.rb_source_manual.toggled.connect( self.slot_toggleRecordingSource) self.ui.rb_source_selected.toggled.connect( self.slot_toggleRecordingSource) self.fTimer.timeout.connect(self.slot_updateProgressbar) for tv in (self.ui.tree_outputs, self.ui.tree_inputs): menu = QMenu() menu.addAction(get_icon("expand-all"), self.tr("E&xpand all"), tv.expandAll) menu.addAction(get_icon("collapse-all"), self.tr("&Collapse all"), tv.collapseAll) menu.addSeparator() menu.addAction( get_icon("list-select-all"), self.tr("&Select all in group"), partial(self.on_select_port_group, tv), ) menu.addAction( get_icon("list-select-none"), self.tr("&Unselect all in group"), partial(self.on_select_port_group, tv, enable=False), ) menu.addSeparator() if tv is self.ui.tree_outputs: menu.addAction( get_icon("select-none"), self.tr("Unselect all &outputs"), partial(self.on_clear_all_ports, tv), ) else: menu.addAction( get_icon("select-none"), self.tr("Unselect all &inputs"), partial(self.on_clear_all_ports, tv), ) tv.setContextMenuPolicy(Qt.CustomContextMenu) tv.customContextMenuRequested.connect( partial(self.on_port_menu, treeview=tv, menu=menu)) tv.clicked.connect(self.on_port_clicked) def enable_port(self, item, enable=True): item.setCheckState(2 if enable else 0) port = item.data() if enable: self.rec_sources.add(port) else: self.rec_sources.discard(port) def on_port_menu(self, pos, treeview=None, menu=None): if treeview and menu: menu.popup(treeview.viewport().mapToGlobal(pos)) def foreach_item(self, model, parent, func, leaves_only=True): for row in range(model.rowCount(parent)): index = model.index(row, 0, parent) is_leaf = not model.hasChildren(index) if is_leaf or not leaves_only: func(model.itemFromIndex(index)) if not is_leaf: self.foreach_item(model, index, func) def on_clear_all_ports(self, treeview): self.foreach_item(treeview.model(), QModelIndex(), partial(self.enable_port, enable=False)) self.checkRecordEnable() def on_port_clicked(self, index): model = index.model() item = model.itemFromIndex(index) if not model.hasChildren(index): self.enable_port(item, not item.checkState()) self.checkRecordEnable() def on_select_port_group(self, treeview, enable=True): index = treeview.currentIndex() model = index.model() if not model.hasChildren(index): index = index.parent() self.foreach_item(model, index, partial(self.enable_port, enable=enable)) self.checkRecordEnable() @Slot() def slot_renderStart(self): if not exists(self.ui.le_folder.text()): QMessageBox.warning( self, self.tr("Warning"), self. tr("The selected directory does not exist. Please choose a valid one." ), ) return timeStart = self.ui.te_start.time() timeEnd = self.ui.te_end.time() minTime = (timeStart.hour() * 3600) + (timeStart.minute() * 60) + (timeStart.second()) maxTime = (timeEnd.hour() * 3600) + (timeEnd.minute() * 60) + (timeEnd.second()) newBufferSize = int(self.ui.cb_buffer_size.currentText()) useTransport = self.ui.group_time.isChecked() self.fFreewheel = self.ui.rb_freewheel.isChecked() self.fLastTime = -1 self.fMaxTime = maxTime if self.fFreewheel: self.fTimer.setInterval(100) else: self.fTimer.setInterval(500) self.ui.group_render.setEnabled(False) self.ui.group_time.setEnabled(False) self.ui.group_encoding.setEnabled(False) self.ui.b_render.setVisible(False) self.ui.b_stop.setVisible(True) self.ui.b_close.setEnabled(False) if useTransport: self.ui.progressBar.setFormat("%p%") self.ui.progressBar.setMinimum(minTime) self.ui.progressBar.setMaximum(maxTime) self.ui.progressBar.setValue(minTime) else: self.ui.progressBar.setFormat("") self.ui.progressBar.setMinimum(0) self.ui.progressBar.setMaximum(0) self.ui.progressBar.setValue(0) self.ui.progressBar.update() arguments = [] # JACK client name arguments.append("-jn") arguments.append(self.fJackName) # Filename prefix arguments.append("-fp") arguments.append(self.ui.le_prefix.text()) # File format arguments.append("-f") arguments.append(self.ui.cb_format.currentText()) # Sanple format (bit depth, int/float) arguments.append("-b") arguments.append(self.ui.cb_depth.currentData()) # Channels arguments.append("-c") if self.ui.rb_mono.isChecked(): arguments.append("1") elif self.ui.rb_stereo.isChecked(): arguments.append("2") else: arguments.append(str(self.ui.sb_channels.value())) # Recording sources if self.ui.rb_source_manual.isChecked(): arguments.append("-mc") elif self.ui.rb_source_selected.isChecked(): for client, port in self.rec_sources: arguments.append("-p") arguments.append("{}:{}".format(client, port)) # Controlled only by freewheel if self.fFreewheel: arguments.append("-jf") # Controlled by transport elif useTransport: arguments.append("-jt") # Silent mode arguments.append("--daemon") # Extra arguments extra_args = self.ui.le_extra_args.text().strip() if extra_args: arg_list = shlex.split(extra_args) arguments.extend(arg_list) # Change current directory os.chdir(self.ui.le_folder.text()) if newBufferSize != self.fJackClient.get_buffer_size(): log.info("Buffer size changed before render.") self.fJackClient.set_buffer_size(newBufferSize) if useTransport: if self.fJackClient.transport_running(): # rolling or starting self.fJackClient.transport_stop() self.fJackClient.transport_locate(minTime * self.fSampleRate) log.debug("jack_capture command line args: %r", arguments) self.fProcess.start(gJackCapturePath, arguments) status = self.fProcess.waitForStarted() if not status: self.fProcess.close() log.error("Could not start jack_capture.") return if self.fFreewheel: log.info("Rendering in freewheel mode.") sleep(1) self.fJackClient.set_freewheel(True) if useTransport: log.info("Rendering using JACK transport.") self.fTimer.start() self.fJackClient.transport_start() @Slot() def slot_renderStop(self): useTransport = self.ui.group_time.isChecked() if useTransport: self.fJackClient.transport_stop() if self.fFreewheel: self.fJackClient.set_freewheel(False) sleep(1) self.fProcess.terminate() # self.fProcess.waitForFinished(5000) if useTransport: self.fTimer.stop() self.ui.group_render.setEnabled(True) self.ui.group_time.setEnabled(True) self.ui.group_encoding.setEnabled(True) self.ui.b_render.setVisible(True) self.ui.b_stop.setVisible(False) self.ui.b_close.setEnabled(True) self.ui.progressBar.setFormat("") self.ui.progressBar.setMinimum(0) self.ui.progressBar.setMaximum(1) self.ui.progressBar.setValue(0) self.ui.progressBar.update() # Restore buffer size newBufferSize = self.fJackClient.get_buffer_size() if newBufferSize != self.fBufferSize: self.fJackClient.set_buffer_size(newBufferSize) @Slot() def slot_getAndSetPath(self): new_path = QFileDialog.getExistingDirectory(self, self.tr("Set Path"), self.ui.le_folder.text(), QFileDialog.ShowDirsOnly) if new_path: self.ui.le_folder.setText(new_path) @Slot() def slot_setStartNow(self): time = self.fJackClient.transport_frame() // self.fSampleRate secs = time % 60 mins = int(time / 60) % 60 hrs = int(time / 3600) % 60 self.ui.te_start.setTime(QTime(hrs, mins, secs)) @Slot() def slot_setEndNow(self): time = self.fJackClient.transport_frame() // self.fSampleRate secs = time % 60 mins = int(time / 60) % 60 hrs = int(time / 3600) % 60 self.ui.te_end.setTime(QTime(hrs, mins, secs)) @Slot(QTime) def slot_updateStartTime(self, time): if time >= self.ui.te_end.time(): self.ui.te_end.setTime(time) renderEnabled = False else: renderEnabled = True if self.ui.group_time.isChecked(): self.ui.b_render.setEnabled(renderEnabled) @Slot(QTime) def slot_updateEndTime(self, time): if time <= self.ui.te_start.time(): self.ui.te_start.setTime(time) renderEnabled = False else: renderEnabled = True if self.ui.group_time.isChecked(): self.ui.b_render.setEnabled(renderEnabled) @Slot(bool) def slot_toggleRecordingSource(self, dummy=None): enabled = self.ui.rb_source_selected.isChecked() self.ui.tree_outputs.setEnabled(enabled) self.ui.tree_inputs.setEnabled(enabled) self.checkRecordEnable() @Slot(bool) def slot_transportChecked(self, dummy=None): self.checkRecordEnable() @Slot() def slot_updateProgressbar(self): time = self.fJackClient.transport_frame() / self.fSampleRate self.ui.progressBar.setValue(time) if time > self.fMaxTime or (self.fLastTime > time and not self.fFreewheel): self.slot_renderStop() self.fLastTime = time def checkRecordEnable(self): enable = True if self.ui.rb_source_selected.isChecked() and not self.rec_sources: enable = False if self.ui.group_time.isChecked( ) and self.ui.te_end.time() <= self.ui.te_start.time(): enable = False self.ui.b_render.setEnabled(enable) log.debug("Recording sources: %s", ", ".join( ("%s:%s" % (c, p) for c, p in self.rec_sources))) def saveSettings(self): settings = QSettings(ORGANIZATION, PROGRAM) if self.ui.rb_mono.isChecked(): channels = 1 elif self.ui.rb_stereo.isChecked(): channels = 2 else: channels = self.ui.sb_channels.value() settings.setValue("Geometry", self.saveGeometry()) settings.setValue("OutputFolder", self.ui.le_folder.text()) settings.setValue("FilenamePrefix", self.ui.le_prefix.text()) settings.setValue("EncodingFormat", self.ui.cb_format.currentText()) settings.setValue("EncodingDepth", self.ui.cb_depth.currentData()) settings.setValue("EncodingChannels", channels) settings.setValue("UseTransport", self.ui.group_time.isChecked()) settings.setValue("StartTime", self.ui.te_start.time()) settings.setValue("EndTime", self.ui.te_end.time()) settings.setValue("ExtraArgs", self.ui.le_extra_args.text().strip()) if self.ui.rb_source_default.isChecked(): settings.setValue("RecordingSource", 0) elif self.ui.rb_source_manual.isChecked(): settings.setValue("RecordingSource", 1) elif self.ui.rb_source_selected.isChecked(): settings.setValue("RecordingSource", 2) settings.beginWriteArray("Sources") for i, (client, port) in enumerate(self.rec_sources): settings.setArrayIndex(i) settings.setValue("Client", client) settings.setValue("Port", port) settings.endArray() def loadSettings(self): settings = QSettings(ORGANIZATION, PROGRAM) self.restoreGeometry(settings.value("Geometry", b"")) outputFolder = settings.value("OutputFolder", get_user_dir("MUSIC")) if isdir(outputFolder): self.ui.le_folder.setText(outputFolder) self.ui.le_prefix.setText( settings.value("FilenamePrefix", "jack_capture_")) encFormat = settings.value("EncodingFormat", "wav", type=str) for i in range(self.ui.cb_format.count()): if self.ui.cb_format.itemText(i) == encFormat: self.ui.cb_format.setCurrentIndex(i) break encDepth = settings.value("EncodingDepth", "FLOAT", type=str) for i in range(self.ui.cb_depth.count()): if self.ui.cb_depth.itemData(i) == encDepth: self.ui.cb_depth.setCurrentIndex(i) break encChannels = settings.value("EncodingChannels", 2, type=int) if encChannels == 1: self.ui.rb_mono.setChecked(True) elif encChannels == 2: self.ui.rb_stereo.setChecked(True) else: self.ui.rb_outro.setChecked(True) self.ui.sb_channels.setValue(encChannels) recSource = settings.value("RecordingSource", 0, type=int) if recSource == 1: self.ui.rb_source_manual.setChecked(True) elif recSource == 2: self.ui.rb_source_selected.setChecked(True) else: self.ui.rb_source_default.setChecked(True) self.ui.group_time.setChecked( settings.value("UseTransport", False, type=bool)) self.ui.te_start.setTime( settings.value("StartTime", self.ui.te_start.time(), type=QTime)) self.ui.te_end.setTime( settings.value("EndTime", self.ui.te_end.time(), type=QTime)) self.ui.le_extra_args.setText(settings.value("ExtraArgs", "", type=str)) size = settings.beginReadArray("Sources") for i in range(size): settings.setArrayIndex(i) client = settings.value("Client", type=str) port = settings.value("Port", type=str) if client and port: self.rec_sources.add((client, port)) settings.endArray() def closeEvent(self, event): self.saveSettings() self.fJackClient.close() QDialog.closeEvent(self, event) def done(self, r): QDialog.done(self, r) self.close()
class MemoryProfilerWidget(QWidget): """ Memory profiler widget. """ DATAPATH = get_conf_path('memoryprofiler.results') VERSION = '0.0.1' redirect_stdio = Signal(bool) sig_finished = Signal() def __init__(self, parent): QWidget.__init__(self, parent) self.setWindowTitle("Memory profiler") self.output = None self.error_output = None self.use_colors = True self._last_wdir = None self._last_args = None self._last_pythonpath = None self.filecombo = PythonModulesComboBox(self) self.start_button = create_toolbutton(self, icon=get_icon('run.png'), text=_("Profile memory usage"), tip=_("Run memory profiler"), triggered=self.start, text_beside_icon=True) self.stop_button = create_toolbutton(self, icon=get_icon('terminate.png'), text=_("Stop"), tip=_("Stop current profiling"), text_beside_icon=True) self.filecombo.valid.connect(self.start_button.setEnabled) #self.connect(self.filecombo, SIGNAL('valid(bool)'), self.show_data) # FIXME: The combobox emits this signal on almost any event # triggering show_data() too early, too often. browse_button = create_toolbutton(self, icon=get_icon('fileopen.png'), tip=_('Select Python script'), triggered=self.select_file) self.datelabel = QLabel() self.log_button = create_toolbutton(self, icon=get_icon('log.png'), text=_("Output"), text_beside_icon=True, tip=_("Show program's output"), triggered=self.show_log) self.datatree = MemoryProfilerDataTree(self) self.collapse_button = create_toolbutton( self, icon=get_icon('collapse.png'), triggered=lambda dD=-1: self.datatree.collapseAll(), tip=_('Collapse all')) self.expand_button = create_toolbutton( self, icon=get_icon('expand.png'), triggered=lambda dD=1: self.datatree.expandAll(), tip=_('Expand all')) hlayout1 = QHBoxLayout() hlayout1.addWidget(self.filecombo) hlayout1.addWidget(browse_button) hlayout1.addWidget(self.start_button) hlayout1.addWidget(self.stop_button) hlayout2 = QHBoxLayout() hlayout2.addWidget(self.collapse_button) hlayout2.addWidget(self.expand_button) hlayout2.addStretch() hlayout2.addWidget(self.datelabel) hlayout2.addStretch() hlayout2.addWidget(self.log_button) layout = QVBoxLayout() layout.addLayout(hlayout1) layout.addLayout(hlayout2) layout.addWidget(self.datatree) self.setLayout(layout) self.process = None self.set_running_state(False) self.start_button.setEnabled(False) if not is_memoryprofiler_installed(): for widget in (self.datatree, self.filecombo, self.log_button, self.start_button, self.stop_button, browse_button, self.collapse_button, self.expand_button): widget.setDisabled(True) text = _( '<b>Please install the <a href="%s">memory_profiler module</a></b>' ) % WEBSITE_URL self.datelabel.setText(text) self.datelabel.setOpenExternalLinks(True) else: pass # self.show_data() def analyze(self, filename, wdir=None, args=None, pythonpath=None, use_colors=True): self.use_colors = use_colors if not is_memoryprofiler_installed(): return self.kill_if_running() #index, _data = self.get_data(filename) index = None # FIXME: storing data is not implemented yet if index is None: self.filecombo.addItem(filename) self.filecombo.setCurrentIndex(self.filecombo.count() - 1) else: self.filecombo.setCurrentIndex(self.filecombo.findText(filename)) self.filecombo.selected() if self.filecombo.is_valid(): if wdir is None: wdir = osp.dirname(filename) self.start(wdir, args, pythonpath) def select_file(self): self.redirect_stdio.emit(False) filename, _selfilter = getopenfilename( self, _("Select Python script"), getcwd(), _("Python scripts") + " (*.py ; *.pyw)") self.redirect_stdio.emit(False) if filename: self.analyze(filename) def show_log(self): if self.output: TextEditor(self.output, title=_("Memory profiler output"), readonly=True, size=(700, 500)).exec_() def show_errorlog(self): if self.error_output: TextEditor(self.error_output, title=_("Memory profiler output"), readonly=True, size=(700, 500)).exec_() def start(self, wdir=None, args=None, pythonpath=None): filename = to_text_string(self.filecombo.currentText()) if wdir is None: wdir = self._last_wdir if wdir is None: wdir = osp.basename(filename) if args is None: args = self._last_args if args is None: args = [] if pythonpath is None: pythonpath = self._last_pythonpath self._last_wdir = wdir self._last_args = args self._last_pythonpath = pythonpath self.datelabel.setText(_('Profiling, please wait...')) self.process = QProcess(self) self.process.setProcessChannelMode(QProcess.SeparateChannels) self.process.setWorkingDirectory(wdir) self.process.readyReadStandardOutput.connect(self.read_output) self.process.readyReadStandardError.connect( lambda: self.read_output(error=True)) self.process.finished.connect(self.finished) self.stop_button.clicked.connect(self.process.kill) if pythonpath is not None: env = [ to_text_string(_pth) for _pth in self.process.systemEnvironment() ] add_pathlist_to_PYTHONPATH(env, pythonpath) processEnvironment = QProcessEnvironment() for envItem in env: envName, separator, envValue = envItem.partition('=') processEnvironment.insert(envName, envValue) self.process.setProcessEnvironment(processEnvironment) self.output = '' self.error_output = '' # remove previous results, since memory_profiler appends to output file # instead of replacing if osp.isfile(self.DATAPATH): os.remove(self.DATAPATH) if os.name == 'nt': # On Windows, one has to replace backslashes by slashes to avoid # confusion with escape characters (otherwise, for example, '\t' # will be interpreted as a tabulation): filename = osp.normpath(filename).replace(os.sep, '/') p_args = [ '-m', 'memory_profiler', '-o', '"' + self.DATAPATH + '"', '"' + filename + '"' ] if args: p_args.extend(programs.shell_split(args)) executable = get_python_executable() executable += ' ' + ' '.join(p_args) executable = executable.replace(os.sep, '/') self.process.start(executable) else: p_args = ['-m', 'memory_profiler', '-o', self.DATAPATH, filename] if args: p_args.extend(programs.shell_split(args)) executable = get_python_executable() self.process.start(executable, p_args) running = self.process.waitForStarted() self.set_running_state(running) if not running: QMessageBox.critical(self, _("Error"), _("Process failed to start")) def set_running_state(self, state=True): self.start_button.setEnabled(not state) self.stop_button.setEnabled(state) def read_output(self, error=False): if error: self.process.setReadChannel(QProcess.StandardError) else: self.process.setReadChannel(QProcess.StandardOutput) qba = QByteArray() while self.process.bytesAvailable(): if error: qba += self.process.readAllStandardError() else: qba += self.process.readAllStandardOutput() text = to_text_string(locale_codec.toUnicode(qba.data())) if error: self.error_output += text else: self.output += text def finished(self): self.set_running_state(False) self.show_errorlog() # If errors occurred, show them. self.output = self.error_output + self.output # FIXME: figure out if show_data should be called here or # as a signal from the combobox self.show_data(justanalyzed=True) self.sig_finished.emit() def kill_if_running(self): if self.process is not None: if self.process.state() == QProcess.Running: self.process.kill() self.process.waitForFinished() def show_data(self, justanalyzed=False): if not justanalyzed: self.output = None self.log_button.setEnabled(self.output is not None and len(self.output) > 0) self.kill_if_running() filename = to_text_string(self.filecombo.currentText()) if not filename: return self.datatree.load_data(self.DATAPATH) self.datelabel.setText(_('Sorting data, please wait...')) QApplication.processEvents() self.datatree.show_tree() text_style = "<span style=\'color: #444444\'><b>%s </b></span>" date_text = text_style % time.strftime("%d %b %Y %H:%M", time.localtime()) self.datelabel.setText(date_text)
class ProfilerWidget(QWidget): """ Profiler widget """ DATAPATH = get_conf_path('profiler.results') VERSION = '0.0.1' redirect_stdio = Signal(bool) def __init__(self, parent, max_entries=100): QWidget.__init__(self, parent) self.setWindowTitle("Profiler") self.output = None self.error_output = None self._last_wdir = None self._last_args = None self._last_pythonpath = None self.filecombo = PythonModulesComboBox(self) self.start_button = create_toolbutton(self, icon=ima.icon('run'), text=_("Profile"), tip=_("Run profiler"), triggered=lambda: self.start(), text_beside_icon=True) self.stop_button = create_toolbutton(self, icon=ima.icon('stop'), text=_("Stop"), tip=_("Stop current profiling"), text_beside_icon=True) self.filecombo.valid.connect(self.start_button.setEnabled) #self.connect(self.filecombo, SIGNAL('valid(bool)'), self.show_data) # FIXME: The combobox emits this signal on almost any event # triggering show_data() too early, too often. browse_button = create_toolbutton(self, icon=ima.icon('fileopen'), tip=_('Select Python script'), triggered=self.select_file) self.datelabel = QLabel() self.log_button = create_toolbutton(self, icon=ima.icon('log'), text=_("Output"), text_beside_icon=True, tip=_("Show program's output"), triggered=self.show_log) self.datatree = ProfilerDataTree(self) self.collapse_button = create_toolbutton( self, icon=ima.icon('collapse'), triggered=lambda dD: self.datatree.change_view(-1), tip=_('Collapse one level up')) self.expand_button = create_toolbutton( self, icon=ima.icon('expand'), triggered=lambda dD: self.datatree.change_view(1), tip=_('Expand one level down')) self.save_button = create_toolbutton(self, text_beside_icon=True, text=_("Save data"), icon=ima.icon('filesave'), triggered=self.save_data, tip=_('Save profiling data')) self.load_button = create_toolbutton( self, text_beside_icon=True, text=_("Load data"), icon=ima.icon('fileimport'), triggered=self.compare, tip=_('Load profiling data for comparison')) self.clear_button = create_toolbutton(self, text_beside_icon=True, text=_("Clear comparison"), icon=ima.icon('editdelete'), triggered=self.clear) hlayout1 = QHBoxLayout() hlayout1.addWidget(self.filecombo) hlayout1.addWidget(browse_button) hlayout1.addWidget(self.start_button) hlayout1.addWidget(self.stop_button) hlayout2 = QHBoxLayout() hlayout2.addWidget(self.collapse_button) hlayout2.addWidget(self.expand_button) hlayout2.addStretch() hlayout2.addWidget(self.datelabel) hlayout2.addStretch() hlayout2.addWidget(self.log_button) hlayout2.addWidget(self.save_button) hlayout2.addWidget(self.load_button) hlayout2.addWidget(self.clear_button) layout = QVBoxLayout() layout.addLayout(hlayout1) layout.addLayout(hlayout2) layout.addWidget(self.datatree) self.setLayout(layout) self.process = None self.set_running_state(False) self.start_button.setEnabled(False) self.clear_button.setEnabled(False) if not is_profiler_installed(): # This should happen only on certain GNU/Linux distributions # or when this a home-made Python build because the Python # profilers are included in the Python standard library for widget in (self.datatree, self.filecombo, self.start_button, self.stop_button): widget.setDisabled(True) url = 'http://docs.python.org/library/profile.html' text = '%s <a href=%s>%s</a>' % (_('Please install'), url, _("the Python profiler modules")) self.datelabel.setText(text) else: pass # self.show_data() def save_data(self): """Save data""" title = _("Save profiler result") filename, _selfilter = getsavefilename( self, title, getcwd_or_home(), _("Profiler result") + " (*.Result)") if filename: self.datatree.save_data(filename) def compare(self): filename, _selfilter = getopenfilename( self, _("Select script to compare"), getcwd_or_home(), _("Profiler result") + " (*.Result)") if filename: self.datatree.compare(filename) self.show_data() self.clear_button.setEnabled(True) def clear(self): self.datatree.compare(None) self.datatree.hide_diff_cols(True) self.show_data() self.clear_button.setEnabled(False) def analyze(self, filename, wdir=None, args=None, pythonpath=None): if not is_profiler_installed(): return self.kill_if_running() #index, _data = self.get_data(filename) index = None # FIXME: storing data is not implemented yet if index is None: self.filecombo.addItem(filename) self.filecombo.setCurrentIndex(self.filecombo.count() - 1) else: self.filecombo.setCurrentIndex(self.filecombo.findText(filename)) self.filecombo.selected() if self.filecombo.is_valid(): if wdir is None: wdir = osp.dirname(filename) self.start(wdir, args, pythonpath) def select_file(self): self.redirect_stdio.emit(False) filename, _selfilter = getopenfilename( self, _("Select Python script"), getcwd_or_home(), _("Python scripts") + " (*.py ; *.pyw)") self.redirect_stdio.emit(True) if filename: self.analyze(filename) def show_log(self): if self.output: TextEditor(self.output, title=_("Profiler output"), readonly=True, size=(700, 500)).exec_() def show_errorlog(self): if self.error_output: TextEditor(self.error_output, title=_("Profiler output"), readonly=True, size=(700, 500)).exec_() def start(self, wdir=None, args=None, pythonpath=None): filename = to_text_string(self.filecombo.currentText()) if wdir is None: wdir = self._last_wdir if wdir is None: wdir = osp.basename(filename) if args is None: args = self._last_args if args is None: args = [] if pythonpath is None: pythonpath = self._last_pythonpath self._last_wdir = wdir self._last_args = args self._last_pythonpath = pythonpath self.datelabel.setText(_('Profiling, please wait...')) self.process = QProcess(self) self.process.setProcessChannelMode(QProcess.SeparateChannels) self.process.setWorkingDirectory(wdir) self.process.readyReadStandardOutput.connect(self.read_output) self.process.readyReadStandardError.connect( lambda: self.read_output(error=True)) self.process.finished.connect( lambda ec, es=QProcess.ExitStatus: self.finished(ec, es)) self.stop_button.clicked.connect(self.process.kill) if pythonpath is not None: env = [ to_text_string(_pth) for _pth in self.process.systemEnvironment() ] add_pathlist_to_PYTHONPATH(env, pythonpath) processEnvironment = QProcessEnvironment() for envItem in env: envName, separator, envValue = envItem.partition('=') processEnvironment.insert(envName, envValue) self.process.setProcessEnvironment(processEnvironment) self.output = '' self.error_output = '' p_args = ['-m', 'cProfile', '-o', self.DATAPATH] if os.name == 'nt': # On Windows, one has to replace backslashes by slashes to avoid # confusion with escape characters (otherwise, for example, '\t' # will be interpreted as a tabulation): p_args.append(osp.normpath(filename).replace(os.sep, '/')) else: p_args.append(filename) if args: p_args.extend(shell_split(args)) executable = sys.executable if executable.endswith("spyder.exe"): # py2exe distribution executable = "python.exe" self.process.start(executable, p_args) running = self.process.waitForStarted() self.set_running_state(running) if not running: QMessageBox.critical(self, _("Error"), _("Process failed to start")) def set_running_state(self, state=True): self.start_button.setEnabled(not state) self.stop_button.setEnabled(state) def read_output(self, error=False): if error: self.process.setReadChannel(QProcess.StandardError) else: self.process.setReadChannel(QProcess.StandardOutput) qba = QByteArray() while self.process.bytesAvailable(): if error: qba += self.process.readAllStandardError() else: qba += self.process.readAllStandardOutput() text = to_text_string(locale_codec.toUnicode(qba.data())) if error: self.error_output += text else: self.output += text def finished(self, exit_code, exit_status): self.set_running_state(False) self.show_errorlog() # If errors occurred, show them. self.output = self.error_output + self.output # FIXME: figure out if show_data should be called here or # as a signal from the combobox self.show_data(justanalyzed=True) def kill_if_running(self): if self.process is not None: if self.process.state() == QProcess.Running: self.process.kill() self.process.waitForFinished() def show_data(self, justanalyzed=False): if not justanalyzed: self.output = None self.log_button.setEnabled(self.output is not None \ and len(self.output) > 0) self.kill_if_running() filename = to_text_string(self.filecombo.currentText()) if not filename: return self.datelabel.setText(_('Sorting data, please wait...')) QApplication.processEvents() self.datatree.load_data(self.DATAPATH) self.datatree.show_tree() text_style = "<span style=\'color: #444444\'><b>%s </b></span>" date_text = text_style % time.strftime("%d %b %Y %H:%M", time.localtime()) self.datelabel.setText(date_text)
class PylintWidget(QWidget): """ Pylint widget """ DATAPATH = get_conf_path('pylint.results') VERSION = '1.1.0' redirect_stdio = Signal(bool) start_analysis = Signal() def __init__(self, parent, max_entries=100, options_button=None, text_color=None, prevrate_color=None): super().__init__(parent) self.setWindowTitle("Pylint") self.output = None self.error_output = None self.filename = None self.text_color = text_color self.prevrate_color = prevrate_color self.max_entries = max_entries self.rdata = [] if osp.isfile(self.DATAPATH): try: data = pickle.loads(open(self.DATAPATH, 'rb').read()) if data[0] == self.VERSION: self.rdata = data[1:] except (EOFError, ImportError): pass self.filecombo = PythonModulesComboBox(self) self.start_button = create_toolbutton( self, icon=ima.icon('run'), text=_("Analyze"), tip=_("Run analysis"), triggered=self.analyze_button_handler, text_beside_icon=True) self.stop_button = create_toolbutton(self, icon=ima.icon('stop'), text=_("Stop"), tip=_("Stop current analysis"), text_beside_icon=True) self.filecombo.valid.connect(self.start_button.setEnabled) self.filecombo.valid.connect(self.check_new_file) browse_button = create_toolbutton(self, icon=ima.icon('fileopen'), tip=_('Select Python file'), triggered=self.select_file) self.ratelabel = QLabel() self.datelabel = QLabel() self.log_button = create_toolbutton(self, icon=ima.icon('log'), text=_("Output"), text_beside_icon=True, tip=_("Complete output"), triggered=self.show_log) self.treewidget = ResultsTree(self) hlayout1 = QHBoxLayout() hlayout1.addWidget(self.filecombo) hlayout1.addWidget(browse_button) hlayout1.addWidget(self.start_button) hlayout1.addWidget(self.stop_button) if options_button: hlayout1.addWidget(options_button) hlayout2 = QHBoxLayout() hlayout2.addWidget(self.ratelabel) hlayout2.addStretch() hlayout2.addWidget(self.datelabel) hlayout2.addStretch() hlayout2.addWidget(self.log_button) layout = QVBoxLayout() layout.addLayout(hlayout1) layout.addLayout(hlayout2) layout.addWidget(self.treewidget) self.setLayout(layout) self.process = None self.set_running_state(False) self.show_data() if self.rdata: self.remove_obsolete_items() self.filecombo.addItems(self.get_filenames()) self.start_button.setEnabled(self.filecombo.is_valid()) else: self.start_button.setEnabled(False) def check_new_file(self): fname = self.get_filename() if fname != self.filename: self.filename = fname self.show_data() def get_filename(self): """Get current filename in combobox.""" return self.filecombo.currentText() @Slot(str) def set_filename(self, filename): """Set filename without performing code analysis.""" filename = str(filename) # filename is a QString instance self.kill_if_running() index, _data = self.get_data(filename) if index is None: self.filecombo.addItem(filename) self.filecombo.setCurrentIndex(self.filecombo.count() - 1) else: self.filecombo.setCurrentIndex(self.filecombo.findText(filename)) self.filecombo.selected() def analyze(self, filename=None): """ Perform code analysis for given `filename`. If `filename` is None default to current filename in combobox. """ if filename is not None: self.set_filename(filename) if self.filecombo.is_valid(): self.start() @Slot() def select_file(self): self.redirect_stdio.emit(False) filename, _selfilter = getopenfilename( self, _("Select Python file"), getcwd_or_home(), _("Python files") + " (*.py ; *.pyw)") self.redirect_stdio.emit(True) if filename: self.analyze(filename) def remove_obsolete_items(self): """Removing obsolete items""" self.rdata = [(filename, data) for filename, data in self.rdata if is_module_or_package(filename)] def get_filenames(self): return [filename for filename, _data in self.rdata] def get_data(self, filename): filename = osp.abspath(filename) for index, (fname, data) in enumerate(self.rdata): if fname == filename: return index, data else: return None, None def set_data(self, filename, data): filename = osp.abspath(filename) index, _data = self.get_data(filename) if index is not None: self.rdata.pop(index) self.rdata.insert(0, (filename, data)) self.save() def save(self): while len(self.rdata) > self.max_entries: self.rdata.pop(-1) pickle.dump([self.VERSION] + self.rdata, open(self.DATAPATH, 'wb'), 2) @Slot() def show_log(self): if self.output: output_dialog = TextEditor(self.output, title=_("Pylint output"), parent=self, readonly=True) output_dialog.resize(700, 500) output_dialog.exec_() @Slot() def analyze_button_handler(self): """Try to start code analysis when Analyze button pressed.""" self.start_analysis.emit() def get_pylintrc_path(self, filename): """Get the path to the most proximate pylintrc config to the file.""" parent = self.parentWidget() if parent is not None: project_dir = parent.main.projects.get_active_project_path() else: project_dir = None search_paths = [ osp.dirname(filename), # File's directory getcwd_or_home(), # Working directory project_dir, # Project directory osp.expanduser("~"), # Home directory ] return get_pylintrc_path(search_paths=search_paths) @Slot() def start(self): """Start the code analysis.""" filename = str(self.filecombo.currentText()) self.process = QProcess(self) self.process.setProcessChannelMode(QProcess.SeparateChannels) self.process.setWorkingDirectory(getcwd_or_home()) self.process.readyReadStandardOutput.connect(self.read_output) self.process.readyReadStandardError.connect( lambda: self.read_output(error=True)) self.process.finished.connect( lambda ec, es=QProcess.ExitStatus: self.finished(ec, es)) self.stop_button.clicked.connect(self.process.kill) self.output = '' self.error_output = '' if PYLINT_VER is not None: pylint_args = [ '-m', 'pylint', '--output-format=text', '--msg-template=' "'{msg_id}:{symbol}:{line:3d},{column}: {msg}'" ] pylintrc_path = self.get_pylintrc_path(filename=filename) if pylintrc_path is not None: pylint_args += ['--rcfile={}'.format(pylintrc_path)] pylint_args.append(filename) processEnvironment = QProcessEnvironment() processEnvironment.insert("PYTHONIOENCODING", "utf8") self.process.setProcessEnvironment(processEnvironment) self.process.start(sys.executable, pylint_args) running = self.process.waitForStarted() self.set_running_state(running) if not running: QMessageBox.critical(self, _("Error"), _("Process failed to start")) def set_running_state(self, state=True): self.start_button.setEnabled(not state) self.stop_button.setEnabled(state) def read_output(self, error=False): if error: self.process.setReadChannel(QProcess.StandardError) else: self.process.setReadChannel(QProcess.StandardOutput) qba = QByteArray() while self.process.bytesAvailable(): if error: qba += self.process.readAllStandardError() else: qba += self.process.readAllStandardOutput() text = str(qba.data(), 'utf-8') if error: self.error_output += text else: self.output += text def finished(self, exit_code, exit_status): self.set_running_state(False) if not self.output: if self.error_output: QMessageBox.critical(self, _("Error"), self.error_output) print("pylint error:\n\n" + self.error_output, file=sys.stderr) return # Convention, Refactor, Warning, Error results = {'C:': [], 'R:': [], 'W:': [], 'E:': []} txt_module = '************* Module ' module = '' # Should not be needed - just in case something goes wrong for line in self.output.splitlines(): if line.startswith(txt_module): # New module module = line[len(txt_module):] continue # Supporting option include-ids: ('R3873:' instead of 'R:') if not re.match(r'^[CRWE]+([0-9]{4})?:', line): continue items = {} idx_0 = 0 idx_1 = 0 key_names = ["msg_id", "message_name", "line_nb", "message"] for key_idx, key_name in enumerate(key_names): if key_idx == len(key_names) - 1: idx_1 = len(line) else: idx_1 = line.find(":", idx_0) if idx_1 < 0: break item = line[(idx_0):idx_1] if not item: break if key_name == "line_nb": item = int(item.split(',')[0]) items[key_name] = item idx_0 = idx_1 + 1 else: pylint_item = (module, items["line_nb"], items["message"], items["msg_id"], items["message_name"]) results[line[0] + ':'].append(pylint_item) # Rate rate = None txt_rate = 'Your code has been rated at ' i_rate = self.output.find(txt_rate) if i_rate > 0: i_rate_end = self.output.find('/10', i_rate) if i_rate_end > 0: rate = self.output[i_rate + len(txt_rate):i_rate_end] # Previous run previous = '' if rate is not None: txt_prun = 'previous run: ' i_prun = self.output.find(txt_prun, i_rate_end) if i_prun > 0: i_prun_end = self.output.find('/10', i_prun) previous = self.output[i_prun + len(txt_prun):i_prun_end] filename = str(self.filecombo.currentText()) self.set_data(filename, (time.localtime(), rate, previous, results)) self.output = self.error_output + self.output self.show_data(justanalyzed=True) def kill_if_running(self): if self.process is not None: if self.process.state() == QProcess.Running: self.process.kill() self.process.waitForFinished() def show_data(self, justanalyzed=False): if not justanalyzed: self.output = None self.log_button.setEnabled(self.output is not None and len(self.output) > 0) self.kill_if_running() filename = str(self.filecombo.currentText()) if not filename: return _index, data = self.get_data(filename) if data is None: text = _('Source code has not been rated yet.') self.treewidget.clear_results() date_text = '' else: datetime, rate, previous_rate, results = data if rate is None: text = _('Analysis did not succeed ' '(see output for more details).') self.treewidget.clear_results() date_text = '' else: text_style = "<span style=\'color: %s\'><b>%s </b></span>" rate_style = "<span style=\'color: %s\'><b>%s</b></span>" prevrate_style = "<span style=\'color: %s\'>%s</span>" color = "#FF0000" if float(rate) > 5.: color = "#22AA22" elif float(rate) > 3.: color = "#EE5500" text = _('Global evaluation:') text = ((text_style % (self.text_color, text)) + (rate_style % (color, ('%s/10' % rate)))) if previous_rate: text_prun = _('previous run:') text_prun = ' (%s %s/10)' % (text_prun, previous_rate) text += prevrate_style % (self.prevrate_color, text_prun) self.treewidget.set_results(filename, results) date = time.strftime("%Y-%m-%d %H:%M:%S", datetime) date_text = text_style % (self.text_color, date) self.ratelabel.setText(text) self.datelabel.setText(date_text)
class LineProfilerWidget(QWidget): """ Line profiler widget. """ DATAPATH = get_conf_path('lineprofiler.results') VERSION = '0.0.1' redirect_stdio = Signal(bool) def __init__(self, parent): QWidget.__init__(self, parent) self.setWindowTitle("Line profiler") self.output = None self.error_output = None self.use_colors = True self._last_wdir = None self._last_args = None self._last_pythonpath = None self.filecombo = PythonModulesComboBox(self) self.start_button = create_toolbutton( self, icon=get_icon('run.png'), text=_("Profile by line"), tip=_("Run line profiler"), triggered=self.start, text_beside_icon=True) self.stop_button = create_toolbutton( self, icon=get_icon('terminate.png'), text=_("Stop"), tip=_("Stop current profiling"), text_beside_icon=True) self.filecombo.valid.connect(self.start_button.setEnabled) #self.filecombo.valid.connect(self.show_data) # FIXME: The combobox emits this signal on almost any event # triggering show_data() too early, too often. browse_button = create_toolbutton( self, icon=get_icon('fileopen.png'), tip=_('Select Python script'), triggered=self.select_file) self.datelabel = QLabel() self.log_button = create_toolbutton( self, icon=get_icon('log.png'), text=_("Output"), text_beside_icon=True, tip=_("Show program's output"), triggered=self.show_log) self.datatree = LineProfilerDataTree(self) self.collapse_button = create_toolbutton( self, icon=get_icon('collapse.png'), triggered=lambda dD=-1: self.datatree.collapseAll(), tip=_('Collapse all')) self.expand_button = create_toolbutton( self, icon=get_icon('expand.png'), triggered=lambda dD=1: self.datatree.expandAll(), tip=_('Expand all')) hlayout1 = QHBoxLayout() hlayout1.addWidget(self.filecombo) hlayout1.addWidget(browse_button) hlayout1.addWidget(self.start_button) hlayout1.addWidget(self.stop_button) hlayout2 = QHBoxLayout() hlayout2.addWidget(self.collapse_button) hlayout2.addWidget(self.expand_button) hlayout2.addStretch() hlayout2.addWidget(self.datelabel) hlayout2.addStretch() hlayout2.addWidget(self.log_button) layout = QVBoxLayout() layout.addLayout(hlayout1) layout.addLayout(hlayout2) layout.addWidget(self.datatree) self.setLayout(layout) self.process = None self.set_running_state(False) self.start_button.setEnabled(False) if not is_lineprofiler_installed(): for widget in (self.datatree, self.filecombo, self.log_button, self.start_button, self.stop_button, browse_button, self.collapse_button, self.expand_button): widget.setDisabled(True) text = _( '<b>Please install the <a href="%s">line_profiler module</a></b>' ) % WEBSITE_URL self.datelabel.setText(text) self.datelabel.setOpenExternalLinks(True) else: pass # self.show_data() def analyze(self, filename, wdir=None, args=None, pythonpath=None, use_colors=True): self.use_colors = use_colors if not is_lineprofiler_installed(): return self.kill_if_running() #index, _data = self.get_data(filename) index = None # FIXME: storing data is not implemented yet if index is None: self.filecombo.addItem(filename) self.filecombo.setCurrentIndex(self.filecombo.count()-1) else: self.filecombo.setCurrentIndex(self.filecombo.findText(filename)) self.filecombo.selected() if self.filecombo.is_valid(): if wdir is None: wdir = osp.dirname(filename) self.start(wdir, args, pythonpath) def select_file(self): self.redirect_stdio.emit(False) filename, _selfilter = getopenfilename( self, _("Select Python script"), getcwd(), _("Python scripts")+" (*.py ; *.pyw)") self.redirect_stdio.emit(False) if filename: self.analyze(filename) def show_log(self): if self.output: TextEditor(self.output, title=_("Line profiler output"), readonly=True, size=(700, 500)).exec_() def show_errorlog(self): if self.error_output: TextEditor(self.error_output, title=_("Line profiler output"), readonly=True, size=(700, 500)).exec_() def start(self, wdir=None, args=None, pythonpath=None): filename = to_text_string(self.filecombo.currentText()) if wdir is None: wdir = self._last_wdir if wdir is None: wdir = osp.basename(filename) if args is None: args = self._last_args if args is None: args = [] if pythonpath is None: pythonpath = self._last_pythonpath self._last_wdir = wdir self._last_args = args self._last_pythonpath = pythonpath self.datelabel.setText(_('Profiling, please wait...')) self.process = QProcess(self) self.process.setProcessChannelMode(QProcess.SeparateChannels) self.process.setWorkingDirectory(wdir) self.process.readyReadStandardOutput.connect(self.read_output) self.process.readyReadStandardError.connect( lambda: self.read_output(error=True)) self.process.finished.connect(self.finished) self.stop_button.clicked.connect(self.process.kill) if pythonpath is not None: env = [to_text_string(_pth) for _pth in self.process.systemEnvironment()] add_pathlist_to_PYTHONPATH(env, pythonpath) processEnvironment = QProcessEnvironment() for envItem in env: envName, separator, envValue = envItem.partition('=') processEnvironment.insert(envName, envValue) self.process.setProcessEnvironment(processEnvironment) self.output = '' self.error_output = '' if os.name == 'nt': # On Windows, one has to replace backslashes by slashes to avoid # confusion with escape characters (otherwise, for example, '\t' # will be interpreted as a tabulation): filename = osp.normpath(filename).replace(os.sep, '/') p_args = ['-lvb', '-o', '"' + self.DATAPATH + '"', '"' + filename + '"'] if args: p_args.extend(programs.shell_split(args)) executable = '"' + programs.find_program('kernprof') + '"' executable += ' ' + ' '.join(p_args) executable = executable.replace(os.sep, '/') self.process.start(executable) else: p_args = ['-lvb', '-o', self.DATAPATH, filename] if args: p_args.extend(programs.shell_split(args)) executable = 'kernprof' self.process.start(executable, p_args) running = self.process.waitForStarted() self.set_running_state(running) if not running: QMessageBox.critical(self, _("Error"), _("Process failed to start")) def set_running_state(self, state=True): self.start_button.setEnabled(not state) self.stop_button.setEnabled(state) def read_output(self, error=False): if error: self.process.setReadChannel(QProcess.StandardError) else: self.process.setReadChannel(QProcess.StandardOutput) qba = QByteArray() while self.process.bytesAvailable(): if error: qba += self.process.readAllStandardError() else: qba += self.process.readAllStandardOutput() text = to_text_string(locale_codec.toUnicode(qba.data())) if error: self.error_output += text else: self.output += text def finished(self): self.set_running_state(False) self.show_errorlog() # If errors occurred, show them. self.output = self.error_output + self.output # FIXME: figure out if show_data should be called here or # as a signal from the combobox self.show_data(justanalyzed=True) def kill_if_running(self): if self.process is not None: if self.process.state() == QProcess.Running: self.process.kill() self.process.waitForFinished() def show_data(self, justanalyzed=False): if not justanalyzed: self.output = None self.log_button.setEnabled( self.output is not None and len(self.output) > 0) self.kill_if_running() filename = to_text_string(self.filecombo.currentText()) if not filename: return self.datatree.load_data(self.DATAPATH) self.datelabel.setText(_('Sorting data, please wait...')) QApplication.processEvents() self.datatree.show_tree() text_style = "<span style=\'color: #444444\'><b>%s </b></span>" date_text = text_style % time.strftime("%d %b %Y %H:%M", time.localtime()) self.datelabel.setText(date_text)
class TestRunner(QObject): """ Class for running tests with py.test or nose. All communication back to the caller is done via signals. Attributes ---------- process : QProcess or None Process running the unit test suite. resultfilename : str Name of file in which test results are stored. Signals ------- sig_finished(list of TestResult, str) Emitted when test process finishes. First argument contains the test results, second argument contains the output of the test process. """ sig_finished = Signal(object, str) def __init__(self, widget, resultfilename=None): """ Construct test runner. Parameters ---------- widget : UnitTestWidget Unit test widget which constructs the test runner. resultfilename : str or None Name of file in which to store test results. If None, use default. """ QObject.__init__(self, widget) self.process = None if resultfilename is None: self.resultfilename = os.path.join(tempfile.gettempdir(), 'unittest.results') else: self.resultfilename = resultfilename def start(self, config, pythonpath): """ Start process which will run the unit test suite. The process is run in the working directory specified in 'config', with the directories in `pythonpath` added to the Python path for the test process. The test results are written to the file `self.resultfilename`. The standard output and error are also recorded. Once the process is finished, `self.finished()` will be called. Parameters ---------- config : TestConfig Unit test configuration. pythonpath : list of str List of directories to be added to the Python path Raises ------ RuntimeError If process failed to start. """ framework = config.framework wdir = config.wdir self.process = QProcess(self) self.process.setProcessChannelMode(QProcess.MergedChannels) self.process.setWorkingDirectory(wdir) self.process.finished.connect(self.finished) if pythonpath is not None: env = [ to_text_string(_pth) for _pth in self.process.systemEnvironment() ] add_pathlist_to_PYTHONPATH(env, pythonpath) processEnvironment = QProcessEnvironment() for envItem in env: envName, separator, envValue = envItem.partition('=') processEnvironment.insert(envName, envValue) self.process.setProcessEnvironment(processEnvironment) if framework == 'nose': executable = 'nosetests' p_args = [ '--with-xunit', '--xunit-file={}'.format(self.resultfilename) ] elif framework == 'py.test': executable = 'py.test' p_args = ['--junit-xml', self.resultfilename] else: raise ValueError('Unknown framework') if os.name == 'nt': executable += '.exe' try: os.remove(self.resultfilename) except OSError: pass self.process.start(executable, p_args) running = self.process.waitForStarted() if not running: raise RuntimeError def finished(self): """ Called when the unit test process has finished. This function reads the results and emits `sig_finished`. """ qbytearray = self.process.readAllStandardOutput() locale_codec = QTextCodec.codecForLocale() output = to_text_string(locale_codec.toUnicode(qbytearray.data())) testresults = self.load_data() self.sig_finished.emit(testresults, output) def kill_if_running(self): """Kill testing process if it is running.""" if self.process and self.process.state() == QProcess.Running: self.process.kill() def load_data(self): """ Read and parse unit test results. This function reads the unit test results from the file with name `self.resultfilename` and parses them. The file should contain the test results in JUnitXML format. Returns ------- list of TestResult Unit test results. """ try: data = etree.parse(self.resultfilename).getroot() except OSError: data = [] testresults = [] for testcase in data: category = Category.OK status = 'ok' name = '{0}.{1}'.format(testcase.get('classname'), testcase.get('name')) message = '' time = float(testcase.get('time')) extras = [] for child in testcase: if child.tag in ('error', 'failure', 'skipped'): if child.tag == 'skipped': category = Category.SKIP else: category = Category.FAIL status = child.tag type_ = child.get('type') message = child.get('message', default='') if type_ and message: message = '{0}: {1}'.format(type_, message) elif type_: message = type_ if child.text: extras.append(child.text) elif child.tag in ('system-out', 'system-err'): if child.tag == 'system-out': heading = _('Captured stdout') else: heading = _('Captured stderr') contents = child.text.rstrip('\n') extras.append('----- {} -----\n{}'.format( heading, contents)) extra_text = '\n\n'.join(extras) testresults.append( TestResult(category, status, name, message, time, extra_text)) return testresults
class CondaProcess(QObject): """Conda API modified to work with QProcess instead of popen.""" # Signals sig_finished = Signal(str, object, str) sig_partial = Signal(str, object, str) sig_started = Signal() ENCODING = "ascii" ROOT_PREFIX = None def __init__(self, parent): QObject.__init__(self, parent) self._parent = parent self._output = None self._partial = None self._stdout = None self._error = None self._parse = False self._function_called = "" self._name = None self._process = QProcess() self.set_root_prefix() # Signals self._process.finished.connect(self._call_conda_ready) self._process.readyReadStandardOutput.connect(self._call_conda_partial) # --- Helpers # ------------------------------------------------------------------------- def _is_running(self): return self._process.state() != QProcess.NotRunning def _is_not_running(self): return self._process.state() == QProcess.NotRunning def _call_conda_partial(self): """ """ stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(stdout, CondaProcess.ENCODING) stderr = self._process.readAllStandardError() stderr = handle_qbytearray(stderr, CondaProcess.ENCODING) if self._parse: try: self._output = json.loads(stdout) except Exception: # Result is a partial json. Can only be parsed when finished self._output = stdout else: self._output = stdout self._partial = self._output self._stdout = self._output self._error = stderr self.sig_partial.emit(self._function_called, self._partial, self._error) def _call_conda_ready(self): """Function called when QProcess in _call_conda finishes task.""" function = self._function_called if self._stdout is None: stdout = to_text_string(self._process.readAllStandardOutput(), encoding=CondaProcess.ENCODING) else: stdout = self._stdout if self._error is None: stderr = to_text_string(self._process.readAllStandardError(), encoding=CondaProcess.ENCODING) else: stderr = self._error if function == "get_conda_version": pat = re.compile(r"conda:?\s+(\d+\.\d\S+|unknown)") m = pat.match(stderr.strip()) if m is None: m = pat.match(stdout.strip()) if m is None: raise Exception("output did not match: {0}".format(stderr)) self._output = m.group(1) elif function == "config_path": result = self._output self._output = result["rc_path"] elif function == "config_get": result = self._output self._output = result["get"] elif ( function == "config_delete" or function == "config_add" or function == "config_set" or function == "config_remove" ): result = self._output self._output = result.get("warnings", []) elif function == "pip": result = [] lines = self._output.split("\n") for line in lines: if "<pip>" in line: temp = line.split()[:-1] + ["pip"] result.append("-".join(temp)) self._output = result if stderr.strip(): self._error = stderr self._parse = False self.sig_finished.emit(self._function_called, self._output, self._error) def _get_prefix_envname_helper(self, name, envs): """ """ if name == "root": return CondaProcess.ROOT_PREFIX for prefix in envs: if basename(prefix) == name: return prefix return None def _abspath(self, abspath): """ """ if abspath: if sys.platform == "win32": python = join(CondaProcess.ROOT_PREFIX, "python.exe") conda = join(CondaProcess.ROOT_PREFIX, "Scripts", "conda-script.py") else: python = join(CondaProcess.ROOT_PREFIX, "bin/python") conda = join(CondaProcess.ROOT_PREFIX, "bin/conda") cmd_list = [python, conda] else: # Just use whatever conda is on the path cmd_list = ["conda"] return cmd_list def _call_conda(self, extra_args, abspath=True): """ """ if self._is_not_running(): cmd_list = self._abspath(abspath) cmd_list.extend(extra_args) self._error, self._output = None, None self._process.start(cmd_list[0], cmd_list[1:]) self.sig_started.emit() def _call_and_parse(self, extra_args, abspath=True): """ """ self._parse = True self._call_conda(extra_args, abspath=abspath) def _setup_install_commands_from_kwargs(self, kwargs, keys=tuple()): """ """ cmd_list = [] if kwargs.get("override_channels", False) and "channel" not in kwargs: raise TypeError("conda search: override_channels requires channel") if "env" in kwargs: cmd_list.extend(["--name", kwargs.pop("env")]) if "prefix" in kwargs: cmd_list.extend(["--prefix", kwargs.pop("prefix")]) if "channel" in kwargs: channel = kwargs.pop("channel") if isinstance(channel, str): cmd_list.extend(["--channel", channel]) else: cmd_list.append("--channel") cmd_list.extend(channel) for key in keys: if key in kwargs and kwargs[key]: cmd_list.append("--" + key.replace("_", "-")) return cmd_list # --- Public api # ------------------------------------------------------------------------ def linked(self, prefix): """ Return the (set of canonical names) of linked packages in `prefix`. """ if not isdir(prefix): raise Exception("no such directory: {0}".format(prefix)) meta_dir = join(prefix, "conda-meta") if not isdir(meta_dir): # we might have nothing in linked (and no conda-meta directory) result = set() result = set(fn[:-5] for fn in os.listdir(meta_dir) if fn.endswith(".json")) self.sig_finished.emit("linked", result, "") return result def split_canonical_name(self, cname): """ Split a canonical package name into (name, version, build) strings. """ result = tuple(cname.rsplit("-", 2)) self.sig_finished.emit("split_canonical_name", result, "") return result def set_root_prefix(self, prefix=None): """ Set the prefix to the root environment (default is /opt/anaconda). """ if prefix: CondaProcess.ROOT_PREFIX = prefix else: # Find conda instance, and then use info() to get 'root_prefix' if CondaProcess.ROOT_PREFIX is None: info = self.info(abspath=False) CondaProcess.ROOT_PREFIX = info["root_prefix"] def get_conda_version(self): """ Return the version of conda being used (invoked) as a string. """ if self._is_not_running(): self._function_called = "get_conda_version" self._call_conda(["--version"]) def get_envs(self): """ Return all of the (named) environment (this does not include the root environment), as a list of absolute path to their prefixes. """ if self._is_not_running(): info = self.info() result = info["envs"] self.sig_finished.emit("get_envs", result, "") return result def get_prefix_envname(self, name): """ Given the name of an environment return its full prefix path, or None if it cannot be found. """ if self._is_not_running(): if name == "root": pref = CondaProcess.ROOT_PREFIX for prefix in self.get_envs(): if basename(prefix) == name: pref = prefix break self.sig_finished.emit("get_prefix_envname", pref, "") return pref def info(self, abspath=True): """ Return a dictionary with configuration information. No guarantee is made about which keys exist. Therefore this function should only be used for testing and debugging. """ if self._is_not_running(): qprocess = QProcess() cmd_list = self._abspath(abspath) cmd_list.extend(["info", "--json"]) qprocess.start(cmd_list[0], cmd_list[1:]) qprocess.waitForFinished() output = qprocess.readAllStandardOutput() output = handle_qbytearray(output, CondaProcess.ENCODING) info = json.loads(output) self.sig_finished.emit("info", str(info), "") return info def package_info(self, package, abspath=True): """ Return a dictionary with package information. Structure is { 'package_name': [{ 'depends': list, 'version': str, 'name': str, 'license': str, 'fn': ..., ... }] } """ if self._is_not_running(): self._function_called = "package_info" self._call_and_parse(["info", package, "--json"], abspath=abspath) def search(self, regex=None, spec=None, **kwargs): """ Search for packages. """ cmd_list = ["search", "--json"] if regex and spec: raise TypeError("conda search: only one of regex or spec allowed") if regex: cmd_list.append(regex) if spec: cmd_list.extend(["--spec", spec]) if "platform" in kwargs: platform = kwargs.pop("platform") platforms = ("win-32", "win-64", "osx-64", "linux-32", "linux-64") if platform not in platforms: raise TypeError("conda search: platform must be one of " + ", ".join(platforms)) cmd_list.extend(["--platform", platform]) cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ("canonical", "unknown", "use_index_cache", "outdated", "override_channels") ) ) if self._is_not_running(): self._function_called = "search" self._call_and_parse(cmd_list, abspath=kwargs.get("abspath", True)) def share(self, prefix): """ Create a "share package" of the environment located in `prefix`, and return a dictionary with (at least) the following keys: - 'path': the full path to the created package - 'warnings': a list of warnings This file is created in a temp directory, and it is the callers responsibility to remove this directory (after the file has been handled in some way). """ if self._is_not_running: self._function_called = "share" self._call_and_parse(["share", "--json", "--prefix", prefix]) def create(self, name=None, path=None, pkgs=None): """ Create an environment either by name or path with a specified set of packages """ if not pkgs or not isinstance(pkgs, (list, tuple)): raise TypeError("must specify a list of one or more packages to " "install into new environment") cmd_list = ["create", "--yes", "--quiet"] if name: ref = name search = [os.path.join(d, name) for d in self.info()["envs_dirs"]] cmd_list = ["create", "--yes", "--quiet", "--name", name] elif path: ref = path search = [path] cmd_list = ["create", "--yes", "--quiet", "--prefix", path] else: raise TypeError("must specify either an environment name or a path" " for new environment") if any(os.path.exists(path) for path in search): raise CondaEnvExistsError("Conda environment [%s] already exists" % ref) cmd_list.extend(pkgs) if self._is_not_running: self._function_called = "create" self._call_conda(cmd_list) def install(self, name=None, path=None, pkgs=None, dep=True): """ Install packages into an environment either by name or path with a specified set of packages """ if not pkgs or not isinstance(pkgs, (list, tuple)): raise TypeError("must specify a list of one or more packages to " "install into existing environment") cmd_list = ["install", "--yes", "--json", "--force-pscheck"] if name: cmd_list.extend(["--name", name]) elif path: cmd_list.extend(["--prefix", path]) else: # just install into the current environment, whatever that is pass cmd_list.extend(pkgs) if not dep: cmd_list.extend(["--no-deps"]) if self._is_not_running: self._function_called = "install" self._call_conda(cmd_list) def update(self, *pkgs, **kwargs): """ Update package(s) (in an environment) by name. """ cmd_list = ["update", "--json", "--quiet", "--yes"] if not pkgs and not kwargs.get("all"): raise TypeError( "Must specify at least one package to update, \ or all=True." ) cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ( "dry_run", "no_deps", "override_channels", "no_pin", "force", "all", "use_index_cache", "use_local", "alt_hint", ), ) ) cmd_list.extend(pkgs) if self._is_not_running: self._function_called = "update" self._call_and_parse(cmd_list, abspath=kwargs.get("abspath", True)) def remove(self, *pkgs, **kwargs): """ Remove a package (from an environment) by name. Returns { success: bool, (this is always true), (other information) } """ cmd_list = ["remove", "--json", "--quiet", "--yes", "--force-pscheck"] if not pkgs and not kwargs.get("all"): raise TypeError( "Must specify at least one package to remove, \ or all=True." ) if kwargs.get("name") and kwargs.get("path"): raise TypeError("conda remove: At most one of name, path allowed") if kwargs.get("name"): cmd_list.extend(["--name", kwargs.pop("name")]) if kwargs.get("path"): cmd_list.extend(["--prefix", kwargs.pop("path")]) cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ("dry_run", "features", "override_channels", "no_pin", "force", "all") ) ) cmd_list.extend(pkgs) if self._is_not_running: self._function_called = "remove" self._call_and_parse(cmd_list, abspath=kwargs.get("abspath", True)) def remove_environment(self, name=None, path=None, **kwargs): """ Remove an environment entirely. See ``remove``. """ if self._is_not_running: self._function_called = "remove_environment" self.remove(name=name, path=path, all=True, **kwargs) def clone_environment(self, clone, name=None, path=None, **kwargs): """ Clone the environment ``clone`` into ``name`` or ``path``. """ cmd_list = ["create", "--json", "--quiet"] if (name and path) or not (name or path): raise TypeError( "conda clone_environment: exactly one of name or \ path required" ) if name: cmd_list.extend(["--name", name]) if path: cmd_list.extend(["--prefix", path]) cmd_list.extend(["--clone", clone]) cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ( "dry_run", "unknown", "use_index_cache", "use_local", "no_pin", "force", "all", "channel", "override_channels", "no_default_packages", ), ) ) if self._is_not_running(): self._function_called = "clone_environment" self._call_and_parse(cmd_list, abspath=kwargs.get("abspath", True)) def clone(self, path, prefix): """ Clone a "share package" (located at `path`) by creating a new environment at `prefix`, and return a dictionary with (at least) the following keys: - 'warnings': a list of warnings The directory `path` is located in, should be some temp directory or some other directory OUTSIDE /opt/anaconda. After calling this function, the original file (at `path`) may be removed (by the caller of this function). The return object is a list of warnings. """ if self._process.state() == QProcess.NotRunning: self._function_called = "clone" self._call_and_parse(["clone", "--json", "--prefix", prefix, path]) def _setup_config_from_kwargs(kwargs): cmd_list = ["--json", "--force"] if "file" in kwargs: cmd_list.extend(["--file", kwargs["file"]]) if "system" in kwargs: cmd_list.append("--system") return cmd_list def config_path(self, **kwargs): """ Get the path to the config file. """ cmd_list = ["config", "--get"] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = "config_path" self._call_and_parse(cmd_list, abspath=kwargs.get("abspath", True)) def config_get(self, *keys, **kwargs): """ Get the values of configuration keys. Returns a dictionary of values. Note, the key may not be in the dictionary if the key wasn't set in the configuration file. """ cmd_list = ["config", "--get"] cmd_list.extend(keys) cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = "config_get" self._call_and_parse(cmd_list, abspath=kwargs.get("abspath", True)) def config_set(self, key, value, **kwargs): """ Set a key to a (bool) value. Returns a list of warnings Conda may have emitted. """ cmd_list = ["config", "--set", key, str(value)] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = "config_set" self._call_and_parse(cmd_list, abspath=kwargs.get("abspath", True)) def config_add(self, key, value, **kwargs): """ Add a value to a key. Returns a list of warnings Conda may have emitted. """ cmd_list = ["config", "--add", key, value] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = "config_add" self._call_and_parse(cmd_list, abspath=kwargs.get("abspath", True)) def config_remove(self, key, value, **kwargs): """ Remove a value from a key. Returns a list of warnings Conda may have emitted. """ cmd_list = ["config", "--remove", key, value] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = "config_remove" self._call_and_parse(cmd_list, abspath=kwargs.get("abspath", True)) def config_delete(self, key, **kwargs): """ Remove a key entirely. Returns a list of warnings Conda may have emitted. """ cmd_list = ["config", "--remove-key", key] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = "config_delete" self._call_and_parse(cmd_list, abspath=kwargs.get("abspath", True)) def run(self, command, abspath=True): """ Launch the specified app by name or full package name. Returns a dictionary containing the key "fn", whose value is the full package (ending in ``.tar.bz2``) of the app. """ cmd_list = ["run", "--json", command] if self._is_not_running: self._function_called = "run" self._call_and_parse(cmd_list, abspath=abspath) # --- Additional methods not in conda-api # ------------------------------------------------------------------------ def pip(self, name): """Get list of pip installed packages.""" cmd_list = ["list", "-n", name] if self._is_not_running: self._function_called = "pip" self._call_conda(cmd_list) def dependencies(self, name=None, path=None, pkgs=None, dep=True): """ Install packages into an environment either by name or path with a specified set of packages. """ if not pkgs or not isinstance(pkgs, (list, tuple)): raise TypeError("must specify a list of one or more packages to " "install into existing environment") cmd_list = ["install", "--dry-run", "--json"] cmd_list = ["install", "--dry-run", "--json", "--force-pscheck"] if not dep: cmd_list.extend(["--no-deps"]) if name: cmd_list.extend(["--name", name]) elif path: cmd_list.extend(["--prefix", path]) else: pass cmd_list.extend(pkgs) if self._is_not_running: self._function_called = "install_dry" self._call_and_parse(cmd_list)
class CondaProcess(QObject): """conda-api modified to work with QProcess instead of popen""" ENCODING = 'ascii' def __init__(self, parent, on_finished=None, on_partial=None): QObject.__init__(self, parent) self._parent = parent self.output = None self.partial = None self.stdout = None self.error = None self._parse = False self._function_called = '' self._name = None self._process = QProcess() self._on_finished = on_finished self._process.finished.connect(self._call_conda_ready) self._process.readyReadStandardOutput.connect(self._call_conda_partial) if on_finished is not None: self._process.finished.connect(on_finished) if on_partial is not None: self._process.readyReadStandardOutput.connect(on_partial) self.set_root_prefix() def _call_conda_partial(self): """ """ stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(stdout, CondaProcess.ENCODING) stderr = self._process.readAllStandardError() stderr = handle_qbytearray(stderr, CondaProcess.ENCODING) if self._parse: self.output = json.loads(stdout) else: self.output = stdout self.partial = self.output self.stdout = self.output self.error = stderr # print(self.partial) # print(self.error) def _call_conda_ready(self): """function called when QProcess in _call_conda finishes task""" function = self._function_called if self.stdout is None: stdout = to_text_string(self._process.readAllStandardOutput(), encoding=CondaProcess.ENCODING) else: stdout = self.stdout if self.error is None: stderr = to_text_string(self._process.readAllStandardError(), encoding=CondaProcess.ENCODING) else: stderr = self.error if function == 'get_conda_version': pat = re.compile(r'conda:?\s+(\d+\.\d\S+|unknown)') m = pat.match(stderr.strip()) if m is None: m = pat.match(stdout.strip()) if m is None: raise Exception('output did not match: %r' % stderr) self.output = m.group(1) # elif function == 'get_envs': # info = self.output # self.output = info['envs'] # elif function == 'get_prefix_envname': # name = self._name # envs = self.output # self.output = self._get_prefix_envname_helper(name, envs) # self._name = None elif function == 'config_path': result = self.output self.output = result['rc_path'] elif function == 'config_get': result = self.output self.output = result['get'] elif (function == 'config_delete' or function == 'config_add' or function == 'config_set' or function == 'config_remove'): result = self.output self.output = result.get('warnings', []) elif function == 'pip': result = [] lines = self.output.split('\n') for line in lines: if '<pip>' in line: temp = line.split()[:-1] + ['pip'] result.append('-'.join(temp)) self.output = result if stderr.strip(): self.error = stderr # raise Exception('conda %r:\nSTDERR:\n%s\nEND' % (extra_args, # stderr.decode())) self._parse = False def _get_prefix_envname_helper(self, name, envs): """ """ global ROOTPREFIX if name == 'root': return ROOT_PREFIX for prefix in envs: if basename(prefix) == name: return prefix return None def _abspath(self, abspath): """ """ if abspath: if sys.platform == 'win32': python = join(ROOT_PREFIX, 'python.exe') conda = join(ROOT_PREFIX, 'Scripts', 'conda-script.py') else: python = join(ROOT_PREFIX, 'bin/python') conda = join(ROOT_PREFIX, 'bin/conda') cmd_list = [python, conda] else: # just use whatever conda is on the path cmd_list = ['conda'] return cmd_list def _call_conda(self, extra_args, abspath=True): """ """ # call conda with the list of extra arguments, and return the tuple # stdout, stderr global ROOT_PREFIX # if abspath: # if sys.platform == 'win32': # python = join(ROOT_PREFIX, 'python.exe') # conda = join(ROOT_PREFIX, # 'Scripts', 'conda-script.py') # else: # python = join(ROOT_PREFIX, 'bin/python') # conda = join(ROOT_PREFIX, 'bin/conda') # cmd_list = [python, conda] # else: # just use whatever conda is on the path # cmd_list = ['conda'] cmd_list = self._abspath(abspath) cmd_list.extend(extra_args) # try: # p = Popen(cmd_list, stdout=PIPE, stderr=PIPE) # except OSError: # raise Exception("could not invoke %r\n" % args) # return p.communicate() # adapted code # ------------ self.error, self.output = None, None self._process.start(cmd_list[0], cmd_list[1:]) def _call_and_parse(self, extra_args, abspath=True): """ """ # stdout, stderr = _call_conda(extra_args, abspath=abspath) # if stderr.decode().strip(): # raise Exception('conda %r:\nSTDERR:\n%s\nEND' % (extra_args, # stderr.decode())) # return json.loads(stdout.decode()) # adapted code # ------------ self._parse = True self._call_conda(extra_args, abspath=abspath) def _setup_install_commands_from_kwargs(self, kwargs, keys=tuple()): """ """ cmd_list = [] if kwargs.get('override_channels', False) and 'channel' not in kwargs: raise TypeError('conda search: override_channels requires channel') if 'env' in kwargs: cmd_list.extend(['--name', kwargs.pop('env')]) if 'prefix' in kwargs: cmd_list.extend(['--prefix', kwargs.pop('prefix')]) if 'channel' in kwargs: channel = kwargs.pop('channel') if isinstance(channel, str): cmd_list.extend(['--channel', channel]) else: cmd_list.append('--channel') cmd_list.extend(channel) for key in keys: if key in kwargs and kwargs[key]: cmd_list.append('--' + key.replace('_', '-')) return cmd_list def set_root_prefix(self, prefix=None): """ Set the prefix to the root environment (default is /opt/anaconda). This function should only be called once (right after importing conda_api). """ global ROOT_PREFIX if prefix: ROOT_PREFIX = prefix # find *some* conda instance, and then use info() to get 'root_prefix' else: pass # i = self.info(abspath=False) # self.ROOT_PREFIX = i['root_prefix'] ''' plat = 'posix' if sys.platform.lower().startswith('win'): listsep = ';' plat = 'win' else: listsep = ':' for p in os.environ['PATH'].split(listsep): if (os.path.exists(os.path.join(p, 'conda')) or os.path.exists(os.path.join(p, 'conda.exe')) or os.path.exists(os.path.join(p, 'conda.bat'))): # TEMPORARY: ROOT_PREFIX = os.path.dirname(p) # root prefix is 1 dir up i = info() # REAL: ROOT_PREFIX = i['root_prefix'] break else: # fall back to standard install location, which may be wrong if plat == 'win': ROOT_PREFIX = 'C:\Anaconda' else: ROOT_PREFIX = '/opt/anaconda' ''' # adapted code # ------------ if ROOT_PREFIX is None: # qprocess = QProcess() # cmd_list = ['conda', 'info', '--json'] # qprocess.start(cmd_list[0], cmd_list[1:]) # qprocess.waitForFinished() # output = qprocess.readAllStandardOutput() # output = handle_qbytearray(output, CondaProcess.ENCODING) # info = json.loads(output) # ROOT_PREFIX = info['root_prefix'] info = self.info(abspath=False) ROOT_PREFIX = info['root_prefix'] def get_conda_version(self): """ return the version of conda being used (invoked) as a string """ # pat = re.compile(r'conda:?\s+(\d+\.\d\S+|unknown)') # stdout, stderr = self._call_conda(['--version']) # # argparse outputs version to stderr in Python < 3.4. # # http://bugs.python.org/issue18920 # m = pat.match(stderr.decode().strip()) # if m is None: # m = pat.match(stdout.decode().strip()) # # if m is None: # raise Exception('output did not match: %r' % stderr) # return m.group(1) # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'get_conda_version' self._call_conda(['--version']) def get_envs(self): """ Return all of the (named) environment (this does not include the root environment), as a list of absolute path to their prefixes. """ # info = self._call_and_parse(['info', '--json']) # return info['envs'] info = self.info() return info['envs'] # adapted code # ------------ # if self._process.state() == QProcess.NotRunning: # self._function_called = 'get_envs' # self._call_and_parse(['info', '--json']) def get_prefix_envname(self, name): """ Given the name of an environment return its full prefix path, or None if it cannot be found. """ if name == 'root': return ROOT_PREFIX for prefix in self.get_envs(): if basename(prefix) == name: return prefix return None # adapted code # ------------ # if self._process.state() == QProcess.NotRunning: # self._name = name # self._function_called = 'get_prefix_envname' # self._call_and_parse(['info', '--json']) def info(self, abspath=True): """ Return a dictionary with configuration information. No guarantee is made about which keys exist. Therefore this function should only be used for testing and debugging. """ # return self._call_and_parse(['info', '--json'], abspath=abspath) qprocess = QProcess() cmd_list = self._abspath(abspath) cmd_list.extend(['info', '--json']) qprocess.start(cmd_list[0], cmd_list[1:]) qprocess.waitForFinished() output = qprocess.readAllStandardOutput() output = handle_qbytearray(output, CondaProcess.ENCODING) info = json.loads(output) return info # adapted code # ------------ # if self._process.state() == QProcess.NotRunning: # self._function_called = 'info' # self._call_and_parse(['info', '--json'], abspath=abspath) def package_info(self, package, abspath=True): """ Return a dictionary with package information. Structure is { 'package_name': [{ 'depends': list, 'version': str, 'name': str, 'license': str, 'fn': ..., ... }] } """ # return self._call_and_parse(['info', package, '--json'], # abspath=abspath) # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'package_info' self._call_and_parse(['info', package, '--json'], abspath=abspath) def search(self, regex=None, spec=None, **kwargs): """ Search for packages. """ cmd_list = ['search', '--json'] if regex and spec: raise TypeError('conda search: only one of regex or spec allowed') if regex: cmd_list.append(regex) if spec: cmd_list.extend(['--spec', spec]) if 'platform' in kwargs: platform = kwargs.pop('platform') platforms = ('win-32', 'win-64', 'osx-64', 'linux-32', 'linux-64') if platform not in platforms: raise TypeError('conda search: platform must be one of ' + ', '.join(platforms)) cmd_list.extend(['--platform', platform]) cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ('canonical', 'unknown', 'use_index_cache', 'outdated', 'override_channels'))) # return self._call_and_parse(cmd_list, # abspath=kwargs.get('abspath', True)) # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'search' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def share(self, prefix): """ Create a "share package" of the environment located in `prefix`, and return a dictionary with (at least) the following keys: - 'path': the full path to the created package - 'warnings': a list of warnings This file is created in a temp directory, and it is the callers responsibility to remove this directory (after the file has been handled in some way). """ # return self._call_and_parse(['share', '--json', '--prefix', prefix]) # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'share' self._call_and_parse(['share', '--json', '--prefix', prefix]) def create(self, name=None, path=None, pkgs=None): """ Create an environment either by name or path with a specified set of packages """ if not pkgs or not isinstance(pkgs, (list, tuple)): raise TypeError('must specify a list of one or more packages to ' 'install into new environment') cmd_list = ['create', '--yes', '--quiet'] if name: ref = name search = [os.path.join(d, name) for d in self.info()['envs_dirs']] cmd_list = ['create', '--yes', '--quiet', '--name', name] elif path: ref = path search = [path] cmd_list = ['create', '--yes', '--quiet', '--prefix', path] else: raise TypeError('must specify either an environment name or a path' ' for new environment') if any(os.path.exists(path) for path in search): raise CondaEnvExistsError('Conda environment [%s] already exists' % ref) cmd_list.extend(pkgs) # (out, err) = self._call_conda(cmd_list) # if err.decode().strip(): # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # err.decode())) # return out # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'create' self._call_conda(cmd_list) def install(self, name=None, path=None, pkgs=None, dep=True): """ Install packages into an environment either by name or path with a specified set of packages """ if not pkgs or not isinstance(pkgs, (list, tuple)): raise TypeError('must specify a list of one or more packages to ' 'install into existing environment') cmd_list = ['install', '--yes', '--json', '--force-pscheck'] # cmd_list = ['install', '--yes', '--quiet'] if name: cmd_list.extend(['--name', name]) elif path: cmd_list.extend(['--prefix', path]) else: # just install into the current environment, whatever that is pass cmd_list.extend(pkgs) # (out, err) = self._call_conda(cmd_list) # if err.decode().strip(): # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # err.decode())) # return out # adapted code # ------------ if not dep: cmd_list.extend(['--no-deps']) if self._process.state() == QProcess.NotRunning: self._function_called = 'install' self._call_conda(cmd_list) def update(self, *pkgs, **kwargs): """ Update package(s) (in an environment) by name. """ cmd_list = ['update', '--json', '--quiet', '--yes'] if not pkgs and not kwargs.get('all'): raise TypeError("Must specify at least one package to update, \ or all=True.") cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ('dry_run', 'no_deps', 'override_channels', 'no_pin', 'force', 'all', 'use_index_cache', 'use_local', 'alt_hint'))) cmd_list.extend(pkgs) # result = self._call_and_parse(cmd_list, # abspath=kwargs.get('abspath', True)) # # if 'error' in result: # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # result['error'])) # # return result # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'update' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def remove(self, *pkgs, **kwargs): """ Remove a package (from an environment) by name. Returns { success: bool, (this is always true), (other information) } """ cmd_list = ['remove', '--json', '--quiet', '--yes', '--force-pscheck'] # cmd_list = ['remove', '--json', '--quiet', '--yes'] if not pkgs and not kwargs.get('all'): raise TypeError("Must specify at least one package to remove, \ or all=True.") if kwargs.get('name') and kwargs.get('path'): raise TypeError('conda remove: At most one of name, path allowed') if kwargs.get('name'): cmd_list.extend(['--name', kwargs.pop('name')]) if kwargs.get('path'): cmd_list.extend(['--prefix', kwargs.pop('path')]) cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ('dry_run', 'features', 'override_channels', 'no_pin', 'force', 'all'))) cmd_list.extend(pkgs) # result = self._call_and_parse(cmd_list, # abspath=kwargs.get('abspath', True)) # # if 'error' in result: # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # result['error'])) # # return result # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'remove' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def remove_environment(self, name=None, path=None, **kwargs): """ Remove an environment entirely. See ``remove``. """ # return self.remove(name=name, path=path, all=True, **kwargs) # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'remove_environment' self.remove(name=name, path=path, all=True, **kwargs) def clone_environment(self, clone, name=None, path=None, **kwargs): """ Clone the environment ``clone`` into ``name`` or ``path``. """ cmd_list = ['create', '--json', '--quiet'] if (name and path) or not (name or path): raise TypeError("conda clone_environment: exactly one of name or \ path required") if name: cmd_list.extend(['--name', name]) if path: cmd_list.extend(['--prefix', path]) cmd_list.extend(['--clone', clone]) cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ('dry_run', 'unknown', 'use_index_cache', 'use_local', 'no_pin', 'force', 'all', 'channel', 'override_channels', 'no_default_packages'))) # result = self._call_and_parse(cmd_list, # abspath=kwargs.get('abspath', True)) # # if 'error' in result: # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # result['error'])) # # return result # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'clone_environment' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) # def process(self, name=None, path=None, cmd=None, args=None, stdin=None, # stdout=None, stderr=None, timeout=None): # """ # Create a Popen process for cmd using the specified args but in the # conda environment specified by name or path. # # The returned object will need to be invoked with p.communicate() or # similar. # # :param name: name of conda environment # :param path: path to conda environment (if no name specified) # :param cmd: command to invoke # :param args: argument # :param stdin: stdin # :param stdout: stdout # :param stderr: stderr # :return: Popen object # """ # # if bool(name) == bool(path): # raise TypeError('exactly one of name or path must be specified') # # if not cmd: # raise TypeError('cmd to execute must be specified') # # if not args: # args = [] # # if name: # path = self.get_prefix_envname(name) # # plat = 'posix' # if sys.platform.lower().startswith('win'): # listsep = ';' # plat = 'win' # else: # listsep = ':' # # conda_env = dict(os.environ) # # if plat == 'posix': # conda_env['PATH'] = path + os.path.sep + 'bin' + listsep + \ # conda_env['PATH'] # else: # win # conda_env['PATH'] = path + os.path.sep + 'Scripts' + listsep + \ # conda_env['PATH'] # # conda_env['PATH'] = path + listsep + conda_env['PATH'] # # cmd_list = [cmd] # cmd_list.extend(args) # # try: # p = Popen(cmd_list, env=conda_env, stdin=stdin, stdout=stdout, # stderr=stderr) # except OSError: # raise Exception("could not invoke %r\n" % cmd_list) # return p def clone(self, path, prefix): """ Clone a "share package" (located at `path`) by creating a new environment at `prefix`, and return a dictionary with (at least) the following keys: - 'warnings': a list of warnings The directory `path` is located in, should be some temp directory or some other directory OUTSIDE /opt/anaconda. After calling this function, the original file (at `path`) may be removed (by the caller of this function). The return object is a list of warnings. """ # return self._call_and_parse(['clone', '--json', '--prefix', prefix, # path]) # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'clone' self._call_and_parse(['clone', '--json', '--prefix', prefix, path]) def _setup_config_from_kwargs(kwargs): cmd_list = ['--json', '--force'] if 'file' in kwargs: cmd_list.extend(['--file', kwargs['file']]) if 'system' in kwargs: cmd_list.append('--system') return cmd_list def config_path(self, **kwargs): """ Get the path to the config file. """ cmd_list = ['config', '--get'] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) # result = self._call_and_parse(cmd_list, # abspath=kwargs.get('abspath', True)) # # if 'error' in result: # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # result['error'])) # return result['rc_path'] # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'config_path' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def config_get(self, *keys, **kwargs): """ Get the values of configuration keys. Returns a dictionary of values. Note, the key may not be in the dictionary if the key wasn't set in the configuration file. """ cmd_list = ['config', '--get'] cmd_list.extend(keys) cmd_list.extend(self._setup_config_from_kwargs(kwargs)) # result = self._call_and_parse(cmd_list, # abspath=kwargs.get('abspath', True)) # # if 'error' in result: # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # result['error'])) # return result['get'] # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'config_get' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def config_set(self, key, value, **kwargs): """ Set a key to a (bool) value. Returns a list of warnings Conda may have emitted. """ cmd_list = ['config', '--set', key, str(value)] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) # result = self._call_and_parse(cmd_list, # abspath=kwargs.get('abspath', True)) # # if 'error' in result: # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # result['error'])) # return result.get('warnings', []) # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'config_set' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def config_add(self, key, value, **kwargs): """ Add a value to a key. Returns a list of warnings Conda may have emitted. """ cmd_list = ['config', '--add', key, value] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) # result = self._call_and_parse(cmd_list, # abspath=kwargs.get('abspath', True)) # # if 'error' in result: # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # result['error'])) # return result.get('warnings', []) # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'config_add' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def config_remove(self, key, value, **kwargs): """ Remove a value from a key. Returns a list of warnings Conda may have emitted. """ cmd_list = ['config', '--remove', key, value] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) # result = self._call_and_parse(cmd_list, # abspath=kwargs.get('abspath', True)) # # if 'error' in result: # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # result['error'])) # return result.get('warnings', []) # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'config_remove' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def config_delete(self, key, **kwargs): """ Remove a key entirely. Returns a list of warnings Conda may have emitted. """ cmd_list = ['config', '--remove-key', key] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) # result = self._call_and_parse(cmd_list, # abspath=kwargs.get('abspath', True)) # # if 'error' in result: # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # result['error'])) # return result.get('warnings', []) # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'config_delete' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def run(self, command, abspath=True): """ Launch the specified app by name or full package name. Returns a dictionary containing the key "fn", whose value is the full package (ending in ``.tar.bz2``) of the app. """ cmd_list = ['run', '--json', command] # result = self._call_and_parse(cmd_list, abspath=abspath) # # if 'error' in result: # raise CondaError('conda %s: %s' % (" ".join(cmd_list), # result['error'])) # return result # adapted code # ------------ if self._process.state() == QProcess.NotRunning: self._function_called = 'run' self._call_and_parse(cmd_list, abspath=abspath) # def test(): # """ # Self-test function, which prints useful debug information. # This function returns None on success, and will crash the interpreter # on failure. # """ # print('sys.version: %r' % sys.version) # print('sys.prefix : %r' % sys.prefix) # print('conda_api.__version__: %r' % __version__) # print('conda_api.ROOT_PREFIX: %r' % ROOT_PREFIX) # if isdir(ROOT_PREFIX): # conda_version = get_conda_version() # print('conda version: %r' % conda_version) # print('conda info:') # d = info() # for kv in d.items(): # print('\t%s=%r' % kv) # assert d['conda_version'] == conda_version # else: # print('Warning: no such directory: %r' % ROOT_PREFIX) # print('OK') # ---- Additional methods not in conda-api def pip(self, name): """Get list of pip installed packages.""" cmd_list = ['list', '-n', name] if self._process.state() == QProcess.NotRunning: self._function_called = 'pip' self._call_conda(cmd_list) def dependencies(self, name=None, path=None, pkgs=None, dep=True): """ Install packages into an environment either by name or path with a specified set of packages """ if not pkgs or not isinstance(pkgs, (list, tuple)): raise TypeError('must specify a list of one or more packages to ' 'install into existing environment') cmd_list = ['install', '--dry-run', '--json'] cmd_list = ['install', '--dry-run', '--json', '--force-pscheck'] if not dep: cmd_list.extend(['--no-deps']) if name: cmd_list.extend(['--name', name]) elif path: cmd_list.extend(['--prefix', path]) else: # just install into the current environment, whatever that is pass cmd_list.extend(pkgs) if self._process.state() == QProcess.NotRunning: self._function_called = 'install_dry' self._call_and_parse(cmd_list)
class JobRunner(QWidget): """objuect to run GUI subprocess calls""" def __init__(self, run_button=None, qwidgets_to_disable=None, text_edit=None, busy=None, timer_label=None, parent=None, setup_layout=True, **kwargs): super(JobRunner, self).__init__(parent, **kwargs) self.is_passed = False self._iteration_edit = QLineEdit(self, placeholderText="Iteration (n)") self._iteration_edit.setText('5') if run_button is None: run_button = QPushButton("Calculate Prime", self) run_button.clicked.connect(self.prime_requested) if text_edit is None: text_edit = QTextEdit("Result:<br>", self) text_edit.setReadOnly(True) if busy is None: busy = QProgressBar(self) self._busy = busy if timer_label is None: timer_label = QLabel('0:00') self._timer_label = timer_label self._busy.setVisible(False) self._timer_label.setVisible(False) self._run_button = run_button self._text_edit = text_edit self._time0 = None self.process = None if qwidgets_to_disable is None: qwidgets_to_disable = [self._run_button, self._iteration_edit] self.qwidgets_to_disable = qwidgets_to_disable if setup_layout: self.setup_layout() def setup_layout(self): hbox_timer = QHBoxLayout() hbox_timer.addWidget(self._busy) hbox_timer.addWidget(self._timer_label) vbox = QVBoxLayout(self) vbox.addWidget(self._iteration_edit) vbox.addWidget(self._run_button) vbox.addLayout(hbox_timer) vbox.addWidget(self._text_edit) self.setLayout(vbox) self.show() def prime_requested(self): try: n = int(self._iteration_edit.text()) except: return #command_list = [str(n)] command_list = ['python', 'ls.py'] self._text_edit.clear() self.run_command(command_list) def run_command(self, command_list): for qwidget in self.qwidgets_to_disable: qwidget.setEnabled(False) self._busy.setRange(0, 0) self._busy.setVisible(True) self._busy.update() self._timer_label.setVisible(True) self._time0 = time.time() self.process = QProcess(self) self.process.readyReadStandardOutput.connect(self.stdout_ready) self.process.finished.connect(self.on_finished) #self.process.started.connect(lambda: print_func('Started!')) #self.process.finished.connect(lambda: print_func('Finished!')) #self.process.startDetached(command_list[0], command_list[1:]) self.process.start(command_list[0], command_list[1:]) for qwidget in self.qwidgets_to_disable: qwidget.setEnabled(True) def stdout_ready(self): text = str(self.process.readAllStandardOutput()) self.append(text) dt = time.time() - self._time0 minutes = dt // 60 seconds = dt % 60 self._timer_label.setText('%i:%02i' % (minutes, seconds)) def append(self, text): cursor = self._text_edit.textCursor() cursor.movePosition(cursor.End) cursor.insertText(text) #self.output.ensureCursorVisible() def on_finished(self, exit_code): #print('exit_code =', exit_code) self.is_passed = True text = str(self.process.readAllStandardError()) self.append(text) dt = time.time() - self._time0 minutes = dt // 60 seconds = dt % 60 time_msg = '' if minutes: time_msg += '%i minutes and ' % minutes time_msg += '%i seconds' % seconds self.append('\nJob completed in %s' % time_msg) self._busy.setVisible(False) self._timer_label.setVisible(False) #self._timer_label.stop() def display_prime(self, msg): text_edit = self._text_edit text_cursor = text_edit.textCursor() end = text_cursor.End # end of text_edit text_cursor.movePosition(end) text_edit.setTextCursor(text_cursor) text_edit.insertHtml(msg) text_edit.ensureCursorVisible() # new message will be visible def unlock(self): #self._busy.setRange(0, 100) for qwidget in self.qwidgets_to_disable: qwidget.setEnabled(True)
class ProcessWorker(QObject): """Conda worker based on a QProcess for non blocking UI.""" sig_finished = Signal(object, object, object) sig_partial = Signal(object, object, object) def __init__(self, cmd_list, parse=False, pip=False, callback=None, extra_kwargs=None): """Conda worker based on a QProcess for non blocking UI. Parameters ---------- cmd_list : list of str Command line arguments to execute. parse : bool (optional) Parse json from output. pip : bool (optional) Define as a pip command. callback : func (optional) If the process has a callback to process output from comd_list. extra_kwargs : dict Arguments for the callback. """ super(ProcessWorker, self).__init__() self._result = None self._cmd_list = cmd_list self._parse = parse self._pip = pip self._conda = not pip self._callback = callback self._fired = False self._communicate_first = False self._partial_stdout = None self._extra_kwargs = extra_kwargs if extra_kwargs else {} self._timer = QTimer() self._process = QProcess() self._timer.setInterval(150) self._timer.timeout.connect(self._communicate) # self._process.finished.connect(self._communicate) self._process.readyReadStandardOutput.connect(self._partial) def _partial(self): """Callback for partial output.""" raw_stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(raw_stdout, _CondaAPI.UTF8) try: json_stdout = [json.loads(s) for s in stdout.split('\x00') if s] json_stdout = json_stdout[-1] except Exception: json_stdout = stdout if self._partial_stdout is None: self._partial_stdout = stdout else: self._partial_stdout += stdout self.sig_partial.emit(self, json_stdout, None) def _communicate(self): """Callback for communicate.""" if (not self._communicate_first and self._process.state() == QProcess.NotRunning): self.communicate() elif self._fired: self._timer.stop() def communicate(self): """Retrieve information.""" self._communicate_first = True self._process.waitForFinished() if self._partial_stdout is None: raw_stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(raw_stdout, _CondaAPI.UTF8) else: stdout = self._partial_stdout raw_stderr = self._process.readAllStandardError() stderr = handle_qbytearray(raw_stderr, _CondaAPI.UTF8) result = [stdout.encode(_CondaAPI.UTF8), stderr.encode(_CondaAPI.UTF8)] # FIXME: Why does anaconda client print to stderr??? if PY2: stderr = stderr.decode() if 'using anaconda' not in stderr.lower(): if stderr.strip() and self._conda: d = {'command': ' '.join(self._cmd_list), 'stderr': stderr} # print(d) # Should we just ignore any message here??? logger.warning('Conda command output on stderr', extra=d) elif stderr.strip() and self._pip: d = {'command': ' '.join(self._cmd_list)} # print(d) # Should we just ignore any message here??? logger.warning('Pip command output on stderr', extra=d) result[-1] = '' if self._parse and stdout: json_stdout = [] json_lines_output = stdout.split('\x00') for i, l in enumerate(json_lines_output): if l: try: json_stdout.append(json.loads(l)) except Exception as error: # An exception here could be product of: # - conda env installing pip stuff that is thrown to # stdout in non json form # - a post link script might be printing stuff to # stdout in non json format logger.warning( ('Problem parsing conda json output. ' 'Line {0}. Data - {1}. Error - {2}'.format( i, l, str(error))), ) if json_stdout: json_stdout = json_stdout[-1] result = json_stdout, result[-1] if 'exception_name' in result[0] or 'exception_type' in result[0]: if not isinstance(result[0], dict): result = {'error': str(result[0])}, None error = '{0}: {1}'.format(" ".join(self._cmd_list), result[0]['message']) result = result[0], error if self._callback: result = self._callback(result[0], result[-1], **self._extra_kwargs), result[-1] self._result = result self.sig_finished.emit(self, result[0], result[-1]) if result[-1]: d = {'stderr': result[-1]} logger.error('error', extra=d) self._fired = True return result def close(self): """Close the running process.""" self._process.close() def is_finished(self): """Return True if worker has finished processing.""" return self._process.state() == QProcess.NotRunning and self._fired def start(self): """Start process.""" logger.debug(str(' '.join(self._cmd_list))) if not self._fired: self._partial_ouput = None self._process.start(self._cmd_list[0], self._cmd_list[1:]) self._timer.start() else: raise CondaProcessWorker('A Conda ProcessWorker can only run once ' 'per method call.')
class QtPipDialog(QDialog): def __init__(self, parent=None): super().__init__(parent) self.setup_ui() self.setAttribute(Qt.WA_DeleteOnClose) # create install process self.process = QProcess(self) self.process.setProgram(sys.executable) self.process.setProcessChannelMode(QProcess.MergedChannels) # setup process path env = QProcessEnvironment() combined_paths = os.pathsep.join( [user_site_packages(), env.systemEnvironment().value("PYTHONPATH")] ) env.insert("PYTHONPATH", combined_paths) self.process.setProcessEnvironment(env) # connections self.install_button.clicked.connect(self._install) self.uninstall_button.clicked.connect(self._uninstall) self.process.readyReadStandardOutput.connect(self._on_stdout_ready) from ..plugins import plugin_manager self.process.finished.connect(plugin_manager.discover) self.process.finished.connect(plugin_manager.prune) def setup_ui(self): layout = QVBoxLayout() self.setLayout(layout) title = QLabel("Install/Uninstall Packages:") title.setObjectName("h2") self.line_edit = QLineEdit() self.install_button = QPushButton("install", self) self.uninstall_button = QPushButton("uninstall", self) self.text_area = QTextEdit(self, readOnly=True) hlay = QHBoxLayout() hlay.addWidget(self.line_edit) hlay.addWidget(self.install_button) hlay.addWidget(self.uninstall_button) layout.addWidget(title) layout.addLayout(hlay) layout.addWidget(self.text_area) self.setFixedSize(700, 400) self.setMaximumHeight(800) self.setMaximumWidth(1280) def _install(self): cmd = ['-m', 'pip', 'install'] if running_as_bundled_app() and sys.platform.startswith('linux'): cmd += [ '--no-warn-script-location', '--prefix', user_plugin_dir(), ] self.process.setArguments(cmd + self.line_edit.text().split()) self.text_area.clear() self.process.start() def _uninstall(self): args = ['-m', 'pip', 'uninstall', '-y'] self.process.setArguments(args + self.line_edit.text().split()) self.text_area.clear() self.process.start() def _on_stdout_ready(self): text = self.process.readAllStandardOutput().data().decode() self.text_area.append(text)
class AsyncClient(QObject): """ A class which handles a connection to a client through a QProcess. """ # Emitted when the client has initialized. initialized = Signal() # Emitted when the client errors. errored = Signal() # Emitted when a request response is received. received = Signal(object) def __init__(self, target, executable=None, name=None, extra_args=None, libs=None, cwd=None, env=None): super(AsyncClient, self).__init__() self.executable = executable or sys.executable self.extra_args = extra_args self.target = target self.name = name or self self.libs = libs self.cwd = cwd self.env = env self.is_initialized = False self.closing = False self.context = zmq.Context() QApplication.instance().aboutToQuit.connect(self.close) # Set up the heartbeat timer. self.timer = QTimer(self) self.timer.timeout.connect(self._heartbeat) def run(self): """Handle the connection with the server. """ # Set up the zmq port. self.socket = self.context.socket(zmq.PAIR) self.port = self.socket.bind_to_random_port('tcp://*') # Set up the process. self.process = QProcess(self) if self.cwd: self.process.setWorkingDirectory(self.cwd) p_args = ['-u', self.target, str(self.port)] if self.extra_args is not None: p_args += self.extra_args # Set up environment variables. processEnvironment = QProcessEnvironment() env = self.process.systemEnvironment() if (self.env and 'PYTHONPATH' not in self.env) or DEV: python_path = osp.dirname(get_module_path('spyderlib')) # Add the libs to the python path. for lib in self.libs: try: path = osp.dirname(imp.find_module(lib)[1]) python_path = osp.pathsep.join([python_path, path]) except ImportError: pass env.append("PYTHONPATH=%s" % python_path) if self.env: env.update(self.env) for envItem in env: envName, separator, envValue = envItem.partition('=') processEnvironment.insert(envName, envValue) self.process.setProcessEnvironment(processEnvironment) # Start the process and wait for started. self.process.start(self.executable, p_args) self.process.finished.connect(self._on_finished) running = self.process.waitForStarted() if not running: raise IOError('Could not start %s' % self) # Set up the socket notifer. fid = self.socket.getsockopt(zmq.FD) self.notifier = QSocketNotifier(fid, QSocketNotifier.Read, self) self.notifier.activated.connect(self._on_msg_received) def request(self, func_name, *args, **kwargs): """Send a request to the server. The response will be a dictionary the 'request_id' and the 'func_name' as well as a 'result' field with the object returned by the function call or or an 'error' field with a traceback. """ if not self.is_initialized: return request_id = uuid.uuid4().hex request = dict(func_name=func_name, args=args, kwargs=kwargs, request_id=request_id) self._send(request) return request_id def close(self): """Cleanly close the connection to the server. """ self.closing = True self.is_initialized = False self.timer.stop() self.notifier.activated.disconnect(self._on_msg_received) self.notifier.setEnabled(False) del self.notifier self.request('server_quit') self.process.waitForFinished(1000) self.process.close() self.context.destroy() def _on_finished(self): """Handle a finished signal from the process. """ if self.closing: return if self.is_initialized: debug_print('Restarting %s' % self.name) debug_print(self.process.readAllStandardOutput()) debug_print(self.process.readAllStandardError()) self.is_initialized = False self.notifier.setEnabled(False) self.run() else: debug_print('Errored %s' % self.name) debug_print(self.process.readAllStandardOutput()) debug_print(self.process.readAllStandardError()) self.errored.emit() def _on_msg_received(self): """Handle a message trigger from the socket. """ self.notifier.setEnabled(False) while 1: try: resp = self.socket.recv_pyobj(flags=zmq.NOBLOCK) except zmq.ZMQError: self.notifier.setEnabled(True) return if not self.is_initialized: self.is_initialized = True debug_print('Initialized %s' % self.name) self.initialized.emit() self.timer.start(HEARTBEAT) continue resp['name'] = self.name self.received.emit(resp) def _heartbeat(self): """Send a heartbeat to keep the server alive. """ self._send(dict(func_name='server_heartbeat')) def _send(self, obj): """Send an object to the server. """ try: self.socket.send_pyobj(obj) except Exception as e: debug_print(e) self.is_initialized = False self._on_finished()
class ProcessWorker(QObject): """ """ sig_finished = Signal(object, object, object) sig_partial = Signal(object, object, object) def __init__(self, cmd_list, parse=False, pip=False, callback=None, extra_kwargs={}): super(ProcessWorker, self).__init__() self._result = None self._cmd_list = cmd_list self._parse = parse self._pip = pip self._conda = not pip self._callback = callback self._fired = False self._communicate_first = False self._partial_stdout = None self._extra_kwargs = extra_kwargs self._timer = QTimer() self._process = QProcess() self._timer.setInterval(50) self._timer.timeout.connect(self._communicate) self._process.finished.connect(self._communicate) self._process.readyReadStandardOutput.connect(self._partial) def _partial(self): raw_stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(raw_stdout, _CondaAPI.UTF8) json_stdout = stdout.replace('\n\x00', '') try: json_stdout = json.loads(json_stdout) except Exception: json_stdout = stdout if self._partial_stdout is None: self._partial_stdout = stdout else: self._partial_stdout += stdout self.sig_partial.emit(self, json_stdout, None) def _communicate(self): """ """ if not self._communicate_first: if self._process.state() == QProcess.NotRunning: self.communicate() elif self._fired: self._timer.stop() def communicate(self): """ """ self._communicate_first = True self._process.waitForFinished() if self._partial_stdout is None: raw_stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(raw_stdout, _CondaAPI.UTF8) else: stdout = self._partial_stdout raw_stderr = self._process.readAllStandardError() stderr = handle_qbytearray(raw_stderr, _CondaAPI.UTF8) result = [stdout.encode(_CondaAPI.UTF8), stderr.encode(_CondaAPI.UTF8)] # FIXME: Why does anaconda client print to stderr??? if PY2: stderr = stderr.decode() if 'using anaconda cloud api site' not in stderr.lower(): if stderr.strip() and self._conda: raise Exception('{0}:\n' 'STDERR:\n{1}\nEND' ''.format(' '.join(self._cmd_list), stderr)) # elif stderr.strip() and self._pip: # raise PipError(self._cmd_list) else: result[-1] = '' if self._parse and stdout: try: result = json.loads(stdout), result[-1] except ValueError as error: result = stdout, error if 'error' in result[0]: error = '{0}: {1}'.format(" ".join(self._cmd_list), result[0]['error']) result = result[0], error if self._callback: result = self._callback(result[0], result[-1], **self._extra_kwargs), result[-1] self._result = result self.sig_finished.emit(self, result[0], result[-1]) if result[-1]: logger.error(str(('error', result[-1]))) self._fired = True return result def close(self): """ """ self._process.close() def is_finished(self): """ """ return self._process.state() == QProcess.NotRunning and self._fired def start(self): """ """ logger.debug(str(' '.join(self._cmd_list))) if not self._fired: self._partial_ouput = None self._process.start(self._cmd_list[0], self._cmd_list[1:]) self._timer.start() else: raise CondaProcessWorker('A Conda ProcessWorker can only run once ' 'per method call.')
class ProfilerWidget(QWidget): """ Profiler widget """ DATAPATH = get_conf_path('profiler.results') VERSION = '0.0.1' redirect_stdio = Signal(bool) def __init__(self, parent, max_entries=100): QWidget.__init__(self, parent) self.setWindowTitle("Profiler") self.output = None self.error_output = None self._last_wdir = None self._last_args = None self._last_pythonpath = None self.filecombo = PythonModulesComboBox(self) self.start_button = create_toolbutton(self, icon=ima.icon('run'), text=_("Profile"), tip=_("Run profiler"), triggered=lambda : self.start(), text_beside_icon=True) self.stop_button = create_toolbutton(self, icon=ima.icon('stop'), text=_("Stop"), tip=_("Stop current profiling"), text_beside_icon=True) self.filecombo.valid.connect(self.start_button.setEnabled) #self.connect(self.filecombo, SIGNAL('valid(bool)'), self.show_data) # FIXME: The combobox emits this signal on almost any event # triggering show_data() too early, too often. browse_button = create_toolbutton(self, icon=ima.icon('fileopen'), tip=_('Select Python script'), triggered=self.select_file) self.datelabel = QLabel() self.log_button = create_toolbutton(self, icon=ima.icon('log'), text=_("Output"), text_beside_icon=True, tip=_("Show program's output"), triggered=self.show_log) self.datatree = ProfilerDataTree(self) self.collapse_button = create_toolbutton(self, icon=ima.icon('collapse'), triggered=lambda dD: self.datatree.change_view(-1), tip=_('Collapse one level up')) self.expand_button = create_toolbutton(self, icon=ima.icon('expand'), triggered=lambda dD: self.datatree.change_view(1), tip=_('Expand one level down')) self.save_button = create_toolbutton(self, text_beside_icon=True, text=_("Save data"), icon=ima.icon('filesave'), triggered=self.save_data, tip=_('Save profiling data')) self.load_button = create_toolbutton(self, text_beside_icon=True, text=_("Load data"), icon=ima.icon('fileimport'), triggered=self.compare, tip=_('Load profiling data for comparison')) self.clear_button = create_toolbutton(self, text_beside_icon=True, text=_("Clear comparison"), icon=ima.icon('editdelete'), triggered=self.clear) hlayout1 = QHBoxLayout() hlayout1.addWidget(self.filecombo) hlayout1.addWidget(browse_button) hlayout1.addWidget(self.start_button) hlayout1.addWidget(self.stop_button) hlayout2 = QHBoxLayout() hlayout2.addWidget(self.collapse_button) hlayout2.addWidget(self.expand_button) hlayout2.addStretch() hlayout2.addWidget(self.datelabel) hlayout2.addStretch() hlayout2.addWidget(self.log_button) hlayout2.addWidget(self.save_button) hlayout2.addWidget(self.load_button) hlayout2.addWidget(self.clear_button) layout = QVBoxLayout() layout.addLayout(hlayout1) layout.addLayout(hlayout2) layout.addWidget(self.datatree) self.setLayout(layout) self.process = None self.set_running_state(False) self.start_button.setEnabled(False) self.clear_button.setEnabled(False) if not is_profiler_installed(): # This should happen only on certain GNU/Linux distributions # or when this a home-made Python build because the Python # profilers are included in the Python standard library for widget in (self.datatree, self.filecombo, self.start_button, self.stop_button): widget.setDisabled(True) url = 'http://docs.python.org/library/profile.html' text = '%s <a href=%s>%s</a>' % (_('Please install'), url, _("the Python profiler modules")) self.datelabel.setText(text) else: pass # self.show_data() def save_data(self): """Save data""" title = _( "Save profiler result") filename, _selfilter = getsavefilename(self, title, getcwd(), _("Profiler result")+" (*.Result)") if filename: self.datatree.save_data(filename) def compare(self): filename, _selfilter = getopenfilename(self, _("Select script to compare"), getcwd(), _("Profiler result")+" (*.Result)") if filename: self.datatree.compare(filename) self.show_data() self.clear_button.setEnabled(True) def clear(self): self.datatree.compare(None) self.datatree.hide_diff_cols(True) self.show_data() self.clear_button.setEnabled(False) def analyze(self, filename, wdir=None, args=None, pythonpath=None): if not is_profiler_installed(): return self.kill_if_running() #index, _data = self.get_data(filename) index = None # FIXME: storing data is not implemented yet if index is None: self.filecombo.addItem(filename) self.filecombo.setCurrentIndex(self.filecombo.count()-1) else: self.filecombo.setCurrentIndex(self.filecombo.findText(filename)) self.filecombo.selected() if self.filecombo.is_valid(): if wdir is None: wdir = osp.dirname(filename) self.start(wdir, args, pythonpath) def select_file(self): self.redirect_stdio.emit(False) filename, _selfilter = getopenfilename(self, _("Select Python script"), getcwd(), _("Python scripts")+" (*.py ; *.pyw)") self.redirect_stdio.emit(True) if filename: self.analyze(filename) def show_log(self): if self.output: TextEditor(self.output, title=_("Profiler output"), readonly=True, size=(700, 500)).exec_() def show_errorlog(self): if self.error_output: TextEditor(self.error_output, title=_("Profiler output"), readonly=True, size=(700, 500)).exec_() def start(self, wdir=None, args=None, pythonpath=None): filename = to_text_string(self.filecombo.currentText()) if wdir is None: wdir = self._last_wdir if wdir is None: wdir = osp.basename(filename) if args is None: args = self._last_args if args is None: args = [] if pythonpath is None: pythonpath = self._last_pythonpath self._last_wdir = wdir self._last_args = args self._last_pythonpath = pythonpath self.datelabel.setText(_('Profiling, please wait...')) self.process = QProcess(self) self.process.setProcessChannelMode(QProcess.SeparateChannels) self.process.setWorkingDirectory(wdir) self.process.readyReadStandardOutput.connect(self.read_output) self.process.readyReadStandardError.connect( lambda: self.read_output(error=True)) self.process.finished.connect(lambda ec, es=QProcess.ExitStatus: self.finished(ec, es)) self.stop_button.clicked.connect(self.process.kill) if pythonpath is not None: env = [to_text_string(_pth) for _pth in self.process.systemEnvironment()] baseshell.add_pathlist_to_PYTHONPATH(env, pythonpath) processEnvironment = QProcessEnvironment() for envItem in env: envName, separator, envValue = envItem.partition('=') processEnvironment.insert(envName, envValue) self.process.setProcessEnvironment(processEnvironment) self.output = '' self.error_output = '' p_args = ['-m', 'cProfile', '-o', self.DATAPATH] if os.name == 'nt': # On Windows, one has to replace backslashes by slashes to avoid # confusion with escape characters (otherwise, for example, '\t' # will be interpreted as a tabulation): p_args.append(osp.normpath(filename).replace(os.sep, '/')) else: p_args.append(filename) if args: p_args.extend(shell_split(args)) executable = sys.executable if executable.endswith("spyder.exe"): # py2exe distribution executable = "python.exe" self.process.start(executable, p_args) running = self.process.waitForStarted() self.set_running_state(running) if not running: QMessageBox.critical(self, _("Error"), _("Process failed to start")) def set_running_state(self, state=True): self.start_button.setEnabled(not state) self.stop_button.setEnabled(state) def read_output(self, error=False): if error: self.process.setReadChannel(QProcess.StandardError) else: self.process.setReadChannel(QProcess.StandardOutput) qba = QByteArray() while self.process.bytesAvailable(): if error: qba += self.process.readAllStandardError() else: qba += self.process.readAllStandardOutput() text = to_text_string( locale_codec.toUnicode(qba.data()) ) if error: self.error_output += text else: self.output += text def finished(self, exit_code, exit_status): self.set_running_state(False) self.show_errorlog() # If errors occurred, show them. self.output = self.error_output + self.output # FIXME: figure out if show_data should be called here or # as a signal from the combobox self.show_data(justanalyzed=True) def kill_if_running(self): if self.process is not None: if self.process.state() == QProcess.Running: self.process.kill() self.process.waitForFinished() def show_data(self, justanalyzed=False): if not justanalyzed: self.output = None self.log_button.setEnabled(self.output is not None \ and len(self.output) > 0) self.kill_if_running() filename = to_text_string(self.filecombo.currentText()) if not filename: return self.datelabel.setText(_('Sorting data, please wait...')) QApplication.processEvents() self.datatree.load_data(self.DATAPATH) self.datatree.show_tree() text_style = "<span style=\'color: #444444\'><b>%s </b></span>" date_text = text_style % time.strftime("%d %b %Y %H:%M", time.localtime()) self.datelabel.setText(date_text)
class PylintWidget(QWidget): """ Pylint widget """ DATAPATH = get_conf_path('pylint.results') VERSION = '1.1.0' redirect_stdio = Signal(bool) def __init__(self, parent, max_entries=100): QWidget.__init__(self, parent) self.setWindowTitle("Pylint") self.output = None self.error_output = None self.max_entries = max_entries self.rdata = [] if osp.isfile(self.DATAPATH): try: data = pickle.loads(open(self.DATAPATH, 'rb').read()) if data[0] == self.VERSION: self.rdata = data[1:] except (EOFError, ImportError): pass self.filecombo = PythonModulesComboBox(self) if self.rdata: self.remove_obsolete_items() self.filecombo.addItems(self.get_filenames()) self.start_button = create_toolbutton(self, icon=ima.icon('run'), text=_("Analyze"), tip=_("Run analysis"), triggered=self.start, text_beside_icon=True) self.stop_button = create_toolbutton(self, icon=ima.icon('stop'), text=_("Stop"), tip=_("Stop current analysis"), text_beside_icon=True) self.filecombo.valid.connect(self.start_button.setEnabled) self.filecombo.valid.connect(self.show_data) browse_button = create_toolbutton(self, icon=ima.icon('fileopen'), tip=_('Select Python file'), triggered=self.select_file) self.ratelabel = QLabel() self.datelabel = QLabel() self.log_button = create_toolbutton(self, icon=ima.icon('log'), text=_("Output"), text_beside_icon=True, tip=_("Complete output"), triggered=self.show_log) self.treewidget = ResultsTree(self) hlayout1 = QHBoxLayout() hlayout1.addWidget(self.filecombo) hlayout1.addWidget(browse_button) hlayout1.addWidget(self.start_button) hlayout1.addWidget(self.stop_button) hlayout2 = QHBoxLayout() hlayout2.addWidget(self.ratelabel) hlayout2.addStretch() hlayout2.addWidget(self.datelabel) hlayout2.addStretch() hlayout2.addWidget(self.log_button) layout = QVBoxLayout() layout.addLayout(hlayout1) layout.addLayout(hlayout2) layout.addWidget(self.treewidget) self.setLayout(layout) self.process = None self.set_running_state(False) self.show_data() def analyze(self, filename): filename = to_text_string(filename) # filename is a QString instance self.kill_if_running() index, _data = self.get_data(filename) if index is None: self.filecombo.addItem(filename) self.filecombo.setCurrentIndex(self.filecombo.count()-1) else: self.filecombo.setCurrentIndex(self.filecombo.findText(filename)) self.filecombo.selected() if self.filecombo.is_valid(): self.start() @Slot() def select_file(self): self.redirect_stdio.emit(False) filename, _selfilter = getopenfilename(self, _("Select Python file"), getcwd(), _("Python files")+" (*.py ; *.pyw)") self.redirect_stdio.emit(True) if filename: self.analyze(filename) def remove_obsolete_items(self): """Removing obsolete items""" self.rdata = [(filename, data) for filename, data in self.rdata if is_module_or_package(filename)] def get_filenames(self): return [filename for filename, _data in self.rdata] def get_data(self, filename): filename = osp.abspath(filename) for index, (fname, data) in enumerate(self.rdata): if fname == filename: return index, data else: return None, None def set_data(self, filename, data): filename = osp.abspath(filename) index, _data = self.get_data(filename) if index is not None: self.rdata.pop(index) self.rdata.insert(0, (filename, data)) self.save() def save(self): while len(self.rdata) > self.max_entries: self.rdata.pop(-1) pickle.dump([self.VERSION]+self.rdata, open(self.DATAPATH, 'wb'), 2) @Slot() def show_log(self): if self.output: TextEditor(self.output, title=_("Pylint output"), readonly=True, size=(700, 500)).exec_() @Slot() def start(self): filename = to_text_string(self.filecombo.currentText()) self.process = QProcess(self) self.process.setProcessChannelMode(QProcess.SeparateChannels) self.process.setWorkingDirectory(osp.dirname(filename)) self.process.readyReadStandardOutput.connect(self.read_output) self.process.readyReadStandardError.connect( lambda: self.read_output(error=True)) self.process.finished.connect(lambda ec, es=QProcess.ExitStatus: self.finished(ec, es)) self.stop_button.clicked.connect(self.process.kill) self.output = '' self.error_output = '' plver = PYLINT_VER if plver is not None: p_args = ['-m', 'pylint', '--output-format=text'] if plver.split('.')[0] == '0': p_args += ['-i', 'yes'] else: # Option '-i' (alias for '--include-ids') was removed in pylint # 1.0 p_args += ["--msg-template='{msg_id}:{line:3d},"\ "{column}: {obj}: {msg}"] p_args += [osp.basename(filename)] else: p_args = [osp.basename(filename)] self.process.start(sys.executable, p_args) running = self.process.waitForStarted() self.set_running_state(running) if not running: QMessageBox.critical(self, _("Error"), _("Process failed to start")) def set_running_state(self, state=True): self.start_button.setEnabled(not state) self.stop_button.setEnabled(state) def read_output(self, error=False): if error: self.process.setReadChannel(QProcess.StandardError) else: self.process.setReadChannel(QProcess.StandardOutput) qba = QByteArray() while self.process.bytesAvailable(): if error: qba += self.process.readAllStandardError() else: qba += self.process.readAllStandardOutput() text = to_text_string( locale_codec.toUnicode(qba.data()) ) if error: self.error_output += text else: self.output += text def finished(self, exit_code, exit_status): self.set_running_state(False) if not self.output: if self.error_output: QMessageBox.critical(self, _("Error"), self.error_output) print("pylint error:\n\n" + self.error_output, file=sys.stderr) return # Convention, Refactor, Warning, Error results = {'C:': [], 'R:': [], 'W:': [], 'E:': []} txt_module = '************* Module ' module = '' # Should not be needed - just in case something goes wrong for line in self.output.splitlines(): if line.startswith(txt_module): # New module module = line[len(txt_module):] continue # Supporting option include-ids: ('R3873:' instead of 'R:') if not re.match('^[CRWE]+([0-9]{4})?:', line): continue i1 = line.find(':') if i1 == -1: continue msg_id = line[:i1] i2 = line.find(':', i1+1) if i2 == -1: continue line_nb = line[i1+1:i2].strip() if not line_nb: continue line_nb = int(line_nb.split(',')[0]) message = line[i2+1:] item = (module, line_nb, message, msg_id) results[line[0]+':'].append(item) # Rate rate = None txt_rate = 'Your code has been rated at ' i_rate = self.output.find(txt_rate) if i_rate > 0: i_rate_end = self.output.find('/10', i_rate) if i_rate_end > 0: rate = self.output[i_rate+len(txt_rate):i_rate_end] # Previous run previous = '' if rate is not None: txt_prun = 'previous run: ' i_prun = self.output.find(txt_prun, i_rate_end) if i_prun > 0: i_prun_end = self.output.find('/10', i_prun) previous = self.output[i_prun+len(txt_prun):i_prun_end] filename = to_text_string(self.filecombo.currentText()) self.set_data(filename, (time.localtime(), rate, previous, results)) self.output = self.error_output + self.output self.show_data(justanalyzed=True) def kill_if_running(self): if self.process is not None: if self.process.state() == QProcess.Running: self.process.kill() self.process.waitForFinished() def show_data(self, justanalyzed=False): if not justanalyzed: self.output = None self.log_button.setEnabled(self.output is not None \ and len(self.output) > 0) self.kill_if_running() filename = to_text_string(self.filecombo.currentText()) if not filename: return _index, data = self.get_data(filename) if data is None: text = _('Source code has not been rated yet.') self.treewidget.clear_results() date_text = '' else: datetime, rate, previous_rate, results = data if rate is None: text = _('Analysis did not succeed ' '(see output for more details).') self.treewidget.clear_results() date_text = '' else: text_style = "<span style=\'color: #444444\'><b>%s </b></span>" rate_style = "<span style=\'color: %s\'><b>%s</b></span>" prevrate_style = "<span style=\'color: #666666\'>%s</span>" color = "#FF0000" if float(rate) > 5.: color = "#22AA22" elif float(rate) > 3.: color = "#EE5500" text = _('Global evaluation:') text = (text_style % text)+(rate_style % (color, ('%s/10' % rate))) if previous_rate: text_prun = _('previous run:') text_prun = ' (%s %s/10)' % (text_prun, previous_rate) text += prevrate_style % text_prun self.treewidget.set_results(filename, results) date = to_text_string(time.strftime("%d %b %Y %H:%M", datetime), encoding='utf8') date_text = text_style % date self.ratelabel.setText(text) self.datelabel.setText(date_text)
class CoalaWidget(QWidget): """ coala Widget """ DATAPATH = get_conf_path('coala.results') VERSION = '' redirect_stdio = Signal(bool) def __init__(self, parent, max_entries=100, options_button=None, text_color=None, prevrate_color=None): QWidget.__init__(self, parent) self.setWindowTitle('coala') self.output = None self.error_output = None self.text_color = text_color self.prevrate_color = prevrate_color self.max_entries = max_entries self.rdata = [] if osp.isfile(self.DATAPATH): try: data = pickle.loads(open(self.DATAPATH, 'rb').read()) self.rdata = data[:] except (EOFError, ImportError): print('error!!') pass self.filecombo = PythonModulesComboBox(self) self.start_button = create_toolbutton(self, icon=ima.icon('run'), text=_("Analyze"), tip=_("Run analysis"), triggered=self.start, text_beside_icon=True) self.stop_button = create_toolbutton(self, icon=ima.icon('stop'), text=_("Stop"), tip=_("Stop current analysis"), text_beside_icon=True) self.filecombo.valid.connect(self.start_button.setEnabled) self.filecombo.valid.connect(self.show_data) browse_button = create_toolbutton(self, icon=ima.icon('fileopen'), tip=_('Select Python file'), triggered=self.select_file) self.ratelabel = QLabel() self.datelabel = QLabel() self.log_button = create_toolbutton(self, icon=ima.icon('log'), text=_("Output"), text_beside_icon=True, tip=_("Complete output"), triggered=self.show_log) self.treewidget = ResultsTree(self) hlayout1 = QHBoxLayout() hlayout1.addWidget(self.filecombo) hlayout1.addWidget(browse_button) hlayout1.addWidget(self.start_button) hlayout1.addWidget(self.stop_button) if options_button: hlayout1.addWidget(options_button) hlayout2 = QHBoxLayout() hlayout2.addWidget(self.ratelabel) hlayout2.addStretch() hlayout2.addWidget(self.datelabel) hlayout2.addStretch() hlayout2.addWidget(self.log_button) layout = QVBoxLayout() layout.addLayout(hlayout1) layout.addLayout(hlayout2) layout.addWidget(self.treewidget) self.setLayout(layout) self.process = None self.set_running_state(False) self.show_data() if self.rdata: self.remove_obsolete_items() self.filecombo.addItems(self.get_filenames()) self.start_button.setEnabled(self.filecombo.is_valid()) else: self.start_button.setEnabled(False) def analyze(self, filename): filename = to_text_string(filename) # filename is a QString instance self.kill_if_running() index, _data = self.get_data(filename) if index is None: self.filecombo.addItem(filename) self.filecombo.setCurrentIndex(self.filecombo.count() - 1) else: self.filecombo.setCurrentIndex(self.filecombo.findText(filename)) self.filecombo.selected() if self.filecombo.is_valid(): self.start() @Slot() def select_file(self): self.redirect_stdio.emit(False) filename, _selfilter = getopenfilename( self, _("Select Python file"), getcwd_or_home(), _("Python files") + " (*.py ; *.pyw)") self.redirect_stdio.emit(True) if filename: self.analyze(filename) def remove_obsolete_items(self): """Removing obsolete items""" self.rdata = [(filename, data) for filename, data in self.rdata if is_module_or_package(filename)] def get_filenames(self): return [filename for filename, _data in self.rdata] def get_data(self, filename): filename = osp.abspath(filename) for index, (fname, data) in enumerate(self.rdata): if fname == filename: return index, data else: return None, None def set_data(self, filename, data): filename = osp.abspath(filename) index, _data = self.get_data(filename) if index is not None: self.rdata.pop(index) self.rdata.insert(0, (filename, data)) self.save() def save(self): while len(self.rdata) > self.max_entries: self.rdata.pop(-1) pickle.dump([self.VERSION] + self.rdata, open(self.DATAPATH, 'wb'), 2) @Slot() def show_log(self): if self.output: TextEditor(self.output, title=_("coala output"), readonly=True, size=(700, 500)).exec_() @Slot() def start(self): filename = to_text_string(self.filecombo.currentText()) self.process = QProcess(self) self.process.setProcessChannelMode(QProcess.SeparateChannels) self.process.setWorkingDirectory(osp.dirname(filename)) self.process.readyReadStandardOutput.connect(self.read_output) self.process.readyReadStandardError.connect( lambda: self.read_output(error=True)) self.process.finished.connect( lambda ec, es=QProcess.ExitStatus: self.finished(ec, es)) self.stop_button.clicked.connect(self.process.kill) self.output = '' self.error_output = '' clver = COALA_VER if clver is not None: c_args = ['-m', 'run_coala'] self.process.start(sys.executable, c_args) running = self.process.waitForStarted() self.set_running_state(running) if not running: QMessageBox.critical(self, _("Error"), _("Process failed to start")) def set_running_state(self, state=True): self.start_button.setEnabled(not state) self.stop_button.setEnabled(state) def read_output(self, error=False): if error: self.process.setReadChannel(QProcess.StandardError) else: self.process.setReadChannel(QProcess.StandardOutput) qba = QByteArray() while self.process.bytesAvailable(): if error: qba += self.process.readAllStandardError() else: qba += self.process.readAllStandardOutput() text = to_text_string(locale_codec.toUnicode(qba.data())) if error: self.error_output += text else: self.output += text def finished(self, exit_code, exit_status): self.set_running_state(False) if not self.output: if self.error_output: QMessageBox.critical(self, _("Error"), self.error_output) print("coala error:\n\n" + self.error_output, file=sys.stderr) return results = {'C:': []} literal_dict = ast.literal_eval(self.output) line_numbers = [] char_numbers = [] bear_values = [] msg_values = [] for line in literal_dict['C']: print(line) for i in line: line_num = re.compile('(.+)~') val = line_num.findall(i) for line_nb in val: if line_nb: line_numbers.append(line_nb) for j in line: char_num = re.compile('(.*);') val = char_num.findall(j) for char_nm in val: if char_nm: char_numbers.append(char_nm) for k in line: bear_val = re.compile('(.*):') val = bear_val.findall(k) for bear_val in val: if bear_val: bear_values.append(bear_val) for m in line: msg_val = re.compile(':(.*)') val = msg_val.findall(m) for msg_val in val: if msg_val: msg_values.append(msg_val) item = list(zip(line_numbers, char_numbers, bear_values, msg_values)) for i in item: results['C:'].append(i) filename = to_text_string(self.filecombo.currentText()) self.set_data(filename, results) self.output = self.error_output + self.output self.show_data(justanalyzed=True) def kill_if_running(self): if self.process is not None: if self.process.state() == QProcess.Running: self.process.kill() self.process.waitForFinished() def show_data(self, justanalyzed=False): if not justanalyzed: self.output = None self.log_button.setEnabled(self.output is not None \ and len(self.output) > 0) self.kill_if_running() filename = to_text_string(self.filecombo.currentText()) if not filename: return _index, data = self.get_data(filename) if data is None: self.treewidget.clear_results() else: results = data self.treewidget.set_results(filename, results)
class PylintWidget(QWidget): """ Pylint widget """ DATAPATH = get_conf_path('pylint.results') VERSION = '1.1.0' redirect_stdio = Signal(bool) def __init__(self, parent, max_entries=100): QWidget.__init__(self, parent) self.setWindowTitle("Pylint") self.output = None self.error_output = None self.max_entries = max_entries self.rdata = [] if osp.isfile(self.DATAPATH): try: data = pickle.loads(open(self.DATAPATH, 'rb').read()) if data[0] == self.VERSION: self.rdata = data[1:] except (EOFError, ImportError): pass self.filecombo = PythonModulesComboBox(self) self.start_button = create_toolbutton(self, icon=ima.icon('run'), text=_("Analyze"), tip=_("Run analysis"), triggered=self.start, text_beside_icon=True) self.stop_button = create_toolbutton(self, icon=ima.icon('stop'), text=_("Stop"), tip=_("Stop current analysis"), text_beside_icon=True) self.filecombo.valid.connect(self.start_button.setEnabled) self.filecombo.valid.connect(self.show_data) browse_button = create_toolbutton(self, icon=ima.icon('fileopen'), tip=_('Select Python file'), triggered=self.select_file) self.ratelabel = QLabel() self.datelabel = QLabel() self.log_button = create_toolbutton(self, icon=ima.icon('log'), text=_("Output"), text_beside_icon=True, tip=_("Complete output"), triggered=self.show_log) self.treewidget = ResultsTree(self) hlayout1 = QHBoxLayout() hlayout1.addWidget(self.filecombo) hlayout1.addWidget(browse_button) hlayout1.addWidget(self.start_button) hlayout1.addWidget(self.stop_button) hlayout2 = QHBoxLayout() hlayout2.addWidget(self.ratelabel) hlayout2.addStretch() hlayout2.addWidget(self.datelabel) hlayout2.addStretch() hlayout2.addWidget(self.log_button) layout = QVBoxLayout() layout.addLayout(hlayout1) layout.addLayout(hlayout2) layout.addWidget(self.treewidget) self.setLayout(layout) self.process = None self.set_running_state(False) self.show_data() if self.rdata: self.remove_obsolete_items() self.filecombo.addItems(self.get_filenames()) else: self.start_button.setEnabled(False) def analyze(self, filename): filename = to_text_string(filename) # filename is a QString instance self.kill_if_running() index, _data = self.get_data(filename) if index is None: self.filecombo.addItem(filename) self.filecombo.setCurrentIndex(self.filecombo.count() - 1) else: self.filecombo.setCurrentIndex(self.filecombo.findText(filename)) self.filecombo.selected() if self.filecombo.is_valid(): self.start() @Slot() def select_file(self): self.redirect_stdio.emit(False) filename, _selfilter = getopenfilename( self, _("Select Python file"), getcwd_or_home(), _("Python files") + " (*.py ; *.pyw)") self.redirect_stdio.emit(True) if filename: self.analyze(filename) def remove_obsolete_items(self): """Removing obsolete items""" self.rdata = [(filename, data) for filename, data in self.rdata if is_module_or_package(filename)] def get_filenames(self): return [filename for filename, _data in self.rdata] def get_data(self, filename): filename = osp.abspath(filename) for index, (fname, data) in enumerate(self.rdata): if fname == filename: return index, data else: return None, None def set_data(self, filename, data): filename = osp.abspath(filename) index, _data = self.get_data(filename) if index is not None: self.rdata.pop(index) self.rdata.insert(0, (filename, data)) self.save() def save(self): while len(self.rdata) > self.max_entries: self.rdata.pop(-1) pickle.dump([self.VERSION] + self.rdata, open(self.DATAPATH, 'wb'), 2) @Slot() def show_log(self): if self.output: TextEditor(self.output, title=_("Pylint output"), readonly=True, size=(700, 500)).exec_() @Slot() def start(self): filename = to_text_string(self.filecombo.currentText()) self.process = QProcess(self) self.process.setProcessChannelMode(QProcess.SeparateChannels) self.process.setWorkingDirectory(osp.dirname(filename)) self.process.readyReadStandardOutput.connect(self.read_output) self.process.readyReadStandardError.connect( lambda: self.read_output(error=True)) self.process.finished.connect( lambda ec, es=QProcess.ExitStatus: self.finished(ec, es)) self.stop_button.clicked.connect(self.process.kill) self.output = '' self.error_output = '' plver = PYLINT_VER if plver is not None: p_args = ['-m', 'pylint', '--output-format=text'] if plver.split('.')[0] == '0': p_args += ['-i', 'yes'] else: # Option '-i' (alias for '--include-ids') was removed in pylint # 1.0 p_args += ["--msg-template='{msg_id}:{line:3d},"\ "{column}: {obj}: {msg}"] p_args += [osp.basename(filename)] else: p_args = [osp.basename(filename)] self.process.start(sys.executable, p_args) running = self.process.waitForStarted() self.set_running_state(running) if not running: QMessageBox.critical(self, _("Error"), _("Process failed to start")) def set_running_state(self, state=True): self.start_button.setEnabled(not state) self.stop_button.setEnabled(state) def read_output(self, error=False): if error: self.process.setReadChannel(QProcess.StandardError) else: self.process.setReadChannel(QProcess.StandardOutput) qba = QByteArray() while self.process.bytesAvailable(): if error: qba += self.process.readAllStandardError() else: qba += self.process.readAllStandardOutput() text = to_text_string(locale_codec.toUnicode(qba.data())) if error: self.error_output += text else: self.output += text def finished(self, exit_code, exit_status): self.set_running_state(False) if not self.output: if self.error_output: QMessageBox.critical(self, _("Error"), self.error_output) print("pylint error:\n\n" + self.error_output, file=sys.stderr) return # Convention, Refactor, Warning, Error results = {'C:': [], 'R:': [], 'W:': [], 'E:': []} txt_module = '************* Module ' module = '' # Should not be needed - just in case something goes wrong for line in self.output.splitlines(): if line.startswith(txt_module): # New module module = line[len(txt_module):] continue # Supporting option include-ids: ('R3873:' instead of 'R:') if not re.match('^[CRWE]+([0-9]{4})?:', line): continue i1 = line.find(':') if i1 == -1: continue msg_id = line[:i1] i2 = line.find(':', i1 + 1) if i2 == -1: continue line_nb = line[i1 + 1:i2].strip() if not line_nb: continue line_nb = int(line_nb.split(',')[0]) message = line[i2 + 1:] item = (module, line_nb, message, msg_id) results[line[0] + ':'].append(item) # Rate rate = None txt_rate = 'Your code has been rated at ' i_rate = self.output.find(txt_rate) if i_rate > 0: i_rate_end = self.output.find('/10', i_rate) if i_rate_end > 0: rate = self.output[i_rate + len(txt_rate):i_rate_end] # Previous run previous = '' if rate is not None: txt_prun = 'previous run: ' i_prun = self.output.find(txt_prun, i_rate_end) if i_prun > 0: i_prun_end = self.output.find('/10', i_prun) previous = self.output[i_prun + len(txt_prun):i_prun_end] filename = to_text_string(self.filecombo.currentText()) self.set_data(filename, (time.localtime(), rate, previous, results)) self.output = self.error_output + self.output self.show_data(justanalyzed=True) def kill_if_running(self): if self.process is not None: if self.process.state() == QProcess.Running: self.process.kill() self.process.waitForFinished() def show_data(self, justanalyzed=False): if not justanalyzed: self.output = None self.log_button.setEnabled(self.output is not None \ and len(self.output) > 0) self.kill_if_running() filename = to_text_string(self.filecombo.currentText()) if not filename: return _index, data = self.get_data(filename) if data is None: text = _('Source code has not been rated yet.') self.treewidget.clear_results() date_text = '' else: datetime, rate, previous_rate, results = data if rate is None: text = _('Analysis did not succeed ' '(see output for more details).') self.treewidget.clear_results() date_text = '' else: text_style = "<span style=\'color: #444444\'><b>%s </b></span>" rate_style = "<span style=\'color: %s\'><b>%s</b></span>" prevrate_style = "<span style=\'color: #666666\'>%s</span>" color = "#FF0000" if float(rate) > 5.: color = "#22AA22" elif float(rate) > 3.: color = "#EE5500" text = _('Global evaluation:') text = (text_style % text) + (rate_style % (color, ('%s/10' % rate))) if previous_rate: text_prun = _('previous run:') text_prun = ' (%s %s/10)' % (text_prun, previous_rate) text += prevrate_style % text_prun self.treewidget.set_results(filename, results) date = to_text_string(time.strftime("%d %b %Y %H:%M", datetime), encoding='utf8') date_text = text_style % date self.ratelabel.setText(text) self.datelabel.setText(date_text)
class CondaProcess(QObject): """Conda API modified to work with QProcess instead of popen.""" # Signals sig_finished = Signal(str, object, str) sig_partial = Signal(str, object, str) sig_started = Signal() ENCODING = 'ascii' ROOT_PREFIX = None def __init__(self, parent): QObject.__init__(self, parent) self._parent = parent self._output = None self._partial = None self._stdout = None self._error = None self._parse = False self._function_called = '' self._name = None self._process = QProcess() self.set_root_prefix() # Signals self._process.finished.connect(self._call_conda_ready) self._process.readyReadStandardOutput.connect(self._call_conda_partial) # --- Helpers # ------------------------------------------------------------------------- def _is_running(self): return self._process.state() != QProcess.NotRunning def _is_not_running(self): return self._process.state() == QProcess.NotRunning def _call_conda_partial(self): """ """ stdout = self._process.readAllStandardOutput() stdout = handle_qbytearray(stdout, CondaProcess.ENCODING) stderr = self._process.readAllStandardError() stderr = handle_qbytearray(stderr, CondaProcess.ENCODING) if self._parse: try: self._output = json.loads(stdout) except Exception: # Result is a partial json. Can only be parsed when finished self._output = stdout else: self._output = stdout self._partial = self._output self._stdout = self._output self._error = stderr self.sig_partial.emit(self._function_called, self._partial, self._error) def _call_conda_ready(self): """Function called when QProcess in _call_conda finishes task.""" function = self._function_called if self._stdout is None: stdout = to_text_string(self._process.readAllStandardOutput(), encoding=CondaProcess.ENCODING) else: stdout = self._stdout if self._error is None: stderr = to_text_string(self._process.readAllStandardError(), encoding=CondaProcess.ENCODING) else: stderr = self._error if function == 'get_conda_version': pat = re.compile(r'conda:?\s+(\d+\.\d\S+|unknown)') m = pat.match(stderr.strip()) if m is None: m = pat.match(stdout.strip()) if m is None: raise Exception('output did not match: {0}'.format(stderr)) self._output = m.group(1) elif function == 'config_path': result = self._output self._output = result['rc_path'] elif function == 'config_get': result = self._output self._output = result['get'] elif (function == 'config_delete' or function == 'config_add' or function == 'config_set' or function == 'config_remove'): result = self._output self._output = result.get('warnings', []) elif function == 'pip': result = [] lines = self._output.split('\n') for line in lines: if '<pip>' in line: temp = line.split()[:-1] + ['pip'] result.append('-'.join(temp)) self._output = result if stderr.strip(): self._error = stderr self._parse = False self.sig_finished.emit(self._function_called, self._output, self._error) def _abspath(self, abspath): """ """ if abspath: if sys.platform == 'win32': python = join(CondaProcess.ROOT_PREFIX, 'python.exe') conda = join(CondaProcess.ROOT_PREFIX, 'Scripts', 'conda-script.py') else: python = join(CondaProcess.ROOT_PREFIX, 'bin/python') conda = join(CondaProcess.ROOT_PREFIX, 'bin/conda') cmd_list = [python, conda] else: # Just use whatever conda/pip is on the path cmd_list = ['conda'] return cmd_list def _call_conda(self, extra_args, abspath=True): """ """ if self._is_not_running(): cmd_list = self._abspath(abspath) cmd_list.extend(extra_args) self._error, self._output = None, None self._process.start(cmd_list[0], cmd_list[1:]) self.sig_started.emit() def _call_pip(self, name=None, prefix=None, extra_args=None): """ """ if self._is_not_running(): cmd_list = self._pip_cmd(name=name, prefix=prefix) cmd_list.extend(extra_args) self._error, self._output = None, None self._parse = False self._process.start(cmd_list[0], cmd_list[1:]) self.sig_started.emit() def _call_and_parse(self, extra_args, abspath=True): """ """ self._parse = True self._call_conda(extra_args, abspath=abspath) def _setup_install_commands_from_kwargs(self, kwargs, keys=tuple()): """ """ cmd_list = [] if kwargs.get('override_channels', False) and 'channel' not in kwargs: raise TypeError('conda search: override_channels requires channel') if 'env' in kwargs: cmd_list.extend(['--name', kwargs.pop('env')]) if 'prefix' in kwargs: cmd_list.extend(['--prefix', kwargs.pop('prefix')]) if 'channel' in kwargs: channel = kwargs.pop('channel') if isinstance(channel, str): cmd_list.extend(['--channel', channel]) else: cmd_list.append('--channel') cmd_list.extend(channel) for key in keys: if key in kwargs and kwargs[key]: cmd_list.append('--' + key.replace('_', '-')) return cmd_list # --- Public api # ------------------------------------------------------------------------ @staticmethod def linked(prefix, as_spec=False): """ Return the (set of canonical names) of linked packages in `prefix`. """ if not isdir(prefix): raise Exception('no such directory: {0}'.format(prefix)) meta_dir = join(prefix, 'conda-meta') if not isdir(meta_dir): # We might have nothing in linked (and no conda-meta directory) result = set() result = set(fn[:-5] for fn in os.listdir(meta_dir) if fn.endswith('.json')) new_result = [] if as_spec: for r in result: n, v, b = CondaProcess.split_canonical_name(r) new_result.append("{0}={1}".format(n, v)) result = "\n".join(new_result) return result @staticmethod def split_canonical_name(cname): """ Split a canonical package name into (name, version, build) strings. """ result = tuple(cname.rsplit('-', 2)) return result def set_root_prefix(self, prefix=None): """ Set the prefix to the root environment (default is /opt/anaconda). """ if prefix: CondaProcess.ROOT_PREFIX = prefix else: # Find conda instance, and then use info() to get 'root_prefix' if CondaProcess.ROOT_PREFIX is None: info = self.info(abspath=False) CondaProcess.ROOT_PREFIX = info['root_prefix'] def get_conda_version(self): """ Return the version of conda being used (invoked) as a string. """ if self._is_not_running(): self._function_called = 'get_conda_version' self._call_conda(['--version']) def get_envs(self, emit=False): """ Return all of the (named) environments (this does not include the root environment), as a list of absolute paths to their prefixes. """ if self._is_not_running(): info = self.info() result = info['envs'] if emit: self.sig_finished.emit('get_envs', result, "") return result def get_prefix_envname(self, name, emit=False): """ Given the name of an environment return its full prefix path, or None if it cannot be found. """ if self._is_not_running(): if name == 'root': prefix = CondaProcess.ROOT_PREFIX for env_prefix in self.get_envs(): if basename(env_prefix) == name: prefix = env_prefix break if emit: self.sig_finished.emit('get_prefix_envname', prefix, "") return prefix def info(self, abspath=True, emit=False): """ Return a dictionary with configuration information. No guarantee is made about which keys exist. Therefore this function should only be used for testing and debugging. """ if self._is_not_running(): qprocess = QProcess() cmd_list = self._abspath(abspath) cmd_list.extend(['info', '--json']) qprocess.start(cmd_list[0], cmd_list[1:]) qprocess.waitForFinished() output = qprocess.readAllStandardOutput() output = handle_qbytearray(output, CondaProcess.ENCODING) info = json.loads(output) if emit: self.sig_finished.emit("info", str(info), "") return info def package_info(self, package, abspath=True): """ Return a dictionary with package information. Structure is { 'package_name': [{ 'depends': list, 'version': str, 'name': str, 'license': str, 'fn': ..., ... }] } """ if self._is_not_running(): self._function_called = 'package_info' self._call_and_parse(['info', package, '--json'], abspath=abspath) def search(self, regex=None, spec=None, **kwargs): """ Search for packages. """ cmd_list = ['search', '--json'] if regex and spec: raise TypeError('conda search: only one of regex or spec allowed') if regex: cmd_list.append(regex) if spec: cmd_list.extend(['--spec', spec]) if 'platform' in kwargs: platform = kwargs.pop('platform') platforms = ('win-32', 'win-64', 'osx-64', 'linux-32', 'linux-64') if platform not in platforms: raise TypeError('conda search: platform must be one of ' + ', '.join(platforms)) cmd_list.extend(['--platform', platform]) cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ('canonical', 'unknown', 'use_index_cache', 'outdated', 'override_channels'))) if self._is_not_running(): self._function_called = 'search' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def share(self, prefix): """ Create a "share package" of the environment located in `prefix`, and return a dictionary with (at least) the following keys: - 'prefix': the full path to the created package - 'warnings': a list of warnings This file is created in a temp directory, and it is the callers responsibility to remove this directory (after the file has been handled in some way). """ if self._is_not_running: self._function_called = 'share' self._call_and_parse(['share', '--json', '--prefix', prefix]) def create(self, name=None, prefix=None, pkgs=None): """ Create an environment either by 'name' or 'prefix' with a specified set of packages. """ # TODO: Fix temporal hack if not pkgs or not isinstance(pkgs, (list, tuple, str)): raise TypeError('must specify a list of one or more packages to ' 'install into new environment') cmd_list = ['create', '--yes', '--quiet', '--json', '--mkdir'] if name: ref = name search = [os.path.join(d, name) for d in self.info()['envs_dirs']] cmd_list.extend(['--name', name]) elif prefix: ref = prefix search = [prefix] cmd_list.extend(['--prefix', prefix]) else: raise TypeError("Must specify either an environment 'name' or a " "'prefix' for new environment.") if any(os.path.exists(path) for path in search): raise CondaEnvExistsError('Conda environment [%s] already exists' % ref) # TODO: Fix temporal hack if isinstance(pkgs, (list, tuple)): cmd_list.extend(pkgs) elif isinstance(pkgs, str): cmd_list.extend(['--file', pkgs]) if self._is_not_running: self._function_called = 'create' self._call_conda(cmd_list) def install(self, name=None, prefix=None, pkgs=None, dep=True): """ Install packages into an environment either by 'name' or 'prefix' with a specified set of packages """ # TODO: Fix temporal hack if not pkgs or not isinstance(pkgs, (list, tuple, str)): raise TypeError('must specify a list of one or more packages to ' 'install into existing environment') cmd_list = ['install', '--yes', '--json', '--force-pscheck'] if name: cmd_list.extend(['--name', name]) elif prefix: cmd_list.extend(['--prefix', prefix]) else: # just install into the current environment, whatever that is pass # TODO: Fix temporal hack if isinstance(pkgs, (list, tuple)): cmd_list.extend(pkgs) elif isinstance(pkgs, str): cmd_list.extend(['--file', pkgs]) if not dep: cmd_list.extend(['--no-deps']) if self._is_not_running: self._function_called = 'install' self._call_conda(cmd_list) def update(self, *pkgs, **kwargs): """ Update package(s) (in an environment) by name. """ cmd_list = ['update', '--json', '--quiet', '--yes'] if not pkgs and not kwargs.get('all'): raise TypeError("Must specify at least one package to update, \ or all=True.") cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ('dry_run', 'no_deps', 'override_channels', 'no_pin', 'force', 'all', 'use_index_cache', 'use_local', 'alt_hint'))) cmd_list.extend(pkgs) if self._is_not_running: self._function_called = 'update' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def remove(self, *pkgs, **kwargs): """ Remove a package (from an environment) by name. Returns { success: bool, (this is always true), (other information) } """ cmd_list = ['remove', '--json', '--quiet', '--yes', '--force-pscheck'] if not pkgs and not kwargs.get('all'): raise TypeError("Must specify at least one package to remove, \ or all=True.") if kwargs.get('name') and kwargs.get('prefix'): raise TypeError("conda remove: At most one of 'name', 'prefix' " "allowed") if kwargs.get('name'): cmd_list.extend(['--name', kwargs.pop('name')]) if kwargs.get('prefix'): cmd_list.extend(['--prefix', kwargs.pop('prefix')]) cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ('dry_run', 'features', 'override_channels', 'no_pin', 'force', 'all'))) cmd_list.extend(pkgs) if self._is_not_running: self._function_called = 'remove' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def remove_environment(self, name=None, prefix=None, **kwargs): """ Remove an environment entirely. See ``remove``. """ if self._is_not_running: self._function_called = 'remove_environment' self.remove(name=name, prefix=prefix, all=True, **kwargs) def clone_environment(self, clone, name=None, prefix=None, **kwargs): """ Clone the environment ``clone`` into ``name`` or ``prefix``. """ cmd_list = ['create', '--json', '--quiet'] if (name and prefix) or not (name or prefix): raise TypeError("conda clone_environment: exactly one of 'name' " "or 'prefix' required.") if name: cmd_list.extend(['--name', name]) if prefix: cmd_list.extend(['--prefix', prefix]) cmd_list.extend(['--clone', clone]) cmd_list.extend( self._setup_install_commands_from_kwargs( kwargs, ('dry_run', 'unknown', 'use_index_cache', 'use_local', 'no_pin', 'force', 'all', 'channel', 'override_channels', 'no_default_packages'))) if self._is_not_running(): self._function_called = 'clone_environment' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def clone(self, path, prefix): """ Clone a "share package" (located at `path`) by creating a new environment at `prefix`, and return a dictionary with (at least) the following keys: - 'warnings': a list of warnings The directory `path` is located in, should be some temp directory or some other directory OUTSIDE /opt/anaconda. After calling this function, the original file (at `path`) may be removed (by the caller of this function). The return object is a list of warnings. """ if self._process.state() == QProcess.NotRunning: self._function_called = 'clone' self._call_and_parse(['clone', '--json', '--prefix', prefix, path]) def _setup_config_from_kwargs(kwargs): cmd_list = ['--json', '--force'] if 'file' in kwargs: cmd_list.extend(['--file', kwargs['file']]) if 'system' in kwargs: cmd_list.append('--system') return cmd_list def config_path(self, **kwargs): """ Get the path to the config file. """ cmd_list = ['config', '--get'] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = 'config_path' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def config_get(self, *keys, **kwargs): """ Get the values of configuration keys. Returns a dictionary of values. Note, the key may not be in the dictionary if the key wasn't set in the configuration file. """ cmd_list = ['config', '--get'] cmd_list.extend(keys) cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = 'config_get' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def config_set(self, key, value, **kwargs): """ Set a key to a (bool) value. Returns a list of warnings Conda may have emitted. """ cmd_list = ['config', '--set', key, str(value)] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = 'config_set' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def config_add(self, key, value, **kwargs): """ Add a value to a key. Returns a list of warnings Conda may have emitted. """ cmd_list = ['config', '--add', key, value] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = 'config_add' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def config_remove(self, key, value, **kwargs): """ Remove a value from a key. Returns a list of warnings Conda may have emitted. """ cmd_list = ['config', '--remove', key, value] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = 'config_remove' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def config_delete(self, key, **kwargs): """ Remove a key entirely. Returns a list of warnings Conda may have emitted. """ cmd_list = ['config', '--remove-key', key] cmd_list.extend(self._setup_config_from_kwargs(kwargs)) if self._is_not_running: self._function_called = 'config_delete' self._call_and_parse(cmd_list, abspath=kwargs.get('abspath', True)) def run(self, command, abspath=True): """ Launch the specified app by name or full package name. Returns a dictionary containing the key "fn", whose value is the full package (ending in ``.tar.bz2``) of the app. """ cmd_list = ['run', '--json', command] if self._is_not_running: self._function_called = 'run' self._call_and_parse(cmd_list, abspath=abspath) # --- Additional methods not in conda-api # ------------------------------------------------------------------------ def dependencies(self, name=None, prefix=None, pkgs=None, dep=True): """ Get dependenciy list for packages to be installed into an environment defined either by 'name' or 'prefix'. """ if not pkgs or not isinstance(pkgs, (list, tuple)): raise TypeError('must specify a list of one or more packages to ' 'install into existing environment') cmd_list = ['install', '--dry-run', '--json'] cmd_list = ['install', '--dry-run', '--json', '--force-pscheck'] if not dep: cmd_list.extend(['--no-deps']) if name: cmd_list.extend(['--name', name]) elif prefix: cmd_list.extend(['--prefix', prefix]) else: pass cmd_list.extend(pkgs) if self._is_not_running: self._function_called = 'install_dry' self._call_and_parse(cmd_list) def environment_exists(self, name=None, prefix=None, abspath=True, emit=False): """ Check if an environment exists by 'name' or by 'prefix'. If query is by 'name' only the default conda environments directory is searched. """ if name and prefix: raise TypeError("Exactly one of 'name' or 'prefix' is required.") qprocess = QProcess() cmd_list = self._abspath(abspath) cmd_list.extend(['list', '--json']) if name: cmd_list.extend(['--name', name]) else: cmd_list.extend(['--prefix', prefix]) qprocess.start(cmd_list[0], cmd_list[1:]) qprocess.waitForFinished() output = qprocess.readAllStandardOutput() output = handle_qbytearray(output, CondaProcess.ENCODING) info = json.loads(output) if emit: self.sig_finished.emit("info", unicode(info), "") return 'error' not in info def export(self, filename, name=None, prefix=None, emit=False): """ Export environment by 'prefix' or 'name' as yaml 'filename'. """ if name and prefix: raise TypeError("Exactly one of 'name' or 'prefix' is required.") if self._is_not_running: if name: temporal_envname = name if prefix: temporal_envname = 'tempenv' + int(random.random()*10000000) envs_dir = self.info()['envs_dirs'][0] os.symlink(prefix, os.sep.join([envs_dir, temporal_envname])) cmd = self._abspath(True) cmd.extend(['env', 'export', '--name', temporal_envname, '--file', os.path.abspath(filename)]) qprocess = QProcess() qprocess.start(cmd[0], cmd[1:]) qprocess.waitForFinished() if prefix: os.unlink(os.sep.join([envs_dir, temporal_envname])) def update_environment(self, filename, name=None, prefix=None, emit=False): """ Set environment at 'prefix' or 'name' to match 'filename' spec as yaml. """ if name and prefix: raise TypeError("Exactly one of 'name' or 'prefix' is required.") if self._is_not_running: if name: temporal_envname = name if prefix: temporal_envname = 'tempenv' + int(random.random()*10000000) envs_dir = self.info()['envs_dirs'][0] os.symlink(prefix, os.sep.join([envs_dir, temporal_envname])) cmd = self._abspath(True) cmd.extend(['env', 'update', '--name', temporal_envname, '--file', os.path.abspath(filename)]) qprocess = QProcess() qprocess.start(cmd[0], cmd[1:]) qprocess.waitForFinished() if prefix: os.unlink(os.sep.join([envs_dir, 'tempenv'])) @property def error(self): return self._error @property def output(self): return self._output # --- Pip commands # ------------------------------------------------------------------------- def _pip_cmd(self, name=None, prefix=None): """ Get pip location based on environment `name` or `prefix`. """ if (name and prefix) or not (name or prefix): raise TypeError("conda pip: exactly one of 'name' ""or 'prefix' " "required.") if name and self.environment_exists(name=name): prefix = self.get_prefix_envname(name) if sys.platform == 'win32': python = join(prefix, 'python.exe') # FIXME: pip = join(prefix, 'pip.exe') # FIXME: else: python = join(prefix, 'bin/python') pip = join(prefix, 'bin/pip') cmd_list = [python, pip] return cmd_list def pip_list(self, name=None, prefix=None, abspath=True, emit=False): """ Get list of pip installed packages. """ if (name and prefix) or not (name or prefix): raise TypeError("conda pip: exactly one of 'name' ""or 'prefix' " "required.") if self._is_not_running: cmd_list = self._abspath(abspath) if name: cmd_list.extend(['list', '--name', name]) if prefix: cmd_list.extend(['list', '--prefix', prefix]) qprocess = QProcess() qprocess.start(cmd_list[0], cmd_list[1:]) qprocess.waitForFinished() output = qprocess.readAllStandardOutput() output = handle_qbytearray(output, CondaProcess.ENCODING) result = [] lines = output.split('\n') for line in lines: if '<pip>' in line: temp = line.split()[:-1] + ['pip'] result.append('-'.join(temp)) if emit: self.sig_finished.emit("pip", str(result), "") return result def pip_remove(self, name=None, prefix=None, pkgs=None): """ Remove a pip pacakge in given environment by 'name' or 'prefix'. """ if isinstance(pkgs, list) or isinstance(pkgs, tuple): pkg = ' '.join(pkgs) else: pkg = pkgs extra_args = ['uninstall', '--yes', pkg] if self._is_not_running(): self._function_called = 'pip_remove' self._call_pip(name=name, prefix=prefix, extra_args=extra_args)
class RunPopup(QDialog): def __init__(self, solver, parent): super(RunPopup, self).__init__(parent) self.commandline_option_exe = solver if solver else None self.mfix_available = False self.mfix_exe_cache = {} self.solver_list = {} self.template_values = {} self.cmdline = [] # List of strings self.parent = parent self.project = parent.project self.settings = parent.settings self.project_dir = parent.get_project_dir() self.gui_comments = self.project.mfix_gui_comments self.flag_processes = {} self.title = 'Run Solver' # load ui ui = self.ui = get_ui('run_popup.ui', self) ui.layout.setSizeConstraint(ui.layout.SetFixedSize) ui.toolbutton_browse.clicked.connect(self.handle_browse_exe) ui.toolbutton_browse.setIcon(get_icon('add.svg')) ui.toolbutton_browse.setIconSize(sub_icon_size()) ui.toolbutton_remove.clicked.connect(self.handle_remove_exe) ui.toolbutton_remove.setIcon(get_icon('remove.svg')) ui.toolbutton_remove.setIconSize(sub_icon_size()) ui.toolbutton_view_error.clicked.connect(self.show_solver_error) ui.listwidget_solver_list.itemSelectionChanged.connect( self.update_dialog_options) ui.combobox_restart.addItems(RESTART_TYPES.keys()) ui.combobox_restart.hide() ui.button_run.clicked.connect(self.handle_run) ui.button_cancel.clicked.connect(self.handle_abort) ui.pushbutton_browse_template.clicked.connect( self.handle_browse_template) n_cpus = multiprocessing.cpu_count() ui.groupbox_smp_options.setTitle("SMP Options (%s available locally)" % plural(n_cpus, "core")) ui.groupbox_queue.toggled.connect(self.toggle_run_btn_text) ui.widget_queue_options.hide() self.initialize_ui() self.init_templates() @property def solver(self): """The currently selected solver""" item = self.ui.listwidget_solver_list.currentItem() if item is None: solver = None else: solver = item.text() if not solver: solver = None return solver def toggle_run_btn_text(self): ui = self.ui ui.button_run.setText( 'Submit' if ui.groupbox_queue.isChecked() else 'Run') def update_gui_comment(self, key, val): if self.gui_comments.get(key) == val: return self.gui_comments[key] = val self.parent.set_unsaved_flag() # UI update functions def initialize_ui(self): ui = self.ui self.setWindowTitle(self.title) get_value = self.parent.project.get_value update_keyword = self.parent.update_keyword # restart spx_files = self.parent.get_output_files(SPX_GLOB) res = self.parent.get_res_files() enable = bool(spx_files) or bool(res) ui.groupbox_restart.setEnabled(enable) if not enable: ui.groupbox_restart.setChecked(False) ui.groupbox_restart.setTitle('Restart - no restart files found') restart_1 = bool(spx_files) and bool(res) if not restart_1: ui.combobox_restart.setCurrentIndex(1) self.enable_restart_item(RESTART_TYPES_INVS['restart_1'], restart_1) self.enable_restart_item(RESTART_TYPES_INVS['restart_2'], bool(res)) # set OMP_NUM_THREADS project_threads = self.gui_comments.get('OMP_NUM_THREADS', '1') env_threads = os.environ.get('OMP_NUM_THREADS', None) if env_threads: project_threads = env_threads ui.spinbox_threads.setValue(safe_int(project_threads, default=1)) # migrate from comments back to keywords, undo issues 149 gui_comment_nodes = self.gui_comments.pop('OMP_NODES', None) if gui_comment_nodes: self.parent.set_unsaved_flag() # modified project nodes = gui_comment_nodes.split(os.path.pathsep) if len(nodes) == 3: nodes = (nodesi, nodesj, nodesk) = [safe_int(n, default=1) for n in nodes] else: nodes = (nodesi, nodesj, nodesk) = 1, 1, 1 update_keyword('nodesi', nodesi) update_keyword('nodesj', nodesj) update_keyword('nodesk', nodesk) else: nodes = (nodesi, nodesj, nodesk) = [ get_value('nodes' + c, default=1) for c in 'ijk' ] for val, spin in zip( nodes, [ui.spinbox_nodesi, ui.spinbox_nodesj, ui.spinbox_nodesk]): spin.setValue(val) # local/queue ui.groupbox_queue.setChecked( int(self.gui_comments.get('submit_to_queue', 0))) # create initial executable list self.get_solver_list() if self.solver_list: self.mfix_available = True self.populate_combobox_solver() for exe in self.solver_list.keys(): self.get_exe_flags(exe) # select solver self.ui.listwidget_solver_list.setCurrentRow(0) self.update_dialog_options() def init_templates(self): # look for templates in MFIX_HOME/queue_templates search_p = os.path.join(get_mfix_templates(), 'queue_templates') self.templates = {} for root, _, files in os.walk(search_p): for f in files: p = os.path.join(root, f) self.add_queue_template(p) # look for recent templates temp_paths = self.settings.value('queue_templates') if temp_paths: for temp_path in temp_paths.split('|'): if os.path.exists(temp_path): self.add_queue_template(temp_path) self.ui.combobox_template.currentIndexChanged.connect( self.update_queue_widgets) temp = self.gui_comments.get('queue_template') if temp: self.template_values = json.loads(temp) t_name = self.template_values.get('template') if t_name: self.set_current_template(t_name) self.update_queue_widgets() def save_template(self): '''Save the current template data''' self.collect_template_values() template_txt = self.ui.combobox_template.currentText() self.template_values['template'] = template_txt self.update_gui_comment('queue_template', json.dumps(self.template_values)) def collect_template_values(self): template_txt = self.ui.combobox_template.currentText() template = self.templates.get(template_txt, {}) replace_dict = {} for name, wid in template.items(): if not isinstance(wid, dict): continue if 'widget_obj' in wid: wid_obj = wid['widget_obj'] if isinstance(wid_obj, (QSpinBox, QDoubleSpinBox)): self.template_values[name] = v = wid_obj.value() elif isinstance(wid_obj, QCheckBox): self.template_values[name] = v = wid_obj.value if v: v = wid.get('true', '') else: v = wid.get('false', '') elif isinstance(wid_obj, QListWidget): self.template_values[name] = v = wid_obj.value v = ' '.join(v) else: self.template_values[name] = v = wid_obj.value replace_dict[name] = v return replace_dict def set_current_template(self, name): '''set the template file combobox''' cb = self.ui.combobox_template for itm in range(cb.count()): if str(name).lower() == str(cb.itemText(itm)).lower(): cb.setCurrentIndex(itm) break def add_queue_template(self, path, select=False): config, script = extract_config(path) c = configparser.ConfigParser() c.readfp(StringIO(config)) d = OrderedDict([(s, dict(c.items(s))) for s in c.sections()]) d['path'] = path d['script'] = script name = os.path.basename(path) if 'options' in d: name = d['options'].get('name', name) self.templates[name] = d self.ui.combobox_template.clear() self.ui.combobox_template.addItems(list(self.templates.keys())) if select: self.set_current_template(name) def update_queue_widgets(self): l = self.ui.groupbox_queue_options_gridlayout clear_layout(l) tp = self.ui.combobox_template.currentText() wids_data = self.templates.get(tp, None) if wids_data is None: return # check to see if qsub command present cmd = wids_data.get('options', {}).get('submit', None) if cmd is not None: cmd = cmd.strip().split()[0] if spawn.find_executable(cmd) is None: label = QLabel( 'The submission command "{}" does not exist in ' 'the current environment. Please select another ' 'template, edit the template, and/or check your ' 'environment.'.format(cmd)) label.setStyleSheet('color:red') label.setWordWrap(True) l.addWidget(label, 0, 0) return # add the widgets for i, wid in enumerate(list(wids_data.keys())): wd = wids_data[wid] if not isinstance(wd, dict) or wid == 'options': continue label = QLabel(wd.get('label', wid)) l.addWidget(label, i, 0) widget = BASE_WIDGETS.get(wd.get('widget', 'lineedit'), BASE_WIDGETS['lineedit'])() items = [it.strip() for it in wd.get('items', '').split('|')] v = self.template_values.get(wid) if not v or self.template_values.get('template') != tp: v = wd.get('value') if isinstance(widget, QComboBox) and items: widget.addItems(items) if v not in items: v = items[0] elif isinstance(widget, QListWidget) and items: widget.add_items(items) widget.setMaximumHeight(100) widget.updateValue('', v) widget.help_text = wd.get('help', 'No help available.') l.addWidget(widget, i, 1) wd['widget_obj'] = widget def populate_combobox_solver(self): """ Add items from self.solver_list to combobox, select the first item """ ui = self.ui ui.listwidget_solver_list.clear() ui.listwidget_solver_list.addItems(self.solver_list.keys()) def enable_restart_item(self, text, enable): cb = self.ui.combobox_restart model = cb.model() index = cb.findText(text) item = model.item(index) if not enable: flags = Qt.NoItemFlags else: flags = Qt.ItemIsSelectable | Qt.ItemIsEnabled item.setFlags(flags) def update_dialog_options(self): """ Enable or disable options based on self.solver features, local or remote settings """ ui = self.ui status_text = '' get_value = self.parent.project.get_value # show status/errors error_msg = self.get_error() status = self.get_status() error = status == ERROR ui.toolbutton_view_error.setVisible(error) if error and error_msg: status_text = 'Solver Error:' # Enable/disable widgets enable = self.mfix_available and self.solver is not None and not error ui.button_run.setEnabled(enable) ui.widget_queue_options.setEnabled(enable) dmp = self.dmp_enabled() smp = self.smp_enabled() ui.groupbox_smp_options.setEnabled(enable and smp) ui.groupbox_dmp_options.setEnabled(enable and dmp) python = self.python_enabled() if status == LOOKING: status_text = 'Checking Solver.' elif not python and not error: status_text = 'Warning: Can not talk to selected solver (not python enabled).' ui.spinbox_nodesi.setEnabled(dmp) ui.spinbox_nodesj.setEnabled(dmp) ui.spinbox_nodesk.setEnabled(dmp and not get_value('no_k')) ui.spinbox_threads.setEnabled(smp) ui.label_solver_error.setText(status_text) def show_solver_error(self): error_msg = '\n'.join(self.get_error()) self.parent.message( text='The solver test failed with the following error:', info_text=error_msg) def popup(self): self.show() self.raise_() self.activateWindow() def closeEvent(self, event): """save information on close""" ui = self.ui # save solver list self.save_selected_exe() # queue self.save_template() self.update_gui_comment('submit_to_queue', int(ui.groupbox_queue.isChecked())) self.update_gui_comment('OMP_NUM_THREADS', str(ui.spinbox_threads.value())) self.parent.save_project() # event handlers def handle_abort(self): self.close() def finish_with_dialog(self): """ save run options in project file, then emit run signal """ ui = self.ui update_keyword = self.parent.update_keyword get_value = self.parent.project.get_value project_dir = self.parent.get_project_dir() udfs = glob(os.path.join(project_dir, "*.f")) udf_msg = ("Warning: Fortran source files exist for this project, but" " the selected mfixsolver is not in the project directory." " This case probably won't run correctly unless this" " project's custom mfixsolver is selected. Proceed anyway?") if udfs and "[project]" + os.sep + "mfixsolver" not in self.solver: response = self.parent.message(title='Warning', icon='question', text=udf_msg, buttons=['yes', 'no'], default='no') if response != 'yes': return thread_count = str(ui.spinbox_threads.value()) # FIXME: should not pollute local env (see saved_env) os.environ['OMP_NUM_THREADS'] = thread_count log.info('SMP enabled with OMP_NUM_THREADS=%s', os.environ["OMP_NUM_THREADS"]) self.update_gui_comment('OMP_NUM_THREADS', thread_count) # restart if ui.groupbox_restart.isChecked(): restart_type = RESTART_TYPES.get(ui.combobox_restart.currentText(), 'restart_1') update_keyword('run_type', restart_type) # restart_2 if restart_type == 'restart_2': spx_files = self.parent.get_output_files(RESTART_2_REMOVE_GLOB) if not self.parent.remove_output_files(spx_files, force_remove=True): log.debug('SP* files exist and run was canceled') return False # normal run else: update_keyword('run_type', 'new') output_files = self.parent.get_output_files() if output_files: message = 'Starting a new run requires the following files to be deleted from the run directory.' if not self.parent.remove_output_files( output_files, message_text=message, force_remove=True): log.info('output files exist and run was canceled') return False # collect nodes[ijk] nodesi = ui.spinbox_nodesi.value() nodesj = ui.spinbox_nodesj.value() nodesk = ui.spinbox_nodesk.value() if not self.dmp_enabled(): nodesi = nodesj = nodesk = 1 # write the correct nodes[ijk] to project file update_keyword('nodesi', nodesi) update_keyword('nodesj', nodesj) if not get_value('no_k'): update_keyword('nodesk', nodesk) else: update_keyword('nodesk', 1) if self.parent.unsaved_flag: # run_type keyword updated and/or nodesi/nodesj/nodesk self.parent.save_project() self.parent.update_source_view() else: stl = os.path.join(self.parent.get_project_dir(), 'geometry.stl') # is this needed? self.parent.vtkwidget.export_stl(stl) self.close() self.parent.signal_update_runbuttons.emit('') return True def handle_run(self): if not self.finish_with_dialog(): self.parent.slot_update_runbuttons() return # reset plots self.parent.reset_plots() self.parent.job_manager.stopping = False self.parent.job_manager.pausing = False self.parent.last_run_msg_time = 0.0 if self.ui.groupbox_queue.isChecked(): self.submit() else: self.run() self.parent.slot_update_runbuttons() def run(self): self.run_cmd = self.get_run_command() msg = 'Starting %s' % ' '.join(self.run_cmd) self.parent.print_internal(msg, color='blue') self.start_command(cmd=self.run_cmd, cwd=self.parent.get_project_dir(), env=os.environ) def submit(self): msg = 'Submitting to queue' self.parent.print_internal(msg, color='blue') self.submit_command(*self.get_submit_command()) def handle_remove_exe(self): ui = self.ui row = ui.listwidget_solver_list.currentRow() item = ui.listwidget_solver_list.currentItem() path = item.text() if not path.startswith('[default]') and not path.startswith( '[project'): ui.listwidget_solver_list.takeItem(row) def handle_browse_exe(self): """ Handle file open dialog for user specified exe """ new_exe, ignore = QFileDialog.getOpenFileName( self, "Select Executable", directory=self.project_dir, options=QFileDialog.DontResolveSymlinks) if not new_exe: return key = self.replace_solver_path(new_exe) lw = self.ui.listwidget_solver_list items = [lw.item(i).text() for i in range(0, lw.count())] if key in items: self.parent.message( text='The selected solver is already in the list of ' 'available solvers.') return # check solver ok, message = self.check_exe(new_exe) if not ok: self.parent.message(text=message) return self.save_selected_exe(new_exe) self.mfix_available = True lw.insertItem(0, key) lw.setCurrentRow(0) log.debug('selected new exe %s', key) def handle_browse_template(self): """ Handle file open dialog for user specified exe """ new_temp, ignore = QFileDialog.getOpenFileName( self, "Select a Template", directory=self.project_dir) if not new_temp: return self.add_queue_template(new_temp, select=True) # add it to the recent settings temp_paths = self.settings.value('queue_templates') good_paths = [os.path.realpath(new_temp)] if temp_paths: for temp_path in temp_paths.split('|'): if os.path.exists(temp_path): good_paths.append(temp_path) self.settings.setValue( 'queue_templates', '|'.join(list(set(good_paths))[:RECENT_EXE_LIMIT])) # utils def save_selected_exe(self, new_solver=None): """ add new executable to recent list, save in project file and config, send signal(s) """ if new_solver is None: new_solver = self.solver if new_solver is None: self.parent.warn('No solver selected') return key = self.replace_solver_path(new_solver) self.settings.setValue('mfix_exe', key) self.update_gui_comment('mfix_exe', key) lw = self.ui.listwidget_solver_list recent_list = [lw.item(i).text() for i in range(0, lw.count())] # truncate to maximum recent_list = recent_list[:RECENT_EXE_LIMIT] # make new solver in front if new_solver is not None: if new_solver in recent_list: recent_list.pop(recent_list.index(new_solver)) recent_list.insert(0, new_solver) # add to solver list nl = OrderedDict([(key, new_solver)]) for key in recent_list: val = self.solver_list.get(key, None) if val is not None: nl[key] = val self.solver_list = nl # save self.settings.setValue('recent_executables', str(os.pathsep).join(recent_list)) def get_solver_list(self): """ assemble list of executables from: - command line - project file 'mfix_exe' - project dir - config item 'recent_executables' - default install location """ def recently_used_executables(): recent_list = self.settings.value('recent_executables') if recent_list: # limit recently used exes to RECENT_EXE_LIMIT recent_lim = recent_list.split(os.pathsep)[:RECENT_EXE_LIMIT] recent_list = [ exe for exe in recent_lim if os.path.exists(exe) ] for recent_exe in recent_list: yield recent_exe def project_directory_executables(): for name in MFIXSOLVER_GLOB_NAMES: for exe in glob(os.path.join(self.project_dir, name)): yield os.path.realpath(exe) def project_file_executable(): project_exe = self.gui_comments.get('mfix_exe', None) if project_exe: yield project_exe def python_path(): for d in sys.path: # filter out empty strings and current directory from $PATH if d and d != os.path.curdir and os.path.isdir(d): for name in MFIXSOLVER_GLOB_NAMES: for exe in glob(os.path.join(d, name)): yield exe def os_path(): PATH = os.environ.get("PATH") if PATH: # using OrderedDict to preserve PATH order dirs = OrderedDict.fromkeys(PATH.split(os.pathsep)) else: dirs = OrderedDict() for d in dirs.keys(): # filter out empty strings and current directory from $PATH if d and d != os.path.curdir and os.path.isdir(d): for name in MFIXSOLVER_GLOB_NAMES: for exe in glob(os.path.join(d, name)): yield exe def mfix_build_directories(): for d in set([get_mfix_home()]): for name in MFIXSOLVER_GLOB_NAMES: for exe in glob(os.path.join(d, name)): yield exe def get_saved_exe(): last_exe = self.settings.value('mfix_exe') if last_exe and os.path.exists(last_exe): yield last_exe def command_line_option(): if self.commandline_option_exe and os.path.exists( self.commandline_option_exe): yield self.commandline_option_exe # Why this order? Shouldn't command_line_option or proj* be first? exe_list_order = [ command_line_option, project_file_executable, project_directory_executables, python_path, os_path, recently_used_executables, mfix_build_directories, get_saved_exe, ] # use an ordered dict because it acts like an ordered set self.solver_list = od = OrderedDict() # look for executables in the order listed in exe_list_order for exe_spec in exe_list_order: for exe in exe_spec(): # expand short hand if '[project]' in exe: exe = exe.replace('[project]', self.prj_dir()) elif '[default]' in exe: exe = exe.replace('[default]', self.def_dir()) # make sure it is an abs path exe = os.path.realpath(exe) # simple checking ok, message = self.check_exe(exe) # truncate paths to [project]/mfixsolver etc. key = self.replace_solver_path(exe) if not ok: self.parent.warn(message) elif key not in od: od[key] = exe def prj_dir(self): return os.path.realpath(self.project_dir) def def_dir(self): return os.path.dirname(os.path.realpath(sys.executable)) def replace_solver_path(self, path): if path.startswith(self.prj_dir()): path = path.replace(self.prj_dir(), '[project]') elif path.startswith(self.def_dir()): path = path.replace(self.def_dir(), '[default]') return path def check_exe(self, path): if not os.path.isfile(path): return False, '{} is not a file.'.format(path) # try executable if not os.access(path, os.X_OK): return False, '{} is not a executable.'.format(path) # windows, check extension if os.name == 'nt': ext = os.path.splitext(path)[-1] if not (ext.endswith('.exe') or ext.endswith('.bat')): return False, 'Extension {} is not recognized, must be .exe or .bat'.format( ext) return True, 'ok' def get_solver_key(self, solver): if solver is None: return None try: stat = os.stat(self.solver_list.get(solver)) except OSError as e: log.debug(str(e)) return None key = (stat, solver) return key def get_exe_flags(self, solver): """ get and cache (and update) executable features """ key = self.get_solver_key(solver) if key is None: return None # stat will have changed if the exe has been modified since last check if key in self.mfix_exe_cache: info = self.mfix_exe_cache[key] return info.get('flags') # spawn process to get flags else: self.set_solver_icon(solver, LOOKING) self.mfix_exe_cache[key] = { 'status': LOOKING, 'flags': None, 'stdout': [], 'stderror': [], 'error': [], } self.spawn_flag_process(*key) return LOOKING def spawn_flag_process(self, stat, solver): log.debug('Feature testing MFiX %s', solver) key = (stat, solver) proc = self.flag_processes.get(key, None) if proc is not None: proc.kill() exe = self.solver_list.get(solver) exe_dir = os.path.dirname(exe) proc = self.flag_processes[key] = QProcess() proc.setProcessEnvironment(QProcessEnvironment.systemEnvironment()) proc.readyReadStandardOutput.connect( lambda k=key: self.flag_process_out(k)) proc.readyReadStandardError.connect( lambda k=key: self.flag_process_error(k)) proc.finished.connect(lambda ecode, estat, k=key: self. flag_process_finished(k, ecode, estat)) proc.error.connect(lambda e, k=key: self.flag_process_error(k, e)) proc.setWorkingDirectory(exe_dir) proc.start(exe, ["--print-flags"]) def flag_process_error(self, key, error=None): info = self.mfix_exe_cache.get(key) self.set_solver_icon(key[1], ERROR) info['status'] = ERROR if error is None: error = bytes( self.flag_processes[key].readAllStandardError()).decode( 'utf-8', errors='ignore') info['stderror'].append(error) else: info['error'].append(error) log.debug("could not run {} --print-flags: {}".format(key[1], error)) def flag_process_out(self, key): info = self.mfix_exe_cache.get(key) out = bytes(self.flag_processes[key].readAllStandardOutput()).decode( 'utf-8', errors='ignore') info['stdout'].append(out) info['flags'] = str('\n'.join(info['stdout'])).strip() log.debug("stdout: {} --print-flags: {}".format(key[1], out)) def flag_process_finished(self, key, exit_code, exit_status): info = self.mfix_exe_cache.get(key, {}) status = info.get('status', ERROR) if info.get('stderror', []): status = ERROR if status == LOOKING: status = OK info['status'] = status self.set_solver_icon(key[1], status) if self.solver == key[1]: self.update_dialog_options() log.debug("finished: {} --print-flags: {}, {}".format( key[1], exit_code, exit_status)) def set_solver_icon(self, solver, status): items = self.ui.listwidget_solver_list.findItems( solver, Qt.MatchExactly) if not items: return item = items[0] icon = 'error_outline.svg' if status == LOOKING: icon = 'timelapse.svg' elif status == OK: icon = 'check_outline.svg' item.setIcon(get_icon(icon)) def dmp_enabled(self): flags = self.get_exe_flags(self.solver) dmp = False if flags is not None: dmp = 'dmp' in str(flags) return dmp def smp_enabled(self): flags = self.get_exe_flags(self.solver) smp = False if flags is not None: smp = 'smp' in str(flags) return smp def python_enabled(self): flags = self.get_exe_flags(self.solver) python = False if flags is not None: python = 'python' in str(flags) return python def get_error(self): key = self.get_solver_key(self.solver) if key is None: return None info = self.mfix_exe_cache.get(key, {}) return info.get('stderror', None) def get_status(self): key = self.get_solver_key(self.solver) if key is None: return None info = self.mfix_exe_cache.get(key, {}) return info.get('status', None) def get_run_command(self): get_value = self.parent.project.get_value # collect nodes[ijk] from project to guarantee that mpirun matches nodesi = get_value('nodesi', 1) nodesj = get_value('nodesj', 1) nodesk = get_value('nodesk', 1) np = nodesi * nodesj * nodesk if self.dmp_enabled() and np > 1: dmp = [ 'mpirun', # '-quiet', # '-mca', 'orte_create_session_dirs', 'true', '-mca', 'mpi_warn_on_fork', '0', '-np', str(nodesi * nodesj * nodesk) ] else: dmp = [] #if self.dmp_enabled(): # run_cmd += ['nodesi=%s'%nodesi, # 'nodesj=%s'%nodesj] # if not self.parent.project.get_value('no_k'): # run_cmd += ['nodesk=%s'%nodesk] if self.smp_enabled(): num_threads = str(self.ui.spinbox_threads.value()) smp = ['env', 'OMP_NUM_THREADS=%s' % num_threads] else: smp = [] run_cmd = smp + dmp + [self.solver_list.get(self.solver)] # Add 'server' flag to start HTTP server run_cmd += ['-s'] # Specify project file project_filename = self.parent.get_project_file() run_cmd += ['-f', project_filename] return run_cmd def get_submit_command(self): cmd = self.get_run_command() template_txt = self.ui.combobox_template.currentText() template = self.templates[template_txt] # collect widget values replace_dict = self.collect_template_values() replace_dict.update({ 'PROJECT_NAME': self.parent.project.get_value('run_name', default=''), 'COMMAND': ' '.join(cmd), 'MFIX_HOME': get_mfix_home(), }) # replace twice to make sure that any references added the first time # get replaced script = replace_with_dict(template['script'], replace_dict) script = replace_with_dict(script, replace_dict) sub_cmd = template['options'].get('submit', False) delete_cmd = template['options'].get('delete', False) # XXX status_cmd = template['options'].get('status', False) job_id_regex = template['options'].get('job_id_regex', None) ## FIXME, return something nicer than this 6-tuple return script, sub_cmd, delete_cmd, status_cmd, job_id_regex, replace_dict def submit_command(self, script, sub_cmd, delete_cmd, status_cmd, job_id_regex, replace_dict): self.remove_mfix_stop() if not sub_cmd: template_txt = self.ui.combobox_template.currentText() self.parent.error( ('The template file at: {}\n' 'does not have a submit_cmd defined').format(template_txt)) return self.parent.job_manager.submit_command(script, sub_cmd, delete_cmd, status_cmd, job_id_regex, replace_dict) def remove_mfix_stop(self): mfix_stop_file = os.path.join(self.parent.get_project_dir(), 'MFIX.STOP') if os.path.exists(mfix_stop_file): try: os.remove(mfix_stop_file) except OSError: self.parent.warn("Cannot remove %s", mfix_stop_file) return def start_command(self, cmd, cwd, env): """Start MFIX in QProcess""" self.cmdline = cmd # List of strings, same as psutil self.remove_mfix_stop() self.mfixproc = QProcess() if not self.mfixproc: log.warning("QProcess creation failed") return self.mfixproc.setWorkingDirectory(cwd) def slot_start(): # processId was only added in qt 5.3 if StrictVersion(QT_VERSION) > StrictVersion('5.3'): pid = self.mfixproc.processId() else: pid = self.mfixproc.pid() msg = "MFiX process %d is running" % pid self.parent.signal_update_runbuttons.emit(msg) def slot_read_out(): # Why convert to bytes then decode? out_str = bytes(self.mfixproc.readAllStandardOutput()).decode( 'utf-8', errors='ignore') self.parent.stdout_signal.emit(out_str) def slot_read_err(): err_str = bytes(self.mfixproc.readAllStandardError()).decode( 'utf-8', errors='ignore') self.parent.stderr_signal.emit(err_str) def slot_finish(status): # This should really be in the job manager if self.parent.job_manager.job: self.parent.job_manager.job.cleanup_and_exit() self.parent.job_manager.job = None msg = "MFiX process has stopped" self.parent.signal_update_runbuttons.emit(msg) if self.parent.job_manager.pidfile: try: os.unlink(self.parent.job_manager.pidfile) self.parent.job_manager.pidfile = None except OSError as e: if e.errno != errno.ENOENT: raise def slot_error(error): cmd_str = ' '.join(self.cmdline) if error == QProcess.FailedToStart: msg = "Process failed to start " + cmd_str elif error == QProcess.Crashed: msg = "Process exit " + cmd_str elif error == QProcess.Timedout: msg = "Process timeout " + cmd_str elif error in (QProcess.WriteError, QProcess.ReadError): msg = "Process communication error " + cmd_str else: msg = "Unknown error " + cmd_str log.warning(msg) # make the message print in red self.parent.stderr_signal.emit(msg) self.mfixproc.started.connect(slot_start) self.mfixproc.readyReadStandardOutput.connect(slot_read_out) self.mfixproc.readyReadStandardError.connect(slot_read_err) self.mfixproc.finished.connect(slot_finish) self.mfixproc.error.connect(slot_error) start_detached = True #if sys.platform.startswith('win') or 'mpirun' not in cmd: # start_detached = False # On Windows, start_detached gives a DOS box # What was the issue with mpirun? start_detached = False # https://bugreports.qt.io/browse/QTBUG-2284 # QProcessEnvironment does not work with startDetached, # fixed in Qt5.10 which we aren't using yet saved_env = None if not start_detached: process_env = QProcessEnvironment() add_env = process_env.insert else: add_env = os.environ.__setitem__ saved_env = os.environ.copy() for key, val in env.items(): add_env(key, val) add_env('MFIX_RUN_CMD', ' '.join(cmd)) if not start_detached: self.mfixproc.setProcessEnvironment(process_env) self.mfixproc.start(cmd[0], cmd[1:]) else: self.mfixproc.startDetached(cmd[0], cmd[1:]) # restore environment if saved_env: for (k, v) in list(os.environ.items()): if k not in saved_env: del os.environ[k] elif v != saved_env[k]: os.environ[k] = saved_env[k] # give gui a reference self.parent.mfix_process = self.mfixproc self.parent.slot_rundir_timer()