class OWFile(widget.OWWidget, RecentPathsWComboMixin): name = "File" id = "orange.widgets.data.file" description = "Read data from an input file or network " \ "and send a data table to the output." icon = "icons/File.svg" priority = 10 category = "Data" keywords = ["data", "file", "load", "read"] outputs = [ widget.OutputSignal( "Data", Table, doc="Attribute-valued data set read from the input file.") ] want_main_area = False SEARCH_PATHS = [("sample-datasets", get_sample_datasets_dir())] SIZE_LIMIT = 1e7 LOCAL_FILE, URL = range(2) settingsHandler = PerfectDomainContextHandler() # Overload RecentPathsWidgetMixin.recent_paths to set defaults recent_paths = Setting([ RecentPath("", "sample-datasets", "iris.tab"), RecentPath("", "sample-datasets", "titanic.tab"), RecentPath("", "sample-datasets", "housing.tab"), RecentPath("", "sample-datasets", "heart_disease.tab"), ]) recent_urls = Setting([]) source = Setting(LOCAL_FILE) xls_sheet = ContextSetting("") sheet_names = Setting({}) url = Setting("") variables = ContextSetting([]) dlg_formats = ("All readable files ({});;".format( '*' + ' *'.join(FileFormat.readers.keys())) + ";;".join( "{} (*{})".format(f.DESCRIPTION, ' *'.join(f.EXTENSIONS)) for f in sorted(set(FileFormat.readers.values()), key=list(FileFormat.readers.values()).index))) domain_editor = SettingProvider(DomainEditor) class Warning(widget.OWWidget.Warning): file_too_big = widget.Msg( "The file is too large to load automatically." " Press Reload to load.") class Error(widget.OWWidget.Error): file_not_found = widget.Msg("File not found.") def __init__(self): super().__init__() RecentPathsWComboMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.reader = None layout = QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) vbox = gui.radioButtons(None, self, "source", box=True, addSpace=True, callback=self.load_data, addToLayout=False) rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False) layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter) box = gui.hBox(None, addToLayout=False, margin=0) box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.activated[int].connect(self.select_file) box.layout().addWidget(self.file_combo) layout.addWidget(box, 0, 1) file_button = gui.button(None, self, '...', callback=self.browse_file, autoDefault=False) file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 2) reload_button = gui.button(None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 3) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox( None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True, ) self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False) layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter) self.url_combo = url_combo = QComboBox() url_model = NamedURLModel(self.sheet_names) url_model.wrap(self.recent_urls) url_combo.setLineEdit(LineEditSelectOnFocus()) url_combo.setModel(url_model) url_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) url_combo.setEditable(True) url_combo.setInsertPolicy(url_combo.InsertAtTop) url_edit = url_combo.lineEdit() l, t, r, b = url_edit.getTextMargins() url_edit.setTextMargins(l + 5, t, r, b) layout.addWidget(url_combo, 3, 1, 3, 3) url_combo.activated.connect(self._url_set) box = gui.vBox(self.controlArea, "Info") self.info = gui.widgetLabel(box, 'No data loaded.') self.warnings = gui.widgetLabel(box, '') box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) box = gui.hBox(self.controlArea) gui.button(box, self, "Browse documentation data sets", callback=lambda: self.browse_file(True), autoDefault=False) gui.rubber(box) box.layout().addWidget(self.report_button) self.report_button.setFixedWidth(170) self.apply_button = gui.button(box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) self.set_file_list() # Must not call open_file from within __init__. open_file # explicitly re-enters the event loop (by a progress bar) self.setAcceptDrops(True) if self.source == self.LOCAL_FILE: last_path = self.last_path() if last_path and os.path.exists(last_path) and \ os.path.getsize(last_path) > self.SIZE_LIMIT: self.Warning.file_too_big() return QTimer.singleShot(0, self.load_data) def sizeHint(self): return QSize(600, 550) def select_file(self, n): assert n < len(self.recent_paths) super().select_file(n) if self.recent_paths: self.source = self.LOCAL_FILE self.load_data() self.set_file_list() def select_sheet(self): self.recent_paths[0].sheet = self.sheet_combo.currentText() self.load_data() def _url_set(self): self.source = self.URL self.load_data() def browse_file(self, in_demos=False): if in_demos: start_file = get_sample_datasets_dir() if not os.path.exists(start_file): QMessageBox.information( None, "File", "Cannot find the directory with documentation data sets") return else: start_file = self.last_path() or os.path.expanduser("~/") filename, _ = QFileDialog.getOpenFileName(self, 'Open Orange Data File', start_file, self.dlg_formats) if not filename: return self.add_path(filename) self.source = self.LOCAL_FILE self.load_data() # Open a file, create data from it and send it over the data channel def load_data(self): # We need to catch any exception type since anything can happen in # file readers # pylint: disable=broad-except self.closeContext() self.domain_editor.set_domain(None) self.apply_button.setEnabled(False) self.clear_messages() self.set_file_list() if self.last_path() and not os.path.exists(self.last_path()): self.Error.file_not_found() self.send("Data", None) self.info.setText("No data.") return error = None try: self.reader = self._get_reader() if self.reader is None: self.data = None self.send("Data", None) self.info.setText("No data.") self.sheet_box.hide() return except Exception as ex: error = ex if not error: self._update_sheet_combo() with catch_warnings(record=True) as warnings: try: data = self.reader.read() except Exception as ex: log.exception(ex) error = ex self.warning(warnings[-1].message.args[0] if warnings else '') if error: self.data = None self.send("Data", None) self.info.setText("An error occurred:\n{}".format(error)) self.sheet_box.hide() return self.info.setText(self._describe(data)) self.loaded_file = self.last_path() add_origin(data, self.loaded_file) self.data = data self.openContext(data.domain) self.apply_domain_edit() # sends data def _get_reader(self): """ Returns ------- FileFormat """ if self.source == self.LOCAL_FILE: reader = FileFormat.get_reader(self.last_path()) if self.recent_paths and self.recent_paths[0].sheet: reader.select_sheet(self.recent_paths[0].sheet) return reader elif self.source == self.URL: url = self.url_combo.currentText().strip() if url: return UrlReader(url) def _update_sheet_combo(self): if len(self.reader.sheets) < 2: self.sheet_box.hide() self.reader.select_sheet(None) return self.sheet_combo.clear() self.sheet_combo.addItems(self.reader.sheets) self._select_active_sheet() self.sheet_box.show() def _select_active_sheet(self): if self.reader.sheet: try: idx = self.reader.sheets.index(self.reader.sheet) self.sheet_combo.setCurrentIndex(idx) except ValueError: # Requested sheet does not exist in this file self.reader.select_sheet(None) else: self.sheet_combo.setCurrentIndex(0) def _describe(self, table): domain = table.domain text = "" attrs = getattr(table, "attributes", {}) descs = [ attrs[desc] for desc in ("Name", "Description") if desc in attrs ] if len(descs) == 2: descs[0] = "<b>{}</b>".format(descs[0]) if descs: text += "<p>{}</p>".format("<br/>".join(descs)) text += "<p>{} instance(s), {} feature(s), {} meta attribute(s)".\ format(len(table), len(domain.attributes), len(domain.metas)) if domain.has_continuous_class: text += "<br/>Regression; numerical class." elif domain.has_discrete_class: text += "<br/>Classification; discrete class with {} values.".\ format(len(domain.class_var.values)) elif table.domain.class_vars: text += "<br/>Multi-target; {} target variables.".format( len(table.domain.class_vars)) else: text += "<br/>Data has no target variable." text += "</p>" if 'Timestamp' in table.domain: # Google Forms uses this header to timestamp responses text += '<p>First entry: {}<br/>Last entry: {}</p>'.format( table[0, 'Timestamp'], table[-1, 'Timestamp']) return text def storeSpecificSettings(self): self.current_context.modified_variables = self.variables[:] def retrieveSpecificSettings(self): if hasattr(self.current_context, "modified_variables"): self.variables[:] = self.current_context.modified_variables def apply_domain_edit(self): if self.data is not None: domain, cols = self.domain_editor.get_domain( self.data.domain, self.data) X, y, m = cols X = np.array(X).T if len(X) else np.empty((len(self.data), 0)) y = np.array(y).T if len(y) else None dtpe = object if any( isinstance(m, StringVariable) for m in domain.metas) else float m = np.array(m, dtype=dtpe).T if len(m) else None table = Table.from_numpy(domain, X, y, m, self.data.W) table.name = self.data.name table.ids = np.array(self.data.ids) table.attributes = getattr(self.data, 'attributes', {}) else: table = self.data self.send("Data", table) self.apply_button.setEnabled(False) def get_widget_name_extension(self): _, name = os.path.split(self.loaded_file) return os.path.splitext(name)[0] def send_report(self): def get_ext_name(filename): try: return FileFormat.names[os.path.splitext(filename)[1]] except KeyError: return "unknown" if self.data is None: self.report_paragraph("File", "No file.") return if self.source == self.LOCAL_FILE: home = os.path.expanduser("~") if self.loaded_file.startswith(home): # os.path.join does not like ~ name = "~" + os.path.sep + \ self.loaded_file[len(home):].lstrip("/").lstrip("\\") else: name = self.loaded_file if self.sheet_combo.isVisible(): name += " ({})".format(self.sheet_combo.currentText()) self.report_items("File", [("File name", name), ("Format", get_ext_name(name))]) else: self.report_items("Data", [("Resource", self.url), ("Format", get_ext_name(self.url))]) self.report_data("Data", self.data) def dragEnterEvent(self, event): """Accept drops of valid file urls""" urls = event.mimeData().urls() if urls: try: FileFormat.get_reader( OSX_NSURL_toLocalFile(urls[0]) or urls[0].toLocalFile()) event.acceptProposedAction() except IOError: pass def dropEvent(self, event): """Handle file drops""" urls = event.mimeData().urls() if urls: self.add_path( OSX_NSURL_toLocalFile(urls[0]) or urls[0].toLocalFile()) # add first file self.source = self.LOCAL_FILE self.load_data()
def __init__(self): self.domain_editor = DomainEditor(self)
def __init__(self): super().__init__() RecentPathsWComboMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.reader = None layout = QtGui.QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) vbox = gui.radioButtons(None, self, "source", box=True, addSpace=True, callback=self.load_data, addToLayout=False) rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False) layout.addWidget(rb_button, 0, 0, QtCore.Qt.AlignVCenter) box = gui.hBox(None, addToLayout=False, margin=0) box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.activated[int].connect(self.select_file) box.layout().addWidget(self.file_combo) layout.addWidget(box, 0, 1) file_button = gui.button( None, self, '...', callback=self.browse_file, autoDefault=False) file_button.setIcon(self.style().standardIcon( QtGui.QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 2) reload_button = gui.button( None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QtGui.QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 3) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox(None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True) self.sheet_combo.setSizePolicy( Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QtGui.QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy( Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget( self.sheet_label, QtCore.Qt.AlignLeft) self.sheet_box.layout().addWidget( self.sheet_combo, QtCore.Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False) layout.addWidget(rb_button, 3, 0, QtCore.Qt.AlignVCenter) self.url_combo = url_combo = QtGui.QComboBox() url_model = NamedURLModel(self.sheet_names) url_model.wrap(self.recent_urls) url_combo.setModel(url_model) url_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) url_combo.setEditable(True) url_combo.setInsertPolicy(url_combo.InsertAtTop) url_edit = url_combo.lineEdit() l, t, r, b = url_edit.getTextMargins() url_edit.setTextMargins(l + 5, t, r, b) layout.addWidget(url_combo, 3, 1, 3, 3) url_combo.activated.connect(self._url_set) box = gui.vBox(self.controlArea, "Info") self.info = gui.widgetLabel(box, 'No data loaded.') self.warnings = gui.widgetLabel(box, '') box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") domain_editor = DomainEditor(self.variables) self.editor_model = domain_editor.model() box.layout().addWidget(domain_editor) box = gui.hBox(self.controlArea) gui.button( box, self, "Browse documentation data sets", callback=lambda: self.browse_file(True), autoDefault=False) gui.rubber(box) box.layout().addWidget(self.report_button) self.report_button.setFixedWidth(170) self.apply_button = gui.button( box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.hide() self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect(self.apply_button.show) self.set_file_list() # Must not call open_file from within __init__. open_file # explicitly re-enters the event loop (by a progress bar) QtCore.QTimer.singleShot(0, self.load_data) self.setAcceptDrops(True)
def __init__(self): super().__init__() self.table = None self._html = None def _loadFinished(is_ok): if is_ok: QTimer.singleShot(1, lambda: setattr(self, '_html', self.webview.html())) self.webview = WebviewWidget(loadFinished=_loadFinished) vb = gui.vBox(self.controlArea, 'Import Data') hb = gui.hBox(vb) self.combo = combo = URLComboBox( hb, self.recent, editable=True, minimumWidth=400, insertPolicy=QComboBox.InsertAtTop, toolTip='Format: ' + VALID_URL_HELP, editTextChanged=self.is_valid_url, # Indirect via QTimer because calling wait() -> processEvents, # while our currentIndexChanged event hadn't yet finished. # Avoids calling handler twice. currentIndexChanged=lambda: QTimer.singleShot(1, self.load_url)) hb.layout().addWidget(QLabel('Public link URL:', hb)) hb.layout().addWidget(combo) hb.layout().setStretch(1, 2) RELOAD_TIMES = ( ('No reload',), ('5 s', 5000), ('10 s', 10000), ('30 s', 30000), ('1 min', 60*1000), ('2 min', 2*60*1000), ('5 min', 5*60*1000), ) reload_timer = QTimer(self, timeout=lambda: self.load_url(from_reload=True)) def _on_reload_changed(): if self.reload_idx == 0: reload_timer.stop() return reload_timer.start(RELOAD_TIMES[self.reload_idx][1]) gui.comboBox(vb, self, 'reload_idx', label='Reload every:', orientation=Qt.Horizontal, items=[i[0] for i in RELOAD_TIMES], callback=_on_reload_changed) box = gui.widgetBox(self.controlArea, "Columns (Double-click to edit)") self.domain_editor = DomainEditor(self) editor_model = self.domain_editor.model() def editorDataChanged(): self.apply_domain_edit() self.commit() editor_model.dataChanged.connect(editorDataChanged) box.layout().addWidget(self.domain_editor) box = gui.widgetBox(self.controlArea, "Info", addSpace=True) info = self.data_info = gui.widgetLabel(box, '') info.setWordWrap(True) self.controlArea.layout().addStretch(1) gui.auto_commit(self.controlArea, self, 'autocommit', label='Commit') self.set_info()
class OW1ka(widget.OWWidget): name = "EnKlik Anketa" description = "Import data from EnKlikAnketa (1ka.si) public URL." icon = "icons/1ka.svg" priority = 30 outputs = [("Data", Table)] want_main_area = False resizing_enabled = False settingsHandler = settings.PerfectDomainContextHandler( match_values=settings.PerfectDomainContextHandler.MATCH_VALUES_ALL ) recent = settings.Setting([]) reload_idx = settings.Setting(0) autocommit = settings.Setting(True) domain_editor = settings.SettingProvider(DomainEditor) UserAdviceMessages = [ widget.Message( 'You can import data from public links to 1ka surveys results. ' 'Click to learn more on how to get a shareable public link URL for ' '1ka surveys that you manage.', 'public-link', icon=widget.Message.Information, moreurl='http://english.1ka.si/db/24/468/Guides/Public_link_to_access_data_and_analysis/' ), ] class Error(widget.OWWidget.Error): net_error = widget.Msg("Couldn't load data: {}. Ensure network connection, firewall ...") parse_error = widget.Msg("Couldn't parse data: {}. Ensure well-formatted data or submit a bug report.") invalid_url = widget.Msg('Invalid URL. Public shareable link should match: ' + VALID_URL_HELP) data_is_anal = widget.Msg("The provided URL is a public link to 'Analysis'. Need public link to 'Data'.") class Information(widget.OWWidget.Information): response_data_empty = widget.Msg('Response data is empty. Get some responses first.') def __init__(self): super().__init__() self.table = None self._html = None def _loadFinished(is_ok): if is_ok: QTimer.singleShot(1, lambda: setattr(self, '_html', self.webview.html())) self.webview = WebviewWidget(loadFinished=_loadFinished) vb = gui.vBox(self.controlArea, 'Import Data') hb = gui.hBox(vb) self.combo = combo = URLComboBox( hb, self.recent, editable=True, minimumWidth=400, insertPolicy=QComboBox.InsertAtTop, toolTip='Format: ' + VALID_URL_HELP, editTextChanged=self.is_valid_url, # Indirect via QTimer because calling wait() -> processEvents, # while our currentIndexChanged event hadn't yet finished. # Avoids calling handler twice. currentIndexChanged=lambda: QTimer.singleShot(1, self.load_url)) hb.layout().addWidget(QLabel('Public link URL:', hb)) hb.layout().addWidget(combo) hb.layout().setStretch(1, 2) RELOAD_TIMES = ( ('No reload',), ('5 s', 5000), ('10 s', 10000), ('30 s', 30000), ('1 min', 60*1000), ('2 min', 2*60*1000), ('5 min', 5*60*1000), ) reload_timer = QTimer(self, timeout=lambda: self.load_url(from_reload=True)) def _on_reload_changed(): if self.reload_idx == 0: reload_timer.stop() return reload_timer.start(RELOAD_TIMES[self.reload_idx][1]) gui.comboBox(vb, self, 'reload_idx', label='Reload every:', orientation=Qt.Horizontal, items=[i[0] for i in RELOAD_TIMES], callback=_on_reload_changed) box = gui.widgetBox(self.controlArea, "Columns (Double-click to edit)") self.domain_editor = DomainEditor(self) editor_model = self.domain_editor.model() def editorDataChanged(): self.apply_domain_edit() self.commit() editor_model.dataChanged.connect(editorDataChanged) box.layout().addWidget(self.domain_editor) box = gui.widgetBox(self.controlArea, "Info", addSpace=True) info = self.data_info = gui.widgetLabel(box, '') info.setWordWrap(True) self.controlArea.layout().addStretch(1) gui.auto_commit(self.controlArea, self, 'autocommit', label='Commit') self.set_info() def set_combo_items(self): self.combo.clear() for sheet in self.recent: self.combo.addItem(sheet.name, sheet.url) def commit(self): self.send('Data', self.table) def is_valid_url(self, url): if is_valid_url(url): self.Error.invalid_url.clear() return True self.Error.invalid_url() QToolTip.showText(self.combo.mapToGlobal(QPoint(0, 0)), self.combo.toolTip()) def load_url(self, from_reload=False): self.closeContext() self.domain_editor.set_domain(None) url = self.combo.currentText() if not self.is_valid_url(url): self.table = None self.commit() return if url not in self.recent: self.recent.insert(0, url) prev_table = self.table with self.progressBar(3) as progress: try: self._html = None self.webview.setUrl(url) wait(until=lambda: self._html is not None) progress.advance() # Wait some seconds for discrete labels to have loaded via AJAX, # then re-query HTML. # *Webview.loadFinished doesn't guarantee it sufficiently try: wait(until=lambda: False, timeout=1200) except TimeoutError: pass progress.advance() html = self.webview.html() except Exception as e: log.exception("Couldn't load data from: %s", url) self.Error.net_error(try_(lambda: e.args[0], '')) self.table = None else: self.Error.clear() self.Information.clear() self.table = None try: table = self.table = self.table_from_html(html) except DataEmptyError: self.Information.response_data_empty() except DataIsAnalError: self.Error.data_is_anal() except Exception as e: log.exception('Parsing error: %s', url) self.Error.parse_error(try_(lambda: e.args[0], '')) else: self.openContext(table.domain) self.combo.setTitleFor(self.combo.currentIndex(), table.name) def _equal(data1, data2): NAN = float('nan') return (try_(lambda: data1.checksum(), NAN) == try_(lambda: data2.checksum(), NAN)) self._orig_table = self.table self.apply_domain_edit() if not (from_reload and _equal(prev_table, self.table)): self.commit() def apply_domain_edit(self): data = self._orig_table if data is None: self.set_info() return domain, cols = self.domain_editor.get_domain(data.domain, data) # Copied verbatim from OWFile if not (domain.variables or domain.metas): table = None else: X, y, m = cols table = Table.from_numpy(domain, X, y, m, data.W) table.name = data.name table.ids = np.array(data.ids) table.attributes = getattr(data, 'attributes', {}) self.table = table self.set_info() DATETIME_VAR = 'Paradata (insert)' def table_from_html(self, html): soup = BeautifulSoup(html, 'html.parser') try: html_table = soup.find_all('table')[-1] except IndexError: raise DataEmptyError if '<h2>Anal' in html or 'div_analiza_' in html: raise DataIsAnalError def _header_row_strings(row): return chain.from_iterable( repeat(th.get_text(), int(th.get('colspan') or 1)) for th in html_table.select('thead tr:nth-of-type(%d) th[title]' % row)) # self.DATETIME_VAR (available when Paradata is enabled in 1ka UI) # should match this variable name format header = [th1.rstrip(':') + ('' if th3 == th1 else ' ({})').format(th3.rstrip(':')) for th1, th3 in zip(_header_row_strings(1), _header_row_strings(3))] values = [[(# If no span, feature is a number or a text field td.get_text() if td.span is None else # If have span, it's a number, but if negative, replace with NaN '' if td.contents[0].strip().startswith('-') else # Else if span, the number is its code, but we want its value td.span.get_text()[1:-1]) for td in tr.select('td') if 'data_uid' not in td.get('class', ())] for tr in html_table.select('tbody tr')] # Save parsed values into in-mem file for default values processing buffer = StringIO() writer = csv.writer(buffer, delimiter='\t') writer.writerow(header) writer.writerows(values) buffer.flush() buffer.seek(0) data = TabReader(buffer).read() title = soup.select('body h2:nth-of-type(1)')[0].get_text().split(': ', maxsplit=1)[-1] data.name = title return data def set_info(self): data = self.table if data is None: self.data_info.setText('No spreadsheet loaded.') return text = "{}\n\n{} instance(s), {} feature(s), {} meta attribute(s)\n".format( data.name, len(data), len(data.domain.attributes), len(data.domain.metas)) text += try_(lambda: '\nFirst entry: {}' '\nLast entry: {}'.format(data[0, self.DATETIME_VAR], data[-1, self.DATETIME_VAR]), '') self.data_info.setText(text)
def __init__(self): super().__init__() RecentPathsWComboMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.reader = None layout = QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) vbox = gui.radioButtons(None, self, "source", box=True, addSpace=True, callback=self.load_data, addToLayout=False) rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False) layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter) box = gui.hBox(None, addToLayout=False, margin=0) box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.activated[int].connect(self.select_file) box.layout().addWidget(self.file_combo) layout.addWidget(box, 0, 1) file_button = gui.button(None, self, '...', callback=self.browse_file, autoDefault=False) file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 2) reload_button = gui.button(None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 3) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox( None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True, ) self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False) layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter) self.url_combo = url_combo = QComboBox() url_model = NamedURLModel(self.sheet_names) url_model.wrap(self.recent_urls) url_combo.setLineEdit(LineEditSelectOnFocus()) url_combo.setModel(url_model) url_combo.setSizePolicy(Policy.Ignored, Policy.Fixed) url_combo.setEditable(True) url_combo.setInsertPolicy(url_combo.InsertAtTop) url_edit = url_combo.lineEdit() l, t, r, b = url_edit.getTextMargins() url_edit.setTextMargins(l + 5, t, r, b) layout.addWidget(url_combo, 3, 1, 3, 3) url_combo.activated.connect(self._url_set) # whit completer we set that combo box is case sensitive when # matching the history completer = QCompleter() completer.setCaseSensitivity(Qt.CaseSensitive) url_combo.setCompleter(completer) box = gui.vBox(self.controlArea, "Info") self.infolabel = gui.widgetLabel(box, 'No data loaded.') self.warnings = gui.widgetLabel(box, '') box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) box = gui.hBox(self.controlArea) gui.button(box, self, "Browse documentation datasets", callback=lambda: self.browse_file(True), autoDefault=False) gui.rubber(box) gui.button(box, self, "Reset", callback=self.reset_domain_edit) self.apply_button = gui.button(box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) self.set_file_list() # Must not call open_file from within __init__. open_file # explicitly re-enters the event loop (by a progress bar) self.setAcceptDrops(True) if self.source == self.LOCAL_FILE: last_path = self.last_path() if last_path and os.path.exists(last_path) and \ os.path.getsize(last_path) > self.SIZE_LIMIT: self.Warning.file_too_big() return QTimer.singleShot(0, self.load_data)
def __init__(self): widget.OWWidget.__init__(self) RelocatablePathsWidgetMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.sheets = [] self.lb = gui.listBox(self.controlArea, self, "file_idx", selectionMode=QListWidget.MultiSelection) self.default_foreground = None layout = QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) file_button = gui.button(None, self, ' ...', callback=self.browse_files, autoDefault=False) file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 0) remove_button = gui.button(None, self, 'Remove', callback=self.remove_item) clear_button = gui.button(None, self, 'Clear', callback=self.clear) layout.addWidget(remove_button, 0, 1) layout.addWidget(clear_button, 0, 2) reload_button = gui.button(None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 7) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_index = 0 self.sheet_combo = gui.comboBox(None, self, "sheet_index", callback=self.select_sheet) self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() layout.addWidget(self.sheet_box, 0, 5) label_box = gui.hBox(None, addToLayout=False, margin=0) gui.lineEdit(label_box, self, "label", callback=self.set_label, label="Label", orientation=Qt.Horizontal) layout.addWidget(label_box, 0, 6) layout.setColumnStretch(3, 2) box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) for rp in self.recent_paths: self.lb.addItem(rp.abspath) box = gui.hBox(self.controlArea) gui.rubber(box) if hasattr(DomainEditor, "reset_domain"): # Orange>=3.21 gui.button(box, self, "Reset", callback=self.reset_domain_edit) self.apply_button = gui.button(box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) self._update_sheet_combo() self.load_data()
class OW1ka(widget.OWWidget): name = "EnKlik Anketa" description = "Import data from EnKlikAnketa (1ka.si) public URL." icon = "icons/1ka.svg" priority = 200 class Outputs: data = Output("Data", Table) want_main_area = False resizing_enabled = False settingsHandler = settings.PerfectDomainContextHandler( match_values=settings.PerfectDomainContextHandler.MATCH_VALUES_ALL) recent = settings.Setting([]) reload_idx = settings.Setting(0) autocommit = settings.Setting(True) domain_editor = settings.SettingProvider(DomainEditor) UserAdviceMessages = [ widget.Message( 'You can import data from public links to 1ka surveys results. ' 'Click to learn more on how to get a shareable public link URL for ' '1ka surveys that you manage.', 'public-link', icon=widget.Message.Information, moreurl= 'http://english.1ka.si/db/24/468/Guides/Public_link_to_access_data_and_analysis/' ), ] class Error(widget.OWWidget.Error): net_error = widget.Msg( "Couldn't load data: {}. Ensure network connection, firewall ...") parse_error = widget.Msg( "Couldn't parse data: {}. Ensure well-formatted data or submit a bug report." ) invalid_url = widget.Msg( 'Invalid URL. Public shareable link should match: ' + VALID_URL_HELP) data_is_anal = widget.Msg( "The provided URL is a public link to 'Analysis'. Need public link to 'Data'." ) class Information(widget.OWWidget.Information): response_data_empty = widget.Msg( 'Response data is empty. Get some responses first.') def __init__(self): super().__init__() self.table = None self._html = None def _loadFinished(is_ok): if is_ok: QTimer.singleShot( 1, lambda: setattr(self, '_html', self.webview.html())) self.webview = WebviewWidget(loadFinished=_loadFinished) vb = gui.vBox(self.controlArea, 'Import Data') hb = gui.hBox(vb) self.combo = combo = URLComboBox( hb, self.recent, editable=True, minimumWidth=400, insertPolicy=QComboBox.InsertAtTop, toolTip='Format: ' + VALID_URL_HELP, editTextChanged=self.is_valid_url, # Indirect via QTimer because calling wait() -> processEvents, # while our currentIndexChanged event hadn't yet finished. # Avoids calling handler twice. currentIndexChanged=lambda: QTimer.singleShot(1, self.load_url)) hb.layout().addWidget(QLabel('Public link URL:', hb)) hb.layout().addWidget(combo) hb.layout().setStretch(1, 2) RELOAD_TIMES = ( ('No reload', ), ('5 s', 5000), ('10 s', 10000), ('30 s', 30000), ('1 min', 60 * 1000), ('2 min', 2 * 60 * 1000), ('5 min', 5 * 60 * 1000), ) reload_timer = QTimer(self, timeout=lambda: self.load_url(from_reload=True)) def _on_reload_changed(): if self.reload_idx == 0: reload_timer.stop() return reload_timer.start(RELOAD_TIMES[self.reload_idx][1]) gui.comboBox(vb, self, 'reload_idx', label='Reload every:', orientation=Qt.Horizontal, items=[i[0] for i in RELOAD_TIMES], callback=_on_reload_changed) box = gui.widgetBox(self.controlArea, "Columns (Double-click to edit)") self.domain_editor = DomainEditor(self) editor_model = self.domain_editor.model() def editorDataChanged(): self.apply_domain_edit() self.commit() editor_model.dataChanged.connect(editorDataChanged) box.layout().addWidget(self.domain_editor) box = gui.widgetBox(self.controlArea, "Info", addSpace=True) info = self.data_info = gui.widgetLabel(box, '') info.setWordWrap(True) self.controlArea.layout().addStretch(1) gui.auto_commit(self.controlArea, self, 'autocommit', label='Commit') self.set_info() def set_combo_items(self): self.combo.clear() for sheet in self.recent: self.combo.addItem(sheet.name, sheet.url) def commit(self): self.Outputs.data.send(self.table) def is_valid_url(self, url): if is_valid_url(url): self.Error.invalid_url.clear() return True self.Error.invalid_url() QToolTip.showText(self.combo.mapToGlobal(QPoint(0, 0)), self.combo.toolTip()) def load_url(self, from_reload=False): self.closeContext() self.domain_editor.set_domain(None) url = self.combo.currentText() if not self.is_valid_url(url): self.table = None self.commit() return if url not in self.recent: self.recent.insert(0, url) prev_table = self.table with self.progressBar(3) as progress: try: self._html = None self.webview.setUrl(url) wait(until=lambda: self._html is not None) progress.advance() # Wait some seconds for discrete labels to have loaded via AJAX, # then re-query HTML. # *Webview.loadFinished doesn't guarantee it sufficiently try: wait(until=lambda: False, timeout=1200) except TimeoutError: pass progress.advance() html = self.webview.html() except Exception as e: log.exception("Couldn't load data from: %s", url) self.Error.net_error(try_(lambda: e.args[0], '')) self.table = None else: self.Error.clear() self.Information.clear() self.table = None try: table = self.table = self.table_from_html(html) except DataEmptyError: self.Information.response_data_empty() except DataIsAnalError: self.Error.data_is_anal() except Exception as e: log.exception('Parsing error: %s', url) self.Error.parse_error(try_(lambda: e.args[0], '')) else: self.openContext(table.domain) self.combo.setTitleFor(self.combo.currentIndex(), table.name) def _equal(data1, data2): NAN = float('nan') return (try_(lambda: data1.checksum(), NAN) == try_(lambda: data2.checksum(), NAN)) self._orig_table = self.table self.apply_domain_edit() if not (from_reload and _equal(prev_table, self.table)): self.commit() def apply_domain_edit(self): data = self._orig_table if data is None: self.set_info() return domain, cols = self.domain_editor.get_domain(data.domain, data) # Copied verbatim from OWFile if not (domain.variables or domain.metas): table = None else: X, y, m = cols table = Table.from_numpy(domain, X, y, m, data.W) table.name = data.name table.ids = np.array(data.ids) table.attributes = getattr(data, 'attributes', {}) self.table = table self.set_info() DATETIME_VAR = 'Paradata (insert)' def table_from_html(self, html): soup = BeautifulSoup(html, 'html.parser') try: html_table = soup.find_all('table')[-1] except IndexError: raise DataEmptyError if '<h2>Anal' in html or 'div_analiza_' in html: raise DataIsAnalError def _header_row_strings(row): return chain.from_iterable( repeat(th.get_text(), int(th.get('colspan') or 1)) for th in html_table.select('thead tr:nth-of-type(%d) th[title]' % row)) # self.DATETIME_VAR (available when Paradata is enabled in 1ka UI) # should match this variable name format header = [ th1.rstrip(':') + ('' if th3 == th1 else ' ({})').format(th3.rstrip(':')) for th1, th3 in zip(_header_row_strings(1), _header_row_strings(3)) ] values = [ [ ( # If no span, feature is a number or a text field td.get_text() if td.span is None else # If have span, it's a number, but if negative, replace with NaN '' if td.contents[0].strip().startswith('-') else # Else if span, the number is its code, but we want its value td.span.get_text()[1:-1]) for td in tr.select('td') if 'data_uid' not in td.get('class', ()) ] for tr in html_table.select('tbody tr') ] # Save parsed values into in-mem file for default values processing buffer = StringIO() writer = csv.writer(buffer, delimiter='\t') writer.writerow(header) writer.writerows(values) buffer.flush() buffer.seek(0) data = TabReader(buffer).read() title = soup.select('body h2:nth-of-type(1)')[0].get_text().split( ': ', maxsplit=1)[-1] data.name = title return data def set_info(self): data = self.table if data is None: self.data_info.setText('No spreadsheet loaded.') return text = "{}\n\n{} instance(s), {} feature(s), {} meta attribute(s)\n".format( data.name, len(data), len(data.domain.attributes), len(data.domain.metas)) text += try_( lambda: '\nFirst entry: {}' '\nLast entry: {}'.format(data[0, self.DATETIME_VAR], data[ -1, self.DATETIME_VAR]), '') self.data_info.setText(text)
def __init__(self): super().__init__() RecentPathsWComboMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.reader = None readers = [ f for f in FileFormat.formats if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None) ] def group_readers_per_addon_key(w): # readers from Orange.data.io should go first def package(w): package = w.qualified_name().split(".")[:-1] package = package[:2] if ".".join(package) == "Orange.data": return ["0"] # force "Orange" to come first return package return package(w), w.DESCRIPTION self.available_readers = sorted(set(readers), key=group_readers_per_addon_key) layout = QGridLayout() layout.setSpacing(4) gui.widgetBox(self.controlArea, orientation=layout, box='Source') vbox = gui.radioButtons(None, self, "source", box=True, callback=self.load_data, addToLayout=False) rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False) layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter) box = gui.hBox(None, addToLayout=False, margin=0) box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.activated[int].connect(self.select_file) box.layout().addWidget(self.file_combo) layout.addWidget(box, 0, 1) file_button = gui.button(None, self, '...', callback=self.browse_file, autoDefault=False) file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 2) reload_button = gui.button(None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 3) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = QComboBox() self.sheet_combo.activated[str].connect(self.select_sheet) self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False) layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter) self.url_combo = url_combo = QComboBox() url_model = NamedURLModel(self.sheet_names) url_model.wrap(self.recent_urls) url_combo.setLineEdit(LineEditSelectOnFocus()) url_combo.setModel(url_model) url_combo.setSizePolicy(Policy.Ignored, Policy.Fixed) url_combo.setEditable(True) url_combo.setInsertPolicy(url_combo.InsertAtTop) url_edit = url_combo.lineEdit() l, t, r, b = url_edit.getTextMargins() url_edit.setTextMargins(l + 5, t, r, b) layout.addWidget(url_combo, 3, 1, 1, 3) url_combo.activated.connect(self._url_set) # whit completer we set that combo box is case sensitive when # matching the history completer = QCompleter() completer.setCaseSensitivity(Qt.CaseSensitive) url_combo.setCompleter(completer) layout = QGridLayout() layout.setSpacing(4) gui.widgetBox(self.controlArea, orientation=layout, box='File Type') box = gui.hBox(None, addToLayout=False, margin=0) box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.reader_combo = QComboBox(self) self.reader_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.reader_combo.activated[int].connect(self.select_reader) box.layout().addWidget(self.reader_combo) layout.addWidget(box, 0, 1) box = gui.vBox(self.controlArea, "Info") self.infolabel = gui.widgetLabel(box, 'No data loaded.') box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) box = gui.hBox(box) gui.button(box, self, "Reset", callback=self.reset_domain_edit, autoDefault=False) gui.rubber(box) self.apply_button = gui.button(box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) hBox = gui.hBox(self.controlArea) gui.rubber(hBox) gui.button(hBox, self, "Browse documentation datasets", callback=lambda: self.browse_file(True), autoDefault=False) gui.rubber(hBox) self.set_file_list() # Must not call open_file from within __init__. open_file # explicitly re-enters the event loop (by a progress bar) self.setAcceptDrops(True) if self.source == self.LOCAL_FILE: last_path = self.last_path() if last_path and os.path.exists(last_path) and \ os.path.getsize(last_path) > self.SIZE_LIMIT: self.Warning.file_too_big() return QTimer.singleShot(0, self.load_data)
class OWMultifile(Orange.widgets.data.owfile.OWFile, RecentPathsWidgetMixin): name = "Multifile" id = "orangecontrib.spectroscopy.widgets.files" icon = "icons/multifile.svg" description = "Read data from input files " \ "and send a data table to the output." priority = 10000 replaces = [ "orangecontrib.infrared.widgets.owfiles.OWFiles", "orangecontrib.infrared.widgets.owmultifile.OWMultifile" ] file_idx = [] sheet = Orange.widgets.settings.Setting(None) label = Orange.widgets.settings.Setting("") recent_paths = Orange.widgets.settings.Setting([]) def __init__(self): widget.OWWidget.__init__(self) RecentPathsWidgetMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.sheets = [] self.lb = gui.listBox(self.controlArea, self, "file_idx", selectionMode=QListWidget.MultiSelection) layout = QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) file_button = gui.button(None, self, ' ...', callback=self.browse_files, autoDefault=False) file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 0) remove_button = gui.button(None, self, 'Remove', callback=self.remove_item) clear_button = gui.button(None, self, 'Clear', callback=self.clear) layout.addWidget(remove_button, 0, 1) layout.addWidget(clear_button, 0, 2) reload_button = gui.button(None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 7) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox(None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True) self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() layout.addWidget(self.sheet_box, 0, 5) label_box = gui.hBox(None, addToLayout=False, margin=0) label = gui.lineEdit(label_box, self, "label", callback=self.set_label, label="Label", orientation=Qt.Horizontal) layout.addWidget(label_box, 0, 6) layout.setColumnStretch(3, 2) box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) for i, rp in enumerate(self.recent_paths): self.lb.addItem(rp.abspath) # TODO unresolved paths just disappear! Modify _relocate_recent_files box = gui.hBox(self.controlArea) gui.rubber(box) box.layout().addWidget(self.report_button) self.report_button.setFixedWidth(170) self.apply_button = gui.button(box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) self._update_sheet_combo() self.load_data() def set_label(self): self.load_data() def add_path(self, filename): recent = RecentPath.create(filename, self._search_paths()) self.recent_paths.append(recent) def set_file_list(self): # need to define it for RecentPathsWidgetMixin pass def _select_active_sheet(self): if self.sheet: try: sheet_list = [s[0] for s in self.sheets] idx = sheet_list.index(self.sheet) self.sheet_combo.setCurrentIndex(idx) except ValueError: # Requested sheet does not exist in this file self.sheet = None else: self.sheet_combo.setCurrentIndex(0) def _update_sheet_combo(self): sheets = Counter() for fn in self.current_filenames(): try: reader = FileFormat.get_reader(fn) sheets.update(reader.sheets) except: pass sheets = sorted(sheets.items(), key=lambda x: x[0]) self.sheets = [(s, s + " (" + str(n) + ")") for s, n in sheets] if len(sheets) < 2: self.sheet_box.hide() self.sheet = None else: self.sheets.insert(0, (None, "(None)")) self.sheet_combo.clear() self.sheet_combo.addItems([s[1] for s in self.sheets]) self._select_active_sheet() self.sheet_box.show() def select_sheet(self): self.sheet = self.sheets[self.sheet_combo.currentIndex()][0] self.load_data() def remove_item(self): ri = [i.row() for i in self.lb.selectedIndexes()] for i in sorted(ri, reverse=True): self.recent_paths.pop(i) self.lb.takeItem(i) self._update_sheet_combo() self.load_data() def clear(self): self.lb.clear() while self.recent_paths: self.recent_paths.pop() self._update_sheet_combo() self.load_data() def browse_files(self, in_demos=False): start_file = self.last_path() or os.path.expanduser("~/") filenames = QFileDialog.getOpenFileNames(self, 'Open Multiple Data Files', start_file, dialog_formats()) if isinstance(filenames, tuple): # has a file description filenames = filenames[0] self.load_files(filenames) def load_files(self, filenames): if not filenames: return for f in filenames: self.add_path(f) self.lb.addItem(f) self._update_sheet_combo() self.load_data() def current_filenames(self): return [rp.abspath for rp in self.recent_paths] def load_data(self): self.closeContext() fns = self.current_filenames() data_list = [] fnok_list = [] empty_domain = Orange.data.Domain(attributes=[]) for fn in fns: reader = FileFormat.get_reader(fn) errors = [] with catch_warnings(record=True) as warnings: try: if self.sheet in reader.sheets: reader.select_sheet(self.sheet) if isinstance(reader, SpectralFileFormat): xs, vals, additional = reader.read_spectra() if additional is None: additional = Orange.data.Table.from_domain( empty_domain, n_rows=len(vals)) data_list.append((xs, vals, additional)) else: data_list.append(reader.read()) fnok_list.append(fn) except Exception as ex: errors.append("An error occurred:") errors.append(str(ex)) #FIXME show error in the list of data self.warning(warnings[-1].message.args[0] if warnings else '') if data_list: data = concatenate_data(data_list, fnok_list, self.label) self.data = data self.openContext(data.domain) else: self.data = None self.domain_editor.set_domain(None) self.apply_domain_edit() # sends data
def __init__(self): super().__init__() self.table = None self._html = None def _loadFinished(is_ok): if is_ok: QTimer.singleShot( 1, lambda: setattr(self, '_html', self.webview.html())) self.webview = WebviewWidget(loadFinished=_loadFinished) vb = gui.vBox(self.controlArea, 'Import Data') hb = gui.hBox(vb) self.combo = combo = URLComboBox( hb, self.recent, editable=True, minimumWidth=400, insertPolicy=QComboBox.InsertAtTop, toolTip='Format: ' + VALID_URL_HELP, editTextChanged=self.is_valid_url, # Indirect via QTimer because calling wait() -> processEvents, # while our currentIndexChanged event hadn't yet finished. # Avoids calling handler twice. currentIndexChanged=lambda: QTimer.singleShot(1, self.load_url)) hb.layout().addWidget(QLabel('Public link URL:', hb)) hb.layout().addWidget(combo) hb.layout().setStretch(1, 2) RELOAD_TIMES = ( ('No reload', ), ('5 s', 5000), ('10 s', 10000), ('30 s', 30000), ('1 min', 60 * 1000), ('2 min', 2 * 60 * 1000), ('5 min', 5 * 60 * 1000), ) reload_timer = QTimer(self, timeout=lambda: self.load_url(from_reload=True)) def _on_reload_changed(): if self.reload_idx == 0: reload_timer.stop() return reload_timer.start(RELOAD_TIMES[self.reload_idx][1]) gui.comboBox(vb, self, 'reload_idx', label='Reload every:', orientation=Qt.Horizontal, items=[i[0] for i in RELOAD_TIMES], callback=_on_reload_changed) box = gui.widgetBox(self.controlArea, "Columns (Double-click to edit)") self.domain_editor = DomainEditor(self) editor_model = self.domain_editor.model() def editorDataChanged(): self.apply_domain_edit() self.commit() editor_model.dataChanged.connect(editorDataChanged) box.layout().addWidget(self.domain_editor) box = gui.widgetBox(self.controlArea, "Info", addSpace=True) info = self.data_info = gui.widgetLabel(box, '') info.setWordWrap(True) self.controlArea.layout().addStretch(1) gui.auto_commit(self.controlArea, self, 'autocommit', label='Commit') self.set_info()
class OWFile(widget.OWWidget, RecentPathsWComboMixin): name = "文件(File)" id = "orange.widgets.data.file" description = "从输入文件或网络读取数据并将数据表发送到输出。" icon = "icons/File.svg" priority = 10 category = "数据(Data)" keywords = ["file", "load", "read", "open", "wenjian"] class Outputs: data = Output("数据(Data)", Table, doc="Attribute-valued dataset read from the input file.", replaces=['Data']) want_main_area = False buttons_area_orientation = None SEARCH_PATHS = [("sample-datasets", get_sample_datasets_dir())] SIZE_LIMIT = 1e7 LOCAL_FILE, URL = range(2) settingsHandler = PerfectDomainContextHandler( match_values=PerfectDomainContextHandler.MATCH_VALUES_ALL) # pylint seems to want declarations separated from definitions recent_paths: List[RecentPath] recent_urls: List[str] variables: list # Overload RecentPathsWidgetMixin.recent_paths to set defaults recent_paths = Setting([ RecentPath("", "sample-datasets", "iris.tab"), RecentPath("", "sample-datasets", "titanic.tab"), RecentPath("", "sample-datasets", "housing.tab"), RecentPath("", "sample-datasets", "heart_disease.tab"), RecentPath("", "sample-datasets", "brown-selected.tab"), RecentPath("", "sample-datasets", "zoo.tab"), ]) recent_urls = Setting([]) source = Setting(LOCAL_FILE) sheet_names = Setting({}) url = Setting("") variables = ContextSetting([]) domain_editor = SettingProvider(DomainEditor) class Information(widget.OWWidget.Information): no_file_selected = Msg("No file selected.") class Warning(widget.OWWidget.Warning): file_too_big = Msg("The file is too large to load automatically." " Press Reload to load.") load_warning = Msg("Read warning:\n{}") performance_warning = Msg( "Categorical variables with >100 values may decrease performance.") renamed_vars = Msg("Some variables have been renamed " "to avoid duplicates.\n{}") multiple_targets = Msg("Most widgets do not support multiple targets") class Error(widget.OWWidget.Error): file_not_found = Msg("File not found.") missing_reader = Msg("Missing reader.") sheet_error = Msg("Error listing available sheets.") unknown = Msg("Read error:\n{}") UserAdviceMessages = [ widget.Message( "Use CSV File Import widget for advanced options " "for comma-separated files", "use-csv-file-import"), widget.Message( "This widget loads only tabular data. Use other widgets to load " "other data types like models, distance matrices and networks.", "other-data-types") ] def __init__(self): super().__init__() RecentPathsWComboMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.reader = None readers = [ f for f in FileFormat.formats if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None) ] def group_readers_per_addon_key(w): # readers from Orange.data.io should go first def package(w): package = w.qualified_name().split(".")[:-1] package = package[:2] if ".".join(package) == "Orange.data": return ["0"] # force "Orange" to come first return package return package(w), w.DESCRIPTION self.available_readers = sorted(set(readers), key=group_readers_per_addon_key) layout = QGridLayout() layout.setSpacing(4) gui.widgetBox(self.controlArea, orientation=layout, box='数据源') vbox = gui.radioButtons(None, self, "source", box=True, callback=self.load_data, addToLayout=False) rb_button = gui.appendRadioButton(vbox, "文件:", addToLayout=False) layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter) box = gui.hBox(None, addToLayout=False, margin=0) box.setSizePolicy(Policy.Expanding, Policy.Fixed) self.file_combo.setSizePolicy(Policy.Expanding, Policy.Fixed) self.file_combo.setMinimumSize(QSize(100, 1)) self.file_combo.activated[int].connect(self.select_file) box.layout().addWidget(self.file_combo) layout.addWidget(box, 0, 1) file_button = gui.button(None, self, '...', callback=self.browse_file, autoDefault=False) file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 2) reload_button = gui.button(None, self, "重新加载", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 3) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = QComboBox() self.sheet_combo.activated[str].connect(self.select_sheet) self.sheet_combo.setSizePolicy(Policy.Expanding, Policy.Fixed) self.sheet_combo.setMinimumSize(QSize(50, 1)) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False) layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter) self.url_combo = url_combo = QComboBox() url_model = NamedURLModel(self.sheet_names) url_model.wrap(self.recent_urls) url_combo.setLineEdit(LineEditSelectOnFocus()) url_combo.setModel(url_model) url_combo.setSizePolicy(Policy.Ignored, Policy.Fixed) url_combo.setEditable(True) url_combo.setInsertPolicy(url_combo.InsertAtTop) url_edit = url_combo.lineEdit() l, t, r, b = url_edit.getTextMargins() url_edit.setTextMargins(l + 5, t, r, b) layout.addWidget(url_combo, 3, 1, 1, 3) url_combo.activated.connect(self._url_set) # whit completer we set that combo box is case sensitive when # matching the history completer = QCompleter() completer.setCaseSensitivity(Qt.CaseSensitive) url_combo.setCompleter(completer) layout = QGridLayout() layout.setSpacing(4) gui.widgetBox(self.controlArea, orientation=layout, box='文件类型') box = gui.hBox(None, addToLayout=False, margin=0) box.setSizePolicy(Policy.Expanding, Policy.Fixed) self.reader_combo = QComboBox(self) self.reader_combo.setSizePolicy(Policy.Expanding, Policy.Fixed) self.reader_combo.setMinimumSize(QSize(100, 1)) self.reader_combo.activated[int].connect(self.select_reader) box.layout().addWidget(self.reader_combo) layout.addWidget(box, 0, 1) box = gui.vBox(self.controlArea, "信息") self.infolabel = gui.widgetLabel(box, '未加载数据.') box = gui.widgetBox(self.controlArea, "列(双击编辑)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) box = gui.hBox(box) gui.button(box, self, "重置", callback=self.reset_domain_edit, autoDefault=False) gui.rubber(box) self.apply_button = gui.button(box, self, "应用", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) hBox = gui.hBox(self.controlArea) gui.rubber(hBox) gui.button(hBox, self, "浏览文档数据集", callback=lambda: self.browse_file(True), autoDefault=False) gui.rubber(hBox) self.set_file_list() # Must not call open_file from within __init__. open_file # explicitly re-enters the event loop (by a progress bar) self.setAcceptDrops(True) if self.source == self.LOCAL_FILE: last_path = self.last_path() if last_path and os.path.exists(last_path) and \ os.path.getsize(last_path) > self.SIZE_LIMIT: self.Warning.file_too_big() return QTimer.singleShot(0, self.load_data) @staticmethod def sizeHint(): return QSize(600, 550) def select_file(self, n): assert n < len(self.recent_paths) super().select_file(n) if self.recent_paths: self.source = self.LOCAL_FILE self.load_data() self.set_file_list() def select_sheet(self): self.recent_paths[0].sheet = self.sheet_combo.currentText() self.load_data() def select_reader(self, n): if self.source != self.LOCAL_FILE: return # ignore for URL's if self.recent_paths: path = self.recent_paths[0] if n == 0: # default path.file_format = None self.load_data() elif n <= len(self.available_readers): reader = self.available_readers[n - 1] path.file_format = reader.qualified_name() self.load_data() else: # the rest include just qualified names path.file_format = self.reader_combo.itemText(n) self.load_data() def _url_set(self): url = self.url_combo.currentText() pos = self.recent_urls.index(url) url = url.strip() if not urlparse(url).scheme: url = 'http://' + url self.url_combo.setItemText(pos, url) self.recent_urls[pos] = url self.source = self.URL self.load_data() def browse_file(self, in_demos=False): if in_demos: start_file = get_sample_datasets_dir() if not os.path.exists(start_file): QMessageBox.information(None, "文件", "无法找到文件") return else: start_file = self.last_path() or os.path.expanduser("~/") filename, reader, _ = open_filename_dialog(start_file, None, self.available_readers) if not filename: return self.add_path(filename) if reader is not None: self.recent_paths[0].file_format = reader.qualified_name() self.source = self.LOCAL_FILE self.load_data() # Open a file, create data from it and send it over the data channel def load_data(self): # We need to catch any exception type since anything can happen in # file readers self.closeContext() self.domain_editor.set_domain(None) self.apply_button.setEnabled(False) self.clear_messages() self.set_file_list() error = self._try_load() if error: error() self.data = None self.sheet_box.hide() self.Outputs.data.send(None) self.infolabel.setText("无数据") def _try_load(self): self._initialize_reader_combo() # pylint: disable=broad-except if self.source == self.LOCAL_FILE: if self.last_path() is None: return self.Information.no_file_selected elif not os.path.exists(self.last_path()): return self.Error.file_not_found else: url = self.url_combo.currentText().strip() if not url: return self.Information.no_file_selected def mark_problematic_reader(): self.reader_combo.setItemData(self.reader_combo.currentIndex(), QBrush(Qt.red), Qt.ForegroundRole) try: self.reader = self._get_reader() # also sets current reader index assert self.reader is not None except MissingReaderException: mark_problematic_reader() return self.Error.missing_reader except Exception as ex: mark_problematic_reader() log.exception(ex) return lambda x=ex: self.Error.unknown(str(x)) try: self._update_sheet_combo() except Exception: return self.Error.sheet_error with log_warnings() as warnings: try: data = self.reader.read() except Exception as ex: mark_problematic_reader() log.exception(ex) return lambda x=ex: self.Error.unknown(str(x)) if warnings: self.Warning.load_warning(warnings[-1].message.args[0]) self.infolabel.setText(self._describe(data)) self.loaded_file = self.last_path() add_origin(data, self.loaded_file) self.data = data self.openContext(data.domain) self.apply_domain_edit() # sends data return None def _get_reader(self) -> FileFormat: if self.source == self.LOCAL_FILE: path = self.last_path() self.reader_combo.setEnabled(True) if self.recent_paths and self.recent_paths[0].file_format: qname = self.recent_paths[0].file_format qname_index = { r.qualified_name(): i for i, r in enumerate(self.available_readers) } if qname in qname_index: self.reader_combo.setCurrentIndex(qname_index[qname] + 1) else: # reader may be accessible, but not in self.available_readers # (perhaps its code was moved) self.reader_combo.addItem(qname) self.reader_combo.setCurrentIndex( len(self.reader_combo) - 1) try: reader_class = class_from_qualified_name(qname) except Exception as ex: raise MissingReaderException( f'Can not find reader "{qname}"') from ex reader = reader_class(path) else: self.reader_combo.setCurrentIndex(0) reader = FileFormat.get_reader(path) if self.recent_paths and self.recent_paths[0].sheet: reader.select_sheet(self.recent_paths[0].sheet) return reader else: url = self.url_combo.currentText().strip() return UrlReader(url) def _update_sheet_combo(self): if len(self.reader.sheets) < 2: self.sheet_box.hide() self.reader.select_sheet(None) return self.sheet_combo.clear() self.sheet_combo.addItems(self.reader.sheets) self._select_active_sheet() self.sheet_box.show() def _select_active_sheet(self): try: idx = self.reader.sheets.index(self.reader.sheet) self.sheet_combo.setCurrentIndex(idx) except ValueError: # Requested sheet does not exist in this file self.reader.select_sheet(None) self.sheet_combo.setCurrentIndex(0) def _initialize_reader_combo(self): self.reader_combo.clear() filters = [format_filter(f) for f in self.available_readers] self.reader_combo.addItems([DEFAULT_READER_TEXT] + filters) self.reader_combo.setCurrentIndex(0) self.reader_combo.setDisabled(True) # additional readers may be added in self._get_reader() @staticmethod def _describe(table): def missing_prop(prop): if prop: return f"({prop * 100:.1f}% 个缺失值)" else: return "(无缺失值)" domain = table.domain text = "" attrs = getattr(table, "attributes", {}) descs = [ attrs[desc] for desc in ("Name", "Description") if desc in attrs ] if len(descs) == 2: descs[0] = f"<b>{descs[0]}</b>" if descs: text += f"<p>{'<br/>'.join(descs)}</p>" text += f"<p>{len(table)} 条数据" missing_in_attr = missing_prop(table.has_missing_attribute() and table.get_nan_frequency_attribute()) missing_in_class = missing_prop(table.has_missing_class() and table.get_nan_frequency_class()) text += f"<br/>特征数目: {len(domain.attributes)} {missing_in_attr}" if domain.has_continuous_class: text += f"<br/>回归; 数值类 {missing_in_class}" elif domain.has_discrete_class: text += "<br/>分类: 分类种类共 " \ f"{len(domain.class_var.values)} 个 {missing_in_class}" elif table.domain.class_vars: text += "<br/>Multi-target; " \ f"{len(table.domain.class_vars)} target variables " \ f"{missing_in_class}" else: text += "<br/>Data has no target variable." text += f"<br/>元属性: { len(domain.metas)}" text += "</p>" if 'Timestamp' in table.domain: # Google Forms uses this header to timestamp responses text += f"<p>First entry: {table[0, 'Timestamp']}<br/>" \ f"Last entry: {table[-1, 'Timestamp']}</p>" return text def storeSpecificSettings(self): self.current_context.modified_variables = self.variables[:] def retrieveSpecificSettings(self): if hasattr(self.current_context, "modified_variables"): self.variables[:] = self.current_context.modified_variables def reset_domain_edit(self): self.domain_editor.reset_domain() self.apply_domain_edit() def _inspect_discrete_variables(self, domain): for var in chain(domain.variables, domain.metas): if var.is_discrete and len(var.values) > 100: self.Warning.performance_warning() def apply_domain_edit(self): self.Warning.performance_warning.clear() self.Warning.renamed_vars.clear() if self.data is None: table = None else: domain, cols, renamed = \ self.domain_editor.get_domain(self.data.domain, self.data, deduplicate=True) if not (domain.variables or domain.metas): table = None elif domain is self.data.domain: table = self.data else: X, y, m = cols table = Table.from_numpy(domain, X, y, m, self.data.W) table.name = self.data.name table.ids = np.array(self.data.ids) table.attributes = getattr(self.data, 'attributes', {}) self._inspect_discrete_variables(domain) if renamed: self.Warning.renamed_vars(f"Renamed: {', '.join(renamed)}") self.Warning.multiple_targets( shown=table is not None and len(table.domain.class_vars) > 1) self.Outputs.data.send(table) self.apply_button.setEnabled(False) def get_widget_name_extension(self): _, name = os.path.split(self.loaded_file) return os.path.splitext(name)[0] def send_report(self): def get_ext_name(filename): try: return FileFormat.names[os.path.splitext(filename)[1]] except KeyError: return "unknown" if self.data is None: self.report_paragraph("File", "No file.") return if self.source == self.LOCAL_FILE: home = os.path.expanduser("~") if self.loaded_file.startswith(home): # os.path.join does not like ~ name = "~" + os.path.sep + \ self.loaded_file[len(home):].lstrip("/").lstrip("\\") else: name = self.loaded_file if self.sheet_combo.isVisible(): name += f" ({self.sheet_combo.currentText()})" self.report_items("File", [("File name", name), ("Format", get_ext_name(name))]) else: self.report_items("Data", [("Resource", self.url), ("Format", get_ext_name(self.url))]) self.report_data("Data", self.data) @staticmethod def dragEnterEvent(event): """Accept drops of valid file urls""" urls = event.mimeData().urls() if urls: try: FileFormat.get_reader(urls[0].toLocalFile()) event.acceptProposedAction() except MissingReaderException: pass def dropEvent(self, event): """Handle file drops""" urls = event.mimeData().urls() if urls: self.add_path(urls[0].toLocalFile()) # add first file self.source = self.LOCAL_FILE self.load_data() def workflowEnvChanged(self, key, value, oldvalue): """ Function called when environment changes (e.g. while saving the scheme) It make sure that all environment connected values are modified (e.g. relative file paths are changed) """ self.update_file_list(key, value, oldvalue)
def __init__(self): widget.OWWidget.__init__(self) RecentPathsWidgetMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.sheets = [] self.lb = gui.listBox(self.controlArea, self, "file_idx", selectionMode=QListWidget.MultiSelection) layout = QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) file_button = gui.button( None, self, ' ...', callback=self.browse_files, autoDefault=False) file_button.setIcon(self.style().standardIcon( QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 0) remove_button = gui.button( None, self, 'Remove', callback=self.remove_item) clear_button = gui.button( None, self, 'Clear', callback=self.clear) layout.addWidget(remove_button, 0, 1) layout.addWidget(clear_button, 0, 2) reload_button = gui.button( None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon(QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 7) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox(None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True) self.sheet_combo.setSizePolicy( Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy( Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget( self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget( self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() layout.addWidget(self.sheet_box, 0, 5) label_box = gui.hBox(None, addToLayout=False, margin=0) label = gui.lineEdit(label_box, self, "label", callback=self.set_label, label="Label", orientation=Qt.Horizontal) layout.addWidget(label_box, 0, 6) layout.setColumnStretch(3, 2) box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) for i, rp in enumerate(self.recent_paths): self.lb.addItem(rp.abspath) # TODO unresolved paths just disappear! Modify _relocate_recent_files box = gui.hBox(self.controlArea) gui.rubber(box) box.layout().addWidget(self.report_button) self.report_button.setFixedWidth(170) self.apply_button = gui.button( box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) self._update_sheet_combo() self.load_data()
class OWMultifile(Orange.widgets.data.owfile.OWFile, RecentPathsWidgetMixin): name = "Multifile" id = "orangecontrib.spectroscopy.widgets.files" icon = "icons/multifile.svg" description = "Read data from input files " \ "and send a data table to the output." priority = 10000 replaces = ["orangecontrib.infrared.widgets.owfiles.OWFiles", "orangecontrib.infrared.widgets.owmultifile.OWMultifile"] file_idx = [] sheet = Orange.widgets.settings.Setting(None) label = Orange.widgets.settings.Setting("") recent_paths = Orange.widgets.settings.Setting([]) def __init__(self): widget.OWWidget.__init__(self) RecentPathsWidgetMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.sheets = [] self.lb = gui.listBox(self.controlArea, self, "file_idx", selectionMode=QListWidget.MultiSelection) layout = QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) file_button = gui.button( None, self, ' ...', callback=self.browse_files, autoDefault=False) file_button.setIcon(self.style().standardIcon( QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 0) remove_button = gui.button( None, self, 'Remove', callback=self.remove_item) clear_button = gui.button( None, self, 'Clear', callback=self.clear) layout.addWidget(remove_button, 0, 1) layout.addWidget(clear_button, 0, 2) reload_button = gui.button( None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon(QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 7) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox(None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True) self.sheet_combo.setSizePolicy( Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy( Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget( self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget( self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() layout.addWidget(self.sheet_box, 0, 5) label_box = gui.hBox(None, addToLayout=False, margin=0) label = gui.lineEdit(label_box, self, "label", callback=self.set_label, label="Label", orientation=Qt.Horizontal) layout.addWidget(label_box, 0, 6) layout.setColumnStretch(3, 2) box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) for i, rp in enumerate(self.recent_paths): self.lb.addItem(rp.abspath) # TODO unresolved paths just disappear! Modify _relocate_recent_files box = gui.hBox(self.controlArea) gui.rubber(box) box.layout().addWidget(self.report_button) self.report_button.setFixedWidth(170) self.apply_button = gui.button( box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) self._update_sheet_combo() self.load_data() def set_label(self): self.load_data() def add_path(self, filename, reader=None): recent = RecentPath.create(filename, self._search_paths()) if reader is not None: recent.file_format = reader.qualified_name() self.recent_paths.append(recent) def set_file_list(self): # need to define it for RecentPathsWidgetMixin pass def _select_active_sheet(self): if self.sheet: try: sheet_list = [s[0] for s in self.sheets] idx = sheet_list.index(self.sheet) self.sheet_combo.setCurrentIndex(idx) except ValueError: # Requested sheet does not exist in this file self.sheet = None else: self.sheet_combo.setCurrentIndex(0) def _update_sheet_combo(self): sheets = Counter() for rp in self.recent_paths: try: reader = _get_reader(rp) sheets.update(reader.sheets) except: pass sheets = sorted(sheets.items(), key=lambda x: x[0]) self.sheets = [(s, s + " (" + str(n) + ")") for s, n in sheets] if len(sheets) < 2: self.sheet_box.hide() self.sheet = None else: self.sheets.insert(0, (None, "(None)")) self.sheet_combo.clear() self.sheet_combo.addItems([s[1] for s in self.sheets]) self._select_active_sheet() self.sheet_box.show() def select_sheet(self): self.sheet = self.sheets[self.sheet_combo.currentIndex()][0] self.load_data() def remove_item(self): ri = [ i.row() for i in self.lb.selectedIndexes() ] for i in sorted(ri, reverse=True): self.recent_paths.pop(i) self.lb.takeItem(i) self._update_sheet_combo() self.load_data() def clear(self): self.lb.clear() while self.recent_paths: self.recent_paths.pop() self._update_sheet_combo() self.load_data() def browse_files(self, in_demos=False): start_file = self.last_path() or os.path.expanduser("~/") readers = [f for f in FileFormat.formats if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None)] filenames, reader, _ = open_filename_dialog(start_file, None, readers, dialog=QFileDialog.getOpenFileNames) self.load_files(filenames, reader) def load_files(self, filenames, reader): if not filenames: return for f in filenames: self.add_path(f, reader) self.lb.addItem(f) self._update_sheet_combo() self.load_data() def load_data(self): self.closeContext() data_list = [] fnok_list = [] empty_domain = Orange.data.Domain(attributes=[]) for rp in self.recent_paths: fn = rp.abspath reader = _get_reader(rp) errors = [] with catch_warnings(record=True) as warnings: try: if self.sheet in reader.sheets: reader.select_sheet(self.sheet) if isinstance(reader, SpectralFileFormat): xs, vals, additional = reader.read_spectra() if additional is None: additional = Orange.data.Table.from_domain(empty_domain, n_rows=len(vals)) data_list.append((xs, vals, additional)) else: data_list.append(reader.read()) fnok_list.append(fn) except Exception as ex: errors.append("An error occurred:") errors.append(str(ex)) #FIXME show error in the list of data self.warning(warnings[-1].message.args[0] if warnings else '') if data_list: data = concatenate_data(data_list, fnok_list, self.label) self.data = data self.openContext(data.domain) else: self.data = None self.domain_editor.set_domain(None) self.apply_domain_edit() # sends data
def __init__(self): super().__init__() RecentPathsWComboMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.reader = None layout = QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) vbox = gui.radioButtons(None, self, "source", box=True, addSpace=True, callback=self.load_data, addToLayout=False) rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False) layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter) box = gui.hBox(None, addToLayout=False, margin=0) box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.activated[int].connect(self.select_file) box.layout().addWidget(self.file_combo) layout.addWidget(box, 0, 1) file_button = gui.button( None, self, '...', callback=self.browse_file, autoDefault=False) file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 2) reload_button = gui.button( None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 3) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox(None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True,) self.sheet_combo.setSizePolicy( Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy( Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget( self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget( self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False) layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter) self.url_combo = url_combo = QComboBox() url_model = NamedURLModel(self.sheet_names) url_model.wrap(self.recent_urls) url_combo.setLineEdit(LineEditSelectOnFocus()) url_combo.setModel(url_model) url_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) url_combo.setEditable(True) url_combo.setInsertPolicy(url_combo.InsertAtTop) url_edit = url_combo.lineEdit() l, t, r, b = url_edit.getTextMargins() url_edit.setTextMargins(l + 5, t, r, b) layout.addWidget(url_combo, 3, 1, 3, 3) url_combo.activated.connect(self._url_set) box = gui.vBox(self.controlArea, "Info") self.infolabel = gui.widgetLabel(box, 'No data loaded.') self.warnings = gui.widgetLabel(box, '') box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) box = gui.hBox(self.controlArea) gui.button( box, self, "Browse documentation datasets", callback=lambda: self.browse_file(True), autoDefault=False) gui.rubber(box) gui.button( box, self, "Reset", callback=self.reset_domain_edit) self.apply_button = gui.button( box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) self.set_file_list() # Must not call open_file from within __init__. open_file # explicitly re-enters the event loop (by a progress bar) self.setAcceptDrops(True) if self.source == self.LOCAL_FILE: last_path = self.last_path() if last_path and os.path.exists(last_path) and \ os.path.getsize(last_path) > self.SIZE_LIMIT: self.Warning.file_too_big() return QTimer.singleShot(0, self.load_data)
class OWFiles(Orange.widgets.data.owfile.OWFile, RecentPathsWidgetMixin): name = "Files" id = "orangecontrib.infrared.widgets.files" icon = "icons/files.svg" description = "Read data from input files " \ "and send a data table to the output." file_idx = -1 sheet = Orange.widgets.settings.Setting(None) label = Orange.widgets.settings.Setting("") recent_paths = Orange.widgets.settings.Setting([]) def __init__(self): widget.OWWidget.__init__(self) RecentPathsWidgetMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.sheets = [] self.lb = gui.listBox(self.controlArea, self, "file_idx") layout = QtGui.QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) file_button = gui.button(None, self, ' ...', callback=self.browse_files, autoDefault=False) file_button.setIcon(self.style().standardIcon( QtGui.QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 0) remove_button = gui.button(None, self, 'Remove', callback=self.remove_item) clear_button = gui.button(None, self, 'Clear', callback=self.clear) layout.addWidget(remove_button, 0, 1) layout.addWidget(clear_button, 0, 2) reload_button = gui.button(None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QtGui.QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 7) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox(None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True) self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QtGui.QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget(self.sheet_label, QtCore.Qt.AlignLeft) self.sheet_box.layout().addWidget(self.sheet_combo, QtCore.Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() layout.addWidget(self.sheet_box, 0, 5) label_box = gui.hBox(None, addToLayout=False, margin=0) label = gui.lineEdit(label_box, self, "label", callback=self.set_label, label="Label", orientation=Qt.Horizontal) layout.addWidget(label_box, 0, 6) layout.setColumnStretch(3, 2) box = gui.widgetBox(self.controlArea, "View loaded columns") if OLD_DOMAINEDITOR: self.domain_editor = DomainEditor(self.variables) else: self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) for i, rp in enumerate(self.recent_paths): self.lb.addItem(rp.abspath) # TODO unresolved paths just disappear! Modify _relocate_recent_files self._update_sheet_combo() self.load_data() def set_label(self): self.load_data() def add_path(self, filename): recent = RecentPath.create(filename, self._search_paths()) self.recent_paths.append(recent) def set_file_list(self): # need to define it for RecentPathsWidgetMixin pass def _select_active_sheet(self): if self.sheet: try: sheet_list = [s[0] for s in self.sheets] idx = sheet_list.index(self.sheet) self.sheet_combo.setCurrentIndex(idx) except ValueError: # Requested sheet does not exist in this file self.sheet = None else: self.sheet_combo.setCurrentIndex(0) def _update_sheet_combo(self): sheets = Counter() for fn in self.current_filenames(): reader = FileFormat.get_reader(fn) sheets.update(reader.sheets) sheets = sorted(sheets.items(), key=lambda x: x[0]) self.sheets = [(s, s + " (" + str(n) + ")") for s, n in sheets] if len(sheets) < 2: self.sheet_box.hide() self.sheet = None else: self.sheets.insert(0, (None, "(None)")) self.sheet_combo.clear() self.sheet_combo.addItems([s[1] for s in self.sheets]) self._select_active_sheet() self.sheet_box.show() def select_sheet(self): self.sheet = self.sheets[self.sheet_combo.currentIndex()][0] self.load_data() def remove_item(self): ri = [i.row() for i in self.lb.selectedIndexes()] for i in sorted(ri, reverse=True): self.recent_paths.pop(i) self.lb.takeItem(i) self._update_sheet_combo() self.load_data() def clear(self): self.lb.clear() while self.recent_paths: self.recent_paths.pop() self._update_sheet_combo() self.load_data() def browse_files(self, in_demos=False): if in_demos: start_file = get_sample_datasets_dir() if not os.path.exists(start_file): QtGui.QMessageBox.information( None, "File", "Cannot find the directory with documentation data sets") return else: start_file = self.last_path() or os.path.expanduser("~/") filenames = QtGui.QFileDialog.getOpenFileNames( self, 'Open Multiple Data Files', start_file, self.dlg_formats) if not filenames: return for f in filenames: self.add_path(f) self.lb.addItem(f) self._update_sheet_combo() self.load_data() def current_filenames(self): return [rp.abspath for rp in self.recent_paths] def load_data(self): self.closeContext() fns = self.current_filenames() data_list = [] fnok_list = [] for fn in fns: reader = FileFormat.get_reader(fn) if self.sheet in reader.sheets: reader.select_sheet(self.sheet) errors = [] with catch_warnings(record=True) as warnings: try: data_list.append(reader.read()) fnok_list.append(fn) except Exception as ex: errors.append("An error occurred:") errors.append(str(ex)) #FIXME show error in the list of data self.warning(warnings[-1].message.args[0] if warnings else '') #code below is from concatenate widget if data_list: tables = data_list domain = reduce(domain_union, (table.domain for table in tables)) tables = [ Orange.data.Table.from_table(domain, table) for table in tables ] data = concat(tables) source_var = Orange.data.StringVariable.make("Filename") source_values = list( chain(*(repeat(fn, len(table)) for fn, table in zip(fnok_list, tables)))) label_var = Orange.data.StringVariable.make("Label") label_values = list( chain(*(repeat(self.label, len(table)) for fn, table in zip(fnok_list, tables)))) data = append_columns( data, **{ "metas": [(source_var, source_values), (label_var, label_values)] }) self.data = data if OLD_DOMAINEDITOR: self.editor_model.set_domain(data.domain) else: self.openContext(data.domain) else: self.data = None if OLD_DOMAINEDITOR: self.editor_model.reset() else: self.domain_editor.set_domain(None) self.send("Data", self.data)
class OWFile(widget.OWWidget, RecentPathsWComboMixin): name = "File" id = "orange.widgets.data.file" description = "Read data from an input file or network " \ "and send a data table to the output." icon = "icons/File.svg" priority = 10 category = "Data" keywords = ["file", "load", "read", "open"] class Outputs: data = Output("Data", Table, doc="Attribute-valued dataset read from the input file.") want_main_area = False SEARCH_PATHS = [("sample-datasets", get_sample_datasets_dir())] SIZE_LIMIT = 1e7 LOCAL_FILE, URL = range(2) settingsHandler = PerfectDomainContextHandler( match_values=PerfectDomainContextHandler.MATCH_VALUES_ALL ) # pylint seems to want declarations separated from definitions recent_paths: List[RecentPath] recent_urls: List[str] variables: list # Overload RecentPathsWidgetMixin.recent_paths to set defaults recent_paths = Setting([ RecentPath("", "sample-datasets", "iris.tab"), RecentPath("", "sample-datasets", "titanic.tab"), RecentPath("", "sample-datasets", "housing.tab"), RecentPath("", "sample-datasets", "heart_disease.tab"), ]) recent_urls = Setting([]) source = Setting(LOCAL_FILE) xls_sheet = ContextSetting("") sheet_names = Setting({}) url = Setting("") variables = ContextSetting([]) domain_editor = SettingProvider(DomainEditor) class Warning(widget.OWWidget.Warning): file_too_big = widget.Msg("The file is too large to load automatically." " Press Reload to load.") load_warning = widget.Msg("Read warning:\n{}") class Error(widget.OWWidget.Error): file_not_found = widget.Msg("File not found.") missing_reader = widget.Msg("Missing reader.") sheet_error = widget.Msg("Error listing available sheets.") unknown = widget.Msg("Read error:\n{}") class NoFileSelected: pass def __init__(self): super().__init__() RecentPathsWComboMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.reader = None layout = QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) vbox = gui.radioButtons(None, self, "source", box=True, addSpace=True, callback=self.load_data, addToLayout=False) rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False) layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter) box = gui.hBox(None, addToLayout=False, margin=0) box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.activated[int].connect(self.select_file) box.layout().addWidget(self.file_combo) layout.addWidget(box, 0, 1) file_button = gui.button( None, self, '...', callback=self.browse_file, autoDefault=False) file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 2) reload_button = gui.button( None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 3) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox(None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True,) self.sheet_combo.setSizePolicy( Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy( Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget( self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget( self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False) layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter) self.url_combo = url_combo = QComboBox() url_model = NamedURLModel(self.sheet_names) url_model.wrap(self.recent_urls) url_combo.setLineEdit(LineEditSelectOnFocus()) url_combo.setModel(url_model) url_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) url_combo.setEditable(True) url_combo.setInsertPolicy(url_combo.InsertAtTop) url_edit = url_combo.lineEdit() l, t, r, b = url_edit.getTextMargins() url_edit.setTextMargins(l + 5, t, r, b) layout.addWidget(url_combo, 3, 1, 3, 3) url_combo.activated.connect(self._url_set) box = gui.vBox(self.controlArea, "Info") self.infolabel = gui.widgetLabel(box, 'No data loaded.') self.warnings = gui.widgetLabel(box, '') box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) box = gui.hBox(self.controlArea) gui.button( box, self, "Browse documentation datasets", callback=lambda: self.browse_file(True), autoDefault=False) gui.rubber(box) gui.button( box, self, "Reset", callback=self.reset_domain_edit) self.apply_button = gui.button( box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) self.set_file_list() # Must not call open_file from within __init__. open_file # explicitly re-enters the event loop (by a progress bar) self.setAcceptDrops(True) if self.source == self.LOCAL_FILE: last_path = self.last_path() if last_path and os.path.exists(last_path) and \ os.path.getsize(last_path) > self.SIZE_LIMIT: self.Warning.file_too_big() return QTimer.singleShot(0, self.load_data) @staticmethod def sizeHint(): return QSize(600, 550) def select_file(self, n): assert n < len(self.recent_paths) super().select_file(n) if self.recent_paths: self.source = self.LOCAL_FILE self.load_data() self.set_file_list() def select_sheet(self): self.recent_paths[0].sheet = self.sheet_combo.currentText() self.load_data() def _url_set(self): url = self.url_combo.currentText() pos = self.recent_urls.index(url) url = url.strip() if not urlparse(url).scheme: url = 'http://' + url self.url_combo.setItemText(pos, url) self.recent_urls[pos] = url self.source = self.URL self.load_data() def browse_file(self, in_demos=False): if in_demos: start_file = get_sample_datasets_dir() if not os.path.exists(start_file): QMessageBox.information( None, "File", "Cannot find the directory with documentation datasets") return else: start_file = self.last_path() or os.path.expanduser("~/") readers = [f for f in FileFormat.formats if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None)] filename, reader, _ = open_filename_dialog(start_file, None, readers) if not filename: return self.add_path(filename) if reader is not None: self.recent_paths[0].file_format = reader.qualified_name() self.source = self.LOCAL_FILE self.load_data() # Open a file, create data from it and send it over the data channel def load_data(self): # We need to catch any exception type since anything can happen in # file readers self.closeContext() self.domain_editor.set_domain(None) self.apply_button.setEnabled(False) self.clear_messages() self.set_file_list() error = self._try_load() if error: error() self.data = None self.sheet_box.hide() self.Outputs.data.send(None) self.infolabel.setText("No data.") def _try_load(self): # pylint: disable=broad-except if self.last_path() and not os.path.exists(self.last_path()): return self.Error.file_not_found try: self.reader = self._get_reader() assert self.reader is not None except Exception: return self.Error.missing_reader if self.reader is self.NoFileSelected: self.Outputs.data.send(None) return None try: self._update_sheet_combo() except Exception: return self.Error.sheet_error with catch_warnings(record=True) as warnings: try: data = self.reader.read() except Exception as ex: log.exception(ex) return lambda x=ex: self.Error.unknown(str(x)) if warnings: self.Warning.load_warning(warnings[-1].message.args[0]) self.infolabel.setText(self._describe(data)) self.loaded_file = self.last_path() add_origin(data, self.loaded_file) self.data = data self.openContext(data.domain) self.apply_domain_edit() # sends data return None def _get_reader(self) -> FileFormat: if self.source == self.LOCAL_FILE: path = self.last_path() if path is None: return self.NoFileSelected if self.recent_paths and self.recent_paths[0].file_format: qname = self.recent_paths[0].file_format reader_class = class_from_qualified_name(qname) reader = reader_class(path) else: reader = FileFormat.get_reader(path) if self.recent_paths and self.recent_paths[0].sheet: reader.select_sheet(self.recent_paths[0].sheet) return reader else: url = self.url_combo.currentText().strip() if url: return UrlReader(url) else: return self.NoFileSelected def _update_sheet_combo(self): if len(self.reader.sheets) < 2: self.sheet_box.hide() self.reader.select_sheet(None) return self.sheet_combo.clear() self.sheet_combo.addItems(self.reader.sheets) self._select_active_sheet() self.sheet_box.show() def _select_active_sheet(self): if self.reader.sheet: try: idx = self.reader.sheets.index(self.reader.sheet) self.sheet_combo.setCurrentIndex(idx) except ValueError: # Requested sheet does not exist in this file self.reader.select_sheet(None) else: self.sheet_combo.setCurrentIndex(0) @staticmethod def _describe(table): def missing_prop(prop): if prop: return f"({prop * 100:.1f}% missing values)" else: return "(no missing values)" domain = table.domain text = "" attrs = getattr(table, "attributes", {}) descs = [attrs[desc] for desc in ("Name", "Description") if desc in attrs] if len(descs) == 2: descs[0] = f"<b>{descs[0]}</b>" if descs: text += f"<p>{'<br/>'.join(descs)}</p>" text += f"<p>{len(table)} instance(s)" missing_in_attr = missing_prop(table.has_missing_attribute() and table.get_nan_frequency_attribute()) missing_in_class = missing_prop(table.has_missing_class() and table.get_nan_frequency_class()) text += f"<br/>{len(domain.attributes)} feature(s) {missing_in_attr}" if domain.has_continuous_class: text += f"<br/>Regression; numerical class {missing_in_class}" elif domain.has_discrete_class: text += "<br/>Classification; categorical class " \ f"with {len(domain.class_var.values)} values {missing_in_class}" elif table.domain.class_vars: text += "<br/>Multi-target; " \ f"{len(table.domain.class_vars)} target variables " \ f"{missing_in_class}" else: text += "<br/>Data has no target variable." text += f"<br/>{len(domain.metas)} meta attribute(s)" text += "</p>" if 'Timestamp' in table.domain: # Google Forms uses this header to timestamp responses text += f"<p>First entry: {table[0, 'Timestamp']}<br/>" \ f"Last entry: {table[-1, 'Timestamp']}</p>" return text def storeSpecificSettings(self): self.current_context.modified_variables = self.variables[:] def retrieveSpecificSettings(self): if hasattr(self.current_context, "modified_variables"): self.variables[:] = self.current_context.modified_variables def reset_domain_edit(self): self.domain_editor.reset_domain() self.apply_domain_edit() def apply_domain_edit(self): if self.data is None: table = None else: domain, cols = self.domain_editor.get_domain(self.data.domain, self.data) if not (domain.variables or domain.metas): table = None else: X, y, m = cols table = Table.from_numpy(domain, X, y, m, self.data.W) table.name = self.data.name table.ids = np.array(self.data.ids) table.attributes = getattr(self.data, 'attributes', {}) self.Outputs.data.send(table) self.apply_button.setEnabled(False) def get_widget_name_extension(self): _, name = os.path.split(self.loaded_file) return os.path.splitext(name)[0] def send_report(self): def get_ext_name(filename): try: return FileFormat.names[os.path.splitext(filename)[1]] except KeyError: return "unknown" if self.data is None: self.report_paragraph("File", "No file.") return if self.source == self.LOCAL_FILE: home = os.path.expanduser("~") if self.loaded_file.startswith(home): # os.path.join does not like ~ name = "~" + os.path.sep + \ self.loaded_file[len(home):].lstrip("/").lstrip("\\") else: name = self.loaded_file if self.sheet_combo.isVisible(): name += f" ({self.sheet_combo.currentText()})" self.report_items("File", [("File name", name), ("Format", get_ext_name(name))]) else: self.report_items("Data", [("Resource", self.url), ("Format", get_ext_name(self.url))]) self.report_data("Data", self.data) @staticmethod def dragEnterEvent(event): """Accept drops of valid file urls""" urls = event.mimeData().urls() if urls: try: FileFormat.get_reader(OSX_NSURL_toLocalFile(urls[0]) or urls[0].toLocalFile()) event.acceptProposedAction() except IOError: pass def dropEvent(self, event): """Handle file drops""" urls = event.mimeData().urls() if urls: self.add_path(OSX_NSURL_toLocalFile(urls[0]) or urls[0].toLocalFile()) # add first file self.source = self.LOCAL_FILE self.load_data() def workflowEnvChanged(self, key, value, oldvalue): """ Function called when environment changes (e.g. while saving the scheme) It make sure that all environment connected values are modified (e.g. relative file paths are changed) """ self.update_file_list(key, value, oldvalue)
def __init__(self): widget.OWWidget.__init__(self) RecentPathsWidgetMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.sheets = [] self.lb = gui.listBox(self.controlArea, self, "file_idx") layout = QtGui.QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) file_button = gui.button(None, self, ' ...', callback=self.browse_files, autoDefault=False) file_button.setIcon(self.style().standardIcon( QtGui.QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 0) remove_button = gui.button(None, self, 'Remove', callback=self.remove_item) clear_button = gui.button(None, self, 'Clear', callback=self.clear) layout.addWidget(remove_button, 0, 1) layout.addWidget(clear_button, 0, 2) reload_button = gui.button(None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QtGui.QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 7) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox(None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True) self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QtGui.QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget(self.sheet_label, QtCore.Qt.AlignLeft) self.sheet_box.layout().addWidget(self.sheet_combo, QtCore.Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() layout.addWidget(self.sheet_box, 0, 5) label_box = gui.hBox(None, addToLayout=False, margin=0) label = gui.lineEdit(label_box, self, "label", callback=self.set_label, label="Label", orientation=Qt.Horizontal) layout.addWidget(label_box, 0, 6) layout.setColumnStretch(3, 2) box = gui.widgetBox(self.controlArea, "View loaded columns") if OLD_DOMAINEDITOR: self.domain_editor = DomainEditor(self.variables) else: self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) for i, rp in enumerate(self.recent_paths): self.lb.addItem(rp.abspath) # TODO unresolved paths just disappear! Modify _relocate_recent_files self._update_sheet_combo() self.load_data()
class OWFile(widget.OWWidget, RecentPathsWComboMixin): name = "File" id = "orange.widgets.data.file" description = "Read data from an input file or network " \ "and send a data table to the output." icon = "icons/File.svg" priority = 10 category = "Data" keywords = ["file", "load", "read", "open"] class Outputs: data = Output("Data", Table, doc="Attribute-valued dataset read from the input file.") want_main_area = False SEARCH_PATHS = [("sample-datasets", get_sample_datasets_dir())] SIZE_LIMIT = 1e7 LOCAL_FILE, URL = range(2) settingsHandler = PerfectDomainContextHandler( match_values=PerfectDomainContextHandler.MATCH_VALUES_ALL) # pylint seems to want declarations separated from definitions recent_paths: List[RecentPath] recent_urls: List[str] variables: list # Overload RecentPathsWidgetMixin.recent_paths to set defaults recent_paths = Setting([ RecentPath("", "sample-datasets", "iris.tab"), RecentPath("", "sample-datasets", "titanic.tab"), RecentPath("", "sample-datasets", "housing.tab"), RecentPath("", "sample-datasets", "heart_disease.tab"), RecentPath("", "sample-datasets", "brown-selected.tab"), RecentPath("", "sample-datasets", "zoo.tab"), ]) recent_urls = Setting([]) source = Setting(LOCAL_FILE) xls_sheet = ContextSetting("") sheet_names = Setting({}) url = Setting("") variables = ContextSetting([]) domain_editor = SettingProvider(DomainEditor) class Warning(widget.OWWidget.Warning): file_too_big = widget.Msg( "The file is too large to load automatically." " Press Reload to load.") load_warning = widget.Msg("Read warning:\n{}") class Error(widget.OWWidget.Error): file_not_found = widget.Msg("File not found.") missing_reader = widget.Msg("Missing reader.") sheet_error = widget.Msg("Error listing available sheets.") unknown = widget.Msg("Read error:\n{}") class NoFileSelected: pass UserAdviceMessages = [ widget.Message( "Use CSV File Import widget for advanced options " "for comma-separated files", "use-csv-file-import"), widget.Message( "This widget loads only tabular data. Use other widgets to load " "other data types like models, distance matrices and networks.", "other-data-types") ] def __init__(self): super().__init__() RecentPathsWComboMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.reader = None layout = QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) vbox = gui.radioButtons(None, self, "source", box=True, addSpace=True, callback=self.load_data, addToLayout=False) rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False) layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter) box = gui.hBox(None, addToLayout=False, margin=0) box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.file_combo.activated[int].connect(self.select_file) box.layout().addWidget(self.file_combo) layout.addWidget(box, 0, 1) file_button = gui.button(None, self, '...', callback=self.browse_file, autoDefault=False) file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 2) reload_button = gui.button(None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 3) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_combo = gui.comboBox( None, self, "xls_sheet", callback=self.select_sheet, sendSelectedValue=True, ) self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False) layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter) self.url_combo = url_combo = QComboBox() url_model = NamedURLModel(self.sheet_names) url_model.wrap(self.recent_urls) url_combo.setLineEdit(LineEditSelectOnFocus()) url_combo.setModel(url_model) url_combo.setSizePolicy(Policy.Ignored, Policy.Fixed) url_combo.setEditable(True) url_combo.setInsertPolicy(url_combo.InsertAtTop) url_edit = url_combo.lineEdit() l, t, r, b = url_edit.getTextMargins() url_edit.setTextMargins(l + 5, t, r, b) layout.addWidget(url_combo, 3, 1, 3, 3) url_combo.activated.connect(self._url_set) # whit completer we set that combo box is case sensitive when # matching the history completer = QCompleter() completer.setCaseSensitivity(Qt.CaseSensitive) url_combo.setCompleter(completer) box = gui.vBox(self.controlArea, "Info") self.infolabel = gui.widgetLabel(box, 'No data loaded.') self.warnings = gui.widgetLabel(box, '') box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) box = gui.hBox(self.controlArea) gui.button(box, self, "Browse documentation datasets", callback=lambda: self.browse_file(True), autoDefault=False) gui.rubber(box) gui.button(box, self, "Reset", callback=self.reset_domain_edit) self.apply_button = gui.button(box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) self.set_file_list() # Must not call open_file from within __init__. open_file # explicitly re-enters the event loop (by a progress bar) self.setAcceptDrops(True) if self.source == self.LOCAL_FILE: last_path = self.last_path() if last_path and os.path.exists(last_path) and \ os.path.getsize(last_path) > self.SIZE_LIMIT: self.Warning.file_too_big() return QTimer.singleShot(0, self.load_data) @staticmethod def sizeHint(): return QSize(600, 550) def select_file(self, n): assert n < len(self.recent_paths) super().select_file(n) if self.recent_paths: self.source = self.LOCAL_FILE self.load_data() self.set_file_list() def select_sheet(self): self.recent_paths[0].sheet = self.sheet_combo.currentText() self.load_data() def _url_set(self): url = self.url_combo.currentText() pos = self.recent_urls.index(url) url = url.strip() if not urlparse(url).scheme: url = 'http://' + url self.url_combo.setItemText(pos, url) self.recent_urls[pos] = url self.source = self.URL self.load_data() def browse_file(self, in_demos=False): if in_demos: start_file = get_sample_datasets_dir() if not os.path.exists(start_file): QMessageBox.information( None, "File", "Cannot find the directory with documentation datasets") return else: start_file = self.last_path() or os.path.expanduser("~/") readers = [ f for f in FileFormat.formats if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None) ] filename, reader, _ = open_filename_dialog(start_file, None, readers) if not filename: return self.add_path(filename) if reader is not None: self.recent_paths[0].file_format = reader.qualified_name() self.source = self.LOCAL_FILE self.load_data() # Open a file, create data from it and send it over the data channel def load_data(self): # We need to catch any exception type since anything can happen in # file readers self.closeContext() self.domain_editor.set_domain(None) self.apply_button.setEnabled(False) self.clear_messages() self.set_file_list() error = self._try_load() if error: error() self.data = None self.sheet_box.hide() self.Outputs.data.send(None) self.infolabel.setText("No data.") def _try_load(self): # pylint: disable=broad-except if self.last_path() and not os.path.exists(self.last_path()): return self.Error.file_not_found try: self.reader = self._get_reader() assert self.reader is not None except Exception: return self.Error.missing_reader if self.reader is self.NoFileSelected: self.Outputs.data.send(None) return None try: self._update_sheet_combo() except Exception: return self.Error.sheet_error with catch_warnings(record=True) as warnings: try: data = self.reader.read() except Exception as ex: log.exception(ex) return lambda x=ex: self.Error.unknown(str(x)) if warnings: self.Warning.load_warning(warnings[-1].message.args[0]) self.infolabel.setText(self._describe(data)) self.loaded_file = self.last_path() add_origin(data, self.loaded_file) self.data = data self.openContext(data.domain) self.apply_domain_edit() # sends data return None def _get_reader(self) -> FileFormat: if self.source == self.LOCAL_FILE: path = self.last_path() if path is None: return self.NoFileSelected if self.recent_paths and self.recent_paths[0].file_format: qname = self.recent_paths[0].file_format reader_class = class_from_qualified_name(qname) reader = reader_class(path) else: reader = FileFormat.get_reader(path) if self.recent_paths and self.recent_paths[0].sheet: reader.select_sheet(self.recent_paths[0].sheet) return reader else: url = self.url_combo.currentText().strip() if url: return UrlReader(url) else: return self.NoFileSelected def _update_sheet_combo(self): if len(self.reader.sheets) < 2: self.sheet_box.hide() self.reader.select_sheet(None) return self.sheet_combo.clear() self.sheet_combo.addItems(self.reader.sheets) self._select_active_sheet() self.sheet_box.show() def _select_active_sheet(self): if self.reader.sheet: try: idx = self.reader.sheets.index(self.reader.sheet) self.sheet_combo.setCurrentIndex(idx) except ValueError: # Requested sheet does not exist in this file self.reader.select_sheet(None) else: self.sheet_combo.setCurrentIndex(0) @staticmethod def _describe(table): def missing_prop(prop): if prop: return f"({prop * 100:.1f}% missing values)" else: return "(no missing values)" domain = table.domain text = "" attrs = getattr(table, "attributes", {}) descs = [ attrs[desc] for desc in ("Name", "Description") if desc in attrs ] if len(descs) == 2: descs[0] = f"<b>{descs[0]}</b>" if descs: text += f"<p>{'<br/>'.join(descs)}</p>" text += f"<p>{len(table)} instance(s)" missing_in_attr = missing_prop(table.has_missing_attribute() and table.get_nan_frequency_attribute()) missing_in_class = missing_prop(table.has_missing_class() and table.get_nan_frequency_class()) text += f"<br/>{len(domain.attributes)} feature(s) {missing_in_attr}" if domain.has_continuous_class: text += f"<br/>Regression; numerical class {missing_in_class}" elif domain.has_discrete_class: text += "<br/>Classification; categorical class " \ f"with {len(domain.class_var.values)} values {missing_in_class}" elif table.domain.class_vars: text += "<br/>Multi-target; " \ f"{len(table.domain.class_vars)} target variables " \ f"{missing_in_class}" else: text += "<br/>Data has no target variable." text += f"<br/>{len(domain.metas)} meta attribute(s)" text += "</p>" if 'Timestamp' in table.domain: # Google Forms uses this header to timestamp responses text += f"<p>First entry: {table[0, 'Timestamp']}<br/>" \ f"Last entry: {table[-1, 'Timestamp']}</p>" return text def storeSpecificSettings(self): self.current_context.modified_variables = self.variables[:] def retrieveSpecificSettings(self): if hasattr(self.current_context, "modified_variables"): self.variables[:] = self.current_context.modified_variables def reset_domain_edit(self): self.domain_editor.reset_domain() self.apply_domain_edit() def apply_domain_edit(self): if self.data is None: table = None else: domain, cols = self.domain_editor.get_domain( self.data.domain, self.data) if not (domain.variables or domain.metas): table = None else: X, y, m = cols table = Table.from_numpy(domain, X, y, m, self.data.W) table.name = self.data.name table.ids = np.array(self.data.ids) table.attributes = getattr(self.data, 'attributes', {}) self.Outputs.data.send(table) self.apply_button.setEnabled(False) def get_widget_name_extension(self): _, name = os.path.split(self.loaded_file) return os.path.splitext(name)[0] def send_report(self): def get_ext_name(filename): try: return FileFormat.names[os.path.splitext(filename)[1]] except KeyError: return "unknown" if self.data is None: self.report_paragraph("File", "No file.") return if self.source == self.LOCAL_FILE: home = os.path.expanduser("~") if self.loaded_file.startswith(home): # os.path.join does not like ~ name = "~" + os.path.sep + \ self.loaded_file[len(home):].lstrip("/").lstrip("\\") else: name = self.loaded_file if self.sheet_combo.isVisible(): name += f" ({self.sheet_combo.currentText()})" self.report_items("File", [("File name", name), ("Format", get_ext_name(name))]) else: self.report_items("Data", [("Resource", self.url), ("Format", get_ext_name(self.url))]) self.report_data("Data", self.data) @staticmethod def dragEnterEvent(event): """Accept drops of valid file urls""" urls = event.mimeData().urls() if urls: try: FileFormat.get_reader(urls[0].toLocalFile()) event.acceptProposedAction() except IOError: pass def dropEvent(self, event): """Handle file drops""" urls = event.mimeData().urls() if urls: self.add_path(urls[0].toLocalFile()) # add first file self.source = self.LOCAL_FILE self.load_data() def workflowEnvChanged(self, key, value, oldvalue): """ Function called when environment changes (e.g. while saving the scheme) It make sure that all environment connected values are modified (e.g. relative file paths are changed) """ self.update_file_list(key, value, oldvalue)
class OWMultifile(widget.OWWidget, RelocatablePathsWidgetMixin): name = "Multifile" id = "orangecontrib.spectroscopy.widgets.files" icon = "icons/multifile.svg" description = "Read data from input files " \ "and send a data table to the output." priority = 10000 replaces = [ "orangecontrib.infrared.widgets.owfiles.OWFiles", "orangecontrib.infrared.widgets.owmultifile.OWMultifile" ] class Outputs: data = Output("Data", Table, doc="Concatenated input files.") want_main_area = False file_idx = [] settingsHandler = PerfectDomainContextHandler( match_values=PerfectDomainContextHandler.MATCH_VALUES_ALL) recent_paths: List[RecentPath] variables: list sheet = Setting(None, schema_only=True) label = Setting("", schema_only=True) recent_paths = Setting([], schema_only=True) variables = ContextSetting([], schema_only=True) class Error(widget.OWWidget.Error): file_not_found = widget.Msg("File(s) not found.") missing_reader = widget.Msg("Missing reader(s).") read_error = widget.Msg("Read error(s).") domain_editor = SettingProvider(DomainEditor) def __init__(self): widget.OWWidget.__init__(self) RelocatablePathsWidgetMixin.__init__(self) self.domain = None self.data = None self.loaded_file = "" self.sheets = [] self.lb = gui.listBox(self.controlArea, self, "file_idx", selectionMode=QListWidget.MultiSelection) self.default_foreground = None layout = QGridLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) file_button = gui.button(None, self, ' ...', callback=self.browse_files, autoDefault=False) file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon)) file_button.setSizePolicy(Policy.Maximum, Policy.Fixed) layout.addWidget(file_button, 0, 0) remove_button = gui.button(None, self, 'Remove', callback=self.remove_item) clear_button = gui.button(None, self, 'Clear', callback=self.clear) layout.addWidget(remove_button, 0, 1) layout.addWidget(clear_button, 0, 2) reload_button = gui.button(None, self, "Reload", callback=self.load_data, autoDefault=False) reload_button.setIcon(self.style().standardIcon( QStyle.SP_BrowserReload)) reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed) layout.addWidget(reload_button, 0, 7) self.sheet_box = gui.hBox(None, addToLayout=False, margin=0) self.sheet_index = 0 self.sheet_combo = gui.comboBox(None, self, "sheet_index", callback=self.select_sheet) self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_label = QLabel() self.sheet_label.setText('Sheet') self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed) self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft) self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter) layout.addWidget(self.sheet_box, 2, 1) self.sheet_box.hide() layout.addWidget(self.sheet_box, 0, 5) label_box = gui.hBox(None, addToLayout=False, margin=0) gui.lineEdit(label_box, self, "label", callback=self.set_label, label="Label", orientation=Qt.Horizontal) layout.addWidget(label_box, 0, 6) layout.setColumnStretch(3, 2) box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)") self.domain_editor = DomainEditor(self) self.editor_model = self.domain_editor.model() box.layout().addWidget(self.domain_editor) for rp in self.recent_paths: self.lb.addItem(rp.abspath) box = gui.hBox(self.controlArea) gui.rubber(box) if hasattr(DomainEditor, "reset_domain"): # Orange>=3.21 gui.button(box, self, "Reset", callback=self.reset_domain_edit) self.apply_button = gui.button(box, self, "Apply", callback=self.apply_domain_edit) self.apply_button.setEnabled(False) self.apply_button.setFixedWidth(170) self.editor_model.dataChanged.connect( lambda: self.apply_button.setEnabled(True)) self._update_sheet_combo() self.load_data() def set_label(self): self.load_data() def _select_active_sheet(self): if self.sheet: try: sheet_list = [s[0] for s in self.sheets] idx = sheet_list.index(self.sheet) self.sheet_combo.setCurrentIndex(idx) except ValueError: # Requested sheet does not exist in this file self.sheet = None else: self.sheet_combo.setCurrentIndex(0) def _update_sheet_combo(self): sheets = Counter() for rp in self.recent_paths: try: reader = _get_reader(rp) sheets.update(reader.sheets) except: pass sheets = sorted(sheets.items(), key=lambda x: x[0]) self.sheets = [(s, s + " (" + str(n) + ")") for s, n in sheets] if len(sheets) < 2: self.sheet_box.hide() self.sheet = None else: self.sheets.insert(0, (None, "(None)")) self.sheet_combo.clear() self.sheet_combo.addItems([s[1] for s in self.sheets]) self._select_active_sheet() self.sheet_box.show() def select_sheet(self): self.sheet = self.sheets[self.sheet_combo.currentIndex()][0] self.load_data() def remove_item(self): ri = [i.row() for i in self.lb.selectedIndexes()] for i in sorted(ri, reverse=True): self.recent_paths.pop(i) self.lb.takeItem(i) self._update_sheet_combo() self.load_data() def clear(self): self.lb.clear() while self.recent_paths: self.recent_paths.pop() self._update_sheet_combo() self.load_data() def browse_files(self, in_demos=False): start_file = self.last_path() or os.path.expanduser("~/") readers = [ f for f in FileFormat.formats if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None) ] filenames, reader, _ = open_filename_dialog( start_file, None, readers, dialog=QFileDialog.getOpenFileNames) self.load_files(filenames, reader) def load_files(self, filenames, reader): if not filenames: return for f in filenames: self.add_path(f, reader) self.lb.addItem(f) self._update_sheet_combo() self.load_data() def load_data(self): self.closeContext() self.Error.file_not_found.clear() self.Error.missing_reader.clear() self.Error.read_error.clear() data_list = [] fnok_list = [] def show_error(li, msg): li.setForeground(Qt.red) li.setToolTip(msg) empty_domain = Domain(attributes=[]) for i, rp in enumerate(self.recent_paths): fn = rp.abspath li = self.lb.item(i) li.setToolTip("") if self.default_foreground is None: self.default_foreground = li.foreground() li.setForeground(self.default_foreground) if not os.path.exists(fn): show_error(li, "File not found.") self.Error.file_not_found() continue try: reader = _get_reader(rp) assert reader is not None except Exception: # pylint: disable=broad-except show_error(li, "Reader not found.") self.Error.missing_reader() continue try: if self.sheet in reader.sheets: reader.select_sheet(self.sheet) if isinstance(reader, SpectralFileFormat): xs, vals, additional = reader.read_spectra() if additional is None: additional = Table.from_domain(empty_domain, n_rows=len(vals)) data_list.append((xs, vals, additional)) else: data_list.append(reader.read()) fnok_list.append(fn) except Exception as ex: # pylint: disable=broad-except show_error(li, "Read error:\n" + str(ex)) self.Error.read_error() if not data_list \ or self.Error.file_not_found.is_shown() \ or self.Error.missing_reader.is_shown() \ or self.Error.read_error.is_shown(): self.data = None self.domain_editor.set_domain(None) else: data = concatenate_data(data_list, fnok_list, self.label) self.data = data self.openContext(data.domain) self.apply_domain_edit() # sends data def storeSpecificSettings(self): self.current_context.modified_variables = self.variables[:] def retrieveSpecificSettings(self): if hasattr(self.current_context, "modified_variables"): self.variables[:] = self.current_context.modified_variables def apply_domain_edit(self): if self.data is None: table = None else: domain, cols = self.domain_editor.get_domain( self.data.domain, self.data) if not (domain.variables or domain.metas): table = None else: X, y, m = cols table = Table.from_numpy(domain, X, y, m, self.data.W) table.name = self.data.name table.ids = np.array(self.data.ids) table.attributes = getattr(self.data, 'attributes', {}) self.Outputs.data.send(table) self.apply_button.setEnabled(False) def reset_domain_edit(self): self.domain_editor.reset_domain() self.apply_domain_edit() def send_report(self): def get_format_name(format): try: return format.DESCRIPTION except AttributeError: return format.__class__.__name__ if self.data is None: self.report_paragraph("File", "No file.") return files = [] for rp in self.recent_paths: format = _get_reader(rp) files.append([rp.abspath, get_format_name(format)]) self.report_table("Files", table=files) self.report_data("Data", self.data) def workflowEnvChanged(self, key, value, oldvalue): """ Function called when environment changes (e.g. while saving the scheme) It make sure that all environment connected values are modified (e.g. relative file paths are changed) """ self.update_file_list(key, value, oldvalue) def update_file_list(self, key, value, oldvalue): if key == "basedir": self._relocate_recent_files()