class OWFile(widget.OWWidget, RecentPathsWComboMixin):
name = "File"
id = "orange.widgets.data.file"
description = "Read data from an input file or network " \
"and send a data table to the output."
icon = "icons/File.svg"
priority = 10
category = "Data"
keywords = ["data", "file", "load", "read"]
outputs = [
widget.OutputSignal(
"Data",
Table,
doc="Attribute-valued data set read from the input file.")
]
want_main_area = False
SEARCH_PATHS = [("sample-datasets", get_sample_datasets_dir())]
SIZE_LIMIT = 1e7
LOCAL_FILE, URL = range(2)
settingsHandler = PerfectDomainContextHandler()
# Overload RecentPathsWidgetMixin.recent_paths to set defaults
recent_paths = Setting([
RecentPath("", "sample-datasets", "iris.tab"),
RecentPath("", "sample-datasets", "titanic.tab"),
RecentPath("", "sample-datasets", "housing.tab"),
RecentPath("", "sample-datasets", "heart_disease.tab"),
])
recent_urls = Setting([])
source = Setting(LOCAL_FILE)
xls_sheet = ContextSetting("")
sheet_names = Setting({})
url = Setting("")
variables = ContextSetting([])
dlg_formats = ("All readable files ({});;".format(
'*' + ' *'.join(FileFormat.readers.keys())) + ";;".join(
"{} (*{})".format(f.DESCRIPTION, ' *'.join(f.EXTENSIONS))
for f in sorted(set(FileFormat.readers.values()),
key=list(FileFormat.readers.values()).index)))
domain_editor = SettingProvider(DomainEditor)
class Warning(widget.OWWidget.Warning):
file_too_big = widget.Msg(
"The file is too large to load automatically."
" Press Reload to load.")
class Error(widget.OWWidget.Error):
file_not_found = widget.Msg("File not found.")
def __init__(self):
super().__init__()
RecentPathsWComboMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.reader = None
layout = QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
vbox = gui.radioButtons(None,
self,
"source",
box=True,
addSpace=True,
callback=self.load_data,
addToLayout=False)
rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.activated[int].connect(self.select_file)
box.layout().addWidget(self.file_combo)
layout.addWidget(box, 0, 1)
file_button = gui.button(None,
self,
'...',
callback=self.browse_file,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 3)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(
None,
self,
"xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True,
)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = NamedURLModel(self.sheet_names)
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 3, 3)
url_combo.activated.connect(self._url_set)
box = gui.vBox(self.controlArea, "Info")
self.info = gui.widgetLabel(box, 'No data loaded.')
self.warnings = gui.widgetLabel(box, '')
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
box = gui.hBox(self.controlArea)
gui.button(box,
self,
"Browse documentation data sets",
callback=lambda: self.browse_file(True),
autoDefault=False)
gui.rubber(box)
box.layout().addWidget(self.report_button)
self.report_button.setFixedWidth(170)
self.apply_button = gui.button(box,
self,
"Apply",
callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
self.set_file_list()
# Must not call open_file from within __init__. open_file
# explicitly re-enters the event loop (by a progress bar)
self.setAcceptDrops(True)
if self.source == self.LOCAL_FILE:
last_path = self.last_path()
if last_path and os.path.exists(last_path) and \
os.path.getsize(last_path) > self.SIZE_LIMIT:
self.Warning.file_too_big()
return
QTimer.singleShot(0, self.load_data)
def sizeHint(self):
return QSize(600, 550)
def select_file(self, n):
assert n < len(self.recent_paths)
super().select_file(n)
if self.recent_paths:
self.source = self.LOCAL_FILE
self.load_data()
self.set_file_list()
def select_sheet(self):
self.recent_paths[0].sheet = self.sheet_combo.currentText()
self.load_data()
def _url_set(self):
self.source = self.URL
self.load_data()
def browse_file(self, in_demos=False):
if in_demos:
start_file = get_sample_datasets_dir()
if not os.path.exists(start_file):
QMessageBox.information(
None, "File",
"Cannot find the directory with documentation data sets")
return
else:
start_file = self.last_path() or os.path.expanduser("~/")
filename, _ = QFileDialog.getOpenFileName(self,
'Open Orange Data File',
start_file, self.dlg_formats)
if not filename:
return
self.add_path(filename)
self.source = self.LOCAL_FILE
self.load_data()
# Open a file, create data from it and send it over the data channel
def load_data(self):
# We need to catch any exception type since anything can happen in
# file readers
# pylint: disable=broad-except
self.closeContext()
self.domain_editor.set_domain(None)
self.apply_button.setEnabled(False)
self.clear_messages()
self.set_file_list()
if self.last_path() and not os.path.exists(self.last_path()):
self.Error.file_not_found()
self.send("Data", None)
self.info.setText("No data.")
return
error = None
try:
self.reader = self._get_reader()
if self.reader is None:
self.data = None
self.send("Data", None)
self.info.setText("No data.")
self.sheet_box.hide()
return
except Exception as ex:
error = ex
if not error:
self._update_sheet_combo()
with catch_warnings(record=True) as warnings:
try:
data = self.reader.read()
except Exception as ex:
log.exception(ex)
error = ex
self.warning(warnings[-1].message.args[0] if warnings else '')
if error:
self.data = None
self.send("Data", None)
self.info.setText("An error occurred:\n{}".format(error))
self.sheet_box.hide()
return
self.info.setText(self._describe(data))
self.loaded_file = self.last_path()
add_origin(data, self.loaded_file)
self.data = data
self.openContext(data.domain)
self.apply_domain_edit() # sends data
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
reader = FileFormat.get_reader(self.last_path())
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
elif self.source == self.URL:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
def _update_sheet_combo(self):
if len(self.reader.sheets) < 2:
self.sheet_box.hide()
self.reader.select_sheet(None)
return
self.sheet_combo.clear()
self.sheet_combo.addItems(self.reader.sheets)
self._select_active_sheet()
self.sheet_box.show()
def _select_active_sheet(self):
if self.reader.sheet:
try:
idx = self.reader.sheets.index(self.reader.sheet)
self.sheet_combo.setCurrentIndex(idx)
except ValueError:
# Requested sheet does not exist in this file
self.reader.select_sheet(None)
else:
self.sheet_combo.setCurrentIndex(0)
def _describe(self, table):
domain = table.domain
text = ""
attrs = getattr(table, "attributes", {})
descs = [
attrs[desc] for desc in ("Name", "Description") if desc in attrs
]
if len(descs) == 2:
descs[0] = "<b>{}</b>".format(descs[0])
if descs:
text += "<p>{}</p>".format("<br/>".join(descs))
text += "<p>{} instance(s), {} feature(s), {} meta attribute(s)".\
format(len(table), len(domain.attributes), len(domain.metas))
if domain.has_continuous_class:
text += "<br/>Regression; numerical class."
elif domain.has_discrete_class:
text += "<br/>Classification; discrete class with {} values.".\
format(len(domain.class_var.values))
elif table.domain.class_vars:
text += "<br/>Multi-target; {} target variables.".format(
len(table.domain.class_vars))
else:
text += "<br/>Data has no target variable."
text += "</p>"
if 'Timestamp' in table.domain:
# Google Forms uses this header to timestamp responses
text += '<p>First entry: {}<br/>Last entry: {}</p>'.format(
table[0, 'Timestamp'], table[-1, 'Timestamp'])
return text
def storeSpecificSettings(self):
self.current_context.modified_variables = self.variables[:]
def retrieveSpecificSettings(self):
if hasattr(self.current_context, "modified_variables"):
self.variables[:] = self.current_context.modified_variables
def apply_domain_edit(self):
if self.data is not None:
domain, cols = self.domain_editor.get_domain(
self.data.domain, self.data)
X, y, m = cols
X = np.array(X).T if len(X) else np.empty((len(self.data), 0))
y = np.array(y).T if len(y) else None
dtpe = object if any(
isinstance(m, StringVariable) for m in domain.metas) else float
m = np.array(m, dtype=dtpe).T if len(m) else None
table = Table.from_numpy(domain, X, y, m, self.data.W)
table.name = self.data.name
table.ids = np.array(self.data.ids)
table.attributes = getattr(self.data, 'attributes', {})
else:
table = self.data
self.send("Data", table)
self.apply_button.setEnabled(False)
def get_widget_name_extension(self):
_, name = os.path.split(self.loaded_file)
return os.path.splitext(name)[0]
def send_report(self):
def get_ext_name(filename):
try:
return FileFormat.names[os.path.splitext(filename)[1]]
except KeyError:
return "unknown"
if self.data is None:
self.report_paragraph("File", "No file.")
return
if self.source == self.LOCAL_FILE:
home = os.path.expanduser("~")
if self.loaded_file.startswith(home):
# os.path.join does not like ~
name = "~" + os.path.sep + \
self.loaded_file[len(home):].lstrip("/").lstrip("\\")
else:
name = self.loaded_file
if self.sheet_combo.isVisible():
name += " ({})".format(self.sheet_combo.currentText())
self.report_items("File", [("File name", name),
("Format", get_ext_name(name))])
else:
self.report_items("Data", [("Resource", self.url),
("Format", get_ext_name(self.url))])
self.report_data("Data", self.data)
def dragEnterEvent(self, event):
"""Accept drops of valid file urls"""
urls = event.mimeData().urls()
if urls:
try:
FileFormat.get_reader(
OSX_NSURL_toLocalFile(urls[0]) or urls[0].toLocalFile())
event.acceptProposedAction()
except IOError:
pass
def dropEvent(self, event):
"""Handle file drops"""
urls = event.mimeData().urls()
if urls:
self.add_path(
OSX_NSURL_toLocalFile(urls[0])
or urls[0].toLocalFile()) # add first file
self.source = self.LOCAL_FILE
self.load_data()
def __init__(self):
self.domain_editor = DomainEditor(self)
def __init__(self):
super().__init__()
RecentPathsWComboMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.reader = None
layout = QtGui.QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
vbox = gui.radioButtons(None, self, "source", box=True, addSpace=True,
callback=self.load_data, addToLayout=False)
rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False)
layout.addWidget(rb_button, 0, 0, QtCore.Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.activated[int].connect(self.select_file)
box.layout().addWidget(self.file_combo)
layout.addWidget(box, 0, 1)
file_button = gui.button(
None, self, '...', callback=self.browse_file, autoDefault=False)
file_button.setIcon(self.style().standardIcon(
QtGui.QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 2)
reload_button = gui.button(
None, self, "Reload", callback=self.load_data, autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QtGui.QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 3)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(None, self, "xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True)
self.sheet_combo.setSizePolicy(
Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QtGui.QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(
Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(
self.sheet_label, QtCore.Qt.AlignLeft)
self.sheet_box.layout().addWidget(
self.sheet_combo, QtCore.Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, QtCore.Qt.AlignVCenter)
self.url_combo = url_combo = QtGui.QComboBox()
url_model = NamedURLModel(self.sheet_names)
url_model.wrap(self.recent_urls)
url_combo.setModel(url_model)
url_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 3, 3)
url_combo.activated.connect(self._url_set)
box = gui.vBox(self.controlArea, "Info")
self.info = gui.widgetLabel(box, 'No data loaded.')
self.warnings = gui.widgetLabel(box, '')
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
domain_editor = DomainEditor(self.variables)
self.editor_model = domain_editor.model()
box.layout().addWidget(domain_editor)
box = gui.hBox(self.controlArea)
gui.button(
box, self, "Browse documentation data sets",
callback=lambda: self.browse_file(True), autoDefault=False)
gui.rubber(box)
box.layout().addWidget(self.report_button)
self.report_button.setFixedWidth(170)
self.apply_button = gui.button(
box, self, "Apply", callback=self.apply_domain_edit)
self.apply_button.hide()
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(self.apply_button.show)
self.set_file_list()
# Must not call open_file from within __init__. open_file
# explicitly re-enters the event loop (by a progress bar)
QtCore.QTimer.singleShot(0, self.load_data)
self.setAcceptDrops(True)
def __init__(self):
super().__init__()
self.table = None
self._html = None
def _loadFinished(is_ok):
if is_ok:
QTimer.singleShot(1, lambda: setattr(self, '_html', self.webview.html()))
self.webview = WebviewWidget(loadFinished=_loadFinished)
vb = gui.vBox(self.controlArea, 'Import Data')
hb = gui.hBox(vb)
self.combo = combo = URLComboBox(
hb, self.recent, editable=True, minimumWidth=400,
insertPolicy=QComboBox.InsertAtTop,
toolTip='Format: ' + VALID_URL_HELP,
editTextChanged=self.is_valid_url,
# Indirect via QTimer because calling wait() -> processEvents,
# while our currentIndexChanged event hadn't yet finished.
# Avoids calling handler twice.
currentIndexChanged=lambda: QTimer.singleShot(1, self.load_url))
hb.layout().addWidget(QLabel('Public link URL:', hb))
hb.layout().addWidget(combo)
hb.layout().setStretch(1, 2)
RELOAD_TIMES = (
('No reload',),
('5 s', 5000),
('10 s', 10000),
('30 s', 30000),
('1 min', 60*1000),
('2 min', 2*60*1000),
('5 min', 5*60*1000),
)
reload_timer = QTimer(self, timeout=lambda: self.load_url(from_reload=True))
def _on_reload_changed():
if self.reload_idx == 0:
reload_timer.stop()
return
reload_timer.start(RELOAD_TIMES[self.reload_idx][1])
gui.comboBox(vb, self, 'reload_idx', label='Reload every:',
orientation=Qt.Horizontal,
items=[i[0] for i in RELOAD_TIMES],
callback=_on_reload_changed)
box = gui.widgetBox(self.controlArea, "Columns (Double-click to edit)")
self.domain_editor = DomainEditor(self)
editor_model = self.domain_editor.model()
def editorDataChanged():
self.apply_domain_edit()
self.commit()
editor_model.dataChanged.connect(editorDataChanged)
box.layout().addWidget(self.domain_editor)
box = gui.widgetBox(self.controlArea, "Info", addSpace=True)
info = self.data_info = gui.widgetLabel(box, '')
info.setWordWrap(True)
self.controlArea.layout().addStretch(1)
gui.auto_commit(self.controlArea, self, 'autocommit', label='Commit')
self.set_info()
class OW1ka(widget.OWWidget):
name = "EnKlik Anketa"
description = "Import data from EnKlikAnketa (1ka.si) public URL."
icon = "icons/1ka.svg"
priority = 30
outputs = [("Data", Table)]
want_main_area = False
resizing_enabled = False
settingsHandler = settings.PerfectDomainContextHandler(
match_values=settings.PerfectDomainContextHandler.MATCH_VALUES_ALL
)
recent = settings.Setting([])
reload_idx = settings.Setting(0)
autocommit = settings.Setting(True)
domain_editor = settings.SettingProvider(DomainEditor)
UserAdviceMessages = [
widget.Message(
'You can import data from public links to 1ka surveys results. '
'Click to learn more on how to get a shareable public link URL for '
'1ka surveys that you manage.',
'public-link',
icon=widget.Message.Information,
moreurl='http://english.1ka.si/db/24/468/Guides/Public_link_to_access_data_and_analysis/'
),
]
class Error(widget.OWWidget.Error):
net_error = widget.Msg("Couldn't load data: {}. Ensure network connection, firewall ...")
parse_error = widget.Msg("Couldn't parse data: {}. Ensure well-formatted data or submit a bug report.")
invalid_url = widget.Msg('Invalid URL. Public shareable link should match: ' + VALID_URL_HELP)
data_is_anal = widget.Msg("The provided URL is a public link to 'Analysis'. Need public link to 'Data'.")
class Information(widget.OWWidget.Information):
response_data_empty = widget.Msg('Response data is empty. Get some responses first.')
def __init__(self):
super().__init__()
self.table = None
self._html = None
def _loadFinished(is_ok):
if is_ok:
QTimer.singleShot(1, lambda: setattr(self, '_html', self.webview.html()))
self.webview = WebviewWidget(loadFinished=_loadFinished)
vb = gui.vBox(self.controlArea, 'Import Data')
hb = gui.hBox(vb)
self.combo = combo = URLComboBox(
hb, self.recent, editable=True, minimumWidth=400,
insertPolicy=QComboBox.InsertAtTop,
toolTip='Format: ' + VALID_URL_HELP,
editTextChanged=self.is_valid_url,
# Indirect via QTimer because calling wait() -> processEvents,
# while our currentIndexChanged event hadn't yet finished.
# Avoids calling handler twice.
currentIndexChanged=lambda: QTimer.singleShot(1, self.load_url))
hb.layout().addWidget(QLabel('Public link URL:', hb))
hb.layout().addWidget(combo)
hb.layout().setStretch(1, 2)
RELOAD_TIMES = (
('No reload',),
('5 s', 5000),
('10 s', 10000),
('30 s', 30000),
('1 min', 60*1000),
('2 min', 2*60*1000),
('5 min', 5*60*1000),
)
reload_timer = QTimer(self, timeout=lambda: self.load_url(from_reload=True))
def _on_reload_changed():
if self.reload_idx == 0:
reload_timer.stop()
return
reload_timer.start(RELOAD_TIMES[self.reload_idx][1])
gui.comboBox(vb, self, 'reload_idx', label='Reload every:',
orientation=Qt.Horizontal,
items=[i[0] for i in RELOAD_TIMES],
callback=_on_reload_changed)
box = gui.widgetBox(self.controlArea, "Columns (Double-click to edit)")
self.domain_editor = DomainEditor(self)
editor_model = self.domain_editor.model()
def editorDataChanged():
self.apply_domain_edit()
self.commit()
editor_model.dataChanged.connect(editorDataChanged)
box.layout().addWidget(self.domain_editor)
box = gui.widgetBox(self.controlArea, "Info", addSpace=True)
info = self.data_info = gui.widgetLabel(box, '')
info.setWordWrap(True)
self.controlArea.layout().addStretch(1)
gui.auto_commit(self.controlArea, self, 'autocommit', label='Commit')
self.set_info()
def set_combo_items(self):
self.combo.clear()
for sheet in self.recent:
self.combo.addItem(sheet.name, sheet.url)
def commit(self):
self.send('Data', self.table)
def is_valid_url(self, url):
if is_valid_url(url):
self.Error.invalid_url.clear()
return True
self.Error.invalid_url()
QToolTip.showText(self.combo.mapToGlobal(QPoint(0, 0)), self.combo.toolTip())
def load_url(self, from_reload=False):
self.closeContext()
self.domain_editor.set_domain(None)
url = self.combo.currentText()
if not self.is_valid_url(url):
self.table = None
self.commit()
return
if url not in self.recent:
self.recent.insert(0, url)
prev_table = self.table
with self.progressBar(3) as progress:
try:
self._html = None
self.webview.setUrl(url)
wait(until=lambda: self._html is not None)
progress.advance()
# Wait some seconds for discrete labels to have loaded via AJAX,
# then re-query HTML.
# *Webview.loadFinished doesn't guarantee it sufficiently
try:
wait(until=lambda: False, timeout=1200)
except TimeoutError:
pass
progress.advance()
html = self.webview.html()
except Exception as e:
log.exception("Couldn't load data from: %s", url)
self.Error.net_error(try_(lambda: e.args[0], ''))
self.table = None
else:
self.Error.clear()
self.Information.clear()
self.table = None
try:
table = self.table = self.table_from_html(html)
except DataEmptyError:
self.Information.response_data_empty()
except DataIsAnalError:
self.Error.data_is_anal()
except Exception as e:
log.exception('Parsing error: %s', url)
self.Error.parse_error(try_(lambda: e.args[0], ''))
else:
self.openContext(table.domain)
self.combo.setTitleFor(self.combo.currentIndex(), table.name)
def _equal(data1, data2):
NAN = float('nan')
return (try_(lambda: data1.checksum(), NAN) ==
try_(lambda: data2.checksum(), NAN))
self._orig_table = self.table
self.apply_domain_edit()
if not (from_reload and _equal(prev_table, self.table)):
self.commit()
def apply_domain_edit(self):
data = self._orig_table
if data is None:
self.set_info()
return
domain, cols = self.domain_editor.get_domain(data.domain, data)
# Copied verbatim from OWFile
if not (domain.variables or domain.metas):
table = None
else:
X, y, m = cols
table = Table.from_numpy(domain, X, y, m, data.W)
table.name = data.name
table.ids = np.array(data.ids)
table.attributes = getattr(data, 'attributes', {})
self.table = table
self.set_info()
DATETIME_VAR = 'Paradata (insert)'
def table_from_html(self, html):
soup = BeautifulSoup(html, 'html.parser')
try:
html_table = soup.find_all('table')[-1]
except IndexError:
raise DataEmptyError
if '<h2>Anal' in html or 'div_analiza_' in html:
raise DataIsAnalError
def _header_row_strings(row):
return chain.from_iterable(
repeat(th.get_text(), int(th.get('colspan') or 1))
for th in html_table.select('thead tr:nth-of-type(%d) th[title]' % row))
# self.DATETIME_VAR (available when Paradata is enabled in 1ka UI)
# should match this variable name format
header = [th1.rstrip(':') + ('' if th3 == th1 else ' ({})').format(th3.rstrip(':'))
for th1, th3 in zip(_header_row_strings(1),
_header_row_strings(3))]
values = [[(# If no span, feature is a number or a text field
td.get_text() if td.span is None else
# If have span, it's a number, but if negative, replace with NaN
'' if td.contents[0].strip().startswith('-') else
# Else if span, the number is its code, but we want its value
td.span.get_text()[1:-1])
for td in tr.select('td')
if 'data_uid' not in td.get('class', ())]
for tr in html_table.select('tbody tr')]
# Save parsed values into in-mem file for default values processing
buffer = StringIO()
writer = csv.writer(buffer, delimiter='\t')
writer.writerow(header)
writer.writerows(values)
buffer.flush()
buffer.seek(0)
data = TabReader(buffer).read()
title = soup.select('body h2:nth-of-type(1)')[0].get_text().split(': ', maxsplit=1)[-1]
data.name = title
return data
def set_info(self):
data = self.table
if data is None:
self.data_info.setText('No spreadsheet loaded.')
return
text = "{}\n\n{} instance(s), {} feature(s), {} meta attribute(s)\n".format(
data.name, len(data), len(data.domain.attributes), len(data.domain.metas))
text += try_(lambda: '\nFirst entry: {}'
'\nLast entry: {}'.format(data[0, self.DATETIME_VAR],
data[-1, self.DATETIME_VAR]), '')
self.data_info.setText(text)
def __init__(self):
super().__init__()
RecentPathsWComboMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.reader = None
layout = QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
vbox = gui.radioButtons(None,
self,
"source",
box=True,
addSpace=True,
callback=self.load_data,
addToLayout=False)
rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.activated[int].connect(self.select_file)
box.layout().addWidget(self.file_combo)
layout.addWidget(box, 0, 1)
file_button = gui.button(None,
self,
'...',
callback=self.browse_file,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 3)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(
None,
self,
"xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True,
)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = NamedURLModel(self.sheet_names)
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(Policy.Ignored, Policy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 3, 3)
url_combo.activated.connect(self._url_set)
# whit completer we set that combo box is case sensitive when
# matching the history
completer = QCompleter()
completer.setCaseSensitivity(Qt.CaseSensitive)
url_combo.setCompleter(completer)
box = gui.vBox(self.controlArea, "Info")
self.infolabel = gui.widgetLabel(box, 'No data loaded.')
self.warnings = gui.widgetLabel(box, '')
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
box = gui.hBox(self.controlArea)
gui.button(box,
self,
"Browse documentation datasets",
callback=lambda: self.browse_file(True),
autoDefault=False)
gui.rubber(box)
gui.button(box, self, "Reset", callback=self.reset_domain_edit)
self.apply_button = gui.button(box,
self,
"Apply",
callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
self.set_file_list()
# Must not call open_file from within __init__. open_file
# explicitly re-enters the event loop (by a progress bar)
self.setAcceptDrops(True)
if self.source == self.LOCAL_FILE:
last_path = self.last_path()
if last_path and os.path.exists(last_path) and \
os.path.getsize(last_path) > self.SIZE_LIMIT:
self.Warning.file_too_big()
return
QTimer.singleShot(0, self.load_data)
def __init__(self):
widget.OWWidget.__init__(self)
RelocatablePathsWidgetMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.sheets = []
self.lb = gui.listBox(self.controlArea,
self,
"file_idx",
selectionMode=QListWidget.MultiSelection)
self.default_foreground = None
layout = QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
file_button = gui.button(None,
self,
' ...',
callback=self.browse_files,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 0)
remove_button = gui.button(None,
self,
'Remove',
callback=self.remove_item)
clear_button = gui.button(None, self, 'Clear', callback=self.clear)
layout.addWidget(remove_button, 0, 1)
layout.addWidget(clear_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 7)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_index = 0
self.sheet_combo = gui.comboBox(None,
self,
"sheet_index",
callback=self.select_sheet)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
layout.addWidget(self.sheet_box, 0, 5)
label_box = gui.hBox(None, addToLayout=False, margin=0)
gui.lineEdit(label_box,
self,
"label",
callback=self.set_label,
label="Label",
orientation=Qt.Horizontal)
layout.addWidget(label_box, 0, 6)
layout.setColumnStretch(3, 2)
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
for rp in self.recent_paths:
self.lb.addItem(rp.abspath)
box = gui.hBox(self.controlArea)
gui.rubber(box)
if hasattr(DomainEditor, "reset_domain"): # Orange>=3.21
gui.button(box, self, "Reset", callback=self.reset_domain_edit)
self.apply_button = gui.button(box,
self,
"Apply",
callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
self._update_sheet_combo()
self.load_data()
class OW1ka(widget.OWWidget):
name = "EnKlik Anketa"
description = "Import data from EnKlikAnketa (1ka.si) public URL."
icon = "icons/1ka.svg"
priority = 200
class Outputs:
data = Output("Data", Table)
want_main_area = False
resizing_enabled = False
settingsHandler = settings.PerfectDomainContextHandler(
match_values=settings.PerfectDomainContextHandler.MATCH_VALUES_ALL)
recent = settings.Setting([])
reload_idx = settings.Setting(0)
autocommit = settings.Setting(True)
domain_editor = settings.SettingProvider(DomainEditor)
UserAdviceMessages = [
widget.Message(
'You can import data from public links to 1ka surveys results. '
'Click to learn more on how to get a shareable public link URL for '
'1ka surveys that you manage.',
'public-link',
icon=widget.Message.Information,
moreurl=
'http://english.1ka.si/db/24/468/Guides/Public_link_to_access_data_and_analysis/'
),
]
class Error(widget.OWWidget.Error):
net_error = widget.Msg(
"Couldn't load data: {}. Ensure network connection, firewall ...")
parse_error = widget.Msg(
"Couldn't parse data: {}. Ensure well-formatted data or submit a bug report."
)
invalid_url = widget.Msg(
'Invalid URL. Public shareable link should match: ' +
VALID_URL_HELP)
data_is_anal = widget.Msg(
"The provided URL is a public link to 'Analysis'. Need public link to 'Data'."
)
class Information(widget.OWWidget.Information):
response_data_empty = widget.Msg(
'Response data is empty. Get some responses first.')
def __init__(self):
super().__init__()
self.table = None
self._html = None
def _loadFinished(is_ok):
if is_ok:
QTimer.singleShot(
1, lambda: setattr(self, '_html', self.webview.html()))
self.webview = WebviewWidget(loadFinished=_loadFinished)
vb = gui.vBox(self.controlArea, 'Import Data')
hb = gui.hBox(vb)
self.combo = combo = URLComboBox(
hb,
self.recent,
editable=True,
minimumWidth=400,
insertPolicy=QComboBox.InsertAtTop,
toolTip='Format: ' + VALID_URL_HELP,
editTextChanged=self.is_valid_url,
# Indirect via QTimer because calling wait() -> processEvents,
# while our currentIndexChanged event hadn't yet finished.
# Avoids calling handler twice.
currentIndexChanged=lambda: QTimer.singleShot(1, self.load_url))
hb.layout().addWidget(QLabel('Public link URL:', hb))
hb.layout().addWidget(combo)
hb.layout().setStretch(1, 2)
RELOAD_TIMES = (
('No reload', ),
('5 s', 5000),
('10 s', 10000),
('30 s', 30000),
('1 min', 60 * 1000),
('2 min', 2 * 60 * 1000),
('5 min', 5 * 60 * 1000),
)
reload_timer = QTimer(self,
timeout=lambda: self.load_url(from_reload=True))
def _on_reload_changed():
if self.reload_idx == 0:
reload_timer.stop()
return
reload_timer.start(RELOAD_TIMES[self.reload_idx][1])
gui.comboBox(vb,
self,
'reload_idx',
label='Reload every:',
orientation=Qt.Horizontal,
items=[i[0] for i in RELOAD_TIMES],
callback=_on_reload_changed)
box = gui.widgetBox(self.controlArea, "Columns (Double-click to edit)")
self.domain_editor = DomainEditor(self)
editor_model = self.domain_editor.model()
def editorDataChanged():
self.apply_domain_edit()
self.commit()
editor_model.dataChanged.connect(editorDataChanged)
box.layout().addWidget(self.domain_editor)
box = gui.widgetBox(self.controlArea, "Info", addSpace=True)
info = self.data_info = gui.widgetLabel(box, '')
info.setWordWrap(True)
self.controlArea.layout().addStretch(1)
gui.auto_commit(self.controlArea, self, 'autocommit', label='Commit')
self.set_info()
def set_combo_items(self):
self.combo.clear()
for sheet in self.recent:
self.combo.addItem(sheet.name, sheet.url)
def commit(self):
self.Outputs.data.send(self.table)
def is_valid_url(self, url):
if is_valid_url(url):
self.Error.invalid_url.clear()
return True
self.Error.invalid_url()
QToolTip.showText(self.combo.mapToGlobal(QPoint(0, 0)),
self.combo.toolTip())
def load_url(self, from_reload=False):
self.closeContext()
self.domain_editor.set_domain(None)
url = self.combo.currentText()
if not self.is_valid_url(url):
self.table = None
self.commit()
return
if url not in self.recent:
self.recent.insert(0, url)
prev_table = self.table
with self.progressBar(3) as progress:
try:
self._html = None
self.webview.setUrl(url)
wait(until=lambda: self._html is not None)
progress.advance()
# Wait some seconds for discrete labels to have loaded via AJAX,
# then re-query HTML.
# *Webview.loadFinished doesn't guarantee it sufficiently
try:
wait(until=lambda: False, timeout=1200)
except TimeoutError:
pass
progress.advance()
html = self.webview.html()
except Exception as e:
log.exception("Couldn't load data from: %s", url)
self.Error.net_error(try_(lambda: e.args[0], ''))
self.table = None
else:
self.Error.clear()
self.Information.clear()
self.table = None
try:
table = self.table = self.table_from_html(html)
except DataEmptyError:
self.Information.response_data_empty()
except DataIsAnalError:
self.Error.data_is_anal()
except Exception as e:
log.exception('Parsing error: %s', url)
self.Error.parse_error(try_(lambda: e.args[0], ''))
else:
self.openContext(table.domain)
self.combo.setTitleFor(self.combo.currentIndex(),
table.name)
def _equal(data1, data2):
NAN = float('nan')
return (try_(lambda: data1.checksum(),
NAN) == try_(lambda: data2.checksum(), NAN))
self._orig_table = self.table
self.apply_domain_edit()
if not (from_reload and _equal(prev_table, self.table)):
self.commit()
def apply_domain_edit(self):
data = self._orig_table
if data is None:
self.set_info()
return
domain, cols = self.domain_editor.get_domain(data.domain, data)
# Copied verbatim from OWFile
if not (domain.variables or domain.metas):
table = None
else:
X, y, m = cols
table = Table.from_numpy(domain, X, y, m, data.W)
table.name = data.name
table.ids = np.array(data.ids)
table.attributes = getattr(data, 'attributes', {})
self.table = table
self.set_info()
DATETIME_VAR = 'Paradata (insert)'
def table_from_html(self, html):
soup = BeautifulSoup(html, 'html.parser')
try:
html_table = soup.find_all('table')[-1]
except IndexError:
raise DataEmptyError
if '<h2>Anal' in html or 'div_analiza_' in html:
raise DataIsAnalError
def _header_row_strings(row):
return chain.from_iterable(
repeat(th.get_text(), int(th.get('colspan') or 1)) for th in
html_table.select('thead tr:nth-of-type(%d) th[title]' % row))
# self.DATETIME_VAR (available when Paradata is enabled in 1ka UI)
# should match this variable name format
header = [
th1.rstrip(':') +
('' if th3 == th1 else ' ({})').format(th3.rstrip(':'))
for th1, th3 in zip(_header_row_strings(1), _header_row_strings(3))
]
values = [
[
( # If no span, feature is a number or a text field
td.get_text() if td.span is None else
# If have span, it's a number, but if negative, replace with NaN
'' if td.contents[0].strip().startswith('-') else
# Else if span, the number is its code, but we want its value
td.span.get_text()[1:-1]) for td in tr.select('td')
if 'data_uid' not in td.get('class', ())
] for tr in html_table.select('tbody tr')
]
# Save parsed values into in-mem file for default values processing
buffer = StringIO()
writer = csv.writer(buffer, delimiter='\t')
writer.writerow(header)
writer.writerows(values)
buffer.flush()
buffer.seek(0)
data = TabReader(buffer).read()
title = soup.select('body h2:nth-of-type(1)')[0].get_text().split(
': ', maxsplit=1)[-1]
data.name = title
return data
def set_info(self):
data = self.table
if data is None:
self.data_info.setText('No spreadsheet loaded.')
return
text = "{}\n\n{} instance(s), {} feature(s), {} meta attribute(s)\n".format(
data.name, len(data), len(data.domain.attributes),
len(data.domain.metas))
text += try_(
lambda: '\nFirst entry: {}'
'\nLast entry: {}'.format(data[0, self.DATETIME_VAR], data[
-1, self.DATETIME_VAR]), '')
self.data_info.setText(text)
def __init__(self):
super().__init__()
RecentPathsWComboMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.reader = None
readers = [
f for f in FileFormat.formats
if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None)
]
def group_readers_per_addon_key(w):
# readers from Orange.data.io should go first
def package(w):
package = w.qualified_name().split(".")[:-1]
package = package[:2]
if ".".join(package) == "Orange.data":
return ["0"] # force "Orange" to come first
return package
return package(w), w.DESCRIPTION
self.available_readers = sorted(set(readers),
key=group_readers_per_addon_key)
layout = QGridLayout()
layout.setSpacing(4)
gui.widgetBox(self.controlArea, orientation=layout, box='Source')
vbox = gui.radioButtons(None,
self,
"source",
box=True,
callback=self.load_data,
addToLayout=False)
rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.activated[int].connect(self.select_file)
box.layout().addWidget(self.file_combo)
layout.addWidget(box, 0, 1)
file_button = gui.button(None,
self,
'...',
callback=self.browse_file,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 3)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = QComboBox()
self.sheet_combo.activated[str].connect(self.select_sheet)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = NamedURLModel(self.sheet_names)
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(Policy.Ignored, Policy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 1, 3)
url_combo.activated.connect(self._url_set)
# whit completer we set that combo box is case sensitive when
# matching the history
completer = QCompleter()
completer.setCaseSensitivity(Qt.CaseSensitive)
url_combo.setCompleter(completer)
layout = QGridLayout()
layout.setSpacing(4)
gui.widgetBox(self.controlArea, orientation=layout, box='File Type')
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.reader_combo = QComboBox(self)
self.reader_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.reader_combo.activated[int].connect(self.select_reader)
box.layout().addWidget(self.reader_combo)
layout.addWidget(box, 0, 1)
box = gui.vBox(self.controlArea, "Info")
self.infolabel = gui.widgetLabel(box, 'No data loaded.')
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
box = gui.hBox(box)
gui.button(box,
self,
"Reset",
callback=self.reset_domain_edit,
autoDefault=False)
gui.rubber(box)
self.apply_button = gui.button(box,
self,
"Apply",
callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
hBox = gui.hBox(self.controlArea)
gui.rubber(hBox)
gui.button(hBox,
self,
"Browse documentation datasets",
callback=lambda: self.browse_file(True),
autoDefault=False)
gui.rubber(hBox)
self.set_file_list()
# Must not call open_file from within __init__. open_file
# explicitly re-enters the event loop (by a progress bar)
self.setAcceptDrops(True)
if self.source == self.LOCAL_FILE:
last_path = self.last_path()
if last_path and os.path.exists(last_path) and \
os.path.getsize(last_path) > self.SIZE_LIMIT:
self.Warning.file_too_big()
return
QTimer.singleShot(0, self.load_data)
class OWMultifile(Orange.widgets.data.owfile.OWFile, RecentPathsWidgetMixin):
name = "Multifile"
id = "orangecontrib.spectroscopy.widgets.files"
icon = "icons/multifile.svg"
description = "Read data from input files " \
"and send a data table to the output."
priority = 10000
replaces = [
"orangecontrib.infrared.widgets.owfiles.OWFiles",
"orangecontrib.infrared.widgets.owmultifile.OWMultifile"
]
file_idx = []
sheet = Orange.widgets.settings.Setting(None)
label = Orange.widgets.settings.Setting("")
recent_paths = Orange.widgets.settings.Setting([])
def __init__(self):
widget.OWWidget.__init__(self)
RecentPathsWidgetMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.sheets = []
self.lb = gui.listBox(self.controlArea,
self,
"file_idx",
selectionMode=QListWidget.MultiSelection)
layout = QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
file_button = gui.button(None,
self,
' ...',
callback=self.browse_files,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 0)
remove_button = gui.button(None,
self,
'Remove',
callback=self.remove_item)
clear_button = gui.button(None, self, 'Clear', callback=self.clear)
layout.addWidget(remove_button, 0, 1)
layout.addWidget(clear_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 7)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(None,
self,
"xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
layout.addWidget(self.sheet_box, 0, 5)
label_box = gui.hBox(None, addToLayout=False, margin=0)
label = gui.lineEdit(label_box,
self,
"label",
callback=self.set_label,
label="Label",
orientation=Qt.Horizontal)
layout.addWidget(label_box, 0, 6)
layout.setColumnStretch(3, 2)
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
for i, rp in enumerate(self.recent_paths):
self.lb.addItem(rp.abspath)
# TODO unresolved paths just disappear! Modify _relocate_recent_files
box = gui.hBox(self.controlArea)
gui.rubber(box)
box.layout().addWidget(self.report_button)
self.report_button.setFixedWidth(170)
self.apply_button = gui.button(box,
self,
"Apply",
callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
self._update_sheet_combo()
self.load_data()
def set_label(self):
self.load_data()
def add_path(self, filename):
recent = RecentPath.create(filename, self._search_paths())
self.recent_paths.append(recent)
def set_file_list(self):
# need to define it for RecentPathsWidgetMixin
pass
def _select_active_sheet(self):
if self.sheet:
try:
sheet_list = [s[0] for s in self.sheets]
idx = sheet_list.index(self.sheet)
self.sheet_combo.setCurrentIndex(idx)
except ValueError:
# Requested sheet does not exist in this file
self.sheet = None
else:
self.sheet_combo.setCurrentIndex(0)
def _update_sheet_combo(self):
sheets = Counter()
for fn in self.current_filenames():
try:
reader = FileFormat.get_reader(fn)
sheets.update(reader.sheets)
except:
pass
sheets = sorted(sheets.items(), key=lambda x: x[0])
self.sheets = [(s, s + " (" + str(n) + ")") for s, n in sheets]
if len(sheets) < 2:
self.sheet_box.hide()
self.sheet = None
else:
self.sheets.insert(0, (None, "(None)"))
self.sheet_combo.clear()
self.sheet_combo.addItems([s[1] for s in self.sheets])
self._select_active_sheet()
self.sheet_box.show()
def select_sheet(self):
self.sheet = self.sheets[self.sheet_combo.currentIndex()][0]
self.load_data()
def remove_item(self):
ri = [i.row() for i in self.lb.selectedIndexes()]
for i in sorted(ri, reverse=True):
self.recent_paths.pop(i)
self.lb.takeItem(i)
self._update_sheet_combo()
self.load_data()
def clear(self):
self.lb.clear()
while self.recent_paths:
self.recent_paths.pop()
self._update_sheet_combo()
self.load_data()
def browse_files(self, in_demos=False):
start_file = self.last_path() or os.path.expanduser("~/")
filenames = QFileDialog.getOpenFileNames(self,
'Open Multiple Data Files',
start_file, dialog_formats())
if isinstance(filenames, tuple): # has a file description
filenames = filenames[0]
self.load_files(filenames)
def load_files(self, filenames):
if not filenames:
return
for f in filenames:
self.add_path(f)
self.lb.addItem(f)
self._update_sheet_combo()
self.load_data()
def current_filenames(self):
return [rp.abspath for rp in self.recent_paths]
def load_data(self):
self.closeContext()
fns = self.current_filenames()
data_list = []
fnok_list = []
empty_domain = Orange.data.Domain(attributes=[])
for fn in fns:
reader = FileFormat.get_reader(fn)
errors = []
with catch_warnings(record=True) as warnings:
try:
if self.sheet in reader.sheets:
reader.select_sheet(self.sheet)
if isinstance(reader, SpectralFileFormat):
xs, vals, additional = reader.read_spectra()
if additional is None:
additional = Orange.data.Table.from_domain(
empty_domain, n_rows=len(vals))
data_list.append((xs, vals, additional))
else:
data_list.append(reader.read())
fnok_list.append(fn)
except Exception as ex:
errors.append("An error occurred:")
errors.append(str(ex))
#FIXME show error in the list of data
self.warning(warnings[-1].message.args[0] if warnings else '')
if data_list:
data = concatenate_data(data_list, fnok_list, self.label)
self.data = data
self.openContext(data.domain)
else:
self.data = None
self.domain_editor.set_domain(None)
self.apply_domain_edit() # sends data
def __init__(self):
super().__init__()
self.table = None
self._html = None
def _loadFinished(is_ok):
if is_ok:
QTimer.singleShot(
1, lambda: setattr(self, '_html', self.webview.html()))
self.webview = WebviewWidget(loadFinished=_loadFinished)
vb = gui.vBox(self.controlArea, 'Import Data')
hb = gui.hBox(vb)
self.combo = combo = URLComboBox(
hb,
self.recent,
editable=True,
minimumWidth=400,
insertPolicy=QComboBox.InsertAtTop,
toolTip='Format: ' + VALID_URL_HELP,
editTextChanged=self.is_valid_url,
# Indirect via QTimer because calling wait() -> processEvents,
# while our currentIndexChanged event hadn't yet finished.
# Avoids calling handler twice.
currentIndexChanged=lambda: QTimer.singleShot(1, self.load_url))
hb.layout().addWidget(QLabel('Public link URL:', hb))
hb.layout().addWidget(combo)
hb.layout().setStretch(1, 2)
RELOAD_TIMES = (
('No reload', ),
('5 s', 5000),
('10 s', 10000),
('30 s', 30000),
('1 min', 60 * 1000),
('2 min', 2 * 60 * 1000),
('5 min', 5 * 60 * 1000),
)
reload_timer = QTimer(self,
timeout=lambda: self.load_url(from_reload=True))
def _on_reload_changed():
if self.reload_idx == 0:
reload_timer.stop()
return
reload_timer.start(RELOAD_TIMES[self.reload_idx][1])
gui.comboBox(vb,
self,
'reload_idx',
label='Reload every:',
orientation=Qt.Horizontal,
items=[i[0] for i in RELOAD_TIMES],
callback=_on_reload_changed)
box = gui.widgetBox(self.controlArea, "Columns (Double-click to edit)")
self.domain_editor = DomainEditor(self)
editor_model = self.domain_editor.model()
def editorDataChanged():
self.apply_domain_edit()
self.commit()
editor_model.dataChanged.connect(editorDataChanged)
box.layout().addWidget(self.domain_editor)
box = gui.widgetBox(self.controlArea, "Info", addSpace=True)
info = self.data_info = gui.widgetLabel(box, '')
info.setWordWrap(True)
self.controlArea.layout().addStretch(1)
gui.auto_commit(self.controlArea, self, 'autocommit', label='Commit')
self.set_info()
class OWFile(widget.OWWidget, RecentPathsWComboMixin):
name = "文件(File)"
id = "orange.widgets.data.file"
description = "从输入文件或网络读取数据并将数据表发送到输出。"
icon = "icons/File.svg"
priority = 10
category = "数据(Data)"
keywords = ["file", "load", "read", "open", "wenjian"]
class Outputs:
data = Output("数据(Data)",
Table,
doc="Attribute-valued dataset read from the input file.",
replaces=['Data'])
want_main_area = False
buttons_area_orientation = None
SEARCH_PATHS = [("sample-datasets", get_sample_datasets_dir())]
SIZE_LIMIT = 1e7
LOCAL_FILE, URL = range(2)
settingsHandler = PerfectDomainContextHandler(
match_values=PerfectDomainContextHandler.MATCH_VALUES_ALL)
# pylint seems to want declarations separated from definitions
recent_paths: List[RecentPath]
recent_urls: List[str]
variables: list
# Overload RecentPathsWidgetMixin.recent_paths to set defaults
recent_paths = Setting([
RecentPath("", "sample-datasets", "iris.tab"),
RecentPath("", "sample-datasets", "titanic.tab"),
RecentPath("", "sample-datasets", "housing.tab"),
RecentPath("", "sample-datasets", "heart_disease.tab"),
RecentPath("", "sample-datasets", "brown-selected.tab"),
RecentPath("", "sample-datasets", "zoo.tab"),
])
recent_urls = Setting([])
source = Setting(LOCAL_FILE)
sheet_names = Setting({})
url = Setting("")
variables = ContextSetting([])
domain_editor = SettingProvider(DomainEditor)
class Information(widget.OWWidget.Information):
no_file_selected = Msg("No file selected.")
class Warning(widget.OWWidget.Warning):
file_too_big = Msg("The file is too large to load automatically."
" Press Reload to load.")
load_warning = Msg("Read warning:\n{}")
performance_warning = Msg(
"Categorical variables with >100 values may decrease performance.")
renamed_vars = Msg("Some variables have been renamed "
"to avoid duplicates.\n{}")
multiple_targets = Msg("Most widgets do not support multiple targets")
class Error(widget.OWWidget.Error):
file_not_found = Msg("File not found.")
missing_reader = Msg("Missing reader.")
sheet_error = Msg("Error listing available sheets.")
unknown = Msg("Read error:\n{}")
UserAdviceMessages = [
widget.Message(
"Use CSV File Import widget for advanced options "
"for comma-separated files", "use-csv-file-import"),
widget.Message(
"This widget loads only tabular data. Use other widgets to load "
"other data types like models, distance matrices and networks.",
"other-data-types")
]
def __init__(self):
super().__init__()
RecentPathsWComboMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.reader = None
readers = [
f for f in FileFormat.formats
if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None)
]
def group_readers_per_addon_key(w):
# readers from Orange.data.io should go first
def package(w):
package = w.qualified_name().split(".")[:-1]
package = package[:2]
if ".".join(package) == "Orange.data":
return ["0"] # force "Orange" to come first
return package
return package(w), w.DESCRIPTION
self.available_readers = sorted(set(readers),
key=group_readers_per_addon_key)
layout = QGridLayout()
layout.setSpacing(4)
gui.widgetBox(self.controlArea, orientation=layout, box='数据源')
vbox = gui.radioButtons(None,
self,
"source",
box=True,
callback=self.load_data,
addToLayout=False)
rb_button = gui.appendRadioButton(vbox, "文件:", addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.Expanding, Policy.Fixed)
self.file_combo.setSizePolicy(Policy.Expanding, Policy.Fixed)
self.file_combo.setMinimumSize(QSize(100, 1))
self.file_combo.activated[int].connect(self.select_file)
box.layout().addWidget(self.file_combo)
layout.addWidget(box, 0, 1)
file_button = gui.button(None,
self,
'...',
callback=self.browse_file,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 2)
reload_button = gui.button(None,
self,
"重新加载",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 3)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = QComboBox()
self.sheet_combo.activated[str].connect(self.select_sheet)
self.sheet_combo.setSizePolicy(Policy.Expanding, Policy.Fixed)
self.sheet_combo.setMinimumSize(QSize(50, 1))
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = NamedURLModel(self.sheet_names)
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(Policy.Ignored, Policy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 1, 3)
url_combo.activated.connect(self._url_set)
# whit completer we set that combo box is case sensitive when
# matching the history
completer = QCompleter()
completer.setCaseSensitivity(Qt.CaseSensitive)
url_combo.setCompleter(completer)
layout = QGridLayout()
layout.setSpacing(4)
gui.widgetBox(self.controlArea, orientation=layout, box='文件类型')
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.Expanding, Policy.Fixed)
self.reader_combo = QComboBox(self)
self.reader_combo.setSizePolicy(Policy.Expanding, Policy.Fixed)
self.reader_combo.setMinimumSize(QSize(100, 1))
self.reader_combo.activated[int].connect(self.select_reader)
box.layout().addWidget(self.reader_combo)
layout.addWidget(box, 0, 1)
box = gui.vBox(self.controlArea, "信息")
self.infolabel = gui.widgetLabel(box, '未加载数据.')
box = gui.widgetBox(self.controlArea, "列(双击编辑)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
box = gui.hBox(box)
gui.button(box,
self,
"重置",
callback=self.reset_domain_edit,
autoDefault=False)
gui.rubber(box)
self.apply_button = gui.button(box,
self,
"应用",
callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
hBox = gui.hBox(self.controlArea)
gui.rubber(hBox)
gui.button(hBox,
self,
"浏览文档数据集",
callback=lambda: self.browse_file(True),
autoDefault=False)
gui.rubber(hBox)
self.set_file_list()
# Must not call open_file from within __init__. open_file
# explicitly re-enters the event loop (by a progress bar)
self.setAcceptDrops(True)
if self.source == self.LOCAL_FILE:
last_path = self.last_path()
if last_path and os.path.exists(last_path) and \
os.path.getsize(last_path) > self.SIZE_LIMIT:
self.Warning.file_too_big()
return
QTimer.singleShot(0, self.load_data)
@staticmethod
def sizeHint():
return QSize(600, 550)
def select_file(self, n):
assert n < len(self.recent_paths)
super().select_file(n)
if self.recent_paths:
self.source = self.LOCAL_FILE
self.load_data()
self.set_file_list()
def select_sheet(self):
self.recent_paths[0].sheet = self.sheet_combo.currentText()
self.load_data()
def select_reader(self, n):
if self.source != self.LOCAL_FILE:
return # ignore for URL's
if self.recent_paths:
path = self.recent_paths[0]
if n == 0: # default
path.file_format = None
self.load_data()
elif n <= len(self.available_readers):
reader = self.available_readers[n - 1]
path.file_format = reader.qualified_name()
self.load_data()
else: # the rest include just qualified names
path.file_format = self.reader_combo.itemText(n)
self.load_data()
def _url_set(self):
url = self.url_combo.currentText()
pos = self.recent_urls.index(url)
url = url.strip()
if not urlparse(url).scheme:
url = 'http://' + url
self.url_combo.setItemText(pos, url)
self.recent_urls[pos] = url
self.source = self.URL
self.load_data()
def browse_file(self, in_demos=False):
if in_demos:
start_file = get_sample_datasets_dir()
if not os.path.exists(start_file):
QMessageBox.information(None, "文件", "无法找到文件")
return
else:
start_file = self.last_path() or os.path.expanduser("~/")
filename, reader, _ = open_filename_dialog(start_file, None,
self.available_readers)
if not filename:
return
self.add_path(filename)
if reader is not None:
self.recent_paths[0].file_format = reader.qualified_name()
self.source = self.LOCAL_FILE
self.load_data()
# Open a file, create data from it and send it over the data channel
def load_data(self):
# We need to catch any exception type since anything can happen in
# file readers
self.closeContext()
self.domain_editor.set_domain(None)
self.apply_button.setEnabled(False)
self.clear_messages()
self.set_file_list()
error = self._try_load()
if error:
error()
self.data = None
self.sheet_box.hide()
self.Outputs.data.send(None)
self.infolabel.setText("无数据")
def _try_load(self):
self._initialize_reader_combo()
# pylint: disable=broad-except
if self.source == self.LOCAL_FILE:
if self.last_path() is None:
return self.Information.no_file_selected
elif not os.path.exists(self.last_path()):
return self.Error.file_not_found
else:
url = self.url_combo.currentText().strip()
if not url:
return self.Information.no_file_selected
def mark_problematic_reader():
self.reader_combo.setItemData(self.reader_combo.currentIndex(),
QBrush(Qt.red), Qt.ForegroundRole)
try:
self.reader = self._get_reader() # also sets current reader index
assert self.reader is not None
except MissingReaderException:
mark_problematic_reader()
return self.Error.missing_reader
except Exception as ex:
mark_problematic_reader()
log.exception(ex)
return lambda x=ex: self.Error.unknown(str(x))
try:
self._update_sheet_combo()
except Exception:
return self.Error.sheet_error
with log_warnings() as warnings:
try:
data = self.reader.read()
except Exception as ex:
mark_problematic_reader()
log.exception(ex)
return lambda x=ex: self.Error.unknown(str(x))
if warnings:
self.Warning.load_warning(warnings[-1].message.args[0])
self.infolabel.setText(self._describe(data))
self.loaded_file = self.last_path()
add_origin(data, self.loaded_file)
self.data = data
self.openContext(data.domain)
self.apply_domain_edit() # sends data
return None
def _get_reader(self) -> FileFormat:
if self.source == self.LOCAL_FILE:
path = self.last_path()
self.reader_combo.setEnabled(True)
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
qname_index = {
r.qualified_name(): i
for i, r in enumerate(self.available_readers)
}
if qname in qname_index:
self.reader_combo.setCurrentIndex(qname_index[qname] + 1)
else:
# reader may be accessible, but not in self.available_readers
# (perhaps its code was moved)
self.reader_combo.addItem(qname)
self.reader_combo.setCurrentIndex(
len(self.reader_combo) - 1)
try:
reader_class = class_from_qualified_name(qname)
except Exception as ex:
raise MissingReaderException(
f'Can not find reader "{qname}"') from ex
reader = reader_class(path)
else:
self.reader_combo.setCurrentIndex(0)
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
return UrlReader(url)
def _update_sheet_combo(self):
if len(self.reader.sheets) < 2:
self.sheet_box.hide()
self.reader.select_sheet(None)
return
self.sheet_combo.clear()
self.sheet_combo.addItems(self.reader.sheets)
self._select_active_sheet()
self.sheet_box.show()
def _select_active_sheet(self):
try:
idx = self.reader.sheets.index(self.reader.sheet)
self.sheet_combo.setCurrentIndex(idx)
except ValueError:
# Requested sheet does not exist in this file
self.reader.select_sheet(None)
self.sheet_combo.setCurrentIndex(0)
def _initialize_reader_combo(self):
self.reader_combo.clear()
filters = [format_filter(f) for f in self.available_readers]
self.reader_combo.addItems([DEFAULT_READER_TEXT] + filters)
self.reader_combo.setCurrentIndex(0)
self.reader_combo.setDisabled(True)
# additional readers may be added in self._get_reader()
@staticmethod
def _describe(table):
def missing_prop(prop):
if prop:
return f"({prop * 100:.1f}% 个缺失值)"
else:
return "(无缺失值)"
domain = table.domain
text = ""
attrs = getattr(table, "attributes", {})
descs = [
attrs[desc] for desc in ("Name", "Description") if desc in attrs
]
if len(descs) == 2:
descs[0] = f"<b>{descs[0]}</b>"
if descs:
text += f"<p>{'<br/>'.join(descs)}</p>"
text += f"<p>{len(table)} 条数据"
missing_in_attr = missing_prop(table.has_missing_attribute()
and table.get_nan_frequency_attribute())
missing_in_class = missing_prop(table.has_missing_class()
and table.get_nan_frequency_class())
text += f"<br/>特征数目: {len(domain.attributes)} {missing_in_attr}"
if domain.has_continuous_class:
text += f"<br/>回归; 数值类 {missing_in_class}"
elif domain.has_discrete_class:
text += "<br/>分类: 分类种类共 " \
f"{len(domain.class_var.values)} 个 {missing_in_class}"
elif table.domain.class_vars:
text += "<br/>Multi-target; " \
f"{len(table.domain.class_vars)} target variables " \
f"{missing_in_class}"
else:
text += "<br/>Data has no target variable."
text += f"<br/>元属性: { len(domain.metas)}"
text += "</p>"
if 'Timestamp' in table.domain:
# Google Forms uses this header to timestamp responses
text += f"<p>First entry: {table[0, 'Timestamp']}<br/>" \
f"Last entry: {table[-1, 'Timestamp']}</p>"
return text
def storeSpecificSettings(self):
self.current_context.modified_variables = self.variables[:]
def retrieveSpecificSettings(self):
if hasattr(self.current_context, "modified_variables"):
self.variables[:] = self.current_context.modified_variables
def reset_domain_edit(self):
self.domain_editor.reset_domain()
self.apply_domain_edit()
def _inspect_discrete_variables(self, domain):
for var in chain(domain.variables, domain.metas):
if var.is_discrete and len(var.values) > 100:
self.Warning.performance_warning()
def apply_domain_edit(self):
self.Warning.performance_warning.clear()
self.Warning.renamed_vars.clear()
if self.data is None:
table = None
else:
domain, cols, renamed = \
self.domain_editor.get_domain(self.data.domain, self.data,
deduplicate=True)
if not (domain.variables or domain.metas):
table = None
elif domain is self.data.domain:
table = self.data
else:
X, y, m = cols
table = Table.from_numpy(domain, X, y, m, self.data.W)
table.name = self.data.name
table.ids = np.array(self.data.ids)
table.attributes = getattr(self.data, 'attributes', {})
self._inspect_discrete_variables(domain)
if renamed:
self.Warning.renamed_vars(f"Renamed: {', '.join(renamed)}")
self.Warning.multiple_targets(
shown=table is not None and len(table.domain.class_vars) > 1)
self.Outputs.data.send(table)
self.apply_button.setEnabled(False)
def get_widget_name_extension(self):
_, name = os.path.split(self.loaded_file)
return os.path.splitext(name)[0]
def send_report(self):
def get_ext_name(filename):
try:
return FileFormat.names[os.path.splitext(filename)[1]]
except KeyError:
return "unknown"
if self.data is None:
self.report_paragraph("File", "No file.")
return
if self.source == self.LOCAL_FILE:
home = os.path.expanduser("~")
if self.loaded_file.startswith(home):
# os.path.join does not like ~
name = "~" + os.path.sep + \
self.loaded_file[len(home):].lstrip("/").lstrip("\\")
else:
name = self.loaded_file
if self.sheet_combo.isVisible():
name += f" ({self.sheet_combo.currentText()})"
self.report_items("File", [("File name", name),
("Format", get_ext_name(name))])
else:
self.report_items("Data", [("Resource", self.url),
("Format", get_ext_name(self.url))])
self.report_data("Data", self.data)
@staticmethod
def dragEnterEvent(event):
"""Accept drops of valid file urls"""
urls = event.mimeData().urls()
if urls:
try:
FileFormat.get_reader(urls[0].toLocalFile())
event.acceptProposedAction()
except MissingReaderException:
pass
def dropEvent(self, event):
"""Handle file drops"""
urls = event.mimeData().urls()
if urls:
self.add_path(urls[0].toLocalFile()) # add first file
self.source = self.LOCAL_FILE
self.load_data()
def workflowEnvChanged(self, key, value, oldvalue):
"""
Function called when environment changes (e.g. while saving the scheme)
It make sure that all environment connected values are modified
(e.g. relative file paths are changed)
"""
self.update_file_list(key, value, oldvalue)
def __init__(self):
widget.OWWidget.__init__(self)
RecentPathsWidgetMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.sheets = []
self.lb = gui.listBox(self.controlArea, self, "file_idx",
selectionMode=QListWidget.MultiSelection)
layout = QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
file_button = gui.button(
None, self, ' ...', callback=self.browse_files, autoDefault=False)
file_button.setIcon(self.style().standardIcon(
QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 0)
remove_button = gui.button(
None, self, 'Remove', callback=self.remove_item)
clear_button = gui.button(
None, self, 'Clear', callback=self.clear)
layout.addWidget(remove_button, 0, 1)
layout.addWidget(clear_button, 0, 2)
reload_button = gui.button(
None, self, "Reload", callback=self.load_data, autoDefault=False)
reload_button.setIcon(self.style().standardIcon(QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 7)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(None, self, "xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True)
self.sheet_combo.setSizePolicy(
Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(
Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(
self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(
self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
layout.addWidget(self.sheet_box, 0, 5)
label_box = gui.hBox(None, addToLayout=False, margin=0)
label = gui.lineEdit(label_box, self, "label", callback=self.set_label,
label="Label", orientation=Qt.Horizontal)
layout.addWidget(label_box, 0, 6)
layout.setColumnStretch(3, 2)
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
for i, rp in enumerate(self.recent_paths):
self.lb.addItem(rp.abspath)
# TODO unresolved paths just disappear! Modify _relocate_recent_files
box = gui.hBox(self.controlArea)
gui.rubber(box)
box.layout().addWidget(self.report_button)
self.report_button.setFixedWidth(170)
self.apply_button = gui.button(
box, self, "Apply", callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
self._update_sheet_combo()
self.load_data()
class OWMultifile(Orange.widgets.data.owfile.OWFile, RecentPathsWidgetMixin):
name = "Multifile"
id = "orangecontrib.spectroscopy.widgets.files"
icon = "icons/multifile.svg"
description = "Read data from input files " \
"and send a data table to the output."
priority = 10000
replaces = ["orangecontrib.infrared.widgets.owfiles.OWFiles",
"orangecontrib.infrared.widgets.owmultifile.OWMultifile"]
file_idx = []
sheet = Orange.widgets.settings.Setting(None)
label = Orange.widgets.settings.Setting("")
recent_paths = Orange.widgets.settings.Setting([])
def __init__(self):
widget.OWWidget.__init__(self)
RecentPathsWidgetMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.sheets = []
self.lb = gui.listBox(self.controlArea, self, "file_idx",
selectionMode=QListWidget.MultiSelection)
layout = QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
file_button = gui.button(
None, self, ' ...', callback=self.browse_files, autoDefault=False)
file_button.setIcon(self.style().standardIcon(
QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 0)
remove_button = gui.button(
None, self, 'Remove', callback=self.remove_item)
clear_button = gui.button(
None, self, 'Clear', callback=self.clear)
layout.addWidget(remove_button, 0, 1)
layout.addWidget(clear_button, 0, 2)
reload_button = gui.button(
None, self, "Reload", callback=self.load_data, autoDefault=False)
reload_button.setIcon(self.style().standardIcon(QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 7)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(None, self, "xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True)
self.sheet_combo.setSizePolicy(
Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(
Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(
self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(
self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
layout.addWidget(self.sheet_box, 0, 5)
label_box = gui.hBox(None, addToLayout=False, margin=0)
label = gui.lineEdit(label_box, self, "label", callback=self.set_label,
label="Label", orientation=Qt.Horizontal)
layout.addWidget(label_box, 0, 6)
layout.setColumnStretch(3, 2)
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
for i, rp in enumerate(self.recent_paths):
self.lb.addItem(rp.abspath)
# TODO unresolved paths just disappear! Modify _relocate_recent_files
box = gui.hBox(self.controlArea)
gui.rubber(box)
box.layout().addWidget(self.report_button)
self.report_button.setFixedWidth(170)
self.apply_button = gui.button(
box, self, "Apply", callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
self._update_sheet_combo()
self.load_data()
def set_label(self):
self.load_data()
def add_path(self, filename, reader=None):
recent = RecentPath.create(filename, self._search_paths())
if reader is not None:
recent.file_format = reader.qualified_name()
self.recent_paths.append(recent)
def set_file_list(self):
# need to define it for RecentPathsWidgetMixin
pass
def _select_active_sheet(self):
if self.sheet:
try:
sheet_list = [s[0] for s in self.sheets]
idx = sheet_list.index(self.sheet)
self.sheet_combo.setCurrentIndex(idx)
except ValueError:
# Requested sheet does not exist in this file
self.sheet = None
else:
self.sheet_combo.setCurrentIndex(0)
def _update_sheet_combo(self):
sheets = Counter()
for rp in self.recent_paths:
try:
reader = _get_reader(rp)
sheets.update(reader.sheets)
except:
pass
sheets = sorted(sheets.items(), key=lambda x: x[0])
self.sheets = [(s, s + " (" + str(n) + ")") for s, n in sheets]
if len(sheets) < 2:
self.sheet_box.hide()
self.sheet = None
else:
self.sheets.insert(0, (None, "(None)"))
self.sheet_combo.clear()
self.sheet_combo.addItems([s[1] for s in self.sheets])
self._select_active_sheet()
self.sheet_box.show()
def select_sheet(self):
self.sheet = self.sheets[self.sheet_combo.currentIndex()][0]
self.load_data()
def remove_item(self):
ri = [ i.row() for i in self.lb.selectedIndexes() ]
for i in sorted(ri, reverse=True):
self.recent_paths.pop(i)
self.lb.takeItem(i)
self._update_sheet_combo()
self.load_data()
def clear(self):
self.lb.clear()
while self.recent_paths:
self.recent_paths.pop()
self._update_sheet_combo()
self.load_data()
def browse_files(self, in_demos=False):
start_file = self.last_path() or os.path.expanduser("~/")
readers = [f for f in FileFormat.formats
if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None)]
filenames, reader, _ = open_filename_dialog(start_file, None, readers,
dialog=QFileDialog.getOpenFileNames)
self.load_files(filenames, reader)
def load_files(self, filenames, reader):
if not filenames:
return
for f in filenames:
self.add_path(f, reader)
self.lb.addItem(f)
self._update_sheet_combo()
self.load_data()
def load_data(self):
self.closeContext()
data_list = []
fnok_list = []
empty_domain = Orange.data.Domain(attributes=[])
for rp in self.recent_paths:
fn = rp.abspath
reader = _get_reader(rp)
errors = []
with catch_warnings(record=True) as warnings:
try:
if self.sheet in reader.sheets:
reader.select_sheet(self.sheet)
if isinstance(reader, SpectralFileFormat):
xs, vals, additional = reader.read_spectra()
if additional is None:
additional = Orange.data.Table.from_domain(empty_domain, n_rows=len(vals))
data_list.append((xs, vals, additional))
else:
data_list.append(reader.read())
fnok_list.append(fn)
except Exception as ex:
errors.append("An error occurred:")
errors.append(str(ex))
#FIXME show error in the list of data
self.warning(warnings[-1].message.args[0] if warnings else '')
if data_list:
data = concatenate_data(data_list, fnok_list, self.label)
self.data = data
self.openContext(data.domain)
else:
self.data = None
self.domain_editor.set_domain(None)
self.apply_domain_edit() # sends data
def __init__(self):
super().__init__()
RecentPathsWComboMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.reader = None
layout = QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
vbox = gui.radioButtons(None, self, "source", box=True, addSpace=True,
callback=self.load_data, addToLayout=False)
rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.activated[int].connect(self.select_file)
box.layout().addWidget(self.file_combo)
layout.addWidget(box, 0, 1)
file_button = gui.button(
None, self, '...', callback=self.browse_file, autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 2)
reload_button = gui.button(
None, self, "Reload", callback=self.load_data, autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 3)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(None, self, "xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True,)
self.sheet_combo.setSizePolicy(
Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(
Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(
self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(
self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = NamedURLModel(self.sheet_names)
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 3, 3)
url_combo.activated.connect(self._url_set)
box = gui.vBox(self.controlArea, "Info")
self.infolabel = gui.widgetLabel(box, 'No data loaded.')
self.warnings = gui.widgetLabel(box, '')
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
box = gui.hBox(self.controlArea)
gui.button(
box, self, "Browse documentation datasets",
callback=lambda: self.browse_file(True), autoDefault=False)
gui.rubber(box)
gui.button(
box, self, "Reset", callback=self.reset_domain_edit)
self.apply_button = gui.button(
box, self, "Apply", callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
self.set_file_list()
# Must not call open_file from within __init__. open_file
# explicitly re-enters the event loop (by a progress bar)
self.setAcceptDrops(True)
if self.source == self.LOCAL_FILE:
last_path = self.last_path()
if last_path and os.path.exists(last_path) and \
os.path.getsize(last_path) > self.SIZE_LIMIT:
self.Warning.file_too_big()
return
QTimer.singleShot(0, self.load_data)
class OWFiles(Orange.widgets.data.owfile.OWFile, RecentPathsWidgetMixin):
name = "Files"
id = "orangecontrib.infrared.widgets.files"
icon = "icons/files.svg"
description = "Read data from input files " \
"and send a data table to the output."
file_idx = -1
sheet = Orange.widgets.settings.Setting(None)
label = Orange.widgets.settings.Setting("")
recent_paths = Orange.widgets.settings.Setting([])
def __init__(self):
widget.OWWidget.__init__(self)
RecentPathsWidgetMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.sheets = []
self.lb = gui.listBox(self.controlArea, self, "file_idx")
layout = QtGui.QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
file_button = gui.button(None,
self,
' ...',
callback=self.browse_files,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(
QtGui.QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 0)
remove_button = gui.button(None,
self,
'Remove',
callback=self.remove_item)
clear_button = gui.button(None, self, 'Clear', callback=self.clear)
layout.addWidget(remove_button, 0, 1)
layout.addWidget(clear_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QtGui.QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 7)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(None,
self,
"xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QtGui.QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label,
QtCore.Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo,
QtCore.Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
layout.addWidget(self.sheet_box, 0, 5)
label_box = gui.hBox(None, addToLayout=False, margin=0)
label = gui.lineEdit(label_box,
self,
"label",
callback=self.set_label,
label="Label",
orientation=Qt.Horizontal)
layout.addWidget(label_box, 0, 6)
layout.setColumnStretch(3, 2)
box = gui.widgetBox(self.controlArea, "View loaded columns")
if OLD_DOMAINEDITOR:
self.domain_editor = DomainEditor(self.variables)
else:
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
for i, rp in enumerate(self.recent_paths):
self.lb.addItem(rp.abspath)
# TODO unresolved paths just disappear! Modify _relocate_recent_files
self._update_sheet_combo()
self.load_data()
def set_label(self):
self.load_data()
def add_path(self, filename):
recent = RecentPath.create(filename, self._search_paths())
self.recent_paths.append(recent)
def set_file_list(self):
# need to define it for RecentPathsWidgetMixin
pass
def _select_active_sheet(self):
if self.sheet:
try:
sheet_list = [s[0] for s in self.sheets]
idx = sheet_list.index(self.sheet)
self.sheet_combo.setCurrentIndex(idx)
except ValueError:
# Requested sheet does not exist in this file
self.sheet = None
else:
self.sheet_combo.setCurrentIndex(0)
def _update_sheet_combo(self):
sheets = Counter()
for fn in self.current_filenames():
reader = FileFormat.get_reader(fn)
sheets.update(reader.sheets)
sheets = sorted(sheets.items(), key=lambda x: x[0])
self.sheets = [(s, s + " (" + str(n) + ")") for s, n in sheets]
if len(sheets) < 2:
self.sheet_box.hide()
self.sheet = None
else:
self.sheets.insert(0, (None, "(None)"))
self.sheet_combo.clear()
self.sheet_combo.addItems([s[1] for s in self.sheets])
self._select_active_sheet()
self.sheet_box.show()
def select_sheet(self):
self.sheet = self.sheets[self.sheet_combo.currentIndex()][0]
self.load_data()
def remove_item(self):
ri = [i.row() for i in self.lb.selectedIndexes()]
for i in sorted(ri, reverse=True):
self.recent_paths.pop(i)
self.lb.takeItem(i)
self._update_sheet_combo()
self.load_data()
def clear(self):
self.lb.clear()
while self.recent_paths:
self.recent_paths.pop()
self._update_sheet_combo()
self.load_data()
def browse_files(self, in_demos=False):
if in_demos:
start_file = get_sample_datasets_dir()
if not os.path.exists(start_file):
QtGui.QMessageBox.information(
None, "File",
"Cannot find the directory with documentation data sets")
return
else:
start_file = self.last_path() or os.path.expanduser("~/")
filenames = QtGui.QFileDialog.getOpenFileNames(
self, 'Open Multiple Data Files', start_file, self.dlg_formats)
if not filenames:
return
for f in filenames:
self.add_path(f)
self.lb.addItem(f)
self._update_sheet_combo()
self.load_data()
def current_filenames(self):
return [rp.abspath for rp in self.recent_paths]
def load_data(self):
self.closeContext()
fns = self.current_filenames()
data_list = []
fnok_list = []
for fn in fns:
reader = FileFormat.get_reader(fn)
if self.sheet in reader.sheets:
reader.select_sheet(self.sheet)
errors = []
with catch_warnings(record=True) as warnings:
try:
data_list.append(reader.read())
fnok_list.append(fn)
except Exception as ex:
errors.append("An error occurred:")
errors.append(str(ex))
#FIXME show error in the list of data
self.warning(warnings[-1].message.args[0] if warnings else '')
#code below is from concatenate widget
if data_list:
tables = data_list
domain = reduce(domain_union, (table.domain for table in tables))
tables = [
Orange.data.Table.from_table(domain, table) for table in tables
]
data = concat(tables)
source_var = Orange.data.StringVariable.make("Filename")
source_values = list(
chain(*(repeat(fn, len(table))
for fn, table in zip(fnok_list, tables))))
label_var = Orange.data.StringVariable.make("Label")
label_values = list(
chain(*(repeat(self.label, len(table))
for fn, table in zip(fnok_list, tables))))
data = append_columns(
data, **{
"metas": [(source_var, source_values),
(label_var, label_values)]
})
self.data = data
if OLD_DOMAINEDITOR:
self.editor_model.set_domain(data.domain)
else:
self.openContext(data.domain)
else:
self.data = None
if OLD_DOMAINEDITOR:
self.editor_model.reset()
else:
self.domain_editor.set_domain(None)
self.send("Data", self.data)
class OWFile(widget.OWWidget, RecentPathsWComboMixin):
name = "File"
id = "orange.widgets.data.file"
description = "Read data from an input file or network " \
"and send a data table to the output."
icon = "icons/File.svg"
priority = 10
category = "Data"
keywords = ["file", "load", "read", "open"]
class Outputs:
data = Output("Data", Table,
doc="Attribute-valued dataset read from the input file.")
want_main_area = False
SEARCH_PATHS = [("sample-datasets", get_sample_datasets_dir())]
SIZE_LIMIT = 1e7
LOCAL_FILE, URL = range(2)
settingsHandler = PerfectDomainContextHandler(
match_values=PerfectDomainContextHandler.MATCH_VALUES_ALL
)
# pylint seems to want declarations separated from definitions
recent_paths: List[RecentPath]
recent_urls: List[str]
variables: list
# Overload RecentPathsWidgetMixin.recent_paths to set defaults
recent_paths = Setting([
RecentPath("", "sample-datasets", "iris.tab"),
RecentPath("", "sample-datasets", "titanic.tab"),
RecentPath("", "sample-datasets", "housing.tab"),
RecentPath("", "sample-datasets", "heart_disease.tab"),
])
recent_urls = Setting([])
source = Setting(LOCAL_FILE)
xls_sheet = ContextSetting("")
sheet_names = Setting({})
url = Setting("")
variables = ContextSetting([])
domain_editor = SettingProvider(DomainEditor)
class Warning(widget.OWWidget.Warning):
file_too_big = widget.Msg("The file is too large to load automatically."
" Press Reload to load.")
load_warning = widget.Msg("Read warning:\n{}")
class Error(widget.OWWidget.Error):
file_not_found = widget.Msg("File not found.")
missing_reader = widget.Msg("Missing reader.")
sheet_error = widget.Msg("Error listing available sheets.")
unknown = widget.Msg("Read error:\n{}")
class NoFileSelected:
pass
def __init__(self):
super().__init__()
RecentPathsWComboMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.reader = None
layout = QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
vbox = gui.radioButtons(None, self, "source", box=True, addSpace=True,
callback=self.load_data, addToLayout=False)
rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.activated[int].connect(self.select_file)
box.layout().addWidget(self.file_combo)
layout.addWidget(box, 0, 1)
file_button = gui.button(
None, self, '...', callback=self.browse_file, autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 2)
reload_button = gui.button(
None, self, "Reload", callback=self.load_data, autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 3)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(None, self, "xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True,)
self.sheet_combo.setSizePolicy(
Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(
Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(
self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(
self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = NamedURLModel(self.sheet_names)
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 3, 3)
url_combo.activated.connect(self._url_set)
box = gui.vBox(self.controlArea, "Info")
self.infolabel = gui.widgetLabel(box, 'No data loaded.')
self.warnings = gui.widgetLabel(box, '')
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
box = gui.hBox(self.controlArea)
gui.button(
box, self, "Browse documentation datasets",
callback=lambda: self.browse_file(True), autoDefault=False)
gui.rubber(box)
gui.button(
box, self, "Reset", callback=self.reset_domain_edit)
self.apply_button = gui.button(
box, self, "Apply", callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
self.set_file_list()
# Must not call open_file from within __init__. open_file
# explicitly re-enters the event loop (by a progress bar)
self.setAcceptDrops(True)
if self.source == self.LOCAL_FILE:
last_path = self.last_path()
if last_path and os.path.exists(last_path) and \
os.path.getsize(last_path) > self.SIZE_LIMIT:
self.Warning.file_too_big()
return
QTimer.singleShot(0, self.load_data)
@staticmethod
def sizeHint():
return QSize(600, 550)
def select_file(self, n):
assert n < len(self.recent_paths)
super().select_file(n)
if self.recent_paths:
self.source = self.LOCAL_FILE
self.load_data()
self.set_file_list()
def select_sheet(self):
self.recent_paths[0].sheet = self.sheet_combo.currentText()
self.load_data()
def _url_set(self):
url = self.url_combo.currentText()
pos = self.recent_urls.index(url)
url = url.strip()
if not urlparse(url).scheme:
url = 'http://' + url
self.url_combo.setItemText(pos, url)
self.recent_urls[pos] = url
self.source = self.URL
self.load_data()
def browse_file(self, in_demos=False):
if in_demos:
start_file = get_sample_datasets_dir()
if not os.path.exists(start_file):
QMessageBox.information(
None, "File",
"Cannot find the directory with documentation datasets")
return
else:
start_file = self.last_path() or os.path.expanduser("~/")
readers = [f for f in FileFormat.formats
if getattr(f, 'read', None)
and getattr(f, "EXTENSIONS", None)]
filename, reader, _ = open_filename_dialog(start_file, None, readers)
if not filename:
return
self.add_path(filename)
if reader is not None:
self.recent_paths[0].file_format = reader.qualified_name()
self.source = self.LOCAL_FILE
self.load_data()
# Open a file, create data from it and send it over the data channel
def load_data(self):
# We need to catch any exception type since anything can happen in
# file readers
self.closeContext()
self.domain_editor.set_domain(None)
self.apply_button.setEnabled(False)
self.clear_messages()
self.set_file_list()
error = self._try_load()
if error:
error()
self.data = None
self.sheet_box.hide()
self.Outputs.data.send(None)
self.infolabel.setText("No data.")
def _try_load(self):
# pylint: disable=broad-except
if self.last_path() and not os.path.exists(self.last_path()):
return self.Error.file_not_found
try:
self.reader = self._get_reader()
assert self.reader is not None
except Exception:
return self.Error.missing_reader
if self.reader is self.NoFileSelected:
self.Outputs.data.send(None)
return None
try:
self._update_sheet_combo()
except Exception:
return self.Error.sheet_error
with catch_warnings(record=True) as warnings:
try:
data = self.reader.read()
except Exception as ex:
log.exception(ex)
return lambda x=ex: self.Error.unknown(str(x))
if warnings:
self.Warning.load_warning(warnings[-1].message.args[0])
self.infolabel.setText(self._describe(data))
self.loaded_file = self.last_path()
add_origin(data, self.loaded_file)
self.data = data
self.openContext(data.domain)
self.apply_domain_edit() # sends data
return None
def _get_reader(self) -> FileFormat:
if self.source == self.LOCAL_FILE:
path = self.last_path()
if path is None:
return self.NoFileSelected
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
reader_class = class_from_qualified_name(qname)
reader = reader_class(path)
else:
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
else:
return self.NoFileSelected
def _update_sheet_combo(self):
if len(self.reader.sheets) < 2:
self.sheet_box.hide()
self.reader.select_sheet(None)
return
self.sheet_combo.clear()
self.sheet_combo.addItems(self.reader.sheets)
self._select_active_sheet()
self.sheet_box.show()
def _select_active_sheet(self):
if self.reader.sheet:
try:
idx = self.reader.sheets.index(self.reader.sheet)
self.sheet_combo.setCurrentIndex(idx)
except ValueError:
# Requested sheet does not exist in this file
self.reader.select_sheet(None)
else:
self.sheet_combo.setCurrentIndex(0)
@staticmethod
def _describe(table):
def missing_prop(prop):
if prop:
return f"({prop * 100:.1f}% missing values)"
else:
return "(no missing values)"
domain = table.domain
text = ""
attrs = getattr(table, "attributes", {})
descs = [attrs[desc]
for desc in ("Name", "Description") if desc in attrs]
if len(descs) == 2:
descs[0] = f"<b>{descs[0]}</b>"
if descs:
text += f"<p>{'<br/>'.join(descs)}</p>"
text += f"<p>{len(table)} instance(s)"
missing_in_attr = missing_prop(table.has_missing_attribute()
and table.get_nan_frequency_attribute())
missing_in_class = missing_prop(table.has_missing_class()
and table.get_nan_frequency_class())
text += f"<br/>{len(domain.attributes)} feature(s) {missing_in_attr}"
if domain.has_continuous_class:
text += f"<br/>Regression; numerical class {missing_in_class}"
elif domain.has_discrete_class:
text += "<br/>Classification; categorical class " \
f"with {len(domain.class_var.values)} values {missing_in_class}"
elif table.domain.class_vars:
text += "<br/>Multi-target; " \
f"{len(table.domain.class_vars)} target variables " \
f"{missing_in_class}"
else:
text += "<br/>Data has no target variable."
text += f"<br/>{len(domain.metas)} meta attribute(s)"
text += "</p>"
if 'Timestamp' in table.domain:
# Google Forms uses this header to timestamp responses
text += f"<p>First entry: {table[0, 'Timestamp']}<br/>" \
f"Last entry: {table[-1, 'Timestamp']}</p>"
return text
def storeSpecificSettings(self):
self.current_context.modified_variables = self.variables[:]
def retrieveSpecificSettings(self):
if hasattr(self.current_context, "modified_variables"):
self.variables[:] = self.current_context.modified_variables
def reset_domain_edit(self):
self.domain_editor.reset_domain()
self.apply_domain_edit()
def apply_domain_edit(self):
if self.data is None:
table = None
else:
domain, cols = self.domain_editor.get_domain(self.data.domain, self.data)
if not (domain.variables or domain.metas):
table = None
else:
X, y, m = cols
table = Table.from_numpy(domain, X, y, m, self.data.W)
table.name = self.data.name
table.ids = np.array(self.data.ids)
table.attributes = getattr(self.data, 'attributes', {})
self.Outputs.data.send(table)
self.apply_button.setEnabled(False)
def get_widget_name_extension(self):
_, name = os.path.split(self.loaded_file)
return os.path.splitext(name)[0]
def send_report(self):
def get_ext_name(filename):
try:
return FileFormat.names[os.path.splitext(filename)[1]]
except KeyError:
return "unknown"
if self.data is None:
self.report_paragraph("File", "No file.")
return
if self.source == self.LOCAL_FILE:
home = os.path.expanduser("~")
if self.loaded_file.startswith(home):
# os.path.join does not like ~
name = "~" + os.path.sep + \
self.loaded_file[len(home):].lstrip("/").lstrip("\\")
else:
name = self.loaded_file
if self.sheet_combo.isVisible():
name += f" ({self.sheet_combo.currentText()})"
self.report_items("File", [("File name", name),
("Format", get_ext_name(name))])
else:
self.report_items("Data", [("Resource", self.url),
("Format", get_ext_name(self.url))])
self.report_data("Data", self.data)
@staticmethod
def dragEnterEvent(event):
"""Accept drops of valid file urls"""
urls = event.mimeData().urls()
if urls:
try:
FileFormat.get_reader(OSX_NSURL_toLocalFile(urls[0]) or
urls[0].toLocalFile())
event.acceptProposedAction()
except IOError:
pass
def dropEvent(self, event):
"""Handle file drops"""
urls = event.mimeData().urls()
if urls:
self.add_path(OSX_NSURL_toLocalFile(urls[0]) or
urls[0].toLocalFile()) # add first file
self.source = self.LOCAL_FILE
self.load_data()
def workflowEnvChanged(self, key, value, oldvalue):
"""
Function called when environment changes (e.g. while saving the scheme)
It make sure that all environment connected values are modified
(e.g. relative file paths are changed)
"""
self.update_file_list(key, value, oldvalue)
def __init__(self):
widget.OWWidget.__init__(self)
RecentPathsWidgetMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.sheets = []
self.lb = gui.listBox(self.controlArea, self, "file_idx")
layout = QtGui.QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
file_button = gui.button(None,
self,
' ...',
callback=self.browse_files,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(
QtGui.QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 0)
remove_button = gui.button(None,
self,
'Remove',
callback=self.remove_item)
clear_button = gui.button(None, self, 'Clear', callback=self.clear)
layout.addWidget(remove_button, 0, 1)
layout.addWidget(clear_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QtGui.QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 7)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(None,
self,
"xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QtGui.QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label,
QtCore.Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo,
QtCore.Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
layout.addWidget(self.sheet_box, 0, 5)
label_box = gui.hBox(None, addToLayout=False, margin=0)
label = gui.lineEdit(label_box,
self,
"label",
callback=self.set_label,
label="Label",
orientation=Qt.Horizontal)
layout.addWidget(label_box, 0, 6)
layout.setColumnStretch(3, 2)
box = gui.widgetBox(self.controlArea, "View loaded columns")
if OLD_DOMAINEDITOR:
self.domain_editor = DomainEditor(self.variables)
else:
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
for i, rp in enumerate(self.recent_paths):
self.lb.addItem(rp.abspath)
# TODO unresolved paths just disappear! Modify _relocate_recent_files
self._update_sheet_combo()
self.load_data()
class OWFile(widget.OWWidget, RecentPathsWComboMixin):
name = "File"
id = "orange.widgets.data.file"
description = "Read data from an input file or network " \
"and send a data table to the output."
icon = "icons/File.svg"
priority = 10
category = "Data"
keywords = ["file", "load", "read", "open"]
class Outputs:
data = Output("Data",
Table,
doc="Attribute-valued dataset read from the input file.")
want_main_area = False
SEARCH_PATHS = [("sample-datasets", get_sample_datasets_dir())]
SIZE_LIMIT = 1e7
LOCAL_FILE, URL = range(2)
settingsHandler = PerfectDomainContextHandler(
match_values=PerfectDomainContextHandler.MATCH_VALUES_ALL)
# pylint seems to want declarations separated from definitions
recent_paths: List[RecentPath]
recent_urls: List[str]
variables: list
# Overload RecentPathsWidgetMixin.recent_paths to set defaults
recent_paths = Setting([
RecentPath("", "sample-datasets", "iris.tab"),
RecentPath("", "sample-datasets", "titanic.tab"),
RecentPath("", "sample-datasets", "housing.tab"),
RecentPath("", "sample-datasets", "heart_disease.tab"),
RecentPath("", "sample-datasets", "brown-selected.tab"),
RecentPath("", "sample-datasets", "zoo.tab"),
])
recent_urls = Setting([])
source = Setting(LOCAL_FILE)
xls_sheet = ContextSetting("")
sheet_names = Setting({})
url = Setting("")
variables = ContextSetting([])
domain_editor = SettingProvider(DomainEditor)
class Warning(widget.OWWidget.Warning):
file_too_big = widget.Msg(
"The file is too large to load automatically."
" Press Reload to load.")
load_warning = widget.Msg("Read warning:\n{}")
class Error(widget.OWWidget.Error):
file_not_found = widget.Msg("File not found.")
missing_reader = widget.Msg("Missing reader.")
sheet_error = widget.Msg("Error listing available sheets.")
unknown = widget.Msg("Read error:\n{}")
class NoFileSelected:
pass
UserAdviceMessages = [
widget.Message(
"Use CSV File Import widget for advanced options "
"for comma-separated files", "use-csv-file-import"),
widget.Message(
"This widget loads only tabular data. Use other widgets to load "
"other data types like models, distance matrices and networks.",
"other-data-types")
]
def __init__(self):
super().__init__()
RecentPathsWComboMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.reader = None
layout = QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
vbox = gui.radioButtons(None,
self,
"source",
box=True,
addSpace=True,
callback=self.load_data,
addToLayout=False)
rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.activated[int].connect(self.select_file)
box.layout().addWidget(self.file_combo)
layout.addWidget(box, 0, 1)
file_button = gui.button(None,
self,
'...',
callback=self.browse_file,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 3)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = gui.comboBox(
None,
self,
"xls_sheet",
callback=self.select_sheet,
sendSelectedValue=True,
)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = NamedURLModel(self.sheet_names)
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(Policy.Ignored, Policy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 3, 3)
url_combo.activated.connect(self._url_set)
# whit completer we set that combo box is case sensitive when
# matching the history
completer = QCompleter()
completer.setCaseSensitivity(Qt.CaseSensitive)
url_combo.setCompleter(completer)
box = gui.vBox(self.controlArea, "Info")
self.infolabel = gui.widgetLabel(box, 'No data loaded.')
self.warnings = gui.widgetLabel(box, '')
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
box = gui.hBox(self.controlArea)
gui.button(box,
self,
"Browse documentation datasets",
callback=lambda: self.browse_file(True),
autoDefault=False)
gui.rubber(box)
gui.button(box, self, "Reset", callback=self.reset_domain_edit)
self.apply_button = gui.button(box,
self,
"Apply",
callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
self.set_file_list()
# Must not call open_file from within __init__. open_file
# explicitly re-enters the event loop (by a progress bar)
self.setAcceptDrops(True)
if self.source == self.LOCAL_FILE:
last_path = self.last_path()
if last_path and os.path.exists(last_path) and \
os.path.getsize(last_path) > self.SIZE_LIMIT:
self.Warning.file_too_big()
return
QTimer.singleShot(0, self.load_data)
@staticmethod
def sizeHint():
return QSize(600, 550)
def select_file(self, n):
assert n < len(self.recent_paths)
super().select_file(n)
if self.recent_paths:
self.source = self.LOCAL_FILE
self.load_data()
self.set_file_list()
def select_sheet(self):
self.recent_paths[0].sheet = self.sheet_combo.currentText()
self.load_data()
def _url_set(self):
url = self.url_combo.currentText()
pos = self.recent_urls.index(url)
url = url.strip()
if not urlparse(url).scheme:
url = 'http://' + url
self.url_combo.setItemText(pos, url)
self.recent_urls[pos] = url
self.source = self.URL
self.load_data()
def browse_file(self, in_demos=False):
if in_demos:
start_file = get_sample_datasets_dir()
if not os.path.exists(start_file):
QMessageBox.information(
None, "File",
"Cannot find the directory with documentation datasets")
return
else:
start_file = self.last_path() or os.path.expanduser("~/")
readers = [
f for f in FileFormat.formats
if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None)
]
filename, reader, _ = open_filename_dialog(start_file, None, readers)
if not filename:
return
self.add_path(filename)
if reader is not None:
self.recent_paths[0].file_format = reader.qualified_name()
self.source = self.LOCAL_FILE
self.load_data()
# Open a file, create data from it and send it over the data channel
def load_data(self):
# We need to catch any exception type since anything can happen in
# file readers
self.closeContext()
self.domain_editor.set_domain(None)
self.apply_button.setEnabled(False)
self.clear_messages()
self.set_file_list()
error = self._try_load()
if error:
error()
self.data = None
self.sheet_box.hide()
self.Outputs.data.send(None)
self.infolabel.setText("No data.")
def _try_load(self):
# pylint: disable=broad-except
if self.last_path() and not os.path.exists(self.last_path()):
return self.Error.file_not_found
try:
self.reader = self._get_reader()
assert self.reader is not None
except Exception:
return self.Error.missing_reader
if self.reader is self.NoFileSelected:
self.Outputs.data.send(None)
return None
try:
self._update_sheet_combo()
except Exception:
return self.Error.sheet_error
with catch_warnings(record=True) as warnings:
try:
data = self.reader.read()
except Exception as ex:
log.exception(ex)
return lambda x=ex: self.Error.unknown(str(x))
if warnings:
self.Warning.load_warning(warnings[-1].message.args[0])
self.infolabel.setText(self._describe(data))
self.loaded_file = self.last_path()
add_origin(data, self.loaded_file)
self.data = data
self.openContext(data.domain)
self.apply_domain_edit() # sends data
return None
def _get_reader(self) -> FileFormat:
if self.source == self.LOCAL_FILE:
path = self.last_path()
if path is None:
return self.NoFileSelected
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
reader_class = class_from_qualified_name(qname)
reader = reader_class(path)
else:
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
else:
return self.NoFileSelected
def _update_sheet_combo(self):
if len(self.reader.sheets) < 2:
self.sheet_box.hide()
self.reader.select_sheet(None)
return
self.sheet_combo.clear()
self.sheet_combo.addItems(self.reader.sheets)
self._select_active_sheet()
self.sheet_box.show()
def _select_active_sheet(self):
if self.reader.sheet:
try:
idx = self.reader.sheets.index(self.reader.sheet)
self.sheet_combo.setCurrentIndex(idx)
except ValueError:
# Requested sheet does not exist in this file
self.reader.select_sheet(None)
else:
self.sheet_combo.setCurrentIndex(0)
@staticmethod
def _describe(table):
def missing_prop(prop):
if prop:
return f"({prop * 100:.1f}% missing values)"
else:
return "(no missing values)"
domain = table.domain
text = ""
attrs = getattr(table, "attributes", {})
descs = [
attrs[desc] for desc in ("Name", "Description") if desc in attrs
]
if len(descs) == 2:
descs[0] = f"<b>{descs[0]}</b>"
if descs:
text += f"<p>{'<br/>'.join(descs)}</p>"
text += f"<p>{len(table)} instance(s)"
missing_in_attr = missing_prop(table.has_missing_attribute()
and table.get_nan_frequency_attribute())
missing_in_class = missing_prop(table.has_missing_class()
and table.get_nan_frequency_class())
text += f"<br/>{len(domain.attributes)} feature(s) {missing_in_attr}"
if domain.has_continuous_class:
text += f"<br/>Regression; numerical class {missing_in_class}"
elif domain.has_discrete_class:
text += "<br/>Classification; categorical class " \
f"with {len(domain.class_var.values)} values {missing_in_class}"
elif table.domain.class_vars:
text += "<br/>Multi-target; " \
f"{len(table.domain.class_vars)} target variables " \
f"{missing_in_class}"
else:
text += "<br/>Data has no target variable."
text += f"<br/>{len(domain.metas)} meta attribute(s)"
text += "</p>"
if 'Timestamp' in table.domain:
# Google Forms uses this header to timestamp responses
text += f"<p>First entry: {table[0, 'Timestamp']}<br/>" \
f"Last entry: {table[-1, 'Timestamp']}</p>"
return text
def storeSpecificSettings(self):
self.current_context.modified_variables = self.variables[:]
def retrieveSpecificSettings(self):
if hasattr(self.current_context, "modified_variables"):
self.variables[:] = self.current_context.modified_variables
def reset_domain_edit(self):
self.domain_editor.reset_domain()
self.apply_domain_edit()
def apply_domain_edit(self):
if self.data is None:
table = None
else:
domain, cols = self.domain_editor.get_domain(
self.data.domain, self.data)
if not (domain.variables or domain.metas):
table = None
else:
X, y, m = cols
table = Table.from_numpy(domain, X, y, m, self.data.W)
table.name = self.data.name
table.ids = np.array(self.data.ids)
table.attributes = getattr(self.data, 'attributes', {})
self.Outputs.data.send(table)
self.apply_button.setEnabled(False)
def get_widget_name_extension(self):
_, name = os.path.split(self.loaded_file)
return os.path.splitext(name)[0]
def send_report(self):
def get_ext_name(filename):
try:
return FileFormat.names[os.path.splitext(filename)[1]]
except KeyError:
return "unknown"
if self.data is None:
self.report_paragraph("File", "No file.")
return
if self.source == self.LOCAL_FILE:
home = os.path.expanduser("~")
if self.loaded_file.startswith(home):
# os.path.join does not like ~
name = "~" + os.path.sep + \
self.loaded_file[len(home):].lstrip("/").lstrip("\\")
else:
name = self.loaded_file
if self.sheet_combo.isVisible():
name += f" ({self.sheet_combo.currentText()})"
self.report_items("File", [("File name", name),
("Format", get_ext_name(name))])
else:
self.report_items("Data", [("Resource", self.url),
("Format", get_ext_name(self.url))])
self.report_data("Data", self.data)
@staticmethod
def dragEnterEvent(event):
"""Accept drops of valid file urls"""
urls = event.mimeData().urls()
if urls:
try:
FileFormat.get_reader(urls[0].toLocalFile())
event.acceptProposedAction()
except IOError:
pass
def dropEvent(self, event):
"""Handle file drops"""
urls = event.mimeData().urls()
if urls:
self.add_path(urls[0].toLocalFile()) # add first file
self.source = self.LOCAL_FILE
self.load_data()
def workflowEnvChanged(self, key, value, oldvalue):
"""
Function called when environment changes (e.g. while saving the scheme)
It make sure that all environment connected values are modified
(e.g. relative file paths are changed)
"""
self.update_file_list(key, value, oldvalue)
class OWMultifile(widget.OWWidget, RelocatablePathsWidgetMixin):
name = "Multifile"
id = "orangecontrib.spectroscopy.widgets.files"
icon = "icons/multifile.svg"
description = "Read data from input files " \
"and send a data table to the output."
priority = 10000
replaces = [
"orangecontrib.infrared.widgets.owfiles.OWFiles",
"orangecontrib.infrared.widgets.owmultifile.OWMultifile"
]
class Outputs:
data = Output("Data", Table, doc="Concatenated input files.")
want_main_area = False
file_idx = []
settingsHandler = PerfectDomainContextHandler(
match_values=PerfectDomainContextHandler.MATCH_VALUES_ALL)
recent_paths: List[RecentPath]
variables: list
sheet = Setting(None, schema_only=True)
label = Setting("", schema_only=True)
recent_paths = Setting([], schema_only=True)
variables = ContextSetting([], schema_only=True)
class Error(widget.OWWidget.Error):
file_not_found = widget.Msg("File(s) not found.")
missing_reader = widget.Msg("Missing reader(s).")
read_error = widget.Msg("Read error(s).")
domain_editor = SettingProvider(DomainEditor)
def __init__(self):
widget.OWWidget.__init__(self)
RelocatablePathsWidgetMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.sheets = []
self.lb = gui.listBox(self.controlArea,
self,
"file_idx",
selectionMode=QListWidget.MultiSelection)
self.default_foreground = None
layout = QGridLayout()
gui.widgetBox(self.controlArea, margin=0, orientation=layout)
file_button = gui.button(None,
self,
' ...',
callback=self.browse_files,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 0)
remove_button = gui.button(None,
self,
'Remove',
callback=self.remove_item)
clear_button = gui.button(None, self, 'Clear', callback=self.clear)
layout.addWidget(remove_button, 0, 1)
layout.addWidget(clear_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 7)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_index = 0
self.sheet_combo = gui.comboBox(None,
self,
"sheet_index",
callback=self.select_sheet)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
layout.addWidget(self.sheet_box, 0, 5)
label_box = gui.hBox(None, addToLayout=False, margin=0)
gui.lineEdit(label_box,
self,
"label",
callback=self.set_label,
label="Label",
orientation=Qt.Horizontal)
layout.addWidget(label_box, 0, 6)
layout.setColumnStretch(3, 2)
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
for rp in self.recent_paths:
self.lb.addItem(rp.abspath)
box = gui.hBox(self.controlArea)
gui.rubber(box)
if hasattr(DomainEditor, "reset_domain"): # Orange>=3.21
gui.button(box, self, "Reset", callback=self.reset_domain_edit)
self.apply_button = gui.button(box,
self,
"Apply",
callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
self._update_sheet_combo()
self.load_data()
def set_label(self):
self.load_data()
def _select_active_sheet(self):
if self.sheet:
try:
sheet_list = [s[0] for s in self.sheets]
idx = sheet_list.index(self.sheet)
self.sheet_combo.setCurrentIndex(idx)
except ValueError:
# Requested sheet does not exist in this file
self.sheet = None
else:
self.sheet_combo.setCurrentIndex(0)
def _update_sheet_combo(self):
sheets = Counter()
for rp in self.recent_paths:
try:
reader = _get_reader(rp)
sheets.update(reader.sheets)
except:
pass
sheets = sorted(sheets.items(), key=lambda x: x[0])
self.sheets = [(s, s + " (" + str(n) + ")") for s, n in sheets]
if len(sheets) < 2:
self.sheet_box.hide()
self.sheet = None
else:
self.sheets.insert(0, (None, "(None)"))
self.sheet_combo.clear()
self.sheet_combo.addItems([s[1] for s in self.sheets])
self._select_active_sheet()
self.sheet_box.show()
def select_sheet(self):
self.sheet = self.sheets[self.sheet_combo.currentIndex()][0]
self.load_data()
def remove_item(self):
ri = [i.row() for i in self.lb.selectedIndexes()]
for i in sorted(ri, reverse=True):
self.recent_paths.pop(i)
self.lb.takeItem(i)
self._update_sheet_combo()
self.load_data()
def clear(self):
self.lb.clear()
while self.recent_paths:
self.recent_paths.pop()
self._update_sheet_combo()
self.load_data()
def browse_files(self, in_demos=False):
start_file = self.last_path() or os.path.expanduser("~/")
readers = [
f for f in FileFormat.formats
if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None)
]
filenames, reader, _ = open_filename_dialog(
start_file, None, readers, dialog=QFileDialog.getOpenFileNames)
self.load_files(filenames, reader)
def load_files(self, filenames, reader):
if not filenames:
return
for f in filenames:
self.add_path(f, reader)
self.lb.addItem(f)
self._update_sheet_combo()
self.load_data()
def load_data(self):
self.closeContext()
self.Error.file_not_found.clear()
self.Error.missing_reader.clear()
self.Error.read_error.clear()
data_list = []
fnok_list = []
def show_error(li, msg):
li.setForeground(Qt.red)
li.setToolTip(msg)
empty_domain = Domain(attributes=[])
for i, rp in enumerate(self.recent_paths):
fn = rp.abspath
li = self.lb.item(i)
li.setToolTip("")
if self.default_foreground is None:
self.default_foreground = li.foreground()
li.setForeground(self.default_foreground)
if not os.path.exists(fn):
show_error(li, "File not found.")
self.Error.file_not_found()
continue
try:
reader = _get_reader(rp)
assert reader is not None
except Exception: # pylint: disable=broad-except
show_error(li, "Reader not found.")
self.Error.missing_reader()
continue
try:
if self.sheet in reader.sheets:
reader.select_sheet(self.sheet)
if isinstance(reader, SpectralFileFormat):
xs, vals, additional = reader.read_spectra()
if additional is None:
additional = Table.from_domain(empty_domain,
n_rows=len(vals))
data_list.append((xs, vals, additional))
else:
data_list.append(reader.read())
fnok_list.append(fn)
except Exception as ex: # pylint: disable=broad-except
show_error(li, "Read error:\n" + str(ex))
self.Error.read_error()
if not data_list \
or self.Error.file_not_found.is_shown() \
or self.Error.missing_reader.is_shown() \
or self.Error.read_error.is_shown():
self.data = None
self.domain_editor.set_domain(None)
else:
data = concatenate_data(data_list, fnok_list, self.label)
self.data = data
self.openContext(data.domain)
self.apply_domain_edit() # sends data
def storeSpecificSettings(self):
self.current_context.modified_variables = self.variables[:]
def retrieveSpecificSettings(self):
if hasattr(self.current_context, "modified_variables"):
self.variables[:] = self.current_context.modified_variables
def apply_domain_edit(self):
if self.data is None:
table = None
else:
domain, cols = self.domain_editor.get_domain(
self.data.domain, self.data)
if not (domain.variables or domain.metas):
table = None
else:
X, y, m = cols
table = Table.from_numpy(domain, X, y, m, self.data.W)
table.name = self.data.name
table.ids = np.array(self.data.ids)
table.attributes = getattr(self.data, 'attributes', {})
self.Outputs.data.send(table)
self.apply_button.setEnabled(False)
def reset_domain_edit(self):
self.domain_editor.reset_domain()
self.apply_domain_edit()
def send_report(self):
def get_format_name(format):
try:
return format.DESCRIPTION
except AttributeError:
return format.__class__.__name__
if self.data is None:
self.report_paragraph("File", "No file.")
return
files = []
for rp in self.recent_paths:
format = _get_reader(rp)
files.append([rp.abspath, get_format_name(format)])
self.report_table("Files", table=files)
self.report_data("Data", self.data)
def workflowEnvChanged(self, key, value, oldvalue):
"""
Function called when environment changes (e.g. while saving the scheme)
It make sure that all environment connected values are modified
(e.g. relative file paths are changed)
"""
self.update_file_list(key, value, oldvalue)
def update_file_list(self, key, value, oldvalue):
if key == "basedir":
self._relocate_recent_files()
