def test_wildcard_extension(self):
self.assertIsInstance(FileFormat.get_reader("t.wild"),
WildcardReader)
self.assertIsInstance(FileFormat.get_reader("t.wild2"),
WildcardReader)
with self.assertRaises(OSError):
FileFormat.get_reader("t.wild2a")
def test_wildcard_extension(self):
self.assertIsInstance(FileFormat.get_reader("t.wild"),
WildcardReader)
self.assertIsInstance(FileFormat.get_reader("t.wild2"),
WildcardReader)
with self.assertRaises(OSError):
FileFormat.get_reader("t.wild2a")
def canDropUrl(self, url: QUrl) -> bool:
if url.isLocalFile():
try:
FileFormat.get_reader(url.toLocalFile())
return True
except Exception: # noqa # pylint:disable=broad-except
return False
else:
return url.scheme().lower() in ("http", "https", "ftp")
def dragEnterEvent(self, event):
"""Accept drops of valid file urls"""
urls = event.mimeData().urls()
if urls:
try:
FileFormat.get_reader(urls[0].toLocalFile())
event.acceptProposedAction()
except IOError:
pass
def dragEnterEvent(event):
"""Accept drops of valid file urls"""
urls = event.mimeData().urls()
if urls:
try:
FileFormat.get_reader(urls[0].toLocalFile())
event.acceptProposedAction()
except IOError:
pass
def _get_reader(self) -> FileFormat:
if self.source == self.LOCAL_FILE:
path = self.last_path()
self.reader_combo.setEnabled(True)
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
qname_index = {
r.qualified_name(): i
for i, r in enumerate(self.available_readers)
}
if qname in qname_index:
self.reader_combo.setCurrentIndex(qname_index[qname] + 1)
else:
# reader may be accessible, but not in self.available_readers
# (perhaps its code was moved)
self.reader_combo.addItem(qname)
self.reader_combo.setCurrentIndex(
len(self.reader_combo) - 1)
try:
reader_class = class_from_qualified_name(qname)
except Exception as ex:
raise MissingReaderException(
f'Can not find reader "{qname}"') from ex
reader = reader_class(path)
else:
self.reader_combo.setCurrentIndex(0)
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
return UrlReader(url)
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
path = self.last_path()
if path is None:
return self.NoFileSelected
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
reader_class = class_from_qualified_name(qname)
reader = reader_class(path)
else:
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
else:
return self.NoFileSelected
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
path = self.last_path()
if path is None:
return self.NoFileSelected
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
reader_class = class_from_qualified_name(qname)
reader = reader_class(path)
else:
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
else:
return self.NoFileSelected
def run_analysis_done(self):
self.infoLabel.setText("Finished running analysis!")
self.btn_run.setEnabled(True)
self.btn_run.setText('Run again')
self.setStatusMessage('Finished!')
#self.progressBarFinished()
self.send("Results", self.host_results_dir)
# Jimmy March 29 2017 added, create a dataset for DataTable widget to show TOP-40.tsv
tsvFile = os.path.join(self.host_results_dir, 'FDR-0.1/TOP-40.tsv')
tsvReader = FileFormat.get_reader(tsvFile)
data = None
try:
data = tsvReader.read()
# TODO Jimmy March 29 2017, tried to define domain for DataTable, didn't work
'''
domain = Orange.data.Domain(["Cell", "Plate", "Gene", "logFC", "logCPM", "PValue"], data.domain)
domain = Orange.data.Domain(data.domain.attributes, data.domain.class_vars,
[Orange.data.DiscreteVariable("Cell"),
Orange.data.DiscreteVariable("Plate"),
Orange.data.DiscreteVariable("Condition1"),
Orange.data.DiscreteVariable("Condition2"),
Orange.data.StringVariable("Gene"),
Orange.data.ContinuousVariable("logFC"),
Orange.data.ContinuousVariable("logCPM"),
Orange.data.StringVariable("PValue")])
data = data.from_table(domain, data)'''
except Exception as ex:
print(ex)
self.send("Top 40", data)
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
path = self.last_path()
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
reader_class = class_from_qualified_name(qname)
reader = reader_class(path)
print('reader_class', reader_class)
else:
reader = FileFormat.get_reader(path)
# Return reader instance that can be used to read the file
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
elif self.source == self.URL:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
def test_single_preproc(self):
# TODO problematic interface design: should be able to use Orange.data.Table directly
path = os.path.join(get_sample_datasets_dir(), AGILENT_TILE)
reader = FileFormat.get_reader(path)
for p in PREPROCESSORS_INDEPENDENT_SAMPLES:
reader.set_preprocessor(p)
reader.read()
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
path = self.last_path()
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
reader_class = class_from_qualified_name(qname)
reader = reader_class(path)
else:
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
# set preprocessor here
if hasattr(reader, "read_tile"):
reader.set_preprocessor(self.preprocessor)
if self.preprocessor is not None:
self.info_preproc.setText(
self._format_preproc_str(
self.preprocessor).lstrip("\n"))
else:
# only allow readers with tile-by-tile support to run.
reader = None
return reader
elif self.source == self.URL:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
def test_preprocessor_list(self):
# TODO problematic interface design: should be able to use Orange.data.Table directly
path = os.path.join(get_sample_datasets_dir(), AGILENT_TILE)
reader = FileFormat.get_reader(path)
pp = PreprocessorList(PREPROCESSORS_INDEPENDENT_SAMPLES[0:7])
reader.set_preprocessor(pp)
t = reader.read()
assert len(t.domain.attributes) == 3
def load_data(self):
self.closeContext()
fns = self.current_filenames()
data_list = []
fnok_list = []
for fn in fns:
reader = FileFormat.get_reader(fn)
if self.sheet in reader.sheets:
reader.select_sheet(self.sheet)
errors = []
with catch_warnings(record=True) as warnings:
try:
data_list.append(reader.read())
fnok_list.append(fn)
except Exception as ex:
errors.append("An error occurred:")
errors.append(str(ex))
#FIXME show error in the list of data
self.warning(warnings[-1].message.args[0] if warnings else '')
#code below is from concatenate widget
if data_list:
tables = data_list
domain = reduce(domain_union, (table.domain for table in tables))
tables = [
Orange.data.Table.from_table(domain, table) for table in tables
]
data = concat(tables)
source_var = Orange.data.StringVariable.make("Filename")
source_values = list(
chain(*(repeat(fn, len(table))
for fn, table in zip(fnok_list, tables))))
label_var = Orange.data.StringVariable.make("Label")
label_values = list(
chain(*(repeat(self.label, len(table))
for fn, table in zip(fnok_list, tables))))
data = append_columns(
data, **{
"metas": [(source_var, source_values),
(label_var, label_values)]
})
self.data = data
if OLD_DOMAINEDITOR:
self.editor_model.set_domain(data.domain)
else:
self.openContext(data.domain)
else:
self.data = None
if OLD_DOMAINEDITOR:
self.editor_model.reset()
else:
self.domain_editor.set_domain(None)
self.send("Data", self.data)
def get_writer_selected(self):
writer = FileFormat.get_reader(self.type_ext)
ext = self.type_ext + self.compress_ext
if ext not in writer.EXTENSIONS:
self.Error.unsupported_extension()
return None
writer.EXTENSIONS = [ext]
return writer
def get_writer_selected(self):
writer = FileFormat.get_reader(self.type_ext)
ext = self.type_ext + self.compress_ext
if ext not in writer.EXTENSIONS:
self.Error.unsupported_extension()
return None
writer.EXTENSIONS = [ext]
return writer
def load_data(self):
self.closeContext()
fns = self.current_filenames()
data_list = []
fnok_list = []
for fn in fns:
reader = FileFormat.get_reader(fn)
errors = []
with catch_warnings(record=True) as warnings:
try:
if self.sheet in reader.sheets:
reader.select_sheet(self.sheet)
data_list.append(reader.read())
fnok_list.append(fn)
except Exception as ex:
errors.append("An error occurred:")
errors.append(str(ex))
#FIXME show error in the list of data
self.warning(warnings[-1].message.args[0] if warnings else '')
#code below is from concatenate widget
if data_list:
tables = data_list
domain = reduce(domain_union, (table.domain for table in tables))
source_var = Orange.data.StringVariable.make("Filename")
label_var = Orange.data.StringVariable.make("Label")
domain = Orange.data.Domain(domain.attributes, domain.class_vars,
domain.metas + (source_var, label_var))
tables = [
Orange.data.Table.from_table(domain, table) for table in tables
]
data = type(tables[0]).concatenate(tables, axis=0)
data[:, source_var] = np.array(
list(
chain(
*(repeat(fn, len(table))
for fn, table in zip(fnok_list, tables))))).reshape(
-1, 1)
data[:, label_var] = np.array(
list(
chain(
*(repeat(self.label, len(table))
for fn, table in zip(fnok_list, tables))))).reshape(
-1, 1)
self.data = data
self.openContext(data.domain)
else:
self.data = None
self.domain_editor.set_domain(None)
self.apply_domain_edit() # sends data
def Event_OnRunFinished(self):
self.btnRun.setEnabled(True)
tsvFile = os.path.join(self.getDirectory('bamfiles'), 'bamcounts.csv')
self.send('Counts', tsvFile)
tsvReader = FileFormat.get_reader(tsvFile)
data = None
try:
data = tsvReader.read()
except Exception as ex:
print(ex)
self.send("DataTable", data)
def Event_OnRunFinished(self):
self.btnRun.setEnabled(True)
self.send('Results', self.output_dir_host)
tsvFile = os.path.join(self.output_dir_host, 'deseq_results.csv')
tsvReader = FileFormat.get_reader(tsvFile)
data = None
try:
data = tsvReader.read()
except Exception as ex:
print(ex)
self.send("DataTable", data)
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
reader = FileFormat.get_reader(self.last_path())
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
elif self.source == self.URL:
return UrlReader(self.url_combo.currentText())
def _update_sheet_combo(self):
sheets = Counter()
for fn in self.current_filenames():
reader = FileFormat.get_reader(fn)
sheets.update(reader.sheets)
sheets = sorted(sheets.items(), key=lambda x: x[0])
self.sheets = [(s, s + " (" + str(n) + ")") for s, n in sheets]
if len(sheets) < 2:
self.sheet_box.hide()
self.sheet = None
else:
self.sheets.insert(0, (None, "(None)"))
self.sheet_combo.clear()
self.sheet_combo.addItems([s[1] for s in self.sheets])
self._select_active_sheet()
self.sheet_box.show()
def load_from_file():
filename = self.last_path()
if not filename:
return None, ""
if not os.path.exists(filename):
filename = os.path.basename(filename)
filename = FileFormat.locate(filename, "")
self.information("Loading '{}' from the current directory."
.format(filename))
reader = FileFormat.get_reader(filename)
if isinstance(reader, ExcelReader):
reader.select_sheet(self.xls_sheet)
try:
return load(lambda x: reader.read(), filename)
except Exception as exc:
self.warnings.setText(str(exc))
# Let us not remove from recent files: user may fix them
raise
def load_from_file():
filename = self.last_path()
if not filename:
return None, ""
if not os.path.exists(filename):
filename = os.path.basename(filename)
filename = FileFormat.locate(filename, "")
self.information(
"Loading '{}' from the current directory.".format(
filename))
reader = FileFormat.get_reader(filename)
if isinstance(reader, ExcelReader):
reader.select_sheet(self.xls_sheet)
try:
return load(lambda x: reader.read(), filename)
except Exception as exc:
self.warnings.setText(str(exc))
# Let us not remove from recent files: user may fix them
raise
def load_data(self):
self.closeContext()
fns = self.current_filenames()
data_list = []
fnok_list = []
empty_domain = Orange.data.Domain(attributes=[])
for fn in fns:
reader = FileFormat.get_reader(fn)
errors = []
with catch_warnings(record=True) as warnings:
try:
if self.sheet in reader.sheets:
reader.select_sheet(self.sheet)
if isinstance(reader, SpectralFileFormat):
xs, vals, additional = reader.read_spectra()
if additional is None:
additional = Orange.data.Table.from_domain(
empty_domain, n_rows=len(vals))
data_list.append((xs, vals, additional))
else:
data_list.append(reader.read())
fnok_list.append(fn)
except Exception as ex:
errors.append("An error occurred:")
errors.append(str(ex))
#FIXME show error in the list of data
self.warning(warnings[-1].message.args[0] if warnings else '')
if data_list:
data = concatenate_data(data_list, fnok_list, self.label)
self.data = data
self.openContext(data.domain)
else:
self.data = None
self.domain_editor.set_domain(None)
self.apply_domain_edit() # sends data
def _get_reader(rp):
if rp.file_format:
reader_class = class_from_qualified_name(rp.file_format)
return reader_class(rp.abspath)
else:
return FileFormat.get_reader(rp.abspath)
def test_find_reader(self):
reader = FileFormat.get_reader("some.same_extension")
self.assertIsInstance(reader, SameExtensionPreferred)
def _get_reader(rp):
if rp.file_format:
reader_class = class_from_qualified_name(rp.file_format)
return reader_class(rp.abspath)
else:
return FileFormat.get_reader(rp.abspath)
def test_usual_extensions(self):
self.assertIsInstance(FileFormat.get_reader("t.tab"), TabReader)
self.assertIsInstance(FileFormat.get_reader("t.csv"), CSVReader)
self.assertIsInstance(FileFormat.get_reader("t.pkl"), PickleReader)
with self.assertRaises(OSError):
FileFormat.get_reader("test.undefined_extension")
def is_multisheet_excel(fn):
return len(FileFormat.get_reader(fn).sheets) > 0
def is_multisheet_excel(fn):
return len(FileFormat.get_reader(fn).sheets) > 0
def test_tile_reader(self):
# Can be removed once the get_reader interface is no logner required.
path = os.path.join(get_sample_datasets_dir(), AGILENT_TILE)
reader = FileFormat.get_reader(path)
reader.read()
def test_usual_extensions(self):
self.assertIsInstance(FileFormat.get_reader("t.tab"), TabReader)
self.assertIsInstance(FileFormat.get_reader("t.csv"), CSVReader)
self.assertIsInstance(FileFormat.get_reader("t.pkl"), PickleReader)
with self.assertRaises(OSError):
FileFormat.get_reader("test.undefined_extension")
def test_find_reader(self):
reader = FileFormat.get_reader("some.same_extension")
self.assertIsInstance(reader, SameExtensionPreferred)
