def test_wildcard_extension(self):
self.assertIsInstance(FileFormat.get_reader("t.wild"),
WildcardReader)
self.assertIsInstance(FileFormat.get_reader("t.wild2"),
WildcardReader)
with self.assertRaises(OSError):
FileFormat.get_reader("t.wild2a")
def test_wildcard_extension(self):
self.assertIsInstance(FileFormat.get_reader("t.wild"),
WildcardReader)
self.assertIsInstance(FileFormat.get_reader("t.wild2"),
WildcardReader)
with self.assertRaises(OSError):
FileFormat.get_reader("t.wild2a")
def test_locate_sample_datasets(self):
with self.assertRaises(OSError):
FileFormat.locate("iris.tab", search_dirs=[os.path.dirname(__file__)])
iris = FileFormat.locate("iris.tab", search_dirs=[get_sample_datasets_dir()])
self.assertEqual(os.path.basename(iris), "iris.tab")
# test extension adding
iris = FileFormat.locate("iris", search_dirs=[get_sample_datasets_dir()])
self.assertEqual(os.path.basename(iris), "iris.tab")
def dragEnterEvent(self, event):
"""Accept drops of valid file urls"""
urls = event.mimeData().urls()
if urls:
try:
FileFormat.get_reader(urls[0].toLocalFile())
event.acceptProposedAction()
except IOError:
pass
def dragEnterEvent(event):
"""Accept drops of valid file urls"""
urls = event.mimeData().urls()
if urls:
try:
FileFormat.get_reader(urls[0].toLocalFile())
event.acceptProposedAction()
except IOError:
pass
def canDropUrl(self, url: QUrl) -> bool:
if url.isLocalFile():
try:
FileFormat.get_reader(url.toLocalFile())
return True
except Exception: # noqa # pylint:disable=broad-except
return False
else:
return url.scheme().lower() in ("http", "https", "ftp")
def test_locate_sample_datasets(self):
with self.assertRaises(OSError):
FileFormat.locate("iris.tab",
search_dirs=[os.path.dirname(__file__)])
iris = FileFormat.locate("iris.tab",
search_dirs=[get_sample_datasets_dir()])
self.assertEqual(os.path.basename(iris), "iris.tab")
# test extension adding
iris = FileFormat.locate("iris",
search_dirs=[get_sample_datasets_dir()])
self.assertEqual(os.path.basename(iris), "iris.tab")
def test_locate_wildcard_extension(self):
tempdir = tempfile.mkdtemp()
with self.assertRaises(OSError):
FileFormat.locate("t.wild9", search_dirs=[tempdir])
fn = os.path.join(tempdir, "t.wild8")
with open(fn, "wt") as f:
f.write("\n")
l = FileFormat.locate("t.wild8", search_dirs=[tempdir])
self.assertEqual(l, fn)
# test extension adding
l = FileFormat.locate("t", search_dirs=[tempdir])
self.assertEqual(l, fn)
shutil.rmtree(tempdir)
def test_locate_wildcard_extension(self):
tempdir = tempfile.mkdtemp()
with self.assertRaises(OSError):
FileFormat.locate("t.wild9", search_dirs=[tempdir])
fn = os.path.join(tempdir, "t.wild8")
with open(fn, "wt") as f:
f.write("\n")
l = FileFormat.locate("t.wild8", search_dirs=[tempdir])
self.assertEqual(l, fn)
# test extension adding
l = FileFormat.locate("t", search_dirs=[tempdir])
self.assertEqual(l, fn)
shutil.rmtree(tempdir)
def run_analysis_done(self):
self.infoLabel.setText("Finished running analysis!")
self.btn_run.setEnabled(True)
self.btn_run.setText('Run again')
self.setStatusMessage('Finished!')
#self.progressBarFinished()
self.send("Results", self.host_results_dir)
# Jimmy March 29 2017 added, create a dataset for DataTable widget to show TOP-40.tsv
tsvFile = os.path.join(self.host_results_dir, 'FDR-0.1/TOP-40.tsv')
tsvReader = FileFormat.get_reader(tsvFile)
data = None
try:
data = tsvReader.read()
# TODO Jimmy March 29 2017, tried to define domain for DataTable, didn't work
'''
domain = Orange.data.Domain(["Cell", "Plate", "Gene", "logFC", "logCPM", "PValue"], data.domain)
domain = Orange.data.Domain(data.domain.attributes, data.domain.class_vars,
[Orange.data.DiscreteVariable("Cell"),
Orange.data.DiscreteVariable("Plate"),
Orange.data.DiscreteVariable("Condition1"),
Orange.data.DiscreteVariable("Condition2"),
Orange.data.StringVariable("Gene"),
Orange.data.ContinuousVariable("logFC"),
Orange.data.ContinuousVariable("logCPM"),
Orange.data.StringVariable("PValue")])
data = data.from_table(domain, data)'''
except Exception as ex:
print(ex)
self.send("Top 40", data)
def on_motion(self, event):
if self.press is None:
return
if event.inaxes not in [self.line1.axes, self.line2.axes]:
return
if event.inaxes == self.line1.axes:
'''
实现ax1到ax2的链接
'''
x0, y0, mx, my = self.press # y0表示on_press选取上的line1的坐标
y0[self.lastind] = event.ydata #上表示改变其中选取的那个点的坐标，其他坐标不变
print(x0, y0, mx, my)
self.line1.set_ydata(y0) #改变line1中个点的y坐标（实际只变动了1维）
iteration_y = np.array(self.line1.get_ydata()).reshape(1, -1)
Data_new = self.pca.transform(iteration_y) #根据line1的坐标确定line2的坐标
self.line2.set_xdata(Data_new[:, 0].reshape(-1, 1))
self.line2.set_ydata(Data_new[:, 1].reshape(-1, 1))
print('123', self.Normal.InverseNormalize(y0))
l = len(self.Normal.InverseNormalize(y0))
m = ["0"] * l
name = m + self.Normal.InverseNormalize(y0)
Mytable = FileFormat.data_table(name)
y1 = self.model(Mytable, 0) #确定选中线的颜色
print('y1', y1)
self.line1.set_color(self.scalarMap.to_rgba(y1))
self.line2.set_color(self.scalarMap.to_rgba(y1))
self.fig.canvas.draw()
def setUp(self):
self.widget = self.create_widget(OWFFT)
self.ifg_single = Orange.data.Table("IFG_single.dpt")
self.ifg_seq = Orange.data.Table("agilent/4_noimage_agg256.seq")
fn = 'NeaReaderGSF_test/NeaReaderGSF_test O2A raw.gsf'
absolute_filename = FileFormat.locate(fn, dataset_dirs)
self.ifg_gsf = NeaReaderGSF(absolute_filename).read()
def initialize_reader(reader, fn):
"""
Returns an initialized reader with the file that can be relative
to Orange's default data set directories.
"""
absolute_filename = FileFormat.locate(fn, Orange.data.table.dataset_dirs)
return reader(absolute_filename)
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
path = self.last_path()
if path is None:
return self.NoFileSelected
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
reader_class = class_from_qualified_name(qname)
reader = reader_class(path)
else:
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
else:
return self.NoFileSelected
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
path = self.last_path()
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
reader_class = class_from_qualified_name(qname)
reader = reader_class(path)
print('reader_class', reader_class)
else:
reader = FileFormat.get_reader(path)
# Return reader instance that can be used to read the file
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
elif self.source == self.URL:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
def test_single_preproc(self):
# TODO problematic interface design: should be able to use Orange.data.Table directly
path = os.path.join(get_sample_datasets_dir(), AGILENT_TILE)
reader = FileFormat.get_reader(path)
for p in PREPROCESSORS_INDEPENDENT_SAMPLES:
reader.set_preprocessor(p)
reader.read()
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
path = self.last_path()
if path is None:
return self.NoFileSelected
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
reader_class = class_from_qualified_name(qname)
reader = reader_class(path)
else:
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
else:
return self.NoFileSelected
def _get_reader(self) -> FileFormat:
if self.source == self.LOCAL_FILE:
path = self.last_path()
self.reader_combo.setEnabled(True)
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
qname_index = {
r.qualified_name(): i
for i, r in enumerate(self.available_readers)
}
if qname in qname_index:
self.reader_combo.setCurrentIndex(qname_index[qname] + 1)
else:
# reader may be accessible, but not in self.available_readers
# (perhaps its code was moved)
self.reader_combo.addItem(qname)
self.reader_combo.setCurrentIndex(
len(self.reader_combo) - 1)
try:
reader_class = class_from_qualified_name(qname)
except Exception as ex:
raise MissingReaderException(
f'Can not find reader "{qname}"') from ex
reader = reader_class(path)
else:
self.reader_combo.setCurrentIndex(0)
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
return UrlReader(url)
def main(argv=sys.argv):
from orangecontrib.spectroscopy.io.neaspec import NeaReaderGSF
from Orange.data.io import FileFormat
from Orange.data import dataset_dirs
fn = 'NeaReaderGSF_test/NeaReaderGSF_test O2A raw.gsf'
absolute_filename = FileFormat.locate(fn, dataset_dirs)
data = NeaReaderGSF(absolute_filename).read()
app = QApplication(list(argv))
filename = "IFG_single.dpt"
ow = OWFFT()
ow.show()
ow.raise_()
dataset = Orange.data.Table(filename)
dataset = data #ComplexFFT, this line can be commented
ow.set_data(dataset)
ow.handleNewSignals()
app.exec_()
ow.set_data(None)
ow.handleNewSignals()
return 0
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
path = self.last_path()
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
reader_class = class_from_qualified_name(qname)
reader = reader_class(path)
else:
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
# set preprocessor here
if hasattr(reader, "read_tile"):
reader.set_preprocessor(self.preprocessor)
if self.preprocessor is not None:
self.info_preproc.setText(
self._format_preproc_str(
self.preprocessor).lstrip("\n"))
else:
# only allow readers with tile-by-tile support to run.
reader = None
return reader
elif self.source == self.URL:
url = self.url_combo.currentText().strip()
if url:
return UrlReader(url)
def test_preprocessor_list(self):
# TODO problematic interface design: should be able to use Orange.data.Table directly
path = os.path.join(get_sample_datasets_dir(), AGILENT_TILE)
reader = FileFormat.get_reader(path)
pp = PreprocessorList(PREPROCESSORS_INDEPENDENT_SAMPLES[0:7])
reader.set_preprocessor(pp)
t = reader.read()
assert len(t.domain.attributes) == 3
def load_data(self):
self.closeContext()
fns = self.current_filenames()
data_list = []
fnok_list = []
for fn in fns:
reader = FileFormat.get_reader(fn)
if self.sheet in reader.sheets:
reader.select_sheet(self.sheet)
errors = []
with catch_warnings(record=True) as warnings:
try:
data_list.append(reader.read())
fnok_list.append(fn)
except Exception as ex:
errors.append("An error occurred:")
errors.append(str(ex))
#FIXME show error in the list of data
self.warning(warnings[-1].message.args[0] if warnings else '')
#code below is from concatenate widget
if data_list:
tables = data_list
domain = reduce(domain_union, (table.domain for table in tables))
tables = [
Orange.data.Table.from_table(domain, table) for table in tables
]
data = concat(tables)
source_var = Orange.data.StringVariable.make("Filename")
source_values = list(
chain(*(repeat(fn, len(table))
for fn, table in zip(fnok_list, tables))))
label_var = Orange.data.StringVariable.make("Label")
label_values = list(
chain(*(repeat(self.label, len(table))
for fn, table in zip(fnok_list, tables))))
data = append_columns(
data, **{
"metas": [(source_var, source_values),
(label_var, label_values)]
})
self.data = data
if OLD_DOMAINEDITOR:
self.editor_model.set_domain(data.domain)
else:
self.openContext(data.domain)
else:
self.data = None
if OLD_DOMAINEDITOR:
self.editor_model.reset()
else:
self.domain_editor.set_domain(None)
self.send("Data", self.data)
def get_writer_selected(self):
writer = FileFormat.get_reader(self.type_ext)
ext = self.type_ext + self.compress_ext
if ext not in writer.EXTENSIONS:
self.Error.unsupported_extension()
return None
writer.EXTENSIONS = [ext]
return writer
def get_writer_selected(self):
writer = FileFormat.get_reader(self.type_ext)
ext = self.type_ext + self.compress_ext
if ext not in writer.EXTENSIONS:
self.Error.unsupported_extension()
return None
writer.EXTENSIONS = [ext]
return writer
def test_read(self):
fn = 'NeaReaderGSF_test/NeaReaderGSF_test O2P raw.gsf'
absolute_filename = FileFormat.locate(fn, dataset_dirs)
data = NeaReaderGSF(absolute_filename).read()
self.assertEqual(len(data), 2)
self.assertEqual("run", data.domain.metas[2].name)
self.assertEqual("O2A", data.metas[0][3])
np.testing.assert_almost_equal(data.X[0, 0], 0.734363853931427)
self.assertEqual("O2P", data.metas[1][3])
np.testing.assert_almost_equal(data.X[1, 43], 0.17290098965168)
def load_data(self):
self.closeContext()
fns = self.current_filenames()
data_list = []
fnok_list = []
for fn in fns:
reader = FileFormat.get_reader(fn)
errors = []
with catch_warnings(record=True) as warnings:
try:
if self.sheet in reader.sheets:
reader.select_sheet(self.sheet)
data_list.append(reader.read())
fnok_list.append(fn)
except Exception as ex:
errors.append("An error occurred:")
errors.append(str(ex))
#FIXME show error in the list of data
self.warning(warnings[-1].message.args[0] if warnings else '')
#code below is from concatenate widget
if data_list:
tables = data_list
domain = reduce(domain_union, (table.domain for table in tables))
source_var = Orange.data.StringVariable.make("Filename")
label_var = Orange.data.StringVariable.make("Label")
domain = Orange.data.Domain(domain.attributes, domain.class_vars,
domain.metas + (source_var, label_var))
tables = [
Orange.data.Table.from_table(domain, table) for table in tables
]
data = type(tables[0]).concatenate(tables, axis=0)
data[:, source_var] = np.array(
list(
chain(
*(repeat(fn, len(table))
for fn, table in zip(fnok_list, tables))))).reshape(
-1, 1)
data[:, label_var] = np.array(
list(
chain(
*(repeat(self.label, len(table))
for fn, table in zip(fnok_list, tables))))).reshape(
-1, 1)
self.data = data
self.openContext(data.domain)
else:
self.data = None
self.domain_editor.set_domain(None)
self.apply_domain_edit() # sends data
def load_from_file():
filename = self.last_path()
if not filename:
return None, ""
if not os.path.exists(filename):
filename = os.path.basename(filename)
filename = FileFormat.locate(filename, "")
self.information("Loading '{}' from the current directory."
.format(filename))
reader = FileFormat.get_reader(filename)
if isinstance(reader, ExcelReader):
reader.select_sheet(self.xls_sheet)
try:
return load(lambda x: reader.read(), filename)
except Exception as exc:
self.warnings.setText(str(exc))
# Let us not remove from recent files: user may fix them
raise
def Event_OnRunFinished(self):
self.btnRun.setEnabled(True)
self.send('Results', self.output_dir_host)
tsvFile = os.path.join(self.output_dir_host, 'deseq_results.csv')
tsvReader = FileFormat.get_reader(tsvFile)
data = None
try:
data = tsvReader.read()
except Exception as ex:
print(ex)
self.send("DataTable", data)
def test_open_v2(self):
fn = "nea_test_v2.txt"
absolute_filename = FileFormat.locate(fn, dataset_dirs)
data = NeaReader(absolute_filename).read()
self.assertEqual(len(data), 12)
self.assertEqual("channel", data.domain.metas[2].name)
np.testing.assert_almost_equal(getx(data), [15., 89.])
self.assertEqual("O0A", data.metas[0][2])
np.testing.assert_almost_equal(data.X[0, 0], 92.0)
self.assertEqual("O0A", data.metas[6][2])
np.testing.assert_almost_equal(data.X[6, 0], 38.0)
def Event_OnRunFinished(self):
self.btnRun.setEnabled(True)
tsvFile = os.path.join(self.getDirectory('bamfiles'), 'bamcounts.csv')
self.send('Counts', tsvFile)
tsvReader = FileFormat.get_reader(tsvFile)
data = None
try:
data = tsvReader.read()
except Exception as ex:
print(ex)
self.send("DataTable", data)
def load_from_file():
filename = self.last_path()
if not filename:
return None, ""
if not os.path.exists(filename):
filename = os.path.basename(filename)
filename = FileFormat.locate(filename, "")
self.information(
"Loading '{}' from the current directory.".format(
filename))
reader = FileFormat.get_reader(filename)
if isinstance(reader, ExcelReader):
reader.select_sheet(self.xls_sheet)
try:
return load(lambda x: reader.read(), filename)
except Exception as exc:
self.warnings.setText(str(exc))
# Let us not remove from recent files: user may fix them
raise
def _get_reader(self):
"""
Returns
-------
FileFormat
"""
if self.source == self.LOCAL_FILE:
reader = FileFormat.get_reader(self.last_path())
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
elif self.source == self.URL:
return UrlReader(self.url_combo.currentText())
def test_read(self):
fn = 'NeaReaderGSF_test/NeaReaderGSF_test O2P raw.gsf'
absolute_filename = FileFormat.locate(fn, dataset_dirs)
data = NeaReaderGSF(absolute_filename).read()
self.assertEqual(len(data), 2)
self.assertEqual("run", data.domain.metas[2].name)
self.assertEqual("O2A", data.metas[0][3])
np.testing.assert_almost_equal(data.X[0, 0], 0.734363853931427)
self.assertEqual("O2P", data.metas[1][3])
np.testing.assert_almost_equal(data.X[1, 43], 0.17290098965168)
n_ifg = int(data.attributes['Pixel Area (X, Y, Z)'][3])
self.assertEqual(n_ifg, 1024)
self.assertEqual(n_ifg, len(data.domain.attributes))
check_attributes(data)
def _update_sheet_combo(self):
sheets = Counter()
for fn in self.current_filenames():
reader = FileFormat.get_reader(fn)
sheets.update(reader.sheets)
sheets = sorted(sheets.items(), key=lambda x: x[0])
self.sheets = [(s, s + " (" + str(n) + ")") for s, n in sheets]
if len(sheets) < 2:
self.sheet_box.hide()
self.sheet = None
else:
self.sheets.insert(0, (None, "(None)"))
self.sheet_combo.clear()
self.sheet_combo.addItems([s[1] for s in self.sheets])
self._select_active_sheet()
self.sheet_box.show()
def test_mosaic_ifg_read(self):
# This reader will only be selected manually due to shared .dmt extension
absolute_filename = FileFormat.locate("agilent/5_mosaic_agg1024.dmt",
Orange.data.table.dataset_dirs)
d = agilentMosaicIFGReader(absolute_filename).read()
self.assertEqual(len(d), 32)
self.assertEqual(len(d.domain.attributes), 311)
# Pixel sizes are 5.5 * 32 = 176.0 (binning to reduce test data)
self.assertAlmostEqual(
d[1]["map_x"] - d[0]["map_x"], 176.0)
self.assertAlmostEqual(
d[4]["map_y"] - d[3]["map_y"], 176.0)
# Last pixel should start at (4 - 1) * 176.0 = 528.0
self.assertAlmostEqual(d[-1]["map_x"], 528.0)
# 1 x 2 mosiac, (8 - 1) * 176.0 = 1232.0
self.assertAlmostEqual(d[-1]["map_y"], 1232.0)
self.assertAlmostEqual(d[21][0], 0.7116039)
self.assertAlmostEqual(d[26][0], 0.48532167)
# Metadata
self.assertEqual(d.metas[0, 2], 1.57980039e+04)
self.assertEqual(d.metas[0, 3], 4)
def load_data(self):
self.closeContext()
fns = self.current_filenames()
data_list = []
fnok_list = []
empty_domain = Orange.data.Domain(attributes=[])
for fn in fns:
reader = FileFormat.get_reader(fn)
errors = []
with catch_warnings(record=True) as warnings:
try:
if self.sheet in reader.sheets:
reader.select_sheet(self.sheet)
if isinstance(reader, SpectralFileFormat):
xs, vals, additional = reader.read_spectra()
if additional is None:
additional = Orange.data.Table.from_domain(
empty_domain, n_rows=len(vals))
data_list.append((xs, vals, additional))
else:
data_list.append(reader.read())
fnok_list.append(fn)
except Exception as ex:
errors.append("An error occurred:")
errors.append(str(ex))
#FIXME show error in the list of data
self.warning(warnings[-1].message.args[0] if warnings else '')
if data_list:
data = concatenate_data(data_list, fnok_list, self.label)
self.data = data
self.openContext(data.domain)
else:
self.data = None
self.domain_editor.set_domain(None)
self.apply_domain_edit() # sends data
def test_find_reader(self):
reader = FileFormat.get_reader("some.same_extension")
self.assertIsInstance(reader, SameExtensionPreferred)
def test_usual_extensions(self):
self.assertIsInstance(FileFormat.get_reader("t.tab"), TabReader)
self.assertIsInstance(FileFormat.get_reader("t.csv"), CSVReader)
self.assertIsInstance(FileFormat.get_reader("t.pkl"), PickleReader)
with self.assertRaises(OSError):
FileFormat.get_reader("test.undefined_extension")
def is_multisheet_excel(fn):
return len(FileFormat.get_reader(fn).sheets) > 0
def test_find_reader(self):
reader = FileFormat.get_reader("some.same_extension")
self.assertIsInstance(reader, SameExtensionPreferred)
def _get_reader(rp):
if rp.file_format:
reader_class = class_from_qualified_name(rp.file_format)
return reader_class(rp.abspath)
else:
return FileFormat.get_reader(rp.abspath)
def test_usual_extensions(self):
self.assertIsInstance(FileFormat.get_reader("t.tab"), TabReader)
self.assertIsInstance(FileFormat.get_reader("t.csv"), CSVReader)
self.assertIsInstance(FileFormat.get_reader("t.pkl"), PickleReader)
with self.assertRaises(OSError):
FileFormat.get_reader("test.undefined_extension")
