def test_read_and_save_attributes(self):
samplefile = """\
Feature 1\tFeature 2\tClass 1\tClass 42
d \tM F \td \td
\ta=1 b=2 \tclass x=a\\ longer\\ string \tclass
1.0 \tM \t5 \trich
"""
file = io.StringIO(samplefile)
table = read_tab_file(file)
f1, f2, c1, c2 = table.domain.variables
self.assertIsInstance(f2, DiscreteVariable)
self.assertEqual(f2.name, "Feature 2")
self.assertEqual(f2.attributes, {'a': 1, 'b': 2})
self.assertIn(c1, table.domain.class_vars)
self.assertIsInstance(c1, DiscreteVariable)
self.assertEqual(c1.name, "Class 1")
self.assertEqual(c1.attributes, {'x': 'a longer string'})
outf = io.StringIO()
outf.close = lambda: None
TabReader.write_file(outf, table)
saved = outf.getvalue()
file = io.StringIO(saved)
table = read_tab_file(file)
f1, f2, c1, c2 = table.domain.variables
self.assertIsInstance(f2, DiscreteVariable)
self.assertEqual(f2.name, "Feature 2")
self.assertEqual(f2.attributes, {'a': 1, 'b': 2})
self.assertIn(c1, table.domain.class_vars)
self.assertIsInstance(c1, DiscreteVariable)
self.assertEqual(c1.name, "Class 1")
self.assertEqual(c1.attributes, {'x': 'a longer string'})
def test_read_and_save_attributes(self):
samplefile = """\
Feature 1\tFeature 2\tClass 1\tClass 42
d \tM F \td \td
\ta=1 b=2 \tclass x=a\\ longer\\ string \tclass
1.0 \tM \t5 \trich
"""
file = io.StringIO(samplefile)
table = read_tab_file(file)
f1, f2, c1, c2 = table.domain.variables
self.assertIsInstance(f2, DiscreteVariable)
self.assertEqual(f2.name, "Feature 2")
self.assertEqual(f2.attributes, {'a': 1, 'b': 2})
self.assertIn(c1, table.domain.class_vars)
self.assertIsInstance(c1, DiscreteVariable)
self.assertEqual(c1.name, "Class 1")
self.assertEqual(c1.attributes, {'x': 'a longer string'})
outf = io.StringIO()
outf.close = lambda: None
TabReader.write_file(outf, table)
saved = outf.getvalue()
file = io.StringIO(saved)
table = read_tab_file(file)
f1, f2, c1, c2 = table.domain.variables
self.assertIsInstance(f2, DiscreteVariable)
self.assertEqual(f2.name, "Feature 2")
self.assertEqual(f2.attributes, {'a': 1, 'b': 2})
self.assertIn(c1, table.domain.class_vars)
self.assertIsInstance(c1, DiscreteVariable)
self.assertEqual(c1.name, "Class 1")
self.assertEqual(c1.attributes, {'x': 'a longer string'})
def test_no_metadata(self):
tempdir = tempfile.mkdtemp()
table = Table("titanic")
table.attributes = OrderedDict()
fname = path.join(tempdir, "out.tab")
TabReader.write_table_metadata(fname, table)
self.assertFalse(path.isfile(fname + ".metadata"))
shutil.rmtree(tempdir)
def test_no_metadata(self):
tempdir = tempfile.mkdtemp()
try:
self.data.attributes = OrderedDict()
fname = path.join(tempdir, "out.tab")
TabReader.write_table_metadata(fname, self.data)
self.assertFalse(path.isfile(fname + ".metadata"))
finally:
shutil.rmtree(tempdir)
def test_no_metadata(self):
tempdir = tempfile.mkdtemp()
try:
self.data.attributes = OrderedDict()
fname = path.join(tempdir, "out.tab")
TabReader.write_table_metadata(fname, self.data)
self.assertFalse(path.isfile(fname + ".metadata"))
finally:
shutil.rmtree(tempdir)
def test_had_metadata_now_there_is_none(self):
tempdir = tempfile.mkdtemp()
try:
self.data.attributes["a"] = "aa"
fname = path.join(tempdir, "out.tab")
TabReader.write_table_metadata(fname, self.data)
self.assertTrue(path.isfile(fname + ".metadata"))
del self.data.attributes["a"]
TabReader.write_table_metadata(fname, self.data)
self.assertFalse(path.isfile(fname + ".metadata"))
finally:
shutil.rmtree(tempdir)
def test_metadata(self):
tempdir = tempfile.mkdtemp()
try:
table = Table("titanic")
table.attributes = OrderedDict()
table.attributes["a"] = "aa"
table.attributes["b"] = "bb"
fname = path.join(tempdir, "out.tab")
TabReader.write_table_metadata(fname, table)
self.assertTrue(path.isfile(fname + ".metadata"))
finally:
shutil.rmtree(tempdir)
def test_metadata(self):
tempdir = tempfile.mkdtemp()
try:
table = Table("titanic")
table.attributes = OrderedDict()
table.attributes["a"] = "aa"
table.attributes["b"] = "bb"
fname = path.join(tempdir, "out.tab")
TabReader.write_table_metadata(fname, table)
self.assertTrue(path.isfile(fname + ".metadata"))
finally:
shutil.rmtree(tempdir)
def test_had_metadata_now_there_is_none(self):
tempdir = tempfile.mkdtemp()
try:
self.data.attributes["a"] = "aa"
fname = path.join(tempdir, "out.tab")
TabReader.write_table_metadata(fname, self.data)
self.assertTrue(path.isfile(fname + ".metadata"))
del self.data.attributes["a"]
TabReader.write_table_metadata(fname, self.data)
self.assertFalse(path.isfile(fname + ".metadata"))
finally:
shutil.rmtree(tempdir)
def table_from_html(self, html):
soup = BeautifulSoup(html, 'html.parser')
try:
html_table = soup.find_all('table')[-1]
except IndexError:
raise DataEmptyError
if '<h2>Anal' in html or 'div_analiza_' in html:
raise DataIsAnalError
def _header_row_strings(row):
return chain.from_iterable(
repeat(th.get_text(), int(th.get('colspan') or 1)) for th in
html_table.select('thead tr:nth-of-type(%d) th[title]' % row))
# self.DATETIME_VAR (available when Paradata is enabled in 1ka UI)
# should match this variable name format
header = [
th1.rstrip(':') +
('' if th3 == th1 else ' ({})').format(th3.rstrip(':'))
for th1, th3 in zip(_header_row_strings(1), _header_row_strings(3))
]
values = [
[
( # If no span, feature is a number or a text field
td.get_text() if td.span is None else
# If have span, it's a number, but if negative, replace with NaN
'' if td.contents[0].strip().startswith('-') else
# Else if span, the number is its code, but we want its value
td.span.get_text()[1:-1]) for td in tr.select('td')
if 'data_uid' not in td.get('class', ())
] for tr in html_table.select('tbody tr')
]
# Save parsed values into in-mem file for default values processing
buffer = StringIO()
writer = csv.writer(buffer, delimiter='\t')
writer.writerow(header)
writer.writerows(values)
buffer.flush()
buffer.seek(0)
data = TabReader(buffer).read()
title = soup.select('body h2:nth-of-type(1)')[0].get_text().split(
': ', maxsplit=1)[-1]
data.name = title
return data
def test_read_and_save_attributes(self):
samplefile = """\
Feature 1\tFeature 2\tClass 1\tClass 42
d \tM F \td \td
\ta=1 b=2 \tclass x=a\\ longer\\ string \tclass
1.0 \tM \t5 \trich
"""
file = io.StringIO(samplefile)
table = read_tab_file(file)
f1, f2, c1, c2 = table.domain.variables
self.assertIsInstance(f2, DiscreteVariable)
self.assertEqual(f2.name, "Feature 2")
self.assertEqual(f2.attributes, {"a": 1, "b": 2})
self.assertIn(c1, table.domain.class_vars)
self.assertIsInstance(c1, DiscreteVariable)
self.assertEqual(c1.name, "Class 1")
self.assertEqual(c1.attributes, {"x": "a longer string"})
outf = io.StringIO()
outf.close = lambda: None
TabReader.write_file(outf, table)
saved = outf.getvalue()
file = io.StringIO(saved)
table = read_tab_file(file)
f1, f2, c1, c2 = table.domain.variables
self.assertIsInstance(f2, DiscreteVariable)
self.assertEqual(f2.name, "Feature 2")
self.assertEqual(f2.attributes, {"a": 1, "b": 2})
self.assertIn(c1, table.domain.class_vars)
self.assertIsInstance(c1, DiscreteVariable)
self.assertEqual(c1.name, "Class 1")
self.assertEqual(c1.attributes, {"x": "a longer string"})
path = "/path/to/somewhere"
c1.attributes["path"] = path
outf = io.StringIO()
outf.close = lambda: None
TabReader.write_file(outf, table)
outf.seek(0)
table = read_tab_file(outf)
f1, f2, c1, c2 = table.domain.variables
self.assertEqual(c1.attributes["path"], path)
def test_many_discrete(self):
b = io.StringIO()
b.write("Poser\nd\n\n")
b.writelines("K" + str(i) + "\n" for i in range(30000))
start = time.time()
_ = TabReader(b).read()
elapsed = time.time() - start
if elapsed > 2:
raise AssertionError()
def table_from_html(self, html):
soup = BeautifulSoup(html, 'html.parser')
try:
html_table = soup.find_all('table')[-1]
except IndexError:
raise DataEmptyError
if '<h2>Anal' in html or 'div_analiza_' in html:
raise DataIsAnalError
def _header_row_strings(row):
return chain.from_iterable(
repeat(th.get_text(), int(th.get('colspan') or 1))
for th in html_table.select('thead tr:nth-of-type(%d) th[title]' % row))
# self.DATETIME_VAR (available when Paradata is enabled in 1ka UI)
# should match this variable name format
header = [th1.rstrip(':') + ('' if th3 == th1 else ' ({})').format(th3.rstrip(':'))
for th1, th3 in zip(_header_row_strings(1),
_header_row_strings(3))]
values = [[(# If no span, feature is a number or a text field
td.get_text() if td.span is None else
# If have span, it's a number, but if negative, replace with NaN
'' if td.contents[0].strip().startswith('-') else
# Else if span, the number is its code, but we want its value
td.span.get_text()[1:-1])
for td in tr.select('td')
if 'data_uid' not in td.get('class', ())]
for tr in html_table.select('tbody tr')]
# Save parsed values into in-mem file for default values processing
buffer = StringIO()
writer = csv.writer(buffer, delimiter='\t')
writer.writerow(header)
writer.writerows(values)
buffer.flush()
buffer.seek(0)
data = TabReader(buffer).read()
title = soup.select('body h2:nth-of-type(1)')[0].get_text().split(': ', maxsplit=1)[-1]
data.name = title
return data
def test_bad_data(self):
"""
Firstly it creates predictions with TreeLearner. Then sends predictions and
different data with different domain to Predictions widget. Those different
data and domain are similar to original data and domain but they have three
different target values instead of two.
GH-2129
"""
Variable._clear_all_caches()
filestr1 = """\
age\tsex\tsurvived
d\td\td
\t\tclass
adult\tmale\tyes
adult\tfemale\tno
child\tmale\tyes
child\tfemale\tyes
"""
file1 = io.StringIO(filestr1)
table = TabReader(file1).read()
learner = TreeLearner()
tree = learner(table)
filestr2 = """\
age\tsex\tsurvived
d\td\td
\t\tclass
adult\tmale\tyes
adult\tfemale\tno
child\tmale\tyes
child\tfemale\tunknown
"""
file2 = io.StringIO(filestr2)
bad_table = TabReader(file2).read()
self.send_signal(self.widget.Inputs.predictors, tree, 1)
with excepthook_catch():
self.send_signal(self.widget.Inputs.data, bad_table)
Variable._clear_all_caches(
) # so that test excepting standard titanic work
def test_read_save_quoted(self):
quoted = '''\
S\tA
s\td
m\t
"""a"""\ti
"""b"""\tj
"""c\td"""\tk
'''
expected = ['"a"', '"b"', '"c\td"']
f = io.StringIO(quoted)
table = read_tab_file(f)
self.assertSequenceEqual(table.metas[:, 0].tolist(), expected)
f = io.StringIO()
f.close = lambda: None
TabReader.write_file(f, table)
saved = f.getvalue()
table1 = read_tab_file(io.StringIO(saved))
self.assertSequenceEqual(table1.metas[:, 0].tolist(), expected)
def test_read_save_quoted(self):
quoted = '''\
S\tA
s\td
m\t
"""a"""\ti
"""b"""\tj
"""c\td"""\tk
'''
expected = ['"a"', '"b"', '"c\td"']
f = io.StringIO(quoted)
table = read_tab_file(f)
self.assertSequenceEqual(table.metas[:, 0].tolist(), expected)
f = io.StringIO()
f.close = lambda: None
TabReader.write_file(f, table)
saved = f.getvalue()
table1 = read_tab_file(io.StringIO(saved))
self.assertSequenceEqual(table1.metas[:, 0].tolist(), expected)
def test_sheets(self):
file1 = io.StringIO("\n".join("xd dbac"))
reader = TabReader(file1)
self.assertEqual(reader.sheets, ())
def read_tab_file(filename):
return TabReader(filename).read()
def test_data_name(self):
table1 = Table('iris')
table2 = TabReader(table1.__file__).read()
self.assertEqual(table1.name, 'iris')
self.assertEqual(table2.name, 'iris')