def test_round_trip_masked_table_default(tmpdir):
"""Test round-trip of MaskedColumn through HDF5 using default serialization
that writes a separate mask column. Note:
>>> simple_table(masked=True)
<Table masked=True length=3>
a b c
int64 float64 str1
----- ------- ----
-- 1.0 c
2 2.0 --
3 -- e
"""
filename = str(tmpdir.join('test.h5'))
t = simple_table(
masked=True) # int, float, and str cols with one masked element
t['c'] = [b'c', b'd', b'e']
t['c'].mask[1] = True
t.write(filename, format='hdf5', path='root', serialize_meta=True)
t2 = Table.read(filename)
assert t2.masked is False
assert t2.colnames == t.colnames
for name in t2.colnames:
assert np.all(t2[name].mask == t[name].mask)
assert np.all(t2[name] == t[name])
# Data under the mask round-trips also (unmask data to show this).
t[name].mask = False
t2[name].mask = False
assert np.all(t2[name] == t[name])
def test_no_deprecation_warning():
# regression test for #5459, where numpy deprecation warnings were
# emitted unnecessarily.
t = simple_table()
with warnings.catch_warnings(record=True) as warns:
t.info()
assert len(warns) == 0
def test_roundtrip_masked(fmt_name_class):
"""
Round trip a simple masked table through every writable format and confirm
that reading back gives the same result.
"""
fmt_name, fmt_cls = fmt_name_class
if not getattr(fmt_cls, '_io_registry_can_write', True):
return
# Skip tests for fixed_width or HTML without bs4
if ((fmt_name == 'html' and not HAS_BEAUTIFUL_SOUP)
or fmt_name == 'fixed_width'):
return
t = simple_table(masked=True)
out = StringIO()
fast = fmt_name in ascii.core.FAST_CLASSES
try:
ascii.write(t, out, format=fmt_name, fast_writer=fast)
except ImportError: # Some failed dependency, e.g. PyYAML, skip test
return
# No-header formats need to be told the column names
kwargs = {'names': t.colnames} if 'no_header' in fmt_name else {}
t2 = ascii.read(out.getvalue(), format=fmt_name, fast_reader=fast, guess=False, **kwargs)
assert t.colnames == t2.colnames
for col, col2 in zip(t.itercols(), t2.itercols()):
assert col.dtype.kind == col2.dtype.kind
assert np.all(col == col2)
def test_round_trip_masked_table_default(tmpdir):
"""Test round-trip of MaskedColumn through HDF5 using default serialization
that writes a separate mask column. Note:
>>> simple_table(masked=True)
<Table masked=True length=3>
a b c
int64 float64 str1
----- ------- ----
-- 1.0 c
2 2.0 --
3 -- e
"""
filename = str(tmpdir.join('test.h5'))
t = simple_table(masked=True) # int, float, and str cols with one masked element
t['c'] = [b'c', b'd', b'e']
t['c'].mask[1] = True
t.write(filename, format='hdf5', path='root', serialize_meta=True)
t2 = Table.read(filename)
assert t2.masked is True
assert t2.colnames == t.colnames
for name in t2.colnames:
assert np.all(t2[name].mask == t[name].mask)
assert np.all(t2[name] == t[name])
# Data under the mask round-trips also (unmask data to show this).
t[name].mask = False
t2[name].mask = False
assert np.all(t2[name] == t[name])
def test_round_trip_masked_table_serialize_mask(tmpdir, method):
"""
Same as previous test but set the serialize_method to 'data_mask' so mask is
written out and the behavior is all correct.
"""
filename = str(tmpdir.join('test.fits'))
t = simple_table(masked=True) # int, float, and str cols with one masked element
if method == 'set_cols':
for col in t.itercols():
col.info.serialize_method['fits'] = 'data_mask'
t.write(filename)
elif method == 'names':
t.write(filename, serialize_method={'a': 'data_mask', 'b': 'data_mask',
'c': 'data_mask'})
elif method == 'class':
t.write(filename, serialize_method='data_mask')
t2 = Table.read(filename)
assert t2.masked is True
assert t2.colnames == t.colnames
for name in t2.colnames:
assert np.all(t2[name].mask == t[name].mask)
assert np.all(t2[name] == t[name])
# Data under the mask round-trips also (unmask data to show this).
t[name].mask = False
t2[name].mask = False
assert np.all(t2[name] == t[name])
def test_round_trip_masked_table_serialize_mask(tmpdir):
"""Same as prev but set the serialize_method to 'data_mask' so mask is written out"""
filename = str(tmpdir.join('test.ecsv'))
t = simple_table(
masked=True) # int, float, and str cols with one masked element
t['c'][0] = '' # This would come back as masked for default "" NULL marker
# MaskedColumn with no masked elements. See table the MaskedColumnInfo class
# _represent_as_dict() method for info about we test a column with no masked elements.
t['d'] = [1, 2, 3]
t.write(filename, serialize_method='data_mask')
t2 = Table.read(filename)
assert t2.masked is False
assert t2.colnames == t.colnames
for name in t2.colnames:
assert np.all(t2[name].mask == t[name].mask)
assert np.all(t2[name] == t[name])
# Data under the mask round-trips also (unmask data to show this).
t[name].mask = False
t2[name].mask = False
assert np.all(t2[name] == t[name])
def test_roundtrip_masked(fmt_name_class):
"""
Round trip a simple masked table through every writable format and confirm
that reading back gives the same result.
"""
fmt_name, fmt_cls = fmt_name_class
if not getattr(fmt_cls, '_io_registry_can_write', True):
return
# Skip tests for fixed_width or HTML without bs4
if ((fmt_name == 'html' and not HAS_BEAUTIFUL_SOUP)
or fmt_name == 'fixed_width'):
return
t = simple_table(masked=True)
out = StringIO()
fast = fmt_name in ascii.core.FAST_CLASSES
try:
ascii.write(t, out, format=fmt_name, fast_writer=fast)
except ImportError: # Some failed dependency, e.g. PyYAML, skip test
return
# No-header formats need to be told the column names
kwargs = {'names': t.colnames} if 'no_header' in fmt_name else {}
t2 = ascii.read(out.getvalue(), format=fmt_name, fast_reader=fast, guess=False, **kwargs)
assert t.colnames == t2.colnames
for col, col2 in zip(t.itercols(), t2.itercols()):
assert col.dtype.kind == col2.dtype.kind
assert np.all(col == col2)
def test_round_trip_masked_table_default(tmpdir):
"""Test (mostly) round-trip of MaskedColumn through ECSV using default serialization
that uses an empty string "" to mark NULL values. Note:
>>> simple_table(masked=True)
<Table masked=True length=3>
a b c
int64 float64 str1
----- ------- ----
-- 1.0 c
2 2.0 --
3 -- e
"""
filename = str(tmpdir.join('test.ecsv'))
t = simple_table(masked=True) # int, float, and str cols with one masked element
t.write(filename)
t2 = Table.read(filename)
assert t2.masked is True
assert t2.colnames == t.colnames
for name in t2.colnames:
# From formal perspective the round-trip columns are the "same"
assert np.all(t2[name].mask == t[name].mask)
assert np.all(t2[name] == t[name])
# But peeking under the mask shows that the underlying data are changed
# because by default ECSV uses "" to represent masked elements.
t[name].mask = False
t2[name].mask = False
assert not np.all(t2[name] == t[name]) # Expected diff
def test_round_trip_masked_table_default(tmpdir):
"""Test (mostly) round-trip of MaskedColumn through ECSV using default serialization
that uses an empty string "" to mark NULL values. Note:
>>> simple_table(masked=True)
<Table masked=True length=3>
a b c
int64 float64 str1
----- ------- ----
-- 1.0 c
2 2.0 --
3 -- e
"""
filename = str(tmpdir.join('test.ecsv'))
t = simple_table(
masked=True) # int, float, and str cols with one masked element
t.write(filename)
t2 = Table.read(filename)
assert t2.masked is True
assert t2.colnames == t.colnames
for name in t2.colnames:
# From formal perspective the round-trip columns are the "same"
assert np.all(t2[name].mask == t[name].mask)
assert np.all(t2[name] == t[name])
# But peeking under the mask shows that the underlying data are changed
# because by default ECSV uses "" to represent masked elements.
t[name].mask = False
t2[name].mask = False
assert not np.all(t2[name] == t[name]) # Expected diff
def test_no_deprecation_warning():
# regression test for #5459, where numpy deprecation warnings were
# emitted unnecessarily.
t = simple_table()
with warnings.catch_warnings(record=True) as warns:
t.info()
assert len(warns) == 0
def test_write_overwrite(tmpdir):
t = simple_table(3, 3)
filename = os.path.join(tmpdir, 'overwrite_test.vot')
t.write(filename, format='votable')
with pytest.raises(OSError, match=_NOT_OVERWRITING_MSG_MATCH):
t.write(filename, format='votable')
t.write(filename, format='votable', overwrite=True)
def test_round_trip_masked_table_serialize_mask(tmpdir, method):
"""
Same as previous test but set the serialize_method to 'data_mask' so mask is
written out and the behavior is all correct.
"""
filename = str(tmpdir.join('test.fits'))
t = simple_table(
masked=True) # int, float, and str cols with one masked element
if method == 'set_cols':
for col in t.itercols():
col.info.serialize_method['fits'] = 'data_mask'
t.write(filename)
elif method == 'names':
t.write(filename,
serialize_method={
'a': 'data_mask',
'b': 'data_mask',
'c': 'data_mask'
})
elif method == 'class':
t.write(filename, serialize_method='data_mask')
t2 = Table.read(filename)
assert t2.masked is True
assert t2.colnames == t.colnames
for name in t2.colnames:
assert np.all(t2[name].mask == t[name].mask)
assert np.all(t2[name] == t[name])
# Data under the mask round-trips also (unmask data to show this).
t[name].mask = False
t2[name].mask = False
assert np.all(t2[name] == t[name])
def test_write_simple():
"""
Write a simple table with common types. This shows the compact version
of serialization with one line per column.
"""
t = simple_table()
out = StringIO()
t.write(out, format='ascii.ecsv')
assert out.getvalue().splitlines() == SIMPLE_LINES
def test_write_simple():
"""
Write a simple table with common types. This shows the compact version
of serialization with one line per column.
"""
t = simple_table()
out = StringIO()
t.write(out, format='ascii.ecsv')
assert out.getvalue().splitlines() == SIMPLE_LINES
def test_pickle_indexed_table(protocol):
"""
Ensure that any indices that have been added will survive pickling.
"""
t = simple_table()
t.add_index('a')
t.add_index(['a', 'b'])
ts = pickle.dumps(t)
tp = pickle.loads(ts)
assert len(t.indices) == len(tp.indices)
for index, indexp in zip(t.indices, tp.indices):
assert np.all(index.data.data == indexp.data.data)
assert index.data.data.colnames == indexp.data.data.colnames
def test_pickle_indexed_table(protocol):
"""
Ensure that any indices that have been added will survive pickling.
"""
t = simple_table()
t.add_index('a')
t.add_index(['a', 'b'])
ts = pickle.dumps(t)
tp = pickle.loads(ts)
assert len(t.indices) == len(tp.indices)
for index, indexp in zip(t.indices, tp.indices):
assert np.all(index.data.data == indexp.data.data)
assert index.data.data.colnames == indexp.data.data.colnames
def test_info_serialize_method_exception():
"""
Unit test of context manager to set info.serialize_method. Normally just
used to set this for writing a Table to file (FITS, ECSV, HDF5).
"""
t = simple_table(masked=True)
origs = deepcopy(t['a'].info.serialize_method)
try:
with serialize_method_as(t, 'test'):
assert all(t['a'].info.serialize_method[key] == 'test'
for key in t['a'].info.serialize_method)
raise ZeroDivisionError()
except ZeroDivisionError:
pass
assert t['a'].info.serialize_method == origs # dict compare
def test_info_serialize_method_exception():
"""
Unit test of context manager to set info.serialize_method. Normally just
used to set this for writing a Table to file (FITS, ECSV, HDF5).
"""
t = simple_table(masked=True)
origs = deepcopy(t['a'].info.serialize_method)
try:
with serialize_method_as(t, 'test'):
assert all(t['a'].info.serialize_method[key] == 'test'
for key in t['a'].info.serialize_method)
raise ZeroDivisionError()
except ZeroDivisionError:
pass
assert t['a'].info.serialize_method == origs # dict compare
def test_round_trip_masked_table_serialize_mask(tmpdir):
"""Same as prev but set the serialize_method to 'data_mask' so mask is written out"""
filename = str(tmpdir.join('test.ecsv'))
t = simple_table(masked=True) # int, float, and str cols with one masked element
t['c'][0] = '' # This would come back as masked for default "" NULL marker
t.write(filename, serialize_method='data_mask')
t2 = Table.read(filename)
assert t2.masked is True
assert t2.colnames == t.colnames
for name in t2.colnames:
assert np.all(t2[name].mask == t[name].mask)
assert np.all(t2[name] == t[name])
# Data under the mask round-trips also (unmask data to show this).
t[name].mask = False
t2[name].mask = False
assert np.all(t2[name] == t[name])
def test_auto_identify_ecsv(tmpdir):
tbl = simple_table()
tmpfile = str(tmpdir.join('/tmpFile.ecsv'))
tbl.write(tmpfile)
tbl2 = Table.read(tmpfile)
assert np.all(tbl == tbl2)
def setup_method(self):
self.t = simple_table(size=1, cols=4, kinds='i')
def test_align():
t = simple_table(2, kinds='iS')
assert t.pformat() == [' a b ',
'--- ---',
' 1 b',
' 2 c']
# Use column format attribute
t['a'].format = '<'
assert t.pformat() == [' a b ',
'--- ---',
'1 b',
'2 c']
# Now override column format attribute with various combinations of align
tpf = [' a b ',
'--- ---',
' 1 b ',
' 2 c ']
for align in ('^', ['^', '^'], ('^', '^')):
assert tpf == t.pformat(align=align)
assert t.pformat(align='<') == [' a b ',
'--- ---',
'1 b ',
'2 c ']
assert t.pformat(align='0=') == [' a b ',
'--- ---',
'001 00b',
'002 00c']
assert t.pformat(align=['<', '^']) == [' a b ',
'--- ---',
'1 b ',
'2 c ']
# Now use fill characters. Stress the system using a fill
# character that is the same as an align character.
t = simple_table(2, kinds='iS')
assert t.pformat(align='^^') == [' a b ',
'--- ---',
'^1^ ^b^',
'^2^ ^c^']
assert t.pformat(align='^>') == [' a b ',
'--- ---',
'^^1 ^^b',
'^^2 ^^c']
assert t.pformat(align='^<') == [' a b ',
'--- ---',
'1^^ b^^',
'2^^ c^^']
# Complicated interaction (same as narrative docs example)
t1 = Table([[1.0, 2.0], [1, 2]], names=['column1', 'column2'])
t1['column1'].format = '#^.2f'
assert t1.pformat() == ['column1 column2',
'------- -------',
'##1.00# 1',
'##2.00# 2']
assert t1.pformat(align='!<') == ['column1 column2',
'------- -------',
'1.00!!! 1!!!!!!',
'2.00!!! 2!!!!!!']
assert t1.pformat(align=[None, '!<']) == ['column1 column2',
'------- -------',
'##1.00# 1!!!!!!',
'##2.00# 2!!!!!!']
# Zero fill
t['a'].format = '+d'
assert t.pformat(align='0=') == [' a b ',
'--- ---',
'+01 00b',
'+02 00c']
with pytest.raises(ValueError):
t.pformat(align=['fail'])
with pytest.raises(TypeError):
t.pformat(align=0)
with pytest.raises(TypeError):
t.pprint(align=0)
# Make sure pprint() does not raise an exception
t.pprint()
with pytest.raises(ValueError):
t.pprint(align=['<', '<', '<'])
with pytest.raises(ValueError):
t.pprint(align='x=')
def test_align():
t = simple_table(2, kinds='iS')
assert t.pformat() == [' a b ',
'--- ---',
' 1 b',
' 2 c']
# Use column format attribute
t['a'].format = '<'
assert t.pformat() == [' a b ',
'--- ---',
'1 b',
'2 c']
# Now override column format attribute with various combinations of align
tpf = [' a b ',
'--- ---',
' 1 b ',
' 2 c ']
for align in ('^', ['^', '^'], ('^', '^')):
assert tpf == t.pformat(align=align)
assert t.pformat(align='<') == [' a b ',
'--- ---',
'1 b ',
'2 c ']
assert t.pformat(align='0=') == [' a b ',
'--- ---',
'001 00b',
'002 00c']
assert t.pformat(align=['<', '^']) == [' a b ',
'--- ---',
'1 b ',
'2 c ']
# Now use fill characters. Stress the system using a fill
# character that is the same as an align character.
t = simple_table(2, kinds='iS')
assert t.pformat(align='^^') == [' a b ',
'--- ---',
'^1^ ^b^',
'^2^ ^c^']
assert t.pformat(align='^>') == [' a b ',
'--- ---',
'^^1 ^^b',
'^^2 ^^c']
assert t.pformat(align='^<') == [' a b ',
'--- ---',
'1^^ b^^',
'2^^ c^^']
# Complicated interaction (same as narrative docs example)
t1 = Table([[1.0, 2.0], [1, 2]], names=['column1', 'column2'])
t1['column1'].format = '#^.2f'
assert t1.pformat() == ['column1 column2',
'------- -------',
'##1.00# 1',
'##2.00# 2']
assert t1.pformat(align='!<') == ['column1 column2',
'------- -------',
'1.00!!! 1!!!!!!',
'2.00!!! 2!!!!!!']
assert t1.pformat(align=[None, '!<']) == ['column1 column2',
'------- -------',
'##1.00# 1!!!!!!',
'##2.00# 2!!!!!!']
# Zero fill
t['a'].format = '+d'
assert t.pformat(align='0=') == [' a b ',
'--- ---',
'+01 00b',
'+02 00c']
with pytest.raises(ValueError):
t.pformat(align=['fail'])
with pytest.raises(TypeError):
t.pformat(align=0)
with pytest.raises(TypeError):
t.pprint(align=0)
# Make sure pprint() does not raise an exception
t.pprint()
with pytest.raises(ValueError):
t.pprint(align=['<', '<', '<'])
with pytest.raises(ValueError):
t.pprint(align='x=')
def test_auto_identify_ecsv(tmpdir):
tbl = simple_table()
tmpfile = str(tmpdir.join('/tmpFile.ecsv'))
tbl.write(tmpfile)
tbl2 = Table.read(tmpfile)
assert np.all(tbl == tbl2)