def _load_votable(self, params):
if 'url' in params.field_results:
buff = CStringIO(curlrequest.urlopen(
params.field_results['url']).read())
return parse(buff, filename=params.filename)
else:
return parse(params.filename)
def test_pedantic_false(self):
with catch_warnings(VOWarning, AstropyDeprecationWarning) as w:
parse(get_pkg_data_filename('data/gemini.xml'), pedantic=False)
assert len(w) == 25
# Make sure we don't yet emit a deprecation warning
assert not any(
isinstance(x.category, AstropyDeprecationWarning) for x in w)
def test_timesys_roundtrip():
orig_votable = parse(get_pkg_data_filename('data/timesys.xml'))
bio = io.BytesIO()
orig_votable.to_xml(bio)
bio.seek(0)
votable = parse(bio)
_timesys_tests(votable)
def test_gzip_filehandles(tmpdir):
votable = parse(get_pkg_data_filename('data/regression.xml'))
with open(str(tmpdir.join("regression.compressed.xml")), 'wb') as fd:
votable.to_xml(fd, compressed=True, _astropy_version="testing")
with open(str(tmpdir.join("regression.compressed.xml")), 'rb') as fd:
votable = parse(fd)
def test_gzip_filehandles(tmpdir):
votable = parse(get_pkg_data_filename('data/regression.xml'))
# W39: Bit values can not be masked
with pytest.warns(W39):
with open(str(tmpdir.join("regression.compressed.xml")), 'wb') as fd:
votable.to_xml(fd, compressed=True, _astropy_version="testing")
with open(str(tmpdir.join("regression.compressed.xml")), 'rb') as fd:
votable = parse(fd)
def setup_class(self):
votable = parse(get_pkg_data_filename('data/regression.xml'))
self.xmlout = bio = io.BytesIO()
votable.to_xml(bio)
bio.seek(0)
self.votable = parse(bio)
self.table = self.votable.get_first_table()
self.array = self.table.array
self.mask = self.table.array.mask
def test_conf_pedantic_true(self, tmpdir):
with set_temp_config(tmpdir.strpath):
with open(tmpdir.join('astropy').join('astropy.cfg').strpath, 'w') as f:
f.write('[io.votable]\npedantic = True')
reload_config('astropy.io.votable')
with pytest.raises(VOWarning):
parse(get_pkg_data_filename('data/gemini.xml'))
def _load_votable(self, params):
if 'url' in params.field_results:
try:
buff = io.StringIO(
request.urlopen(params.field_results['url']).read())
except TypeError:
buff = io.BytesIO(
request.urlopen(params.field_results['url']).read())
return parse(buff, filename=params.filename)
else:
return parse(params.filename)
def setup_class(self):
votable = parse(
get_pkg_data_filename('data/regression.xml'),
pedantic=False)
self.xmlout = bio = io.BytesIO()
votable.to_xml(bio)
bio.seek(0)
self.votable = parse(bio, pedantic=False)
self.table = self.votable.get_first_table()
self.array = self.table.array
self.mask = self.table.array.mask
def setup_class(self):
votable = parse(get_pkg_data_filename('data/regression.xml'))
self.xmlout = bio = io.BytesIO()
# W39: Bit values can not be masked
with pytest.warns(W39):
votable.to_xml(bio)
bio.seek(0)
self.votable = parse(bio)
self.table = self.votable.get_first_table()
self.array = self.table.array
self.mask = self.table.array.mask
def test_gzip_filehandles(tmpdir):
with np.errstate(over="ignore"):
# https://github.com/astropy/astropy/issues/13341
votable = parse(get_pkg_data_filename('data/regression.xml'))
# W39: Bit values can not be masked
with pytest.warns(W39):
with open(str(tmpdir.join("regression.compressed.xml")), 'wb') as fd:
votable.to_xml(fd, compressed=True, _astropy_version="testing")
with open(str(tmpdir.join("regression.compressed.xml")), 'rb') as fd:
votable = parse(fd)
def setup_class(self):
votable = parse(get_pkg_data_filename('data/regression.xml'),
pedantic=False)
votable.get_first_table().format = 'binary'
self.xmlout = bio = io.BytesIO()
votable.to_xml(bio)
bio.seek(0)
self.votable = parse(bio, pedantic=False)
self.table = self.votable.get_first_table()
self.array = self.table.array
self.mask = self.table.array.mask
def test_nonstandard_units():
from astropy import units as u
votable = parse(get_pkg_data_filename('data/nonstandard_units.xml'))
assert isinstance(
votable.get_first_table().fields[0].unit, u.UnrecognizedUnit)
votable = parse(get_pkg_data_filename('data/nonstandard_units.xml'),
unit_format='generic')
assert not isinstance(
votable.get_first_table().fields[0].unit, u.UnrecognizedUnit)
def test_conf_pedantic_false(self, tmpdir):
with set_temp_config(tmpdir.strpath):
with open(tmpdir.join('astropy').join('astropy.cfg').strpath, 'w') as f:
f.write('[io.votable]\npedantic = False')
reload_config('astropy.io.votable')
with catch_warnings(VOWarning, AstropyDeprecationWarning) as w:
parse(get_pkg_data_filename('data/gemini.xml'))
assert len(w) == 25
# Make sure we don't yet emit a deprecation warning
assert not any(isinstance(x.category, AstropyDeprecationWarning) for x in w)
def setup_class(self):
votable = parse(get_pkg_data_filename('data/regression.xml'))
votable.version = '1.3'
votable.get_first_table()._config['version_1_3_or_later'] = True
votable.get_first_table().format = 'binary2'
self.xmlout = bio = io.BytesIO()
votable.to_xml(bio)
bio.seek(0)
self.votable = parse(bio)
self.table = self.votable.get_first_table()
self.array = self.table.array
self.mask = self.table.array.mask
def setup_class(self):
with np.errstate(over="ignore"):
# https://github.com/astropy/astropy/issues/13341
votable = parse(get_pkg_data_filename('data/regression.xml'))
self.xmlout = bio = io.BytesIO()
# W39: Bit values can not be masked
with pytest.warns(W39):
votable.to_xml(bio)
bio.seek(0)
self.votable = parse(bio)
self.table = self.votable.get_first_table()
self.array = self.table.array
self.mask = self.table.array.mask
def test_gzip_filehandles(tmpdir):
votable = parse(
get_pkg_data_filename('data/regression.xml'),
pedantic=False)
with open(str(tmpdir.join("regression.compressed.xml")), 'wb') as fd:
votable.to_xml(
fd,
compressed=True,
_astropy_version="testing")
with open(str(tmpdir.join("regression.compressed.xml")), 'rb') as fd:
votable = parse(
fd,
pedantic=False)
def test_custom_datatype():
votable = parse(get_pkg_data_filename('data/custom_datatype.xml'),
pedantic=False,
datatype_mapping={'bar': 'int'})
table = votable.get_first_table()
assert table.array.dtype['foo'] == np.int32
def setup_class(self):
with np.errstate(over="ignore"):
# https://github.com/astropy/astropy/issues/13341
self.votable = parse(get_pkg_data_filename('data/regression.xml'))
self.table = self.votable.get_first_table()
self.array = self.table.array
self.mask = self.table.array.mask
def setup_class(self):
votable = parse(
get_pkg_data_filename('data/regression.xml'),
pedantic=False)
votable.version = '1.3'
votable.get_first_table()._config['version_1_3_or_later'] = True
votable.get_first_table().format = 'binary2'
self.xmlout = bio = io.BytesIO()
votable.to_xml(bio)
bio.seek(0)
self.votable = parse(bio, pedantic=False)
self.table = self.votable.get_first_table()
self.array = self.table.array
self.mask = self.table.array.mask
def setup_class(self):
self.votable = parse(
get_pkg_data_filename('data/regression.xml'),
pedantic=False)
self.table = self.votable.get_first_table()
self.array = self.table.array
self.mask = self.table.array.mask
def _vo_service_request(url, pedantic, kwargs, cache=True, verbose=False):
"""
This is called by :func:`call_vo_service`.
Raises
------
InvalidAccessURL
Invalid access URL.
"""
if len(kwargs) and not url.endswith(('?', '&')):
raise InvalidAccessURL("url should already end with '?' or '&'")
query = []
for key, value in six.iteritems(kwargs):
query.append('{0}={1}'.format(urllib.parse.quote(key),
urllib.parse.quote_plus(str(value))))
parsed_url = url + '&'.join(query)
with get_readable_fileobj(parsed_url,
encoding='binary',
cache=cache,
show_progress=verbose) as req:
tab = table.parse(req, filename=parsed_url, pedantic=pedantic)
return vo_tab_parse(tab, url, kwargs)
def test_unit_format():
data = parse(get_pkg_data_filename('data/irsa-nph-error.xml'))
assert data._config['version'] == '1.0'
assert tree._get_default_unit_format(data._config) == 'cds'
data = parse(get_pkg_data_filename('data/names.xml'))
assert data._config['version'] == '1.1'
assert tree._get_default_unit_format(data._config) == 'cds'
data = parse(get_pkg_data_filename('data/gemini.xml'))
assert data._config['version'] == '1.2'
assert tree._get_default_unit_format(data._config) == 'cds'
data = parse(get_pkg_data_filename('data/binary2_masked_strings.xml'))
assert data._config['version'] == '1.3'
assert tree._get_default_unit_format(data._config) == 'cds'
data = parse(get_pkg_data_filename('data/timesys.xml'))
assert data._config['version'] == '1.4'
assert tree._get_default_unit_format(data._config) == 'vounit'
def _parse_result(self, response, url, pars={}, verbose=False):
"""
Parse the raw HTTP response and return it as a table.
"""
# Suppress any VOTable related warnings.
if not verbose:
commons.suppress_vo_warnings()
query = []
for key, value in pars.items():
query.append('{0}={1}'.format(urllib.parse.quote(key),
urllib.parse.quote_plus(str(value))))
parsed_url = url + '&'.join(query)
# Parse the result
pedantic = 'exception' if conf.pedantic else 'warn'
tab = table.parse(BytesIO(response.content),
filename=parsed_url,
verify=pedantic)
try:
result = vo_tab_parse(tab, url, pars)
except VOSError as exc:
result = None
warnings.warn(str(exc), NoResultsWarning)
return result
def validate_vo(self):
path = self.get_vo_xml_path()
if not os.path.exists(path):
self.download_xml_content()
self['version'] = ''
if 'network_error' in self and self['network_error'] is not None:
self['nwarnings'] = 0
self['nexceptions'] = 0
self['warnings'] = []
self['xmllint'] = None
self['warning_types'] = set()
return
nexceptions = 0
nwarnings = 0
t = None
lines = []
with open(path, 'rb') as input:
with warnings.catch_warnings(record=True) as warning_lines:
try:
t = table.parse(input, verify='warn', filename=path)
except (ValueError, TypeError, ExpatError) as e:
lines.append(str(e))
nexceptions += 1
lines = [str(x.message) for x in warning_lines] + lines
if t is not None:
self['version'] = version = t.version
else:
self['version'] = version = "1.0"
if 'xmllint' not in self:
# Now check the VO schema based on the version in
# the file.
try:
success, stdout, stderr = xmlutil.validate_schema(
path, version)
# OSError is raised when XML file eats all memory and
# system sends kill signal.
except OSError as e:
self['xmllint'] = None
self['xmllint_content'] = str(e)
else:
self['xmllint'] = (success == 0)
self['xmllint_content'] = stderr
warning_types = set()
for line in lines:
w = exceptions.parse_vowarning(line)
if w['is_warning']:
nwarnings += 1
if w['is_exception']:
nexceptions += 1
warning_types.add(w['warning'])
self['nwarnings'] = nwarnings
self['nexceptions'] = nexceptions
self['warnings'] = lines
self['warning_types'] = warning_types
def test_nonstandard_units():
from astropy import units as u
votable = parse(
get_pkg_data_filename('data/nonstandard_units.xml'),
pedantic=False)
assert isinstance(
votable.get_first_table().fields[0].unit, u.UnrecognizedUnit)
votable = parse(
get_pkg_data_filename('data/nonstandard_units.xml'),
pedantic=False,
unit_format='generic')
assert not isinstance(
votable.get_first_table().fields[0].unit, u.UnrecognizedUnit)
def validate_vo(self):
path = self.get_vo_xml_path()
if not os.path.exists(path):
self.download_xml_content()
self['version'] = ''
if 'network_error' in self and self['network_error'] is not None:
self['nwarnings'] = 0
self['nexceptions'] = 0
self['warnings'] = []
self['xmllint'] = None
self['warning_types'] = set()
return
nexceptions = 0
nwarnings = 0
t = None
lines = []
with open(path, 'rb') as input:
with warnings.catch_warnings(record=True) as warning_lines:
try:
t = table.parse(input, pedantic=False, filename=path)
except (ValueError, TypeError, ExpatError) as e:
lines.append(str(e))
nexceptions += 1
lines = [str(x.message) for x in warning_lines] + lines
if t is not None:
self['version'] = version = t.version
else:
self['version'] = version = "1.0"
if 'xmllint' not in self:
# Now check the VO schema based on the version in
# the file.
try:
success, stdout, stderr = xmlutil.validate_schema(path, version)
# OSError is raised when XML file eats all memory and
# system sends kill signal.
except OSError as e:
self['xmllint'] = None
self['xmllint_content'] = str(e)
else:
self['xmllint'] = (success == 0)
self['xmllint_content'] = stderr
warning_types = set()
for line in lines:
w = exceptions.parse_vowarning(line)
if w['is_warning']:
nwarnings += 1
if w['is_exception']:
nexceptions += 1
warning_types.add(w['warning'])
self['nwarnings'] = nwarnings
self['nexceptions'] = nexceptions
self['warnings'] = lines
self['warning_types'] = warning_types
def test_votable_path_object():
"""
Testing when votable is passed as pathlib.Path object #4412.
"""
fpath = pathlib.Path(get_pkg_data_filename('data/names.xml'))
table = parse(fpath).get_first_table().to_table()
assert len(table) == 1
assert int(table[0][3]) == 266
def test_custom_datatype():
votable = parse(
get_pkg_data_filename('data/custom_datatype.xml'),
pedantic=False,
datatype_mapping={'bar': 'int'}
)
table = votable.get_first_table()
assert table.array.dtype['foo'] == np.int32
def test_votable_path_object():
"""
Testing when votable is passed as pathlib.Path object #4412.
"""
fpath = pathlib.Path(get_pkg_data_filename('data/names.xml'))
table = parse(fpath).get_first_table().to_table()
assert len(table) == 1
assert int(table[0][3]) == 266
def test_explicit_ids():
with get_pkg_data_fileobj('data/names.xml', encoding='binary') as fd:
votable = parse(fd)
table = votable.get_first_table().to_table(use_names_over_ids=False)
assert table.colnames == [
'col1', 'col2', 'col3', 'col4', 'col5', 'col6', 'col7', 'col8', 'col9',
'col10', 'col11', 'col12', 'col13', 'col14', 'col15', 'col16', 'col17']
def test_explicit_ids():
with get_pkg_data_fileobj('data/names.xml', encoding='binary') as fd:
votable = parse(fd)
table = votable.get_first_table().to_table(use_names_over_ids=False)
assert table.colnames == [
'col1', 'col2', 'col3', 'col4', 'col5', 'col6', 'col7', 'col8', 'col9',
'col10', 'col11', 'col12', 'col13', 'col14', 'col15', 'col16', 'col17']
def test_names_over_ids():
with get_pkg_data_fileobj('data/names.xml', encoding='binary') as fd:
votable = parse(fd)
table = votable.get_first_table().to_table(use_names_over_ids=True)
assert table.colnames == [
'Name', 'GLON', 'GLAT', 'RAdeg', 'DEdeg', 'Jmag', 'Hmag', 'Kmag',
'G3.6mag', 'G4.5mag', 'G5.8mag', 'G8.0mag', '4.5mag', '8.0mag',
'Emag', '24mag', 'f_Name']
def test_select_columns_by_name():
columns = ['string_test', 'unsignedByte', 'bitarray']
table = parse(
get_pkg_data_filename('data/regression.xml'), columns=columns).get_first_table()
array = table.array
mask = table.array.mask
assert array['string_test'][0] == b"String & test"
for c in columns:
assert not np.all(mask[c])
assert np.all(mask['unicode_test'])
def test_names_over_ids():
with get_pkg_data_fileobj('data/names.xml', encoding='binary') as fd:
votable = parse(fd)
table = votable.get_first_table().to_table(use_names_over_ids=True)
assert table.colnames == [
'Name', 'GLON', 'GLAT', 'RAdeg', 'DEdeg', 'Jmag', 'Hmag', 'Kmag',
'G3.6mag', 'G4.5mag', 'G5.8mag', 'G8.0mag', '4.5mag', '8.0mag',
'Emag', '24mag', 'f_Name']
def test_table(tmpdir):
# Read the VOTABLE
with np.errstate(over="ignore"):
# https://github.com/astropy/astropy/issues/13341
votable = parse(get_pkg_data_filename('data/regression.xml'))
table = votable.get_first_table()
astropy_table = table.to_table()
for name in table.array.dtype.names:
assert np.all(astropy_table.mask[name] == table.array.mask[name])
votable2 = tree.VOTableFile.from_table(astropy_table)
t = votable2.get_first_table()
field_types = [
('string_test', {'datatype': 'char', 'arraysize': '*'}),
('string_test_2', {'datatype': 'char', 'arraysize': '10'}),
('unicode_test', {'datatype': 'unicodeChar', 'arraysize': '*'}),
('fixed_unicode_test', {'datatype': 'unicodeChar', 'arraysize': '10'}),
('string_array_test', {'datatype': 'char', 'arraysize': '4'}),
('unsignedByte', {'datatype': 'unsignedByte'}),
('short', {'datatype': 'short'}),
('int', {'datatype': 'int'}),
('long', {'datatype': 'long'}),
('double', {'datatype': 'double'}),
('float', {'datatype': 'float'}),
('array', {'datatype': 'long', 'arraysize': '2*'}),
('bit', {'datatype': 'bit'}),
('bitarray', {'datatype': 'bit', 'arraysize': '3x2'}),
('bitvararray', {'datatype': 'bit', 'arraysize': '*'}),
('bitvararray2', {'datatype': 'bit', 'arraysize': '3x2*'}),
('floatComplex', {'datatype': 'floatComplex'}),
('doubleComplex', {'datatype': 'doubleComplex'}),
('doubleComplexArray', {'datatype': 'doubleComplex', 'arraysize': '*'}),
('doubleComplexArrayFixed', {'datatype': 'doubleComplex', 'arraysize': '2'}),
('boolean', {'datatype': 'bit'}),
('booleanArray', {'datatype': 'bit', 'arraysize': '4'}),
('nulls', {'datatype': 'int'}),
('nulls_array', {'datatype': 'int', 'arraysize': '2x2'}),
('precision1', {'datatype': 'double'}),
('precision2', {'datatype': 'double'}),
('doublearray', {'datatype': 'double', 'arraysize': '*'}),
('bitarray2', {'datatype': 'bit', 'arraysize': '16'})]
for field, type in zip(t.fields, field_types):
name, d = type
assert field.ID == name
assert field.datatype == d['datatype'], f'{name} expected {d["datatype"]} but get {field.datatype}' # noqa
if 'arraysize' in d:
assert field.arraysize == d['arraysize']
# W39: Bit values can not be masked
with pytest.warns(W39):
writeto(votable2, os.path.join(str(tmpdir), "through_table.xml"))
def test_select_columns_by_name():
columns = ['string_test', 'unsignedByte', 'bitarray']
table = parse(
get_pkg_data_filename('data/regression.xml'),
pedantic=False, columns=columns).get_first_table()
array = table.array
mask = table.array.mask
assert array['string_test'][0] == b"String & test"
for c in columns:
assert not np.all(mask[c])
assert np.all(mask['unicode_test'])
def test_namespace_warning():
"""
A version 1.4 VOTable must use the same namespace as 1.3.
(see https://www.ivoa.net/documents/VOTable/20191021/REC-VOTable-1.4-20191021.html#ToC16)
"""
bad_namespace = b'''<?xml version="1.0" encoding="utf-8"?>
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.4"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<RESOURCE/>
</VOTABLE>
'''
with pytest.warns(W41):
parse(io.BytesIO(bad_namespace), verify='exception')
good_namespace_14 = b'''<?xml version="1.0" encoding="utf-8"?>
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<RESOURCE/>
</VOTABLE>
'''
parse(io.BytesIO(good_namespace_14), verify='exception')
good_namespace_13 = b'''<?xml version="1.0" encoding="utf-8"?>
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<RESOURCE/>
</VOTABLE>
'''
parse(io.BytesIO(good_namespace_13), verify='exception')
def test_table(tmpdir):
# Read the VOTABLE
votable = parse(
get_pkg_data_filename('data/regression.xml'),
pedantic=False)
table = votable.get_first_table()
astropy_table = table.to_table()
for name in table.array.dtype.names:
assert np.all(astropy_table.mask[name] == table.array.mask[name])
votable2 = tree.VOTableFile.from_table(astropy_table)
t = votable2.get_first_table()
field_types = [
('string_test', {'datatype': 'char', 'arraysize': '*'}),
('string_test_2', {'datatype': 'char', 'arraysize': '10'}),
('unicode_test', {'datatype': 'unicodeChar', 'arraysize': '*'}),
('fixed_unicode_test', {'datatype': 'unicodeChar', 'arraysize': '10'}),
('string_array_test', {'datatype': 'char', 'arraysize': '4'}),
('unsignedByte', {'datatype': 'unsignedByte'}),
('short', {'datatype': 'short'}),
('int', {'datatype': 'int'}),
('long', {'datatype': 'long'}),
('double', {'datatype': 'double'}),
('float', {'datatype': 'float'}),
('array', {'datatype': 'long', 'arraysize': '2*'}),
('bit', {'datatype': 'bit'}),
('bitarray', {'datatype': 'bit', 'arraysize': '3x2'}),
('bitvararray', {'datatype': 'bit', 'arraysize': '*'}),
('bitvararray2', {'datatype': 'bit', 'arraysize': '3x2*'}),
('floatComplex', {'datatype': 'floatComplex'}),
('doubleComplex', {'datatype': 'doubleComplex'}),
('doubleComplexArray', {'datatype': 'doubleComplex', 'arraysize': '*'}),
('doubleComplexArrayFixed', {'datatype': 'doubleComplex', 'arraysize': '2'}),
('boolean', {'datatype': 'bit'}),
('booleanArray', {'datatype': 'bit', 'arraysize': '4'}),
('nulls', {'datatype': 'int'}),
('nulls_array', {'datatype': 'int', 'arraysize': '2x2'}),
('precision1', {'datatype': 'double'}),
('precision2', {'datatype': 'double'}),
('doublearray', {'datatype': 'double', 'arraysize': '*'}),
('bitarray2', {'datatype': 'bit', 'arraysize': '16'})]
for field, type in zip(t.fields, field_types):
name, d = type
assert field.ID == name
assert field.datatype == d['datatype']
if 'arraysize' in d:
assert field.arraysize == d['arraysize']
writeto(votable2, os.path.join(str(tmpdir), "through_table.xml"))
def test_binary2_masked_strings():
"""
Issue #8995
"""
# Read a VOTable which sets the null mask bit for each empty string value.
votable = parse(get_pkg_data_filename('data/binary2_masked_strings.xml'))
table = votable.get_first_table()
astropy_table = table.to_table()
# Ensure string columns have no masked values and can be written out
assert not np.any(table.array.mask['epoch_photometry_url'])
output = io.BytesIO()
astropy_table.write(output, format='votable')
def _vo_service_request(url, pedantic, kwargs):
if len(kwargs) and not (url.endswith('?') or url.endswith('&')):
raise VOSError("url should already end with '?' or '&'")
query = []
for key, value in kwargs.iteritems():
query.append('{}={}'.format(
urllib.quote(key), urllib.quote_plus(str(value))))
parsed_url = url + '&'.join(query)
with get_readable_fileobj(parsed_url) as req:
tab = table.parse(req, filename=parsed_url, pedantic=pedantic)
return vo_tab_parse(tab, url, kwargs)
def vo_service_request(url, **kwargs):
if len(kwargs) and not (url.endswith('?') or url.endswith('&')):
raise VOSError("url should already end with '?' or '&'")
query = []
for key, value in kwargs.iteritems():
query.append('{}={}'.format(urllib.quote(key),
urllib.quote_plus(str(value))))
parsed_url = url + '&'.join(query)
with get_readable_fileobj(parsed_url) as req:
tab = table.parse(req, filename=parsed_url, pedantic=False)
#outfile = open('test.dat','wb')
#outfile.write(req.read())
#outfile.close()
return vo_tab_parse(tab, url, kwargs)
def complex_table():
"""
Return a masked table from the io.votable test set that has a wide variety
of stressing types.
"""
from astropy.utils.data import get_pkg_data_filename
from astropy.io.votable.table import parse
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
votable = parse(get_pkg_data_filename('../io/votable/tests/data/regression.xml'),
pedantic=False)
first_table = votable.get_first_table()
table = first_table.to_table()
return table
def test_build_from_scratch(tmpdir):
# Create a new VOTable file...
votable = tree.VOTableFile()
# ...with one resource...
resource = tree.Resource()
votable.resources.append(resource)
# ... with one table
table = tree.Table(votable)
resource.tables.append(table)
# Define some fields
table.fields.extend([
tree.Field(votable,
ID="filename",
name='filename',
datatype="char",
arraysize='1'),
tree.Field(votable,
ID="matrix",
name='matrix',
datatype="double",
arraysize="2x2")
])
# Now, use those field definitions to create the numpy record arrays, with
# the given number of rows
table.create_arrays(2)
# Now table.array can be filled with data
table.array[0] = ('test1.xml', [[1, 0], [0, 1]])
table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]])
# Now write the whole thing to a file.
# Note, we have to use the top-level votable file object
votable.to_xml(str(tmpdir.join("new_votable.xml")))
votable = parse(str(tmpdir.join("new_votable.xml")))
table = votable.get_first_table()
assert_array_equal(
table.array.mask,
np.array([(False, [[False, False], [False, False]]),
(False, [[False, False], [False, False]])],
dtype=[('filename', '?'), ('matrix', '?', (2, 2))]))
def test_resource_structure():
# Based on issue #1223, as reported by @astro-friedel and @RayPlante
from astropy.io.votable import tree as vot
vtf = vot.VOTableFile()
r1 = vot.Resource()
vtf.resources.append(r1)
t1 = vot.Table(vtf)
t1.name = "t1"
t2 = vot.Table(vtf)
t2.name = 't2'
r1.tables.append(t1)
r1.tables.append(t2)
r2 = vot.Resource()
vtf.resources.append(r2)
t3 = vot.Table(vtf)
t3.name = "t3"
t4 = vot.Table(vtf)
t4.name = "t4"
r2.tables.append(t3)
r2.tables.append(t4)
r3 = vot.Resource()
vtf.resources.append(r3)
t5 = vot.Table(vtf)
t5.name = "t5"
t6 = vot.Table(vtf)
t6.name = "t6"
r3.tables.append(t5)
r3.tables.append(t6)
buff = io.BytesIO()
vtf.to_xml(buff)
buff.seek(0)
vtf2 = parse(buff)
assert len(vtf2.resources) == 3
for r in range(len(vtf2.resources)):
res = vtf2.resources[r]
assert len(res.tables) == 2
assert len(res.resources) == 0
def _parse_result(self, response, pars={}, verbose=False):
"""
Parse the raw HTTP response and return it as a table.
"""
# Suppress any VOTable related warnings.
if not verbose:
commons.suppress_vo_warnings()
query = []
for key, value in six.iteritems(pars):
query.append('{0}={1}'.format(urllib.parse.quote(key),
urllib.parse.quote_plus(str(value))))
parsed_url = self.URL + '&'.join(query)
# Parse the result
tab = table.parse(BytesIO(response.content), filename=parsed_url,
pedantic=self.PEDANTIC)
return vo_tab_parse(tab, self.URL, pars)
def test_build_from_scratch(tmpdir):
# Create a new VOTable file...
votable = tree.VOTableFile()
# ...with one resource...
resource = tree.Resource()
votable.resources.append(resource)
# ... with one table
table = tree.Table(votable)
resource.tables.append(table)
# Define some fields
table.fields.extend([
tree.Field(votable, ID="filename", datatype="char"),
tree.Field(votable, ID="matrix", datatype="double", arraysize="2x2")])
# Now, use those field definitions to create the numpy record arrays, with
# the given number of rows
table.create_arrays(2)
# Now table.array can be filled with data
table.array[0] = ('test1.xml', [[1, 0], [0, 1]])
table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]])
# Now write the whole thing to a file.
# Note, we have to use the top-level votable file object
votable.to_xml(str(tmpdir.join("new_votable.xml")))
votable = parse(str(tmpdir.join("new_votable.xml")))
table = votable.get_first_table()
assert_array_equal(
table.array.mask, np.array([(False, [[False, False], [False, False]]),
(False, [[False, False], [False, False]])],
dtype=[(str('filename'), str('?')),
(str('matrix'), str('?'), (2, 2))]))
def _vo_service_request(url, pedantic, kwargs, cache=True, verbose=False):
"""
This is called by :func:`call_vo_service`.
Raises
------
InvalidAccessURL
Invalid access URL.
"""
if len(kwargs) and not url.endswith(('?', '&')):
raise InvalidAccessURL("url should already end with '?' or '&'")
query = []
for key, value in six.iteritems(kwargs):
query.append('{0}={1}'.format(
urllib.parse.quote(key), urllib.parse.quote_plus(str(value))))
parsed_url = url + '&'.join(query)
with get_readable_fileobj(parsed_url, encoding='binary', cache=cache,
show_progress=verbose) as req:
tab = table.parse(req, filename=parsed_url, pedantic=pedantic)
return vo_tab_parse(tab, url, kwargs)
def _list_tables(filename, pedantic=False):
votable = parse(filename, pedantic=pedantic)
tables = {}
for i, table in enumerate(votable.iter_tables()):
tables[i] = table.name
return tables
def read(self, filename, pedantic=False, tid=-1, verbose=True):
'''
Read a table from a VOT file
Required Arguments:
*filename*: [ string ]
The VOT file to read the table from
Optional Keyword Arguments:
*tid*: [ integer ]
The ID of the table to read from the VO file (this is
only required if there are more than one table in the VO file)
*pedantic*: [ True | False ]
When *pedantic* is True, raise an error when the file violates
the VO Table specification, otherwise issue a warning.
'''
self.reset()
# If no table is requested, check that there is only one table
if tid==-1:
tables = _list_tables(filename, pedantic=pedantic)
if len(tables) == 1:
tid = 0
elif len(tables) == 0:
raise Exception("There are no tables present in this file")
else:
raise TableException(tables, 'tid')
votable = parse(filename, pedantic=pedantic)
for id, table in enumerate(votable.iter_tables()):
if id==tid:
break
if table.ID:
self.table_name = str(table.ID)
elif table.name:
self.table_name = str(table.name)
for field in table.fields:
colname = field.ID
if table.array.size:
data = table.array[colname]
else:
data = np.array([], dtype=field.converter.format)
if len(data) > 0 and data.ndim == 1 and not np.all([np.isscalar(x) for x in data]):
warnings.warn("VO Variable length vector column detected (%s) - converting to string" % colname)
data = np.array([str(x) for x in data])
if self._masked:
self.add_column(colname, data, \
unit=field.unit, mask=table.mask[colname], \
description=field.description)
else:
self.add_column(colname, data, \
unit=field.unit, description=field.description)
for param in table.params:
self.add_keyword(param.ID, param.value)
def test_open_files():
for filename in get_pkg_data_filenames('data', pattern='*.xml'):
if filename.endswith('custom_datatype.xml'):
continue
parse(filename, pedantic=False)
def test_too_many_columns():
parse(
get_pkg_data_filename('data/too_many_columns.xml.gz'),
pedantic=False)
def _test_regression(tmpdir, _python_based=False, binary_mode=1):
# Read the VOTABLE
votable = parse(
get_pkg_data_filename('data/regression.xml'),
pedantic=False,
_debug_python_based_parser=_python_based)
table = votable.get_first_table()
dtypes = [
((str('string test'), str('string_test')), str('|O8')),
((str('fixed string test'), str('string_test_2')), str('|S10')),
(str('unicode_test'), str('|O8')),
((str('unicode test'), str('fixed_unicode_test')), str('<U10')),
((str('string array test'), str('string_array_test')), str('|S4')),
(str('unsignedByte'), str('|u1')),
(str('short'), str('<i2')),
(str('int'), str('<i4')),
(str('long'), str('<i8')),
(str('double'), str('<f8')),
(str('float'), str('<f4')),
(str('array'), str('|O8')),
(str('bit'), str('|b1')),
(str('bitarray'), str('|b1'), (3, 2)),
(str('bitvararray'), str('|O8')),
(str('bitvararray2'), str('|O8')),
(str('floatComplex'), str('<c8')),
(str('doubleComplex'), str('<c16')),
(str('doubleComplexArray'), str('|O8')),
(str('doubleComplexArrayFixed'), str('<c16'), (2,)),
(str('boolean'), str('|b1')),
(str('booleanArray'), str('|b1'), (4,)),
(str('nulls'), str('<i4')),
(str('nulls_array'), str('<i4'), (2, 2)),
(str('precision1'), str('<f8')),
(str('precision2'), str('<f8')),
(str('doublearray'), str('|O8')),
(str('bitarray2'), str('|b1'), (16,))
]
if sys.byteorder == 'big':
new_dtypes = []
for dtype in dtypes:
dtype = list(dtype)
dtype[1] = dtype[1].replace(str('<'), str('>'))
new_dtypes.append(tuple(dtype))
dtypes = new_dtypes
assert table.array.dtype == dtypes
votable.to_xml(str(tmpdir.join("regression.tabledata.xml")),
_debug_python_based_parser=_python_based)
assert_validate_schema(str(tmpdir.join("regression.tabledata.xml")),
votable.version)
if binary_mode == 1:
votable.get_first_table().format = 'binary'
votable.version = '1.1'
elif binary_mode == 2:
votable.get_first_table()._config['version_1_3_or_later'] = True
votable.get_first_table().format = 'binary2'
votable.version = '1.3'
# Also try passing a file handle
with open(str(tmpdir.join("regression.binary.xml")), "wb") as fd:
votable.to_xml(fd, _debug_python_based_parser=_python_based)
assert_validate_schema(str(tmpdir.join("regression.binary.xml")),
votable.version)
# Also try passing a file handle
with open(str(tmpdir.join("regression.binary.xml")), "rb") as fd:
votable2 = parse(fd, pedantic=False,
_debug_python_based_parser=_python_based)
votable2.get_first_table().format = 'tabledata'
votable2.to_xml(str(tmpdir.join("regression.bin.tabledata.xml")),
_astropy_version="testing",
_debug_python_based_parser=_python_based)
assert_validate_schema(str(tmpdir.join("regression.bin.tabledata.xml")),
votable.version)
with open(
get_pkg_data_filename(
'data/regression.bin.tabledata.truth.{0}.xml'.format(
votable.version)),
'rt', encoding='utf-8') as fd:
truth = fd.readlines()
with open(str(tmpdir.join("regression.bin.tabledata.xml")),
'rt', encoding='utf-8') as fd:
output = fd.readlines()
# If the lines happen to be different, print a diff
# This is convenient for debugging
sys.stdout.writelines(
difflib.unified_diff(truth, output, fromfile='truth', tofile='output'))
assert truth == output
# Test implicit gzip saving
votable2.to_xml(
str(tmpdir.join("regression.bin.tabledata.xml.gz")),
_astropy_version="testing",
_debug_python_based_parser=_python_based)
with gzip.GzipFile(
str(tmpdir.join("regression.bin.tabledata.xml.gz")), 'rb') as gzfd:
output = gzfd.readlines()
output = [x.decode('utf-8').rstrip() for x in output]
truth = [x.rstrip() for x in truth]
assert truth == output
def test_empty_table():
votable = parse(
get_pkg_data_filename('data/empty_table.xml'),
pedantic=False)
table = votable.get_first_table()
astropy_table = table.to_table() # noqa
