def _load_votable(self, params): if 'url' in params.field_results: buff = CStringIO(curlrequest.urlopen( params.field_results['url']).read()) return parse(buff, filename=params.filename) else: return parse(params.filename)
def test_pedantic_false(self): with catch_warnings(VOWarning, AstropyDeprecationWarning) as w: parse(get_pkg_data_filename('data/gemini.xml'), pedantic=False) assert len(w) == 25 # Make sure we don't yet emit a deprecation warning assert not any( isinstance(x.category, AstropyDeprecationWarning) for x in w)
def test_timesys_roundtrip(): orig_votable = parse(get_pkg_data_filename('data/timesys.xml')) bio = io.BytesIO() orig_votable.to_xml(bio) bio.seek(0) votable = parse(bio) _timesys_tests(votable)
def test_gzip_filehandles(tmpdir): votable = parse(get_pkg_data_filename('data/regression.xml')) with open(str(tmpdir.join("regression.compressed.xml")), 'wb') as fd: votable.to_xml(fd, compressed=True, _astropy_version="testing") with open(str(tmpdir.join("regression.compressed.xml")), 'rb') as fd: votable = parse(fd)
def test_gzip_filehandles(tmpdir): votable = parse(get_pkg_data_filename('data/regression.xml')) # W39: Bit values can not be masked with pytest.warns(W39): with open(str(tmpdir.join("regression.compressed.xml")), 'wb') as fd: votable.to_xml(fd, compressed=True, _astropy_version="testing") with open(str(tmpdir.join("regression.compressed.xml")), 'rb') as fd: votable = parse(fd)
def setup_class(self): votable = parse(get_pkg_data_filename('data/regression.xml')) self.xmlout = bio = io.BytesIO() votable.to_xml(bio) bio.seek(0) self.votable = parse(bio) self.table = self.votable.get_first_table() self.array = self.table.array self.mask = self.table.array.mask
def test_conf_pedantic_true(self, tmpdir): with set_temp_config(tmpdir.strpath): with open(tmpdir.join('astropy').join('astropy.cfg').strpath, 'w') as f: f.write('[io.votable]\npedantic = True') reload_config('astropy.io.votable') with pytest.raises(VOWarning): parse(get_pkg_data_filename('data/gemini.xml'))
def _load_votable(self, params): if 'url' in params.field_results: try: buff = io.StringIO( request.urlopen(params.field_results['url']).read()) except TypeError: buff = io.BytesIO( request.urlopen(params.field_results['url']).read()) return parse(buff, filename=params.filename) else: return parse(params.filename)
def setup_class(self): votable = parse( get_pkg_data_filename('data/regression.xml'), pedantic=False) self.xmlout = bio = io.BytesIO() votable.to_xml(bio) bio.seek(0) self.votable = parse(bio, pedantic=False) self.table = self.votable.get_first_table() self.array = self.table.array self.mask = self.table.array.mask
def setup_class(self): votable = parse(get_pkg_data_filename('data/regression.xml')) self.xmlout = bio = io.BytesIO() # W39: Bit values can not be masked with pytest.warns(W39): votable.to_xml(bio) bio.seek(0) self.votable = parse(bio) self.table = self.votable.get_first_table() self.array = self.table.array self.mask = self.table.array.mask
def test_gzip_filehandles(tmpdir): with np.errstate(over="ignore"): # https://github.com/astropy/astropy/issues/13341 votable = parse(get_pkg_data_filename('data/regression.xml')) # W39: Bit values can not be masked with pytest.warns(W39): with open(str(tmpdir.join("regression.compressed.xml")), 'wb') as fd: votable.to_xml(fd, compressed=True, _astropy_version="testing") with open(str(tmpdir.join("regression.compressed.xml")), 'rb') as fd: votable = parse(fd)
def setup_class(self): votable = parse(get_pkg_data_filename('data/regression.xml'), pedantic=False) votable.get_first_table().format = 'binary' self.xmlout = bio = io.BytesIO() votable.to_xml(bio) bio.seek(0) self.votable = parse(bio, pedantic=False) self.table = self.votable.get_first_table() self.array = self.table.array self.mask = self.table.array.mask
def test_nonstandard_units(): from astropy import units as u votable = parse(get_pkg_data_filename('data/nonstandard_units.xml')) assert isinstance( votable.get_first_table().fields[0].unit, u.UnrecognizedUnit) votable = parse(get_pkg_data_filename('data/nonstandard_units.xml'), unit_format='generic') assert not isinstance( votable.get_first_table().fields[0].unit, u.UnrecognizedUnit)
def test_conf_pedantic_false(self, tmpdir): with set_temp_config(tmpdir.strpath): with open(tmpdir.join('astropy').join('astropy.cfg').strpath, 'w') as f: f.write('[io.votable]\npedantic = False') reload_config('astropy.io.votable') with catch_warnings(VOWarning, AstropyDeprecationWarning) as w: parse(get_pkg_data_filename('data/gemini.xml')) assert len(w) == 25 # Make sure we don't yet emit a deprecation warning assert not any(isinstance(x.category, AstropyDeprecationWarning) for x in w)
def setup_class(self): votable = parse(get_pkg_data_filename('data/regression.xml')) votable.version = '1.3' votable.get_first_table()._config['version_1_3_or_later'] = True votable.get_first_table().format = 'binary2' self.xmlout = bio = io.BytesIO() votable.to_xml(bio) bio.seek(0) self.votable = parse(bio) self.table = self.votable.get_first_table() self.array = self.table.array self.mask = self.table.array.mask
def setup_class(self): with np.errstate(over="ignore"): # https://github.com/astropy/astropy/issues/13341 votable = parse(get_pkg_data_filename('data/regression.xml')) self.xmlout = bio = io.BytesIO() # W39: Bit values can not be masked with pytest.warns(W39): votable.to_xml(bio) bio.seek(0) self.votable = parse(bio) self.table = self.votable.get_first_table() self.array = self.table.array self.mask = self.table.array.mask
def test_gzip_filehandles(tmpdir): votable = parse( get_pkg_data_filename('data/regression.xml'), pedantic=False) with open(str(tmpdir.join("regression.compressed.xml")), 'wb') as fd: votable.to_xml( fd, compressed=True, _astropy_version="testing") with open(str(tmpdir.join("regression.compressed.xml")), 'rb') as fd: votable = parse( fd, pedantic=False)
def test_custom_datatype(): votable = parse(get_pkg_data_filename('data/custom_datatype.xml'), pedantic=False, datatype_mapping={'bar': 'int'}) table = votable.get_first_table() assert table.array.dtype['foo'] == np.int32
def setup_class(self): with np.errstate(over="ignore"): # https://github.com/astropy/astropy/issues/13341 self.votable = parse(get_pkg_data_filename('data/regression.xml')) self.table = self.votable.get_first_table() self.array = self.table.array self.mask = self.table.array.mask
def setup_class(self): votable = parse( get_pkg_data_filename('data/regression.xml'), pedantic=False) votable.version = '1.3' votable.get_first_table()._config['version_1_3_or_later'] = True votable.get_first_table().format = 'binary2' self.xmlout = bio = io.BytesIO() votable.to_xml(bio) bio.seek(0) self.votable = parse(bio, pedantic=False) self.table = self.votable.get_first_table() self.array = self.table.array self.mask = self.table.array.mask
def setup_class(self): self.votable = parse( get_pkg_data_filename('data/regression.xml'), pedantic=False) self.table = self.votable.get_first_table() self.array = self.table.array self.mask = self.table.array.mask
def _vo_service_request(url, pedantic, kwargs, cache=True, verbose=False): """ This is called by :func:`call_vo_service`. Raises ------ InvalidAccessURL Invalid access URL. """ if len(kwargs) and not url.endswith(('?', '&')): raise InvalidAccessURL("url should already end with '?' or '&'") query = [] for key, value in six.iteritems(kwargs): query.append('{0}={1}'.format(urllib.parse.quote(key), urllib.parse.quote_plus(str(value)))) parsed_url = url + '&'.join(query) with get_readable_fileobj(parsed_url, encoding='binary', cache=cache, show_progress=verbose) as req: tab = table.parse(req, filename=parsed_url, pedantic=pedantic) return vo_tab_parse(tab, url, kwargs)
def test_unit_format(): data = parse(get_pkg_data_filename('data/irsa-nph-error.xml')) assert data._config['version'] == '1.0' assert tree._get_default_unit_format(data._config) == 'cds' data = parse(get_pkg_data_filename('data/names.xml')) assert data._config['version'] == '1.1' assert tree._get_default_unit_format(data._config) == 'cds' data = parse(get_pkg_data_filename('data/gemini.xml')) assert data._config['version'] == '1.2' assert tree._get_default_unit_format(data._config) == 'cds' data = parse(get_pkg_data_filename('data/binary2_masked_strings.xml')) assert data._config['version'] == '1.3' assert tree._get_default_unit_format(data._config) == 'cds' data = parse(get_pkg_data_filename('data/timesys.xml')) assert data._config['version'] == '1.4' assert tree._get_default_unit_format(data._config) == 'vounit'
def _parse_result(self, response, url, pars={}, verbose=False): """ Parse the raw HTTP response and return it as a table. """ # Suppress any VOTable related warnings. if not verbose: commons.suppress_vo_warnings() query = [] for key, value in pars.items(): query.append('{0}={1}'.format(urllib.parse.quote(key), urllib.parse.quote_plus(str(value)))) parsed_url = url + '&'.join(query) # Parse the result pedantic = 'exception' if conf.pedantic else 'warn' tab = table.parse(BytesIO(response.content), filename=parsed_url, verify=pedantic) try: result = vo_tab_parse(tab, url, pars) except VOSError as exc: result = None warnings.warn(str(exc), NoResultsWarning) return result
def validate_vo(self): path = self.get_vo_xml_path() if not os.path.exists(path): self.download_xml_content() self['version'] = '' if 'network_error' in self and self['network_error'] is not None: self['nwarnings'] = 0 self['nexceptions'] = 0 self['warnings'] = [] self['xmllint'] = None self['warning_types'] = set() return nexceptions = 0 nwarnings = 0 t = None lines = [] with open(path, 'rb') as input: with warnings.catch_warnings(record=True) as warning_lines: try: t = table.parse(input, verify='warn', filename=path) except (ValueError, TypeError, ExpatError) as e: lines.append(str(e)) nexceptions += 1 lines = [str(x.message) for x in warning_lines] + lines if t is not None: self['version'] = version = t.version else: self['version'] = version = "1.0" if 'xmllint' not in self: # Now check the VO schema based on the version in # the file. try: success, stdout, stderr = xmlutil.validate_schema( path, version) # OSError is raised when XML file eats all memory and # system sends kill signal. except OSError as e: self['xmllint'] = None self['xmllint_content'] = str(e) else: self['xmllint'] = (success == 0) self['xmllint_content'] = stderr warning_types = set() for line in lines: w = exceptions.parse_vowarning(line) if w['is_warning']: nwarnings += 1 if w['is_exception']: nexceptions += 1 warning_types.add(w['warning']) self['nwarnings'] = nwarnings self['nexceptions'] = nexceptions self['warnings'] = lines self['warning_types'] = warning_types
def test_nonstandard_units(): from astropy import units as u votable = parse( get_pkg_data_filename('data/nonstandard_units.xml'), pedantic=False) assert isinstance( votable.get_first_table().fields[0].unit, u.UnrecognizedUnit) votable = parse( get_pkg_data_filename('data/nonstandard_units.xml'), pedantic=False, unit_format='generic') assert not isinstance( votable.get_first_table().fields[0].unit, u.UnrecognizedUnit)
def validate_vo(self): path = self.get_vo_xml_path() if not os.path.exists(path): self.download_xml_content() self['version'] = '' if 'network_error' in self and self['network_error'] is not None: self['nwarnings'] = 0 self['nexceptions'] = 0 self['warnings'] = [] self['xmllint'] = None self['warning_types'] = set() return nexceptions = 0 nwarnings = 0 t = None lines = [] with open(path, 'rb') as input: with warnings.catch_warnings(record=True) as warning_lines: try: t = table.parse(input, pedantic=False, filename=path) except (ValueError, TypeError, ExpatError) as e: lines.append(str(e)) nexceptions += 1 lines = [str(x.message) for x in warning_lines] + lines if t is not None: self['version'] = version = t.version else: self['version'] = version = "1.0" if 'xmllint' not in self: # Now check the VO schema based on the version in # the file. try: success, stdout, stderr = xmlutil.validate_schema(path, version) # OSError is raised when XML file eats all memory and # system sends kill signal. except OSError as e: self['xmllint'] = None self['xmllint_content'] = str(e) else: self['xmllint'] = (success == 0) self['xmllint_content'] = stderr warning_types = set() for line in lines: w = exceptions.parse_vowarning(line) if w['is_warning']: nwarnings += 1 if w['is_exception']: nexceptions += 1 warning_types.add(w['warning']) self['nwarnings'] = nwarnings self['nexceptions'] = nexceptions self['warnings'] = lines self['warning_types'] = warning_types
def test_votable_path_object(): """ Testing when votable is passed as pathlib.Path object #4412. """ fpath = pathlib.Path(get_pkg_data_filename('data/names.xml')) table = parse(fpath).get_first_table().to_table() assert len(table) == 1 assert int(table[0][3]) == 266
def test_custom_datatype(): votable = parse( get_pkg_data_filename('data/custom_datatype.xml'), pedantic=False, datatype_mapping={'bar': 'int'} ) table = votable.get_first_table() assert table.array.dtype['foo'] == np.int32
def test_explicit_ids(): with get_pkg_data_fileobj('data/names.xml', encoding='binary') as fd: votable = parse(fd) table = votable.get_first_table().to_table(use_names_over_ids=False) assert table.colnames == [ 'col1', 'col2', 'col3', 'col4', 'col5', 'col6', 'col7', 'col8', 'col9', 'col10', 'col11', 'col12', 'col13', 'col14', 'col15', 'col16', 'col17']
def test_names_over_ids(): with get_pkg_data_fileobj('data/names.xml', encoding='binary') as fd: votable = parse(fd) table = votable.get_first_table().to_table(use_names_over_ids=True) assert table.colnames == [ 'Name', 'GLON', 'GLAT', 'RAdeg', 'DEdeg', 'Jmag', 'Hmag', 'Kmag', 'G3.6mag', 'G4.5mag', 'G5.8mag', 'G8.0mag', '4.5mag', '8.0mag', 'Emag', '24mag', 'f_Name']
def test_select_columns_by_name(): columns = ['string_test', 'unsignedByte', 'bitarray'] table = parse( get_pkg_data_filename('data/regression.xml'), columns=columns).get_first_table() array = table.array mask = table.array.mask assert array['string_test'][0] == b"String & test" for c in columns: assert not np.all(mask[c]) assert np.all(mask['unicode_test'])
def test_table(tmpdir): # Read the VOTABLE with np.errstate(over="ignore"): # https://github.com/astropy/astropy/issues/13341 votable = parse(get_pkg_data_filename('data/regression.xml')) table = votable.get_first_table() astropy_table = table.to_table() for name in table.array.dtype.names: assert np.all(astropy_table.mask[name] == table.array.mask[name]) votable2 = tree.VOTableFile.from_table(astropy_table) t = votable2.get_first_table() field_types = [ ('string_test', {'datatype': 'char', 'arraysize': '*'}), ('string_test_2', {'datatype': 'char', 'arraysize': '10'}), ('unicode_test', {'datatype': 'unicodeChar', 'arraysize': '*'}), ('fixed_unicode_test', {'datatype': 'unicodeChar', 'arraysize': '10'}), ('string_array_test', {'datatype': 'char', 'arraysize': '4'}), ('unsignedByte', {'datatype': 'unsignedByte'}), ('short', {'datatype': 'short'}), ('int', {'datatype': 'int'}), ('long', {'datatype': 'long'}), ('double', {'datatype': 'double'}), ('float', {'datatype': 'float'}), ('array', {'datatype': 'long', 'arraysize': '2*'}), ('bit', {'datatype': 'bit'}), ('bitarray', {'datatype': 'bit', 'arraysize': '3x2'}), ('bitvararray', {'datatype': 'bit', 'arraysize': '*'}), ('bitvararray2', {'datatype': 'bit', 'arraysize': '3x2*'}), ('floatComplex', {'datatype': 'floatComplex'}), ('doubleComplex', {'datatype': 'doubleComplex'}), ('doubleComplexArray', {'datatype': 'doubleComplex', 'arraysize': '*'}), ('doubleComplexArrayFixed', {'datatype': 'doubleComplex', 'arraysize': '2'}), ('boolean', {'datatype': 'bit'}), ('booleanArray', {'datatype': 'bit', 'arraysize': '4'}), ('nulls', {'datatype': 'int'}), ('nulls_array', {'datatype': 'int', 'arraysize': '2x2'}), ('precision1', {'datatype': 'double'}), ('precision2', {'datatype': 'double'}), ('doublearray', {'datatype': 'double', 'arraysize': '*'}), ('bitarray2', {'datatype': 'bit', 'arraysize': '16'})] for field, type in zip(t.fields, field_types): name, d = type assert field.ID == name assert field.datatype == d['datatype'], f'{name} expected {d["datatype"]} but get {field.datatype}' # noqa if 'arraysize' in d: assert field.arraysize == d['arraysize'] # W39: Bit values can not be masked with pytest.warns(W39): writeto(votable2, os.path.join(str(tmpdir), "through_table.xml"))
def test_select_columns_by_name(): columns = ['string_test', 'unsignedByte', 'bitarray'] table = parse( get_pkg_data_filename('data/regression.xml'), pedantic=False, columns=columns).get_first_table() array = table.array mask = table.array.mask assert array['string_test'][0] == b"String & test" for c in columns: assert not np.all(mask[c]) assert np.all(mask['unicode_test'])
def test_namespace_warning(): """ A version 1.4 VOTable must use the same namespace as 1.3. (see https://www.ivoa.net/documents/VOTable/20191021/REC-VOTable-1.4-20191021.html#ToC16) """ bad_namespace = b'''<?xml version="1.0" encoding="utf-8"?> <VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.4" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <RESOURCE/> </VOTABLE> ''' with pytest.warns(W41): parse(io.BytesIO(bad_namespace), verify='exception') good_namespace_14 = b'''<?xml version="1.0" encoding="utf-8"?> <VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <RESOURCE/> </VOTABLE> ''' parse(io.BytesIO(good_namespace_14), verify='exception') good_namespace_13 = b'''<?xml version="1.0" encoding="utf-8"?> <VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <RESOURCE/> </VOTABLE> ''' parse(io.BytesIO(good_namespace_13), verify='exception')
def test_table(tmpdir): # Read the VOTABLE votable = parse( get_pkg_data_filename('data/regression.xml'), pedantic=False) table = votable.get_first_table() astropy_table = table.to_table() for name in table.array.dtype.names: assert np.all(astropy_table.mask[name] == table.array.mask[name]) votable2 = tree.VOTableFile.from_table(astropy_table) t = votable2.get_first_table() field_types = [ ('string_test', {'datatype': 'char', 'arraysize': '*'}), ('string_test_2', {'datatype': 'char', 'arraysize': '10'}), ('unicode_test', {'datatype': 'unicodeChar', 'arraysize': '*'}), ('fixed_unicode_test', {'datatype': 'unicodeChar', 'arraysize': '10'}), ('string_array_test', {'datatype': 'char', 'arraysize': '4'}), ('unsignedByte', {'datatype': 'unsignedByte'}), ('short', {'datatype': 'short'}), ('int', {'datatype': 'int'}), ('long', {'datatype': 'long'}), ('double', {'datatype': 'double'}), ('float', {'datatype': 'float'}), ('array', {'datatype': 'long', 'arraysize': '2*'}), ('bit', {'datatype': 'bit'}), ('bitarray', {'datatype': 'bit', 'arraysize': '3x2'}), ('bitvararray', {'datatype': 'bit', 'arraysize': '*'}), ('bitvararray2', {'datatype': 'bit', 'arraysize': '3x2*'}), ('floatComplex', {'datatype': 'floatComplex'}), ('doubleComplex', {'datatype': 'doubleComplex'}), ('doubleComplexArray', {'datatype': 'doubleComplex', 'arraysize': '*'}), ('doubleComplexArrayFixed', {'datatype': 'doubleComplex', 'arraysize': '2'}), ('boolean', {'datatype': 'bit'}), ('booleanArray', {'datatype': 'bit', 'arraysize': '4'}), ('nulls', {'datatype': 'int'}), ('nulls_array', {'datatype': 'int', 'arraysize': '2x2'}), ('precision1', {'datatype': 'double'}), ('precision2', {'datatype': 'double'}), ('doublearray', {'datatype': 'double', 'arraysize': '*'}), ('bitarray2', {'datatype': 'bit', 'arraysize': '16'})] for field, type in zip(t.fields, field_types): name, d = type assert field.ID == name assert field.datatype == d['datatype'] if 'arraysize' in d: assert field.arraysize == d['arraysize'] writeto(votable2, os.path.join(str(tmpdir), "through_table.xml"))
def test_binary2_masked_strings(): """ Issue #8995 """ # Read a VOTable which sets the null mask bit for each empty string value. votable = parse(get_pkg_data_filename('data/binary2_masked_strings.xml')) table = votable.get_first_table() astropy_table = table.to_table() # Ensure string columns have no masked values and can be written out assert not np.any(table.array.mask['epoch_photometry_url']) output = io.BytesIO() astropy_table.write(output, format='votable')
def _vo_service_request(url, pedantic, kwargs): if len(kwargs) and not (url.endswith('?') or url.endswith('&')): raise VOSError("url should already end with '?' or '&'") query = [] for key, value in kwargs.iteritems(): query.append('{}={}'.format( urllib.quote(key), urllib.quote_plus(str(value)))) parsed_url = url + '&'.join(query) with get_readable_fileobj(parsed_url) as req: tab = table.parse(req, filename=parsed_url, pedantic=pedantic) return vo_tab_parse(tab, url, kwargs)
def vo_service_request(url, **kwargs): if len(kwargs) and not (url.endswith('?') or url.endswith('&')): raise VOSError("url should already end with '?' or '&'") query = [] for key, value in kwargs.iteritems(): query.append('{}={}'.format(urllib.quote(key), urllib.quote_plus(str(value)))) parsed_url = url + '&'.join(query) with get_readable_fileobj(parsed_url) as req: tab = table.parse(req, filename=parsed_url, pedantic=False) #outfile = open('test.dat','wb') #outfile.write(req.read()) #outfile.close() return vo_tab_parse(tab, url, kwargs)
def complex_table(): """ Return a masked table from the io.votable test set that has a wide variety of stressing types. """ from astropy.utils.data import get_pkg_data_filename from astropy.io.votable.table import parse import warnings with warnings.catch_warnings(): warnings.simplefilter("ignore") votable = parse(get_pkg_data_filename('../io/votable/tests/data/regression.xml'), pedantic=False) first_table = votable.get_first_table() table = first_table.to_table() return table
def test_build_from_scratch(tmpdir): # Create a new VOTable file... votable = tree.VOTableFile() # ...with one resource... resource = tree.Resource() votable.resources.append(resource) # ... with one table table = tree.Table(votable) resource.tables.append(table) # Define some fields table.fields.extend([ tree.Field(votable, ID="filename", name='filename', datatype="char", arraysize='1'), tree.Field(votable, ID="matrix", name='matrix', datatype="double", arraysize="2x2") ]) # Now, use those field definitions to create the numpy record arrays, with # the given number of rows table.create_arrays(2) # Now table.array can be filled with data table.array[0] = ('test1.xml', [[1, 0], [0, 1]]) table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]]) # Now write the whole thing to a file. # Note, we have to use the top-level votable file object votable.to_xml(str(tmpdir.join("new_votable.xml"))) votable = parse(str(tmpdir.join("new_votable.xml"))) table = votable.get_first_table() assert_array_equal( table.array.mask, np.array([(False, [[False, False], [False, False]]), (False, [[False, False], [False, False]])], dtype=[('filename', '?'), ('matrix', '?', (2, 2))]))
def test_resource_structure(): # Based on issue #1223, as reported by @astro-friedel and @RayPlante from astropy.io.votable import tree as vot vtf = vot.VOTableFile() r1 = vot.Resource() vtf.resources.append(r1) t1 = vot.Table(vtf) t1.name = "t1" t2 = vot.Table(vtf) t2.name = 't2' r1.tables.append(t1) r1.tables.append(t2) r2 = vot.Resource() vtf.resources.append(r2) t3 = vot.Table(vtf) t3.name = "t3" t4 = vot.Table(vtf) t4.name = "t4" r2.tables.append(t3) r2.tables.append(t4) r3 = vot.Resource() vtf.resources.append(r3) t5 = vot.Table(vtf) t5.name = "t5" t6 = vot.Table(vtf) t6.name = "t6" r3.tables.append(t5) r3.tables.append(t6) buff = io.BytesIO() vtf.to_xml(buff) buff.seek(0) vtf2 = parse(buff) assert len(vtf2.resources) == 3 for r in range(len(vtf2.resources)): res = vtf2.resources[r] assert len(res.tables) == 2 assert len(res.resources) == 0
def _parse_result(self, response, pars={}, verbose=False): """ Parse the raw HTTP response and return it as a table. """ # Suppress any VOTable related warnings. if not verbose: commons.suppress_vo_warnings() query = [] for key, value in six.iteritems(pars): query.append('{0}={1}'.format(urllib.parse.quote(key), urllib.parse.quote_plus(str(value)))) parsed_url = self.URL + '&'.join(query) # Parse the result tab = table.parse(BytesIO(response.content), filename=parsed_url, pedantic=self.PEDANTIC) return vo_tab_parse(tab, self.URL, pars)
def test_build_from_scratch(tmpdir): # Create a new VOTable file... votable = tree.VOTableFile() # ...with one resource... resource = tree.Resource() votable.resources.append(resource) # ... with one table table = tree.Table(votable) resource.tables.append(table) # Define some fields table.fields.extend([ tree.Field(votable, ID="filename", datatype="char"), tree.Field(votable, ID="matrix", datatype="double", arraysize="2x2")]) # Now, use those field definitions to create the numpy record arrays, with # the given number of rows table.create_arrays(2) # Now table.array can be filled with data table.array[0] = ('test1.xml', [[1, 0], [0, 1]]) table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]]) # Now write the whole thing to a file. # Note, we have to use the top-level votable file object votable.to_xml(str(tmpdir.join("new_votable.xml"))) votable = parse(str(tmpdir.join("new_votable.xml"))) table = votable.get_first_table() assert_array_equal( table.array.mask, np.array([(False, [[False, False], [False, False]]), (False, [[False, False], [False, False]])], dtype=[(str('filename'), str('?')), (str('matrix'), str('?'), (2, 2))]))
def _vo_service_request(url, pedantic, kwargs, cache=True, verbose=False): """ This is called by :func:`call_vo_service`. Raises ------ InvalidAccessURL Invalid access URL. """ if len(kwargs) and not url.endswith(('?', '&')): raise InvalidAccessURL("url should already end with '?' or '&'") query = [] for key, value in six.iteritems(kwargs): query.append('{0}={1}'.format( urllib.parse.quote(key), urllib.parse.quote_plus(str(value)))) parsed_url = url + '&'.join(query) with get_readable_fileobj(parsed_url, encoding='binary', cache=cache, show_progress=verbose) as req: tab = table.parse(req, filename=parsed_url, pedantic=pedantic) return vo_tab_parse(tab, url, kwargs)
def _list_tables(filename, pedantic=False): votable = parse(filename, pedantic=pedantic) tables = {} for i, table in enumerate(votable.iter_tables()): tables[i] = table.name return tables
def read(self, filename, pedantic=False, tid=-1, verbose=True): ''' Read a table from a VOT file Required Arguments: *filename*: [ string ] The VOT file to read the table from Optional Keyword Arguments: *tid*: [ integer ] The ID of the table to read from the VO file (this is only required if there are more than one table in the VO file) *pedantic*: [ True | False ] When *pedantic* is True, raise an error when the file violates the VO Table specification, otherwise issue a warning. ''' self.reset() # If no table is requested, check that there is only one table if tid==-1: tables = _list_tables(filename, pedantic=pedantic) if len(tables) == 1: tid = 0 elif len(tables) == 0: raise Exception("There are no tables present in this file") else: raise TableException(tables, 'tid') votable = parse(filename, pedantic=pedantic) for id, table in enumerate(votable.iter_tables()): if id==tid: break if table.ID: self.table_name = str(table.ID) elif table.name: self.table_name = str(table.name) for field in table.fields: colname = field.ID if table.array.size: data = table.array[colname] else: data = np.array([], dtype=field.converter.format) if len(data) > 0 and data.ndim == 1 and not np.all([np.isscalar(x) for x in data]): warnings.warn("VO Variable length vector column detected (%s) - converting to string" % colname) data = np.array([str(x) for x in data]) if self._masked: self.add_column(colname, data, \ unit=field.unit, mask=table.mask[colname], \ description=field.description) else: self.add_column(colname, data, \ unit=field.unit, description=field.description) for param in table.params: self.add_keyword(param.ID, param.value)
def test_open_files(): for filename in get_pkg_data_filenames('data', pattern='*.xml'): if filename.endswith('custom_datatype.xml'): continue parse(filename, pedantic=False)
def test_too_many_columns(): parse( get_pkg_data_filename('data/too_many_columns.xml.gz'), pedantic=False)
def _test_regression(tmpdir, _python_based=False, binary_mode=1): # Read the VOTABLE votable = parse( get_pkg_data_filename('data/regression.xml'), pedantic=False, _debug_python_based_parser=_python_based) table = votable.get_first_table() dtypes = [ ((str('string test'), str('string_test')), str('|O8')), ((str('fixed string test'), str('string_test_2')), str('|S10')), (str('unicode_test'), str('|O8')), ((str('unicode test'), str('fixed_unicode_test')), str('<U10')), ((str('string array test'), str('string_array_test')), str('|S4')), (str('unsignedByte'), str('|u1')), (str('short'), str('<i2')), (str('int'), str('<i4')), (str('long'), str('<i8')), (str('double'), str('<f8')), (str('float'), str('<f4')), (str('array'), str('|O8')), (str('bit'), str('|b1')), (str('bitarray'), str('|b1'), (3, 2)), (str('bitvararray'), str('|O8')), (str('bitvararray2'), str('|O8')), (str('floatComplex'), str('<c8')), (str('doubleComplex'), str('<c16')), (str('doubleComplexArray'), str('|O8')), (str('doubleComplexArrayFixed'), str('<c16'), (2,)), (str('boolean'), str('|b1')), (str('booleanArray'), str('|b1'), (4,)), (str('nulls'), str('<i4')), (str('nulls_array'), str('<i4'), (2, 2)), (str('precision1'), str('<f8')), (str('precision2'), str('<f8')), (str('doublearray'), str('|O8')), (str('bitarray2'), str('|b1'), (16,)) ] if sys.byteorder == 'big': new_dtypes = [] for dtype in dtypes: dtype = list(dtype) dtype[1] = dtype[1].replace(str('<'), str('>')) new_dtypes.append(tuple(dtype)) dtypes = new_dtypes assert table.array.dtype == dtypes votable.to_xml(str(tmpdir.join("regression.tabledata.xml")), _debug_python_based_parser=_python_based) assert_validate_schema(str(tmpdir.join("regression.tabledata.xml")), votable.version) if binary_mode == 1: votable.get_first_table().format = 'binary' votable.version = '1.1' elif binary_mode == 2: votable.get_first_table()._config['version_1_3_or_later'] = True votable.get_first_table().format = 'binary2' votable.version = '1.3' # Also try passing a file handle with open(str(tmpdir.join("regression.binary.xml")), "wb") as fd: votable.to_xml(fd, _debug_python_based_parser=_python_based) assert_validate_schema(str(tmpdir.join("regression.binary.xml")), votable.version) # Also try passing a file handle with open(str(tmpdir.join("regression.binary.xml")), "rb") as fd: votable2 = parse(fd, pedantic=False, _debug_python_based_parser=_python_based) votable2.get_first_table().format = 'tabledata' votable2.to_xml(str(tmpdir.join("regression.bin.tabledata.xml")), _astropy_version="testing", _debug_python_based_parser=_python_based) assert_validate_schema(str(tmpdir.join("regression.bin.tabledata.xml")), votable.version) with open( get_pkg_data_filename( 'data/regression.bin.tabledata.truth.{0}.xml'.format( votable.version)), 'rt', encoding='utf-8') as fd: truth = fd.readlines() with open(str(tmpdir.join("regression.bin.tabledata.xml")), 'rt', encoding='utf-8') as fd: output = fd.readlines() # If the lines happen to be different, print a diff # This is convenient for debugging sys.stdout.writelines( difflib.unified_diff(truth, output, fromfile='truth', tofile='output')) assert truth == output # Test implicit gzip saving votable2.to_xml( str(tmpdir.join("regression.bin.tabledata.xml.gz")), _astropy_version="testing", _debug_python_based_parser=_python_based) with gzip.GzipFile( str(tmpdir.join("regression.bin.tabledata.xml.gz")), 'rb') as gzfd: output = gzfd.readlines() output = [x.decode('utf-8').rstrip() for x in output] truth = [x.rstrip() for x in truth] assert truth == output
def test_empty_table(): votable = parse( get_pkg_data_filename('data/empty_table.xml'), pedantic=False) table = votable.get_first_table() astropy_table = table.to_table() # noqa