def _parse_result(self, response, verbose=False): if not verbose: commons.suppress_vo_warnings() # fix to replace non standard datatype 'integer' in returned VOTable # with 'int' to make it parsable by astropy.io.votable integer_re = re.compile(r'datatype="integer"') content = response.text new_content = integer_re.sub(r'datatype="int"', content) # these are pretty bad hacks, but also needed... days_re = re.compile(r'unit="days" datatype="double"') new_content = days_re.sub( r'unit="days" datatype="char" arraysize="*"', new_content) degrees_re = re.compile(r'unit="degrees" datatype="double"') new_content = degrees_re.sub( r'unit="degrees" datatype="char" arraysize="*"', new_content) try: tf = six.BytesIO(new_content.encode()) first_table = votable.parse(tf, pedantic=False).get_first_table() try: table = first_table.to_table(use_names_over_ids=True) except TypeError: warnings.warn( "NRAO table parsing: astropy versions prior to 6558975c use " "the table column IDs instead of names.") table = first_table.to_table() return table except Exception as ex: self.response = response self.table_parse_error = ex raise TableParseError( "Failed to parse NRAO votable result! The raw response can be found " "in self.response, and the error in self.table_parse_error.")
def _parse_result(self, response, verbose=False): """ Parses the results form the HTTP response to `~astropy.table.Table`. Parameters ---------- response : `requests.Response` The HTTP response object verbose : bool, optional Defaults to `False`. When true it will display warnings whenever the VOtable returned from the Service doesn't conform to the standard. Returns ------- table : `~astropy.table.Table` """ if not verbose: commons.suppress_vo_warnings() content = response.text logging.debug(content) # Check if results were returned if 'The catalog is not in the list' in content: raise Exception("Catalogue not found") # Check that object name was not malformed if 'Either wrong or missing coordinate/object name' in content: raise Exception("Malformed coordinate/object name") # Check that the results are not of length zero if len(content) == 0: raise Exception("The LCOGT server sent back an empty reply") # Read it in using the astropy VO table reader try: first_table = votable.parse(six.BytesIO(response.content), pedantic=False).get_first_table() except Exception as ex: self.response = response self.table_parse_error = ex raise TableParseError("Failed to parse LCOGT votable! The raw " " response can be found in self.response," " and the error in self.table_parse_error.") # Convert to astropy.table.Table instance table = first_table.to_table() # Check if table is empty if len(table) == 0: warnings.warn( "Query returned no results, so the table will " "be empty", NoResultsWarning) return table
def get_stringio(self): """ Return the file as an io.StringIO object """ s = self.get_string() # TODO: replace with six.BytesIO try: return six.BytesIO(s) except TypeError: return six.StringIO(s)
def test_griddata_fits(): """Round tripping with write_griddata_fits() and read_griddata_fits()""" x0 = np.array([0., 1.]) x1 = np.array([0., 1., 2.]) y = np.zeros((2, 3)) f = six.BytesIO() sncosmo.write_griddata_fits(x0, x1, y, f) # Read it back f.seek(0) x0_in, x1_in, y_in = sncosmo.read_griddata_fits(f) assert_allclose(x0_in, x0) assert_allclose(x1_in, x1) assert_allclose(y_in, y) f.close() # Test reading 3-d grid data. We don't have a writer for # this, so we write a temporary FITS file by hand. x2 = np.array([3., 5., 7., 9]) y = np.zeros((len(x0), len(x1), len(x2))) # write a FITS file that represents x0, x1, x2, y w = wcs.WCS(naxis=3) w.wcs.crpix = [1, 1, 1] w.wcs.crval = [x2[0], x1[0], x0[0]] w.wcs.cdelt = [2., 1., 1.] hdu = fits.PrimaryHDU(y, header=w.to_header()) f = six.BytesIO() hdu.writeto(f) # Read it back f.seek(0) x0_in, x1_in, x2_in, y_in = sncosmo.read_griddata_fits(f) f.close() assert_allclose(x0_in, x0) assert_allclose(x1_in, x1) assert_allclose(x2_in, x2) assert_allclose(y_in, y)
def _parse_result(self, response, verbose=False): """ Parse a VOtable response """ if not verbose: commons.suppress_vo_warnings() tf = six.BytesIO(response.content) vo_tree = votable.parse(tf, pedantic=False, invalid='mask') first_table = vo_tree.get_first_table() table = first_table.to_table(use_names_over_ids=True) return table
def _parse_xml_table(self, response): # try to parse the result into an astropy.Table, else # return the raw result with an informative error message. try: tf = six.BytesIO(response.content) vo_table = votable.parse(tf, pedantic=False) first_table = vo_table.get_first_table() table = first_table.to_table(use_names_over_ids=True) return table except Exception as ex: self.response = response self.table_parse_error = ex raise TableParseError( "Failed to parse ESASky VOTABLE result! The raw response can be " "found in self.response, and the error in " "self.table_parse_error.")
def test_griddata_fits(): """Round tripping with write_griddata_fits() and read_griddata_fits()""" x0 = np.array([0., 1.]) x1 = np.array([0., 1., 2.]) y = np.zeros((2, 3)) f = six.BytesIO() sncosmo.write_griddata_fits(x0, x1, y, f) # Read it back f.seek(0) x0_in, x1_in, y_in = sncosmo.read_griddata_fits(f) assert_allclose(x0_in, x0) assert_allclose(x1_in, x1) assert_allclose(y_in, y)
def _parse_result(self, response, verbose=False): """ Parses the raw HTTP response and returns it as an `astropy.table.Table`. Parameters ---------- response : `requests.Response` The HTTP response object verbose : bool, optional Defaults to false. When true it will display warnings whenever the VOtable returned from the service doesn't conform to the standard. Returns ------- table : `astropy.table.Table` """ if not verbose: commons.suppress_vo_warnings() try: tf = six.BytesIO(response.content) first_table = votable.parse(tf, pedantic=False).get_first_table() # For astropy version < 0.3 returns tables that have field ids as col names if ASTROPY_VERSION < '0.3': table = first_table.to_table() # For astropy versions >= 0.3 return the field names as col names else: table = first_table.to_table(use_names_over_ids=True) return table except Exception as ex: (is_valid, err_msg) = _check_ned_valid(response.content) if not is_valid: if err_msg: raise RemoteServiceError( "The remote service returned the following error message.\nERROR: {err_msg}" .format(err_msg=err_msg)) else: raise RemoteServiceError( "The remote service returned an error, but with no message." ) else: self.response = response self.table_parse_error = ex raise TableParseError( "Failed to parse NED result! The raw response can be found " "in self.response, and the error in self.table_parse_error." )
def _parse_votable_result(self, response, verbose=False): if not verbose: commons.suppress_vo_warnings() new_content = response.text # these are pretty bad hacks, but also needed... days_re = re.compile(r'unit="days" datatype="double"') new_content = days_re.sub( r'unit="days" datatype="char" ' 'arraysize="*"', new_content) degrees_re = re.compile(r'unit="degrees" datatype="double"') new_content = degrees_re.sub( r'unit="degrees" datatype="char" ' 'arraysize="*"', new_content) telconfig_re = re.compile(r'datatype="char" name="Telescope:config"') new_content = telconfig_re.sub( r'datatype="unicodeChar" ' 'name="Telescope:config" ' ' arraysize="*" ', new_content) datatype_mapping = {'integer': 'long'} try: tf = six.BytesIO(new_content.encode()) first_table = votable.parse( tf, pedantic=False, datatype_mapping=datatype_mapping).get_first_table() try: table = first_table.to_table(use_names_over_ids=True) except TypeError: warnings.warn("NRAO table parsing: astropy versions prior " "to 6558975c use the table column IDs instead " "of names.") table = first_table.to_table() return table except Exception as ex: self.response = response self.table_parse_error = ex raise TableParseError("Failed to parse NRAO votable result! The " "raw response can be found in self.response," " and the error in self.table_parse_error.")
def _parse_result(self, response, verbose=False): """ Parse a VOtable response """ if not verbose: commons.suppress_vo_warnings() if 'run?' in response.url: if response.text == "": raise RemoteServiceError("Empty return.") # this is a CSV-like table returned via a direct browser request import pandas table = Table.from_pandas(pandas.read_csv(StringIO(response.text))) else: fixed_content = self._hack_bad_arraysize_vofix(response.content) tf = six.BytesIO(fixed_content) vo_tree = votable.parse(tf, pedantic=False, invalid='mask') first_table = vo_tree.get_first_table() table = first_table.to_table(use_names_over_ids=True) return table
def parse_votable(content): """ Parse a votable in string format """ tables = votable.parse(six.BytesIO(content), pedantic=False) return tables
def _parse_result(self, response, get_catalog_names=False, verbose=False, invalid='warn'): """ Parses the HTTP response to create a `~astropy.table.Table`. Returns the raw result as a string in case of parse errors. Parameters ---------- response : `requests.Response` The response of the HTTP POST request get_catalog_names : bool If specified, return only the table names (useful for table discovery) invalid : 'warn', 'mask' or 'raise' The behavior if a VOTABLE cannot be parsed. Default is 'warn', which will try to parse the table, then if an exception is raised, it will be printent but the masked table will be returned Returns ------- table_list : `astroquery.utils.TableList` or str If there are errors in the parsing, then returns the raw results as a string. """ if not verbose: commons.suppress_vo_warnings() try: tf = six.BytesIO(response.content) if invalid == 'mask': vo_tree = votable.parse(tf, pedantic=False, invalid='mask') elif invalid == 'warn': try: vo_tree = votable.parse(tf, pedantic=False, invalid='raise') except Exception as ex: warnings.warn("VOTABLE parsing raised exception: {0}".format(ex)) vo_tree = votable.parse(tf, pedantic=False, invalid='mask') elif invalid == 'raise': vo_tree = votable.parse(tf, pedantic=False, invalid='raise') else: raise ValueError("Invalid keyword 'invalid'. Must be raise, mask, or warn") if get_catalog_names: return dict([(R.name, R) for R in vo_tree.resources]) else: table_dict = OrderedDict() for t in vo_tree.iter_tables(): if len(t.array) > 0: if t.ref is not None: name = vo_tree.get_table_by_id(t.ref).name else: name = t.name if name not in table_dict.keys(): table_dict[name] = [] table_dict[name] += [t.to_table()] for name in table_dict.keys(): if len(table_dict[name]) > 1: table_dict[name] = tbl.vstack(table_dict[name]) else: table_dict[name] = table_dict[name][0] return commons.TableList(table_dict) except Exception as ex: self.response = response self.table_parse_error = ex raise TableParseError("Failed to parse VIZIER result! The raw response can be found " "in self.response, and the error in self.table_parse_error." " The attempted parsed result is in self.parsed_result.\n" "Exception: " + str(self.table_parse_error))