def _parse_result(self, response, get_catalog_names=False, verbose=False):
"""
Parses the HTTP response to create an `astropy.table.Table`.
Returns the raw result as a string in case of parse errors.
Parameters
----------
response : `requests.Response`
The response of the HTTP POST request
get_catalog_names : bool
If specified, return only the table names (useful for table
discovery)
Returns
-------
`astroquery.utils.commons.TableList`
An OrderedDict of `astropy.table.Table` objects.
If there are errors in the parsing, then returns the raw results as a string.
"""
if not verbose:
commons.suppress_vo_warnings()
try:
tf = tempfile.NamedTemporaryFile()
if PY3:
tf.write(response.content)
else:
tf.write(response.content.encode('utf-8'))
tf.file.flush()
vo_tree = votable.parse(tf.name, pedantic=False)
if get_catalog_names:
return dict([(R.name,R) for R in vo_tree.resources])
else:
table_dict = OrderedDict()
for t in vo_tree.iter_tables():
if len(t.array) > 0:
if t.ref is not None:
name = vo_tree.get_table_by_id(t.ref).name
else:
name = t.name
if name not in table_dict.keys():
table_dict[name] = []
table_dict[name] += [t.to_table()]
for name in table_dict.keys():
if len(table_dict[name]) > 1:
table_dict[name] = tbl.vstack(table_dict[name])
else:
table_dict[name] = table_dict[name][0]
return commons.TableList(table_dict)
except:
traceback.print_exc() # temporary for debugging
warnings.warn(
"Error in parsing result, returning raw result instead")
return response.content
class VizierKeyword(list):
"""Helper class for setting keywords for Vizier queries"""
def __init__(self, keywords):
file_name = aud.get_pkg_data_filename(
os.path.join("data", "inverse_dict.json"))
with open(file_name, 'r') as f:
kwd = json.load(f)
self.keyword_types = sorted(kwd.values())
self.keyword_dict = OrderedDict([(k, kwd[k]) for k in sorted(kwd)])
self._keywords = None
self.keywords = keywords
@property
def keywords(self):
"""List or string for keyword(s) that must be set for the Vizier object."""
return self._keywords
@keywords.setter
def keywords(self, values):
if isinstance(values, six.string_types):
values = list(values)
keys = [key.lower() for key in self.keyword_dict]
values = [val.lower() for val in values]
# warn about unknown keywords
for val in set(values) - set(keys):
warnings.warn("{val} : No such keyword".format(val=val))
valid_keys = [
key for key in self.keyword_dict.keys()
if key.lower() in list(map(str.lower, values))]
# create a dict for each type of keyword
set_keywords = OrderedDict()
for key in self.keyword_dict:
if key in valid_keys:
if self.keyword_dict[key] in set_keywords:
set_keywords[self.keyword_dict[key]].append(key)
else:
set_keywords[self.keyword_dict[key]] = [key]
self._keywords = OrderedDict(
[(k, sorted(set_keywords[k]))
for k in set_keywords]
)
@keywords.deleter
def keywords(self):
del self._keywords
def __repr__(self):
return "\n".join([x for key in self.keywords for x in self.get_keyword_str(key)])
def get_keyword_str(self, key):
"""
Helper function that returns the keywords, grouped into appropriate
categories and suitable for the Vizier votable CGI.
Comma-separated is not valid!!!
"""
keyword_name = "-kw." + key
return [keyword_name + "=" + s for s in self.keywords[key]]
def parse_vizier_votable(data,
verbose=False,
invalid='warn',
get_catalog_names=False):
"""
Given a votable as string, parse it into tables
"""
if not verbose:
commons.suppress_vo_warnings()
tf = BytesIO(data)
if invalid == 'mask':
vo_tree = votable.parse(tf, pedantic=False, invalid='mask')
elif invalid == 'warn':
try:
vo_tree = votable.parse(tf, pedantic=False, invalid='exception')
except Exception as ex:
warnings.warn("VOTABLE parsing raised exception: {0}".format(ex))
vo_tree = votable.parse(tf, pedantic=False, invalid='mask')
elif invalid == 'exception':
vo_tree = votable.parse(tf, pedantic=False, invalid='exception')
else:
raise ValueError("Invalid keyword for 'invalid'. "
"Must be exception, mask, or warn")
if get_catalog_names:
return dict([(R.name, R) for R in vo_tree.resources])
else:
table_dict = OrderedDict()
for t in vo_tree.iter_tables():
if len(t.array) > 0:
if t.ref is not None:
name = vo_tree.get_table_by_id(t.ref).name
else:
name = t.name
if name not in table_dict.keys():
table_dict[name] = []
table_dict[name] += [t.to_table()]
for name in table_dict.keys():
if len(table_dict[name]) > 1:
table_dict[name] = tbl.vstack(table_dict[name])
else:
table_dict[name] = table_dict[name][0]
return commons.TableList(table_dict)
def parse_vizier_votable(data, verbose=False, invalid='warn',
get_catalog_names=False):
"""
Given a votable as string, parse it into tables
"""
if not verbose:
commons.suppress_vo_warnings()
tf = BytesIO(data)
if invalid == 'mask':
vo_tree = votable.parse(tf, pedantic=False, invalid='mask')
elif invalid == 'warn':
try:
vo_tree = votable.parse(tf, pedantic=False, invalid='exception')
except Exception as ex:
warnings.warn("VOTABLE parsing raised exception: {0}".format(ex))
vo_tree = votable.parse(tf, pedantic=False, invalid='mask')
elif invalid == 'exception':
vo_tree = votable.parse(tf, pedantic=False, invalid='exception')
else:
raise ValueError("Invalid keyword for 'invalid'. "
"Must be exception, mask, or warn")
if get_catalog_names:
return dict([(R.name, R) for R in vo_tree.resources])
else:
table_dict = OrderedDict()
for t in vo_tree.iter_tables():
if len(t.array) > 0:
if t.ref is not None:
name = vo_tree.get_table_by_id(t.ref).name
else:
name = t.name
if name not in table_dict.keys():
table_dict[name] = []
table_dict[name] += [t.to_table()]
for name in table_dict.keys():
if len(table_dict[name]) > 1:
table_dict[name] = tbl.vstack(table_dict[name])
else:
table_dict[name] = table_dict[name][0]
return commons.TableList(table_dict)
def _parse_result(self, response, get_catalog_names=False, verbose=False, invalid='warn'):
"""
Parses the HTTP response to create a `~astropy.table.Table`.
Returns the raw result as a string in case of parse errors.
Parameters
----------
response : `requests.Response`
The response of the HTTP POST request
get_catalog_names : bool
If specified, return only the table names (useful for table
discovery)
invalid : 'warn', 'mask' or 'raise'
The behavior if a VOTABLE cannot be parsed. Default is 'warn',
which will try to parse the table, then if an exception is raised,
it will be printent but the masked table will be returned
Returns
-------
table_list : `astroquery.utils.TableList` or str
If there are errors in the parsing, then returns the raw results as a string.
"""
if not verbose:
commons.suppress_vo_warnings()
try:
tf = six.BytesIO(response.content)
if invalid == 'mask':
vo_tree = votable.parse(tf, pedantic=False, invalid='mask')
elif invalid == 'warn':
try:
vo_tree = votable.parse(tf, pedantic=False, invalid='raise')
except Exception as ex:
warnings.warn("VOTABLE parsing raised exception: {0}".format(ex))
vo_tree = votable.parse(tf, pedantic=False, invalid='mask')
elif invalid == 'raise':
vo_tree = votable.parse(tf, pedantic=False, invalid='raise')
else:
raise ValueError("Invalid keyword 'invalid'. Must be raise, mask, or warn")
if get_catalog_names:
return dict([(R.name, R) for R in vo_tree.resources])
else:
table_dict = OrderedDict()
for t in vo_tree.iter_tables():
if len(t.array) > 0:
if t.ref is not None:
name = vo_tree.get_table_by_id(t.ref).name
else:
name = t.name
if name not in table_dict.keys():
table_dict[name] = []
table_dict[name] += [t.to_table()]
for name in table_dict.keys():
if len(table_dict[name]) > 1:
table_dict[name] = tbl.vstack(table_dict[name])
else:
table_dict[name] = table_dict[name][0]
return commons.TableList(table_dict)
except Exception as ex:
self.response = response
self.table_parse_error = ex
raise TableParseError("Failed to parse VIZIER result! The raw response can be found "
"in self.response, and the error in self.table_parse_error."
" The attempted parsed result is in self.parsed_result.\n"
"Exception: " + str(self.table_parse_error))
def read_table_fits(input, hdu=None):
"""
Read a Table object from an FITS file
Parameters
----------
input : str or file-like object or compatible `astropy.io.fits` HDU object
If a string, the filename to read the table from. If a file object, or
a compatible HDU object, the object to extract the table from. The
following `astropy.io.fits` HDU objects can be used as input:
- :class:`~astropy.io.fits.hdu.table.TableHDU`
- :class:`~astropy.io.fits.hdu.table.BinTableHDU`
- :class:`~astropy.io.fits.hdu.table.GroupsHDU`
- :class:`~astropy.io.fits.hdu.hdulist.HDUList`
hdu : int or str, optional
The HDU to read the table from.
"""
if isinstance(input, basestring):
input = fits_open(input)
to_close = input
else:
to_close = None
if hasattr(input, 'read'):
input = fits_open(input)
try:
# Parse all table objects
tables = OrderedDict()
if isinstance(input, HDUList):
for ihdu, hdu_item in enumerate(input):
if isinstance(hdu_item, (TableHDU, BinTableHDU, GroupsHDU)):
tables[ihdu] = hdu_item
if len(tables) > 1:
if hdu is None:
warnings.warn("hdu= was not specified but multiple tables"
" are present, reading in first available"
" table (hdu={0})".format(tables.keys()[0]))
hdu = tables.keys()[0]
# hdu might not be an integer, so we first need to convert it
# to the correct HDU index
hdu = input.index_of(hdu)
if hdu in tables:
table = tables[hdu]
else:
raise ValueError("No table found in hdu={0}".format(hdu))
elif len(tables) == 1:
table = tables[tables.keys()[0]]
else:
raise ValueError("No table found")
elif isinstance(input, (TableHDU, BinTableHDU, GroupsHDU)):
table = input
else:
raise ValueError("Input should be a string, a file-like object, "
"an HDUList, TableHDU, BinTableHDU, or "
"GroupsHDU instance")
# Check if table is masked
masked = False
for col in table.columns:
if col.null is not None:
masked = True
break
# Convert to an astropy.table.Table object
t = Table(table.data, masked=masked)
# Copy over null values if needed
if masked:
for col in table.columns:
t[col.name].set_fill_value(col.null)
t[col.name].mask[t[col.name] == col.null] = True
# Copy over units
for col in table.columns:
if col.unit is not None:
try:
t[col.name].units = u.Unit(col.unit, format='fits')
except ValueError:
t[col.name].units = u.UnrecognizedUnit(col.unit)
# TODO: deal properly with unsigned integers
for key, value, comment in table.header.cards:
if key in ['COMMENT', 'HISTORY']:
if key in t.meta:
t.meta[key].append(value)
else:
t.meta[key] = [value]
elif key in t.meta: # key is duplicate
if isinstance(t.meta[key], list):
t.meta[key].append(value)
else:
t.meta[key] = [t.meta[key], value]
elif (is_column_keyword(key.upper()) or
key.upper() in REMOVE_KEYWORDS):
pass
else:
t.meta[key] = value
# TODO: implement masking
finally:
if to_close is not None:
to_close.close()
return t
def read_table_fits(input, hdu=None):
"""
Read a Table object from an FITS file
Parameters
----------
input : str or file-like object or compatible `astropy.io.fits` HDU object
If a string, the filename to read the table from. If a file object, or
a compatible HDU object, the object to extract the table from. The
following `astropy.io.fits` HDU objects can be used as input:
- :class:`~astropy.io.fits.hdu.table.TableHDU`
- :class:`~astropy.io.fits.hdu.table.BinTableHDU`
- :class:`~astropy.io.fits.hdu.table.GroupsHDU`
- :class:`~astropy.io.fits.hdu.hdulist.HDUList`
hdu : int or str, optional
The HDU to read the table from.
"""
if isinstance(input, six.string_types):
input = fits_open(input)
to_close = input
else:
to_close = None
if hasattr(input, 'read'):
input = fits_open(input)
try:
# Parse all table objects
tables = OrderedDict()
if isinstance(input, HDUList):
for ihdu, hdu_item in enumerate(input):
if isinstance(hdu_item, (TableHDU, BinTableHDU, GroupsHDU)):
tables[ihdu] = hdu_item
if len(tables) > 1:
if hdu is None:
warnings.warn("hdu= was not specified but multiple tables"
" are present, reading in first available"
" table (hdu={0})".format(
list(tables.keys())[0]))
hdu = list(tables.keys())[0]
# hdu might not be an integer, so we first need to convert it
# to the correct HDU index
hdu = input.index_of(hdu)
if hdu in tables:
table = tables[hdu]
else:
raise ValueError("No table found in hdu={0}".format(hdu))
elif len(tables) == 1:
table = tables[list(tables.keys())[0]]
else:
raise ValueError("No table found")
elif isinstance(input, (TableHDU, BinTableHDU, GroupsHDU)):
table = input
else:
raise ValueError("Input should be a string, a file-like object, "
"an HDUList, TableHDU, BinTableHDU, or "
"GroupsHDU instance")
# Check if table is masked
masked = False
for col in table.columns:
if col.null is not None:
masked = True
break
# Convert to an astropy.table.Table object
t = Table(table.data, masked=masked)
# Copy over null values if needed
if masked:
for col in table.columns:
t[col.name].set_fill_value(col.null)
t[col.name].mask[t[col.name] == col.null] = True
# Copy over units
for col in table.columns:
if col.unit is not None:
try:
t[col.name].units = u.Unit(col.unit, format='fits')
except ValueError:
t[col.name].units = u.UnrecognizedUnit(col.unit)
# TODO: deal properly with unsigned integers
for key, value, comment in table.header.cards:
if key in ['COMMENT', 'HISTORY']:
if key in t.meta:
t.meta[key].append(value)
else:
t.meta[key] = [value]
elif key in t.meta: # key is duplicate
if isinstance(t.meta[key], list):
t.meta[key].append(value)
else:
t.meta[key] = [t.meta[key], value]
elif (is_column_keyword(key.upper())
or key.upper() in REMOVE_KEYWORDS):
pass
else:
t.meta[key] = value
# TODO: implement masking
finally:
if to_close is not None:
to_close.close()
return t
def _parse_result(self, response, get_catalog_names=False, verbose=False):
"""
Parses the HTTP response to create a `~astropy.table.Table`.
Returns the raw result as a string in case of parse errors.
Parameters
----------
response : `requests.Response`
The response of the HTTP POST request
get_catalog_names : bool
If specified, return only the table names (useful for table
discovery)
Returns
-------
table_list : `astroquery.utils.TableList` or str
If there are errors in the parsing, then returns the raw results as a string.
"""
if not verbose:
commons.suppress_vo_warnings()
try:
tf = tempfile.NamedTemporaryFile()
if six.PY3:
# This is an exceedingly confusing section
# It is likely to be doubly wrong, but has caused issue #185
try:
# Case 1: data is read in as unicode
tf.write(response.content.encode())
except AttributeError:
# Case 2: data is read in as a byte string
tf.write(response.content.decode().encode('utf-8'))
else:
tf.write(response.content.encode('utf-8'))
tf.file.flush()
vo_tree = votable.parse(tf, pedantic=False)
if get_catalog_names:
return dict([(R.name,R) for R in vo_tree.resources])
else:
table_dict = OrderedDict()
for t in vo_tree.iter_tables():
if len(t.array) > 0:
if t.ref is not None:
name = vo_tree.get_table_by_id(t.ref).name
else:
name = t.name
if name not in table_dict.keys():
table_dict[name] = []
table_dict[name] += [t.to_table()]
for name in table_dict.keys():
if len(table_dict[name]) > 1:
table_dict[name] = tbl.vstack(table_dict[name])
else:
table_dict[name] = table_dict[name][0]
return commons.TableList(table_dict)
except Exception as ex:
self.response = response
self.table_parse_error = ex
raise TableParseError("Failed to parse VIZIER result! The raw response can be found "
"in self.response, and the error in self.table_parse_error."
" The attempted parsed result is in self.parsed_result.\n"
"Exception: " + str(self.table_parse_error))
def _parse_result(self, response, get_catalog_names=False, verbose=False):
"""
Parses the HTTP response to create a `~astropy.table.Table`.
Returns the raw result as a string in case of parse errors.
Parameters
----------
response : `requests.Response`
The response of the HTTP POST request
get_catalog_names : bool
If specified, return only the table names (useful for table
discovery)
Returns
-------
table_list : `astroquery.utils.TableList` or str
If there are errors in the parsing, then returns the raw results as a string.
"""
if not verbose:
commons.suppress_vo_warnings()
try:
tf = tempfile.NamedTemporaryFile()
if six.PY3:
# This is an exceedingly confusing section
# It is likely to be doubly wrong, but has caused issue #185
try:
# Case 1: data is read in as unicode
tf.write(response.content.encode())
except AttributeError:
# Case 2: data is read in as a byte string
tf.write(response.content.decode().encode('utf-8'))
else:
tf.write(response.content.encode('utf-8'))
tf.file.flush()
vo_tree = votable.parse(tf, pedantic=False)
if get_catalog_names:
return dict([(R.name, R) for R in vo_tree.resources])
else:
table_dict = OrderedDict()
for t in vo_tree.iter_tables():
if len(t.array) > 0:
if t.ref is not None:
name = vo_tree.get_table_by_id(t.ref).name
else:
name = t.name
if name not in table_dict.keys():
table_dict[name] = []
table_dict[name] += [t.to_table()]
for name in table_dict.keys():
if len(table_dict[name]) > 1:
table_dict[name] = tbl.vstack(table_dict[name])
else:
table_dict[name] = table_dict[name][0]
return commons.TableList(table_dict)
except Exception as ex:
self.response = response
self.table_parse_error = ex
raise TableParseError(
"Failed to parse VIZIER result! The raw response can be found "
"in self.response, and the error in self.table_parse_error."
" The attempted parsed result is in self.parsed_result.\n"
"Exception: " + str(self.table_parse_error))