def _parse_result(self, response):
data = StringIO(BeautifulSoup(response.text).find('pre').text.strip())
# `header` is e.g.:
# "u'-LAMBDA-VAC-ANG-|-SPECTRUM--|TT|--------TERM---------|---J-J---|----LEVEL-ENERGY--CM-1----'"
# `colnames` is then
# [u'LAMBDA VAC ANG', u'SPECTRUM', u'TT', u'TERM', u'J J',
# u'LEVEL ENERGY CM 1']
header = data.readline().strip().strip('|')
colnames = [
colname.strip('-').replace('-', ' ')
for colname in header.split('|') if colname.strip()
]
indices = [i for i, c in enumerate(header) if c == '|']
input = []
for line in data:
row = []
for start, end in zip([0] + indices, indices + [None]):
# `value` will hold all cell values in the line, so
# `u'1.010799'`, `u'Zn XXX'` etc.
value = line[start:end].strip()
if value:
row.append(value)
if row:
input.append('\t'.join(row))
if input:
return ascii.read(input,
data_start=0,
delimiter='\t',
names=colnames)
else:
# return an empty table if the query yielded no results
return Table()
def _parse_result(self, response):
data = StringIO(BeautifulSoup(response.text).find('pre').text.strip())
# `header` is e.g.:
# "u'-LAMBDA-VAC-ANG-|-SPECTRUM--|TT|--------TERM---------|---J-J---|----LEVEL-ENERGY--CM-1----'"
# `colnames` is then
# [u'LAMBDA VAC ANG', u'SPECTRUM', u'TT', u'TERM', u'J J',
# u'LEVEL ENERGY CM 1']
header = data.readline().strip().strip('|')
colnames = [colname.strip('-').replace('-', ' ')
for colname in header.split('|') if colname.strip()]
indices = [i for i, c in enumerate(header) if c == '|']
input = []
for line in data:
row = []
for start, end in zip([0] + indices, indices + [None]):
# `value` will hold all cell values in the line, so
# `u'1.010799'`, `u'Zn XXX'` etc.
value = line[start:end].strip()
if value:
row.append(value)
else:
# maintain table dimensions when data missing
row.append('None')
if row:
input.append('\t'.join(row))
if input:
return ascii.read(input, data_start=0, delimiter='\t',
names=colnames)
else:
# return an empty table if the query yielded no results
return Table()
def retrieve(url, outfile, opener=None, overwrite=False):
"""
"retrieve" (i.e., download to file) a URL.
"""
if opener is None:
opener = urllib.build_opener()
page = opener.open(url)
results = chunk_read(page, report_hook=chunk_report)
S = StringIO(results)
try:
fitsfile = fits.open(S,ignore_missing_end=True)
except IOError:
S.seek(0)
G = gzip.GzipFile(fileobj=S)
fitsfile = fits.open(G,ignore_missing_end=True)
fitsfile.writeto(outfile, clobber=overwrite)
def query_survey(self, survey, **kwargs):
"""
Query survey Phase 3 data contained in the ESO archive.
Parameters
----------
survey : string
Name of the survey to query, one of the names returned by
`list_surveys()`.
Returns
-------
table : `~astropy.table.Table` or `None`
A table representing the data available in the archive for the
specified survey, matching the constraints specified in ``kwargs``.
The number of rows returned is capped by the ROW_LIMIT
configuration item. `None` is returned when the query has no
results.
"""
if survey not in self.list_surveys():
raise ValueError("Survey %s is not in the survey list." % survey)
url = "http://archive.eso.org/wdb/wdb/adp/phase3_main/form"
survey_form = self.request("GET", url)
query_dict = kwargs
query_dict["wdbo"] = "csv/download"
query_dict['phase3_program'] = survey
if self.ROW_LIMIT >= 0:
query_dict["max_rows_returned"] = self.ROW_LIMIT
else:
query_dict["max_rows_returned"] = 10000
survey_response = self._activate_form(survey_form,
form_index=0,
inputs=query_dict)
if _check_response(survey_response.content):
table = ascii.read(StringIO(
survey_response.content.decode(survey_response.encoding)),
format='csv',
comment='#',
delimiter=',',
header_start=1)
return table
else:
warnings.warn("Query returned no results")
def test_read_lc():
from astropy.extern.six import StringIO
f = StringIO("""
@id 1
@RA 36.0
@description good
time band flux fluxerr zp zpsys
50000. g 1. 0.1 25. ab
50000.1 r 2. 0.1 25. ab
""")
t = sncosmo.read_lc(f, format='ascii')
assert str(t) == (" time band flux fluxerr zp zpsys\n"
"------- ---- ---- ------- ---- -----\n"
"50000.0 g 1.0 0.1 25.0 ab\n"
"50000.1 r 2.0 0.1 25.0 ab")
assert t.meta['id'] == 1
assert t.meta['RA'] == 36.0
assert t.meta['description'] == 'good'
def _parse_result(self, response, verbose=False):
"""
Parse a VOtable response
"""
if not verbose:
commons.suppress_vo_warnings()
if 'run?' in response.url:
if response.text == "":
raise RemoteServiceError("Empty return.")
# this is a CSV-like table returned via a direct browser request
import pandas
table = Table.from_pandas(pandas.read_csv(StringIO(response.text)))
else:
fixed_content = self._hack_bad_arraysize_vofix(response.content)
tf = six.BytesIO(fixed_content)
vo_tree = votable.parse(tf, pedantic=False, invalid='mask')
first_table = vo_tree.get_first_table()
table = first_table.to_table(use_names_over_ids=True)
return table
def download_list_of_fitsfiles(linklist, output_directory=None,
output_prefix=None, save=False, overwrite=False, verbose=False,
output_coord_format=None, filename_header_keywords=None,
include_input_filename=True):
"""
Given a list of file URLs, download them and (optionally) rename them.
Examples
--------
>>> linklist = ['http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH00.fits',
... 'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH01.fits',
... 'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_SC00.fits']
>>> download_list_of_fitsfiles(linklist,
... output_directory='fermi_m31',
... output_prefix='FermiLAT',
... save=True,
... overwrite=False,
... verbose=True,
... output_coord_format=None, # FITS tables don't have crval/crpix, good one is: "%08.3g%+08.3g",
... filename_header_keywords=None, # couldn't find any useful ones
... include_input_filename=True)
"""
# Loop through links and retrieve FITS images
images = {}
for link in linklist:
if output_directory is None:
output_directory = ""
elif output_directory[-1] != "/":
output_directory += "/"
if not os.path.exists(output_directory):
os.mkdir(output_directory)
with get_readable_fileobj(link, cache=True) as f:
results = f.read()
S = StringIO(results)
try:
# try to open as a fits file
fitsfile = fits.open(S, ignore_missing_end=True)
except IOError:
# if that fails, try to open as a gzip'd fits file
# have to rewind to the start
S.seek(0)
G = gzip.GzipFile(fileobj=S)
fitsfile = fits.open(G, ignore_missing_end=True)
# Get Multiframe ID from the header
images[link] = fitsfile
if save:
h0 = fitsfile[0].header
if filename_header_keywords: # is not None or empty
nametxt = "_".join([validify_filename(str(h0[key])) for key in filename_header_keywords])
else:
nametxt = ""
if output_coord_format:
lon = h0['CRVAL1']
lat = h0['CRVAL2']
# this part will eventually be handled by astropy.coordinates directly
# ctype = h0['CTYPE1']
# if 'RA' in ctype:
# coordinate = coord.SkyCoord(lon,lat,unit=('deg','deg'))
# elif 'GLON' in ctype:
# coordinate = coord.Galactic(lon,lat,unit=('deg','deg'))
# else:
# raise TypeError("Don't recognize ctype %s" % ctype)
# coordstr = coordinate.format(output_coord_format)
try:
coordstr = output_coord_format.format(lon, lat)
except TypeError:
coordstr = output_coord_format % (lon, lat)
nametxt += "_" + coordstr
if include_input_filename:
filename_root = os.path.split(link)[1]
else:
filename_root = ""
savename = output_prefix if output_prefix else ""
savename += nametxt
savename += "_" + filename_root
# Set final directory and file names
final_file = output_directory + savename
if verbose:
print("Saving file %s" % final_file)
try:
fitsfile.writeto(final_file, clobber=overwrite)
except IOError:
print("Skipped writing file {0} because it exists "
"and overwrite=False".format(final_file))
return images
def download_list_of_fitsfiles(linklist,
output_directory=None,
output_prefix=None,
save=False,
overwrite=False,
verbose=False,
output_coord_format=None,
filename_header_keywords=None,
include_input_filename=True):
"""
Given a list of file URLs, download them and (optionally) rename them.
Examples
--------
>>> linklist = ['http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH00.fits',
... 'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH01.fits',
... 'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_SC00.fits']
>>> download_list_of_fitsfiles(linklist,
... output_directory='fermi_m31',
... output_prefix='FermiLAT',
... save=True,
... overwrite=False,
... verbose=True,
... output_coord_format=None, # FITS tables don't have crval/crpix, good one is: "%08.3g%+08.3g",
... filename_header_keywords=None, # couldn't find any useful ones
... include_input_filename=True)
"""
# Loop through links and retrieve FITS images
images = {}
for link in linklist:
if output_directory is None:
output_directory = ""
elif output_directory[-1] != "/":
output_directory += "/"
if not os.path.exists(output_directory):
os.mkdir(output_directory)
with get_readable_fileobj(link, cache=True) as f:
results = f.read()
S = StringIO(results)
try:
# try to open as a fits file
fitsfile = fits.open(S, ignore_missing_end=True)
except IOError:
# if that fails, try to open as a gzip'd fits file
# have to rewind to the start
S.seek(0)
G = gzip.GzipFile(fileobj=S)
fitsfile = fits.open(G, ignore_missing_end=True)
# Get Multiframe ID from the header
images[link] = fitsfile
if save:
h0 = fitsfile[0].header
if filename_header_keywords: # is not None or empty
nametxt = "_".join([
validify_filename(str(h0[key]))
for key in filename_header_keywords
])
else:
nametxt = ""
if output_coord_format:
lon = h0['CRVAL1']
lat = h0['CRVAL2']
# this part will eventually be handled by astropy.coordinates directly
# ctype = h0['CTYPE1']
# if 'RA' in ctype:
# coordinate = coord.ICRS(lon,lat,unit=('deg','deg'))
# elif 'GLON' in ctype:
# coordinate = coord.Galactic(lon,lat,unit=('deg','deg'))
# else:
# raise TypeError("Don't recognize ctype %s" % ctype)
# coordstr = coordinate.format(output_coord_format)
try:
coordstr = output_coord_format.format(lon, lat)
except TypeError:
coordstr = output_coord_format % (lon, lat)
nametxt += "_" + coordstr
if include_input_filename:
filename_root = os.path.split(link)[1]
else:
filename_root = ""
savename = output_prefix if output_prefix else ""
savename += nametxt
savename += "_" + filename_root
# Set final directory and file names
final_file = output_directory + savename
if verbose:
print("Saving file %s" % final_file)
try:
fitsfile.writeto(final_file, clobber=overwrite)
except IOError:
print("Skipped writing file {0} because it exists "
"and overwrite=False".format(final_file))
return images
