def _parse_result(self, response): data = StringIO(BeautifulSoup(response.text).find('pre').text.strip()) # `header` is e.g.: # "u'-LAMBDA-VAC-ANG-|-SPECTRUM--|TT|--------TERM---------|---J-J---|----LEVEL-ENERGY--CM-1----'" # `colnames` is then # [u'LAMBDA VAC ANG', u'SPECTRUM', u'TT', u'TERM', u'J J', # u'LEVEL ENERGY CM 1'] header = data.readline().strip().strip('|') colnames = [ colname.strip('-').replace('-', ' ') for colname in header.split('|') if colname.strip() ] indices = [i for i, c in enumerate(header) if c == '|'] input = [] for line in data: row = [] for start, end in zip([0] + indices, indices + [None]): # `value` will hold all cell values in the line, so # `u'1.010799'`, `u'Zn XXX'` etc. value = line[start:end].strip() if value: row.append(value) if row: input.append('\t'.join(row)) if input: return ascii.read(input, data_start=0, delimiter='\t', names=colnames) else: # return an empty table if the query yielded no results return Table()
def _parse_result(self, response): data = StringIO(BeautifulSoup(response.text).find('pre').text.strip()) # `header` is e.g.: # "u'-LAMBDA-VAC-ANG-|-SPECTRUM--|TT|--------TERM---------|---J-J---|----LEVEL-ENERGY--CM-1----'" # `colnames` is then # [u'LAMBDA VAC ANG', u'SPECTRUM', u'TT', u'TERM', u'J J', # u'LEVEL ENERGY CM 1'] header = data.readline().strip().strip('|') colnames = [colname.strip('-').replace('-', ' ') for colname in header.split('|') if colname.strip()] indices = [i for i, c in enumerate(header) if c == '|'] input = [] for line in data: row = [] for start, end in zip([0] + indices, indices + [None]): # `value` will hold all cell values in the line, so # `u'1.010799'`, `u'Zn XXX'` etc. value = line[start:end].strip() if value: row.append(value) else: # maintain table dimensions when data missing row.append('None') if row: input.append('\t'.join(row)) if input: return ascii.read(input, data_start=0, delimiter='\t', names=colnames) else: # return an empty table if the query yielded no results return Table()
def retrieve(url, outfile, opener=None, overwrite=False): """ "retrieve" (i.e., download to file) a URL. """ if opener is None: opener = urllib.build_opener() page = opener.open(url) results = chunk_read(page, report_hook=chunk_report) S = StringIO(results) try: fitsfile = fits.open(S,ignore_missing_end=True) except IOError: S.seek(0) G = gzip.GzipFile(fileobj=S) fitsfile = fits.open(G,ignore_missing_end=True) fitsfile.writeto(outfile, clobber=overwrite)
def query_survey(self, survey, **kwargs): """ Query survey Phase 3 data contained in the ESO archive. Parameters ---------- survey : string Name of the survey to query, one of the names returned by `list_surveys()`. Returns ------- table : `~astropy.table.Table` or `None` A table representing the data available in the archive for the specified survey, matching the constraints specified in ``kwargs``. The number of rows returned is capped by the ROW_LIMIT configuration item. `None` is returned when the query has no results. """ if survey not in self.list_surveys(): raise ValueError("Survey %s is not in the survey list." % survey) url = "http://archive.eso.org/wdb/wdb/adp/phase3_main/form" survey_form = self.request("GET", url) query_dict = kwargs query_dict["wdbo"] = "csv/download" query_dict['phase3_program'] = survey if self.ROW_LIMIT >= 0: query_dict["max_rows_returned"] = self.ROW_LIMIT else: query_dict["max_rows_returned"] = 10000 survey_response = self._activate_form(survey_form, form_index=0, inputs=query_dict) if _check_response(survey_response.content): table = ascii.read(StringIO( survey_response.content.decode(survey_response.encoding)), format='csv', comment='#', delimiter=',', header_start=1) return table else: warnings.warn("Query returned no results")
def test_read_lc(): from astropy.extern.six import StringIO f = StringIO(""" @id 1 @RA 36.0 @description good time band flux fluxerr zp zpsys 50000. g 1. 0.1 25. ab 50000.1 r 2. 0.1 25. ab """) t = sncosmo.read_lc(f, format='ascii') assert str(t) == (" time band flux fluxerr zp zpsys\n" "------- ---- ---- ------- ---- -----\n" "50000.0 g 1.0 0.1 25.0 ab\n" "50000.1 r 2.0 0.1 25.0 ab") assert t.meta['id'] == 1 assert t.meta['RA'] == 36.0 assert t.meta['description'] == 'good'
def _parse_result(self, response, verbose=False): """ Parse a VOtable response """ if not verbose: commons.suppress_vo_warnings() if 'run?' in response.url: if response.text == "": raise RemoteServiceError("Empty return.") # this is a CSV-like table returned via a direct browser request import pandas table = Table.from_pandas(pandas.read_csv(StringIO(response.text))) else: fixed_content = self._hack_bad_arraysize_vofix(response.content) tf = six.BytesIO(fixed_content) vo_tree = votable.parse(tf, pedantic=False, invalid='mask') first_table = vo_tree.get_first_table() table = first_table.to_table(use_names_over_ids=True) return table
def download_list_of_fitsfiles(linklist, output_directory=None, output_prefix=None, save=False, overwrite=False, verbose=False, output_coord_format=None, filename_header_keywords=None, include_input_filename=True): """ Given a list of file URLs, download them and (optionally) rename them. Examples -------- >>> linklist = ['http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH00.fits', ... 'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH01.fits', ... 'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_SC00.fits'] >>> download_list_of_fitsfiles(linklist, ... output_directory='fermi_m31', ... output_prefix='FermiLAT', ... save=True, ... overwrite=False, ... verbose=True, ... output_coord_format=None, # FITS tables don't have crval/crpix, good one is: "%08.3g%+08.3g", ... filename_header_keywords=None, # couldn't find any useful ones ... include_input_filename=True) """ # Loop through links and retrieve FITS images images = {} for link in linklist: if output_directory is None: output_directory = "" elif output_directory[-1] != "/": output_directory += "/" if not os.path.exists(output_directory): os.mkdir(output_directory) with get_readable_fileobj(link, cache=True) as f: results = f.read() S = StringIO(results) try: # try to open as a fits file fitsfile = fits.open(S, ignore_missing_end=True) except IOError: # if that fails, try to open as a gzip'd fits file # have to rewind to the start S.seek(0) G = gzip.GzipFile(fileobj=S) fitsfile = fits.open(G, ignore_missing_end=True) # Get Multiframe ID from the header images[link] = fitsfile if save: h0 = fitsfile[0].header if filename_header_keywords: # is not None or empty nametxt = "_".join([validify_filename(str(h0[key])) for key in filename_header_keywords]) else: nametxt = "" if output_coord_format: lon = h0['CRVAL1'] lat = h0['CRVAL2'] # this part will eventually be handled by astropy.coordinates directly # ctype = h0['CTYPE1'] # if 'RA' in ctype: # coordinate = coord.SkyCoord(lon,lat,unit=('deg','deg')) # elif 'GLON' in ctype: # coordinate = coord.Galactic(lon,lat,unit=('deg','deg')) # else: # raise TypeError("Don't recognize ctype %s" % ctype) # coordstr = coordinate.format(output_coord_format) try: coordstr = output_coord_format.format(lon, lat) except TypeError: coordstr = output_coord_format % (lon, lat) nametxt += "_" + coordstr if include_input_filename: filename_root = os.path.split(link)[1] else: filename_root = "" savename = output_prefix if output_prefix else "" savename += nametxt savename += "_" + filename_root # Set final directory and file names final_file = output_directory + savename if verbose: print("Saving file %s" % final_file) try: fitsfile.writeto(final_file, clobber=overwrite) except IOError: print("Skipped writing file {0} because it exists " "and overwrite=False".format(final_file)) return images
def download_list_of_fitsfiles(linklist, output_directory=None, output_prefix=None, save=False, overwrite=False, verbose=False, output_coord_format=None, filename_header_keywords=None, include_input_filename=True): """ Given a list of file URLs, download them and (optionally) rename them. Examples -------- >>> linklist = ['http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH00.fits', ... 'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_PH01.fits', ... 'http://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L130413170713F15B52BC06_SC00.fits'] >>> download_list_of_fitsfiles(linklist, ... output_directory='fermi_m31', ... output_prefix='FermiLAT', ... save=True, ... overwrite=False, ... verbose=True, ... output_coord_format=None, # FITS tables don't have crval/crpix, good one is: "%08.3g%+08.3g", ... filename_header_keywords=None, # couldn't find any useful ones ... include_input_filename=True) """ # Loop through links and retrieve FITS images images = {} for link in linklist: if output_directory is None: output_directory = "" elif output_directory[-1] != "/": output_directory += "/" if not os.path.exists(output_directory): os.mkdir(output_directory) with get_readable_fileobj(link, cache=True) as f: results = f.read() S = StringIO(results) try: # try to open as a fits file fitsfile = fits.open(S, ignore_missing_end=True) except IOError: # if that fails, try to open as a gzip'd fits file # have to rewind to the start S.seek(0) G = gzip.GzipFile(fileobj=S) fitsfile = fits.open(G, ignore_missing_end=True) # Get Multiframe ID from the header images[link] = fitsfile if save: h0 = fitsfile[0].header if filename_header_keywords: # is not None or empty nametxt = "_".join([ validify_filename(str(h0[key])) for key in filename_header_keywords ]) else: nametxt = "" if output_coord_format: lon = h0['CRVAL1'] lat = h0['CRVAL2'] # this part will eventually be handled by astropy.coordinates directly # ctype = h0['CTYPE1'] # if 'RA' in ctype: # coordinate = coord.ICRS(lon,lat,unit=('deg','deg')) # elif 'GLON' in ctype: # coordinate = coord.Galactic(lon,lat,unit=('deg','deg')) # else: # raise TypeError("Don't recognize ctype %s" % ctype) # coordstr = coordinate.format(output_coord_format) try: coordstr = output_coord_format.format(lon, lat) except TypeError: coordstr = output_coord_format % (lon, lat) nametxt += "_" + coordstr if include_input_filename: filename_root = os.path.split(link)[1] else: filename_root = "" savename = output_prefix if output_prefix else "" savename += nametxt savename += "_" + filename_root # Set final directory and file names final_file = output_directory + savename if verbose: print("Saving file %s" % final_file) try: fitsfile.writeto(final_file, clobber=overwrite) except IOError: print("Skipped writing file {0} because it exists " "and overwrite=False".format(final_file)) return images