def download_cutouts(sbid, username, password, destination_dir, catalogue_query, do_cutouts, cutout_radius_degrees=0.1): # 2) Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id print ("\n\n** Finding images and image cubes for scheduling block {} ... \n\n".format(sbid)) data_product_id_query = "select * from ivoa.obscore where obs_id = '" + str( sbid) + "' and dataproduct_type = 'cube' and dataproduct_subtype in ('cont.restored.t0', 'spectral.restored.3d')" filename = destination_dir + "image_cubes_" + str(sbid) + ".xml" casda.sync_tap_query(data_product_id_query, filename, username, password) image_cube_votable = parse(filename, pedantic=False) results_array = image_cube_votable.get_table_by_id('results').array service = 'cutout_service' if do_cutouts else 'async_service' # 3) For each of the image cubes, query datalink to get the secure datalink details print ("\n\n** Retrieving datalink for each image and image cube...\n\n") authenticated_id_tokens = [] for image_cube_result in results_array: image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8') async_url, authenticated_id_token = casda.get_service_link_and_id(image_cube_id, username, password, service=service, destination_dir=destination_dir) if authenticated_id_token is not None and len(authenticated_id_tokens) < 10: authenticated_id_tokens.append(authenticated_id_token) if len(authenticated_id_tokens) == 0: print ("No image cubes for scheduling_block_id " + str(sbid)) return 1 # Run the catalogue_query to find catalogue entries that are of interest if do_cutouts: print ("\n\n** Finding components in each image and image cube...\n\n") filename = destination_dir + "catalogue_query_" + str(sbid) + ".xml" casda.sync_tap_query(catalogue_query, filename, username, password) catalogue_vo_table = parse(filename, pedantic=False) catalogue_results_array = catalogue_vo_table.get_table_by_id('results').array print ("\n\n** Found %d components...\n\n" % (len(catalogue_results_array))) if len(catalogue_results_array) == 0: print ("No catalogue entries matching the criteria found for scheduling_block_id " + str(sbid)) return 1 # For each source found in the catalogue query, create a position filter pos_list = [] for entry in catalogue_results_array: ra = entry['ra_deg_cont'] dec = entry['dec_deg_cont'] circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str(cutout_radius_degrees) pos_list.append(circle) # Generate cutouts from each image around each source # where there is no overlap an error file is generated but can be ignored. job_location = casda.create_async_soda_job(authenticated_id_tokens) if do_cutouts: casda.add_params_to_async_job(job_location, 'pos', pos_list) job_status = casda.run_async_job(job_location) print ('\nJob finished with status %s address is %s\n\n' % (job_status, job_location)) if job_status != 'ERROR': casda.download_all(job_location, destination_dir) return 0
def get_vizier_resume(): with cd(PATH): bzip2("-f", "-dk", "vizier_votable.vot.bz2") votable = parse("vizier_votable.vot") table = votable.get_first_table().to_table(use_names_over_ids=True) df = table.to_pandas() del votable, table df = df[["OID", "RAJ2000", "DEJ2000", "Type"]].copy() df.columns = "ID", "ra", "dec", "cls" df["catalog"] = "vizier" # solo las RRLyrae flt = 'RRAB', 'RRC', 'RRD' df = df[df.cls.isin(flt)] def change_type(t): subpart = {'RRAB': "RRab", 'RRC': "RRc", 'RRD': "RRd"}[t] return "RRLyr-" + subpart df["cls"] = df.cls.apply(change_type) # ~ os.remove("vizier_votable.vot") return df
def get_output(self, astable=True): from astropy.io import ascii if hasattr(self.cfg.CATALOG_NAME, 'content'): if self.verbose: print("Getting output from stored content") content = self.cfg.CATALOG_NAME.content else: if self.verbose: print("Can't get output if a proxy output was not used - " "reading from file " + self.cfg.CATALOG_NAME) if self.renameoutputs: catfn = self.get_renamed_output_fns()[0].values()[0] else: catfn = self.cfg.CATALOG_NAME with open(catfn, 'rb') as f: content = f.read() if astable: lcattype = self.cfg.CATALOG_TYPE.lower() if 'votable' in lcattype: from astropy.io import votable from io import BytesIO return votable.parse(BytesIO(content)) elif 'ascii' in lcattype: return ascii.read(content, Reader=ascii.SExtractor) elif 'fits' in lcattype: from astropy.io import fits return fits.HDUList.fromstring(content)[2].data else: return content
def make_Dias2014_cut_vot(): inpath = ( '/home/luke/local/tess-trex/catalogs/' 'Dias_2014_prob_gt_50_pct_vizier.vot') outpath = ('../data/cluster_data/' 'Dias_2014_prob_gt_50_pct_to_gaia_archive.vot') if os.path.exists(outpath): print('found {}, skipping'.format(outpath)) return tab = parse(inpath) t = tab.get_first_table().to_table() J_mag = np.array(t['Jmag']) Ks_mag = np.array(t['Kmag']) # is actually Ks G_mag_estimate = estimate_Gaia_G_given_2mass_J_Ks(J_mag, Ks_mag) okinds = np.isfinite(G_mag_estimate) d14_ucac4_ids = np.array(t['UCAC4'])[okinds] #TODO: check if J2015 transformation needed. ra_deg = np.array(t['RAJ2000'])[okinds] dec_deg = np.array(t['DEJ2000'])[okinds] import IPython; IPython.embed() outtab = Table() outtab['ra'] = ra_deg*u.deg outtab['dec'] = dec_deg*u.deg outtab['gmag_estimate'] = G_mag_estimate[okinds]*u.mag outtab['ucac_id'] = d14_ucac4_ids v_outtab = from_table(outtab) writeto(v_outtab, outpath) print('--> made {}'.format(outpath))
def make_Dias2014_cut_csv(): inpath = ( '/home/luke/local/tess-trex/catalogs/' 'Dias_2014_prob_gt_50_pct_vizier.vot') outpath = ('../data/cluster_data/' 'Dias_2014_prob_gt_50_pct_to_gaia_archive.csv') if os.path.exists(outpath): print('found {}, skipping'.format(outpath)) return tab = parse(inpath) t = tab.get_first_table().to_table() J_mag = np.array(t['Jmag']) Ks_mag = np.array(t['Kmag']) # is actually Ks G_mag_estimate = estimate_Gaia_G_given_2mass_J_Ks(J_mag, Ks_mag) okinds = np.isfinite(G_mag_estimate) d14_ucac4_ids = np.array(t['UCAC4'])[okinds] #TODO: check if J2015 transformation needed. ra_deg = np.array(t['RAJ2000'])[okinds] dec_deg = np.array(t['DEJ2000'])[okinds] outdf = pd.DataFrame( {'RA':ra_deg, 'DEC':dec_deg, 'gmag_estimate':G_mag_estimate[okinds], 'ucac_id':d14_ucac4_ids}) outdf.to_csv(outpath, index=False) print('--> made {}'.format(outpath))
def GaiaCollaboration2018_clusters_to_csv(): inpaths = [ os.path.join(clusterdatadir,'GaiaCollaboration2018_616_A10_tablea1b_beyond_250pc.vot'), os.path.join(clusterdatadir,'GaiaCollaboration2018_616_A10_tablea1a_within_250pc.vot') ] for inpath in inpaths: tab = parse(inpath) t = tab.get_first_table().to_table() df = pd.DataFrame({'source':t['Source'], 'cluster':t['Cluster']}) cnames = np.array(df['cluster'].str.decode('utf-8')) # you don't want the globular clusters from this work! bad_ngcs = [104, 288, 362, 1851, 5272, 5904, 6205, 6218, 6341, 6397, 6656, 6752, 6809, 7099] bad_ngcs = ['NGC{:s}'.format(str(bn).zfill(4)) for bn in bad_ngcs] # however, I manually checked: none of the globular clusters are in the # these source catalogs. outpath = inpath.replace('.vot','_cut_only_source_cluster.csv') df.to_csv(outpath, index=False) print('made {}'.format(outpath))
def find_images(pos_criteria, username, password): """ Run an SIA2 query against CASDA to find images and cubes that contain any of the specified locations. See http://www.ivoa.net/documents/SIA/ for how to specify criteria. :param pos_criteria: An array of POS criteria (CIRCLE, POLYGON or RANGE) specifying the locations to be found. :param username: The OPAL username of the user. :param password: The OPAL password of the user. :return: A VOTableFile object containing the SIA2 response. This will list the images along with extensive metadata. """ url = _casda_query_base_url + _sia2_endpoint req = urllib2.Request(url) # Uses basic auth to securely access the data access information for the image cube base64string = base64.encodestring('%s:%s' % (username, password)).replace( '\n', '') req.add_header("Authorization", "Basic %s" % base64string) params = list(map((lambda value: ('POS', value)), pos_criteria)) data = urllib.urlencode(params) print(url) response = urllib2.urlopen(req, data) filename = 'temp/sia-resp.xml' with open(filename, 'wb') as f: f.write(response.read()) votable = parse(filename, pedantic=False) return votable
def _make_fov_footprint(self, footprint): """Making the instrument field of view footprint using a template from http://aladin.u-strasbg.fr/footprint_editor/""" votable = parse(footprint) # reading footprint template table = votable.get_first_table() for param in table.params: # retrieving table.params param # box or circle footprint if param.ID == 'radius': param.value = self._fov_radius*3600.0 votable.to_xml('user_fov.vot') # VOTable file output else: data = table.array fov_width_arcsec = self._fov_width*3600.0 fov_height_arcsec = self._fov_height*3600.0 data[0] = - fov_width_arcsec / 2.0, fov_height_arcsec / 2.0 data[1] = fov_width_arcsec / 2.0, fov_height_arcsec / 2.0 data[2] = fov_width_arcsec / 2.0, - fov_height_arcsec / 2.0 data[3] = - fov_width_arcsec / 2.0, - fov_height_arcsec / 2.0 votable.to_xml('user_fov.vot') # VOTable file output return samp.send_file( 'user_fov.vot' ) # sending to Aladin
def read_sources(filename, min_sn=10, min_flux=0.02): print("Extracting sources from " + filename) sources = [] if not os.path.exists(filename): print ("Warning: File %s does not exist, skipping source read." % \ filename) return sources src_votable = votable.parse(filename, pedantic=False) results = src_votable.get_first_table().array for row in results: id = str(row['island']) + "-" + str(row['source']) ra = row['ra'] dec = row['dec'] rms = row['local_rms'] flux = row['peak_flux'] sn = flux / rms print("Found source %s at %.4f, %.4f with flux %.4f and rms of %.4f " "giving S/N of %.4f" % (id, ra, dec, flux, rms, sn)) if sn > min_sn and flux > min_flux: src = dict(zip(results.dtype.names, row)) src['id'] = id src['sn'] = sn #sources.append([ra, dec, id, flux, row['island']]) sources.append(src) else: print("Ignoring source at %.4f, %.4f due to low S/N of %.4f or " "flux of %.4f" % (ra, dec, sn, flux)) return sources
def votable_format(ra, dec, radius, nearest, result): ztfdic = {} votable = result.text.encode(encoding='UTF-8') bio = io.BytesIO(votable) votable = parse(bio) table = parse_single_table(bio).to_table() if len(table) <= 0: ztfdic['0'] = 'not found' return ztfdic #'not found' tablas = table.group_by('oid') #the most close object to radius if nearest is True: minztf = id_nearest(ra, dec, radius, tablas) buf = io.BytesIO() votable = from_table(tablas.groups[minztf]) writeto(votable, buf) ztfdic[str(tablas.groups[minztf]['oid'][0])] = ( buf.getvalue().decode("utf-8")) return ztfdic # all objects in radius else: for group in tablas.groups: buf = io.BytesIO() votable = from_table(group) writeto(votable, buf) ztfdic[str(group['oid'][0])] = (buf.getvalue().decode("utf-8")) return ztfdic
def query(**kwargs): """ For details, see the SVO FPS documentation [1]. Parameters ---------- ID : Filter ID (e.g. 2MASS/2MASS.H) PhotCalID : Photometric Calibration ID (e.g. 2MASS/2MASS.H/AB) WavelengthMean (range) WavelengthEff (range) WavelengthMin (range) WavelengthMax (range) WidthEff (range) FWHM (range) Instrument (value) Facility (value) PhotSystem (value) [1] http://svo2.cab.inta-csic.es/svo/theory/fps3/index.php?mode=voservice """ url = "http://svo2.cab.inta-csic.es/svo/theory/fps3/fps.php" payload = {key: value for key, value in kwargs.items()} r = requests.get(url, params=payload) return parse(BytesIO(r.content), pedantic=False)
def read_sources(filename): print ("Extracting sources from " + filename) sources = [] if not os.path.exists(filename): print ("Warning: File %s does not exist, skipping source read." % \ filename) return sources src_votable = votable.parse(filename, pedantic=False) results = src_votable.get_first_table().array for row in results: id = str(row['island']) + "-" + str(row['source']) ra = row['ra'] dec = row['dec'] rms = row['local_rms'] flux = row['peak_flux'] sn = flux / rms print ("Found source %s at %.4f, %.4f with flux %.4f and rms of %.4f " "giving S/N of %.4f" % (id, ra, dec, flux, rms, sn)) if sn > 10 and flux > 0.02: src = dict(zip(results.dtype.names,row)) src['id'] = id src['sn'] = sn #sources.append([ra, dec, id, flux, row['island']]) sources.append(src) else: print ("Ignoring source at %.4f, %.4f due to low S/N of %.4f or " "flux of %.4f" % (ra, dec, sn, flux)) return sources
def query_vizier(query_text): # Do the search using VizieR r = requests.post("http://vizier.u-strasbg.fr/viz-bin/votable", { '-words': query_text.split(), '-meta.all': 1 }) # We now loop over the results and construct a list of dictionaries, where # each dictionary contains information about one set of tables. votablefile = parse(BytesIO(r.content)) result = [] for resource in votablefile.resources: catalog_set = {} catalog_set['description'] = resource.description catalog_set['tables'] = [] for table in resource.tables: catalog = {} catalog['description'] = table.description catalog['nrows'] = str(table.nrows) catalog['name'] = str(table.name) catalog_set['tables'].append(catalog) result.append(catalog_set) return result
def test_exec_sync(): # save results in a file # create the VOTable result # example from http://docs.astropy.org/en/stable/io/votable/ votable = VOTableFile() resource = Resource() votable.resources.append(resource) table = Table(votable) resource.tables.append(table) table.fields.extend([ Field(votable, name="filename", datatype="char", arraysize="*"), Field(votable, name="matrix", datatype="double", arraysize="2x2")]) table.create_arrays(2) table.array[0] = ('test1.xml', [[1, 0], [0, 1]]) table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]]) buffer = BytesIO() votable.to_xml(buffer) cadc = Cadc(auth_session=requests.Session()) response = Mock() response.to_table.return_value = buffer.getvalue() cadc.cadctap.search = Mock(return_value=response) output_file = '{}/test_vooutput.xml'.format(tempfile.tempdir) cadc.exec_sync('some query', output_file=output_file) actual = parse(output_file) assert len(votable.resources) == len(actual.resources) == 1 assert len(votable.resources[0].tables) ==\ len(actual.resources[0].tables) == 1 actual_table = actual.resources[0].tables[0] try: # TODO remove when astropy LTS upgraded from astropy.utils.diff import report_diff_values assert report_diff_values(table, actual_table, fileobj=sys.stdout) except ImportError: pass
def _parse_result(self, response, verbose=False): if not verbose: commons.suppress_vo_warnings() # fix to replace non standard datatype 'integer' in returned VOTable # with 'int' to make it parsable by astropy.io.votable integer_re = re.compile(r'datatype="integer"') new_content = integer_re.sub(r'datatype="int"', response.content) # these are pretty bad hacks, but also needed... days_re = re.compile(r'unit="days" datatype="double"') new_content = days_re.sub(r'unit="days" datatype="char" arraysize="*"', new_content) degrees_re = re.compile(r'unit="degrees" datatype="double"') new_content = degrees_re.sub(r'unit="degrees" datatype="char" arraysize="*"', new_content) try: tf = tempfile.NamedTemporaryFile() tf.write(new_content.encode('utf-8')) tf.flush() first_table = votable.parse(tf.name, pedantic=False).get_first_table() try: table = first_table.to_table(use_names_over_ids=True) except TypeError: warnings.warn("NRAO table parsing: astropy versions prior to 6558975c use " "the table column IDs instead of names.") table = first_table.to_table() return table except Exception as ex: self.response = response self.table_parse_error = ex raise TableParseError("Failed to parse NRAO votable result! The raw response can be found " "in self.response, and the error in self.table_parse_error.")
def _parse_result(self, response, verbose=False): if not verbose: commons.suppress_vo_warnings() # fix to replace non standard datatype 'integer' in returned VOTable # with 'int' to make it parsable by astropy.io.votable integer_re = re.compile(r'datatype="integer"') content = response.text new_content = integer_re.sub(r'datatype="int"', content) # these are pretty bad hacks, but also needed... days_re = re.compile(r'unit="days" datatype="double"') new_content = days_re.sub( r'unit="days" datatype="char" arraysize="*"', new_content) degrees_re = re.compile(r'unit="degrees" datatype="double"') new_content = degrees_re.sub( r'unit="degrees" datatype="char" arraysize="*"', new_content) try: tf = six.BytesIO(new_content.encode()) first_table = votable.parse(tf, pedantic=False).get_first_table() try: table = first_table.to_table(use_names_over_ids=True) except TypeError: warnings.warn( "NRAO table parsing: astropy versions prior to 6558975c use " "the table column IDs instead of names.") table = first_table.to_table() return table except Exception as ex: self.response = response self.table_parse_error = ex raise TableParseError( "Failed to parse NRAO votable result! The raw response can be found " "in self.response, and the error in self.table_parse_error.")
def find_images(pos_criteria, username, password): """ Run an SIA2 query against CASDA to find images and cubes that contain any of the specified locations. See http://www.ivoa.net/documents/SIA/ for how to specify criteria. :param pos_criteria: An array of POS criteria (CIRCLE, POLYGON or RANGE) specifying the locations to be found. :param username: The OPAL username of the user. :param password: The OPAL password of the user. :return: A VOTableFile object containing the SIA2 response. This will list the images along with extensive metadata. """ url = _casda_query_base_url + _sia2_endpoint req = urllib2.Request(url) # Uses basic auth to securely access the data access information for the image cube base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '') req.add_header("Authorization", "Basic %s" % base64string) params = list(map((lambda value: ('POS', value)), pos_criteria)) data = urllib.urlencode(params) print(url) response = urllib2.urlopen(req, data) filename = 'temp/sia-resp.xml' with open(filename, 'wb') as f: f.write(response.read()) votable = parse(filename, pedantic=False) return votable
def getLightCurveDataPanSTARRS(coordinates: CoordClass, radius, return_type, column_filters=None): """Pan-STARRS light curve data retrieval The Pan-STARRs catalog API allows the ability to search the Pan-STARRS catalogs. For additional information on the catalogs please visit the Pan-STARRS Data Archive Home Page. Ref. https://outerspace.stsci.edu/display/PANSTARRS/Pan-STARRS1+data+archive+home+page :param coordinates: Coordinates of object expressed CoordClass notation in J2000 RA Dec (Decimal) format. :type coordinates: CoordClass :param radius: Radius of cone search ** in degrees ** for passing to Pan-STARRS :type radius: float :param return_type: For selection of different return types, e.g. "VOTABLE" (Default), "HTML", "CSV" :type return_type: str :param column_filters: Not used currently :returns: (boolean) Valid data return (DataFrame) Data payload :rtype: tuple """ # TODO Note yet completed. Access to Pan-STARRS data requires more development status = True delim = "&" ra = coordinates.ra_str() + delim dec = coordinates.dec_str() + delim radius_str = to_string(radius, 4) + delim if column_filters is None: column_filters = {} base_url = "https://catalogs.mast.stsci.edu/api/v0.1/panstarrs/" query_url_part = "dr1/mean?" url_pos = ra + dec + radius_str url_bandname = "&BANDNAME=g,r,i" url_format = "&FORMAT=" + return_type url_badCatFlagsMask = "&BAD_CATFLAGS_MASK=32768" url_payload = base_url + query_url_part + \ url_pos + \ url_bandname + \ url_format + \ url_badCatFlagsMask # establish http connection # http = urllib3.PoolManager() # siteData = http.request('GET', url_payload) siteData = urlopen(url_payload) if siteData.status != 200: status = False memFile = io.BytesIO(siteData.read()) votable = parse(memFile) table = votable.get_first_table().to_table(use_names_over_ids=True) return status, table.to_pandas()
def parse_datalink_for_service_and_id(filename, service_name): """ Parses a datalink file into a vo table, and returns the async service url and the authenticated id token """ # Parse the datalink file into a vo table, and get the results votable = parse(filename, pedantic=False) results = next(resource for resource in votable.resources if resource.type == "results") if results is None: return None results_array = results.tables[0].array async_url = None authenticated_id_token = None # Find the authenticated id token for accessing the image cube for x in results_array: if x['service_def'] == service_name: authenticated_id_token = x['authenticated_id_token'] # Find the async url for x in votable.resources: if x.type == "meta": if x.ID == service_name: for p in x.params: if p.name == "accessURL": async_url = p.value # print "Async url:", async_url # print "Authenticated id token for async access:", authenticated_id_token return async_url, authenticated_id_token
def find_images(pos_criteria, username, password, maxrec=0): """ Run an SIA2 query against CASDA to find images and cubes that contain any of the specified locations. See http://www.ivoa.net/documents/SIA/ for how to specify criteria. :param pos_criteria: An array of POS criteria (CIRCLE, POLYGON or RANGE) specifying the locations to be found. :param username: The OPAL username of the user. :param password: The OPAL password of the user. :param maxrec: The maximum number of images to retrieve, default is no limit :return: A VOTableFile object containing the SIA2 response. This will list the images along with extensive metadata. """ url = _casda_query_base_url + _sia2_endpoint params = list(map((lambda value: ('POS', value)), pos_criteria)) if maxrec > 0: params.append(('MAXREC', maxrec)) response = requests.get(url, params=params, auth=(username, password)) response.raise_for_status() if not os.path.exists('temp'): os.makedirs('temp') filename = 'temp/sia-resp.xml' with open(filename, 'wb') as f: f.write(response.content) votable = parse(filename, pedantic=False) return votable
def run(self): while True: #grabs host from queue host = self.queue.get() #grabs urls of hosts and reads or parses them according to their type try: # s1 = time.time() url = urllib2.urlopen(host[0]) # print 'fetch time: ',time.time() - s1 except Exception, e: sys.stderr.write( "\n Unable to return url file object. Here is the error message built into the exception: \n %s\n" % e) if host[1] == 1: warnings.simplefilter("ignore") votable1 = 'x' try: votable1 = parse(url, pedantic=False) except Exception, e: sys.stderr.write( "\n Unable to parse voTable. Here is the message built into the exception: \n %s \n" % e) self.outQueue.put(votable1) warnings.resetwarnings()
def make_fov_footprint(self, footprint): """Creating the user-defined field-of-view footprint using a template from http://aladin.u-strasbg.fr/footprint_editor/""" votable = parse(footprint) # reading footprint template table = votable.get_first_table() for param in table.params: # retrieving table.params param # box or circle footprint if param.ID == 'radius': try: param.value = float(self.entry_radius.get()) * 3600.0 votable.to_xml('GWsky_fov.vot') # VOTable file output except ValueError as value_error: raise else: try: data = table.array fov_width_arcsec = float(self.entry_width.get()) * 3600.0 fov_height_arcsec = float(self.entry_height.get()) * 3600.0 data[0] = -fov_width_arcsec / 2.0, fov_height_arcsec / 2.0 data[1] = fov_width_arcsec / 2.0, fov_height_arcsec / 2.0 data[2] = fov_width_arcsec / 2.0, -fov_height_arcsec / 2.0 data[3] = -fov_width_arcsec / 2.0, -fov_height_arcsec / 2.0 votable.to_xml('GWsky_fov.vot') # VOTable file output except UnboundLocalError: raise except ValueError: raise return aladin.send_file('GWsky_fov.vot')
def cat_read_SpARCS1634(): #read in catalogs #official GOGREEN imaging catalog catgogreen = '/Users/grudnick/Work/GOGREEN/Catalogs/Preimaging/SpARCS1634/SPARCS1634_zband_IRAC_v2_handcheck_cat.fits' gg_hdul = fits.open(catgogreen) gg_dat = gg_hdul[1].data #select the subset of data with a z-band detection izdet = np.where(gg_dat['zmag'] < 90.) gg_dat = gg_dat[izdet] #refcat = '/Users/grudnick/Work/GOGREEN/Catalogs/Astrometric/SpARCS1634/SpARCS1634_J.v0.sexcat' #ref_dat = ascii.read(refcat) #this is a GAIA reference catalog refcat = '/Users/grudnick/Work/GOGREEN/Catalogs/Astrometric/GOGREEN_GAIA/sparcs1634_gaia_votable.xml' votable = parse(refcat) table = votable.get_first_table() ref_dat = table.array #read the catalog spectroscopic redshifts speczcat = "/Users/grudnick/Work/GOGREEN/Data/Spectroscopy/v0.3/SpARCS1634_final.fits" return gg_dat, ref_dat, catgogreen, refcat, speczcat
def search_chandra_database(self): u = "https://cxcfps.cfa.harvard.edu/cgi-bin/cda/footprint/get_vo_table.pl?pos={0},{1}&size={2}" url = u.format(self.ra, self.dec, self.radius) url += "&inst=ACIS-I,ACIS-S" url += "&grating=NONE" r = requests.get(url) if r.status_code != 200: return f = io.BytesIO(r.text.encode()) votable = parse(f) tbdata = votable.get_first_table().to_table() if len(tbdata) > 0: table = unique(tbdata, keys='ObsId') self.total_exp = np.sum(table['Exposure']) self.obsids = table['ObsId'] self.targnames = table['target_name'] self.n_obsid = len(table) self.targ_ras = table['RA'] self.targ_decs = table['Dec'] self.exp_times = table['Exposure'] self.obs_dates = table['obs_date'] self.jpegs = table['preview_uri'] self.images = table['full_uri']
def get_SIMBAD_coordinates(name): url = VOTABLE_OPTIONS + SIMBAD_VOTABLE_SCRIPT_START + QUERY_VOTABLE_FULLCOORDINATES + SIMBAD_VOTABLE_SCRIPT_MIDDLE + name + SIMBAD_VOTABLE_SCRIPT_END try: response = urllib2.urlopen(SIMBAD_ROOT_1 + NAME_SCRIPT + urllib2.quote(url)) except urllib2.URLError: try: response = urllib2.urlopen(SIMBAD_ROOT_2 + NAME_SCRIPT + urllib2.quote(url)) except urllib2.URLError: return None try: response_votable = votable.parse(response.fp) first_table = response_votable.get_first_table() except: return None else: ra = float(first_table.array[0][0]) dec = float(first_table.array[0][1]) try: coords, created = AstronomicalCoordinates.objects.get_or_create( right_ascension=ra, declination=dec) except MultipleObjectsReturned: coords = AstronomicalCoordinates.objects.filter( right_ascension=ra, declination=dec).first() return coords
def _make_fov_footprint(self, footprint): """Making the instrument field of view footprint using a template from http://aladin.u-strasbg.fr/footprint_editor/""" votable = parse(footprint) # reading footprint template table = votable.get_first_table() for param in table.params: # retrieving table.params param # box or circle footprint if param.ID == 'radius': param.value = self._fov_radius * 3600.0 votable.to_xml('user_fov.vot') # VOTable file output else: data = table.array fov_width_arcsec = self._fov_width * 3600.0 fov_height_arcsec = self._fov_height * 3600.0 data[0] = -fov_width_arcsec / 2.0, fov_height_arcsec / 2.0 data[1] = fov_width_arcsec / 2.0, fov_height_arcsec / 2.0 data[2] = fov_width_arcsec / 2.0, -fov_height_arcsec / 2.0 data[3] = -fov_width_arcsec / 2.0, -fov_height_arcsec / 2.0 votable.to_xml('user_fov.vot') # VOTable file output return samp.send_file('user_fov.vot') # sending to Aladin
def read_votable(self, voname=None): "rad a votable" # if no voname, find it with name, raduis, errtol if voname == None: voname = "%s-%3.1fdeg-%serr.vot" % (self.name, self.radius, self.errtol) else: loc_name = voname.find('-') loc_deg = voname.find('deg') loc_err = voname.find('err') self.name = voname[:loc_name] self.radius = float(voname[loc_deg - 3:loc_deg]) self.errtol = float(voname[loc_deg + 4:loc_err]) print("## %s read..." % (voname)) votable = parse(voname) for table in votable.iter_tables(): data = table.array #print(data.dtype.names) self.data = data distance_max = np.max(1000. / np.ma.filled(data['parallax'], -999999.)) self.distmax = round(distance_max / 100, 0) * 100 cone_volume = np.pi * self.distmax * ( np.tan(self.radius * np.pi / 180.) * self.distmax)**2 / 3 self.density = len(self.data) / cone_volume print("## Total stars: %d" % (len(data))) print("## Density star per pc^3: %.5f" % (self.density)) return (len(data))
def process_gc_votable(): """ Executed this query: SELECT TOP 500 * FROM gaiadr1.gaia_source WHERE CONTAINS(POINT('ICRS',gaiadr1.gaia_source.ra,gaiadr1.gaia_source.dec),CIRCLE('ICRS',266.41683333333333,-29.00777777777778,0.016666666666666666))=1 on the gaia archive: http://gea.esac.esa.int/archive/ Saved to gc_result.vot (in Downloads) """ result_file = '/Users/jlu/work/tmt/iris/gc_result.vot' tab = parse(result_file) foo = tab.get_first_table() # this is an astropy table data_tab = foo.to_table() data_tab.write('/Users/jlu/work/tmt/iris/gc_gaia.txt', format='ascii.fixed_width', delimiter=' ') return
def get_SIMBAD_coordinates(name): url = VOTABLE_OPTIONS + SIMBAD_VOTABLE_SCRIPT_START + QUERY_VOTABLE_FULLCOORDINATES + SIMBAD_VOTABLE_SCRIPT_MIDDLE + name + SIMBAD_VOTABLE_SCRIPT_END try: response = urllib2.urlopen(SIMBAD_ROOT_1+NAME_SCRIPT+urllib2.quote(url)) except urllib2.URLError: try: response = urllib2.urlopen(SIMBAD_ROOT_2+NAME_SCRIPT+urllib2.quote(url)) except urllib2.URLError: return None try: response_votable = votable.parse(response.fp) first_table = response_votable.get_first_table() except: return None else: ra = float(first_table.array[0][0]) dec = float(first_table.array[0][1]) try: coords, created = AstronomicalCoordinates.objects.get_or_create(right_ascension=ra, declination=dec) except MultipleObjectsReturned: coords = AstronomicalCoordinates.objects.filter(right_ascension=ra, declination=dec).first() return coords
def build_table_mapper_map(self): logger.info("Looking for tables matching TABLE_MAPPING ") votable = parse(self.votable_path) for template_key in self.json_view["MODEL_INSTANCE"][ "TABLE_MAPPING"].keys(): logger.info("Looking for a table matching TABLE_MAPPING %s", template_key) name = None parsed_table = None for table in votable.iter_tables(): if template_key == table.ID: logger.info("Table with ID = %s found", template_key) name = table.ID parsed_table = table break if name == None: for table in votable.iter_tables(): if template_key == table.name: logger.info("Table with name = %s found", template_key) name = table.name parsed_table = table break if name == None: raise Exception("Cannot find table with name or ID = " + name) else: logger.info("Add TableMapper for table %s", name) self.table_mappers[template_key] = TableMapper( template_key, self.votable_path, parsed_table=parsed_table, json_inst_dict=self.json_view)
def Zari18_stars_to_csv(): """ The Zari+ 2018 work constructs large samples of young stars within 500 pc. It includes many upper main sequence and pre-main sequence members of the Sco-Cen, Orion and Vela star-forming regions. There are also less massive regions represented in Taurus, Perseus, Cepheus, etc. Annoyingly, no attempt is made to map between the star and the name of the cluster. So the "cluster name" column for this one will be "N/A". """ indir = clusterdatadir inpaths = [ os.path.join(indir,'Zari_2018_pms_tab.vot'), os.path.join(indir,'Zari_2018_ums_tab.vot') ] for inpath in inpaths: tab = parse(inpath) t = tab.get_first_table().to_table() df = pd.DataFrame({'source':t['Source'], 'cluster':np.repeat("N/A", len(t))}) outpath = inpath.replace('.vot','_cut_only_source_cluster_MATCH.csv') outdir = os.path.join(indir, 'moving_groups') outpath = os.path.join( outdir, os.path.basename(outpath) ) df.to_csv(outpath, index=False) print('made {}'.format(outpath))
def setUp(self): self.data_path = os.path.dirname(os.path.realpath(__file__)) self.votable_path = os.path.join(self.data_path, "./data/test_joint_instances.xml") votable = parse(self.votable_path) for table in votable.iter_tables(): if "OtherResults" == table.name: self.parsed_table = table self.join_iterator = JoinIterator( "OtherResults", "_poserr_148", "_foreign", { "@foreign": "_foreign", "@primary": "_poserr_148", "@tableref": "OtherResults", "test:detection": { "@dmtype": "test:Detection", "test:detection.id": { "@dmtype": "ivoa:real", "@ref": "_foreign", "@value": "" }, "test:detection.num": { "@dmtype": "ivoa:real", "@ref": "_num_148", "@value": "" } } }) self.join_iterator.connect_votable(self.parsed_table)
def postQuery(self, tap_params): """ Post query according to given parameters Parameters ----------- tap_params : dict Tap query parameters. It has to contains four keys. Dict keys: URL(str) Url of tap server table(str) Name of table for query select(str/list) Select string or list of column names conditions(list/tuple) For each condition in the list of conditions there is a tuple - ("name of column", "condition") or ("name of column", "lower value", "upper value" for search in the range Returns -------- list of lists Result from the query as nested lists """ # Load tap protocol parameters self.URL = tap_params["URL"] self.table = tap_params["table"] if "/" in self.table and not (self.table.startswith('"') or self.table.startswith("'")): self.table = '"' + self.table + '"' self.conditions = tap_params["conditions"] self.select = tap_params["select"] query = self._get_select_text() + self._get_from_text() + \ self._get_where_text() query_url = '%s/sync' % self.URL params = {"REQUEST": "doQuery", "LANG": "ADQL", "QUERY": query} # Run query res = requests.post(query_url, params=params) f = io.BytesIO(res.content) tab = parse(f) df = tab.get_first_table().to_table().to_pandas() for col in df.columns: if df[col].dtype == "object": df[col] = df[col].map(lambda x: x.decode() if isinstance(x, bytes) else x) else: df[col] = pd.to_numeric(df[col], downcast="signed") return df
def load_timeseries(in_file, value_name, obs_location): """returns a time series from in_file in JD for TCB BARYCENTER. in_file must give a VOTable literal. value_name names the column to take the value from. obs_location is a SkyCoordinate giving the (ICRS) position of the object observed. The time column will be the first FIELD that references TIMESYS, [the label column is just for debugging) """ vot = votable.parse(in_file) timesystems = vot.resources[0].timesystems time_series = vot.resources[0].tables[0] times, times_meta = locate_time_column(time_series, timesystems) times = to_jd(times, times_meta) times = to_tcb_barycenter(times, times_meta, obs_location) apy_table = time_series.to_table() values = apy_table.columns[value_name] # need to use times.value here since astropy vstack can't deal with # mixin columns. return table.Table(data=[ times.value, apy_table.columns[value_name], apy_table.columns["label"]], names=("time", "value", "label"))
def main_deprecated(): df = _get_cks_data() sel = _apply_cks_IV_metallicity_study_filters(df) p17_df = df[sel] vot = parse('../data/McQuillan_2014_ApJS_211_24_table1.vot') m14 = vot.get_first_table() m14_df = m14.to_table().to_pandas() p17_df['id_kic'] = p17_df['id_kic'].astype(str) m14_df['KIC'] = m14_df['KIC'].astype(str) # # no matches. wat. # mdf = p17_df.merge(m14_df, how='inner', left_on='id_kic', right_on='KIC') q = NasaExoplanetArchive.query_criteria(table='koi', select='kepoi_name,ra,dec', order='kepoi_name') q_df = q.to_pandas() mdf0 = p17_df.merge(q_df, how='inner', left_on='id_koicand', right_on='kepoi_name').drop_duplicates('id_koicand', keep='first') assert len(mdf0) == len(p17_df) c_p17 = SkyCoord(ra=nparr(mdf0.ra) * u.deg, dec=nparr(mdf0.dec) * u.deg) c_m14 = SkyCoord(ra=nparr(m14_df._RA) * u.deg, dec=nparr(m14_df._DE) * u.deg) idx_m14, idx_p17, d2d, _ = c_p17.search_around_sky(c_m14, 1 * u.arcsec)
def data_from_svo(query, error_msg='No data found for requested query'): """Get data in response to the query send to SVO FPS Parameters ---------- query : dict Used to create a HTTP query string i.e. send to SVO FPS to get data. In dictionary, specify keys as search parameters (str) and values as required. List of search parameters can be found at http://svo2.cab.inta-csic.es/theory/fps/fps.php?FORMAT=metadata error_msg : str, optional Error message to be shown in case no table element found in the responded VOTable. Use this to make error message verbose in context of the query made (default is 'No data found for requested query') Returns ------- astropy.io.votable.tree.VOTableFile object Entire VOTable fetched from SVO (in response to query) """ response = session.get(SVO_MAIN_URL, params=query) response.raise_for_status() votable = io.BytesIO(response.content) try: parse_single_table(votable) return parse(votable) # to keep metadata stored as PARAMS except IndexError: # If no table element found in VOTable raise ValueError(error_msg)
def parse_datalink_for_service_and_id(filename, service_name): """ Parses a datalink file into a vo table, and returns the async service url and the authenticated id token """ # Parse the datalink file into a vo table, and get the results votable = parse(filename, pedantic=False) results = next(resource for resource in votable.resources if resource.type == "results") if results is None: return None results_array = results.tables[0].array async_url = None authenticated_id_token = None # Find the authenticated id token for accessing the image cube for x in results_array: if x['service_def'].decode("utf8") == service_name: authenticated_id_token = x['authenticated_id_token'] # Find the async url for x in votable.resources: if x.type == "meta": if x.ID == service_name: for p in x.params: if p.name == "accessURL": async_url = p.value # print "Async url:", async_url # print "Authenticated id token for async access:", authenticated_id_token return async_url, authenticated_id_token
def download_cutouts(sbid, username, password, destination_dir, num_channels, data_product_sub_type): print ("\n\n** Finding images and image cubes for scheduling block {} ... \n\n".format(sbid)) sbid_multi_channel_query = "SELECT TOP 1000 * FROM ivoa.obscore where obs_id='" + str(sbid) \ + "' and dataproduct_subtype='" + str(data_product_sub_type) \ + "' and em_xel > 1 and dataproduct_type = 'cube'" # create async TAP query and wait for query to complete result_file_path = casda.async_tap_query(sbid_multi_channel_query, username, password, destination_dir) image_cube_votable = parse(result_file_path, pedantic=False) results_array = image_cube_votable.get_table_by_id('results').array # 3) For each of the image cubes, query datalink to get the secure datalink details print ("\n\n** Retrieving datalink for each image and image cube...\n\n") authenticated_id_tokens = [] for image_cube_result in results_array: image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8') async_url, authenticated_id_token = casda.get_service_link_and_id(image_cube_id, username, password, service='cutout_service', destination_dir=destination_dir) if authenticated_id_token is not None: authenticated_id_tokens.append([authenticated_id_token, image_cube_result]) if len(authenticated_id_tokens) == 0: print ("No image cubes for scheduling_block_id " + str(sbid)) return 1 # For each image cube, slice by channels using num_channels specified by the user. job_locations = [] for entry in authenticated_id_tokens: auth_id_token = entry[0] # get the image cube and number of channels ic = entry[1] channel_count = ic['em_xel'] channel_list = [] # wrap to max number of channels, if provided value exceeds channel count if num_channels > channel_count: num_channels = channel_count # slice up cube into chunks using the number of channels as the size of each chunk slices = math.ceil(channel_count / num_channels) current_step = 1 for s in range(slices): channel_list.append(str(current_step) + " " + str(num_channels * int(s+1))) current_step = (num_channels * int(s+1)) + 1 # create job for given band params job_location = casda.create_async_soda_job([auth_id_token]) casda.add_params_to_async_job(job_location, 'CHANNEL', channel_list) job_locations.append(job_location) # run all jobs and download casda.run_async_jobs_and_download(job_locations, destination_dir) return 0
def get_magmo_table(): votable = parse("magmo-spectra.vot", pedantic=False) table = votable.get_first_table() table = table.to_table() magmo_table = table[table['Duplicate'] == False] magmo_table = magmo_table[magmo_table['Rating'] <= 'C'] magmo_coords = SkyCoord(magmo_table['Longitude'], magmo_table['Latitude'], frame='galactic', unit="deg") return magmo_coords, magmo_table
def __init__(self, catalogpath=None, **specs): """ Constructor for EXOCAT1 Args: catalogpath (string): Full path to catalog VOTABLE. Defaults to mission_exocat.votable """ if catalogpath is None: classpath = os.path.split(inspect.getfile(self.__class__))[0] filename = 'mission_exocat.votable' catalogpath = os.path.join(classpath, filename) if not os.path.exists(catalogpath): raise IOError('Catalog File %s Not Found.'%catalogpath) #read votable with warnings.catch_warnings(): # warnings for IPAC votables are out of control # they are not moderated by pedantic=False # they all have to do with units, which we handle independently anyway warnings.simplefilter('ignore', astropy.io.votable.exceptions.VOTableSpecWarning) warnings.simplefilter('ignore', astropy.io.votable.exceptions.VOTableChangeWarning) votable = parse(catalogpath) table = votable.get_first_table() data = table.array StarCatalog.__init__(self, ntargs=len(data), **specs) # list of astropy attributes self.dist = data['st_dist'].data*u.pc #Distance to the planetary system in units of parsecs self.parx = self.dist.to('mas', equivalencies=u.parallax()) # parallactic angle in units of mas self.coords = SkyCoord(ra=data['ra']*u.deg, dec=data['dec']*u.deg, distance=self.dist) #Right Ascension of the planetary system in decimal degrees, Declination of the planetary system in decimal degrees self.pmra = data['st_pmra'].data*u.mas/u.yr #Angular change in right ascension over time as seen from the center of mass of the Solar System, units (mas/yr) self.pmdec = data['st_pmdec'].data*u.mas/u.yr #Angular change in declination over time as seen from the center of mass of the Solar System, units (mas/yr) self.L = data['st_lbol'].data #Amount of energy emitted by a star per unit time, measured in units of solar luminosities. The bolometric corrections are derived from V-K or B-V colors, units [log(solar)] # list of non-astropy attributes self.Name = data['hip_name'].astype(str) #Name of the star as given by the Hipparcos Catalog. self.Spec = data['st_spttype'].astype(str) #Classification of the star based on their spectral characteristics following the Morgan-Keenan system self.Vmag = data['st_vmag'] #Brightness of the host star as measured using the V band in units of magnitudes self.Jmag = data['st_j2m'] #Stellar J (2MASS) Magnitude Value self.Hmag = data['st_h2m'] #Stellar H (2MASS) Magnitude Value self.BV = data['st_bmv'] #Color of the star as measured by the difference between B and V bands, units of [mag] self.Bmag = self.Vmag + data['st_bmv'] #calculation of the B band #st_vmk Color of the star as measured by the difference between V and K bands. The source column is for the K magnitude, units of [mag] self.Kmag = self.Vmag - data['st_vmk'] #calculation of the K band #st_mbol Apparent magnitude of the star at a distance of 10 parsec units of [mag] self.BC = -self.Vmag + data['st_mbol'] # bolometric correction https://www.astro.princeton.edu/~gk/A403/constants.pdf self.MV = self.Vmag - 5.*(np.log10(self.dist.to('pc').value) - 1.) # apparent "visual" magnitude https://www.astro.princeton.edu/~gk/A403/constants.pdf self.stellar_diameters = data['st_rad']*2.*R_sun # stellar_diameters in solar diameters self.Binary_Cut = ~data['wds_sep'].mask #WDS (Washington Double Star) Catalog separation (arcsecs)
def parse_vizier_votable(data, verbose=False, invalid='warn', get_catalog_names=False): """ Given a votable as string, parse it into dict or tables """ if not verbose: commons.suppress_vo_warnings() tf = BytesIO(data) if invalid == 'mask': vo_tree = votable.parse(tf, pedantic=False, invalid='mask') elif invalid == 'warn': try: vo_tree = votable.parse(tf, pedantic=False, invalid='exception') except Exception as ex: warnings.warn("VOTABLE parsing raised exception: {0}".format(ex)) vo_tree = votable.parse(tf, pedantic=False, invalid='mask') elif invalid == 'exception': vo_tree = votable.parse(tf, pedantic=False, invalid='exception') else: raise ValueError("Invalid keyword for 'invalid'. " "Must be exception, mask, or warn") if get_catalog_names: return OrderedDict([(R.name, R) for R in vo_tree.resources]) else: table_dict = OrderedDict() for t in vo_tree.iter_tables(): if len(t.array) > 0: if t.ref is not None: name = vo_tree.get_table_by_id(t.ref).name else: name = t.name if name not in table_dict.keys(): table_dict[name] = [] table_dict[name] += [t.to_table()] for name in table_dict.keys(): if len(table_dict[name]) > 1: table_dict[name] = tbl.vstack(table_dict[name]) else: table_dict[name] = table_dict[name][0] return commons.TableList(table_dict)
def plot_USNO_B1_stars(ax): t = votable.parse(usnoB1.TAPQuery(ra_cen, dec_cen, width, height)).get_first_table() Rmag = t.array['Bmag'][t.array['Bmag'] < 15] min = max(Rmag.min(),11) max = Rmag.max() scale = 0.5*10**((min - Rmag)/2.5) print scale.min(), scale.max() ax.scatter(t.array['RAJ2000'], t.array['DEJ2000'], s=scale, marker='o', facecolor='y', alpha=0.8, edgecolor='', zorder=-10) return ax
def query_for_observations(mjd, observable, runids): """Do a QUERY on the TAP service for all observations that are part of runid, where taken after mjd and have calibration 'observable'. Schema is at: http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/tap/tables mjd : float observable: str ( 2 or 1 ) runid: tuple eg. ('13AP05', '13AP06') """ data = {"QUERY": ("SELECT Observation.target_name as TargetName, " "COORD1(CENTROID(Plane.position_bounds)) AS RA," "COORD2(CENTROID(Plane.position_bounds)) AS DEC, " "Plane.time_bounds_lower AS StartDate, " "Plane.time_exposure AS ExposureTime, " "Observation.instrument_name AS Instrument, " "Plane.energy_bandpassName AS Filter, " "Observation.observationID AS dataset_name, " "Observation.proposal_id AS ProposalID, " "Observation.proposal_pi AS PI " "FROM caom2.Observation AS Observation " "JOIN caom2.Plane AS Plane ON " "Observation.obsID = Plane.obsID " "WHERE ( Observation.collection = 'CFHT' ) " "AND Plane.time_bounds_lower > %d " "AND Plane.calibrationLevel=%s " "AND Observation.proposal_id IN %s " ) % ( mjd, observable, str(runids)), "REQUEST": "doQuery", "LANG": "ADQL", "FORMAT": "votable"} result = requests.get(storage.TAP_WEB_SERVICE, params=data, verify=False) assert isinstance(result, requests.Response) logging.debug("Doing TAP Query using url: %s" % (str(result.url))) tmpFile = tempfile.NamedTemporaryFile() with open(tmpFile.name, 'w') as outfile: outfile.write(result.text) try: vot = parse(tmpFile.name).get_first_table() except: print result.text raise vot.array.sort(order='StartDate') t = vot.array tmpFile.close() print t logging.debug("Got {} lines from tap query".format(len(t))) return t
def _parse_result(self, response, verbose=False): """ Parses the results form the HTTP response to `~astropy.table.Table`. Parameters ---------- response : `requests.Response` The HTTP response object verbose : bool, optional Defaults to `False`. When true it will display warnings whenever the VOtable returned from the Service doesn't conform to the standard. Returns ------- table : `~astropy.table.Table` """ if not verbose: commons.suppress_vo_warnings() content = response.text logging.debug(content) # Check if results were returned if 'The catalog is not in the list' in content: raise Exception("Catalogue not found") # Check that object name was not malformed if 'Either wrong or missing coordinate/object name' in content: raise Exception("Malformed coordinate/object name") # Check that the results are not of length zero if len(content) == 0: raise Exception("The LCOGT server sent back an empty reply") # Read it in using the astropy VO table reader try: first_table = votable.parse(six.BytesIO(response.content), pedantic=False).get_first_table() except Exception as ex: self.response = response self.table_parse_error = ex raise TableParseError("Failed to parse LCOGT votable! The raw " " response can be found in self.response," " and the error in self.table_parse_error.") # Convert to astropy.table.Table instance table = first_table.to_table() # Check if table is empty if len(table) == 0: warnings.warn("Query returned no results, so the table will " "be empty", NoResultsWarning) return table
def xml2csv(self): """Convert the saved xml file to csv and read with pandas""" with warnings.catch_warnings(): warnings.simplefilter("ignore") vo = votable.parse('exo.xml', invalid='mask', pedantic=False) vo = vo.get_first_table().to_table(use_names_over_ids=True) df = vo.to_pandas() # Divide the data in Confirmed and not. df[df.planet_status == 'Confirmed'].to_csv('exo.csv', index=False) df[(df.planet_status == 'Unconfirmed') | (df.planet_status == 'Candidate')].to_csv('exo_cont.csv', index=False)
def _parse_result(self, response, get_catalog_names=False, verbose=False): """ Parses the HTTP response to create an `astropy.table.Table`. Returns the raw result as a string in case of parse errors. Parameters ---------- response : `requests.Response` The response of the HTTP POST request get_catalog_names : bool If specified, return only the table names (useful for table discovery) Returns ------- `astroquery.utils.commons.TableList` An OrderedDict of `astropy.table.Table` objects. If there are errors in the parsing, then returns the raw results as a string. """ if not verbose: commons.suppress_vo_warnings() try: tf = tempfile.NamedTemporaryFile() if PY3: tf.write(response.content) else: tf.write(response.content.encode('utf-8')) tf.file.flush() vo_tree = votable.parse(tf.name, pedantic=False) if get_catalog_names: return dict([(R.name,R) for R in vo_tree.resources]) else: table_dict = OrderedDict() for t in vo_tree.iter_tables(): if len(t.array) > 0: if t.ref is not None: name = vo_tree.get_table_by_id(t.ref).name else: name = t.name if name not in table_dict.keys(): table_dict[name] = [] table_dict[name] += [t.to_table()] for name in table_dict.keys(): if len(table_dict[name]) > 1: table_dict[name] = tbl.vstack(table_dict[name]) else: table_dict[name] = table_dict[name][0] return commons.TableList(table_dict) except: traceback.print_exc() # temporary for debugging warnings.warn( "Error in parsing result, returning raw result instead") return response.content
def _parse_result(self, response, verbose=False): """ Parse a VOtable response """ if not verbose: commons.suppress_vo_warnings() tf = six.BytesIO(response.content) vo_tree = votable.parse(tf, pedantic=False, invalid='mask') first_table = vo_tree.get_first_table() table = first_table.to_table(use_names_over_ids=True) return table
def _parse_result(self, response, verbose=False): """ Parses the results form the HTTP response to `~astropy.table.Table`. Parameters ---------- response : `requests.Response` The HTTP response object verbose : bool, optional Defaults to `False`. When true it will display warnings whenever the VOtable returned from the Service doesn't conform to the standard. Returns ------- table : `~astropy.table.Table` """ if not verbose: commons.suppress_vo_warnings() # Check if results were returned if 'The catalog is not on the list' in response.content: raise Exception("Catalog not found") # Check that object name was not malformed if 'Either wrong or missing coordinate/object name' in response.content: raise Exception("Malformed coordinate/object name") # Check that the results are not of length zero if len(response.content) == 0: raise Exception("The IRSA server sent back an empty reply") # Write table to temporary file output = tempfile.NamedTemporaryFile() output.write(response.content.encode()) output.flush() # Read it in using the astropy VO table reader try: first_table = votable.parse(output.name, pedantic=False).get_first_table() except Exception as ex: self.response = response self.table_parse_error = ex raise TableParseError("Failed to parse IRSA votable! The raw response can be found " "in self.response, and the error in self.table_parse_error.") # Convert to astropy.table.Table instance table = first_table.to_table() # Check if table is empty if len(table) == 0: warnings.warn("Query returned no results, so the table will be empty") return table
def _parse_result(self, response, verbose=False): """ Parses the raw HTTP response and returns it as an `astropy.table.Table`. Parameters ---------- response : `requests.Response` The HTTP response object verbose : bool, optional Defaults to false. When true it will display warnings whenever the VOtable returned from the service doesn't conform to the standard. Returns ------- table : `astropy.table.Table` """ if not verbose: commons.suppress_vo_warnings() try: tf = tempfile.NamedTemporaryFile() if six.PY3: # This is an exceedingly confusing section # It is likely to be doubly wrong, but has caused issue #185 try: # Case 1: data is read in as unicode tf.write(response.content.encode()) except AttributeError: # Case 2: data is read in as a byte string tf.write(response.content.decode().encode('utf-8')) else: tf.write(response.content.encode('utf-8')) tf.file.flush() first_table = votable.parse(tf.name, pedantic=False).get_first_table() # For astropy version < 0.3 returns tables that have field ids as col names if ASTROPY_VERSION < '0.3': table = first_table.to_table() # For astropy versions >= 0.3 return the field names as col names else: table = first_table.to_table(use_names_over_ids=True) return table except Exception as ex: (is_valid, err_msg) = _check_ned_valid(response.content) if not is_valid: if err_msg: raise RemoteServiceError("The remote service returned the following error message.\nERROR: {err_msg}".format(err_msg=err_msg)) else: raise RemoteServiceError("The remote service returned an error, but with no message.") else: self.response = response self.table_parse_error = ex raise TableParseError("Failed to parse NED result! The raw response can be found " "in self.response, and the error in self.table_parse_error.")
def run(self): # load the votable content from previous task votable = parse(self.input().path) data_arr = votable.get_first_table().array # Extract plot data dist = data_arr["dist"] phot_g_mean_mag = data_arr["phot_g_mean_mag"] # highcharts_data = zip(list_a, list_b) highcharts_data = {"data": map(lambda x, y: [x, y], dist, phot_g_mean_mag)} with open(self.output().path, "w") as out: json.dump(highcharts_data, out)
def find_redshifted_lines(self): # next: read atomic.xml and find relevant lab wavelengths mypath = os.path.dirname(os.path.realpath(__file__)) atoms = parse(os.path.expanduser(mypath+'/atomic.xml')) atomstable = atoms.get_first_table() self.atomstable = atomstable promels = ['Si', 'C', 'N', 'O', 'Al', 'Fe', 'Ca', 'B', 'Be'] ionallow= ['I', 'II', 'III', 'IV', 'V', 'VI', 'VII', 'IX', 'X'] isoallow= [28, 12, 14, 16, 27, 56, 40, 10, 9] wlshift = 5.0 # WL displacement in A wls = self.wls z_sel = self.z_sel lymaninf = where((atomstable.array['Element'] == 'H') \ & (atomstable.array['Ion'] == 'I') \ & (atomstable.array['MassNumber'] == 1) ) deutinf = where((atomstable.array['Element'] == 'H') \ & (atomstable.array['Ion'] == 'I') \ & (atomstable.array['MassNumber'] == 2) ) promlinesBOOL = in1d(atomstable.array['Element'], promels) promlinesIonBOOL = in1d(atomstable.array['Ion'], ionallow) promlinesIsoBOOL = in1d(atomstable.array['MassNumber'], isoallow) promlines = where((promlinesBOOL) \ & (promlinesIonBOOL) \ & (promlinesIsoBOOL) \ & (atomstable.array['fval'] >= 0.05) \ & (atomstable.array['RestWave'] <= max(wls)/(1+z_sel[0]) )\ & (atomstable.array['RestWave'] >= min(wls)/(1+z_sel[0]) ) ) implines = where((atomstable.array['fval'] >= 0.05) \ & (atomstable.array['RestWave'] <= max(wls)/(1+z_sel[0]) )\ & (atomstable.array['RestWave'] >= min(wls)/(1+z_sel[0]) ) ) # Sort prominent lines, most important first promlines_id = atomstable.array['fval'][promlines].argsort() self.promlines1 = promlines[0][promlines_id] # sort by strenght self.promlines1 = self.promlines1[::-1] self.promlines_wl = atomstable.array['RestWave'][promlines].argsort() self.promlines2 = promlines[0][promlines_id] self.lymanseries = outer((1 + z_sel), atomstable.array['RestWave'][lymaninf]) self.lymanserieslab = atomstable.array['RestWave'][lymaninf] self.deutseries = outer((1 + z_sel), atomstable.array['RestWave'][deutinf]) self.promseries1 = outer((1 + z_sel), atomstable.array['RestWave'][self.promlines1]) self.promseries2 = outer((1 + z_sel), atomstable.array['RestWave'][self.promlines2]) self.promels = promels self.ionallow = ionallow self.isoallow = isoallow self.wlshift = wlshift
def __init__(self, catalogpath=None, **specs): """ Constructor for EXOCAT1 Args: catalogpath (string): Full path to catalog VOTABLE. Defaults to mission_exocat.votable """ if catalogpath is None: classpath = os.path.split(inspect.getfile(self.__class__))[0] filename = 'mission_exocat.votable' catalogpath = os.path.join(classpath, filename) if not os.path.exists(catalogpath): raise IOError('Catalog File %s Not Found.'%catalogpath) #read votable with warnings.catch_warnings(): # warnings for IPAC votables are out of control # they are not moderated by pedantic=False # they all have to do with units, which we handle independently anyway warnings.simplefilter('ignore', astropy.io.votable.exceptions.VOTableSpecWarning) warnings.simplefilter('ignore', astropy.io.votable.exceptions.VOTableChangeWarning) votable = parse(catalogpath) table = votable.get_first_table() data = table.array StarCatalog.__init__(self, ntargs=len(data), **specs) # list of astropy attributes self.dist = data['st_dist'].data*u.pc self.parx = self.dist.to('mas',equivalencies=u.parallax()) self.coords = SkyCoord(ra=data['ra']*u.deg, dec=data['dec']*u.deg, distance=self.dist) self.pmra = data['st_pmra'].data*u.mas/u.yr self.pmdec = data['st_pmdec'].data*u.mas/u.yr # list of non-astropy attributes self.Name = data['hip_name'] self.Spec = data['st_spttype'] self.Vmag = data['st_vmag'] self.Jmag = data['st_j2m'] self.Hmag = data['st_h2m'] self.BV = data['st_bmv'] self.L = data['st_lbol'] self.Bmag = self.Vmag + data['st_bmv'] self.Kmag = self.Vmag - data['st_vmk'] self.BC = -self.Vmag + data['st_mbol'] self.MV = self.Vmag - 5*(np.log10(self.dist.value) - 1) self.Binary_Cut = ~data['wds_sep'].mask
def read_islands(filename): print ("Extracting islands from " + filename) islands = {} if not os.path.exists(filename): print ("Warning: File %s does not exist, skipping island read." % \ filename) return {} isle_votable = votable.parse(filename, pedantic=False) results = isle_votable.get_first_table().array for row in results: islands[row['island']] = row return islands
def readData(): x = [] y = [] c = [] bad_spectra = 0 prev_field = '' num_fields = 0 vo_files = glob.glob('day*/*.votable.xml') for filename in sorted(vo_files): print 'Reading', filename votable = parse(filename, pedantic=False) results = next(resource for resource in votable.resources if resource.type == "results") if results is not None: gal_info = None for info in votable.infos: if info.name == 'longitude': gal_info = info #gal_info = votable.get_info_by_id('longitude') if gal_info is None: print "No longitude provided for %s, skipping" % filename continue gal_long = float(gal_info.value) if gal_long > 180: gal_long -= 360 results_array = results.tables[0].array poor_sn = False #print gal_long for row in results_array: opacity = row['opacity'] if opacity > 6 or opacity < -8: poor_sn = True if not poor_sn: for row in results_array: x.append(gal_long) y.append(row['velocity']/1000.0) # Convert from m/s to km/s opacity = row['opacity'] c.append(opacity) if opacity > 10 or opacity < -15: poor_sn = True field = filename.split('_') if field[0] != prev_field: prev_field = field[0] num_fields += 1 if poor_sn: bad_spectra += 1 print "In %d fields read %d spectra of which %d had reasonable S/N. " % ( num_fields, len(vo_files), len(vo_files)-bad_spectra) return x, y, c
def _readData(self): """ Read data from VO table """ # Temporarily suppress warnings (astropy issues one on reading this table) with warnings.catch_warnings(): warnings.simplefilter("ignore") try: self.vot = votable.parse(self._fs.requestFile(self.dataFileName, 'r', gzip.open), pedantic=False, invalid="mask") except Exception as e: votable.validate(self._fs.requestFile(self.dataFileName, 'r', gzip.open)) raise e # Use 'name' over ID field to specify column names self.vot = self.vot.get_first_table().to_table(use_names_over_ids=True)
def get_proc_table(self): """ retrieve the table of processes from the proc service. @return: Table """ url = self.server+self.endpoint f = cStringIO.StringIO(self.client.open(uri=None, URL=url).read()) f.seek(0) logging.debug(f.read()) f.seek(0) with warnings.catch_warnings(): warnings.simplefilter("ignore") return votable.parse(f, invalid='mask').get_first_table().to_table()
def produce_cutouts(source_list, image_id, username, password, destination_dir): # Use CASDA VO (secure) to query for the images associated with the given scheduling_block_id print ("\n\n** Retreiving image details for %s ... \n\n" % image_id) filename = destination_dir + str(image_id) + ".xml" data_product_id_query = "select * from ivoa.obscore where obs_publisher_did = '" + image_id + \ "' and dataproduct_type = 'cube'" casda.sync_tap_query(data_product_id_query, filename, username=username, password=password) image_cube_votable = votable.parse(filename, pedantic=False) results_array = image_cube_votable.get_table_by_id('results').array # For each of the image cubes, query datalink to get the secure datalink details print ("\n\n** Retrieving datalink for each image and image cube...\n\n") authenticated_id_tokens = [] for image_cube_result in results_array: image_cube_id = image_cube_result['obs_publisher_did'].decode('utf-8') async_url, authenticated_id_token = casda.get_service_link_and_id(image_cube_id, username, password, service='cutout_service', destination_dir=destination_dir) if authenticated_id_token is not None: authenticated_id_tokens.append(authenticated_id_token) if len(authenticated_id_tokens) == 0: print ("No image cubes found") return 1 # Create the async job job_location = casda.create_async_soda_job(authenticated_id_tokens) # For each entry in the results of the catalogue query, add the position filter as a parameter to the async job cutout_filters = [] for sky_loc in source_list: ra = sky_loc.ra.degree dec = sky_loc.dec.degree circle = "CIRCLE " + str(ra) + " " + str(dec) + " " + str(cutout_radius_degrees) cutout_filters.append(circle) casda.add_params_to_async_job(job_location, 'pos', cutout_filters) # Run the job status = casda.run_async_job(job_location) # Download all of the files, or alert if it didn't complete if status == 'COMPLETED': print ("\n\n** Downloading results...\n\n") casda.download_all(job_location, destination_dir) else: print ("Job did not complete: Status was %s." % status) return 1 return 0
def queryFitsImage(fitsImage,voTableName,pos='104.9566125,14.2341555'): ''' retrieve fits image from 2MASS http://irsa.ipac.caltech.edu/applications/2MASS/IM/docs/siahelp.html note that fitsImage has to be in gz file in order to be extracted and opened correctly ''' print 'loading 2MASS fits images' conn1 = hlib.HTTPConnection('irsa.ipac.caltech.edu') params1 = ulib.urlencode({'POS':pos,'INTERSECT':'ENCLOSED'}) conn1.request('POST','/cgi-bin/2MASS/IM/nph-im_sia',params1) resp1 = conn1.getresponse() xml = open(voTableName,'w') xml.write(resp1.read()) conn1.close() xml.close() #parsing the votable and download the selected image tree = ET.parse(voTableName) root = tree.getroot() #map all null values to 0 which fixes the error when open the votable with astropy for name in root.iter('TD'): if name.text == 'null': name.text = 0 tree.write(voTableName) #warning suppression when open votable print 'downloading fits image...' warnings.resetwarnings() warnings.filterwarnings('ignore', category=Warning, append=True) voTable = VT.parse(voTableName) warnings.resetwarnings() warnings.filterwarnings('always', category=Warning, append=True) #raise warning if the image cannot be found try: table = voTable.get_first_table() except: raise ValueError('Cannot Locate the fits image!') imageUrls = table.array['download'] print imageUrls download = imageUrls[0] print download ulib.urlretrieve(download,fitsImage) print 'done'
def plot_existing_CFHT_Megacam_observations_in_area(ax): t = votable.parse(megacam.TAPQuery(ra_cen, dec_cen, width, height)).get_first_table() ra = t.array['RAJ2000'] dec = t.array['DEJ2000'] rects = [Rectangle(xy=(ra[idx]-dimen/2.0,dec[idx]-dimen/2.0), height=camera_dimen, width=camera_dimen, edgecolor='k', alpha=0.1, lw=0.1, zorder=-100, fill=False) for idx in xrange(ra.size)] for r in rects: ax.add_artist(r) return ax