def getObstable(self): options = self.options table = Observations.query_region(self.coord, radius=self.options['radius']) # HST-specific masks filmask = [ table['filters'] == good for good in options['mask']['filters'] ] filmask = [any(l) for l in list(map(list, zip(*filmask)))] expmask = table['t_exptime'] > options['mask']['t_exptime'] obsmask = [ table['obs_collection'] == good for good in options['mask']['obs_collection'] ] obsmask = [any(l) for l in list(map(list, zip(*obsmask)))] detmask = [ table['instrument_name'] == good for good in options['mask']['instrument_name'] ] detmask = [any(l) for l in list(map(list, zip(*detmask)))] # Construct and apply mask mask = [all(l) for l in zip(filmask, expmask, obsmask, detmask)] self.obstable = table[mask] self.Nimages = len(self.obstable)
def check_exist_location(ra, dec, radius): # This function takes the columns of RA and DEC to check if there are any observations # of that region, and how many are there in MAST. # RA and DEC must be in decimal format, NOT in DEG:MM:SS. # print("\nChecking...\n") results = [] location = str(ra) + ' ' + str(dec) results = Observations.query_region(location, radius=radius) final = results[(results['obs_collection'] == 'GALEX') & (results['dataproduct_type'] == 'image')] data_products_by_obs = Observations.get_product_list(final) ObjIDs = data_products_by_obs[( (data_products_by_obs['description'] == descript) | (data_products_by_obs['description'] == skybkg) | (data_products_by_obs['description'] == intmap)) & ( (data_products_by_obs['project'] == 'MIS') | (data_products_by_obs['project'] == 'AIS'))] counts = len(ObjIDs) if counts == 0: counts = -9999 ObjIDs = -9999 return counts, ObjIDs
def getObstable(self): self.obstable = Observations.query_region( str(self.ra) + " " + str(self.dec), radius=str(self.radius) + " deg") detmasks = [ self.obstable['instrument_name'] == name for name in self.allowed_detector ] filmasks = [self.obstable['filters'] != bad for bad in self.badfilter] expmasks = [self.obstable['t_exptime'] >= self.minexp] colmasks = [ self.obstable['obs_collection'] == coll for coll in self.collection ] good=[all(l) for l in list(zip(\ [any(l) for l in list(map(list,zip(*detmasks)))],\ [any(l) for l in list(map(list,zip(*colmasks)))],\ [any(l) for l in list(map(list,zip(*expmasks)))],\ [any(l) for l in list(map(list,zip(*filmasks)))]))] self.obstable = self.obstable[good] self.Nimages = len(self.obstable) self.filtlist = self.obstable['filters'] self.detlist = self.obstable['instrument_name'] self.obsid = self.obstable['obs_id'] self.jdlist = self.obstable['t_min'] + 2400000.5
def get_galex_image(ra, dec, imsize): """ get galex image of a galaxy Input: * ra in deg * dec in deg * imsize in arcsec Returns: * image """ # following procedure outlined here: # https://astroquery.readthedocs.io/en/latest/mast/mast.html # get data products in region near ra,dec obs_table = Observations.query_region("%12.8f %12.8f" % (c.ra, c.dec), radius=.1 * u.arcmin) # create a flag to select galex data galexFlag = obs_table['obs_collection'] == 'GALEX' # separate out galex data data_products = Observations.get_product_list(obs_table[galexFlag]) # download the observations manifest = Observations.download_products(data_products, productType="SCIENCE") for m in manifest: # choose the first NUV image if m['Local Path'].find('nd-int') > -1: nuv_path = m['Local Path'] break # should be able to construct path from the obs_id in data_products # this will let us check if the image is already downloaded # # I can also save the cutout in a GALEX folder, and # look for the image before calling this function nuv, nuv_header = fits.getdata(nuv_path, header=True) # this is a big image, so we need to get a cutout nuv_wcs = WCS(nuv_header) position = SkyCoord(ra, dec, unit="deg", frame='icrs') cutout = Cutout2D(nuv, position, (imsize * u.arcsec, imsize * u.arcsec), wcs=nuv_wcs) return cutout
def find_object(catalog, name, raname='ra', decname='dec', namename='Name', verbose=True): m = catalog['Name'] == name target = catalog[m][0] ra, dec = target[raname, decname] obsTable = Observations.query_region("%s %s " % (ra, dec), radius=10 * u.arcsec) return obsTable
def find_in_tess(catalog, raname='ra', decname='dec', namename='Name', verbose=True): names = [] for j, name in enumerate(catalog['Name']): ra, dec = catalog[j][raname, decname] obsTable = Observations.query_region("%s %s " % (ra, dec), radius=10 * u.arcsec) m = [obsTable['obs_collection'] == 'TESS'] if np.sum(m) != 0: names.append(name) if verbose: print(name) in_tess = join(Table({namename: np.array(names)}), catalog, keys=namename) return in_tess
def getObstable(self): options = self.options table = Observations.query_region(self.coord, radius=self.options['radius']) # HST-specific masks filmask = [ table['filters'] == good for good in options['mask']['filters'] ] filmask = [any(l) for l in list(map(list, zip(*filmask)))] expmask = table['t_exptime'] > options['mask']['t_exptime'] obsmask = [ table['obs_collection'] == good for good in options['mask']['obs_collection'] ] obsmask = [any(l) for l in list(map(list, zip(*obsmask)))] detmask = [ table['instrument_name'] == good for good in options['mask']['instrument_name'] ] detmask = [any(l) for l in list(map(list, zip(*detmask)))] # Construct and apply mask mask = [all(l) for l in zip(filmask, expmask, obsmask, detmask)] self.obstable = table[mask] self.Nimages = 0 if self.obstable: if len(self.obstable) > 0: self.obstable['t_min'] = np.around(self.obstable['t_min'], decimals=4) self.obstable['t_max'] = np.around(self.obstable['t_max'], decimals=4) self.obstable.sort('obs_collection') self.obstable = unique(self.obstable, keys=['t_min'], keep='last') self.obstable = unique(self.obstable, keys=['t_max'], keep='last') self.Nimages = len(self.obstable)
def downloadPS1(coord, filt, outdir='data/'): # Check for filter if not filt: filt='r' obs = Observations.query_region(coord, radius=5*u.arcmin) masks = [] masks.append([str(t).upper()=='PS1' for t in obs['obs_collection']]) masks.append([str(p).upper()=='IMAGE' for p in obs['dataproduct_type']]) masks.append([str(r).upper()=='PUBLIC' for r in obs['dataRights']]) masks.append([str(f).lower()==filt for f in obs['filters']]) masks = list(map(list, zip(*masks))) mask = np.array([all(l) for l in masks]) obsTable = obs[mask] if len(obsTable)==0: error = 'ERROR: no matching PS1 images for ra={ra}, dec={dec}.' print(error.format(ra=ra, dec=dec)) return(1) else: prodList = Observations.get_product_list(obsTable) prodList = prodList[prodList['productType']=='SCIENCE'] prodList = prodList[prodList['description']=='stack data image'] message = 'We need to download {n} PS1 images.' print(message.format(n=len(prodList))) for product in prodList: filename = product['productFilename'] outfile = outdir+filename url = 'https://mast.stsci.edu/api/v0/download/file?uri=' url += product['dataURI'] if os.path.isfile(outfile): message = '{filename} exists. Continuing...' print(message.format(filename=outfile)) return(outfile) else: errflag = download_url_to_file(url, outfile) return(outfile)
) print( '------------------------------------------------------------------------------------------------------------------' ) for i in range(len(teff)): #-------------------------------------- #--------Downloading Data-------------- #-------------------------------------- print('----------------------------------------------------------------') print('---------------Working on ' + name[i] + ' -------') print('----------------------------------------------------------------') ra1 = utils.RA_to_deg(ra[i]) dec1 = utils.DEC_to_deg(dec[i]) coord = str(ra1) + ' ' + str(dec1) obt = obs.query_region(coord, radius=0.001) b = np.array([]) for j in range(len(obt['intentType'])): if obt['obs_collection'][j] == 'TESS' and obt['dataproduct_type'][ j] == 'timeseries': b = np.hstack((b, j)) if len(b) == 0: print( '******************************************************************************************' ) print( ' ' ) print('Completed analysis for ' + str(i + 1) + ' system(s) / out of ' + str(len(name)) + ' systems') print(
def app_caom(): global blc global trc global im if blc is None or trc is None or im is None: load_image() searchString = '{} {}'.format(np.mean([blc[0], trc[0]]), np.mean([blc[1], trc[1]])) obsTable = Observations.query_region(searchString, radius=0.2) obsDF = obsTable.to_pandas() obsDF = obsDF[(obsDF['obs_collection'].isin(['HST', 'SWIFT'])) & ( obsDF['instrument_name'].isin(['WFC3/IR', 'ACS/WFC', 'STIS/CCD']))] obsDF['coords'] = obsDF.apply(lambda x: parse_s_region(x['s_region']), axis=1) obsDF['coords'].head() # get plot p = make_base_bokeh() # Loop over instruments, coloring each separately for ins, color in zip(obsDF['instrument_name'].unique(), Spectral4): ind = obsDF['instrument_name'] == ins # Add patches with the observation footprings patch_xs = [c['ra'] for c in obsDF['coords'][ind]] patch_ys = [c['dec'] for c in obsDF['coords'][ind]] data = { 'x': patch_xs, 'y': patch_ys, 'obs_collection': obsDF['obs_collection'][ind], 'instrument_name': obsDF['instrument_name'][ind], 'obs_id': obsDF['obs_id'][ind], 'target_name': obsDF['target_name'][ind], 'proposal_pi': obsDF['proposal_pi'][ind], 'obsID': obsDF['obsid'][ind] } p.patches('x', 'y', source=data, legend=ins, fill_color=color, fill_alpha=0.1, line_color="white", line_width=0.5) # Add hover tooltip for MAST observations tooltip = [("obs_collection", "@obs_collection"), ("instrument_name", "@instrument_name"), ("obs_id", "@obs_id"), ("target_name", "@target_name"), ('proposal_pi', '@proposal_pi')] p.add_tools(HoverTool(tooltips=tooltip)) # Set up tap/click tool # Only gets top one? url = 'getproducts/@obsID' p.add_tools(TapTool()) taptool = p.select(type=TapTool) taptool.callback = OpenURL(url=url) p.legend.click_policy = "hide" script, div = components(p) return render_template('base.html', script=script, plot=div)
quasar_list = pd.read_csv('/Users/lcgordon/Downloads/5000-hmq') #import list n = 0 #counts through coordinate list m = 0 #counter for coordinate pairs WITH DATA #find all of the while n < 5000: #produce coordinate string from list RA_float = quasar_list.iloc[n, 0] RA = str(RA_float) DEC_float = quasar_list.iloc[n, 1] DEC = str(DEC_float) coords = RA + "_" + DEC #query MAST, produce table of all data products associated with those coordinates table_of_observations = Observations.query_region(coords, radius=0.02) product_list = Observations.get_product_list( product_list) #makes list of all products i = 0 #counter for obsTable rows while i < len(table_of_observations): obs_collection = str(table_of_observations[i] [1]) #converts the type to a string (essential) if obs_collection == "HST": #check to see if the observatory collection is HST, caNNOT USE 'and' comparison m = m + 1 #add one to number of coordinate pairs that have data print("Data at", coords) #tells you which coordinate pairs, optional i = len( obsTable ) + 1 #makes i out of the range of obstable, ending this while loop
redshift_target = quasar_list.iloc[n][3] #redshift of the target. r_mag_target = quasar_list.iloc[n][2] #magnitude of target #calculate the absolute magnitude of the target from the red magnitude dL = Distance(unit = u.Mpc, z=redshift_target, cosmology = cosmo) / u.Mpc magnitude_target = r_mag_target - 5*np.log(dL) - 25 #if you have the absolute magnitude of your target, remove the above calculation and use: #magnitude_target = quasar_list.iloc[n][2] #instead #print(r_magnitude_target, magnitude_target, redshift_target) #search for and download all data products associated with those coordinates obsTable = Observations.query_region(coords, radius = 0.02) dataProductsByObservation = Observations.get_product_list(obsTable) manifest = Observations.download_products(dataProductsByObservation, download_dir=coords, obs_collection = "HST", dataproduct_type = "image", productType = "SCIENCE") except TimeoutError: n = n + 1 print("a timeout error occurred at", coords) #isolate drizzle files findQSOdrz = 'find . -type f \( -name "*drz.fits" \)' out = subprocess.Popen(findQSOdrz, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def get_table(self, telescope): options = self.options[telescope] if (telescope.upper() == 'HST'): from astroquery.mast import Observations # Get table table = Observations.query_region(self.coord, radius=options['radius']) # HST-specific masks filmask = [ table['filters'] == good for good in options['mask']['filters'] ] filmask = [any(l) for l in list(map(list, zip(*filmask)))] expmask = table['t_exptime'] > options['mask']['t_exptime'] obsmask = [ table['obs_collection'] == good for good in options['mask']['obs_collection'] ] obsmask = [any(l) for l in list(map(list, zip(*obsmask)))] detmask = [ table['instrument_name'] == good for good in options['mask']['instrument_name'] ] detmask = [any(l) for l in list(map(list, zip(*detmask)))] # Construct and apply mask mask = [all(l) for l in zip(filmask, expmask, obsmask, detmask)] table = table[mask] if (telescope.upper() == 'SPITZER'): import requests from dateutil.parser import parse params = { 'RA': self.coord.ra.degree, 'DEC': self.coord.dec.degree, 'DATASET': 'ivo://irsa.ipac.spitzer.level2', 'SIZE': options['radius'].to(u.degree).value, 'VERB': 3 } url = options['url'] r = requests.get(url, params=params) if 'No Matches' in r.text: return None if r.status_code != 200: message = 'status message: {message}.' print(message.format(message=r.text)) error = 'ERROR: could not get url {url}, status code {stat}.' raise RuntimeError(error.format(url=url, stat=r.status_code)) return None # Parse table from html text # This is borrowed from astroquery.sha raw_data = [line for line in r.text.split('\n')] field_widths = [len(s) + 1 for s in raw_data[1].split('|')][1:-1] col_names = [s.strip() for s in raw_data[1].split('|')][1:-1] type_names = [s.strip() for s in raw_data[2].split('|')][1:-1] cs = [0] + np.cumsum(field_widths).tolist() def parse_line(line, cs=cs): return [line[a:b] for a, b in zip(cs[:-1], cs[1:])] data = [parse_line(row) for row in raw_data[4:-1]] # Parse type names dtypes = self.map_dtypes(type_names, field_widths) # To table # transpose data for appropriate table instance handling table = Table(list(zip(*data)), names=col_names, dtype=dtypes) # Split wavelength into instrument and filter instrument = [] filt = [] for w in table['wavelength']: dat = w.split() inst = dat[0].strip() if 'IRS' in inst: f = dat[2] elif 'IRAC' in inst: f = dat[1] elif 'MIPS' in inst: if 'SED' in dat[1]: f = dat[2] else: f = dat[1] else: print('Unknown instrument: ', inst) instrument.append(inst) filt.append(f) table.add_column( astropy.table.Column(name='instrument', data=instrument)) table.add_column(astropy.table.Column(name='filter', data=filt)) detmask = [ table['instrument'] == good for good in options['mask']['instrument'] ] detmask = [any(l) for l in list(map(list, zip(*detmask)))] promask = [[good in x for x in table['externalname']] for good in options['mask']['externalname']] promask = [any(l) for l in list(map(list, zip(*promask)))] # Construct and apply mask mask = [all(l) for l in zip(detmask, promask)] table = table[mask] table.add_column( astropy.table.Column(name='jpg', data=[None] * len(table))) # Estimate exposure time. This is only defined for Level 1 products # since all other images are stacks. But we can guesstimate the # exposure time by looking at the beginning and end time of the # observation exptime = [ (parse(end) - parse(start)).seconds for start, end in zip(table['begintime'], table['endtime']) ] table.add_column(astropy.table.Column(name='exptime', data=exptime)) if (telescope.upper() == 'CHANDRA'): import requests from astropy.io.votable import parse url = 'https://cxcfps.cfa.harvard.edu/cgi-bin/'+\ 'cda/footprint/get_vo_table.pl' ra = self.coord.ra.degree dec = self.coord.dec.degree params = { 'pos': '{0},{1}'.format(ra, dec), 'size': 0.2, 'grating': 'NONE', 'inst': 'ACIS-I,ACIS-S' } if True: # Try to get a response from the URL r = requests.get(url, params=params) # Create a temporary file and then read it into a votable object. # This is terrible, but best way I've found to complete this part fname = 'test.xml' f = open(fname, 'w') f.write(r.text) f.close() votable = parse(fname) os.remove(fname) #except: # return(None) tbdata = votable.get_first_table().to_table() # Although using a large search radius, check to make sure that the # input coordinates are < 5 arcmin off axis remove = [] for i, row in enumerate(copy.copy(tbdata)): coord = SkyCoord(row['RA'], row['Dec'], unit=(u.deg, u.deg), frame='icrs') if self.coord.separation(coord).arcmin > 5: remove.append(i) tbdata.remove_rows(remove) # Require that all observations have SI Mode = TE (timed exposure) # rather than CC = continuous clocking (and thus not imaging) remove = [] for i, row in enumerate(copy.copy(tbdata)): if not row['SIMode'].decode('utf-8').startswith('TE'): remove.append(i) tbdata.remove_rows(remove) table = tbdata # Rename column names to standard names for keypair in options['keymap']: oldname = keypair[0] newname = keypair[1] table.rename_column(oldname, newname) # Reorder table and make sure only standard keys are used return table