def tess_target_download(targID, sectors='all', sc=True, lc_format='pdc', delete_fits='n'): ### this function interfaces with MASS to download light curves based on the TIC #. if os.path.exists(moonpydir + '/TESS_lcs'): pass else: os.system('mkdir ' + moonpydir + '/TESS_lcs') ### first try a simple curl script! all_times = [] all_fluxes = [] all_errors = [] all_flags = [] sectors = [] lcfiles = [] try: if targID.startswith('TIC'): ticnum = str(targID)[3:] if ticnum.startswith(' '): ticnum = str(ticnum)[1:] else: ticnum = str(targID) ### you've already handled the TOI already! if ticnum.startswith(' '): ticnum = ticnum[1:] ### prepare for the query query_num = ticnum while len(query_num) < 16: query_num = '0' + query_num assert len(query_num) == 16 sector_prefixes, sector_suffixes = {}, {} ### these can be found at archive.stsci.edu/tess/bulk_downloads/bulk_downloads_ffi-tp-lc-dv.html, ### in the tesscurl_sector_NN_lc.sh files. nsectors = 99 nactual_sectors = 0 for sector in np.arange(1, nsectors, 1): ### get the curl script... then extract the prefixes and suffixes from the first line. try: if os.path.exists(moonpydir + '/sector' + str(sector) + "_curlscript.txt"): pass else: sector_curl_URL = 'http://archive.stsci.edu/missions/tess/download_scripts/sector/tesscurl_sector_' + str( sector) + '_lc.sh' os.system('wget --tries=1 -N "' + sector_curl_URL + '" -O ' + moonpydir + '/sector' + str(sector) + "_curlscript.txt") curltxt = open(moonpydir + '/sector' + str(sector) + '_curlscript.txt', mode='r') first_line = curltxt.readline() second_line = curltxt.readline() sector_prefix = second_line[16:40] sector_suffix = second_line[56:71] ### now read the first line of that sector_prefixes[sector], sector_suffixes[ sector] = sector_prefix, sector_suffix if len(sector_prefix) > 0: print("sector_prefix, sector_suffix = ", sector_prefix, sector_suffix) nactual_sectors += 1 else: break except: traceback.print_exc() break nsectors = nactual_sectors print('nsectors = ', nsectors) """ sector_prefixes[1], sector_suffixes[1] = 'tess2018206045859-s0001-', '-0120-s_lc.fits' sector_prefixes[2], sector_suffixes[2] = 'tess2018234235059-s002-', '-0121-s_lc.fits' sector_prefixes[3], sector_suffixes[3] = 'tess2018263035959-s0003-', '-0123-s_lc.fits' sector_prefixes[4], sector_suffixes[4] = 'tess2018292075959-s0004-', '-0124-s_lc.fits' sector_prefixes[5], sector_suffixes[5] = 'tess2018319095959-s0005-', '-0125-s_lc.fits' sector_prefixes[6], sector_suffixes[6] = 'tess2018349182459-s0006-', '-0126-s_lc.fits' sector_prefixes[7], sector_suffixes[7] = 'tess2019006130736-s0007-', '-0131-s_lc.fits' sector_prefixes[8], sector_suffixes[8] = 'tess2019032160000-s0008-', '-0136-s_lc.fits' sector_prefixes[9], sector_suffixes[9] = 'tess2019058134432-s0009-', '-0139-s_lc.fits' sector_prefixes[10], sector_suffixes[10] = 'tess2019085135100-s0010-', '-0140-s_lc.fits' sector_prefixes[11], sector_suffixes[11] = 'tess2019112060037-s0011-', '-0143-s_lc.fits' sector_prefixes[12], sector_suffixes[12] = 'tess2019140104343-s0012-', '-0144-s_lc.fits' sector_prefixes[13], sector_suffixes[13] = 'tess2019169103026-s0013-', '-0146-s_lc.fits' sector_prefixes[14], sector_suffixes[14] = 'tess2019198215352-s0014-', '-0150-s_lc.fits' sector_prefixes[15], sector_suffixes[15] = 'tess2019226182529-s0015-', '-0151-s_lc.fits' sector nsectors = 28 """ for sector in np.arange(1, nsectors + 1, 1): download_directory = central_data_dir + 'TESS_lightcurves/TIC' + str( ticnum) if os.path.exists(download_directory): pass else: os.system('mkdir ' + download_directory) lcdownload_name = 'TIC' + ticnum + '_sector' + str( sector) + '-s_lc.fits' os.system( 'curl -C - -L -o ' + download_directory + '/' + lcdownload_name + ' https://mast.stsci.edu/api/v0.1/Download/file/?uri=mast:TESS/product/' + sector_prefixes[sector] + query_num + sector_suffixes[sector]) print( 'downloading the light curve for ' + str(targID) + ' in sector ', sector) try: lcfile = pyfits.open(moonpydir + '/TESS_lcs/' + lcdownload_name) except: os.system('rm -rf ' + moonpydir + '/TESS_lcs/' + lcdownload_name) continue lcfiles.append(lcfile) lcdata = lcfile[1].data lctimes = np.array(lcdata['TIME']) if lc_format == 'pdc': lcfluxes = np.array(lcdata['PDCSAP_FLUX']) lcerrors = np.array(lcdata['PDCSAP_FLUX_ERR']) elif lc_format == 'sap': lcfluxes = np.array(lcdata['SAP_FLUX']) lcerrors = np.array(lcdata['SAP_FLUX_ERR']) lcflags = np.array(lcdata['QUALITY']) sector = lcfile[0].header['SECTOR'] all_times.append(lctimes) all_fluxes.append(lcfluxes) all_errors.append(lcerrors) all_flags.append(lcflags) sectors.append(sector) if delete_fits == 'y': os.system('rm -rf ' + moonpydir + '/TESS_lcs/' + lcdownload_name) except: traceback.print_exc() time.sleep(60) obsTable = Observations.query_object(targID, radius='0.001 deg') TESS_idxs = np.where(np.array(obsTable['obs_collection']) == 'TESS')[0] minTESSidx, maxTESSidx = np.nanmin(TESS_idxs), np.nanmax(TESS_idxs) + 1 dataproducts = Observations.get_product_list( obsTable[minTESSidx:maxTESSidx]) timeseries_idxs = np.where( np.array(dataproducts['dataproduct_type']) == 'timeseries')[0] obsids = np.array(dataproducts)['obsID'][timeseries_idxs] for obsid in np.unique(obsids): print("obsid = ", obsid) dataproductsbyID = Observations.get_product_list(obsid) manifest = Observations.download_products( dataproductsbyID, download_dir=moonpydir + '/TESS_lcs', dataproduct_type='timeseries', extension='lc.fits', mrp_only=True) for nmanfile, manfile in enumerate(manifest): manfilepath = manfile[0] if "_lc.fits" in manfilepath: print('found the light curve!') ### this is the only one you want to save! lcpath = manfilepath print("lcpath = ", lcpath) ### open the file, grab the data! lcfile = pyfits.open(lcpath) lcfiles.append(lcfile) lcdata = lcfile[1].data lctimes = np.array(lcdata['TIME']) if lc_format == 'pdc': lcfluxes = np.array(lcdata['PDCSAP_FLUX']) lcerrors = np.array(lcdata['PDCSAP_FLUX_ERR']) elif lc_format == 'sap': lcfluxes = np.array(lcdata['SAP_FLUX']) lcerrors = np.array(lcdata['SAP_FLUX_ERR']) lcflags = np.array(lcdata['QUALITY']) sector = lcfile[0].header['SECTOR'] all_times.append(lctimes) all_fluxes.append(lcfluxes) all_errors.append(lcerrors) all_flags.append(lcflags) sectors.append(sector) if delete_fits == 'y': os.system('rm ' + lcpath) break else: pass #os.system('rm -rf '+manfilepath) print(" ") print(" ") all_times, all_fluxes, all_errors, all_flags, sectors = np.array( all_times), np.array(all_fluxes), np.array(all_errors), np.array( all_flags), np.array(sectors) return all_times, all_fluxes, all_errors, all_flags, sectors
def _query_mast(target, radius=None, project=['Kepler', 'K2', 'TESS']): """Helper function which wraps `astroquery.mast.Observations.query_criteria()` to returns a table of all Kepler or K2 observations of a given target. Parameters ---------- target : str, int, or `astropy.coordinates.SkyCoord` object See docstrings above. radius : float or `astropy.units.Quantity` object Conesearch radius. If a float is given it will be assumed to be in units of arcseconds. If `None` then we default to 0.0001 arcsec. project : str, list of str 'Kepler', 'K2', and/or 'TESS'. Returns ------- obs : astropy.Table Table detailing the available observations on MAST. """ # If passed a SkyCoord, convert it to an RA and Dec if isinstance(target, SkyCoord): target = '{}, {}'.format(target.ra.deg, target.dec.deg) project = np.atleast_1d(project) if radius is None: radius = .0001 * u.arcsec elif not isinstance(radius, u.quantity.Quantity): radius = radius * u.arcsec try: # If `target` looks like a KIC or EPIC ID, we will pass the exact # `target_name` under which MAST will know the object to prevent # source confusion (see GitHub issue #148). target = int(target) if (target > 0) and (target < 200000000): target_name = 'kplr{:09d}'.format(target) elif (target > 200000000) and (target < 300000000): target_name = 'ktwo{:09d}'.format(target) else: raise ValueError( "{:09d}: not in the KIC or EPIC ID range".format(target)) # query_criteria does not allow a cone search when target_name is passed in # so first grab desired target with ~0 arcsecond radius with warnings.catch_warnings(): # suppress misleading AstropyWarning warnings.simplefilter('ignore', AstropyWarning) target_obs = Observations.query_criteria(target_name=target_name, radius=str( radius.to(u.deg)), project=project, obs_collection=project) if len(target_obs) == 0: raise ValueError( "No observations found for {}.".format(target_name)) # check if a cone search is being performed # if yes, perform a cone search around coordinates of desired target if radius < (0.1 * u.arcsec): obs = target_obs # astroquery does not return distance if target_name is given; # we add it here so that the table returned always has this column. obs['distance'] = 0. else: ra = target_obs['s_ra'][0] dec = target_obs['s_dec'][0] with warnings.catch_warnings(): # suppress misleading AstropyWarning warnings.simplefilter('ignore', AstropyWarning) obs = Observations.query_criteria(coordinates='{} {}'.format( ra, dec), radius=str(radius.to(u.deg)), project=project, obs_collection=project) obs.sort('distance') return obs except ValueError: pass # If `target` did not look like a KIC or EPIC ID, then we let MAST # resolve the target name to a sky position. Convert radius from arcsec # to degrees for query_criteria(). try: with warnings.catch_warnings(): # suppress misleading AstropyWarning warnings.simplefilter('ignore', AstropyWarning) obs = Observations.query_criteria(objectname=target, radius=str(radius.to(u.deg)), project=project, obs_collection=project) obs.sort('distance') return obs except ResolverError as exc: raise SearchError(exc)
def get_all_TESS_data(object_name, radius=".02 deg", get_PDC=True, get_all=False, get_lightcurves_only=True): """ Given a planet name, this function returns a dictionary of times, fluxes and errors on fluxes in a juliet-friendly format for usage. The function does an astroquery to MAST using a default radius of .02 deg around the target name. If get_PDC is True, this function returns PDC fluxes. False returns SAP fluxes. If get_all is true, this function returns a dictionary that in addition to the times, fluxes and errors, returns other metadata. """ if not has_astroquery: print( "Error on using juliet function `get_all_TESS_data`: astroquery.mast not found." ) obs_table = Observations.query_object(object_name, radius=radius) out_dict = {} times = {} fluxes = {} fluxes_errors = {} for i in range(len(obs_table['dataURL'])): if 's_lc.fits' in obs_table['dataURL'][i]: fname = obs_table['dataURL'][i].split('/')[-1] metadata = fname.split('-') if len(metadata) == 5: # Extract metadata: sector = int(metadata[1].split('s')[-1]) ticid = int(metadata[2]) # Download files: data_products = Observations.get_product_list(obs_table[i]) if get_lightcurves_only: want = data_products['description'] == "Light curves" else: want = (data_products['description'] == "Light curves") or (data_products['description'] == "Target pixel files") manifest = Observations.download_products(data_products[want]) # Read lightcurve file: d, h = fits.getdata('mastDownload/TESS/' + fname[:-8] + '/' + fname, header=True) t,fs,fserr,f,ferr = d['TIME']+h['BJDREFI'],d['SAP_FLUX'],d['SAP_FLUX_ERR'],\ d['PDCSAP_FLUX'],d['PDCSAP_FLUX_ERR'] idx_goodpdc = np.where((f != 0.) & (~np.isnan(f)))[0] idx_goodsap = np.where((fs != 0.) & (~np.isnan(fs)))[0] # Save to output dictionary: if 'TIC' not in out_dict.keys(): out_dict['TIC'] = ticid out_dict[sector] = {} out_dict[sector]['TIME_PDCSAP_FLUX'] = t[idx_goodpdc] out_dict[sector]['PDCSAP_FLUX'] = f[idx_goodpdc] out_dict[sector]['PDCSAP_FLUX_ERR'] = ferr[idx_goodpdc] out_dict[sector]['TIME_SAP_FLUX'] = t[idx_goodsap] out_dict[sector]['SAP_FLUX'] = fs[idx_goodsap] out_dict[sector]['SAP_FLUX_ERR'] = fserr[idx_goodsap] if get_PDC: times['TESS' + str(sector)] = t[idx_goodpdc] med = np.median(f[idx_goodpdc]) fluxes['TESS' + str(sector)] = f[idx_goodpdc] / med fluxes_errors['TESS' + str(sector)] = ferr[idx_goodpdc] / med else: times['TESS' + str(sector)] = t[idx_goodsap] med = np.median(fs[idx_goodsap]) fluxes['TESS' + str(sector)] = fs[idx_goodsap] / med fluxes_errors['TESS' + str(sector)] = fserr[idx_goodsap] / med # Remove downloaded folder: os.system('rm -r mastDownload') if get_all: return out_dict, times, fluxes, fluxes_errors else: return times, fluxes, fluxes_errors
def locate_postcard(self, local): """ Finds the eleanor postcard, if available, this star falls on. Attributes ---------- postcard : str postcard_bkg : str postcard_path : str position_on_postcard : list all_postcards : list mast_list : astropy.table.Table """ self.mast_list = None info_str = "{0:04d}-{1}-{2}-{3}".format(self.sector, self.camera, self.chip, "cal") postcard_fmt = "postcard-s{0}-{{0:04d}}-{{1:04d}}" postcard_fmt = postcard_fmt.format(info_str) eleanorpath = os.path.dirname(__file__) guide_url = eleanorpath + '/postcard_centers.txt' guide = Table.read(guide_url, format="ascii") col, row = self.position_on_chip[0], self.position_on_chip[1] post_args = np.where((np.abs(guide['x'].data - col) <= 100) & (np.abs(guide['y'].data - row) <= 100)) post_cens = guide[post_args] # Finds the mostest closest postcard closest_x, closest_y = np.argmin( np.abs(post_cens['x'] - col)), np.argmin( np.abs(post_cens['y'] - row)) self.postcard = postcard_fmt.format(post_cens['x'].data[closest_x], post_cens['y'].data[closest_y]) # Keeps track of all postcards that the star falls on all_postcards = [] for i in range(len(post_cens)): name = postcard_fmt.format(post_cens['x'].data[i], post_cens['y'].data[i]) all_postcards.append(name) self.all_postcards = np.array(all_postcards) if local == False: postcard_obs = Observations.query_criteria( provenance_name="ELEANOR", target_name=self.postcard, obs_collection="HLSP") if len(postcard_obs) > 0: product_list = Observations.get_product_list(postcard_obs) # we're just checking if the pointing file already exists. we don't need # to be warned that it doesn't. with warnings.catch_warnings(): warnings.simplefilter("ignore", category=EleanorWarning) self.pointing = check_pointing(self.sector, self.camera, self.chip, self.pm_dir) if self.pointing is None: extension = ["pc.fits", "bkg.fits", "pm.txt"] else: extension = ["pc.fits", "bkg.fits"] results = Observations.download_products( product_list, extension=extension, download_dir=self.postcard_path) postcard_path = results['Local Path'][0] self.postcard_path = '/'.join( e for e in postcard_path.split('/')[:-1]) self.postcard = results['Local Path'][1].split('/')[-1] self.postcard_bkg = results['Local Path'][0].split('/')[-1] self.mast_results = results self.cutout = None # Attribute for TessCut only # only downloaded the pointing model if the search for it above failed, so only # update it in that case here if self.pointing is None: self.pm_dir = self.postcard_path self.pointing = check_pointing(self.sector, self.camera, self.chip, self.pm_dir) else: print( "No eleanor postcard has been made for your target (yet). Using TessCut instead." ) self.locate_with_tesscut() else: self.cutout = None #Attribute for TessCut only self.postcard_bkg = 'hlsp_eleanor_tess_ffi_' + self.postcard + '_tess_v2_bkg.fits' self.postcard = 'hlsp_eleanor_tess_ffi_' + self.postcard + '_tess_v2_pc.fits' self.pointing = check_pointing(self.sector, self.camera, self.chip, self.pm_dir)
def download_all(self, quality_bitmask='default', download_dir=None, cutout_size=None): """Returns a `TargetPixelFileCollection or `LightCurveFileCollection`. Parameters ---------- quality_bitmask : str or int Bitmask (integer) which identifies the quality flag bitmask that should be used to mask out bad cadences. If a string is passed, it has the following meaning: * "none": no cadences will be ignored (`quality_bitmask=0`). * "default": cadences with severe quality issues will be ignored (`quality_bitmask=1130799`). * "hard": more conservative choice of flags to ignore (`quality_bitmask=1664431`). This is known to remove good data. * "hardest": removes all data that has been flagged (`quality_bitmask=2096639`). This mask is not recommended. See the :class:`KeplerQualityFlags` class for details on the bitmasks. download_dir : str Location where the data files will be stored. Defaults to "~/.lightkurve-cache" if `None` is passed. cutout_size : int, float or tuple Side length of cutout in pixels. Tuples should have dimensions (y, x). Default size is (5, 5) Returns ------- collection : `lightkurve.Collection` object Returns a `LightCurveFileCollection` or `TargetPixelFileCollection`, containing all entries in the products table """ if len(self.table) == 0: warnings.warn("Cannot download from an empty search result.", LightkurveWarning) return None # Make sure astroquery uses the same level of verbosity logging.getLogger('astropy').setLevel(log.getEffectiveLevel()) # download all products listed in self.products if download_dir is None: download_dir = self._default_download_dir() # if table contains TESScut search results, download cutouts if 'FFI Cutout' in self.table[0]['description']: path = [ self._fetch_tesscut_path(t, s, download_dir, cutout_size) for t, s in zip(self.table['target_name'], self.table['sequence_number']) ] else: if cutout_size is not None: warnings.warn( '`cutout_size` can only be specified for TESS ' 'Full Frame Image cutouts.', LightkurveWarning) path = Observations.download_products( self.table, mrp_only=False, download_dir=download_dir)['Local Path'] # open() will determine filetype and return # return a collection containing opened files tpf_extensions = ['lpd-targ.fits', 'spd-targ.fits', '_tp.fits', 'n/a'] lcf_extensions = ['llc.fits', 'slc.fits', '_lc.fits'] if any(e in self.table['productFilename'][0] for e in tpf_extensions): return TargetPixelFileCollection([open(p) for p in path]) elif any(e in self.table['productFilename'][0] for e in lcf_extensions): return LightCurveFileCollection([open(p) for p in path])
import matplotlib.pyplot as plt from astroquery.mast import Observations from scipy import signal from Time_and_flux import gettimeflux_1800 ids = ['167602025', '167695269', '167695269', '167814740', '167814740'] cadence = '1800' flare_times = [2458335.233, 2458336.636, 2458343.026, 2458344.888, 2458344.875] for this_id in ids, bjd in flare_times: #for bjd in flare_times: tbjd = bjd - 2457000 obsTable = Observations.query_criteria(target_name=this_id, obs_collection="HLSP", filters="TESS", t_exptime=[1799, 1801]) sector = obsTable['sequence_number'] tess_bjds = gettimeflux_1800(this_id, str(sector[0]))[0] sap_fluxes = gettimeflux_1800(this_id, str(sector[0]))[1] if (cadence == '1800'): med_flux_1 = signal.medfilt(sap_fluxes, kernel_size=65) else: med_flux_1 = signal.medfilt(sap_fluxes, kernel_size=101) t0 = tbjd #transit time fig, ax = plt.subplots() #ax.plot(tess_bjds, pdcsap_fluxes, 'ko')#BKG black #ax.plot(tess_bjds, med_flux_1, 'b-')
def mvm_retrieve_files(products, archive=False, clobber=False, log_level=logutil.logging.INFO): """ This function retrieves specified files from the archive - unless the file is found to be locally resident on disk. Upon completion, The function returns a list of filenames available on disk. Parameters ---------- products : `~astropy.table.Table` object A Table of products as returned by the mvm_id_filenames function. archive : Boolean, optional Retain copies of the downloaded files in the astroquery created sub-directories? Default is "False". clobber : Boolean, optional Download and Overwrite existing files? Default is "False". log_level : int, optional The desired level of verbosity in the log statements displayed on the screen and written to the .log file. Default value is 20, or 'info'. Returns ------- local_files : list List of filenames Note: Code here cribbed from retrieve_obsevation in astroquery_utils module. """ # set logging level to user-specified level log.setLevel(log_level) # Determine if the files of interest are already on the local disk. If so, # remove the filename from the download list. all_images = [] all_images = products['productFilename'].tolist() if not clobber: rows_to_remove = [] for row_idx, row in enumerate(products): fname = row['productFilename'] if os.path.isfile(fname): log.info(fname + " already exists. File download skipped.") rows_to_remove.append(row_idx) products.remove_rows(rows_to_remove) # Only download files as necessary if products: # Actual download of products log.info("Downloading files now...") manifest = Observations.download_products(products, mrp_only=False) else: log.info( "There are no files to download as they are all resident on disk.") # Manifest has the following columns: "Local Path", "Status", "Message", and "URL" if not clobber: for rownum in rows_to_remove[::-1]: if manifest: manifest.insert_row( rownum, vals=[all_images[rownum], "LOCAL", "None", "None"]) else: return all_images download_dir = None local_files = [] for file, file_status in zip(manifest['Local Path'], manifest['Status']): if file_status != "LOCAL": # Identify what sub-directory was created by astroquery for the # download if download_dir is None: download_dir = os.path.dirname(os.path.abspath(file)) # Move or copy downloaded file to current directory local_file = os.path.abspath(os.path.basename(file)) if archive: shutil.copy(file, local_file) else: shutil.move(file, local_file) # Record what files were downloaded and their current location local_files.append(os.path.basename(local_file)) else: local_files.append(file) if not archive: # Remove astroquery created sub-directories shutil.rmtree('mastDownload') return local_files
def tasoc_lc(self): """ Grabs the T'DA available light curves for your target. For more information, see the TASOC light curve documentation: https://tasoc.dk/code/. Parameters ---------- Attributes ---------- tasoc_header : tasoc_tpf : np.2darray tasoc_aperture : np.2darray tasoc_time : np.array tasoc_quality : np.array tasoc_timecorr : np.array tasoc_cadenceno : np.array tasoc_flux_raw : np.array tasoc_flux_raw_err : np.array tasoc_flux_corr : np.array tasoc_flux_corr_err : np.array tasoc_flux_bkg : np.array tasoc_pixel_quality : np.array Quality flags for the data; use these not `tasoc_quality`. tasoc_pos_corr1 : np.array tasoc_pos_corr2 : np.array tasoc_mom_centr1 : np.array tasoc_mom_centr2 : np.array """ products = Observations.query_object(objectname="TIC" + str(self.tic)) column = np.where((products['provenance_name'] == 'TASOC') & (products['target_name'] == str(self.tic)) & (products['sequence_number'] == self.sector))[0] if len(column) > 0: download = Observations.get_product_list(products[column]) manifest = Observations.download_products( download, download_dir=self.download_dir) self.tasoc_path = manifest["Local Path"].data[0] hdu = fits.open(self.tasoc_path) self.tasoc_header = hdu[0].header self.tasoc_tpf = hdu[2].data self.tasoc_aperture = hdu[3].data self.tasoc_time = hdu[1].data['TIME'] self.tasoc_quality = hdu[1].data['QUALITY'] self.tasoc_timecorr = hdu[1].data['TIMECORR'] self.tasoc_cadenceno = hdu[1].data['CADENCENO'] self.tasoc_flux_raw = hdu[1].data['FLUX_RAW'] self.tasoc_flux_bkg = hdu[1].data['FLUX_BKG'] self.tasoc_flux_corr = hdu[1].data['FLUX_CORR'] self.tasoc_pos_corr1 = hdu[1].data['POS_CORR1'] self.tasoc_pos_corr2 = hdu[1].data['POS_CORR2'] self.tasoc_mom_centr1 = hdu[1].data['MOM_CENTR1'] self.tasoc_mom_centr2 = hdu[1].data['MOM_CENTR2'] self.tasoc_pixel_quality = hdu[1].data['PIXEL_QUALITY'] self.tasoc_flux_raw_err = hdu[1].data['FLUX_RAW_ERR'] self.tasoc_flux_corr_err = hdu[1].data['FLUX_CORR_ERR'] else: raise SearchError("No TASOC light curve found.")
#!/usr/bin/env python3 from astroquery.mast import Observations import IPython import boto3 from s3_query import find_product_in_s3 Observations.enable_s3_hst_dataset() obs = Observations.query_criteria( dataproduct_type=['image'], project='HST', instrument_name='ACS/WFC', filters='F555W', calib_level=3, ) print("Observations: ", len(obs)) products = Observations.get_product_list(obs) print("Products: ", len(products)) filtered_products = Observations.filter_products( products, productType='SCIENCE', extension='fits', description='DADS FLT file - Calibrated exposure ACS/WFC3/STIS/COS', ) print("Filtered products: ", len(filtered_products)) print()
def get_mast_file_list(star_id, provenance, sequence_name): """ Queries MAST to retrieve archive data based on object name (``star_id``), data provenance, and sequence name. Parameters ---------- star_id: int, str EPIC ID of the star. provenance: str Provenance name in MAST archive, e.g., ``'K2'``, ``'EVEREST'``, ``'K2SFF'``. sequence_name: str Campaign number. Returns ------- file_names: list of str A Python list of all the file names retrieved from archive. """ star_id = str(star_id) target_name = '*' + star_id.split()[-1] # Make sure that there are data for the criteria mission = _MISSION_MAPPING[provenance] print("\n===== Retrieving data for observation: =====") print("** star_id: {}\n** provenance: {}\n** sequence_name: {}".format( star_id, provenance, sequence_name)) if sequence_name == '*': sequence_name = '' obs_count = Observations.query_criteria_count( obs_collection=mission, dataproduct_type=["timeseries"], instrument_name="Kepler", objectname=star_id, target_name=target_name, project="K2", provenance_name=provenance, sequence_number=sequence_name + '*') if obs_count == 0: raise RuntimeError("No data found in archive.") obs_table = Observations.query_criteria(obs_collection=mission, dataproduct_type=["timeseries"], instrument_name="Kepler", objectname=star_id, target_name=target_name, project="K2", provenance_name=provenance, sequence_number=sequence_name + '*') data_products = Observations.get_product_list(obs_table) lc_mask = [ "lightcurve" in x or "light curve" in x for x in map(str.lower, data_products['description']) ] if not any(lc_mask): raise RuntimeError("Retrieved data products do not contain light " "curve data.") data_products = data_products[lc_mask] # keep only rows with light curves manifest = Observations.download_products(data_products) files = list(manifest['Local Path']) # get local file names # sort results: idx = np.argsort(files) file_names = [files[i] for i in idx] print("Download Status: SUCCESS\n") return file_names
def retrieve_observation(obsid, suffix=['FLC'], archive=False,clobber=False): """Simple interface for retrieving an observation from the MAST archive If the input obsid is for an association, it will request all members with the specified suffixes. Parameters ----------- obsid : string ID for observation to be retrieved from the MAST archive. Only the IPPSSOOT (rootname) of exposure or ASN needs to be provided; eg., ib6v06060. suffix : list List containing suffixes of files which should be requested from MAST. path : string Directory to use for writing out downloaded files. If `None` (default), the current working directory will be used. archive : Boolean Retain copies of the downloaded files in the astroquery created sub-directories? Default is 'False'. clobber : Boolean Download and Overwrite existing files? Default is 'False'. Returns ------- local_files : list List of filenames """ local_files = [] # Query MAST for the data with an observation type of either "science" or "calibration" obsTable = Observations.query_criteria(obs_id=obsid, obstype='all') # Catch the case where no files are found for download if len(obsTable) == 0: log.info("WARNING: Query for {} returned NO RESULTS!".format(obsid)) return local_files dpobs = Observations.get_product_list(obsTable) dataProductsByID = Observations.filter_products(dpobs, productSubGroupDescription=suffix, extension='fits', mrp_only=False) # After the filtering has been done, ensure there is still data in the table for download. # If the table is empty, look for FLT images in lieu of FLC images. Only want one # or the other (not both!), so just do the filtering again. if len(dataProductsByID) == 0: log.info("WARNING: No FLC files found for {} - will look for FLT files instead.".format(obsid)) suffix = ['FLT'] dataProductsByID = Observations.filter_products(dpobs, productSubGroupDescription=suffix, extension='fits', mrp_only=False) # If still no data, then return. An exception will eventually be thrown in # the higher level code. if len(dataProductsByID) == 0: log.info("WARNING: No FLC or FLT files found for {}.".format(obsid)) return local_files allImages = [] for tableLine in dataProductsByID: allImages.append(tableLine['productFilename']) log.info(allImages) if not clobber: rowsToRemove = [] for rowCtr in range(0,len(dataProductsByID)): if os.path.exists(dataProductsByID[rowCtr]['productFilename']): log.info("{} already exists. File download skipped.".format(dataProductsByID[rowCtr]['productFilename'])) rowsToRemove.append(rowCtr) if rowsToRemove: rowsToRemove.reverse() for rowNum in rowsToRemove: dataProductsByID.remove_row(rowNum) manifest = Observations.download_products(dataProductsByID, mrp_only=False) if not clobber: rowsToRemove.reverse() for rownum in rowsToRemove: if not manifest: local_files = allImages return local_files else: manifest.insert_row(rownum,vals=[allImages[rownum],"LOCAL","None","None"]) download_dir = None for file,fileStatus in zip(manifest['Local Path'],manifest['Status']): if fileStatus != "LOCAL": # Identify what sub-directory was created by astroquery for the download if download_dir is None: file_path = file.split(os.sep) file_path.remove('.') download_dir = file_path[0] # Move or copy downloaded file to current directory local_file = os.path.abspath(os.path.basename(file)) if archive: shutil.copy(file, local_file) else: shutil.move(file, local_file) # Record what files were downloaded and their current location local_files.append(os.path.basename(local_file)) else: local_files.append(file) if not archive: # Remove astroquery created sub-directories shutil.rmtree(download_dir) return local_files
def _search_products(target, radius=None, filetype="Lightcurve", cadence='long', mission=('Kepler', 'K2', 'TESS'), quarter=None, month=None, campaign=None, sector=None, limit=None, **extra_query_criteria): """Helper function which returns a SearchResult object containing MAST products that match several criteria. Parameters ---------- target : str, int, or `astropy.coordinates.SkyCoord` object See docstrings above. radius : float or `astropy.units.Quantity` object Conesearch radius. If a float is given it will be assumed to be in units of arcseconds. If `None` then we default to 0.0001 arcsec. filetype : {'Target pixel', 'Lightcurve', 'FFI'} Type of files queried at MAST. cadence : str Desired cadence (`long`, `short`, `any`) mission : str, list of str 'Kepler', 'K2', or 'TESS'. By default, all will be returned. quarter, campaign, sector : int, list of ints Kepler Quarter, K2 Campaign, or TESS Sector number. By default all quarters/campaigns/sectors will be returned. month : 1, 2, 3, 4 or list of int For Kepler's prime mission, there are three short-cadence TargetPixelFiles for each quarter, each covering one month. Hence, if cadence='short' you can specify month=1, 2, 3, or 4. By default all months will be returned. limit : int Maximum number of products to return Returns ------- SearchResult : :class:`SearchResult` object. """ if isinstance(target, int): if (0 < target) and (target < 13161030): log.warning( "Warning: {} may refer to a different Kepler or TESS target. " "Please add the prefix 'KIC' or 'TIC' to disambiguate." "".format(target)) elif (0 < 200000000) and (target < 251813739): log.warning( "Warning: {} may refer to a different K2 or TESS target. " "Please add the prefix 'EPIC' or 'TIC' to disambiguate." "".format(target)) # Speed up by restricting the MAST query if we don't want FFI image data extra_query_criteria = {} if filetype in ['Lightcurve', 'Target Pixel']: # At MAST, non-FFI Kepler pipeline products are known as "cube" products, # and non-FFI TESS pipeline products are listed as "timeseries". extra_query_criteria['dataproduct_type'] = ['cube', 'timeseries'] observations = _query_mast(target, project=mission, radius=radius, **extra_query_criteria) log.debug("MAST found {} observations. " "Now querying MAST for the corresponding data products." "".format(len(observations))) if len(observations) == 0: raise SearchError('No data found for target "{}".'.format(target)) # Light curves and target pixel files if filetype.lower() != 'ffi': from astroquery.mast import Observations products = Observations.get_product_list(observations) result = join(products, observations, keys="obs_id", join_type='left', uniq_col_name='{col_name}{table_name}', table_names=['', '_2']) result.sort(['distance', 'obs_id']) masked_result = _filter_products(result, filetype=filetype, campaign=campaign, quarter=quarter, cadence=cadence, project=mission, month=month, sector=sector, limit=limit) log.debug("MAST found {} matching data products.".format( len(masked_result))) return SearchResult(masked_result) # Full Frame Images else: cutouts = [] for idx in np.where( ['TESS FFI' in t for t in observations['target_name']])[0]: # if target passed in is a SkyCoord object, convert to RA, dec pair if isinstance(target, SkyCoord): target = '{}, {}'.format(target.ra.deg, target.dec.deg) # pull sector numbers s = observations['sequence_number'][idx] # if the desired sector is available, add a row if s in np.atleast_1d(sector) or sector is None: cutouts.append({ 'description': 'TESS FFI Cutout (sector {})'.format(s), 'target_name': str(target), 'targetid': str(target), 'productFilename': 'TESSCut', 'distance': 0.0, 'sequence_number': s, 'obs_collection': 'TESS' }) if len(cutouts) > 0: log.debug("Found {} matching cutouts.".format(len(cutouts))) masked_result = Table(cutouts) masked_result.sort(['distance', 'sequence_number']) else: masked_result = None return SearchResult(masked_result)
if simbadResults is not None: break if simbadResults is not None: coord = SkyCoord(simbadResults["RA"][0], simbadResults["DEC"][0], unit=[u.hourangle, u.deg]) ras.append(coord.ra.value) raUnit = coord.ra.unit decs.append(coord.dec.value) decUnit = coord.dec.unit else: ras.append(StarListTics["ra"][where_closest]) decs.append(StarListTics["dec"][where_closest]) #Retrieve data obsTable = Observations.query_criteria(filters=["TESS"], objectname=targetName, dataproduct_type=["TIMESERIES"], radius="0.01 deg") proceed = True if len(obsTable) == 0: proceed = False obsIDList.append(np.nan) pathList.append("NA") sectorsTxt, sectorsNb = TESSSectorLookUp(targetName) if len(sectorsNb) == 0: sectorList.append("") elif len(sectorsNb) == 1: sectorList.append(sectorsNb[0]) else: sectorList.append(",".join(["{}".format(s) for s in sectorsNb]))
#!/usr/bin/env python3 # # from astroquery.mast import Observations import boto3 import json import os import IPython wfc = Observations.query_criteria( dataproduct_type=['image'], project='HST', instrument_name='ACS/WFC', ) print("WFC: ", len(wfc)) hrc = Observations.query_criteria( dataproduct_type=['image'], project='HST', instrument_name='ACS/HRC', ) print("HRC: ", len(hrc)) IPython.embed()
def get_tess_data(u_ticid, max_flag=16, out_dir='./download'): """Given a TIC-ID, return time,flux,ferr,itime u_ticid : (int) TIC ID returns lc_time,flux,ferr,int_time """ tic_str = 'TIC' + str(u_ticid) #out_dir='/data/rowe/TESS/download/' # Search MAST for TIC ID obs_table = Observations.query_object(tic_str, radius=".002 deg") # Identify TESS timeseries data sets (i.e. ignore FFIs) oti=(obs_table["obs_collection"] == "TESS") & \ (obs_table["dataproduct_type"] == "timeseries") if oti.any() == True: data_products = Observations.get_product_list(obs_table[oti]) dpi = [ j for j, s in enumerate(data_products["productFilename"]) if "lc.fits" in s ] manifest = Observations.download_products(data_products[dpi], download_dir=out_dir) else: manifest = [] lc_time = [] flux = [] ferr = [] int_time = [] for j in range(0, len(manifest)): fits_fname = str(manifest["Local Path"][j]) print(fits_fname) hdu = fits.open(fits_fname) tmp_bjd = hdu[1].data['TIME'] tmp_flux = hdu[1].data['PDCSAP_FLUX'] tmp_ferr = hdu[1].data['PDCSAP_FLUX_ERR'] tmp_int_time = hdu[1].header['INT_TIME'] + np.zeros(len(tmp_bjd)) tmp_flag = hdu[1].data['QUALITY'] ii = (tmp_flag <= max_flag) & (~np.isnan(tmp_flux)) tmp_bjd = tmp_bjd[ii] tmp_flux = tmp_flux[ii] tmp_ferr = tmp_ferr[ii] tmp_int_time = tmp_int_time[ii] # Shift flux measurements median_flux = np.median(tmp_flux) tmp_flux = tmp_flux / median_flux tmp_ferr = tmp_ferr / median_flux # Append to output columns lc_time = np.append(lc_time, tmp_bjd) flux = np.append(flux, tmp_flux) ferr = np.append(ferr, tmp_ferr) int_time = np.append(int_time, tmp_int_time) hdu.close() # Sort by time si = np.argsort(lc_time) lc_time = np.asarray(lc_time)[si] flux = np.asarray(flux)[si] ferr = np.asarray(ferr)[si] int_time = np.asarray(int_time)[si] phot = phot_class() phot.time = np.copy(lc_time) phot.flux = np.copy(flux) phot.ferr = np.copy(ferr) phot.itime = np.copy(int_time) phot.itime = phot.itime / (60 * 24) #convert minutes to days return phot
from astroquery.mast import Observations import numpy as np import os,sys search_radius = 120 ## import files ## inp0 = 'ALMA_z6qso_all.list' data = np.loadtxt(inp0,comments='#',dtype={'names':('proj_id','name','ra','dec','PI'),'formats':('S32','S32','f16','f16','S32')}) proj_id = data["proj_id"] name = data["name"] ra = data["ra"] dec = data["dec"] for i in range(len(name)): obs_table = Observations.query_criteria(coordinates="%3.5f %3.5f" %(ra[i],dec[i]), radius="%3.5f arcsec" % search_radius, intentType=["science","SCIENCE"], obs_collection=["HST"]) __,uidx = np.unique(obs_table['target_name'],return_index=True) target_table = obs_table[uidx]['target_name','s_ra','s_dec','filters','t_exptime','proposal_id','dataURL','obsid'] if len(uidx) > 0: os.system('mkdir download_sh/'+name[i]) for u in range(len(uidx)): data_products = Observations.get_product_list(target_table[u][-1]) Observations.download_products(data_products, calib_level=[2,3], productType="SCIENCE", curl_flag=True,mrp_only=True,download_dir='download_sh/'+name[i])
def download_lc(target_name: str, flux_type='PDCSAP', mission: str = 'TESS',sigma_clip=4,iters=1,do_pca : bool = False,do_psf :bool= False) -> Tuple[ LightCurve, Union[List[Figure],None]]: """ Downloads a light curve using the TESS mission. If the star has been observed in the SC mode, it will download the original light curve from MAST. You can also choose the flux type you want to use. If it wasn't observed in SC mode, it will try to extract a light curve from the FFIs if the target has been observed by TESS. You can also download light curves of stars that are observed by the K2 or Kepler mission, by setting the mission parameter. :param target_name: Name of the target. You can either provide the TIC ID (TIC ...), Kepler ID (KIC ...), K2 ID(EPIC ...) or a name that is resolvable by Simbad. :param flux_type: Type of flux in the SC mode. Can be either PDCSAP or SAP or PSF for long cadence data :param mission: Mission from which the light curves are extracted. By default TESS only is used. You can consider all missions by passing 'all' (TESS, Kepler, K2) :param sigma_clip: Sigma clip parameter. Defines the number of standard deviations that are clipped. :param iters: Iterations for the sigma clipping :return: lightkurve.LightCurve object and validation page if extracted from FFI """ chosen_mission = [mission] if mission != 'all' else ('Kepler', 'K2', 'TESS') mprint(f"Searching processed light curves for {target_name} on mission(s) {','.join(chosen_mission)} ... ", log) if chosen_mission == ['TESS']: if target_name.startswith('TIC'): tic_id = re.findall(r'\d+', target_name) if len(tic_id) == 0: raise ValueError(ctext("A Tic ID needs to consist of TIC and a number!", error)) tic_id = int(tic_id[0]) else: mprint(f"Resolving {target_name} to TIC using MAST ...",log) try: tic_id = Catalogs.query_object(target_name,catalog='TIC',radius=0.003)[0]['ID'] except KeyError: raise ValueError(ctext(f"No TESS observations available for {target_name}", error)) mprint(f"TIC ID for {target_name}: TIC {tic_id}",log) o = Observations.query_criteria(objectname=target_name, radius=str(0 * u.deg), project='TESS', obs_collection='TESS').to_pandas() if len(o) > 0 and len(o[o.target_name != 'TESS FFI']) > 0: mprint(f"Short cadence observations available for {target_name}. Downloading ...",info) res = search_lightcurvefile(target_name, mission=chosen_mission) else: #Only FFI available mprint(f"No short cadence data available for {target_name}, extracting from FFI ...",info) lc, fig,_ = cut_ffi(tic_id,sigma_clip,iters,do_pca,do_psf,flux_type) mprint(f"Total observation length: {'%.2f' % (lc.time[-1] - lc.time[0])} days.", log) return lc, fig else: res = search_lightcurvefile(target_name, mission=chosen_mission) if len(res) != 0: fig = None mprint(f"Found processed light curve for {target_name}!", info) res = res.download_all() types = [] for d in res.data: type = 'TESS' if isinstance(d, lk.TessLightCurveFile) else 'Kepler' if type not in types: types.append(type) mprint(f"Using {','.join(types)} observations! Combining sectors ...", log) if flux_type == 'PSF': mprint(f"PSF not available for short cadence data. Reverting to PDCSAP",warn) flux_type = 'PDCSAP' if flux_type == 'PDCSAP': lc_set: List[Union[lk.TessLightCurve, lk.KeplerLightCurve]] = [i for i in res.PDCSAP_FLUX.data] elif flux_type == 'SAP': lc_set: List[Union[lk.TessLightCurve, lk.KeplerLightCurve]] = [i for i in res.SAP_FLUX.data] else: raise ValueError(ctext("Flux type needs to be either PDCSAP or SAP", error)) lc = combine_light_curves(lc_set,sigma_clip=sigma_clip,iters=iters) else: raise ValueError(ctext(f"No light curve available for {target_name} on mission(s) {chosen_mission}",error)) mprint(f"Total observation length: {'%.2f' % (lc.time[-1] - lc.time[0])} days.", log) return lc, fig
def get_hlsp_lightcurves(tic_id, hlsp_products=('CDIPS', 'TASOC', 'PATHOS'), download_dir=None, verbose=True): """This downloads TESS HLSP light curves for a given TIC ID. Parameters ---------- tic_id : str The TIC ID of the object as a string. hlsp_products : sequence of str List of desired HLSP products to search. For instance, ["CDIPS"]. download_dir : str Path of directory to which light-curve will be downloaded. Returns ------- lcfiles : list or None List of light-curve file paths. None if none are found and downloaded. """ if not astroquery_dependency: LOGERROR( "The astroquery package is required for this function to work.") return None lcfiles = [] for hlsp in hlsp_products: obs_table = Observations.query_criteria(target_name=tic_id, provenance_name=hlsp) if verbose: LOGINFO(f'Found {len(obs_table)} {hlsp} light-curves.') if len(obs_table) == 0: if verbose: LOGINFO("Did not find light-curves. Escaping.") return None # Get list of available products for this Observation. cdips_products = Observations.get_product_list(obs_table) # Download the products for this Observation. manifest = Observations.download_products(cdips_products, download_dir=download_dir) if verbose: LOGINFO("Done") if len(manifest) >= 1: lcfiles.append(list(manifest['Local Path'])) # # flatten lcfiles list # if len(lcfiles) >= 1: return_lcfiles = [item for sublist in lcfiles for item in sublist] else: return_lcfiles = None return return_lcfiles
redshift_target = quasar_list.iloc[n][3] #redshift of the target. r_mag_target = quasar_list.iloc[n][2] #magnitude of target #calculate the absolute magnitude of the target from the red magnitude dL = Distance(unit = u.Mpc, z=redshift_target, cosmology = cosmo) / u.Mpc magnitude_target = r_mag_target - 5*np.log(dL) - 25 #if you have the absolute magnitude of your target, remove the above calculation and use: #magnitude_target = quasar_list.iloc[n][2] #instead #print(r_magnitude_target, magnitude_target, redshift_target) #search for and download all data products associated with those coordinates obsTable = Observations.query_region(coords, radius = 0.02) dataProductsByObservation = Observations.get_product_list(obsTable) manifest = Observations.download_products(dataProductsByObservation, download_dir=coords, obs_collection = "HST", dataproduct_type = "image", productType = "SCIENCE") except TimeoutError: n = n + 1 print("a timeout error occurred at", coords) #isolate drizzle files findQSOdrz = 'find . -type f \( -name "*drz.fits" \)' out = subprocess.Popen(findQSOdrz, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
""" Example 10 ++++++++++ Retrieve Hubble archival data of M83 and make a figure """ from astroquery.mast import Mast, Observations from astropy.visualization import make_lupton_rgb, ImageNormalize import matplotlib.pyplot as plt import reproject result = Observations.query_object('M83') selected_bands = result[(result['obs_collection'] == 'HST') & (result['instrument_name'] == 'WFC3/UVIS') & ((result['filters'] == 'F657N') | (result['filters'] == 'F487N') | (result['filters'] == 'F336W')) & (result['target_name'] == 'MESSIER-083')] prodlist = Observations.get_product_list(selected_bands) filtered_prodlist = Observations.filter_products(prodlist) downloaded = Observations.download_products(filtered_prodlist) blue = fits.open(downloaded['Local Path'][2]) red = fits.open(downloaded['Local Path'][5]) green = fits.open(downloaded['Local Path'][8]) target_header = red['SCI'].header green_repr, _ = reproject.reproject_interp(green['SCI'], target_header) blue_repr, _ = reproject.reproject_interp(blue['SCI'], target_header)
def mvm_id_filenames(sky_coord, cutout_size, log_level=logutil.logging.INFO): """ This function retrieves a table of MVM drizzled image filenames with additional information from the archive. The user can then further cull the table to use as input to obtain a list of files from the archive. This function will return filter-level products. At this time, both ACS and WFC3 are searched by default. Parameters ---------- sky_coord : str or `~astropy.coordinates.SkyCoord` object The position around which to cutout. It may be specified as a string ("ra dec" in degrees) or as the appropriate `~astropy.coordinates.SkyCoord` object. cutout_size : int, array-like, `~astropy.units.Quantity` The size of the cutout array. If ``cutout_size`` is a scalar number or a scalar `~astropy.units.Quantity`, then a square cutout of ``cutout_size`` will be created. If ``cutout_size`` has two elements, they should be in ``(ny, nx)`` order. Scalar numbers in ``cutout_size`` are assumed to be in units of arcseconds. `~astropy.units.Quantity` objects must be in angular units. log_level : int, optional The desired level of verbosity in the log statements displayed on the screen and written to the .log file. Default value is 20, or 'info'. Returns ------- final_table : `~astropy.table.Table` object This utility also writes an output ECSV file version of the in-memory filtered data product table, final_table. The output filename is in the form: mvm_query-ra<###>d<####>-dec<n|s><##>d<####>_<radius>_cutout.ecsv (e.g., mvm_query-ra84d8208-decs69d8516_354_cutout.ecsv, where radius has been computed from the cutout dimensions. """ # set logging level to user-specified level log.setLevel(log_level) # If the cutout_size is not an astropy.units.Quantity object, the scalar(s) # are assumed to be arcseconds. The variable must be cast as a Quantity. if not isinstance(cutout_size, Quantity): cutout_size *= u.arcsec cutout_size = np.atleast_1d(cutout_size) if len(cutout_size) == 1: cutout_size = np.repeat(cutout_size, 2) if not isinstance(sky_coord, SkyCoord): sky_coord = SkyCoord(sky_coord, unit="deg") # From HST data, Search for the list of images based upon: coordinates, search region, data # product type, and the instrument name (with wildcard), project (HAP), and observation # collection (HST). Use the wildcard to get all the detectors for the instrument. Multiple # instruments cannot be searched at the same time. Use the diagonal of the cutout to define # the search radius for the archive. Images which fall outside the desired cutout need to # be filtered from the solution later. radius = math.ceil( math.sqrt( math.pow(cutout_size.value[0], 2) + math.pow(cutout_size.value[1], 2)) / 2.0) # Careful - the radius must be a str or Quantity radius *= u.arcsec log.info("Radius for query: {}.".format(radius)) log.info("Performing query for ACS images.") acs_query_table = Observations.query_criteria(coordinates=sky_coord, radius=radius, dataproduct_type="IMAGE", instrument_name="ACS*", project="HAP", obs_collection="HST") log.info("Performing query for WFC3 images.") wfc3_query_table = Observations.query_criteria(coordinates=sky_coord, radius=radius, dataproduct_type="IMAGE", instrument_name="WFC3*", project="HAP", obs_collection="HST") query_table = vstack([acs_query_table, wfc3_query_table]) del acs_query_table del wfc3_query_table # Catch the case where no files are found which satisfied the Query if not query_table: log.warning( "Query for objects within {} of {} returned NO RESULTS!".format( radius, (str_ra, str_dec))) return query_table # Compute the limits of the cutout region deg_cutout_size = cutout_size.to(u.deg) ra_min = sky_coord.ra.degree - deg_cutout_size.value[0] ra_max = sky_coord.ra.degree + deg_cutout_size.value[0] dec_min = sky_coord.dec.degree - deg_cutout_size.value[1] dec_max = sky_coord.dec.degree + deg_cutout_size.value[1] str_ra = "{:.4f}".format(sky_coord.ra.degree) str_dec = "{:.4f}".format(sky_coord.dec.degree) # Filter the output as necessary to include only MVM filenames (MVM prefix: hst_skycell). # Also, filter out images which are not actually in the requested cutout region as the # archive search had to be done using a radius. good_rows = [] updated_query_table = None for old_row in query_table: if old_row["obs_id"].startswith("hst_skycell"): if old_row["s_ra"] >= ra_min and old_row["s_ra"] <= ra_max and \ old_row["s_dec"] >= dec_min and old_row["s_dec"] <= dec_max: good_rows.append(old_row) # Catch the case where no files are found which satisfy the clean up criteria if len(good_rows) == 0: log.warning( "Query for objects within cutout {} of {} returned NO RESULTS!". format(cutout_size, (str_ra, str_dec))) return updated_query_table # Make the cleaned up table updated_query_table = Table(rows=good_rows, names=query_table.colnames) del query_table # Get the data product list associated with the elements of the table log.info("Get the product list for all entries in the query table.") dp_table = Observations.get_product_list(updated_query_table) del updated_query_table # Filter on MVM drizzled products only suffix = ["DRZ", "DRC"] log.info( "Filter the product list table for only {} filenames.".format(suffix)) filtered_dp_table = Observations.filter_products( dp_table, productSubGroupDescription=suffix, extension="fits") if not filtered_dp_table: log.warning( "No MVM drizzle product datasets (DRZ/DRC) found within {} of {}.". format(radius, (str_ra, str_dec))) return filtered_dp_table del dp_table # Need to filter out any non-hst-skycell entries AGAIN which may have # crept back into the list via the get_product_list() function. good_rows = [] output_table = None for old_row in filtered_dp_table: if old_row["obs_id"].startswith("hst_skycell"): good_rows.append(old_row) # Catch the case where no files are found which satisfy the criteria if len(good_rows) == 0: log.warning( "After filtering datasets there are NO RESULTS within {} of {}!". format(radius, (str_ra, str_dec))) return output_table # Make the output table output_table = Table(rows=good_rows, names=filtered_dp_table.colnames) del filtered_dp_table # Finally, make sure the entries are unique final_table = None final_table = unique(output_table, keys="productFilename") del output_table # Write the table to a file. This allows for further manipulation of # the information before a list of filenames is distilled from the table. # Output filename in the form: mvm_query-ra<###>d<####>-dec<n|s><##>d<####>_<radius>_cutout.ecsv # (e.g., mvm_query-ra84d9208-decs69d1483_71_cutout.ecsv), where radius has been computed from the # cutout dimensions. # # Get the whole number and fractional components of the RA and Dec ns = "s" if sky_coord.dec.degree < 0.0 else "n" ra_whole = int(sky_coord.ra.value) ra_frac = str(sky_coord.ra.value).split(".")[1][0:4] dec_whole = abs(int(sky_coord.dec.value)) dec_frac = str(sky_coord.dec.value).split(".")[1][0:4] log.info("coords2. {} {} {}".format(sky_coord.ra.value, sky_coord.dec.value, dec_frac)) query_filename = "mvm_query-ra" + str(ra_whole) + "d" + ra_frac + "-dec" + ns + \ str(dec_whole) + "d" + dec_frac + "_{:.0f}".format(radius.value) + "_cutout.ecsv" log.info( "Writing out the MVM product list table to {}.".format(query_filename)) log.info("Number of entries in table: {}.".format(len(final_table))) final_table.write(query_filename, format="ascii.ecsv") return final_table
def main(): # Read in the Pantheon+ catalog from Ben adap_dir = home + '/Documents/adap2021/' pantheon_datadir = adap_dir + 'pantheon_data/' cat = np.genfromtxt(adap_dir + 'pantheon_plus.csv', dtype=None, names=True, delimiter=',', encoding='ascii') print("Read in Pantheon+ catalog with the following header names:") print(cat.dtype.names) num_orig_cols = len(cat.dtype.names) # Open a new file to write an updated catalog # Adds the following columns # HST data # GALEX data # if yes to any of the above observatories then give # Inst/Camera field, and filters. # if no then leave these cols blank. fh = open(adap_dir + 'pantheon_plus_data.csv', 'w') # Write header fh.write("Serial_num,CID,CIDint,IDSURVEY,zHEL,zHELERR,zCMB,zCMBERR,zHD,zHDERR," +\ "HOST_LOGMASS,HOST_LOGMASS_ERR,RA,DEC,HOST_RA,HOST_DEC," +\ "HST_data,Inst/Cam,Filters" + "\n") # Loop over all objects in the catalog # and search for HST data at the SN and Host location for i in range(27, len(cat)): #tqdm(range(len(cat)), desc="Processing SN"): # Get coords sn_ra = cat['RA'][i] sn_dec = cat['DEC'][i] host_ra = cat['HOST_RA'][i] host_dec = cat['HOST_DEC'][i] # Print info #print(f"{bcolors.CYAN}") #print("SN identifier:", cat['CID'][i], " at:", sn_ra, sn_dec) #print("Host galaxy coords:", host_ra, host_dec) #print(f"{bcolors.ENDC}") # Set up query sn_coords = SkyCoord(ra=sn_ra*u.degree, dec=sn_dec*u.degree, frame='icrs') print("SN coordinates:", sn_coords) obs_table = Observations.query_criteria(coordinates=sn_coords, radius="0.5 arcsec", \ intentType='science', obs_collection=['HST']) #print(obs_table) print(obs_table.columns) print("\nRows in obs table:", len(obs_table)) print("HST filters available for this SN:") all_instr = np.unique(obs_table['instrument_name']) print(all_instr) print("--------------------------------------\n") sys.exit(0) # Download any existing wfc3 data for r in range(len(obs_table)): instr = obs_table['instrument_name'][r] if 'WFC3' in instr: data_products = Observations.get_product_list(obs_table[r]) Observations.download_products(data_products, download_dir=pantheon_datadir, productType="SCIENCE", mrp_only=True) sys.exit(0) # Now loop over all the observations ra_one = sn_ra dec_one = sn_dec dist = [] inst_cam = [] exptimes = [] filt = [] for o in range(len(obs_table)): ra_two = obs_table['s_ra'][o] dec_two = obs_table['s_dec'][o] dist_to_sn = np.arccos(np.cos(dec_one*np.pi/180) * \ np.cos(dec_two*np.pi/180) * np.cos(ra_one*np.pi/180 - ra_two*np.pi/180) + \ np.sin(dec_one*np.pi/180) * np.sin(dec_two*np.pi/180)) # print("{:.7}".format(ra_two), "{:.7}".format(dec_two)) #print("Distance from SN [arcsec]:", "{:.5}".format(dist_to_sn * (180/np.pi) * 3600), \ # "Inst/Cam:", obs_table['instrument_name'][o], "Filter(s):", obs_table['filters'][o], \ # "ExpTime:", obs_table['t_exptime'][o]) dist.append(dist_to_sn * (180/np.pi) * 3600) # The dist returned by the line above is in radians inst_cam.append(obs_table['instrument_name'][o]) filt.append(obs_table['filters'][o]) exptimes.append(obs_table['t_exptime'][o]) # Add to original catalog # Need to loop over the original row to do this for j in range(num_orig_cols): fh.write(str(cat[i][j]) + ',') # Now add the new cols # First convert to numpy arrays inst_cam = np.unique(np.asarray(inst_cam)) filt = np.unique(np.asarray(filt)) if len(obs_table) > 0: hst_data = True else: hst_data = False fh.write(str(hst_data) + ",") if len(inst_cam) > 1: for w in range(len(inst_cam)): fh.write(str(inst_cam[w]) + ";") fh.write(",") else: fh.write(str(inst_cam) + ",") if len(filt) > 1: for v in range(len(filt)): fh.write(str(filt[v]) + ";") else: fh.write(str(filt)) fh.write("\n") # Check that the distances are within FoV of the instrument specified # dist = np.asarray(dist) * (180/np.pi) * 3600 # radians to degrees to arcseconds fh.close() return None
def download(self, quality_bitmask='default', download_dir=None, cutout_size=None): """Returns a single `KeplerTargetPixelFile` or `KeplerLightCurveFile` object. If multiple files are present in `SearchResult.table`, only the first will be downloaded. Parameters ---------- quality_bitmask : str or int Bitmask (integer) which identifies the quality flag bitmask that should be used to mask out bad cadences. If a string is passed, it has the following meaning: * "none": no cadences will be ignored (`quality_bitmask=0`). * "default": cadences with severe quality issues will be ignored (`quality_bitmask=1130799`). * "hard": more conservative choice of flags to ignore (`quality_bitmask=1664431`). This is known to remove good data. * "hardest": removes all data that has been flagged (`quality_bitmask=2096639`). This mask is not recommended. See the :class:`KeplerQualityFlags` class for details on the bitmasks. download_dir : str Location where the data files will be stored. Defaults to "~/.lightkurve-cache" if `None` is passed. cutout_size : int, float or tuple Side length of cutout in pixels. Tuples should have dimensions (y, x). Default size is (5, 5) Returns ------- data : `TargetPixelFile` or `LightCurveFile` object The first entry in the products table. """ if len(self.table) == 0: warnings.warn("Cannot download from an empty search result.", LightkurveWarning) return None if len(self.table) != 1: warnings.warn( 'Warning: {} files available to download. ' 'Only the first file has been downloaded. ' 'Please use `download_all()` or specify additional ' 'criteria (e.g. quarter, campaign, or sector) ' 'to limit your search.'.format(len(self.table)), LightkurveWarning) # Make sure astroquery uses the same level of verbosity logging.getLogger('astropy').setLevel(log.getEffectiveLevel()) # download first product in table if download_dir is None: download_dir = self._default_download_dir() # if table contains TESScut search results, download cutout if 'FFI Cutout' in self.table[0]['description']: try: path = self._fetch_tesscut_path( self.table[0]['target_name'], self.table[0]['sequence_number'], download_dir, cutout_size) except: raise SearchError( 'Unable to download FFI cutout. Desired target ' 'coordinates may be too near the edge of the FFI.') else: if cutout_size is not None: warnings.warn( '`cutout_size` can only be specified for TESS ' 'Full Frame Image cutouts.', LightkurveWarning) path = Observations.download_products( self.table[:1], mrp_only=False, download_dir=download_dir)['Local Path'][0] # open() will determine filetype and return return open(path, quality_bitmask=quality_bitmask)
def retrieve_observation(obsid, suffix=['FLC'], archive=False, clobber=False): """Simple interface for retrieving an observation from the MAST archive If the input obsid is for an association, it will request all members with the specified suffixes. Parameters ----------- obsid : string ID for observation to be retrieved from the MAST archive. Only the IPPSSOOT (rootname) of exposure or ASN needs to be provided; eg., ib6v06060. suffix : list, optional List containing suffixes of files which should be requested from MAST. Default value "['FLC']". archive : Boolean, optional Retain copies of the downloaded files in the astroquery created sub-directories? Default is "False". clobber : Boolean, optional Download and Overwrite existing files? Default is "False". Returns ------- local_files : list List of filenames """ local_files = [] if Observations is None: log.warning("The astroquery package was not found. No files retrieved!") return local_files # Query MAST for the data with an observation type of either "science" or # "calibration" obs_table = Observations.query_criteria(obs_id=obsid, obstype='all') # Catch the case where no files are found for download if not obs_table: log.info("WARNING: Query for {} returned NO RESULTS!".format(obsid)) return local_files dpobs = Observations.get_product_list(obs_table) data_products_by_id = Observations.filter_products(dpobs, productSubGroupDescription=suffix, extension='fits', mrp_only=False) # After the filtering has been done, ensure there is still data in the # table for download. If the table is empty, look for FLT images in lieu # of FLC images. Only want one or the other (not both!), so just do the # filtering again. if not data_products_by_id: log.info("WARNING: No FLC files found for {} - will look for FLT " "files instead.".format(obsid)) suffix = ['FLT'] data_products_by_id = Observations.filter_products(dpobs, productSubGroupDescription=suffix, extension='fits', mrp_only=False) # If still no data, then return. An exception will eventually be # thrown in the higher level code. if not data_products_by_id: log.info( "WARNING: No FLC or FLT files found for {}.".format(obsid)) return local_files all_images = data_products_by_id['productFilename'].tolist() log.info(all_images) if not clobber: rows_to_remove = [] for row_idx, row in enumerate(data_products_by_id): fname = row['productFilename'] if os.path.isfile(fname): log.info(fname + " already exists. File download skipped.") rows_to_remove.append(row_idx) data_products_by_id.remove_rows(rows_to_remove) manifest = Observations.download_products(data_products_by_id, mrp_only=False) if not clobber: for rownum in rows_to_remove[::-1]: if manifest: manifest.insert_row(rownum, vals=[all_images[rownum], "LOCAL", "None", "None"]) else: return all_images download_dir = None for file, file_status in zip(manifest['Local Path'], manifest['Status']): if file_status != "LOCAL": # Identify what sub-directory was created by astroquery for the # download if download_dir is None: download_dir = os.path.dirname(os.path.abspath(file)) # Move or copy downloaded file to current directory local_file = os.path.abspath(os.path.basename(file)) if archive: shutil.copy(file, local_file) else: shutil.move(file, local_file) # Record what files were downloaded and their current location local_files.append(os.path.basename(local_file)) else: local_files.append(file) if not archive: # Remove astroquery created sub-directories shutil.rmtree(download_dir) return local_files
def _search_products(target, radius=None, filetype="Lightcurve", cadence='long', mission=['Kepler', 'K2', 'TESS'], quarter=None, month=None, campaign=None, sector=None, limit=None): """Helper function which returns a SearchResult object containing MAST products that match several criteria. Parameters ---------- target : str, int, or `astropy.coordinates.SkyCoord` object See docstrings above. radius : float or `astropy.units.Quantity` object Conesearch radius. If a float is given it will be assumed to be in units of arcseconds. If `None` then we default to 0.0001 arcsec. filetype : {'Target pixel', 'Lightcurve', 'FFI'} Type of files queried at MAST. cadence : str Desired cadence (`long`, `short`, `any`) mission : str, list of str 'Kepler', 'K2', or 'TESS'. By default, all will be returned. quarter, campaign, sector : int, list of ints Kepler Quarter, K2 Campaign, or TESS Sector number. By default all quarters/campaigns/sectors will be returned. month : 1, 2, 3, 4 or list of int For Kepler's prime mission, there are three short-cadence TargetPixelFiles for each quarter, each covering one month. Hence, if cadence='short' you can specify month=1, 2, 3, or 4. By default all months will be returned. limit : int Maximum number of products to return Returns ------- SearchResult : :class:`SearchResult` object. """ observations = _query_mast(target, project=mission, radius=radius) if len(observations) == 0: raise SearchError('No data found for target "{}".'.format(target)) # Light curves and target pixel files if filetype.lower() != 'ffi': products = Observations.get_product_list(observations) result = join(products, observations, keys="obs_id", join_type='left', uniq_col_name='{col_name}{table_name}', table_names=['', '_2']) result.sort(['distance', 'obs_id']) masked_result = _filter_products(result, filetype=filetype, campaign=campaign, quarter=quarter, cadence=cadence, project=mission, month=month, sector=sector, limit=limit) return SearchResult(masked_result) # Full Frame Images else: cutouts = [] for idx in np.where( ['TESS FFI' in t for t in observations['target_name']])[0]: # if target passed in is a SkyCoord object, convert to RA, dec pair if isinstance(target, SkyCoord): target = '{}, {}'.format(target.ra.deg, target.dec.deg) # pull sector numbers s = observations['sequence_number'][idx] # if the desired sector is available, add a row if s in np.atleast_1d(sector) or sector is None: cutouts.append({ 'description': 'TESS FFI Cutout (sector {})'.format(s), 'target_name': str(target), 'targetid': str(target), 'productFilename': 'n/a', 'distance': 0.0, 'sequence_number': s }) masked_result = Table(cutouts) masked_result.sort(['distance', 'sequence_number']) return SearchResult(masked_result)
def _download_one(self, table, quality_bitmask, download_dir, cutout_size, **kwargs): """Private method used by `download()` and `download_all()` to download exactly one file from the MAST archive. Always returns a `TargetPixelFile` or `LightCurve` object. """ # Make sure astroquery uses the same level of verbosity logging.getLogger('astropy').setLevel(log.getEffectiveLevel()) if download_dir is None: download_dir = self._default_download_dir() # if the SearchResult row is a TESScut entry, then download cutout if 'FFI Cutout' in table[0]['description']: try: log.debug("Started downloading TESSCut for '{}' sector {}." "".format(table[0]['target_name'], table[0]['sequence_number'])) path = self._fetch_tesscut_path(table[0]['target_name'], table[0]['sequence_number'], download_dir, cutout_size) except Exception as exc: msg = str(exc) if "504" in msg: # TESSCut will occasionally return a "504 Gateway Timeout # error" when it is overloaded. raise HTTPError('The TESS FFI cutout service at MAST appears ' 'to be temporarily unavailable. It returned ' 'the following error: {}'.format(exc)) else: raise SearchError('Unable to download FFI cutout. Desired target ' 'coordinates may be too near the edge of the FFI.' 'Error: {}'.format(exc)) return read(path, quality_bitmask=quality_bitmask, targetid=table[0]['targetid']) else: if cutout_size is not None: warnings.warn('`cutout_size` can only be specified for TESS ' 'Full Frame Image cutouts.', LightkurveWarning) # Whenever `astroquery.mast.Observations.download_products` is called, # a HTTP request will be sent to determine the length of the file # prior to checking if the file already exists in the local cache. # For performance, we skip this HTTP request and immediately try to # find the file in the cache. The path we check here is consistent # with the one hard-coded inside `astroquery.mast.Observations._download_files()` # in Astroquery v0.4.1. It would be good to submit a PR to astroquery # so we can avoid having to use this hard-coded hack. path = os.path.join(download_dir.rstrip('/'), "mastDownload", table['obs_collection'][0], table['obs_id'][0], table['productFilename'][0]) if os.path.exists(path): log.debug("File found in local cache.") else: from astroquery.mast import Observations log.debug("Started downloading {}.".format(table[:1]['dataURL'][0])) path = Observations.download_products(table[:1], mrp_only=False, download_dir=download_dir)['Local Path'][0] log.debug("Finished downloading.") return read(path, quality_bitmask=quality_bitmask, **kwargs)
distVec = [] m = len(quasar_list) for n in np.arange(0,m): try: #pull the coordinate pairs from the csv file RA_float = quasar_list.iloc[n,0] RA = str(RA_float) DEC_float = quasar_list.iloc[n,1] DEC = str(DEC_float) coords = RA + "_" + DEC #search for and download all data products associated with those coordinates print("coordinates:", coords, " n:", n) table_of_observations = Observations.query_region(coords, radius = 0.02) data_product_list = Observations.get_product_list(table_of_observations) downloads = Observations.download_products(data_product_list, download_dir=coords, obs_collection = "HST", dataproduct_type = "image", productType = "SCIENCE") #Filter out all files that are not drz.fits files (drizzle reduced)---> findQSOdrz = 'find . -type f \( -name "*drz.fits" \)' out = subprocess.Popen(findQSOdrz, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = out.communicate() filelist = stdout.decode().split() length = len(filelist) print("drz files found:", length)
def _search_products(target, radius=None, filetype="Lightcurve", cadence=None, mission=('Kepler', 'K2', 'TESS'), provenance_name=('Kepler', 'K2', 'SPOC'), t_exptime=(0, 9999), quarter=None, month=None, campaign=None, sector=None, limit=None, **extra_query_criteria): """Helper function which returns a SearchResult object containing MAST products that match several criteria. Parameters ---------- target : str, int, or `astropy.coordinates.SkyCoord` object See docstrings above. radius : float or `astropy.units.Quantity` object Conesearch radius. If a float is given it will be assumed to be in units of arcseconds. If `None` then we default to 0.0001 arcsec. filetype : {'Target pixel', 'Lightcurve', 'FFI'} Type of files queried at MAST. cadence : 'long', 'short', 'fast', or float 'long' selects 10-min and 30-min cadence products; 'short' selects 1-min and 2-min products; 'fast' selects 20-sec products. Alternatively, you can pass the exact exposure time in seconds as an int or a float, e.g. ``cadence=600`` selects 10-minute cadence. By default, all cadence modes are returned. mission : str, list of str 'Kepler', 'K2', or 'TESS'. By default, all will be returned. provenance_name : str, list of str Provenance of the data product. Defaults to official products, i.e. ('Kepler', 'K2', 'SPOC'). Community-provided products such as 'K2SFF' are supported as well. quarter, campaign, sector : int, list of ints Kepler Quarter, K2 Campaign, or TESS Sector number. By default all quarters/campaigns/sectors will be returned. month : 1, 2, 3, 4 or list of int For Kepler's prime mission, there are three short-cadence TargetPixelFiles for each quarter, each covering one month. Hence, if cadence='short' you can specify month=1, 2, 3, or 4. By default all months will be returned. limit : int Maximum number of products to return Returns ------- SearchResult : :class:`SearchResult` object. """ if isinstance(target, int): if (0 < target) and (target < 13161030): log.warning("Warning: {} may refer to a different Kepler or TESS target. " "Please add the prefix 'KIC' or 'TIC' to disambiguate." "".format(target)) elif (0 < 200000000) and (target < 251813739): log.warning("Warning: {} may refer to a different K2 or TESS target. " "Please add the prefix 'EPIC' or 'TIC' to disambiguate." "".format(target)) # Specifying quarter, campaign, or quarter should constrain the mission if quarter: mission = "Kepler" if campaign: mission = "K2" if sector: mission = "TESS" # Ensure mission is a list mission = np.atleast_1d(mission).tolist() # Avoid filtering on `provenance_name` if `author` equals "any" or "all" if provenance_name in ("any", "all") or provenance_name is None: provenance_name = None else: provenance_name = np.atleast_1d(provenance_name).tolist() # Speed up by restricting the MAST query if we don't want FFI image data extra_query_criteria = {} if filetype in ['Lightcurve', 'Target Pixel']: # At MAST, non-FFI Kepler pipeline products are known as "cube" products, # and non-FFI TESS pipeline products are listed as "timeseries". extra_query_criteria['dataproduct_type'] = ['cube', 'timeseries'] # Make sure `search_tesscut` always performs a cone search (i.e. always # passed a radius value), because strict target name search does not apply. if filetype.lower() == 'ffi' and radius is None: radius = .0001 * u.arcsec observations = _query_mast(target, radius=radius, project=mission, provenance_name=provenance_name, t_exptime=t_exptime, sequence_number=campaign or sector, **extra_query_criteria) log.debug("MAST found {} observations. " "Now querying MAST for the corresponding data products." "".format(len(observations))) if len(observations) == 0: raise SearchError('No data found for target "{}".'.format(target)) # Light curves and target pixel files if filetype.lower() != 'ffi': from astroquery.mast import Observations products = Observations.get_product_list(observations) result = join(observations, products, keys="obs_id", join_type='right', uniq_col_name='{col_name}{table_name}', table_names=['', '_products']) result.sort(['distance', 'obs_id']) # Add the user-friendly 'author' column (synonym for 'provenance_name') result['author'] = result['provenance_name'] # Add the user-friendly 'observation' column result['observation'] = None obs_prefix = {'Kepler': 'Quarter', 'K2': 'Campaign', 'TESS': 'Sector'} for idx in range(len(result)): obs_project = result['project'][idx] obs_seqno = result['sequence_number'][idx] # Kepler sequence_number values were not populated at the time of # writing this code, so we parse them from the description field. if obs_project == 'Kepler' and result['sequence_number'].mask[idx]: try: obs_seqno = re.findall(r".*Q(\d+)", result['description'][idx])[0] except IndexError: obs_seqno = "" result['observation'][idx] = "{} {} {}".format(obs_project, obs_prefix.get(obs_project, ""), obs_seqno) masked_result = _filter_products(result, filetype=filetype, campaign=campaign, quarter=quarter, cadence=cadence, project=mission, provenance_name=provenance_name, month=month, sector=sector, limit=limit) log.debug("MAST found {} matching data products.".format(len(masked_result))) masked_result['distance'].info.format = '.1f' # display <0.1 arcsec return SearchResult(masked_result) # Full Frame Images else: cutouts = [] for idx in np.where(['TESS FFI' in t for t in observations['target_name']])[0]: # if target passed in is a SkyCoord object, convert to RA, dec pair if isinstance(target, SkyCoord): target = '{}, {}'.format(target.ra.deg, target.dec.deg) # pull sector numbers s = observations['sequence_number'][idx] # if the desired sector is available, add a row if s in np.atleast_1d(sector) or sector is None: cutouts.append({'description': f'TESS FFI Cutout (sector {s})', 'observation': f'TESS Sector {s}', 'target_name': str(target), 'targetid': str(target), 't_exptime': observations['t_exptime'][idx], 'productFilename': 'TESSCut', 'provenance_name': 'MAST', 'author': 'MAST', 'distance': 0.0, 'sequence_number': s, 'project': 'TESS', 'obs_collection': 'TESS'} ) if len(cutouts) > 0: log.debug("Found {} matching cutouts.".format(len(cutouts))) masked_result = Table(cutouts) masked_result.sort(['distance', 'sequence_number']) else: masked_result = None return SearchResult(masked_result)
def lambda_handler(event, context): """Extract light curve data from one TESS full frame image. Parameters ---------- event : dict API Gateway Lambda Proxy Input Format. Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format context : object Lambda Context runtime methods and attributes. Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html Returns ------ result : dict API Gateway Lambda Proxy Output Format. Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html """ # noqa obs_id = event['id'] # TESS observation ID; Example: 'tess-s0001-1-1' # TODO: Calculate some of these from the 10th frame? # For now, also takes these and pass them onto worker: payload = { 'xpos': event['xpos'], 'ypos': event['ypos'], 'radius': event['radius'], 'bright_pixel_threshold': event['bright_pixel_threshold'] } # Find full frame dataset for the observation ID. obs_table = Observations.query_criteria(obs_id=obs_id) products = Observations.get_product_list(obs_table) filtered = Observations.filter_products(products, productSubGroupDescription="FFIC", mrp_only=False) # Use AWS S3 bucket to pull data from. Observations.enable_cloud_dataset() # TODO: verbose=False ? s3_urls = Observations.get_cloud_uris(filtered, include_bucket=False) # TODO: Timed out! Try https://docs.python.org/3/library/asyncio.html ? # TODO: Handle same Lambda call invoked multiple times by AWS? # Call tess_fullframe_worker AWS Lambda function in parallel # https://aws.amazon.com/blogs/compute/parallel-processing-in-python-with-aws-lambda/ parent_connections = [] processes = [] data = [] for url in s3_urls[:2]: # TODO: Remove [:2] when done testing payload['key'] = url parent_conn, child_conn = Pipe() parent_connections.append(parent_conn) arg = json.dumps(payload) process = Process(target=_pipe_worker, args=(arg, child_conn)) processes.append(process) for process in processes: process.start() for process in processes: process.join() for parent_connection in parent_connections: try: response = parent_connection.recv()[0] except EOFError: response = {} if 'body' not in response: # Worker Lambda threw exception continue body = json.loads(response['body']) row = (body['midtime'], body['signal'], body['background']) if np.all(list(map(np.isfinite, row))): data.append(row) # TODO: Save data as table. # filename = f'/tmp/{obs_id}_lightcurve.csv' # with open(filename) as fout: # for row in data: # fout.write(f'{row[0]},{row[1]},{row[2]}{os.linesep}') # TODO: Upload table to S3 and then delete the table locally. # TODO: Return table S3 URL below. # TODO: Do we want to plot it and upload the plot too? # If so, need to add matplotlib as dependency. return { "statusCode": 200, "body": json.dumps({ 'n_rows': len(data), 'data_url': 'TODO' }) }
def _query_mast(target, radius=None, project=('Kepler', 'K2', 'TESS'), provenance_name=("Kepler", "K2", "SPOC"), t_exptime=(0, 9999), sequence_number=None, **extra_query_criteria): """Helper function which wraps `astroquery.mast.Observations.query_criteria()` to return a table of all Kepler/K2/TESS observations of a given target. By default only the official data products are returned, but this can be adjusted by adding alternative data product names into `provenance_name`. Parameters ---------- target : str, int, or `astropy.coordinates.SkyCoord` object See docstrings above. radius : float or `astropy.units.Quantity` object Conesearch radius. If a float is given it will be assumed to be in units of arcseconds. If `None` then we default to 0.0001 arcsec. project : str, list of str Mission name. Typically 'Kepler', 'K2', or 'TESS'. This parameter is case-insensitive. provenance_name : str, list of str Provenance of the observation. Common options include 'Kepler', 'K2', 'SPOC', 'K2SFF', 'EVEREST', 'KEPSEISMIC'. This parameter is case-insensitive. t_exptime : (float, float) tuple Exposure time range in seconds. Common values include `(59, 61)` for Kepler short cadence and `(1799, 1801)` for Kepler long cadence. sequence_number : int, list of int Quarter, Campaign, or Sector number. **extra_query_criteria : kwargs Extra criteria to be passed to `astroquery.mast.Observations.query_criteria`. Returns ------- obs : astropy.Table Table detailing the available observations on MAST. """ # Local astroquery import because the package is not used elsewhere from astroquery.mast import Observations from astroquery.exceptions import ResolverError, NoResultsWarning # If passed a SkyCoord, convert it to an "ra, dec" string for MAST if isinstance(target, SkyCoord): target = '{}, {}'.format(target.ra.deg, target.dec.deg) # We pass the following `query_criteria` to MAST regardless of whether # we search by position or target name: query_criteria = { 'project': project, **extra_query_criteria } if provenance_name is not None: query_criteria['provenance_name'] = provenance_name if sequence_number is not None: query_criteria['sequence_number'] = sequence_number if t_exptime is not None: query_criteria['t_exptime'] = t_exptime # If an exact KIC ID is passed, we will search by the exact `target_name` # under which MAST will know the object to prevent source confusion. # For discussion, see e.g. GitHub issues #148, #718. exact_target_name = None target_lower = str(target).lower() # Was a Kepler target ID passed? kplr_match = re.match("^(kplr|kic) ?(\d+)$", target_lower) if kplr_match: exact_target_name = f"kplr{kplr_match.group(2).zfill(9)}" # Was a K2 target ID passed? ktwo_match = re.match("^(ktwo|epic) ?(\d+)$", target_lower) if ktwo_match: exact_target_name = f"ktwo{ktwo_match.group(2).zfill(9)}" # Was a TESS target ID passed? tess_match = re.match("^(tess|tic) ?(\d+)$", target_lower) if tess_match: exact_target_name = f"{tess_match.group(2).zfill(9)}" if exact_target_name and radius is None: log.debug("Started querying MAST for observations with the exact " f"target_name='{exact_target_name}'.") with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=NoResultsWarning) obs = Observations.query_criteria(target_name=exact_target_name, **query_criteria) if len(obs) > 0: # astroquery does not report distance when querying by `target_name`; # we add it here so that the table returned always has this column. obs['distance'] = 0. return obs else: log.debug(f"No observations found. Now performing a cone search instead.") # If the above did not return a result, then do a cone search using the MAST name resolver # `radius` defaults to 0.0001 and unit arcsecond if radius is None: radius = .0001 * u.arcsec elif not isinstance(radius, u.quantity.Quantity): radius = radius * u.arcsec query_criteria['radius'] = str(radius.to(u.deg)) try: log.debug("Started querying MAST for observations within " f"{radius.to(u.arcsec)} arcsec of objectname='{target}'.") with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=NoResultsWarning) obs = Observations.query_criteria(objectname=target, **query_criteria) obs.sort('distance') return obs except ResolverError as exc: # MAST failed to resolve the object name to sky coordinates raise SearchError(exc) from exc
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Sun Apr 19 19:31:11 2020 @author: smullally """ from astroquery.mast import Observations Observations.enable_cloud_dataset(provider='AWS') target = "Kepler-10" #Do a cone search and find the Kepler long cadence data for your target obs = Observations.query_object(target, radius="0s") want = (obs['obs_collection'] == "Kepler") & (obs['t_exptime'] == 1800.0) #Pick which data you want to retrieve data_prod = Observations.get_product_list(obs[want]) filt_prod = Observations.filter_products( data_prod, description="Lightcurve Long Cadence (CLC) - Q4") #Move data from the S3 bucket to the default astroquery location. #cloud_only=True means that data will only be retrieved if available on AWS S3 manifest = Observations.download_products(filt_prod) #%% import pdb from lightkurve import search_targetpixelfile
input( "Would you like an ascii file of the phased data?\n0 = no, 1 = yes: ")) # Is the period actually 2*P? flag_p2 = int( input("Would you like to multiply the period by two?\n" "(useful for ellipsoidal variables and some eclipsing systems)\n" "0 = no, 1 = yes: ")) ################################ ####### DOWNLOAD DATA ######## # Searching for data at MAST obsTable = Observations.query_criteria(dataproduct_type="timeseries", project="TESS", target_name=TIC) # Download the 2-minute cadence light curves data = Observations.get_product_list(obsTable) download_lc = Observations.download_products(data, productSubGroupDescription="LC") infile = download_lc[0][:] n_slow = len(infile) print("I have found a total of " + str(len(infile)) + " 2-min light curve(s).") # Download the 20-second cadence light curves download_fast_lc = Observations.download_products(