def tess_target_download(targID,
sectors='all',
sc=True,
lc_format='pdc',
delete_fits='n'):
### this function interfaces with MASS to download light curves based on the TIC #.
if os.path.exists(moonpydir + '/TESS_lcs'):
pass
else:
os.system('mkdir ' + moonpydir + '/TESS_lcs')
### first try a simple curl script!
all_times = []
all_fluxes = []
all_errors = []
all_flags = []
sectors = []
lcfiles = []
try:
if targID.startswith('TIC'):
ticnum = str(targID)[3:]
if ticnum.startswith(' '):
ticnum = str(ticnum)[1:]
else:
ticnum = str(targID) ### you've already handled the TOI already!
if ticnum.startswith(' '):
ticnum = ticnum[1:]
### prepare for the query
query_num = ticnum
while len(query_num) < 16:
query_num = '0' + query_num
assert len(query_num) == 16
sector_prefixes, sector_suffixes = {}, {}
### these can be found at archive.stsci.edu/tess/bulk_downloads/bulk_downloads_ffi-tp-lc-dv.html,
### in the tesscurl_sector_NN_lc.sh files.
nsectors = 99
nactual_sectors = 0
for sector in np.arange(1, nsectors, 1):
### get the curl script... then extract the prefixes and suffixes from the first line.
try:
if os.path.exists(moonpydir + '/sector' + str(sector) +
"_curlscript.txt"):
pass
else:
sector_curl_URL = 'http://archive.stsci.edu/missions/tess/download_scripts/sector/tesscurl_sector_' + str(
sector) + '_lc.sh'
os.system('wget --tries=1 -N "' + sector_curl_URL +
'" -O ' + moonpydir + '/sector' + str(sector) +
"_curlscript.txt")
curltxt = open(moonpydir + '/sector' + str(sector) +
'_curlscript.txt',
mode='r')
first_line = curltxt.readline()
second_line = curltxt.readline()
sector_prefix = second_line[16:40]
sector_suffix = second_line[56:71]
### now read the first line of that
sector_prefixes[sector], sector_suffixes[
sector] = sector_prefix, sector_suffix
if len(sector_prefix) > 0:
print("sector_prefix, sector_suffix = ", sector_prefix,
sector_suffix)
nactual_sectors += 1
else:
break
except:
traceback.print_exc()
break
nsectors = nactual_sectors
print('nsectors = ', nsectors)
"""
sector_prefixes[1], sector_suffixes[1] = 'tess2018206045859-s0001-', '-0120-s_lc.fits'
sector_prefixes[2], sector_suffixes[2] = 'tess2018234235059-s002-', '-0121-s_lc.fits'
sector_prefixes[3], sector_suffixes[3] = 'tess2018263035959-s0003-', '-0123-s_lc.fits'
sector_prefixes[4], sector_suffixes[4] = 'tess2018292075959-s0004-', '-0124-s_lc.fits'
sector_prefixes[5], sector_suffixes[5] = 'tess2018319095959-s0005-', '-0125-s_lc.fits'
sector_prefixes[6], sector_suffixes[6] = 'tess2018349182459-s0006-', '-0126-s_lc.fits'
sector_prefixes[7], sector_suffixes[7] = 'tess2019006130736-s0007-', '-0131-s_lc.fits'
sector_prefixes[8], sector_suffixes[8] = 'tess2019032160000-s0008-', '-0136-s_lc.fits'
sector_prefixes[9], sector_suffixes[9] = 'tess2019058134432-s0009-', '-0139-s_lc.fits'
sector_prefixes[10], sector_suffixes[10] = 'tess2019085135100-s0010-', '-0140-s_lc.fits'
sector_prefixes[11], sector_suffixes[11] = 'tess2019112060037-s0011-', '-0143-s_lc.fits'
sector_prefixes[12], sector_suffixes[12] = 'tess2019140104343-s0012-', '-0144-s_lc.fits'
sector_prefixes[13], sector_suffixes[13] = 'tess2019169103026-s0013-', '-0146-s_lc.fits'
sector_prefixes[14], sector_suffixes[14] = 'tess2019198215352-s0014-', '-0150-s_lc.fits'
sector_prefixes[15], sector_suffixes[15] = 'tess2019226182529-s0015-', '-0151-s_lc.fits'
sector
nsectors = 28
"""
for sector in np.arange(1, nsectors + 1, 1):
download_directory = central_data_dir + 'TESS_lightcurves/TIC' + str(
ticnum)
if os.path.exists(download_directory):
pass
else:
os.system('mkdir ' + download_directory)
lcdownload_name = 'TIC' + ticnum + '_sector' + str(
sector) + '-s_lc.fits'
os.system(
'curl -C - -L -o ' + download_directory + '/' +
lcdownload_name +
' https://mast.stsci.edu/api/v0.1/Download/file/?uri=mast:TESS/product/'
+ sector_prefixes[sector] + query_num +
sector_suffixes[sector])
print(
'downloading the light curve for ' + str(targID) +
' in sector ', sector)
try:
lcfile = pyfits.open(moonpydir + '/TESS_lcs/' +
lcdownload_name)
except:
os.system('rm -rf ' + moonpydir + '/TESS_lcs/' +
lcdownload_name)
continue
lcfiles.append(lcfile)
lcdata = lcfile[1].data
lctimes = np.array(lcdata['TIME'])
if lc_format == 'pdc':
lcfluxes = np.array(lcdata['PDCSAP_FLUX'])
lcerrors = np.array(lcdata['PDCSAP_FLUX_ERR'])
elif lc_format == 'sap':
lcfluxes = np.array(lcdata['SAP_FLUX'])
lcerrors = np.array(lcdata['SAP_FLUX_ERR'])
lcflags = np.array(lcdata['QUALITY'])
sector = lcfile[0].header['SECTOR']
all_times.append(lctimes)
all_fluxes.append(lcfluxes)
all_errors.append(lcerrors)
all_flags.append(lcflags)
sectors.append(sector)
if delete_fits == 'y':
os.system('rm -rf ' + moonpydir + '/TESS_lcs/' +
lcdownload_name)
except:
traceback.print_exc()
time.sleep(60)
obsTable = Observations.query_object(targID, radius='0.001 deg')
TESS_idxs = np.where(np.array(obsTable['obs_collection']) == 'TESS')[0]
minTESSidx, maxTESSidx = np.nanmin(TESS_idxs), np.nanmax(TESS_idxs) + 1
dataproducts = Observations.get_product_list(
obsTable[minTESSidx:maxTESSidx])
timeseries_idxs = np.where(
np.array(dataproducts['dataproduct_type']) == 'timeseries')[0]
obsids = np.array(dataproducts)['obsID'][timeseries_idxs]
for obsid in np.unique(obsids):
print("obsid = ", obsid)
dataproductsbyID = Observations.get_product_list(obsid)
manifest = Observations.download_products(
dataproductsbyID,
download_dir=moonpydir + '/TESS_lcs',
dataproduct_type='timeseries',
extension='lc.fits',
mrp_only=True)
for nmanfile, manfile in enumerate(manifest):
manfilepath = manfile[0]
if "_lc.fits" in manfilepath:
print('found the light curve!')
### this is the only one you want to save!
lcpath = manfilepath
print("lcpath = ", lcpath)
### open the file, grab the data!
lcfile = pyfits.open(lcpath)
lcfiles.append(lcfile)
lcdata = lcfile[1].data
lctimes = np.array(lcdata['TIME'])
if lc_format == 'pdc':
lcfluxes = np.array(lcdata['PDCSAP_FLUX'])
lcerrors = np.array(lcdata['PDCSAP_FLUX_ERR'])
elif lc_format == 'sap':
lcfluxes = np.array(lcdata['SAP_FLUX'])
lcerrors = np.array(lcdata['SAP_FLUX_ERR'])
lcflags = np.array(lcdata['QUALITY'])
sector = lcfile[0].header['SECTOR']
all_times.append(lctimes)
all_fluxes.append(lcfluxes)
all_errors.append(lcerrors)
all_flags.append(lcflags)
sectors.append(sector)
if delete_fits == 'y':
os.system('rm ' + lcpath)
break
else:
pass
#os.system('rm -rf '+manfilepath)
print(" ")
print(" ")
all_times, all_fluxes, all_errors, all_flags, sectors = np.array(
all_times), np.array(all_fluxes), np.array(all_errors), np.array(
all_flags), np.array(sectors)
return all_times, all_fluxes, all_errors, all_flags, sectors
def _query_mast(target, radius=None, project=['Kepler', 'K2', 'TESS']):
"""Helper function which wraps `astroquery.mast.Observations.query_criteria()`
to returns a table of all Kepler or K2 observations of a given target.
Parameters
----------
target : str, int, or `astropy.coordinates.SkyCoord` object
See docstrings above.
radius : float or `astropy.units.Quantity` object
Conesearch radius. If a float is given it will be assumed to be in
units of arcseconds. If `None` then we default to 0.0001 arcsec.
project : str, list of str
'Kepler', 'K2', and/or 'TESS'.
Returns
-------
obs : astropy.Table
Table detailing the available observations on MAST.
"""
# If passed a SkyCoord, convert it to an RA and Dec
if isinstance(target, SkyCoord):
target = '{}, {}'.format(target.ra.deg, target.dec.deg)
project = np.atleast_1d(project)
if radius is None:
radius = .0001 * u.arcsec
elif not isinstance(radius, u.quantity.Quantity):
radius = radius * u.arcsec
try:
# If `target` looks like a KIC or EPIC ID, we will pass the exact
# `target_name` under which MAST will know the object to prevent
# source confusion (see GitHub issue #148).
target = int(target)
if (target > 0) and (target < 200000000):
target_name = 'kplr{:09d}'.format(target)
elif (target > 200000000) and (target < 300000000):
target_name = 'ktwo{:09d}'.format(target)
else:
raise ValueError(
"{:09d}: not in the KIC or EPIC ID range".format(target))
# query_criteria does not allow a cone search when target_name is passed in
# so first grab desired target with ~0 arcsecond radius
with warnings.catch_warnings():
# suppress misleading AstropyWarning
warnings.simplefilter('ignore', AstropyWarning)
target_obs = Observations.query_criteria(target_name=target_name,
radius=str(
radius.to(u.deg)),
project=project,
obs_collection=project)
if len(target_obs) == 0:
raise ValueError(
"No observations found for {}.".format(target_name))
# check if a cone search is being performed
# if yes, perform a cone search around coordinates of desired target
if radius < (0.1 * u.arcsec):
obs = target_obs
# astroquery does not return distance if target_name is given;
# we add it here so that the table returned always has this column.
obs['distance'] = 0.
else:
ra = target_obs['s_ra'][0]
dec = target_obs['s_dec'][0]
with warnings.catch_warnings():
# suppress misleading AstropyWarning
warnings.simplefilter('ignore', AstropyWarning)
obs = Observations.query_criteria(coordinates='{} {}'.format(
ra, dec),
radius=str(radius.to(u.deg)),
project=project,
obs_collection=project)
obs.sort('distance')
return obs
except ValueError:
pass
# If `target` did not look like a KIC or EPIC ID, then we let MAST
# resolve the target name to a sky position. Convert radius from arcsec
# to degrees for query_criteria().
try:
with warnings.catch_warnings():
# suppress misleading AstropyWarning
warnings.simplefilter('ignore', AstropyWarning)
obs = Observations.query_criteria(objectname=target,
radius=str(radius.to(u.deg)),
project=project,
obs_collection=project)
obs.sort('distance')
return obs
except ResolverError as exc:
raise SearchError(exc)
def get_all_TESS_data(object_name,
radius=".02 deg",
get_PDC=True,
get_all=False,
get_lightcurves_only=True):
"""
Given a planet name, this function returns a dictionary of times, fluxes and
errors on fluxes in a juliet-friendly format for usage. The function does an
astroquery to MAST using a default radius of .02 deg around the target name. If get_PDC is True,
this function returns PDC fluxes. False returns SAP fluxes. If get_all is true, this function
returns a dictionary that in addition to the times, fluxes and errors, returns other
metadata.
"""
if not has_astroquery:
print(
"Error on using juliet function `get_all_TESS_data`: astroquery.mast not found."
)
obs_table = Observations.query_object(object_name, radius=radius)
out_dict = {}
times = {}
fluxes = {}
fluxes_errors = {}
for i in range(len(obs_table['dataURL'])):
if 's_lc.fits' in obs_table['dataURL'][i]:
fname = obs_table['dataURL'][i].split('/')[-1]
metadata = fname.split('-')
if len(metadata) == 5:
# Extract metadata:
sector = int(metadata[1].split('s')[-1])
ticid = int(metadata[2])
# Download files:
data_products = Observations.get_product_list(obs_table[i])
if get_lightcurves_only:
want = data_products['description'] == "Light curves"
else:
want = (data_products['description']
== "Light curves") or (data_products['description']
== "Target pixel files")
manifest = Observations.download_products(data_products[want])
# Read lightcurve file:
d, h = fits.getdata('mastDownload/TESS/' + fname[:-8] + '/' +
fname,
header=True)
t,fs,fserr,f,ferr = d['TIME']+h['BJDREFI'],d['SAP_FLUX'],d['SAP_FLUX_ERR'],\
d['PDCSAP_FLUX'],d['PDCSAP_FLUX_ERR']
idx_goodpdc = np.where((f != 0.) & (~np.isnan(f)))[0]
idx_goodsap = np.where((fs != 0.) & (~np.isnan(fs)))[0]
# Save to output dictionary:
if 'TIC' not in out_dict.keys():
out_dict['TIC'] = ticid
out_dict[sector] = {}
out_dict[sector]['TIME_PDCSAP_FLUX'] = t[idx_goodpdc]
out_dict[sector]['PDCSAP_FLUX'] = f[idx_goodpdc]
out_dict[sector]['PDCSAP_FLUX_ERR'] = ferr[idx_goodpdc]
out_dict[sector]['TIME_SAP_FLUX'] = t[idx_goodsap]
out_dict[sector]['SAP_FLUX'] = fs[idx_goodsap]
out_dict[sector]['SAP_FLUX_ERR'] = fserr[idx_goodsap]
if get_PDC:
times['TESS' + str(sector)] = t[idx_goodpdc]
med = np.median(f[idx_goodpdc])
fluxes['TESS' + str(sector)] = f[idx_goodpdc] / med
fluxes_errors['TESS' +
str(sector)] = ferr[idx_goodpdc] / med
else:
times['TESS' + str(sector)] = t[idx_goodsap]
med = np.median(fs[idx_goodsap])
fluxes['TESS' + str(sector)] = fs[idx_goodsap] / med
fluxes_errors['TESS' +
str(sector)] = fserr[idx_goodsap] / med
# Remove downloaded folder:
os.system('rm -r mastDownload')
if get_all:
return out_dict, times, fluxes, fluxes_errors
else:
return times, fluxes, fluxes_errors
def locate_postcard(self, local):
""" Finds the eleanor postcard, if available, this star falls on.
Attributes
----------
postcard : str
postcard_bkg : str
postcard_path : str
position_on_postcard : list
all_postcards : list
mast_list : astropy.table.Table
"""
self.mast_list = None
info_str = "{0:04d}-{1}-{2}-{3}".format(self.sector, self.camera,
self.chip, "cal")
postcard_fmt = "postcard-s{0}-{{0:04d}}-{{1:04d}}"
postcard_fmt = postcard_fmt.format(info_str)
eleanorpath = os.path.dirname(__file__)
guide_url = eleanorpath + '/postcard_centers.txt'
guide = Table.read(guide_url, format="ascii")
col, row = self.position_on_chip[0], self.position_on_chip[1]
post_args = np.where((np.abs(guide['x'].data - col) <= 100)
& (np.abs(guide['y'].data - row) <= 100))
post_cens = guide[post_args]
# Finds the mostest closest postcard
closest_x, closest_y = np.argmin(
np.abs(post_cens['x'] - col)), np.argmin(
np.abs(post_cens['y'] - row))
self.postcard = postcard_fmt.format(post_cens['x'].data[closest_x],
post_cens['y'].data[closest_y])
# Keeps track of all postcards that the star falls on
all_postcards = []
for i in range(len(post_cens)):
name = postcard_fmt.format(post_cens['x'].data[i],
post_cens['y'].data[i])
all_postcards.append(name)
self.all_postcards = np.array(all_postcards)
if local == False:
postcard_obs = Observations.query_criteria(
provenance_name="ELEANOR",
target_name=self.postcard,
obs_collection="HLSP")
if len(postcard_obs) > 0:
product_list = Observations.get_product_list(postcard_obs)
# we're just checking if the pointing file already exists. we don't need
# to be warned that it doesn't.
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=EleanorWarning)
self.pointing = check_pointing(self.sector, self.camera,
self.chip, self.pm_dir)
if self.pointing is None:
extension = ["pc.fits", "bkg.fits", "pm.txt"]
else:
extension = ["pc.fits", "bkg.fits"]
results = Observations.download_products(
product_list,
extension=extension,
download_dir=self.postcard_path)
postcard_path = results['Local Path'][0]
self.postcard_path = '/'.join(
e for e in postcard_path.split('/')[:-1])
self.postcard = results['Local Path'][1].split('/')[-1]
self.postcard_bkg = results['Local Path'][0].split('/')[-1]
self.mast_results = results
self.cutout = None # Attribute for TessCut only
# only downloaded the pointing model if the search for it above failed, so only
# update it in that case here
if self.pointing is None:
self.pm_dir = self.postcard_path
self.pointing = check_pointing(self.sector, self.camera,
self.chip, self.pm_dir)
else:
print(
"No eleanor postcard has been made for your target (yet). Using TessCut instead."
)
self.locate_with_tesscut()
else:
self.cutout = None #Attribute for TessCut only
self.postcard_bkg = 'hlsp_eleanor_tess_ffi_' + self.postcard + '_tess_v2_bkg.fits'
self.postcard = 'hlsp_eleanor_tess_ffi_' + self.postcard + '_tess_v2_pc.fits'
self.pointing = check_pointing(self.sector, self.camera, self.chip,
self.pm_dir)
def download_all(self,
quality_bitmask='default',
download_dir=None,
cutout_size=None):
"""Returns a `TargetPixelFileCollection or `LightCurveFileCollection`.
Parameters
----------
quality_bitmask : str or int
Bitmask (integer) which identifies the quality flag bitmask that should
be used to mask out bad cadences. If a string is passed, it has the
following meaning:
* "none": no cadences will be ignored (`quality_bitmask=0`).
* "default": cadences with severe quality issues will be ignored
(`quality_bitmask=1130799`).
* "hard": more conservative choice of flags to ignore
(`quality_bitmask=1664431`). This is known to remove good data.
* "hardest": removes all data that has been flagged
(`quality_bitmask=2096639`). This mask is not recommended.
See the :class:`KeplerQualityFlags` class for details on the bitmasks.
download_dir : str
Location where the data files will be stored.
Defaults to "~/.lightkurve-cache" if `None` is passed.
cutout_size : int, float or tuple
Side length of cutout in pixels. Tuples should have dimensions (y, x).
Default size is (5, 5)
Returns
-------
collection : `lightkurve.Collection` object
Returns a `LightCurveFileCollection` or `TargetPixelFileCollection`,
containing all entries in the products table
"""
if len(self.table) == 0:
warnings.warn("Cannot download from an empty search result.",
LightkurveWarning)
return None
# Make sure astroquery uses the same level of verbosity
logging.getLogger('astropy').setLevel(log.getEffectiveLevel())
# download all products listed in self.products
if download_dir is None:
download_dir = self._default_download_dir()
# if table contains TESScut search results, download cutouts
if 'FFI Cutout' in self.table[0]['description']:
path = [
self._fetch_tesscut_path(t, s, download_dir, cutout_size)
for t, s in zip(self.table['target_name'],
self.table['sequence_number'])
]
else:
if cutout_size is not None:
warnings.warn(
'`cutout_size` can only be specified for TESS '
'Full Frame Image cutouts.', LightkurveWarning)
path = Observations.download_products(
self.table, mrp_only=False,
download_dir=download_dir)['Local Path']
# open() will determine filetype and return
# return a collection containing opened files
tpf_extensions = ['lpd-targ.fits', 'spd-targ.fits', '_tp.fits', 'n/a']
lcf_extensions = ['llc.fits', 'slc.fits', '_lc.fits']
if any(e in self.table['productFilename'][0] for e in tpf_extensions):
return TargetPixelFileCollection([open(p) for p in path])
elif any(e in self.table['productFilename'][0]
for e in lcf_extensions):
return LightCurveFileCollection([open(p) for p in path])
import matplotlib.pyplot as plt
from astroquery.mast import Observations
from scipy import signal
from Time_and_flux import gettimeflux_1800
ids = ['167602025', '167695269', '167695269', '167814740', '167814740']
cadence = '1800'
flare_times = [2458335.233, 2458336.636, 2458343.026, 2458344.888, 2458344.875]
for this_id in ids, bjd in flare_times:
#for bjd in flare_times:
tbjd = bjd - 2457000
obsTable = Observations.query_criteria(target_name=this_id,
obs_collection="HLSP",
filters="TESS",
t_exptime=[1799, 1801])
sector = obsTable['sequence_number']
tess_bjds = gettimeflux_1800(this_id, str(sector[0]))[0]
sap_fluxes = gettimeflux_1800(this_id, str(sector[0]))[1]
if (cadence == '1800'):
med_flux_1 = signal.medfilt(sap_fluxes, kernel_size=65)
else:
med_flux_1 = signal.medfilt(sap_fluxes, kernel_size=101)
t0 = tbjd #transit time
fig, ax = plt.subplots()
#ax.plot(tess_bjds, pdcsap_fluxes, 'ko')#BKG black
#ax.plot(tess_bjds, med_flux_1, 'b-')
def mvm_retrieve_files(products,
archive=False,
clobber=False,
log_level=logutil.logging.INFO):
"""
This function retrieves specified files from the archive - unless the file is found
to be locally resident on disk. Upon completion, The function returns a list of
filenames available on disk.
Parameters
----------
products : `~astropy.table.Table` object
A Table of products as returned by the mvm_id_filenames function.
archive : Boolean, optional
Retain copies of the downloaded files in the astroquery created
sub-directories? Default is "False".
clobber : Boolean, optional
Download and Overwrite existing files? Default is "False".
log_level : int, optional
The desired level of verbosity in the log statements displayed on the screen and written to the
.log file. Default value is 20, or 'info'.
Returns
-------
local_files : list
List of filenames
Note: Code here cribbed from retrieve_obsevation in astroquery_utils module.
"""
# set logging level to user-specified level
log.setLevel(log_level)
# Determine if the files of interest are already on the local disk. If so,
# remove the filename from the download list.
all_images = []
all_images = products['productFilename'].tolist()
if not clobber:
rows_to_remove = []
for row_idx, row in enumerate(products):
fname = row['productFilename']
if os.path.isfile(fname):
log.info(fname + " already exists. File download skipped.")
rows_to_remove.append(row_idx)
products.remove_rows(rows_to_remove)
# Only download files as necessary
if products:
# Actual download of products
log.info("Downloading files now...")
manifest = Observations.download_products(products, mrp_only=False)
else:
log.info(
"There are no files to download as they are all resident on disk.")
# Manifest has the following columns: "Local Path", "Status", "Message", and "URL"
if not clobber:
for rownum in rows_to_remove[::-1]:
if manifest:
manifest.insert_row(
rownum, vals=[all_images[rownum], "LOCAL", "None", "None"])
else:
return all_images
download_dir = None
local_files = []
for file, file_status in zip(manifest['Local Path'], manifest['Status']):
if file_status != "LOCAL":
# Identify what sub-directory was created by astroquery for the
# download
if download_dir is None:
download_dir = os.path.dirname(os.path.abspath(file))
# Move or copy downloaded file to current directory
local_file = os.path.abspath(os.path.basename(file))
if archive:
shutil.copy(file, local_file)
else:
shutil.move(file, local_file)
# Record what files were downloaded and their current location
local_files.append(os.path.basename(local_file))
else:
local_files.append(file)
if not archive:
# Remove astroquery created sub-directories
shutil.rmtree('mastDownload')
return local_files
def tasoc_lc(self):
"""
Grabs the T'DA available light curves for your target.
For more information, see the TASOC light curve documentation: https://tasoc.dk/code/.
Parameters
----------
Attributes
----------
tasoc_header :
tasoc_tpf : np.2darray
tasoc_aperture : np.2darray
tasoc_time : np.array
tasoc_quality : np.array
tasoc_timecorr : np.array
tasoc_cadenceno : np.array
tasoc_flux_raw : np.array
tasoc_flux_raw_err : np.array
tasoc_flux_corr : np.array
tasoc_flux_corr_err : np.array
tasoc_flux_bkg : np.array
tasoc_pixel_quality : np.array
Quality flags for the data; use these not `tasoc_quality`.
tasoc_pos_corr1 : np.array
tasoc_pos_corr2 : np.array
tasoc_mom_centr1 : np.array
tasoc_mom_centr2 : np.array
"""
products = Observations.query_object(objectname="TIC" + str(self.tic))
column = np.where((products['provenance_name'] == 'TASOC')
& (products['target_name'] == str(self.tic))
& (products['sequence_number'] == self.sector))[0]
if len(column) > 0:
download = Observations.get_product_list(products[column])
manifest = Observations.download_products(
download, download_dir=self.download_dir)
self.tasoc_path = manifest["Local Path"].data[0]
hdu = fits.open(self.tasoc_path)
self.tasoc_header = hdu[0].header
self.tasoc_tpf = hdu[2].data
self.tasoc_aperture = hdu[3].data
self.tasoc_time = hdu[1].data['TIME']
self.tasoc_quality = hdu[1].data['QUALITY']
self.tasoc_timecorr = hdu[1].data['TIMECORR']
self.tasoc_cadenceno = hdu[1].data['CADENCENO']
self.tasoc_flux_raw = hdu[1].data['FLUX_RAW']
self.tasoc_flux_bkg = hdu[1].data['FLUX_BKG']
self.tasoc_flux_corr = hdu[1].data['FLUX_CORR']
self.tasoc_pos_corr1 = hdu[1].data['POS_CORR1']
self.tasoc_pos_corr2 = hdu[1].data['POS_CORR2']
self.tasoc_mom_centr1 = hdu[1].data['MOM_CENTR1']
self.tasoc_mom_centr2 = hdu[1].data['MOM_CENTR2']
self.tasoc_pixel_quality = hdu[1].data['PIXEL_QUALITY']
self.tasoc_flux_raw_err = hdu[1].data['FLUX_RAW_ERR']
self.tasoc_flux_corr_err = hdu[1].data['FLUX_CORR_ERR']
else:
raise SearchError("No TASOC light curve found.")
#!/usr/bin/env python3
from astroquery.mast import Observations
import IPython
import boto3
from s3_query import find_product_in_s3
Observations.enable_s3_hst_dataset()
obs = Observations.query_criteria(
dataproduct_type=['image'],
project='HST',
instrument_name='ACS/WFC',
filters='F555W',
calib_level=3,
)
print("Observations: ", len(obs))
products = Observations.get_product_list(obs)
print("Products: ", len(products))
filtered_products = Observations.filter_products(
products,
productType='SCIENCE',
extension='fits',
description='DADS FLT file - Calibrated exposure ACS/WFC3/STIS/COS',
)
print("Filtered products: ", len(filtered_products))
print()
def get_mast_file_list(star_id, provenance, sequence_name):
"""
Queries MAST to retrieve archive data based on object name (``star_id``),
data provenance, and sequence name.
Parameters
----------
star_id: int, str
EPIC ID of the star.
provenance: str
Provenance name in MAST archive, e.g., ``'K2'``, ``'EVEREST'``,
``'K2SFF'``.
sequence_name: str
Campaign number.
Returns
-------
file_names: list of str
A Python list of all the file names retrieved from archive.
"""
star_id = str(star_id)
target_name = '*' + star_id.split()[-1]
# Make sure that there are data for the criteria
mission = _MISSION_MAPPING[provenance]
print("\n===== Retrieving data for observation: =====")
print("** star_id: {}\n** provenance: {}\n** sequence_name: {}".format(
star_id, provenance, sequence_name))
if sequence_name == '*':
sequence_name = ''
obs_count = Observations.query_criteria_count(
obs_collection=mission,
dataproduct_type=["timeseries"],
instrument_name="Kepler",
objectname=star_id,
target_name=target_name,
project="K2",
provenance_name=provenance,
sequence_number=sequence_name + '*')
if obs_count == 0:
raise RuntimeError("No data found in archive.")
obs_table = Observations.query_criteria(obs_collection=mission,
dataproduct_type=["timeseries"],
instrument_name="Kepler",
objectname=star_id,
target_name=target_name,
project="K2",
provenance_name=provenance,
sequence_number=sequence_name +
'*')
data_products = Observations.get_product_list(obs_table)
lc_mask = [
"lightcurve" in x or "light curve" in x
for x in map(str.lower, data_products['description'])
]
if not any(lc_mask):
raise RuntimeError("Retrieved data products do not contain light "
"curve data.")
data_products = data_products[lc_mask] # keep only rows with light curves
manifest = Observations.download_products(data_products)
files = list(manifest['Local Path']) # get local file names
# sort results:
idx = np.argsort(files)
file_names = [files[i] for i in idx]
print("Download Status: SUCCESS\n")
return file_names
def retrieve_observation(obsid, suffix=['FLC'], archive=False,clobber=False):
"""Simple interface for retrieving an observation from the MAST archive
If the input obsid is for an association, it will request all members with
the specified suffixes.
Parameters
-----------
obsid : string
ID for observation to be retrieved from the MAST archive. Only the
IPPSSOOT (rootname) of exposure or ASN needs to be provided; eg., ib6v06060.
suffix : list
List containing suffixes of files which should be requested from MAST.
path : string
Directory to use for writing out downloaded files. If `None` (default),
the current working directory will be used.
archive : Boolean
Retain copies of the downloaded files in the astroquery created sub-directories? Default is 'False'.
clobber : Boolean
Download and Overwrite existing files? Default is 'False'.
Returns
-------
local_files : list
List of filenames
"""
local_files = []
# Query MAST for the data with an observation type of either "science" or "calibration"
obsTable = Observations.query_criteria(obs_id=obsid, obstype='all')
# Catch the case where no files are found for download
if len(obsTable) == 0:
log.info("WARNING: Query for {} returned NO RESULTS!".format(obsid))
return local_files
dpobs = Observations.get_product_list(obsTable)
dataProductsByID = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# After the filtering has been done, ensure there is still data in the table for download.
# If the table is empty, look for FLT images in lieu of FLC images. Only want one
# or the other (not both!), so just do the filtering again.
if len(dataProductsByID) == 0:
log.info("WARNING: No FLC files found for {} - will look for FLT files instead.".format(obsid))
suffix = ['FLT']
dataProductsByID = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# If still no data, then return. An exception will eventually be thrown in
# the higher level code.
if len(dataProductsByID) == 0:
log.info("WARNING: No FLC or FLT files found for {}.".format(obsid))
return local_files
allImages = []
for tableLine in dataProductsByID:
allImages.append(tableLine['productFilename'])
log.info(allImages)
if not clobber:
rowsToRemove = []
for rowCtr in range(0,len(dataProductsByID)):
if os.path.exists(dataProductsByID[rowCtr]['productFilename']):
log.info("{} already exists. File download skipped.".format(dataProductsByID[rowCtr]['productFilename']))
rowsToRemove.append(rowCtr)
if rowsToRemove:
rowsToRemove.reverse()
for rowNum in rowsToRemove:
dataProductsByID.remove_row(rowNum)
manifest = Observations.download_products(dataProductsByID, mrp_only=False)
if not clobber:
rowsToRemove.reverse()
for rownum in rowsToRemove:
if not manifest:
local_files = allImages
return local_files
else:
manifest.insert_row(rownum,vals=[allImages[rownum],"LOCAL","None","None"])
download_dir = None
for file,fileStatus in zip(manifest['Local Path'],manifest['Status']):
if fileStatus != "LOCAL":
# Identify what sub-directory was created by astroquery for the download
if download_dir is None:
file_path = file.split(os.sep)
file_path.remove('.')
download_dir = file_path[0]
# Move or copy downloaded file to current directory
local_file = os.path.abspath(os.path.basename(file))
if archive:
shutil.copy(file, local_file)
else:
shutil.move(file, local_file)
# Record what files were downloaded and their current location
local_files.append(os.path.basename(local_file))
else:
local_files.append(file)
if not archive:
# Remove astroquery created sub-directories
shutil.rmtree(download_dir)
return local_files
def _search_products(target,
radius=None,
filetype="Lightcurve",
cadence='long',
mission=('Kepler', 'K2', 'TESS'),
quarter=None,
month=None,
campaign=None,
sector=None,
limit=None,
**extra_query_criteria):
"""Helper function which returns a SearchResult object containing MAST
products that match several criteria.
Parameters
----------
target : str, int, or `astropy.coordinates.SkyCoord` object
See docstrings above.
radius : float or `astropy.units.Quantity` object
Conesearch radius. If a float is given it will be assumed to be in
units of arcseconds. If `None` then we default to 0.0001 arcsec.
filetype : {'Target pixel', 'Lightcurve', 'FFI'}
Type of files queried at MAST.
cadence : str
Desired cadence (`long`, `short`, `any`)
mission : str, list of str
'Kepler', 'K2', or 'TESS'. By default, all will be returned.
quarter, campaign, sector : int, list of ints
Kepler Quarter, K2 Campaign, or TESS Sector number.
By default all quarters/campaigns/sectors will be returned.
month : 1, 2, 3, 4 or list of int
For Kepler's prime mission, there are three short-cadence
TargetPixelFiles for each quarter, each covering one month.
Hence, if cadence='short' you can specify month=1, 2, 3, or 4.
By default all months will be returned.
limit : int
Maximum number of products to return
Returns
-------
SearchResult : :class:`SearchResult` object.
"""
if isinstance(target, int):
if (0 < target) and (target < 13161030):
log.warning(
"Warning: {} may refer to a different Kepler or TESS target. "
"Please add the prefix 'KIC' or 'TIC' to disambiguate."
"".format(target))
elif (0 < 200000000) and (target < 251813739):
log.warning(
"Warning: {} may refer to a different K2 or TESS target. "
"Please add the prefix 'EPIC' or 'TIC' to disambiguate."
"".format(target))
# Speed up by restricting the MAST query if we don't want FFI image data
extra_query_criteria = {}
if filetype in ['Lightcurve', 'Target Pixel']:
# At MAST, non-FFI Kepler pipeline products are known as "cube" products,
# and non-FFI TESS pipeline products are listed as "timeseries".
extra_query_criteria['dataproduct_type'] = ['cube', 'timeseries']
observations = _query_mast(target,
project=mission,
radius=radius,
**extra_query_criteria)
log.debug("MAST found {} observations. "
"Now querying MAST for the corresponding data products."
"".format(len(observations)))
if len(observations) == 0:
raise SearchError('No data found for target "{}".'.format(target))
# Light curves and target pixel files
if filetype.lower() != 'ffi':
from astroquery.mast import Observations
products = Observations.get_product_list(observations)
result = join(products,
observations,
keys="obs_id",
join_type='left',
uniq_col_name='{col_name}{table_name}',
table_names=['', '_2'])
result.sort(['distance', 'obs_id'])
masked_result = _filter_products(result,
filetype=filetype,
campaign=campaign,
quarter=quarter,
cadence=cadence,
project=mission,
month=month,
sector=sector,
limit=limit)
log.debug("MAST found {} matching data products.".format(
len(masked_result)))
return SearchResult(masked_result)
# Full Frame Images
else:
cutouts = []
for idx in np.where(
['TESS FFI' in t for t in observations['target_name']])[0]:
# if target passed in is a SkyCoord object, convert to RA, dec pair
if isinstance(target, SkyCoord):
target = '{}, {}'.format(target.ra.deg, target.dec.deg)
# pull sector numbers
s = observations['sequence_number'][idx]
# if the desired sector is available, add a row
if s in np.atleast_1d(sector) or sector is None:
cutouts.append({
'description':
'TESS FFI Cutout (sector {})'.format(s),
'target_name':
str(target),
'targetid':
str(target),
'productFilename':
'TESSCut',
'distance':
0.0,
'sequence_number':
s,
'obs_collection':
'TESS'
})
if len(cutouts) > 0:
log.debug("Found {} matching cutouts.".format(len(cutouts)))
masked_result = Table(cutouts)
masked_result.sort(['distance', 'sequence_number'])
else:
masked_result = None
return SearchResult(masked_result)
if simbadResults is not None: break
if simbadResults is not None:
coord = SkyCoord(simbadResults["RA"][0],
simbadResults["DEC"][0],
unit=[u.hourangle, u.deg])
ras.append(coord.ra.value)
raUnit = coord.ra.unit
decs.append(coord.dec.value)
decUnit = coord.dec.unit
else:
ras.append(StarListTics["ra"][where_closest])
decs.append(StarListTics["dec"][where_closest])
#Retrieve data
obsTable = Observations.query_criteria(filters=["TESS"],
objectname=targetName,
dataproduct_type=["TIMESERIES"],
radius="0.01 deg")
proceed = True
if len(obsTable) == 0:
proceed = False
obsIDList.append(np.nan)
pathList.append("NA")
sectorsTxt, sectorsNb = TESSSectorLookUp(targetName)
if len(sectorsNb) == 0:
sectorList.append("")
elif len(sectorsNb) == 1:
sectorList.append(sectorsNb[0])
else:
sectorList.append(",".join(["{}".format(s) for s in sectorsNb]))
#!/usr/bin/env python3
#
#
from astroquery.mast import Observations
import boto3
import json
import os
import IPython
wfc = Observations.query_criteria(
dataproduct_type=['image'],
project='HST',
instrument_name='ACS/WFC',
)
print("WFC: ", len(wfc))
hrc = Observations.query_criteria(
dataproduct_type=['image'],
project='HST',
instrument_name='ACS/HRC',
)
print("HRC: ", len(hrc))
IPython.embed()
def get_tess_data(u_ticid, max_flag=16, out_dir='./download'):
"""Given a TIC-ID, return time,flux,ferr,itime
u_ticid : (int) TIC ID
returns lc_time,flux,ferr,int_time
"""
tic_str = 'TIC' + str(u_ticid)
#out_dir='/data/rowe/TESS/download/'
# Search MAST for TIC ID
obs_table = Observations.query_object(tic_str, radius=".002 deg")
# Identify TESS timeseries data sets (i.e. ignore FFIs)
oti=(obs_table["obs_collection"] == "TESS") & \
(obs_table["dataproduct_type"] == "timeseries")
if oti.any() == True:
data_products = Observations.get_product_list(obs_table[oti])
dpi = [
j for j, s in enumerate(data_products["productFilename"])
if "lc.fits" in s
]
manifest = Observations.download_products(data_products[dpi],
download_dir=out_dir)
else:
manifest = []
lc_time = []
flux = []
ferr = []
int_time = []
for j in range(0, len(manifest)):
fits_fname = str(manifest["Local Path"][j])
print(fits_fname)
hdu = fits.open(fits_fname)
tmp_bjd = hdu[1].data['TIME']
tmp_flux = hdu[1].data['PDCSAP_FLUX']
tmp_ferr = hdu[1].data['PDCSAP_FLUX_ERR']
tmp_int_time = hdu[1].header['INT_TIME'] + np.zeros(len(tmp_bjd))
tmp_flag = hdu[1].data['QUALITY']
ii = (tmp_flag <= max_flag) & (~np.isnan(tmp_flux))
tmp_bjd = tmp_bjd[ii]
tmp_flux = tmp_flux[ii]
tmp_ferr = tmp_ferr[ii]
tmp_int_time = tmp_int_time[ii]
# Shift flux measurements
median_flux = np.median(tmp_flux)
tmp_flux = tmp_flux / median_flux
tmp_ferr = tmp_ferr / median_flux
# Append to output columns
lc_time = np.append(lc_time, tmp_bjd)
flux = np.append(flux, tmp_flux)
ferr = np.append(ferr, tmp_ferr)
int_time = np.append(int_time, tmp_int_time)
hdu.close()
# Sort by time
si = np.argsort(lc_time)
lc_time = np.asarray(lc_time)[si]
flux = np.asarray(flux)[si]
ferr = np.asarray(ferr)[si]
int_time = np.asarray(int_time)[si]
phot = phot_class()
phot.time = np.copy(lc_time)
phot.flux = np.copy(flux)
phot.ferr = np.copy(ferr)
phot.itime = np.copy(int_time)
phot.itime = phot.itime / (60 * 24) #convert minutes to days
return phot
from astroquery.mast import Observations
import numpy as np
import os,sys
search_radius = 120
## import files ##
inp0 = 'ALMA_z6qso_all.list'
data = np.loadtxt(inp0,comments='#',dtype={'names':('proj_id','name','ra','dec','PI'),'formats':('S32','S32','f16','f16','S32')})
proj_id = data["proj_id"]
name = data["name"]
ra = data["ra"]
dec = data["dec"]
for i in range(len(name)):
obs_table = Observations.query_criteria(coordinates="%3.5f %3.5f" %(ra[i],dec[i]), radius="%3.5f arcsec" % search_radius, intentType=["science","SCIENCE"], obs_collection=["HST"])
__,uidx = np.unique(obs_table['target_name'],return_index=True)
target_table = obs_table[uidx]['target_name','s_ra','s_dec','filters','t_exptime','proposal_id','dataURL','obsid']
if len(uidx) > 0:
os.system('mkdir download_sh/'+name[i])
for u in range(len(uidx)):
data_products = Observations.get_product_list(target_table[u][-1])
Observations.download_products(data_products, calib_level=[2,3], productType="SCIENCE", curl_flag=True,mrp_only=True,download_dir='download_sh/'+name[i])
def download_lc(target_name: str, flux_type='PDCSAP', mission: str = 'TESS',sigma_clip=4,iters=1,do_pca : bool = False,do_psf :bool= False) -> Tuple[
LightCurve, Union[List[Figure],None]]:
"""
Downloads a light curve using the TESS mission. If the star has been observed in the SC mode, it
will download the original light curve from MAST. You can also choose the flux type you want to use.
If it wasn't observed in SC mode, it will try to extract a light curve from the FFIs if the target has
been observed by TESS.
You can also download light curves of stars that are observed by the K2 or Kepler mission, by setting
the mission parameter.
:param target_name: Name of the target. You can either provide the TIC ID (TIC ...), Kepler ID (KIC ...), K2 ID(EPIC ...) or a name that is resolvable by Simbad.
:param flux_type: Type of flux in the SC mode. Can be either PDCSAP or SAP or PSF for long cadence data
:param mission: Mission from which the light curves are extracted. By default TESS only is used. You can consider all missions by passing 'all' (TESS, Kepler, K2)
:param sigma_clip: Sigma clip parameter. Defines the number of standard deviations that are clipped.
:param iters: Iterations for the sigma clipping
:return: lightkurve.LightCurve object and validation page if extracted from FFI
"""
chosen_mission = [mission] if mission != 'all' else ('Kepler', 'K2', 'TESS')
mprint(f"Searching processed light curves for {target_name} on mission(s) {','.join(chosen_mission)} ... ", log)
if chosen_mission == ['TESS']:
if target_name.startswith('TIC'):
tic_id = re.findall(r'\d+', target_name)
if len(tic_id) == 0:
raise ValueError(ctext("A Tic ID needs to consist of TIC and a number!", error))
tic_id = int(tic_id[0])
else:
mprint(f"Resolving {target_name} to TIC using MAST ...",log)
try:
tic_id = Catalogs.query_object(target_name,catalog='TIC',radius=0.003)[0]['ID']
except KeyError:
raise ValueError(ctext(f"No TESS observations available for {target_name}", error))
mprint(f"TIC ID for {target_name}: TIC {tic_id}",log)
o = Observations.query_criteria(objectname=target_name, radius=str(0 * u.deg), project='TESS',
obs_collection='TESS').to_pandas()
if len(o) > 0 and len(o[o.target_name != 'TESS FFI']) > 0:
mprint(f"Short cadence observations available for {target_name}. Downloading ...",info)
res = search_lightcurvefile(target_name, mission=chosen_mission)
else: #Only FFI available
mprint(f"No short cadence data available for {target_name}, extracting from FFI ...",info)
lc, fig,_ = cut_ffi(tic_id,sigma_clip,iters,do_pca,do_psf,flux_type)
mprint(f"Total observation length: {'%.2f' % (lc.time[-1] - lc.time[0])} days.", log)
return lc, fig
else:
res = search_lightcurvefile(target_name, mission=chosen_mission)
if len(res) != 0:
fig = None
mprint(f"Found processed light curve for {target_name}!", info)
res = res.download_all()
types = []
for d in res.data:
type = 'TESS' if isinstance(d, lk.TessLightCurveFile) else 'Kepler'
if type not in types:
types.append(type)
mprint(f"Using {','.join(types)} observations! Combining sectors ...", log)
if flux_type == 'PSF':
mprint(f"PSF not available for short cadence data. Reverting to PDCSAP",warn)
flux_type = 'PDCSAP'
if flux_type == 'PDCSAP':
lc_set: List[Union[lk.TessLightCurve, lk.KeplerLightCurve]] = [i for i in res.PDCSAP_FLUX.data]
elif flux_type == 'SAP':
lc_set: List[Union[lk.TessLightCurve, lk.KeplerLightCurve]] = [i for i in res.SAP_FLUX.data]
else:
raise ValueError(ctext("Flux type needs to be either PDCSAP or SAP", error))
lc = combine_light_curves(lc_set,sigma_clip=sigma_clip,iters=iters)
else:
raise ValueError(ctext(f"No light curve available for {target_name} on mission(s) {chosen_mission}",error))
mprint(f"Total observation length: {'%.2f' % (lc.time[-1] - lc.time[0])} days.", log)
return lc, fig
def get_hlsp_lightcurves(tic_id,
hlsp_products=('CDIPS', 'TASOC', 'PATHOS'),
download_dir=None,
verbose=True):
"""This downloads TESS HLSP light curves for a given TIC ID.
Parameters
----------
tic_id : str
The TIC ID of the object as a string.
hlsp_products : sequence of str
List of desired HLSP products to search. For instance, ["CDIPS"].
download_dir : str
Path of directory to which light-curve will be downloaded.
Returns
-------
lcfiles : list or None
List of light-curve file paths. None if none are found and downloaded.
"""
if not astroquery_dependency:
LOGERROR(
"The astroquery package is required for this function to work.")
return None
lcfiles = []
for hlsp in hlsp_products:
obs_table = Observations.query_criteria(target_name=tic_id,
provenance_name=hlsp)
if verbose:
LOGINFO(f'Found {len(obs_table)} {hlsp} light-curves.')
if len(obs_table) == 0:
if verbose:
LOGINFO("Did not find light-curves. Escaping.")
return None
# Get list of available products for this Observation.
cdips_products = Observations.get_product_list(obs_table)
# Download the products for this Observation.
manifest = Observations.download_products(cdips_products,
download_dir=download_dir)
if verbose:
LOGINFO("Done")
if len(manifest) >= 1:
lcfiles.append(list(manifest['Local Path']))
#
# flatten lcfiles list
#
if len(lcfiles) >= 1:
return_lcfiles = [item for sublist in lcfiles for item in sublist]
else:
return_lcfiles = None
return return_lcfiles
redshift_target = quasar_list.iloc[n][3] #redshift of the target.
r_mag_target = quasar_list.iloc[n][2] #magnitude of target
#calculate the absolute magnitude of the target from the red magnitude
dL = Distance(unit = u.Mpc, z=redshift_target, cosmology = cosmo) / u.Mpc
magnitude_target = r_mag_target - 5*np.log(dL) - 25
#if you have the absolute magnitude of your target, remove the above calculation and use:
#magnitude_target = quasar_list.iloc[n][2]
#instead
#print(r_magnitude_target, magnitude_target, redshift_target)
#search for and download all data products associated with those coordinates
obsTable = Observations.query_region(coords, radius = 0.02)
dataProductsByObservation = Observations.get_product_list(obsTable)
manifest = Observations.download_products(dataProductsByObservation, download_dir=coords,
obs_collection = "HST",
dataproduct_type = "image",
productType = "SCIENCE")
except TimeoutError:
n = n + 1
print("a timeout error occurred at", coords)
#isolate drizzle files
findQSOdrz = 'find . -type f \( -name "*drz.fits" \)'
out = subprocess.Popen(findQSOdrz, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
"""
Example 10
++++++++++
Retrieve Hubble archival data of M83 and make a figure
"""
from astroquery.mast import Mast, Observations
from astropy.visualization import make_lupton_rgb, ImageNormalize
import matplotlib.pyplot as plt
import reproject
result = Observations.query_object('M83')
selected_bands = result[(result['obs_collection'] == 'HST') &
(result['instrument_name'] == 'WFC3/UVIS') &
((result['filters'] == 'F657N') |
(result['filters'] == 'F487N') |
(result['filters'] == 'F336W')) &
(result['target_name'] == 'MESSIER-083')]
prodlist = Observations.get_product_list(selected_bands)
filtered_prodlist = Observations.filter_products(prodlist)
downloaded = Observations.download_products(filtered_prodlist)
blue = fits.open(downloaded['Local Path'][2])
red = fits.open(downloaded['Local Path'][5])
green = fits.open(downloaded['Local Path'][8])
target_header = red['SCI'].header
green_repr, _ = reproject.reproject_interp(green['SCI'], target_header)
blue_repr, _ = reproject.reproject_interp(blue['SCI'], target_header)
def mvm_id_filenames(sky_coord, cutout_size, log_level=logutil.logging.INFO):
"""
This function retrieves a table of MVM drizzled image filenames with additional
information from the archive. The user can then further cull the table to use as
input to obtain a list of files from the archive. This function will return filter-level
products. At this time, both ACS and WFC3 are searched by default.
Parameters
----------
sky_coord : str or `~astropy.coordinates.SkyCoord` object
The position around which to cutout. It may be specified as a string ("ra dec" in degrees)
or as the appropriate `~astropy.coordinates.SkyCoord` object.
cutout_size : int, array-like, `~astropy.units.Quantity`
The size of the cutout array. If ``cutout_size`` is a scalar number or a scalar
`~astropy.units.Quantity`, then a square cutout of ``cutout_size`` will be created.
If ``cutout_size`` has two elements, they should be in ``(ny, nx)`` order. Scalar numbers
in ``cutout_size`` are assumed to be in units of arcseconds. `~astropy.units.Quantity` objects
must be in angular units.
log_level : int, optional
The desired level of verbosity in the log statements displayed on the screen and written to the
.log file. Default value is 20, or 'info'.
Returns
-------
final_table : `~astropy.table.Table` object
This utility also writes an output ECSV file version of the in-memory filtered data product table,
final_table. The output filename is in the form:
mvm_query-ra<###>d<####>-dec<n|s><##>d<####>_<radius>_cutout.ecsv
(e.g., mvm_query-ra84d8208-decs69d8516_354_cutout.ecsv, where radius has been computed from the
cutout dimensions.
"""
# set logging level to user-specified level
log.setLevel(log_level)
# If the cutout_size is not an astropy.units.Quantity object, the scalar(s)
# are assumed to be arcseconds. The variable must be cast as a Quantity.
if not isinstance(cutout_size, Quantity):
cutout_size *= u.arcsec
cutout_size = np.atleast_1d(cutout_size)
if len(cutout_size) == 1:
cutout_size = np.repeat(cutout_size, 2)
if not isinstance(sky_coord, SkyCoord):
sky_coord = SkyCoord(sky_coord, unit="deg")
# From HST data, Search for the list of images based upon: coordinates, search region, data
# product type, and the instrument name (with wildcard), project (HAP), and observation
# collection (HST). Use the wildcard to get all the detectors for the instrument. Multiple
# instruments cannot be searched at the same time. Use the diagonal of the cutout to define
# the search radius for the archive. Images which fall outside the desired cutout need to
# be filtered from the solution later.
radius = math.ceil(
math.sqrt(
math.pow(cutout_size.value[0], 2) +
math.pow(cutout_size.value[1], 2)) / 2.0)
# Careful - the radius must be a str or Quantity
radius *= u.arcsec
log.info("Radius for query: {}.".format(radius))
log.info("Performing query for ACS images.")
acs_query_table = Observations.query_criteria(coordinates=sky_coord,
radius=radius,
dataproduct_type="IMAGE",
instrument_name="ACS*",
project="HAP",
obs_collection="HST")
log.info("Performing query for WFC3 images.")
wfc3_query_table = Observations.query_criteria(coordinates=sky_coord,
radius=radius,
dataproduct_type="IMAGE",
instrument_name="WFC3*",
project="HAP",
obs_collection="HST")
query_table = vstack([acs_query_table, wfc3_query_table])
del acs_query_table
del wfc3_query_table
# Catch the case where no files are found which satisfied the Query
if not query_table:
log.warning(
"Query for objects within {} of {} returned NO RESULTS!".format(
radius, (str_ra, str_dec)))
return query_table
# Compute the limits of the cutout region
deg_cutout_size = cutout_size.to(u.deg)
ra_min = sky_coord.ra.degree - deg_cutout_size.value[0]
ra_max = sky_coord.ra.degree + deg_cutout_size.value[0]
dec_min = sky_coord.dec.degree - deg_cutout_size.value[1]
dec_max = sky_coord.dec.degree + deg_cutout_size.value[1]
str_ra = "{:.4f}".format(sky_coord.ra.degree)
str_dec = "{:.4f}".format(sky_coord.dec.degree)
# Filter the output as necessary to include only MVM filenames (MVM prefix: hst_skycell).
# Also, filter out images which are not actually in the requested cutout region as the
# archive search had to be done using a radius.
good_rows = []
updated_query_table = None
for old_row in query_table:
if old_row["obs_id"].startswith("hst_skycell"):
if old_row["s_ra"] >= ra_min and old_row["s_ra"] <= ra_max and \
old_row["s_dec"] >= dec_min and old_row["s_dec"] <= dec_max:
good_rows.append(old_row)
# Catch the case where no files are found which satisfy the clean up criteria
if len(good_rows) == 0:
log.warning(
"Query for objects within cutout {} of {} returned NO RESULTS!".
format(cutout_size, (str_ra, str_dec)))
return updated_query_table
# Make the cleaned up table
updated_query_table = Table(rows=good_rows, names=query_table.colnames)
del query_table
# Get the data product list associated with the elements of the table
log.info("Get the product list for all entries in the query table.")
dp_table = Observations.get_product_list(updated_query_table)
del updated_query_table
# Filter on MVM drizzled products only
suffix = ["DRZ", "DRC"]
log.info(
"Filter the product list table for only {} filenames.".format(suffix))
filtered_dp_table = Observations.filter_products(
dp_table, productSubGroupDescription=suffix, extension="fits")
if not filtered_dp_table:
log.warning(
"No MVM drizzle product datasets (DRZ/DRC) found within {} of {}.".
format(radius, (str_ra, str_dec)))
return filtered_dp_table
del dp_table
# Need to filter out any non-hst-skycell entries AGAIN which may have
# crept back into the list via the get_product_list() function.
good_rows = []
output_table = None
for old_row in filtered_dp_table:
if old_row["obs_id"].startswith("hst_skycell"):
good_rows.append(old_row)
# Catch the case where no files are found which satisfy the criteria
if len(good_rows) == 0:
log.warning(
"After filtering datasets there are NO RESULTS within {} of {}!".
format(radius, (str_ra, str_dec)))
return output_table
# Make the output table
output_table = Table(rows=good_rows, names=filtered_dp_table.colnames)
del filtered_dp_table
# Finally, make sure the entries are unique
final_table = None
final_table = unique(output_table, keys="productFilename")
del output_table
# Write the table to a file. This allows for further manipulation of
# the information before a list of filenames is distilled from the table.
# Output filename in the form: mvm_query-ra<###>d<####>-dec<n|s><##>d<####>_<radius>_cutout.ecsv
# (e.g., mvm_query-ra84d9208-decs69d1483_71_cutout.ecsv), where radius has been computed from the
# cutout dimensions.
#
# Get the whole number and fractional components of the RA and Dec
ns = "s" if sky_coord.dec.degree < 0.0 else "n"
ra_whole = int(sky_coord.ra.value)
ra_frac = str(sky_coord.ra.value).split(".")[1][0:4]
dec_whole = abs(int(sky_coord.dec.value))
dec_frac = str(sky_coord.dec.value).split(".")[1][0:4]
log.info("coords2. {} {} {}".format(sky_coord.ra.value,
sky_coord.dec.value, dec_frac))
query_filename = "mvm_query-ra" + str(ra_whole) + "d" + ra_frac + "-dec" + ns + \
str(dec_whole) + "d" + dec_frac + "_{:.0f}".format(radius.value) + "_cutout.ecsv"
log.info(
"Writing out the MVM product list table to {}.".format(query_filename))
log.info("Number of entries in table: {}.".format(len(final_table)))
final_table.write(query_filename, format="ascii.ecsv")
return final_table
def main():
# Read in the Pantheon+ catalog from Ben
adap_dir = home + '/Documents/adap2021/'
pantheon_datadir = adap_dir + 'pantheon_data/'
cat = np.genfromtxt(adap_dir + 'pantheon_plus.csv',
dtype=None, names=True, delimiter=',', encoding='ascii')
print("Read in Pantheon+ catalog with the following header names:")
print(cat.dtype.names)
num_orig_cols = len(cat.dtype.names)
# Open a new file to write an updated catalog
# Adds the following columns
# HST data
# GALEX data
# if yes to any of the above observatories then give
# Inst/Camera field, and filters.
# if no then leave these cols blank.
fh = open(adap_dir + 'pantheon_plus_data.csv', 'w')
# Write header
fh.write("Serial_num,CID,CIDint,IDSURVEY,zHEL,zHELERR,zCMB,zCMBERR,zHD,zHDERR," +\
"HOST_LOGMASS,HOST_LOGMASS_ERR,RA,DEC,HOST_RA,HOST_DEC," +\
"HST_data,Inst/Cam,Filters" + "\n")
# Loop over all objects in the catalog
# and search for HST data at the SN and Host location
for i in range(27, len(cat)): #tqdm(range(len(cat)), desc="Processing SN"):
# Get coords
sn_ra = cat['RA'][i]
sn_dec = cat['DEC'][i]
host_ra = cat['HOST_RA'][i]
host_dec = cat['HOST_DEC'][i]
# Print info
#print(f"{bcolors.CYAN}")
#print("SN identifier:", cat['CID'][i], " at:", sn_ra, sn_dec)
#print("Host galaxy coords:", host_ra, host_dec)
#print(f"{bcolors.ENDC}")
# Set up query
sn_coords = SkyCoord(ra=sn_ra*u.degree, dec=sn_dec*u.degree, frame='icrs')
print("SN coordinates:", sn_coords)
obs_table = Observations.query_criteria(coordinates=sn_coords, radius="0.5 arcsec", \
intentType='science', obs_collection=['HST'])
#print(obs_table)
print(obs_table.columns)
print("\nRows in obs table:", len(obs_table))
print("HST filters available for this SN:")
all_instr = np.unique(obs_table['instrument_name'])
print(all_instr)
print("--------------------------------------\n")
sys.exit(0)
# Download any existing wfc3 data
for r in range(len(obs_table)):
instr = obs_table['instrument_name'][r]
if 'WFC3' in instr:
data_products = Observations.get_product_list(obs_table[r])
Observations.download_products(data_products, download_dir=pantheon_datadir,
productType="SCIENCE", mrp_only=True)
sys.exit(0)
# Now loop over all the observations
ra_one = sn_ra
dec_one = sn_dec
dist = []
inst_cam = []
exptimes = []
filt = []
for o in range(len(obs_table)):
ra_two = obs_table['s_ra'][o]
dec_two = obs_table['s_dec'][o]
dist_to_sn = np.arccos(np.cos(dec_one*np.pi/180) * \
np.cos(dec_two*np.pi/180) * np.cos(ra_one*np.pi/180 - ra_two*np.pi/180) + \
np.sin(dec_one*np.pi/180) * np.sin(dec_two*np.pi/180))
# print("{:.7}".format(ra_two), "{:.7}".format(dec_two))
#print("Distance from SN [arcsec]:", "{:.5}".format(dist_to_sn * (180/np.pi) * 3600), \
# "Inst/Cam:", obs_table['instrument_name'][o], "Filter(s):", obs_table['filters'][o], \
# "ExpTime:", obs_table['t_exptime'][o])
dist.append(dist_to_sn * (180/np.pi) * 3600) # The dist returned by the line above is in radians
inst_cam.append(obs_table['instrument_name'][o])
filt.append(obs_table['filters'][o])
exptimes.append(obs_table['t_exptime'][o])
# Add to original catalog
# Need to loop over the original row to do this
for j in range(num_orig_cols):
fh.write(str(cat[i][j]) + ',')
# Now add the new cols
# First convert to numpy arrays
inst_cam = np.unique(np.asarray(inst_cam))
filt = np.unique(np.asarray(filt))
if len(obs_table) > 0:
hst_data = True
else:
hst_data = False
fh.write(str(hst_data) + ",")
if len(inst_cam) > 1:
for w in range(len(inst_cam)):
fh.write(str(inst_cam[w]) + ";")
fh.write(",")
else:
fh.write(str(inst_cam) + ",")
if len(filt) > 1:
for v in range(len(filt)):
fh.write(str(filt[v]) + ";")
else:
fh.write(str(filt))
fh.write("\n")
# Check that the distances are within FoV of the instrument specified
# dist = np.asarray(dist) * (180/np.pi) * 3600 # radians to degrees to arcseconds
fh.close()
return None
def download(self,
quality_bitmask='default',
download_dir=None,
cutout_size=None):
"""Returns a single `KeplerTargetPixelFile` or `KeplerLightCurveFile` object.
If multiple files are present in `SearchResult.table`, only the first
will be downloaded.
Parameters
----------
quality_bitmask : str or int
Bitmask (integer) which identifies the quality flag bitmask that should
be used to mask out bad cadences. If a string is passed, it has the
following meaning:
* "none": no cadences will be ignored (`quality_bitmask=0`).
* "default": cadences with severe quality issues will be ignored
(`quality_bitmask=1130799`).
* "hard": more conservative choice of flags to ignore
(`quality_bitmask=1664431`). This is known to remove good data.
* "hardest": removes all data that has been flagged
(`quality_bitmask=2096639`). This mask is not recommended.
See the :class:`KeplerQualityFlags` class for details on the bitmasks.
download_dir : str
Location where the data files will be stored.
Defaults to "~/.lightkurve-cache" if `None` is passed.
cutout_size : int, float or tuple
Side length of cutout in pixels. Tuples should have dimensions (y, x).
Default size is (5, 5)
Returns
-------
data : `TargetPixelFile` or `LightCurveFile` object
The first entry in the products table.
"""
if len(self.table) == 0:
warnings.warn("Cannot download from an empty search result.",
LightkurveWarning)
return None
if len(self.table) != 1:
warnings.warn(
'Warning: {} files available to download. '
'Only the first file has been downloaded. '
'Please use `download_all()` or specify additional '
'criteria (e.g. quarter, campaign, or sector) '
'to limit your search.'.format(len(self.table)),
LightkurveWarning)
# Make sure astroquery uses the same level of verbosity
logging.getLogger('astropy').setLevel(log.getEffectiveLevel())
# download first product in table
if download_dir is None:
download_dir = self._default_download_dir()
# if table contains TESScut search results, download cutout
if 'FFI Cutout' in self.table[0]['description']:
try:
path = self._fetch_tesscut_path(
self.table[0]['target_name'],
self.table[0]['sequence_number'], download_dir,
cutout_size)
except:
raise SearchError(
'Unable to download FFI cutout. Desired target '
'coordinates may be too near the edge of the FFI.')
else:
if cutout_size is not None:
warnings.warn(
'`cutout_size` can only be specified for TESS '
'Full Frame Image cutouts.', LightkurveWarning)
path = Observations.download_products(
self.table[:1], mrp_only=False,
download_dir=download_dir)['Local Path'][0]
# open() will determine filetype and return
return open(path, quality_bitmask=quality_bitmask)
def retrieve_observation(obsid, suffix=['FLC'], archive=False, clobber=False):
"""Simple interface for retrieving an observation from the MAST archive
If the input obsid is for an association, it will request all members with
the specified suffixes.
Parameters
-----------
obsid : string
ID for observation to be retrieved from the MAST archive. Only the
IPPSSOOT (rootname) of exposure or ASN needs to be provided; eg.,
ib6v06060.
suffix : list, optional
List containing suffixes of files which should be requested from MAST.
Default value "['FLC']".
archive : Boolean, optional
Retain copies of the downloaded files in the astroquery created
sub-directories? Default is "False".
clobber : Boolean, optional
Download and Overwrite existing files? Default is "False".
Returns
-------
local_files : list
List of filenames
"""
local_files = []
if Observations is None:
log.warning("The astroquery package was not found. No files retrieved!")
return local_files
# Query MAST for the data with an observation type of either "science" or
# "calibration"
obs_table = Observations.query_criteria(obs_id=obsid, obstype='all')
# Catch the case where no files are found for download
if not obs_table:
log.info("WARNING: Query for {} returned NO RESULTS!".format(obsid))
return local_files
dpobs = Observations.get_product_list(obs_table)
data_products_by_id = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# After the filtering has been done, ensure there is still data in the
# table for download. If the table is empty, look for FLT images in lieu
# of FLC images. Only want one or the other (not both!), so just do the
# filtering again.
if not data_products_by_id:
log.info("WARNING: No FLC files found for {} - will look for FLT "
"files instead.".format(obsid))
suffix = ['FLT']
data_products_by_id = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# If still no data, then return. An exception will eventually be
# thrown in the higher level code.
if not data_products_by_id:
log.info(
"WARNING: No FLC or FLT files found for {}.".format(obsid))
return local_files
all_images = data_products_by_id['productFilename'].tolist()
log.info(all_images)
if not clobber:
rows_to_remove = []
for row_idx, row in enumerate(data_products_by_id):
fname = row['productFilename']
if os.path.isfile(fname):
log.info(fname + " already exists. File download skipped.")
rows_to_remove.append(row_idx)
data_products_by_id.remove_rows(rows_to_remove)
manifest = Observations.download_products(data_products_by_id,
mrp_only=False)
if not clobber:
for rownum in rows_to_remove[::-1]:
if manifest:
manifest.insert_row(rownum,
vals=[all_images[rownum], "LOCAL", "None", "None"])
else:
return all_images
download_dir = None
for file, file_status in zip(manifest['Local Path'], manifest['Status']):
if file_status != "LOCAL":
# Identify what sub-directory was created by astroquery for the
# download
if download_dir is None:
download_dir = os.path.dirname(os.path.abspath(file))
# Move or copy downloaded file to current directory
local_file = os.path.abspath(os.path.basename(file))
if archive:
shutil.copy(file, local_file)
else:
shutil.move(file, local_file)
# Record what files were downloaded and their current location
local_files.append(os.path.basename(local_file))
else:
local_files.append(file)
if not archive:
# Remove astroquery created sub-directories
shutil.rmtree(download_dir)
return local_files
def _search_products(target,
radius=None,
filetype="Lightcurve",
cadence='long',
mission=['Kepler', 'K2', 'TESS'],
quarter=None,
month=None,
campaign=None,
sector=None,
limit=None):
"""Helper function which returns a SearchResult object containing MAST
products that match several criteria.
Parameters
----------
target : str, int, or `astropy.coordinates.SkyCoord` object
See docstrings above.
radius : float or `astropy.units.Quantity` object
Conesearch radius. If a float is given it will be assumed to be in
units of arcseconds. If `None` then we default to 0.0001 arcsec.
filetype : {'Target pixel', 'Lightcurve', 'FFI'}
Type of files queried at MAST.
cadence : str
Desired cadence (`long`, `short`, `any`)
mission : str, list of str
'Kepler', 'K2', or 'TESS'. By default, all will be returned.
quarter, campaign, sector : int, list of ints
Kepler Quarter, K2 Campaign, or TESS Sector number.
By default all quarters/campaigns/sectors will be returned.
month : 1, 2, 3, 4 or list of int
For Kepler's prime mission, there are three short-cadence
TargetPixelFiles for each quarter, each covering one month.
Hence, if cadence='short' you can specify month=1, 2, 3, or 4.
By default all months will be returned.
limit : int
Maximum number of products to return
Returns
-------
SearchResult : :class:`SearchResult` object.
"""
observations = _query_mast(target, project=mission, radius=radius)
if len(observations) == 0:
raise SearchError('No data found for target "{}".'.format(target))
# Light curves and target pixel files
if filetype.lower() != 'ffi':
products = Observations.get_product_list(observations)
result = join(products,
observations,
keys="obs_id",
join_type='left',
uniq_col_name='{col_name}{table_name}',
table_names=['', '_2'])
result.sort(['distance', 'obs_id'])
masked_result = _filter_products(result,
filetype=filetype,
campaign=campaign,
quarter=quarter,
cadence=cadence,
project=mission,
month=month,
sector=sector,
limit=limit)
return SearchResult(masked_result)
# Full Frame Images
else:
cutouts = []
for idx in np.where(
['TESS FFI' in t for t in observations['target_name']])[0]:
# if target passed in is a SkyCoord object, convert to RA, dec pair
if isinstance(target, SkyCoord):
target = '{}, {}'.format(target.ra.deg, target.dec.deg)
# pull sector numbers
s = observations['sequence_number'][idx]
# if the desired sector is available, add a row
if s in np.atleast_1d(sector) or sector is None:
cutouts.append({
'description':
'TESS FFI Cutout (sector {})'.format(s),
'target_name':
str(target),
'targetid':
str(target),
'productFilename':
'n/a',
'distance':
0.0,
'sequence_number':
s
})
masked_result = Table(cutouts)
masked_result.sort(['distance', 'sequence_number'])
return SearchResult(masked_result)
def _download_one(self, table, quality_bitmask, download_dir, cutout_size, **kwargs):
"""Private method used by `download()` and `download_all()` to download
exactly one file from the MAST archive.
Always returns a `TargetPixelFile` or `LightCurve` object.
"""
# Make sure astroquery uses the same level of verbosity
logging.getLogger('astropy').setLevel(log.getEffectiveLevel())
if download_dir is None:
download_dir = self._default_download_dir()
# if the SearchResult row is a TESScut entry, then download cutout
if 'FFI Cutout' in table[0]['description']:
try:
log.debug("Started downloading TESSCut for '{}' sector {}."
"".format(table[0]['target_name'], table[0]['sequence_number']))
path = self._fetch_tesscut_path(table[0]['target_name'],
table[0]['sequence_number'],
download_dir,
cutout_size)
except Exception as exc:
msg = str(exc)
if "504" in msg:
# TESSCut will occasionally return a "504 Gateway Timeout
# error" when it is overloaded.
raise HTTPError('The TESS FFI cutout service at MAST appears '
'to be temporarily unavailable. It returned '
'the following error: {}'.format(exc))
else:
raise SearchError('Unable to download FFI cutout. Desired target '
'coordinates may be too near the edge of the FFI.'
'Error: {}'.format(exc))
return read(path,
quality_bitmask=quality_bitmask,
targetid=table[0]['targetid'])
else:
if cutout_size is not None:
warnings.warn('`cutout_size` can only be specified for TESS '
'Full Frame Image cutouts.', LightkurveWarning)
# Whenever `astroquery.mast.Observations.download_products` is called,
# a HTTP request will be sent to determine the length of the file
# prior to checking if the file already exists in the local cache.
# For performance, we skip this HTTP request and immediately try to
# find the file in the cache. The path we check here is consistent
# with the one hard-coded inside `astroquery.mast.Observations._download_files()`
# in Astroquery v0.4.1. It would be good to submit a PR to astroquery
# so we can avoid having to use this hard-coded hack.
path = os.path.join(download_dir.rstrip('/'),
"mastDownload",
table['obs_collection'][0],
table['obs_id'][0],
table['productFilename'][0])
if os.path.exists(path):
log.debug("File found in local cache.")
else:
from astroquery.mast import Observations
log.debug("Started downloading {}.".format(table[:1]['dataURL'][0]))
path = Observations.download_products(table[:1], mrp_only=False,
download_dir=download_dir)['Local Path'][0]
log.debug("Finished downloading.")
return read(path, quality_bitmask=quality_bitmask, **kwargs)
distVec = []
m = len(quasar_list)
for n in np.arange(0,m):
try:
#pull the coordinate pairs from the csv file
RA_float = quasar_list.iloc[n,0]
RA = str(RA_float)
DEC_float = quasar_list.iloc[n,1]
DEC = str(DEC_float)
coords = RA + "_" + DEC
#search for and download all data products associated with those coordinates
print("coordinates:", coords, " n:", n)
table_of_observations = Observations.query_region(coords, radius = 0.02)
data_product_list = Observations.get_product_list(table_of_observations)
downloads = Observations.download_products(data_product_list, download_dir=coords,
obs_collection = "HST",
dataproduct_type = "image",
productType = "SCIENCE")
#Filter out all files that are not drz.fits files (drizzle reduced)--->
findQSOdrz = 'find . -type f \( -name "*drz.fits" \)'
out = subprocess.Popen(findQSOdrz, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(stdout, stderr) = out.communicate()
filelist = stdout.decode().split()
length = len(filelist)
print("drz files found:", length)
def _search_products(target, radius=None, filetype="Lightcurve", cadence=None,
mission=('Kepler', 'K2', 'TESS'),
provenance_name=('Kepler', 'K2', 'SPOC'),
t_exptime=(0, 9999), quarter=None, month=None,
campaign=None, sector=None, limit=None,
**extra_query_criteria):
"""Helper function which returns a SearchResult object containing MAST
products that match several criteria.
Parameters
----------
target : str, int, or `astropy.coordinates.SkyCoord` object
See docstrings above.
radius : float or `astropy.units.Quantity` object
Conesearch radius. If a float is given it will be assumed to be in
units of arcseconds. If `None` then we default to 0.0001 arcsec.
filetype : {'Target pixel', 'Lightcurve', 'FFI'}
Type of files queried at MAST.
cadence : 'long', 'short', 'fast', or float
'long' selects 10-min and 30-min cadence products;
'short' selects 1-min and 2-min products;
'fast' selects 20-sec products.
Alternatively, you can pass the exact exposure time in seconds as
an int or a float, e.g. ``cadence=600`` selects 10-minute cadence.
By default, all cadence modes are returned.
mission : str, list of str
'Kepler', 'K2', or 'TESS'. By default, all will be returned.
provenance_name : str, list of str
Provenance of the data product. Defaults to official products, i.e.
('Kepler', 'K2', 'SPOC'). Community-provided products such as 'K2SFF'
are supported as well.
quarter, campaign, sector : int, list of ints
Kepler Quarter, K2 Campaign, or TESS Sector number.
By default all quarters/campaigns/sectors will be returned.
month : 1, 2, 3, 4 or list of int
For Kepler's prime mission, there are three short-cadence
TargetPixelFiles for each quarter, each covering one month.
Hence, if cadence='short' you can specify month=1, 2, 3, or 4.
By default all months will be returned.
limit : int
Maximum number of products to return
Returns
-------
SearchResult : :class:`SearchResult` object.
"""
if isinstance(target, int):
if (0 < target) and (target < 13161030):
log.warning("Warning: {} may refer to a different Kepler or TESS target. "
"Please add the prefix 'KIC' or 'TIC' to disambiguate."
"".format(target))
elif (0 < 200000000) and (target < 251813739):
log.warning("Warning: {} may refer to a different K2 or TESS target. "
"Please add the prefix 'EPIC' or 'TIC' to disambiguate."
"".format(target))
# Specifying quarter, campaign, or quarter should constrain the mission
if quarter:
mission = "Kepler"
if campaign:
mission = "K2"
if sector:
mission = "TESS"
# Ensure mission is a list
mission = np.atleast_1d(mission).tolist()
# Avoid filtering on `provenance_name` if `author` equals "any" or "all"
if provenance_name in ("any", "all") or provenance_name is None:
provenance_name = None
else:
provenance_name = np.atleast_1d(provenance_name).tolist()
# Speed up by restricting the MAST query if we don't want FFI image data
extra_query_criteria = {}
if filetype in ['Lightcurve', 'Target Pixel']:
# At MAST, non-FFI Kepler pipeline products are known as "cube" products,
# and non-FFI TESS pipeline products are listed as "timeseries".
extra_query_criteria['dataproduct_type'] = ['cube', 'timeseries']
# Make sure `search_tesscut` always performs a cone search (i.e. always
# passed a radius value), because strict target name search does not apply.
if filetype.lower() == 'ffi' and radius is None:
radius = .0001 * u.arcsec
observations = _query_mast(target, radius=radius,
project=mission,
provenance_name=provenance_name,
t_exptime=t_exptime,
sequence_number=campaign or sector,
**extra_query_criteria)
log.debug("MAST found {} observations. "
"Now querying MAST for the corresponding data products."
"".format(len(observations)))
if len(observations) == 0:
raise SearchError('No data found for target "{}".'.format(target))
# Light curves and target pixel files
if filetype.lower() != 'ffi':
from astroquery.mast import Observations
products = Observations.get_product_list(observations)
result = join(observations, products, keys="obs_id", join_type='right',
uniq_col_name='{col_name}{table_name}', table_names=['', '_products'])
result.sort(['distance', 'obs_id'])
# Add the user-friendly 'author' column (synonym for 'provenance_name')
result['author'] = result['provenance_name']
# Add the user-friendly 'observation' column
result['observation'] = None
obs_prefix = {'Kepler': 'Quarter', 'K2': 'Campaign', 'TESS': 'Sector'}
for idx in range(len(result)):
obs_project = result['project'][idx]
obs_seqno = result['sequence_number'][idx]
# Kepler sequence_number values were not populated at the time of
# writing this code, so we parse them from the description field.
if obs_project == 'Kepler' and result['sequence_number'].mask[idx]:
try:
obs_seqno = re.findall(r".*Q(\d+)", result['description'][idx])[0]
except IndexError:
obs_seqno = ""
result['observation'][idx] = "{} {} {}".format(obs_project,
obs_prefix.get(obs_project, ""),
obs_seqno)
masked_result = _filter_products(result, filetype=filetype,
campaign=campaign, quarter=quarter,
cadence=cadence, project=mission,
provenance_name=provenance_name,
month=month, sector=sector, limit=limit)
log.debug("MAST found {} matching data products.".format(len(masked_result)))
masked_result['distance'].info.format = '.1f' # display <0.1 arcsec
return SearchResult(masked_result)
# Full Frame Images
else:
cutouts = []
for idx in np.where(['TESS FFI' in t for t in observations['target_name']])[0]:
# if target passed in is a SkyCoord object, convert to RA, dec pair
if isinstance(target, SkyCoord):
target = '{}, {}'.format(target.ra.deg, target.dec.deg)
# pull sector numbers
s = observations['sequence_number'][idx]
# if the desired sector is available, add a row
if s in np.atleast_1d(sector) or sector is None:
cutouts.append({'description': f'TESS FFI Cutout (sector {s})',
'observation': f'TESS Sector {s}',
'target_name': str(target),
'targetid': str(target),
't_exptime': observations['t_exptime'][idx],
'productFilename': 'TESSCut',
'provenance_name': 'MAST',
'author': 'MAST',
'distance': 0.0,
'sequence_number': s,
'project': 'TESS',
'obs_collection': 'TESS'}
)
if len(cutouts) > 0:
log.debug("Found {} matching cutouts.".format(len(cutouts)))
masked_result = Table(cutouts)
masked_result.sort(['distance', 'sequence_number'])
else:
masked_result = None
return SearchResult(masked_result)
def lambda_handler(event, context):
"""Extract light curve data from one TESS full frame image.
Parameters
----------
event : dict
API Gateway Lambda Proxy Input Format.
Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format
context : object
Lambda Context runtime methods and attributes.
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
result : dict
API Gateway Lambda Proxy Output Format.
Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
""" # noqa
obs_id = event['id'] # TESS observation ID; Example: 'tess-s0001-1-1'
# TODO: Calculate some of these from the 10th frame?
# For now, also takes these and pass them onto worker:
payload = {
'xpos': event['xpos'],
'ypos': event['ypos'],
'radius': event['radius'],
'bright_pixel_threshold': event['bright_pixel_threshold']
}
# Find full frame dataset for the observation ID.
obs_table = Observations.query_criteria(obs_id=obs_id)
products = Observations.get_product_list(obs_table)
filtered = Observations.filter_products(products,
productSubGroupDescription="FFIC",
mrp_only=False)
# Use AWS S3 bucket to pull data from.
Observations.enable_cloud_dataset() # TODO: verbose=False ?
s3_urls = Observations.get_cloud_uris(filtered, include_bucket=False)
# TODO: Timed out! Try https://docs.python.org/3/library/asyncio.html ?
# TODO: Handle same Lambda call invoked multiple times by AWS?
# Call tess_fullframe_worker AWS Lambda function in parallel
# https://aws.amazon.com/blogs/compute/parallel-processing-in-python-with-aws-lambda/
parent_connections = []
processes = []
data = []
for url in s3_urls[:2]: # TODO: Remove [:2] when done testing
payload['key'] = url
parent_conn, child_conn = Pipe()
parent_connections.append(parent_conn)
arg = json.dumps(payload)
process = Process(target=_pipe_worker, args=(arg, child_conn))
processes.append(process)
for process in processes:
process.start()
for process in processes:
process.join()
for parent_connection in parent_connections:
try:
response = parent_connection.recv()[0]
except EOFError:
response = {}
if 'body' not in response: # Worker Lambda threw exception
continue
body = json.loads(response['body'])
row = (body['midtime'], body['signal'], body['background'])
if np.all(list(map(np.isfinite, row))):
data.append(row)
# TODO: Save data as table.
# filename = f'/tmp/{obs_id}_lightcurve.csv'
# with open(filename) as fout:
# for row in data:
# fout.write(f'{row[0]},{row[1]},{row[2]}{os.linesep}')
# TODO: Upload table to S3 and then delete the table locally.
# TODO: Return table S3 URL below.
# TODO: Do we want to plot it and upload the plot too?
# If so, need to add matplotlib as dependency.
return {
"statusCode": 200,
"body": json.dumps({
'n_rows': len(data),
'data_url': 'TODO'
})
}
def _query_mast(target, radius=None,
project=('Kepler', 'K2', 'TESS'),
provenance_name=("Kepler", "K2", "SPOC"),
t_exptime=(0, 9999),
sequence_number=None,
**extra_query_criteria):
"""Helper function which wraps `astroquery.mast.Observations.query_criteria()`
to return a table of all Kepler/K2/TESS observations of a given target.
By default only the official data products are returned, but this can be
adjusted by adding alternative data product names into `provenance_name`.
Parameters
----------
target : str, int, or `astropy.coordinates.SkyCoord` object
See docstrings above.
radius : float or `astropy.units.Quantity` object
Conesearch radius. If a float is given it will be assumed to be in
units of arcseconds. If `None` then we default to 0.0001 arcsec.
project : str, list of str
Mission name. Typically 'Kepler', 'K2', or 'TESS'.
This parameter is case-insensitive.
provenance_name : str, list of str
Provenance of the observation. Common options include 'Kepler', 'K2',
'SPOC', 'K2SFF', 'EVEREST', 'KEPSEISMIC'.
This parameter is case-insensitive.
t_exptime : (float, float) tuple
Exposure time range in seconds. Common values include `(59, 61)`
for Kepler short cadence and `(1799, 1801)` for Kepler long cadence.
sequence_number : int, list of int
Quarter, Campaign, or Sector number.
**extra_query_criteria : kwargs
Extra criteria to be passed to `astroquery.mast.Observations.query_criteria`.
Returns
-------
obs : astropy.Table
Table detailing the available observations on MAST.
"""
# Local astroquery import because the package is not used elsewhere
from astroquery.mast import Observations
from astroquery.exceptions import ResolverError, NoResultsWarning
# If passed a SkyCoord, convert it to an "ra, dec" string for MAST
if isinstance(target, SkyCoord):
target = '{}, {}'.format(target.ra.deg, target.dec.deg)
# We pass the following `query_criteria` to MAST regardless of whether
# we search by position or target name:
query_criteria = {
'project': project,
**extra_query_criteria
}
if provenance_name is not None:
query_criteria['provenance_name'] = provenance_name
if sequence_number is not None:
query_criteria['sequence_number'] = sequence_number
if t_exptime is not None:
query_criteria['t_exptime'] = t_exptime
# If an exact KIC ID is passed, we will search by the exact `target_name`
# under which MAST will know the object to prevent source confusion.
# For discussion, see e.g. GitHub issues #148, #718.
exact_target_name = None
target_lower = str(target).lower()
# Was a Kepler target ID passed?
kplr_match = re.match("^(kplr|kic) ?(\d+)$", target_lower)
if kplr_match:
exact_target_name = f"kplr{kplr_match.group(2).zfill(9)}"
# Was a K2 target ID passed?
ktwo_match = re.match("^(ktwo|epic) ?(\d+)$", target_lower)
if ktwo_match:
exact_target_name = f"ktwo{ktwo_match.group(2).zfill(9)}"
# Was a TESS target ID passed?
tess_match = re.match("^(tess|tic) ?(\d+)$", target_lower)
if tess_match:
exact_target_name = f"{tess_match.group(2).zfill(9)}"
if exact_target_name and radius is None:
log.debug("Started querying MAST for observations with the exact "
f"target_name='{exact_target_name}'.")
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=NoResultsWarning)
obs = Observations.query_criteria(target_name=exact_target_name,
**query_criteria)
if len(obs) > 0:
# astroquery does not report distance when querying by `target_name`;
# we add it here so that the table returned always has this column.
obs['distance'] = 0.
return obs
else:
log.debug(f"No observations found. Now performing a cone search instead.")
# If the above did not return a result, then do a cone search using the MAST name resolver
# `radius` defaults to 0.0001 and unit arcsecond
if radius is None:
radius = .0001 * u.arcsec
elif not isinstance(radius, u.quantity.Quantity):
radius = radius * u.arcsec
query_criteria['radius'] = str(radius.to(u.deg))
try:
log.debug("Started querying MAST for observations within "
f"{radius.to(u.arcsec)} arcsec of objectname='{target}'.")
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=NoResultsWarning)
obs = Observations.query_criteria(objectname=target,
**query_criteria)
obs.sort('distance')
return obs
except ResolverError as exc:
# MAST failed to resolve the object name to sky coordinates
raise SearchError(exc) from exc
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Apr 19 19:31:11 2020
@author: smullally
"""
from astroquery.mast import Observations
Observations.enable_cloud_dataset(provider='AWS')
target = "Kepler-10"
#Do a cone search and find the Kepler long cadence data for your target
obs = Observations.query_object(target, radius="0s")
want = (obs['obs_collection'] == "Kepler") & (obs['t_exptime'] == 1800.0)
#Pick which data you want to retrieve
data_prod = Observations.get_product_list(obs[want])
filt_prod = Observations.filter_products(
data_prod, description="Lightcurve Long Cadence (CLC) - Q4")
#Move data from the S3 bucket to the default astroquery location.
#cloud_only=True means that data will only be retrieved if available on AWS S3
manifest = Observations.download_products(filt_prod)
#%%
import pdb
from lightkurve import search_targetpixelfile
input(
"Would you like an ascii file of the phased data?\n0 = no, 1 = yes: "))
# Is the period actually 2*P?
flag_p2 = int(
input("Would you like to multiply the period by two?\n"
"(useful for ellipsoidal variables and some eclipsing systems)\n"
"0 = no, 1 = yes: "))
################################
####### DOWNLOAD DATA ########
# Searching for data at MAST
obsTable = Observations.query_criteria(dataproduct_type="timeseries",
project="TESS",
target_name=TIC)
# Download the 2-minute cadence light curves
data = Observations.get_product_list(obsTable)
download_lc = Observations.download_products(data,
productSubGroupDescription="LC")
infile = download_lc[0][:]
n_slow = len(infile)
print("I have found a total of " + str(len(infile)) + " 2-min light curve(s).")
# Download the 20-second cadence light curves
download_fast_lc = Observations.download_products(
