def retrieve_observation(obsid, suffix=['FLC'], archive=False,clobber=False):
"""Simple interface for retrieving an observation from the MAST archive
If the input obsid is for an association, it will request all members with
the specified suffixes.
Parameters
-----------
obsid : string
ID for observation to be retrieved from the MAST archive. Only the
IPPSSOOT (rootname) of exposure or ASN needs to be provided; eg., ib6v06060.
suffix : list
List containing suffixes of files which should be requested from MAST.
path : string
Directory to use for writing out downloaded files. If `None` (default),
the current working directory will be used.
archive : Boolean
Retain copies of the downloaded files in the astroquery created sub-directories? Default is 'False'.
clobber : Boolean
Download and Overwrite existing files? Default is 'False'.
Returns
-------
local_files : list
List of filenames
"""
local_files = []
# Query MAST for the data with an observation type of either "science" or "calibration"
obsTable = Observations.query_criteria(obs_id=obsid, obstype='all')
# Catch the case where no files are found for download
if len(obsTable) == 0:
log.info("WARNING: Query for {} returned NO RESULTS!".format(obsid))
return local_files
dpobs = Observations.get_product_list(obsTable)
dataProductsByID = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# After the filtering has been done, ensure there is still data in the table for download.
# If the table is empty, look for FLT images in lieu of FLC images. Only want one
# or the other (not both!), so just do the filtering again.
if len(dataProductsByID) == 0:
log.info("WARNING: No FLC files found for {} - will look for FLT files instead.".format(obsid))
suffix = ['FLT']
dataProductsByID = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# If still no data, then return. An exception will eventually be thrown in
# the higher level code.
if len(dataProductsByID) == 0:
log.info("WARNING: No FLC or FLT files found for {}.".format(obsid))
return local_files
allImages = []
for tableLine in dataProductsByID:
allImages.append(tableLine['productFilename'])
log.info(allImages)
if not clobber:
rowsToRemove = []
for rowCtr in range(0,len(dataProductsByID)):
if os.path.exists(dataProductsByID[rowCtr]['productFilename']):
log.info("{} already exists. File download skipped.".format(dataProductsByID[rowCtr]['productFilename']))
rowsToRemove.append(rowCtr)
if rowsToRemove:
rowsToRemove.reverse()
for rowNum in rowsToRemove:
dataProductsByID.remove_row(rowNum)
manifest = Observations.download_products(dataProductsByID, mrp_only=False)
if not clobber:
rowsToRemove.reverse()
for rownum in rowsToRemove:
if not manifest:
local_files = allImages
return local_files
else:
manifest.insert_row(rownum,vals=[allImages[rownum],"LOCAL","None","None"])
download_dir = None
for file,fileStatus in zip(manifest['Local Path'],manifest['Status']):
if fileStatus != "LOCAL":
# Identify what sub-directory was created by astroquery for the download
if download_dir is None:
file_path = file.split(os.sep)
file_path.remove('.')
download_dir = file_path[0]
# Move or copy downloaded file to current directory
local_file = os.path.abspath(os.path.basename(file))
if archive:
shutil.copy(file, local_file)
else:
shutil.move(file, local_file)
# Record what files were downloaded and their current location
local_files.append(os.path.basename(local_file))
else:
local_files.append(file)
if not archive:
# Remove astroquery created sub-directories
shutil.rmtree(download_dir)
return local_files
++++++++++
Retrieve Hubble archival data of M83 and make a figure
"""
from astroquery.mast import Mast, Observations
from astropy.visualization import make_lupton_rgb, ImageNormalize
import matplotlib.pyplot as plt
import reproject
result = Observations.query_object('M83')
selected_bands = result[(result['obs_collection'] == 'HST') &
(result['instrument_name'] == 'WFC3/UVIS') &
((result['filters'] == 'F657N') |
(result['filters'] == 'F487N') |
(result['filters'] == 'F336W')) &
(result['target_name'] == 'MESSIER-083')]
prodlist = Observations.get_product_list(selected_bands)
filtered_prodlist = Observations.filter_products(prodlist)
downloaded = Observations.download_products(filtered_prodlist)
blue = fits.open(downloaded['Local Path'][2])
red = fits.open(downloaded['Local Path'][5])
green = fits.open(downloaded['Local Path'][8])
target_header = red['SCI'].header
green_repr, _ = reproject.reproject_interp(green['SCI'], target_header)
blue_repr, _ = reproject.reproject_interp(blue['SCI'], target_header)
rgb_img = make_lupton_rgb(ImageNormalize(vmin=0, vmax=1)(red['SCI'].data),
ImageNormalize(vmin=0, vmax=0.3)(green_repr),
"(useful for ellipsoidal variables and some eclipsing systems)\n"
"0 = no, 1 = yes: "))
################################
####### DOWNLOAD DATA ########
# Searching for data at MAST
obsTable = Observations.query_criteria(dataproduct_type="timeseries",
project="TESS",
target_name=TIC)
# Download the 2-minute cadence light curves
data = Observations.get_product_list(obsTable)
download_lc = Observations.download_products(data,
productSubGroupDescription="LC")
infile = download_lc[0][:]
n_slow = len(infile)
print("I have found a total of " + str(len(infile)) + " 2-min light curve(s).")
# Download the 20-second cadence light curves
download_fast_lc = Observations.download_products(
data, productSubGroupDescription="FAST-LC")
if download_fast_lc is None:
print("I have found no 20-sec light curves.")
fast = False
def get_tess_data(u_ticid, max_flag=16, out_dir='./download'):
"""Given a TIC-ID, return time,flux,ferr,itime
u_ticid : (int) TIC ID
returns lc_time,flux,ferr,int_time
"""
tic_str = 'TIC' + str(u_ticid)
#out_dir='/data/rowe/TESS/download/'
# Search MAST for TIC ID
obs_table = Observations.query_object(tic_str, radius=".002 deg")
# Identify TESS timeseries data sets (i.e. ignore FFIs)
oti=(obs_table["obs_collection"] == "TESS") & \
(obs_table["dataproduct_type"] == "timeseries")
if oti.any() == True:
data_products = Observations.get_product_list(obs_table[oti])
dpi = [
j for j, s in enumerate(data_products["productFilename"])
if "lc.fits" in s
]
manifest = Observations.download_products(data_products[dpi],
download_dir=out_dir)
else:
manifest = []
lc_time = []
flux = []
ferr = []
int_time = []
for j in range(0, len(manifest)):
fits_fname = str(manifest["Local Path"][j])
print(fits_fname)
hdu = fits.open(fits_fname)
tmp_bjd = hdu[1].data['TIME']
tmp_flux = hdu[1].data['PDCSAP_FLUX']
tmp_ferr = hdu[1].data['PDCSAP_FLUX_ERR']
tmp_int_time = hdu[1].header['INT_TIME'] + np.zeros(len(tmp_bjd))
tmp_flag = hdu[1].data['QUALITY']
ii = (tmp_flag <= max_flag) & (~np.isnan(tmp_flux))
tmp_bjd = tmp_bjd[ii]
tmp_flux = tmp_flux[ii]
tmp_ferr = tmp_ferr[ii]
tmp_int_time = tmp_int_time[ii]
# Shift flux measurements
median_flux = np.median(tmp_flux)
tmp_flux = tmp_flux / median_flux
tmp_ferr = tmp_ferr / median_flux
# Append to output columns
lc_time = np.append(lc_time, tmp_bjd)
flux = np.append(flux, tmp_flux)
ferr = np.append(ferr, tmp_ferr)
int_time = np.append(int_time, tmp_int_time)
hdu.close()
# Sort by time
si = np.argsort(lc_time)
lc_time = np.asarray(lc_time)[si]
flux = np.asarray(flux)[si]
ferr = np.asarray(ferr)[si]
int_time = np.asarray(int_time)[si]
phot = phot_class()
phot.time = np.copy(lc_time)
phot.flux = np.copy(flux)
phot.ferr = np.copy(ferr)
phot.itime = np.copy(int_time)
phot.itime = phot.itime / (60 * 24) #convert minutes to days
return phot
def _search_products(target, radius=None, filetype="Lightcurve", cadence=None,
mission=('Kepler', 'K2', 'TESS'),
provenance_name=('Kepler', 'K2', 'SPOC'),
t_exptime=(0, 9999), quarter=None, month=None,
campaign=None, sector=None, limit=None,
**extra_query_criteria):
"""Helper function which returns a SearchResult object containing MAST
products that match several criteria.
Parameters
----------
target : str, int, or `astropy.coordinates.SkyCoord` object
See docstrings above.
radius : float or `astropy.units.Quantity` object
Conesearch radius. If a float is given it will be assumed to be in
units of arcseconds. If `None` then we default to 0.0001 arcsec.
filetype : {'Target pixel', 'Lightcurve', 'FFI'}
Type of files queried at MAST.
cadence : 'long', 'short', 'fast', or float
'long' selects 10-min and 30-min cadence products;
'short' selects 1-min and 2-min products;
'fast' selects 20-sec products.
Alternatively, you can pass the exact exposure time in seconds as
an int or a float, e.g. ``cadence=600`` selects 10-minute cadence.
By default, all cadence modes are returned.
mission : str, list of str
'Kepler', 'K2', or 'TESS'. By default, all will be returned.
provenance_name : str, list of str
Provenance of the data product. Defaults to official products, i.e.
('Kepler', 'K2', 'SPOC'). Community-provided products such as 'K2SFF'
are supported as well.
quarter, campaign, sector : int, list of ints
Kepler Quarter, K2 Campaign, or TESS Sector number.
By default all quarters/campaigns/sectors will be returned.
month : 1, 2, 3, 4 or list of int
For Kepler's prime mission, there are three short-cadence
TargetPixelFiles for each quarter, each covering one month.
Hence, if cadence='short' you can specify month=1, 2, 3, or 4.
By default all months will be returned.
limit : int
Maximum number of products to return
Returns
-------
SearchResult : :class:`SearchResult` object.
"""
if isinstance(target, int):
if (0 < target) and (target < 13161030):
log.warning("Warning: {} may refer to a different Kepler or TESS target. "
"Please add the prefix 'KIC' or 'TIC' to disambiguate."
"".format(target))
elif (0 < 200000000) and (target < 251813739):
log.warning("Warning: {} may refer to a different K2 or TESS target. "
"Please add the prefix 'EPIC' or 'TIC' to disambiguate."
"".format(target))
# Ensure mission is a list
mission = np.atleast_1d(mission).tolist()
# Avoid filtering on `provenance_name` if `author` equals "any" or "all"
if provenance_name in ("any", "all") or provenance_name is None:
provenance_name = None
else:
provenance_name = np.atleast_1d(provenance_name).tolist()
# Speed up by restricting the MAST query if we don't want FFI image data
extra_query_criteria = {}
if filetype in ['Lightcurve', 'Target Pixel']:
# At MAST, non-FFI Kepler pipeline products are known as "cube" products,
# and non-FFI TESS pipeline products are listed as "timeseries".
extra_query_criteria['dataproduct_type'] = ['cube', 'timeseries']
# Make sure `search_tesscut` always performs a cone search (i.e. always
# passed a radius value), because strict target name search does not apply.
if filetype.lower() == 'ffi' and radius is None:
radius = .0001 * u.arcsec
observations = _query_mast(target, radius=radius,
project=mission,
provenance_name=provenance_name,
t_exptime=t_exptime,
sequence_number=campaign or sector,
**extra_query_criteria)
log.debug("MAST found {} observations. "
"Now querying MAST for the corresponding data products."
"".format(len(observations)))
if len(observations) == 0:
raise SearchError('No data found for target "{}".'.format(target))
# Light curves and target pixel files
if filetype.lower() != 'ffi':
from astroquery.mast import Observations
products = Observations.get_product_list(observations)
result = join(observations, products, keys="obs_id", join_type='right',
uniq_col_name='{col_name}{table_name}', table_names=['', '_products'])
result.sort(['distance', 'obs_id'])
# Add the user-friendly 'author' column (synonym for 'provenance_name')
result['author'] = result['provenance_name']
# Add the user-friendly 'observation' column
result['observation'] = None
obs_prefix = {'Kepler': 'Quarter', 'K2': 'Campaign', 'TESS': 'Sector'}
for idx in range(len(result)):
obs_project = result['project'][idx]
obs_seqno = result['sequence_number'][idx]
# Kepler sequence_number values were not populated at the time of
# writing this code, so we parse them from the description field.
if obs_project == 'Kepler' and result['sequence_number'].mask[idx]:
try:
obs_seqno = re.findall(r".*Q(\d+)", result['description'][idx])[0]
except IndexError:
obs_seqno = ""
result['observation'][idx] = "{} {} {}".format(obs_project,
obs_prefix.get(obs_project, ""),
obs_seqno)
masked_result = _filter_products(result, filetype=filetype,
campaign=campaign, quarter=quarter,
cadence=cadence, project=mission,
provenance_name=provenance_name,
month=month, sector=sector, limit=limit)
log.debug("MAST found {} matching data products.".format(len(masked_result)))
masked_result['distance'].info.format = '.1f' # display <0.1 arcsec
return SearchResult(masked_result)
# Full Frame Images
else:
cutouts = []
for idx in np.where(['TESS FFI' in t for t in observations['target_name']])[0]:
# if target passed in is a SkyCoord object, convert to RA, dec pair
if isinstance(target, SkyCoord):
target = '{}, {}'.format(target.ra.deg, target.dec.deg)
# pull sector numbers
s = observations['sequence_number'][idx]
# if the desired sector is available, add a row
if s in np.atleast_1d(sector) or sector is None:
cutouts.append({'description': f'TESS FFI Cutout (sector {s})',
'observation': f'TESS Sector {s}',
'target_name': str(target),
'targetid': str(target),
't_exptime': observations['t_exptime'][idx],
'productFilename': 'TESSCut',
'provenance_name': 'MAST',
'author': 'MAST',
'distance': 0.0,
'sequence_number': s,
'project': 'TESS',
'obs_collection': 'TESS'}
)
if len(cutouts) > 0:
log.debug("Found {} matching cutouts.".format(len(cutouts)))
masked_result = Table(cutouts)
masked_result.sort(['distance', 'sequence_number'])
else:
masked_result = None
return SearchResult(masked_result)
def _search_products(target, radius=None, filetype="Lightcurve", cadence='long',
mission=['Kepler', 'K2', 'TESS'], quarter=None, month=None,
campaign=None, sector=None, limit=None):
"""Helper function which returns a SearchResult object containing MAST
products that match several criteria.
Parameters
----------
target : str, int, or `astropy.coordinates.SkyCoord` object
See docstrings above.
radius : float or `astropy.units.Quantity` object
Conesearch radius. If a float is given it will be assumed to be in
units of arcseconds. If `None` then we default to 0.0001 arcsec.
filetype : {'Target pixel', 'Lightcurve', 'FFI'}
Type of files queried at MAST.
cadence : str
Desired cadence (`long`, `short`, `any`)
mission : str, list of str
'Kepler', 'K2', or 'TESS'. By default, all will be returned.
quarter, campaign, sector : int, list of ints
Kepler Quarter, K2 Campaign, or TESS Sector number.
By default all quarters/campaigns/sectors will be returned.
month : 1, 2, 3, 4 or list of int
For Kepler's prime mission, there are three short-cadence
TargetPixelFiles for each quarter, each covering one month.
Hence, if cadence='short' you can specify month=1, 2, 3, or 4.
By default all months will be returned.
limit : int
Maximum number of products to return
Returns
-------
SearchResult : :class:`SearchResult` object.
"""
observations = _query_mast(target, project=mission, radius=radius)
if len(observations) == 0:
raise SearchError('No data found for target "{}".'.format(target))
# Light curves and target pixel files
if filetype.lower() != 'ffi':
from astroquery.mast import Observations
products = Observations.get_product_list(observations)
result = join(products, observations, keys="obs_id", join_type='left',
uniq_col_name='{col_name}{table_name}', table_names=['', '_2'])
result.sort(['distance', 'obs_id'])
masked_result = _filter_products(result, filetype=filetype,
campaign=campaign, quarter=quarter,
cadence=cadence, project=mission,
month=month, sector=sector, limit=limit)
return SearchResult(masked_result)
# Full Frame Images
else:
cutouts = []
for idx in np.where(['TESS FFI' in t for t in observations['target_name']])[0]:
# if target passed in is a SkyCoord object, convert to RA, dec pair
if isinstance(target, SkyCoord):
target = '{}, {}'.format(target.ra.deg, target.dec.deg)
# pull sector numbers
s = observations['sequence_number'][idx]
# if the desired sector is available, add a row
if s in np.atleast_1d(sector) or sector is None:
cutouts.append({'description': 'TESS FFI Cutout (sector {})'.format(s),
'target_name': str(target),
'targetid': str(target),
'productFilename': 'n/a',
'distance': 0.0,
'sequence_number': s}
)
if len(cutouts) > 0:
masked_result = Table(cutouts)
masked_result.sort(['distance', 'sequence_number'])
else:
masked_result = None
return SearchResult(masked_result)
def tasoc_lc(self):
"""
Grabs the T'DA available light curves for your target.
For more information, see the TASOC light curve documentation: https://tasoc.dk/code/.
Parameters
----------
Attributes
----------
tasoc_header :
tasoc_tpf : np.2darray
tasoc_aperture : np.2darray
tasoc_time : np.array
tasoc_quality : np.array
tasoc_timecorr : np.array
tasoc_cadenceno : np.array
tasoc_flux_raw : np.array
tasoc_flux_raw_err : np.array
tasoc_flux_corr : np.array
tasoc_flux_corr_err : np.array
tasoc_flux_bkg : np.array
tasoc_pixel_quality : np.array
Quality flags for the data; use these not `tasoc_quality`.
tasoc_pos_corr1 : np.array
tasoc_pos_corr2 : np.array
tasoc_mom_centr1 : np.array
tasoc_mom_centr2 : np.array
"""
products = Observations.query_object(objectname="TIC" + str(self.tic))
column = np.where((products['provenance_name'] == 'TASOC')
& (products['target_name'] == str(self.tic))
& (products['sequence_number'] == self.sector))[0]
if len(column) > 0:
download = Observations.get_product_list(products[column])
manifest = Observations.download_products(
download, download_dir=self.download_dir)
self.tasoc_path = manifest["Local Path"].data[0]
hdu = fits.open(self.tasoc_path)
self.tasoc_header = hdu[0].header
self.tasoc_tpf = hdu[2].data
self.tasoc_aperture = hdu[3].data
self.tasoc_time = hdu[1].data['TIME']
self.tasoc_quality = hdu[1].data['QUALITY']
self.tasoc_timecorr = hdu[1].data['TIMECORR']
self.tasoc_cadenceno = hdu[1].data['CADENCENO']
self.tasoc_flux_raw = hdu[1].data['FLUX_RAW']
self.tasoc_flux_bkg = hdu[1].data['FLUX_BKG']
self.tasoc_flux_corr = hdu[1].data['FLUX_CORR']
self.tasoc_pos_corr1 = hdu[1].data['POS_CORR1']
self.tasoc_pos_corr2 = hdu[1].data['POS_CORR2']
self.tasoc_mom_centr1 = hdu[1].data['MOM_CENTR1']
self.tasoc_mom_centr2 = hdu[1].data['MOM_CENTR2']
self.tasoc_pixel_quality = hdu[1].data['PIXEL_QUALITY']
self.tasoc_flux_raw_err = hdu[1].data['FLUX_RAW_ERR']
self.tasoc_flux_corr_err = hdu[1].data['FLUX_CORR_ERR']
else:
raise SearchError("No TASOC light curve found.")
#Goals: Obtain all time series data from Kepler and TESS on a given target
#using astroquery.
star_name = "L98-59"
#Use Astroquery to find all dvt products
#Start by getting all time series observations at exactly the coordinates of our star.
observations = Observations.query_object(star_name, radius="0 deg")
obs_wanted = observations['dataproduct_type'] == 'timeseries'
print(observations[obs_wanted]['obs_collection', 'project', 'obs_id'])
#Get all Products and determine which have DV files.
#Then plot the time series for each that has DVT files.
data_products = Observations.get_product_list(observations[obs_wanted])
files_wanted = (data_products["productSubGroupDescription"] == "DVT") | \
(data_products["productSubGroupDescription"] == "DVM") | \
(data_products["productSubGroupDescription"] == "DVS") | \
(data_products["productSubGroupDescription"] == "DVR")
#files_wanted = list(
# map( lambda x: x[-8:] == 'dvt.fits', data_products['productFilename'] ))
print(data_products["productFilename"][files_wanted])
manifest = Observations.download_products(data_products[files_wanted])
#%%
#!/usr/bin/env python3
from astroquery.mast import Observations
obs = Observations.query_criteria(
dataproduct_type=['image'],
project='HST',
instrument_name='ACS/WFC',
filters='F555W',
calib_level=3,
)
print("Observations: ", len(obs))
products = Observations.get_product_list(obs)
print("Products: ", len(products))
filtered_products = Observations.filter_products(
products,
productType='SCIENCE',
extension='fits',
description='DADS FLT file - Calibrated exposure ACS/WFC3/STIS/COS',
)
print("Filtered products: ", len(filtered_products))
print("Downloading first 3 products")
Observations.download_products(filtered_products[0:3])
if len(sectorsNb) == 0:
sectorList.append("")
elif len(sectorsNb) == 1:
sectorList.append(sectorsNb[0])
else:
sectorList.append(",".join(["{}".format(s) for s in sectorsNb]))
logging.info("TESS observations for {} will be in\n{}".format(
targetName, sectorsTxt))
if proceed == False:
logging.info("Retrieved info: name={}, obsID={}".format(
targetName, "N/A"))
continue
dataProductsByObservation = Observations.get_product_list(obsTable)
obsID = obsTable["obsid"]
logging.debug("Dumping obsTable info: {}".format(obsTable))
if len(obsID) > 1:
logging.info("There is more than 1 obs to download: {}".format(obsID))
#raise RuntimeError("You'll have to check this one by hand, there are objects close by.")
sec_ = []
for obsrow in obsTable["obs_id"]:
sec_.append(str(int(obsrow.split("-s")[1].split("-")[0])))
sectorList.append(",".join(sec_))
pathList.append("|".join(obsTable["obs_id"]))
elif len(obsID) == 0:
raise RuntimeError(
"No observation found --> you should change something")
def get_products_table(query_tab, extensions=['RAW'], use_astroquery=True):
"""
Get a new table with the association products
"""
from astropy import table
from . import utils
obsid = ','.join(['{0}'.format(o) for o in query_tab['obsid']])
if use_astroquery:
try:
from astroquery.mast import Observations
prod_tab = Observations.get_product_list(obsid)
force_manual = False
except:
force_manual = True
else:
force_manual = True
if force_manual:
print('Query with `utils.mastQuery`')
request = {
'service': 'Mast.Caom.Products',
'params': {
'obsid': obsid
},
'format': 'json',
'pagesize': 10000,
'page': 1
}
headers, outString = utils.mastQuery(request)
outData = json.loads(outString)
prod_tab = utils.mastJson2Table(outData)
if hasattr(prod_tab['parent_obsid'], 'filled'):
obsint = np.cast[int](prod_tab['parent_obsid'].filled('0'))
else:
obsint = np.cast[int](prod_tab['parent_obsid'])
if obsint.sum() == 0:
print(
'MAST product database problem with ``parent_obsid``, try one-by-one...'
)
# Problem with database, so query one by one
prods = []
if force_manual:
for obs in query_tab['obsid']:
request['params']['obsid'] = '{0}'.format(obs)
headers, outString = utils.mastQuery(request)
outData = json.loads(outString)
prod = utils.mastJson2Table(outData)
prod.remove_column('parent_obsid')
prod['parent_obsid'] = '{0}'.format(obs)
prods.append(prod)
else:
for obs in query_tab['obsid']:
prod = Observations.get_product_list('{0}'.format(obs))
prod.remove_column('parent_obsid')
prod['parent_obsid'] = '{0}'.format(obs)
prods.append(prod)
prod_tab = table.vstack(prods)
prod_tab.rename_column('parent_obsid', 'obsid')
if query_tab['obsid'].dtype != prod_tab['obsid'].dtype:
prod_tab['obsid'] = np.cast[query_tab['obsid']](prod_tab['obsid'])
prod_tab.remove_column('proposal_id')
prod_tab.rename_column('obs_id', 'observation_id')
col = 'productSubGroupDescription'
ext = np.vstack([prod_tab[col] == e for e in extensions]).sum(axis=0) > 0
full_tab = table.join(prod_tab[ext],
query_tab,
keys='obsid',
join_type='inner')
full_tab.sort(['observation_id'])
return full_tab
