def locate_postcard(self, local):
""" Finds the eleanor postcard, if available, this star falls on.
Attributes
----------
postcard : str
postcard_bkg : str
postcard_path : str
position_on_postcard : list
all_postcards : list
mast_list : astropy.table.Table
"""
self.mast_list = None
info_str = "{0:04d}-{1}-{2}-{3}".format(self.sector, self.camera,
self.chip, "cal")
postcard_fmt = "postcard-s{0}-{{0:04d}}-{{1:04d}}"
postcard_fmt = postcard_fmt.format(info_str)
eleanorpath = os.path.dirname(__file__)
guide_url = eleanorpath + '/postcard_centers.txt'
guide = Table.read(guide_url, format="ascii")
col, row = self.position_on_chip[0], self.position_on_chip[1]
post_args = np.where((np.abs(guide['x'].data - col) <= 100)
& (np.abs(guide['y'].data - row) <= 100))
post_cens = guide[post_args]
# Finds the mostest closest postcard
closest_x, closest_y = np.argmin(
np.abs(post_cens['x'] - col)), np.argmin(
np.abs(post_cens['y'] - row))
self.postcard = postcard_fmt.format(post_cens['x'].data[closest_x],
post_cens['y'].data[closest_y])
# Keeps track of all postcards that the star falls on
all_postcards = []
for i in range(len(post_cens)):
name = postcard_fmt.format(post_cens['x'].data[i],
post_cens['y'].data[i])
all_postcards.append(name)
self.all_postcards = np.array(all_postcards)
if local == False:
postcard_obs = Observations.query_criteria(
provenance_name="ELEANOR",
target_name=self.postcard,
obs_collection="HLSP")
if len(postcard_obs) > 0:
product_list = Observations.get_product_list(postcard_obs)
self.pointing = check_pointing(self.sector, self.camera,
self.chip, self.pm_dir)
if self.pointing is None:
extension = ["pc.fits", "bkg.fits", "pm.txt"]
else:
extension = ["pc.fits", "bkg.fits"]
results = Observations.download_products(
product_list,
extension=extension,
download_dir=self.fn_dir)
postcard_path = results['Local Path'][0]
self.postcard_path = '/'.join(
e for e in postcard_path.split('/')[:-1])
self.postcard = results['Local Path'][1].split('/')[-1]
self.postcard_bkg = results['Local Path'][0].split('/')[-1]
self.mast_results = results
self.cutout = None # Attribute for TessCut only
# only downloaded the pointing model if the search for it above failed, so only
# update it in that case here
if self.pointing is None:
self.pm_dir = self.postcard_path
self.pointing = check_pointing(self.sector, self.camera,
self.chip, self.pm_dir)
else:
print(
"No eleanor postcard has been made for your target (yet). Using TessCut instead."
)
self.locate_with_tesscut()
else:
self.cutout = None #Attribute for TessCut only
self.postcard_bkg = 'hlsp_eleanor_tess_ffi_' + self.postcard + '_tess_v2_bkg.fits'
self.postcard = 'hlsp_eleanor_tess_ffi_' + self.postcard + '_tess_v2_pc.fits'
self.pointing = check_pointing(self.sector, self.camera, self.chip,
self.pm_dir)
from astroquery.mast import Catalogs
from astroquery.vizier import Vizier
from astroquery.mast import Observations
import urllib.request
import os
filter_wide = ["F555W","F606W"]
for f in filter_wide:
if not os.path.isdir("data/" + f + "/"):
os.makedirs("data/" + f + "/")
obsTable = Observations.query_criteria(calib_level= 3, dataproduct_type = 'image',
obs_collection = ["HLA"],
instrument_name = ["WFC3/UVIS"], filters = [f])
for url in obsTable['dataURL']:
if isinstance(url, type(obsTable[7]['dataURL'])):
if url[-5:] == str(".fits"):
file = "data/" + f + "/" + str(url[50:])
urllib.request.urlretrieve(url,file)
import configparser
config = configparser.ConfigParser()
config.read(os.path.expanduser('~/.aws/credentials'))
os.environ["AWS_ACCESS_KEY_ID"] = config.get('default', 'aws_access_key_id')
os.environ["AWS_SECRET_ACCESS_KEY"] = config.get('default',
'aws_secret_access_key')
# Create a directory for downloaded files
if not os.path.exists('DATA'):
os.makedirs('DATA')
# This downloads all the F160W DRZ images from CANDELS in the GOODS-South field
print('\nQuerying the MAST archive.\n')
obsTable = Observations.query_criteria(obs_collection='HST',
filters='F160W',
instrument_name='WFC3/IR',
proposal_id=['12062', '12061', '12062'])
# Grab the list of available data products for these observations
products = Observations.get_product_list(obsTable)
# Select only drizzled (DRZ) files
filtered = Observations.filter_products(products,
mrp_only=False,
productSubGroupDescription='DRZ')
# Enable 'S3 mode' for module which will return S3-like URLs for FITs files
# e.g. s3://stpubdata/hst/public/icde/icde43l0q/icde43l0q_drz.fits
Observations.enable_s3_hst_dataset()
# Grab the S3 URLs for each of the observations
def _query_mast(target, radius=None, project=('Kepler', 'K2', 'TESS')):
"""Helper function which wraps `astroquery.mast.Observations.query_criteria()`
to returns a table of all Kepler or K2 observations of a given target.
Parameters
----------
target : str, int, or `astropy.coordinates.SkyCoord` object
See docstrings above.
radius : float or `astropy.units.Quantity` object
Conesearch radius. If a float is given it will be assumed to be in
units of arcseconds. If `None` then we default to 0.0001 arcsec.
project : str, list of str
'Kepler', 'K2', and/or 'TESS'.
Returns
-------
obs : astropy.Table
Table detailing the available observations on MAST.
"""
# If passed a SkyCoord, convert it to an RA and Dec
if isinstance(target, SkyCoord):
target = '{}, {}'.format(target.ra.deg, target.dec.deg)
project = np.atleast_1d(project)
if radius is None:
radius = .0001 * u.arcsec
elif not isinstance(radius, u.quantity.Quantity):
radius = radius * u.arcsec
try:
# If `target` looks like a KIC or EPIC ID, we will pass the exact
# `target_name` under which MAST will know the object to prevent
# source confusion (see GitHub issue #148).
target = int(target)
if (target > 0) and (target < 200000000):
target_name = 'kplr{:09d}'.format(target)
elif (target > 200000000) and (target < 300000000):
target_name = 'ktwo{:09d}'.format(target)
else:
raise ValueError(
"{:09d}: not in the KIC or EPIC ID range".format(target))
# query_criteria does not allow a cone search when target_name is passed in
# so first grab desired target with ~0 arcsecond radius
with warnings.catch_warnings():
# suppress misleading AstropyWarning
warnings.simplefilter('ignore', AstropyWarning)
from astroquery.mast import Observations
log.debug(
"Started querying MAST for observations within {} of target_name='{}'."
"".format(radius.to(u.arcsec), target_name))
target_obs = Observations.query_criteria(target_name=target_name,
radius=str(
radius.to(u.deg)),
project=project,
obs_collection=project)
if len(target_obs) == 0:
raise ValueError(
"No observations found for '{}'.".format(target_name))
# check if a cone search is being performed
# if yes, perform a cone search around coordinates of desired target
if radius < (0.1 * u.arcsec):
obs = target_obs
# astroquery does not return distance if target_name is given;
# we add it here so that the table returned always has this column.
obs['distance'] = 0.
else:
ra = target_obs['s_ra'][0]
dec = target_obs['s_dec'][0]
with warnings.catch_warnings():
# suppress misleading AstropyWarning
warnings.simplefilter('ignore', AstropyWarning)
from astroquery.mast import Observations
log.debug(
"Started querying MAST for observations within {} of coordinates='{} {}'."
"".format(radius.to(u.arcsec), ra, dec))
obs = Observations.query_criteria(coordinates='{} {}'.format(
ra, dec),
radius=str(radius.to(u.deg)),
project=project,
obs_collection=project)
obs.sort('distance')
return obs
except ValueError:
pass
# If `target` did not look like a KIC or EPIC ID, then we let MAST
# resolve the target name to a sky position. Convert radius from arcsec
# to degrees for query_criteria().
from astroquery.exceptions import ResolverError
try:
with warnings.catch_warnings():
# suppress misleading AstropyWarning
warnings.simplefilter('ignore', AstropyWarning)
from astroquery.mast import Observations
log.debug(
"Started querying MAST for observations within {} of objectname='{}'."
"".format(radius.to(u.arcsec), target))
obs = Observations.query_criteria(objectname=target,
radius=str(radius.to(u.deg)),
project=project,
obs_collection=project)
obs.sort('distance')
return obs
except ResolverError as exc:
raise SearchError(exc)
sector = 1
tpeak = 2458341.89227 # Julian Day
# %%
'''
### Querying MAST
#### Mission light curves
Here we choose the TESS mission (`obs_collection`) and query on our TIC ID and sector.
'''
# %%
mission_res = Observations.query_criteria(obs_collection="TESS",
target_name=tic_id,
sequence_number=sector)
mission_res
# %%
'''
#### TASOC light curves
MAST also hosts a variety of community contributed High Level Science Products (HLSPs), all of which are given the mission "HLSP". In this case we will specifically search for HLSPs in the TESS project, which will return the light curves provided by the TASOC (note the `provenance_name` of "TASOC").
'''
# %%
tasoc_res = Observations.query_criteria(target_name=tic_id,
obs_collection="HLSP",
project="TESS",
sequence_number=sector)
# ofile1.write(",".join(fieldnames1)+'\n')
for this_id in ids:
try:
target_name = this_id
radius = 0.2
catalogTIC = Catalogs.query_object(target_name, radius, catalog = "TIC")
numObj = "Number of TIC objects within %f deg of %s: %u" % (radius, target_name, len(catalogTIC))
where_dwarfs = np.where(catalogTIC['lumclass'] == 'DWARF')[0]
where_giants = np.where(catalogTIC['lumclass'] == 'GIANT')[0]
dwarfs = "Number of objects classified as 'DWARF' within %f deg of %s: %u" % (radius, target_name, len(where_dwarfs))
giants = "Number of objects classified as 'GIANT' within %f deg of %s: %u" % (radius, target_name, len(where_giants))
where_closest = np.argmin(catalogTIC['dstArcSec'])
closest = "Closest TIC ID to %s: TIC %s, seperation of %f arcsec. and a TESS mag. of %f" % (target_name, catalogTIC['ID'][where_closest], catalogTIC['dstArcSec'][where_closest], catalogTIC['Tmag'][where_closest])
#sectors_search = Observations.query_criteria(target_name=this_id, provenance_project='TASOC')
sectors_search = Observations.query_criteria(target_name=this_id, obs_collection="HLSP", filters="TESS",
t_exptime=[1799, 1801])
#print('Getting sectors')
#import pdb; pdb.set_trace()
sector_length = len(sectors_search)
if sector_length !=0:
sector = sectors_search['sequence_number']
sector_length = sector_length-1
print('Getting times')
#import pdb; pdb.set_trace()
time = gettimeflux_1800(this_id, str(sector[0]))[0]
print(time)
flux = gettimeflux_1800(this_id, str(sector[0]))[1]
Tmag = get_catalog_data(this_id)[0]
Teff = get_catalog_data(this_id)[1]
def get_hlsp_lightcurves(tic_id,
hlsp_products=('CDIPS', 'TASOC', 'PATHOS'),
download_dir=None,
verbose=True):
"""This downloads TESS HLSP light curves for a given TIC ID.
Parameters
----------
tic_id : str
The TIC ID of the object as a string.
hlsp_products : sequence of str
List of desired HLSP products to search. For instance, ["CDIPS"].
download_dir : str
Path of directory to which light-curve will be downloaded.
Returns
-------
lcfiles : list or None
List of light-curve file paths. None if none are found and downloaded.
"""
if not astroquery_dependency:
LOGERROR(
"The astroquery package is required for this function to work.")
return None
lcfiles = []
for hlsp in hlsp_products:
obs_table = Observations.query_criteria(target_name=tic_id,
provenance_name=hlsp)
if verbose:
LOGINFO(f'Found {len(obs_table)} {hlsp} light-curves.')
if len(obs_table) == 0:
if verbose:
LOGINFO("Did not find light-curves. Escaping.")
return None
# Get list of available products for this Observation.
cdips_products = Observations.get_product_list(obs_table)
# Download the products for this Observation.
manifest = Observations.download_products(cdips_products,
download_dir=download_dir)
if verbose:
LOGINFO("Done")
if len(manifest) >= 1:
lcfiles.append(list(manifest['Local Path']))
#
# flatten lcfiles list
#
if len(lcfiles) >= 1:
return_lcfiles = [item for sublist in lcfiles for item in sublist]
else:
return_lcfiles = None
return return_lcfiles
def run_query(box=None,
get_exptime=True,
rename_columns=DEFAULT_RENAME,
sort_column=['obs_id', 'filter'],
position_box=True,
base_query=DEFAULT_QUERY_ASTROQUERY.copy(),
**kwargs):
"""
Run MAST query with astroquery.mast.
All columns listed at https://mast.stsci.edu/api/v0/_c_a_o_mfields.html
can be used for the query.
position_box: query on s_ra / s_dec positions rather than position
coordinates
"""
# arguments
frame = inspect.currentframe()
msg = utils.log_function_arguments(None, frame, 'query.run_query')
import time
from astroquery.mast import Observations
from astropy.coordinates import SkyCoord
from astropy.io.misc import yaml
import astropy.units as u
query_args = {}
for k in base_query:
query_args[k] = base_query[k]
# JWST "expected data" won't have datasets to query for actual exptimes...
if 'obs_collection' in base_query:
if 'JWST' in base_query['obs_collection']:
get_exptime = False
for k in kwargs:
if k == 'instruments':
query_args['instrument_name'] = kwargs[k]
elif k == 'proposal_id':
query_args['proposal_id'] = ['{0}'.format(p) for p in kwargs[k]]
elif k == 'extensions':
continue
else:
query_args[k] = kwargs[k]
if (box is not None):
ra, dec, radius = box
#coo = SkyCoord(ra*u.deg, dec*u.deg)
#query_args['coordinates'] = coo
#query_args['radius'] = radius*u.arcmin
cosd = np.cos(box[1] / 180 * np.pi)
query_args['s_ra'] = box[0] + np.array([-1, 1]) * box[2] / 60 / cosd
query_args['s_dec'] = box[1] + np.array([-1, 1]) * box[2] / 60
try:
tab = Observations.query_criteria(**query_args)
except:
return query_args
tab.meta['qtime'] = time.ctime(), 'Query timestamp'
if box is not None:
tab.meta['boxra'] = ra, 'Query RA, degrees'
tab.meta['boxdec'] = dec, 'Query Decl., degrees'
tab.meta['boxrad'] = radius, 'Query radius, arcmin'
str_args = yaml.dump(query_args).replace('\n', ';;')
tab.meta[
'obsquery'] = str_args, 'Full query string, replace ;; with newline'
if len(tab) == 0:
return tab
tab = modify_table(tab,
get_exptime=get_exptime,
rename_columns=rename_columns,
sort_column=sort_column)
return tab
#!/usr/bin/env python3
from astroquery.mast import Observations
import IPython
import boto3
from s3_query import find_product_in_s3
Observations.enable_s3_hst_dataset()
obs = Observations.query_criteria(
dataproduct_type=['image'],
project='HST',
instrument_name='ACS/WFC',
filters='F555W',
calib_level=3,
)
print("Observations: ", len(obs))
products = Observations.get_product_list(obs)
print("Products: ", len(products))
filtered_products = Observations.filter_products(
products,
productType='SCIENCE',
extension='fits',
description='DADS FLT file - Calibrated exposure ACS/WFC3/STIS/COS',
)
print("Filtered products: ", len(filtered_products))
print()
from astroquery.mast import Observations
import numpy as np
import os,sys
search_radius = 120
## import files ##
inp0 = 'ALMA_z6qso_all.list'
data = np.loadtxt(inp0,comments='#',dtype={'names':('proj_id','name','ra','dec','PI'),'formats':('S32','S32','f16','f16','S32')})
proj_id = data["proj_id"]
name = data["name"]
ra = data["ra"]
dec = data["dec"]
for i in range(len(name)):
obs_table = Observations.query_criteria(coordinates="%3.5f %3.5f" %(ra[i],dec[i]), radius="%3.5f arcsec" % search_radius, intentType=["science","SCIENCE"], obs_collection=["HST"])
__,uidx = np.unique(obs_table['target_name'],return_index=True)
target_table = obs_table[uidx]['target_name','s_ra','s_dec','filters','t_exptime','proposal_id','dataURL','obsid']
if len(uidx) > 0:
os.system('mkdir download_sh/'+name[i])
for u in range(len(uidx)):
data_products = Observations.get_product_list(target_table[u][-1])
Observations.download_products(data_products, calib_level=[2,3], productType="SCIENCE", curl_flag=True,mrp_only=True,download_dir='download_sh/'+name[i])
def _query_mast(target, radius=None, cadence='long'):
"""Helper function which wraps `astroquery.mast.Observations.query_criteria()`
to returns a table of all Kepler or K2 observations of a given target.
Parameters
----------
target : str, int, or `astropy.coordinates.SkyCoord` object
See docstrings above.
radius : float or `astropy.units.Quantity` object
Conesearch radius. If a float is given it will be assumed to be in
units of arcseconds. If `None` then we default to 0.0001 arcsec.
cadence: 'short' or 'long'
Specify short (1-min) or long (30-min) cadence data.
Returns
-------
obs : astropy.Table
Table detailing the available observations on MAST.
"""
# If passed a SkyCoord, convert it to an RA and Dec
if isinstance(target, SkyCoord):
target = '{}, {}'.format(target.ra.deg, target.dec.deg)
if radius is None:
radius = .0001 * u.arcsec
elif not isinstance(radius, u.quantity.Quantity):
radius = radius * u.arcsec
try:
# If `target` looks like a KIC or EPIC ID, we will pass the exact
# `target_name` under which MAST will know the object to prevent
# source confusion (see GitHub issue #148).
target = int(target)
if (target > 0) and (target < 200000000):
target_name = 'kplr{:09d}'.format(target)
elif (target > 200000000) and (target < 300000000):
target_name = 'ktwo{:09d}'.format(target)
else:
raise ValueError("{:09d}: not in the KIC or EPIC ID range".format(target))
# query_criteria does not allow a cone search when target_name is passed in
# so first grab desired target with ~0 arcsecond radius
target_obs = Observations.query_criteria(target_name=target_name,
radius=str(radius.to(u.deg)),
project=["Kepler", "K2"],
obs_collection=["Kepler", "K2"])
if len(target_obs) == 0:
raise ValueError("No observations found for {}".format(target_name))
# check if a cone search is being performed
# if yes, perform a cone search around coordinates of desired target
if radius < (0.1 * u.arcsec):
obs = target_obs
# astroquery does not return distance if target_name is given;
# we add it here so that the table returned always has this column.
obs['distance'] = 0.
else:
ra = target_obs['s_ra'][0]
dec = target_obs['s_dec'][0]
obs = Observations.query_criteria(coordinates='{} {}'.format(ra, dec),
radius=str(radius.to(u.deg)),
project=["Kepler", "K2"],
obs_collection=["Kepler", "K2"])
except ValueError:
# If `target` did not look like a KIC or EPIC ID, then we let MAST
# resolve the target name to a sky position. Convert radius from arcsec
# to degrees for query_criteria().
try:
obs = Observations.query_criteria(objectname=target,
radius=str(radius.to(u.deg)),
project=["Kepler", "K2"],
obs_collection=["Kepler", "K2"])
except ResolverError as exc:
raise SearchError(exc)
obs.sort('distance') # ensure table returned is sorted by distance
return obs
def retrieve_observation(obsid, suffix=['FLC'], archive=False,clobber=False):
"""Simple interface for retrieving an observation from the MAST archive
If the input obsid is for an association, it will request all members with
the specified suffixes.
Parameters
-----------
obsid : string
ID for observation to be retrieved from the MAST archive. Only the
IPPSSOOT (rootname) of exposure or ASN needs to be provided; eg., ib6v06060.
suffix : list
List containing suffixes of files which should be requested from MAST.
path : string
Directory to use for writing out downloaded files. If `None` (default),
the current working directory will be used.
archive : Boolean
Retain copies of the downloaded files in the astroquery created sub-directories? Default is 'False'.
clobber : Boolean
Download and Overwrite existing files? Default is 'False'.
Returns
-------
local_files : list
List of filenames
"""
local_files = []
# Query MAST for the data with an observation type of either "science" or "calibration"
obsTable = Observations.query_criteria(obs_id=obsid, obstype='all')
# Catch the case where no files are found for download
if len(obsTable) == 0:
log.info("WARNING: Query for {} returned NO RESULTS!".format(obsid))
return local_files
dpobs = Observations.get_product_list(obsTable)
dataProductsByID = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# After the filtering has been done, ensure there is still data in the table for download.
# If the table is empty, look for FLT images in lieu of FLC images. Only want one
# or the other (not both!), so just do the filtering again.
if len(dataProductsByID) == 0:
log.info("WARNING: No FLC files found for {} - will look for FLT files instead.".format(obsid))
suffix = ['FLT']
dataProductsByID = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# If still no data, then return. An exception will eventually be thrown in
# the higher level code.
if len(dataProductsByID) == 0:
log.info("WARNING: No FLC or FLT files found for {}.".format(obsid))
return local_files
allImages = []
for tableLine in dataProductsByID:
allImages.append(tableLine['productFilename'])
log.info(allImages)
if not clobber:
rowsToRemove = []
for rowCtr in range(0,len(dataProductsByID)):
if os.path.exists(dataProductsByID[rowCtr]['productFilename']):
log.info("{} already exists. File download skipped.".format(dataProductsByID[rowCtr]['productFilename']))
rowsToRemove.append(rowCtr)
if rowsToRemove:
rowsToRemove.reverse()
for rowNum in rowsToRemove:
dataProductsByID.remove_row(rowNum)
manifest = Observations.download_products(dataProductsByID, mrp_only=False)
if not clobber:
rowsToRemove.reverse()
for rownum in rowsToRemove:
if not manifest:
local_files = allImages
return local_files
else:
manifest.insert_row(rownum,vals=[allImages[rownum],"LOCAL","None","None"])
download_dir = None
for file,fileStatus in zip(manifest['Local Path'],manifest['Status']):
if fileStatus != "LOCAL":
# Identify what sub-directory was created by astroquery for the download
if download_dir is None:
file_path = file.split(os.sep)
file_path.remove('.')
download_dir = file_path[0]
# Move or copy downloaded file to current directory
local_file = os.path.abspath(os.path.basename(file))
if archive:
shutil.copy(file, local_file)
else:
shutil.move(file, local_file)
# Record what files were downloaded and their current location
local_files.append(os.path.basename(local_file))
else:
local_files.append(file)
if not archive:
# Remove astroquery created sub-directories
shutil.rmtree(download_dir)
return local_files
def get_mast_file_list(star_id, provenance, sequence_name):
"""
Queries MAST to retrieve archive data based on object name (``star_id``),
data provenance, and sequence name.
Parameters
----------
star_id: int, str
EPIC ID of the star.
provenance: str
Provenance name in MAST archive, e.g., ``'K2'``, ``'EVEREST'``,
``'K2SFF'``.
sequence_name: str
Campaign number.
Returns
-------
file_names: list of str
A Python list of all the file names retrieved from archive.
"""
star_id = str(star_id)
target_name = '*' + star_id.split()[-1]
# Make sure that there are data for the criteria
mission = _MISSION_MAPPING[provenance]
print("\n===== Retrieving data for observation: =====")
print("** star_id: {}\n** provenance: {}\n** sequence_name: {}".format(
star_id, provenance, sequence_name))
if sequence_name == '*':
sequence_name = ''
obs_count = Observations.query_criteria_count(
obs_collection=mission,
dataproduct_type=["timeseries"],
instrument_name="Kepler",
objectname=star_id,
target_name=target_name,
project="K2",
provenance_name=provenance,
sequence_number=sequence_name + '*')
if obs_count == 0:
raise RuntimeError("No data found in archive.")
obs_table = Observations.query_criteria(obs_collection=mission,
dataproduct_type=["timeseries"],
instrument_name="Kepler",
objectname=star_id,
target_name=target_name,
project="K2",
provenance_name=provenance,
sequence_number=sequence_name +
'*')
data_products = Observations.get_product_list(obs_table)
lc_mask = [
"lightcurve" in x or "light curve" in x
for x in map(str.lower, data_products['description'])
]
if not any(lc_mask):
raise RuntimeError("Retrieved data products do not contain light "
"curve data.")
data_products = data_products[lc_mask] # keep only rows with light curves
manifest = Observations.download_products(data_products)
files = list(manifest['Local Path']) # get local file names
# sort results:
idx = np.argsort(files)
file_names = [files[i] for i in idx]
print("Download Status: SUCCESS\n")
return file_names
if simbadResults is not None: break
if simbadResults is not None:
coord = SkyCoord(simbadResults["RA"][0],
simbadResults["DEC"][0],
unit=[u.hourangle, u.deg])
ras.append(coord.ra.value)
raUnit = coord.ra.unit
decs.append(coord.dec.value)
decUnit = coord.dec.unit
else:
ras.append(StarListTics["ra"][where_closest])
decs.append(StarListTics["dec"][where_closest])
#Retrieve data
obsTable = Observations.query_criteria(filters=["TESS"],
objectname=targetName,
dataproduct_type=["TIMESERIES"],
radius="0.01 deg")
proceed = True
if len(obsTable) == 0:
proceed = False
obsIDList.append(np.nan)
pathList.append("NA")
sectorsTxt, sectorsNb = TESSSectorLookUp(targetName)
if len(sectorsNb) == 0:
sectorList.append("")
elif len(sectorsNb) == 1:
sectorList.append(sectorsNb[0])
else:
sectorList.append(",".join(["{}".format(s) for s in sectorsNb]))
def main():
# Read in the Pantheon+ catalog from Ben
adap_dir = home + '/Documents/adap2021/'
pantheon_datadir = adap_dir + 'pantheon_data/'
cat = np.genfromtxt(adap_dir + 'pantheon_plus.csv',
dtype=None, names=True, delimiter=',', encoding='ascii')
print("Read in Pantheon+ catalog with the following header names:")
print(cat.dtype.names)
num_orig_cols = len(cat.dtype.names)
# Open a new file to write an updated catalog
# Adds the following columns
# HST data
# GALEX data
# if yes to any of the above observatories then give
# Inst/Camera field, and filters.
# if no then leave these cols blank.
fh = open(adap_dir + 'pantheon_plus_data.csv', 'w')
# Write header
fh.write("Serial_num,CID,CIDint,IDSURVEY,zHEL,zHELERR,zCMB,zCMBERR,zHD,zHDERR," +\
"HOST_LOGMASS,HOST_LOGMASS_ERR,RA,DEC,HOST_RA,HOST_DEC," +\
"HST_data,Inst/Cam,Filters" + "\n")
# Loop over all objects in the catalog
# and search for HST data at the SN and Host location
for i in range(27, len(cat)): #tqdm(range(len(cat)), desc="Processing SN"):
# Get coords
sn_ra = cat['RA'][i]
sn_dec = cat['DEC'][i]
host_ra = cat['HOST_RA'][i]
host_dec = cat['HOST_DEC'][i]
# Print info
#print(f"{bcolors.CYAN}")
#print("SN identifier:", cat['CID'][i], " at:", sn_ra, sn_dec)
#print("Host galaxy coords:", host_ra, host_dec)
#print(f"{bcolors.ENDC}")
# Set up query
sn_coords = SkyCoord(ra=sn_ra*u.degree, dec=sn_dec*u.degree, frame='icrs')
print("SN coordinates:", sn_coords)
obs_table = Observations.query_criteria(coordinates=sn_coords, radius="0.5 arcsec", \
intentType='science', obs_collection=['HST'])
#print(obs_table)
print(obs_table.columns)
print("\nRows in obs table:", len(obs_table))
print("HST filters available for this SN:")
all_instr = np.unique(obs_table['instrument_name'])
print(all_instr)
print("--------------------------------------\n")
sys.exit(0)
# Download any existing wfc3 data
for r in range(len(obs_table)):
instr = obs_table['instrument_name'][r]
if 'WFC3' in instr:
data_products = Observations.get_product_list(obs_table[r])
Observations.download_products(data_products, download_dir=pantheon_datadir,
productType="SCIENCE", mrp_only=True)
sys.exit(0)
# Now loop over all the observations
ra_one = sn_ra
dec_one = sn_dec
dist = []
inst_cam = []
exptimes = []
filt = []
for o in range(len(obs_table)):
ra_two = obs_table['s_ra'][o]
dec_two = obs_table['s_dec'][o]
dist_to_sn = np.arccos(np.cos(dec_one*np.pi/180) * \
np.cos(dec_two*np.pi/180) * np.cos(ra_one*np.pi/180 - ra_two*np.pi/180) + \
np.sin(dec_one*np.pi/180) * np.sin(dec_two*np.pi/180))
# print("{:.7}".format(ra_two), "{:.7}".format(dec_two))
#print("Distance from SN [arcsec]:", "{:.5}".format(dist_to_sn * (180/np.pi) * 3600), \
# "Inst/Cam:", obs_table['instrument_name'][o], "Filter(s):", obs_table['filters'][o], \
# "ExpTime:", obs_table['t_exptime'][o])
dist.append(dist_to_sn * (180/np.pi) * 3600) # The dist returned by the line above is in radians
inst_cam.append(obs_table['instrument_name'][o])
filt.append(obs_table['filters'][o])
exptimes.append(obs_table['t_exptime'][o])
# Add to original catalog
# Need to loop over the original row to do this
for j in range(num_orig_cols):
fh.write(str(cat[i][j]) + ',')
# Now add the new cols
# First convert to numpy arrays
inst_cam = np.unique(np.asarray(inst_cam))
filt = np.unique(np.asarray(filt))
if len(obs_table) > 0:
hst_data = True
else:
hst_data = False
fh.write(str(hst_data) + ",")
if len(inst_cam) > 1:
for w in range(len(inst_cam)):
fh.write(str(inst_cam[w]) + ";")
fh.write(",")
else:
fh.write(str(inst_cam) + ",")
if len(filt) > 1:
for v in range(len(filt)):
fh.write(str(filt[v]) + ";")
else:
fh.write(str(filt))
fh.write("\n")
# Check that the distances are within FoV of the instrument specified
# dist = np.asarray(dist) * (180/np.pi) * 3600 # radians to degrees to arcseconds
fh.close()
return None
#!/usr/bin/env python3
import numpy
import sys
import boto3
import astropy.io.fits
from astroquery.mast import Observations
s3 = boto3.resource('s3')
stpubdata = s3.Bucket('stpubdata')
Observations.enable_s3_hst_dataset()
obs = Observations.query_criteria(
project='HST',
instrument_name='ACS/WFC',
)
print("Observations: ", len(obs), file=sys.stderr)
proposal_ids = numpy.unique(obs['proposal_id'])
print("Proposals: ", len(proposal_ids), file=sys.stderr)
shape_tally = {}
for prop in proposal_ids:
proposal_obs = Observations.query_criteria(
project='HST',
instrument_name='ACS/WFC',
proposal_id=[prop],
)
def retrieve_observation(obsid, suffix=['FLC'], archive=False, clobber=False):
"""Simple interface for retrieving an observation from the MAST archive
If the input obsid is for an association, it will request all members with
the specified suffixes.
Parameters
-----------
obsid : string
ID for observation to be retrieved from the MAST archive. Only the
IPPSSOOT (rootname) of exposure or ASN needs to be provided; eg.,
ib6v06060.
suffix : list, optional
List containing suffixes of files which should be requested from MAST.
Default value "['FLC']".
archive : Boolean, optional
Retain copies of the downloaded files in the astroquery created
sub-directories? Default is "False".
clobber : Boolean, optional
Download and Overwrite existing files? Default is "False".
Returns
-------
local_files : list
List of filenames
"""
local_files = []
if Observations is None:
log.warning("The astroquery package was not found. No files retrieved!")
return local_files
# Query MAST for the data with an observation type of either "science" or
# "calibration"
obs_table = Observations.query_criteria(obs_id=obsid, obstype='all')
# Catch the case where no files are found for download
if not obs_table:
log.info("WARNING: Query for {} returned NO RESULTS!".format(obsid))
return local_files
dpobs = Observations.get_product_list(obs_table)
data_products_by_id = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# After the filtering has been done, ensure there is still data in the
# table for download. If the table is empty, look for FLT images in lieu
# of FLC images. Only want one or the other (not both!), so just do the
# filtering again.
if not data_products_by_id:
log.info("WARNING: No FLC files found for {} - will look for FLT "
"files instead.".format(obsid))
suffix = ['FLT']
data_products_by_id = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# If still no data, then return. An exception will eventually be
# thrown in the higher level code.
if not data_products_by_id:
log.info(
"WARNING: No FLC or FLT files found for {}.".format(obsid))
return local_files
all_images = data_products_by_id['productFilename'].tolist()
log.info(all_images)
if not clobber:
rows_to_remove = []
for row_idx, row in enumerate(data_products_by_id):
fname = row['productFilename']
if os.path.isfile(fname):
log.info(fname + " already exists. File download skipped.")
rows_to_remove.append(row_idx)
data_products_by_id.remove_rows(rows_to_remove)
manifest = Observations.download_products(data_products_by_id,
mrp_only=False)
if not clobber:
for rownum in rows_to_remove[::-1]:
if manifest:
manifest.insert_row(rownum,
vals=[all_images[rownum], "LOCAL", "None", "None"])
else:
return all_images
download_dir = None
for file, file_status in zip(manifest['Local Path'], manifest['Status']):
if file_status != "LOCAL":
# Identify what sub-directory was created by astroquery for the
# download
if download_dir is None:
download_dir = os.path.dirname(os.path.abspath(file))
# Move or copy downloaded file to current directory
local_file = os.path.abspath(os.path.basename(file))
if archive:
shutil.copy(file, local_file)
else:
shutil.move(file, local_file)
# Record what files were downloaded and their current location
local_files.append(os.path.basename(local_file))
else:
local_files.append(file)
if not archive:
# Remove astroquery created sub-directories
shutil.rmtree(download_dir)
return local_files
import time
import boto3
import numpy as np
from astropy.io import fits
from astroquery.mast import Observations
# NOTE: Use your own key values here.
os.environ['AWS_ACCESS_KEY_ID'] = 'somekey'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'somesecret'
# NOTE: Change TESS observation ID as needed.
obs_id = 'tess-s0001-1-1'
# Find full frame dataset for the observation ID.
obs_table = Observations.query_criteria(obs_id=obs_id)
products = Observations.get_product_list(obs_table)
filtered = Observations.filter_products(products,
productSubGroupDescription="FFIC",
mrp_only=False)
# Set up AWS S3 bucket to pull data from.
Observations.enable_cloud_dataset()
s3_urls = Observations.get_cloud_uris(filtered, include_bucket=False)
s3 = boto3.resource('s3')
bucket = s3.Bucket('stpubdata')
def time_mean():
"""Loop through full frame files, extract a subarray, and calculate mean.
This must be done in a way that the file is deleted as soon as it is
def download_lc(target_name: str, flux_type='PDCSAP', mission: str = 'TESS',sigma_clip=4,iters=1,do_pca : bool = False,do_psf :bool= False) -> Tuple[
LightCurve, Union[List[Figure],None]]:
"""
Downloads a light curve using the TESS mission. If the star has been observed in the SC mode, it
will download the original light curve from MAST. You can also choose the flux type you want to use.
If it wasn't observed in SC mode, it will try to extract a light curve from the FFIs if the target has
been observed by TESS.
You can also download light curves of stars that are observed by the K2 or Kepler mission, by setting
the mission parameter.
:param target_name: Name of the target. You can either provide the TIC ID (TIC ...), Kepler ID (KIC ...), K2 ID(EPIC ...) or a name that is resolvable by Simbad.
:param flux_type: Type of flux in the SC mode. Can be either PDCSAP or SAP or PSF for long cadence data
:param mission: Mission from which the light curves are extracted. By default TESS only is used. You can consider all missions by passing 'all' (TESS, Kepler, K2)
:param sigma_clip: Sigma clip parameter. Defines the number of standard deviations that are clipped.
:param iters: Iterations for the sigma clipping
:return: lightkurve.LightCurve object and validation page if extracted from FFI
"""
chosen_mission = [mission] if mission != 'all' else ('Kepler', 'K2', 'TESS')
mprint(f"Searching processed light curves for {target_name} on mission(s) {','.join(chosen_mission)} ... ", log)
if chosen_mission == ['TESS']:
if target_name.startswith('TIC'):
tic_id = re.findall(r'\d+', target_name)
if len(tic_id) == 0:
raise ValueError(ctext("A Tic ID needs to consist of TIC and a number!", error))
tic_id = int(tic_id[0])
else:
mprint(f"Resolving {target_name} to TIC using MAST ...",log)
try:
tic_id = Catalogs.query_object(target_name,catalog='TIC',radius=0.003)[0]['ID']
except KeyError:
raise ValueError(ctext(f"No TESS observations available for {target_name}", error))
mprint(f"TIC ID for {target_name}: TIC {tic_id}",log)
o = Observations.query_criteria(objectname=target_name, radius=str(0 * u.deg), project='TESS',
obs_collection='TESS').to_pandas()
if len(o) > 0 and len(o[o.target_name != 'TESS FFI']) > 0:
mprint(f"Short cadence observations available for {target_name}. Downloading ...",info)
res = search_lightcurvefile(target_name, mission=chosen_mission)
else: #Only FFI available
mprint(f"No short cadence data available for {target_name}, extracting from FFI ...",info)
lc, fig,_ = cut_ffi(tic_id,sigma_clip,iters,do_pca,do_psf,flux_type)
mprint(f"Total observation length: {'%.2f' % (lc.time[-1] - lc.time[0])} days.", log)
return lc, fig
else:
res = search_lightcurvefile(target_name, mission=chosen_mission)
if len(res) != 0:
fig = None
mprint(f"Found processed light curve for {target_name}!", info)
res = res.download_all()
types = []
for d in res.data:
type = 'TESS' if isinstance(d, lk.TessLightCurveFile) else 'Kepler'
if type not in types:
types.append(type)
mprint(f"Using {','.join(types)} observations! Combining sectors ...", log)
if flux_type == 'PSF':
mprint(f"PSF not available for short cadence data. Reverting to PDCSAP",warn)
flux_type = 'PDCSAP'
if flux_type == 'PDCSAP':
lc_set: List[Union[lk.TessLightCurve, lk.KeplerLightCurve]] = [i for i in res.PDCSAP_FLUX.data]
elif flux_type == 'SAP':
lc_set: List[Union[lk.TessLightCurve, lk.KeplerLightCurve]] = [i for i in res.SAP_FLUX.data]
else:
raise ValueError(ctext("Flux type needs to be either PDCSAP or SAP", error))
lc = combine_light_curves(lc_set,sigma_clip=sigma_clip,iters=iters)
else:
raise ValueError(ctext(f"No light curve available for {target_name} on mission(s) {chosen_mission}",error))
mprint(f"Total observation length: {'%.2f' % (lc.time[-1] - lc.time[0])} days.", log)
return lc, fig
def _query_mast(target,
radius=None,
project=('Kepler', 'K2', 'TESS'),
provenance_name=("Kepler", "K2", "SPOC"),
t_exptime=(0, 9999),
sequence_number=None,
**extra_query_criteria):
"""Helper function which wraps `astroquery.mast.Observations.query_criteria()`
to return a table of all Kepler/K2/TESS observations of a given target.
By default only the official data products are returned, but this can be
adjusted by adding alternative data product names into `provenance_name`.
Parameters
----------
target : str, int, or `astropy.coordinates.SkyCoord` object
See docstrings above.
radius : float or `astropy.units.Quantity` object
Conesearch radius. If a float is given it will be assumed to be in
units of arcseconds. If `None` then we default to 0.0001 arcsec.
project : str, list of str
Mission name. Typically 'Kepler', 'K2', or 'TESS'.
This parameter is case-insensitive.
provenance_name : str, list of str
Provenance of the observation. Common options include 'Kepler', 'K2',
'SPOC', 'K2SFF', 'EVEREST', 'KEPSEISMIC'.
This parameter is case-insensitive.
t_exptime : (float, float) tuple
Exposure time range in seconds. Common values include `(59, 61)`
for Kepler short cadence and `(1799, 1801)` for Kepler long cadence.
sequence_number : int, list of int
Quarter, Campaign, or Sector number.
**extra_query_criteria : kwargs
Extra criteria to be passed to `astroquery.mast.Observations.query_criteria`.
Returns
-------
obs : astropy.Table
Table detailing the available observations on MAST.
"""
# Local astroquery import because the package is not used elsewhere
from astroquery.mast import Observations
from astroquery.exceptions import ResolverError, NoResultsWarning
# If passed a SkyCoord, convert it to an "ra, dec" string for MAST
if isinstance(target, SkyCoord):
target = '{}, {}'.format(target.ra.deg, target.dec.deg)
# We pass the following `query_criteria` to MAST regardless of whether
# we search by position or target name:
query_criteria = {'project': project, **extra_query_criteria}
if provenance_name is not None:
query_criteria['provenance_name'] = provenance_name
if sequence_number is not None:
query_criteria['sequence_number'] = sequence_number
if t_exptime is not None:
query_criteria['t_exptime'] = t_exptime
# If an exact KIC ID is passed, we will search by the exact `target_name`
# under which MAST will know the object to prevent source confusion.
# For discussion, see e.g. GitHub issues #148, #718.
exact_target_name = None
target_lower = str(target).lower()
# Was a Kepler target ID passed?
kplr_match = re.match("^(kplr|kic) ?(\d+)$", target_lower)
if kplr_match:
exact_target_name = f"kplr{kplr_match.group(2).zfill(9)}"
# Was a K2 target ID passed?
ktwo_match = re.match("^(ktwo|epic) ?(\d+)$", target_lower)
if ktwo_match:
exact_target_name = f"ktwo{ktwo_match.group(2).zfill(9)}"
# Was a TESS target ID passed?
tess_match = re.match("^(tess|tic) ?(\d+)$", target_lower)
if tess_match:
exact_target_name = f"{tess_match.group(2).zfill(9)}"
if exact_target_name and radius is None:
log.debug("Started querying MAST for observations with the exact "
f"target_name='{exact_target_name}'.")
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=NoResultsWarning)
obs = Observations.query_criteria(target_name=exact_target_name,
**query_criteria)
if len(obs) > 0:
# astroquery does not report distance when querying by `target_name`;
# we add it here so that the table returned always has this column.
obs['distance'] = 0.
return obs
else:
log.debug(
f"No observations found. Now performing a cone search instead."
)
# If the above did not return a result, then do a cone search using the MAST name resolver
# `radius` defaults to 0.0001 and unit arcsecond
if radius is None:
radius = .0001 * u.arcsec
elif not isinstance(radius, u.quantity.Quantity):
radius = radius * u.arcsec
query_criteria['radius'] = str(radius.to(u.deg))
try:
log.debug("Started querying MAST for observations within "
f"{radius.to(u.arcsec)} arcsec of objectname='{target}'.")
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=NoResultsWarning)
obs = Observations.query_criteria(objectname=target,
**query_criteria)
obs.sort('distance')
return obs
except ResolverError as exc:
# MAST failed to resolve the object name to sky coordinates
raise SearchError(exc) from exc
def mvm_id_filenames(sky_coord, cutout_size, log_level=logutil.logging.INFO):
"""
This function retrieves a table of MVM drizzled image filenames with additional
information from the archive. The user can then further cull the table to use as
input to obtain a list of files from the archive. This function will return filter-level
products. At this time, both ACS and WFC3 are searched by default.
Parameters
----------
sky_coord : str or `~astropy.coordinates.SkyCoord` object
The position around which to cutout. It may be specified as a string ("ra dec" in degrees)
or as the appropriate `~astropy.coordinates.SkyCoord` object.
cutout_size : int, array-like, `~astropy.units.Quantity`
The size of the cutout array. If ``cutout_size`` is a scalar number or a scalar
`~astropy.units.Quantity`, then a square cutout of ``cutout_size`` will be created.
If ``cutout_size`` has two elements, they should be in ``(ny, nx)`` order. Scalar numbers
in ``cutout_size`` are assumed to be in units of arcseconds. `~astropy.units.Quantity` objects
must be in angular units.
log_level : int, optional
The desired level of verbosity in the log statements displayed on the screen and written to the
.log file. Default value is 20, or 'info'.
Returns
-------
final_table : `~astropy.table.Table` object
This utility also writes an output ECSV file version of the in-memory filtered data product table,
final_table. The output filename is in the form:
mvm_query-ra<###>d<####>-dec<n|s><##>d<####>_<radius>_cutout.ecsv
(e.g., mvm_query-ra84d8208-decs69d8516_354_cutout.ecsv, where radius has been computed from the
cutout dimensions.
"""
# set logging level to user-specified level
log.setLevel(log_level)
# If the cutout_size is not an astropy.units.Quantity object, the scalar(s)
# are assumed to be arcseconds. The variable must be cast as a Quantity.
if not isinstance(cutout_size, Quantity):
cutout_size *= u.arcsec
cutout_size = np.atleast_1d(cutout_size)
if len(cutout_size) == 1:
cutout_size = np.repeat(cutout_size, 2)
if not isinstance(sky_coord, SkyCoord):
sky_coord = SkyCoord(sky_coord, unit="deg")
# From HST data, Search for the list of images based upon: coordinates, search region, data
# product type, and the instrument name (with wildcard), project (HAP), and observation
# collection (HST). Use the wildcard to get all the detectors for the instrument. Multiple
# instruments cannot be searched at the same time. Use the diagonal of the cutout to define
# the search radius for the archive. Images which fall outside the desired cutout need to
# be filtered from the solution later.
radius = math.ceil(
math.sqrt(
math.pow(cutout_size.value[0], 2) +
math.pow(cutout_size.value[1], 2)) / 2.0)
# Careful - the radius must be a str or Quantity
radius *= u.arcsec
log.info("Radius for query: {}.".format(radius))
log.info("Performing query for ACS images.")
acs_query_table = Observations.query_criteria(coordinates=sky_coord,
radius=radius,
dataproduct_type="IMAGE",
instrument_name="ACS*",
project="HAP",
obs_collection="HST")
log.info("Performing query for WFC3 images.")
wfc3_query_table = Observations.query_criteria(coordinates=sky_coord,
radius=radius,
dataproduct_type="IMAGE",
instrument_name="WFC3*",
project="HAP",
obs_collection="HST")
query_table = vstack([acs_query_table, wfc3_query_table])
del acs_query_table
del wfc3_query_table
# Catch the case where no files are found which satisfied the Query
if not query_table:
log.warning(
"Query for objects within {} of {} returned NO RESULTS!".format(
radius, (str_ra, str_dec)))
return query_table
# Compute the limits of the cutout region
deg_cutout_size = cutout_size.to(u.deg)
ra_min = sky_coord.ra.degree - deg_cutout_size.value[0]
ra_max = sky_coord.ra.degree + deg_cutout_size.value[0]
dec_min = sky_coord.dec.degree - deg_cutout_size.value[1]
dec_max = sky_coord.dec.degree + deg_cutout_size.value[1]
str_ra = "{:.4f}".format(sky_coord.ra.degree)
str_dec = "{:.4f}".format(sky_coord.dec.degree)
# Filter the output as necessary to include only MVM filenames (MVM prefix: hst_skycell).
# Also, filter out images which are not actually in the requested cutout region as the
# archive search had to be done using a radius.
good_rows = []
updated_query_table = None
for old_row in query_table:
if old_row["obs_id"].startswith("hst_skycell"):
if old_row["s_ra"] >= ra_min and old_row["s_ra"] <= ra_max and \
old_row["s_dec"] >= dec_min and old_row["s_dec"] <= dec_max:
good_rows.append(old_row)
# Catch the case where no files are found which satisfy the clean up criteria
if len(good_rows) == 0:
log.warning(
"Query for objects within cutout {} of {} returned NO RESULTS!".
format(cutout_size, (str_ra, str_dec)))
return updated_query_table
# Make the cleaned up table
updated_query_table = Table(rows=good_rows, names=query_table.colnames)
del query_table
# Get the data product list associated with the elements of the table
log.info("Get the product list for all entries in the query table.")
dp_table = Observations.get_product_list(updated_query_table)
del updated_query_table
# Filter on MVM drizzled products only
suffix = ["DRZ", "DRC"]
log.info(
"Filter the product list table for only {} filenames.".format(suffix))
filtered_dp_table = Observations.filter_products(
dp_table, productSubGroupDescription=suffix, extension="fits")
if not filtered_dp_table:
log.warning(
"No MVM drizzle product datasets (DRZ/DRC) found within {} of {}.".
format(radius, (str_ra, str_dec)))
return filtered_dp_table
del dp_table
# Need to filter out any non-hst-skycell entries AGAIN which may have
# crept back into the list via the get_product_list() function.
good_rows = []
output_table = None
for old_row in filtered_dp_table:
if old_row["obs_id"].startswith("hst_skycell"):
good_rows.append(old_row)
# Catch the case where no files are found which satisfy the criteria
if len(good_rows) == 0:
log.warning(
"After filtering datasets there are NO RESULTS within {} of {}!".
format(radius, (str_ra, str_dec)))
return output_table
# Make the output table
output_table = Table(rows=good_rows, names=filtered_dp_table.colnames)
del filtered_dp_table
# Finally, make sure the entries are unique
final_table = None
final_table = unique(output_table, keys="productFilename")
del output_table
# Write the table to a file. This allows for further manipulation of
# the information before a list of filenames is distilled from the table.
# Output filename in the form: mvm_query-ra<###>d<####>-dec<n|s><##>d<####>_<radius>_cutout.ecsv
# (e.g., mvm_query-ra84d9208-decs69d1483_71_cutout.ecsv), where radius has been computed from the
# cutout dimensions.
#
# Get the whole number and fractional components of the RA and Dec
ns = "s" if sky_coord.dec.degree < 0.0 else "n"
ra_whole = int(sky_coord.ra.value)
ra_frac = str(sky_coord.ra.value).split(".")[1][0:4]
dec_whole = abs(int(sky_coord.dec.value))
dec_frac = str(sky_coord.dec.value).split(".")[1][0:4]
log.info("coords2. {} {} {}".format(sky_coord.ra.value,
sky_coord.dec.value, dec_frac))
query_filename = "mvm_query-ra" + str(ra_whole) + "d" + ra_frac + "-dec" + ns + \
str(dec_whole) + "d" + dec_frac + "_{:.0f}".format(radius.value) + "_cutout.ecsv"
log.info(
"Writing out the MVM product list table to {}.".format(query_filename))
log.info("Number of entries in table: {}.".format(len(final_table)))
final_table.write(query_filename, format="ascii.ecsv")
return final_table
def hst_button(
galaxies,
skymethod='globalmin+match',
instruments="ACS/WFC",
prop_ids=None,
filters=None,
radius=None,
filepath=None,
download_data=True,
correct_astrometry=True,
create_mosaic=True,
jy_conversion=True,
verbose=False,
log_filename='hst.log',
):
"""Create a HST mosaic, given a galaxy name.
Using a galaxy name and radius, queries around that object,
downloads available HST data and mosaics into a final product. It
will create separate mosaics for each proposal ID, and the file structure
will look like ``/galaxy/HST/proposal_id/galaxy_instrument_filter_proposal_id.fits``.
N.B. I must confess to not being well-versed with HST data, so if
anyone can help improve this please let me know.
This data button uses a number of tools included in the drizzlepac
Python package. This includes alignimages/tweakreg and astrodrizzle,
which correct astrometry and are specifically tailored for the setup
of HST data. This means that 1) creating mosaics with this will likely
take a long time and 2) you will need a beefy computer (especially with
regards to hard drive space).
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
Resolved by NED.
skymethod (str, optional): Method used for AstroDrizzle's background
matching step. In general, this can be left untouched but for
mosaics with little overlap, it may be worth playing around
with this. For instance, I've had some luck when there isn't
much overlap between exposures using 'globalmin'. Options are
'localmin', 'globalmin+match', 'globalmin', and 'match'. Defaults
to 'globalmin+match'.
instruments (str or list, optional): Instrument to download data
for. Can be any combination of 'ACS/WFC', 'WFC3/IR',
'WFC3/UVIS', 'WFPC2/PC', or 'WFPC2/WFC'. If you want all
available data for all these instruments, select 'all', but
this is not recommended! Defaults to 'ACS/WFC'.
prop_ids (str or list, optional): Proposal IDs to download data for.
Defaults to None, which will pull out all proposal IDs for each
instrument.
filters (str or list, optional): Filters to download data for.
The script will look for each filter, for each instrument.
Defaults to None, which will pull out all applicable filters
for each instrument, for each proposal ID.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data from
MAST. Defaults to True.
correct_astrometry (bool, optional): If True, will perform astrometric
corrections to the downloaded data using alignimages. Defaults
to True.
create_mosaic (bool, optional): Switching this to True will
mosaic the data using astrodrizzle as appropriate. Defaults
to True.
jy_conversion (bool, optional): Convert the mosaicked file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Can be used to suppress most of the
output messages produced during the process. Mainly useful
for debugging. Defaults to False.
log_filename (str, optional): Will produce a stripped down log
of what data the code is reducing. By default, will save to
galaxy/hst.log.
"""
if isinstance(galaxies, str):
galaxies = [galaxies]
if isinstance(instruments,str):
instruments = [instruments]
if instruments == 'all':
instruments = ['ACS/WFC',
'WFC3/IR','WFC3/UVIS',
'WFPC2/PC','WFPC2/WFC']
if isinstance(filters,str):
filters = [filters]
if isinstance(prop_ids,str):
prop_ids = [prop_ids]
if filepath is not None:
os.chdir(filepath)
orig_dir = os.getcwd()
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if correct_astrometry:
steps.append(2)
if create_mosaic:
steps.append(3)
if jy_conversion:
steps.append(4)
# Set up folders for various corrections
os.environ['CRDS_SERVER_URL'] = 'https://hst-crds.stsci.edu'
os.environ['CRDS_PATH'] = orig_dir+'/reference_files'
os.environ['iref'] = orig_dir+'/reference_files/references/hst/wfc3/'
os.environ['jref'] = orig_dir+'/reference_files/references/hst/acs/'
os.environ['uref'] = orig_dir+'/reference_files/references/hst/wfpc2/'
# For large proposals, astrodrizzle can run into file open
# issues so raise the max file open amount.
_, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE,(hard,hard))
# Change the temp directory -- if this gets filled up it can cause
# problems.
orig_tmpdir = os.environ['TMPDIR']
if not os.path.exists('tmp'):
os.mkdir('tmp')
os.environ['TMPDIR'] = orig_dir+'/tmp'
for galaxy in galaxies:
if not os.path.exists(galaxy):
os.mkdir(galaxy)
if not os.path.exists(galaxy+'/HST'):
os.mkdir(galaxy+'/HST')
if not verbose:
# Various packages used here put out a lot of messages. Silence info messages.
loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]
for logger in loggers:
logger.setLevel(logging.ERROR)
# Even if verbose is not True, still print out some useful messages to the
# console.
hst_logger = logging.getLogger('data_buttons')
handler = logging.FileHandler(galaxy+'/'+log_filename,mode='w')
hst_logger.addHandler(handler)
hst_logger.addHandler(logging.StreamHandler())
hst_logger.setLevel(logging.INFO)
hst_logger.info('Beginning '+galaxy)
hst_logger.info(' ')
hst_logger.info(' ')
if radius is None:
try:
size_query = Ned.get_table(galaxy,table='diameters')
radius = np.max(size_query['NED Major Axis'])/2*u.arcsec
radius = radius.to(u.deg)
except:
hst_logger.warning(galaxy+' not resolved by Ned, using 0.2deg radius.')
radius = 0.2*u.degree
obs_table = Observations.query_criteria(objectname=galaxy,
radius=radius,
obs_type='all',
obs_collection='HST')
# Ignore any calibration observations.
obs_table = obs_table[obs_table['intentType'] == 'science']
for instrument in instruments:
# Pixel sizes for final mosaics selected to match the HLA.
pix_size = {'ACS/HRC':0.025,
'ACS/SBC':0.03,
'ACS/WFC':0.05,
'NICMOS/NIC1':0.025,
'NICMOS/NIC2':0.05,
'NICMOS/NIC3':0.1,
'WFC3/IR':0.09,
'WFC3/UVIS':0.04,
'WFPC2/PC':0.05,
'WFPC2/WFC':0.1}[instrument]
# Bits to consider good for drizzling.
bits = {'ACS/HRC':256,
'ACS/SBC':256,
'ACS/WFC':256,
'NICMOS/NIC1':0,
'NICMOS/NIC2':0,
'NICMOS/NIC3':0,
'WFC3/IR':768,
'WFC3/UVIS':256,
'WFPC2/PC':'8,1024',
'WFPC2/WFC':'8,1024'}[instrument]
# Filename extension, in order of preference.
suffixes = {'ACS/WFC':['FLC','FLT'],
'WFC3/IR':['FLT'],
'WFC3/UVIS':['FLC','FLT'],
'WFPC2/PC':[['C0M','C1M']],
'WFPC2/WFC':[['C0M','C1M']],
}[instrument]
# The instruments often have / in the name, so account for
# this in making folders and files.
hst_logger.info('Beginning '+instrument)
if not os.path.exists(galaxy+'/HST/'+instrument.replace('/','_')):
os.mkdir(galaxy+'/HST/'+instrument.replace('/','_'))
reset_filters = False
instrument_table = obs_table[obs_table['instrument_name'] == instrument]
reset_prop_ids = False
if not prop_ids:
prop_ids = list(np.unique(instrument_table['proposal_id']))
reset_prop_ids = True
hst_logger.info('Available proposal IDs: '+','.join(prop_ids))
hst_logger.info(' ')
for prop_id in prop_ids:
hst_logger.info('Proposal ID: '+str(prop_id))
prop_table = instrument_table[instrument_table['proposal_id'] == prop_id]
if not filters:
filters = list(np.unique(prop_table['filters']))
reset_filters = True
hst_logger.info('Available filters: '+','.join(filters))
for hst_filter in filters:
# If we have a highly illegal filter, just skip.
# TODO: This needs to be sorted for some fringe
# cases, probably.
if not hst_filter[0] == 'F':
continue
hst_logger.info('Filter: '+str(hst_filter))
# Pull out available data and download.
filter_table = prop_table[prop_table['filters'] == hst_filter]
if len(filter_table) == 0:
hst_logger.warning('No available data to download. Skipping...')
continue
data_products_id = Observations.get_product_list(filter_table)
for suffix in suffixes:
download_table = Observations.filter_products(data_products_id,
productSubGroupDescription=suffix,
mrp_only=False)
if len(download_table) > 0:
break
if isinstance(suffix,list):
filename_exts = [ext.lower() for ext in suffix]
else:
filename_exts = [suffix.lower()]
hst_logger.info(instrument+'/'+prop_id+'/'+hst_filter)
if not os.path.exists(galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter):
os.mkdir(galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter)
if not os.path.exists(galaxy+'/HST/'+prop_id):
os.mkdir(galaxy+'/HST/'+prop_id)
full_filepath = (galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter+
'/'
+prop_id)
if not os.path.exists(full_filepath):
os.mkdir(full_filepath)
if 1 in steps:
# Download files
download_mast(download_table,
download_dir="hst_temp/" + galaxy)
if not os.path.exists(full_filepath+'/raw'):
os.mkdir(full_filepath+'/raw')
if not os.path.exists(full_filepath+'/outputs'):
os.mkdir(full_filepath+'/outputs')
# Pull out the relevant files, and move to base folder.
for filename_ext in filename_exts:
matches = []
for root, _, filenames in os.walk("hst_temp/" + galaxy):
for filename in fnmatch.filter(
filenames, "*_"+filename_ext+".fits"
):
matches.append(os.path.join(root, filename))
for match in matches:
filename = match.split('/')
os.rename(match,full_filepath+'/raw/'+filename[-1])
# Clean up any temporary files.
shutil.rmtree("hst_temp/" + galaxy, ignore_errors=True)
filename_ext = filename_exts[0]
hst_files = glob.glob(full_filepath+'/raw/*_'+filename_ext+'.fits')
if 2 in steps:
# First, update the WCS information in case it's
# required.
for filename_ext in filename_exts:
hst_files = glob.glob(full_filepath+'/raw/*_'+filename_ext+'.fits')
crds.assign_bestrefs(hst_files,
sync_references=True)
# For WFPC2, the CRDS doesn't download everything
# needed. Download the GEIS data files and
# rerun the bestrefs assignment.
if 'WFPC2' in instrument:
geis_hdrs = glob.glob(os.environ['uref']+'/*h')
for geis_hdr in geis_hdrs:
geis_data = geis_hdr[:-1]+'d'
if not os.path.exists(geis_data):
geis_data = geis_data.split('/')[-1]
print(geis_data)
print(os.environ['uref'])
wget.download(
os.environ['CRDS_SERVER_URL']+'/unchecked_get/references/hst/'+geis_data,
out=os.environ['uref'])
crds.assign_bestrefs(hst_files,sync_references=True)
for hst_file in hst_files:
stwcs.updatewcs.updatewcs(hst_file,
use_db=False)
os.chdir(full_filepath+'/raw')
filename_ext = filename_exts[0]
hst_files = glob.glob('*_'+filename_ext+'.fits')
# Normalize all files.
photeq.photeq(', '.join(hst_files),readonly=False)
os.rename('photeq.log','../outputs/photeq.log')
if 'WFPC' in instrument:
# Using tweakreg, align each frame to GAIA.
gaia_table = Gaia.query_object_async(coordinate=galaxy,
radius=2*radius)
ras = gaia_table['ra']
decs = gaia_table['dec']
source_table = Table([ras,decs])
source_table.write('gaia.cat',
format='ascii.fast_commented_header')
tweakreg.TweakReg(hst_files,
imagefindcfg={'threshold':5,'conv_width':3},
refcat='gaia.cat',
#expand_refcat=True,
enforce_user_order=False,
shiftfile=True,
outshifts='shifts.txt',
searchrad=10,
minobj=5,
separation=0,
updatehdr=True,
reusename=True,
wcsname='TWEAK',
interactive=False,
fitgeometry='general',
clean=True,
see2dplot=False
)
# Update the c1m files to use the TWEAK
# wcs
for hst_file in hst_files:
dq_file = hst_file.replace('c0','c1')
tweakback.tweakback(hst_file,
dq_file,
newname='TWEAK')
plot_files = glob.glob('*.png')
for plot_file in plot_files:
os.remove(plot_file)
cat_files = glob.glob('*.coo')
for cat_file in cat_files:
os.remove(cat_file)
os.rename('shifts_wcs.fits','../outputs/shifts_wcs.fits')
os.rename('tweakreg.log','../outputs/tweakreg.log')
os.rename('shifts.txt','../outputs/shifts.txt')
elif 'ACS' in instrument or 'WFC3' in instrument:
# Correct astrometry using alignimages. First,
# correct each frame separately.
pool = mp.Pool(mp.cpu_count())
suitable_hst_files = pool.map(astrometric_correction,
hst_files)
pool.close()
suitable_hst_files = [x for x in suitable_hst_files
if x is not None]
if len(suitable_hst_files) == 0:
hst_logger.warning('Failure with astrometry corrections. Skipping')
os.chdir(orig_dir)
continue
# Now, align every suitable frame simultaneously.
output_table = astrometric_correction(suitable_hst_files)
with open('../outputs/astrometry.pkl','wb') as table_file:
pickle.dump(output_table,table_file)
else:
raise Exception('Unknown instrument!')
os.chdir(orig_dir)
os.chdir(full_filepath)
if 3 in steps:
os.chdir('raw')
if 'WFPC2' in instrument:
hst_files = glob.glob('*_c0m.fits')
wcskey = 'TWEAK'
elif 'ACS' in instrument or 'WFC3' in instrument:
with open('../outputs/astrometry.pkl','rb') as table_file:
output_table = pickle.load(table_file)
# We only want fits where an acceptable astrometric
# solution has been found.
suitable_fits = np.where(output_table['fit_qual'] < 5)
# (output_table['fit_qual'] >= 1)
hst_files = list(output_table[suitable_fits]['imageName'])
if len(output_table[suitable_fits]) == 0:
hst_logger.warning('Failure with astrometry corrections. Skipping')
os.chdir(orig_dir)
continue
wcskey = ' '
else:
raise Exception('Unknown instrument!')
# Following Dalcanton+ (2012), group exposures into
# long (>50s) and short (<=50s), and process for cosmic
# rays separately
exp_times = []
for hst_file in hst_files:
hdu = fits.open(hst_file)[0]
exp_time = hdu.header['EXPTIME']
exp_times.append(exp_time)
for exp_group in ['short','long']:
hst_files_group = []
for i in range(len(exp_times)):
if exp_times[i] > 50 and exp_group == 'long':
hst_files_group.append(hst_files[i])
elif exp_times[i] <= 50 and exp_group == 'short':
hst_files_group.append(hst_files[i])
if len(hst_files_group) == len(hst_files):
exp_group = ''
if len(hst_files_group) == 0:
continue
if len(exp_group) > 0:
output_name = '../outputs/'+galaxy+'_'+exp_group
drizzle_log_name = '../outputs/astrodrizzle_'+exp_group+'.log'
else:
output_name = '../outputs/'+galaxy
drizzle_log_name = '../outputs/astrodrizzle.log'
# Perform the mosaicking. Generally, use iminmed.
# However, sometimes iminmed will fail so
# for the other instruments we'll use imedian as
# a fallback.
combine_types = ['iminmed','imedian']
if 'WFPC2' in instrument:
combine_nhigh = 1
else:
combine_nhigh = 0
for combine_type in combine_types:
try:
astrodrizzle.AstroDrizzle(
input=hst_files_group,
output=output_name,
preserve=False,
clean=True,
combine_type=combine_type,
combine_nhigh=combine_nhigh,
skymethod=skymethod,
sky_bits=bits,
driz_sep_bits=bits,
driz_sep_fillval=99999,
combine_hthresh=90000,
final_scale=pix_size,
final_bits=bits,
final_fillval=0,
wcskey=wcskey,
final_rot=0,
)
break
except ValueError:
pass
# Move the AstroDrizzle log.
os.rename('astrodrizzle.log',
drizzle_log_name)
# Move back to the original directory.
os.chdir(orig_dir)
if 4 in steps:
mosaic_outputs = glob.glob(full_filepath+'/outputs/*_sci.fits')
for mosaic_output in mosaic_outputs:
# Replace any fillvals with NaNs.
hdu = fits.open(mosaic_output)[0]
hdu.data[hdu.data == 0] = np.nan
fits.writeto(mosaic_output,
hdu.data,hdu.header,
overwrite=True)
if '_long_' in mosaic_output.split('/')[-1]:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'_long.fits')
elif '_short_' in mosaic_output.split('/')[-1]:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'_short.fits')
else:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'.fits')
convert_to_jy(mosaic_output,
new_filename)
if reset_filters:
filters = None
hst_logger.info(' ')
if reset_prop_ids:
prop_ids = None
hst_logger.info(' ')
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
# Clear out the tmp folder and reset to the original.
shutil.rmtree('tmp/', ignore_errors=True)
os.environ['TMPDIR'] = orig_tmpdir
def lambda_handler(event, context):
"""Extract light curve data from one TESS full frame image.
Parameters
----------
event : dict
API Gateway Lambda Proxy Input Format.
Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format
context : object
Lambda Context runtime methods and attributes.
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
result : dict
API Gateway Lambda Proxy Output Format.
Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
""" # noqa
obs_id = event['id'] # TESS observation ID; Example: 'tess-s0001-1-1'
# TODO: Calculate some of these from the 10th frame?
# For now, also takes these and pass them onto worker:
payload = {
'xpos': event['xpos'],
'ypos': event['ypos'],
'radius': event['radius'],
'bright_pixel_threshold': event['bright_pixel_threshold']
}
# Find full frame dataset for the observation ID.
obs_table = Observations.query_criteria(obs_id=obs_id)
products = Observations.get_product_list(obs_table)
filtered = Observations.filter_products(products,
productSubGroupDescription="FFIC",
mrp_only=False)
# Use AWS S3 bucket to pull data from.
Observations.enable_cloud_dataset() # TODO: verbose=False ?
s3_urls = Observations.get_cloud_uris(filtered, include_bucket=False)
# TODO: Timed out! Try https://docs.python.org/3/library/asyncio.html ?
# TODO: Handle same Lambda call invoked multiple times by AWS?
# Call tess_fullframe_worker AWS Lambda function in parallel
# https://aws.amazon.com/blogs/compute/parallel-processing-in-python-with-aws-lambda/
parent_connections = []
processes = []
data = []
for url in s3_urls[:2]: # TODO: Remove [:2] when done testing
payload['key'] = url
parent_conn, child_conn = Pipe()
parent_connections.append(parent_conn)
arg = json.dumps(payload)
process = Process(target=_pipe_worker, args=(arg, child_conn))
processes.append(process)
for process in processes:
process.start()
for process in processes:
process.join()
for parent_connection in parent_connections:
try:
response = parent_connection.recv()[0]
except EOFError:
response = {}
if 'body' not in response: # Worker Lambda threw exception
continue
body = json.loads(response['body'])
row = (body['midtime'], body['signal'], body['background'])
if np.all(list(map(np.isfinite, row))):
data.append(row)
# TODO: Save data as table.
# filename = f'/tmp/{obs_id}_lightcurve.csv'
# with open(filename) as fout:
# for row in data:
# fout.write(f'{row[0]},{row[1]},{row[2]}{os.linesep}')
# TODO: Upload table to S3 and then delete the table locally.
# TODO: Return table S3 URL below.
# TODO: Do we want to plot it and upload the plot too?
# If so, need to add matplotlib as dependency.
return {
"statusCode": 200,
"body": json.dumps({
'n_rows': len(data),
'data_url': 'TODO'
})
}
def galex_button(
galaxies,
filters="both",
radius=None,
filepath=None,
download_data=True,
create_mosaic=True,
jy_conversion=True,
verbose=False,
):
"""Create a GALEX mosaic, given a galaxy name.
Using a galaxy name and radius, queries around that object,
downloads available GALEX data and mosaics into a final product.
Because GALEX images are in counts/s and the integrations may be
different lengths, we convert back to a raw count, add the frames
and convert back to counts/s at the end. This effectively weights
the frame by exposure time.
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
Resolved by NED.
filters (str, optional): One of 'FUV', 'NUV', or 'both'. Selects
which GALEX filters to create a mosaic for. Defaults to
'both'.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data from
MAST. Defaults to True.
create_mosaic (bool, optional): Switching this to True will
mosaic data as appropriate. Defaults to True.
jy_conversion (bool, optional): Convert the mosaicked file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Print out messages during the process.
Useful mainly for debugging purposes or large images.
Defaults to False.
"""
if isinstance(galaxies, str):
galaxies = [galaxies]
if filters == "both":
filters = ["NUV", "FUV"]
else:
filters = [filters]
if filepath is not None:
os.chdir(filepath)
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if create_mosaic:
steps.append(2)
if jy_conversion:
steps.append(3)
for galaxy in galaxies:
if verbose:
print('Beginning ' + galaxy)
if radius is None:
try:
size_query = Ned.get_table(galaxy, table='diameters')
radius = 1.2 * np.max(
size_query['NED Major Axis']) / 2 * u.arcsec
radius = radius.to(u.deg)
except:
raise Warning(galaxy +
' not resolved by Ned, using 0.2deg radius.')
radius = 0.2 * u.degree
if not os.path.exists(galaxy):
os.mkdir(galaxy)
obs_table = Observations.query_criteria(objectname=galaxy,
radius=radius,
obs_type='all',
obs_collection='GALEX')
# Ignore any calibration observations.
obs_table = obs_table[obs_table['intentType'] == 'science']
for galex_filter in filters:
if verbose:
print('Beginning GALEX ' + galex_filter)
if 1 in steps:
# Pull out available data, and download it
query_results = np.where(
obs_table["filters"] == galex_filter)[0]
# If there isn't any GALEX coverage, just skip
if len(query_results) == 0:
print(galaxy + " missing!")
continue
# We only want to print out download messages if
# verbose is True, so redirect otherwise.
if not verbose:
sys.stdout = open(os.devnull, 'w')
dataProductsByID = Observations.get_product_list(
obs_table[query_results])
Observations.download_products(
dataProductsByID,
download_dir="galex_temp/" + galaxy,
mrp_only=True,
)
# And set back to the original for printing.
if not verbose:
sys.stdout = sys.__stdout__
if not os.path.exists(galaxy + "/GALEX/"):
os.mkdir(galaxy + "/GALEX/")
if not os.path.exists(galaxy + "/GALEX/" + galex_filter):
os.mkdir(galaxy + "/GALEX/" + galex_filter)
if not os.path.exists(galaxy + "/GALEX/" + galex_filter +
'/raw'):
os.mkdir(galaxy + "/GALEX/" + galex_filter + '/raw')
if not os.path.exists(galaxy + "/GALEX/" + galex_filter +
'/data'):
os.mkdir(galaxy + "/GALEX/" + galex_filter + '/data')
if not os.path.exists(galaxy + "/GALEX/" + galex_filter +
'/reprojected'):
os.mkdir(galaxy + "/GALEX/" + galex_filter +
'/reprojected')
if not os.path.exists(galaxy + "/GALEX/" + galex_filter +
'/weight'):
os.mkdir(galaxy + "/GALEX/" + galex_filter + '/weight')
if not os.path.exists(galaxy + "/GALEX/" + galex_filter +
'/outputs'):
os.mkdir(galaxy + "/GALEX/" + galex_filter + '/outputs')
ext_name = {"NUV": "nd", "FUV": "fd"}[galex_filter]
# Pull out the relevant filter files files (either *nd*
# or *fd*), extract and move to base folder.
matches = []
for root, _, filenames in os.walk("galex_temp/" + galaxy):
for filename in fnmatch.filter(
filenames, "*" + ext_name + "-int.fits.gz"):
matches.append(os.path.join(root, filename))
for match in matches:
with gzip.open(match, "rb") as f_in:
with open(match[:-3], "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
filename = match[:-3].split('/')
os.rename(
match[:-3],
galaxy + "/GALEX/" + galex_filter + "/raw/" +
filename[-1],
)
# Clean up any temporary files.
shutil.rmtree("galex_temp/" + galaxy, ignore_errors=True)
mHdr(
galaxy,
2 * radius.value,
2 * radius.value,
galaxy + "/GALEX/" + galex_filter + "/outputs/header.hdr",
resolution=1.5,
)
if 2 in steps:
# Read in these files and set anything more than 35
# arcmin out to NaN.
if verbose:
print('Performing initial weighted reprojections')
galex_files = glob.glob(galaxy + "/GALEX/" + galex_filter +
"/raw/*")
for galex_file in galex_files:
hdu = fits.open(galex_file)[0]
i = np.linspace(-hdu.data.shape[0] / 2,
hdu.data.shape[0] / 2, hdu.data.shape[0])
j = np.linspace(-hdu.data.shape[1] / 2,
hdu.data.shape[1] / 2, hdu.data.shape[1])
iv, jv = np.meshgrid(i, j)
r = iv**2 + jv**2
hdu.data[r >= 1400**2] = np.nan
hdu.writeto(galex_file.replace('/raw/', '/data/'),
overwrite=True)
# Also create a weight map (sqrt EXPTIME).
exp_time = np.ones(
hdu.data.shape) * hdu.header['EXPTIME']**0.5
exp_time[np.isnan(hdu.data) == True] = np.nan
fits.writeto(galex_file.replace('/raw/', '/weight/'),
exp_time,
hdu.header,
overwrite=True)
# And reproject each map separately using this weighting.
mProject(galex_file.replace('/raw/', '/data/'),
galex_file.replace('/raw/', '/reprojected/'),
galaxy + "/GALEX/" + galex_filter +
"/outputs/header.hdr",
weight_file=galex_file.replace(
'/raw/', '/weight/'))
# And mosaic!
# Montage uses its size as the length of the square,
# since we want a radius use twice that.
mHdr(
galaxy,
2 * radius.value,
2 * radius.value,
galaxy + "/GALEX/" + galex_filter + "/outputs/header.hdr",
resolution=1.5,
)
tools.mosaic(
galaxy + "/GALEX/" + galex_filter + '/reprojected',
header=galaxy + "/GALEX/" + galex_filter +
"/outputs/header.hdr",
verbose=verbose,
reproject=False,
haveAreas=True,
)
os.rename(
"mosaic/mosaic.fits", galaxy + "/GALEX/" + galex_filter +
"/outputs/" + galaxy + '.fits')
shutil.rmtree("mosaic/", ignore_errors=True)
if 3 in steps:
if verbose:
print('Converting to Jy')
# Convert to Jy.
convert_to_jy(galaxy + "/GALEX/" + galex_filter + '/outputs/' +
galaxy + ".fits",
galex_filter,
hdu_out=galaxy + "/GALEX/" + galaxy + '_' +
galex_filter + ".fits")
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
def bert_tess_fullframe_main_2():
"""Continuation of main function to run it across different sectors."""
import os
import time
import boto3
from astropy.io import fits
from astropy.wcs import WCS
from astroquery.mast import Observations
s3 = boto3.resource('s3')
bucket = s3.Bucket(name=os.environ.get('AWSBUCKETNAME'))
outbucket = s3.Bucket(name=os.environ.get('CACHEBUCKETNAME'))
homedir = os.environ.get('HOME')
work_queue, done_queue, ologger = utils.comm_binders(
bert_tess_fullframe_main_2)
# Example event:
# {
# "tic_id": "25155310",
# "sec_id": "tess-s0001-4-1",
# "ra": 63.3739396231274,
# "dec": -69.226822697583,
# "radius": 2.5,
# "cutout_width": 30,
# "use_cache": "true"
# }
#
# work_queue populated by calling Lambda
for event in work_queue:
tic_id = event['tic_id']
sec_id = event['sec_id']
basename = f'{sec_id}_s3_uris.txt' # noqa
filename = os.path.join(homedir, basename)
try:
# Check if URI list already cached.
# According to MAST, there is no need to invalidate cache here.
ologger.info(f'Attempting to download {basename} from S3')
outbucket.download_file(
basename, filename,
ExtraArgs={"RequestPayer": "requester"})
except Exception:
# Find full frame dataset for the observation ID.
ologger.info('Started quering Observations...')
obs_table = Observations.query_criteria(obs_id=sec_id)
products = Observations.get_product_list(obs_table)
filtered = Observations.filter_products(
products, productSubGroupDescription="FFIC",
mrp_only=False)
# Use AWS S3 bucket to pull data from.
Observations.enable_cloud_dataset(verbose=False)
ologger.info('Started obtaining cloud URIs...')
t_start = time.time()
s3_urls = Observations.get_cloud_uris(
filtered, include_bucket=False)
t_end = time.time()
ologger.info(f'Got {len(s3_urls)} URIs in {t_end - t_start} s')
# Upload URI list to cache.
with open(filename, 'w') as fout:
for url in s3_urls:
fout.write(url + os.linesep)
try:
outbucket.upload_file(
filename, basename,
ExtraArgs={"RequestPayer": "requester"})
except Exception as exc:
ologger.error(str(exc))
else:
ologger.info(f'Uploaded {basename} to S3')
else:
# Use cache if it exists.
with open(filename, 'r') as fin:
s3_urls = [url.strip() for url in fin.readlines()]
ologger.info(f'Read {len(s3_urls)} URIs from {basename}')
finally:
# Clean up
if os.path.exists(filename):
os.remove(filename)
ra = float(event['ra'])
dec = float(event['dec'])
# TODO: Cache good WCS for a given sector/camera/ccd combo and use
# known good cache if available.
# Find pixel coordinates from sky from first frame header.
key = s3_urls[0]
basename = key.split('/')[-1]
filename = os.path.join(homedir, basename)
ologger.info(f'Resolving WCS from {key}')
bucket.download_file(
key, filename, ExtraArgs={"RequestPayer": "requester"})
hdr = fits.getheader(filename, ext=1)
if hdr.get('WCSAXES', 0) != 2: # Good WCS according to MIT
ologger.error(f'{key} has invalid WCS')
continue
w = WCS(hdr)
pix = w.all_world2pix(ra, dec, 0)
xpos = round(float(pix[0])) # float needed to get rid of 0-D array
ypos = round(float(pix[1]))
# Clean up
os.remove(filename)
# The star needs to be at least 2*radii pixels away in both X and Y.
radius = float(event['radius'])
edge_r = 2 * radius
naxis1, naxis2 = w.pixel_shape # X Y
if (xpos < edge_r or xpos >= (naxis1 - edge_r) or
ypos < edge_r or ypos >= (naxis2 - edge_r)):
ologger.error(
f'TIC f{tic_id} in {sec_id}: X={xpos},Y={ypos} not at least '
f'{edge_r} pixels away from the edge, skipping...')
continue
# Pass data into the next AWS Lambda function.
ologger.info(f'TIC f{tic_id} in {sec_id}: Started processing '
'full frame URIs...')
for url in s3_urls:
done_queue.put({
'key': url,
'tic_id': tic_id,
'ra': ra,
'dec': dec,
'xpos': xpos,
'ypos': ypos,
'radius': radius,
'cutout_width': event['cutout_width'],
'use_cache': event['use_cache']})
def get_lc_file_and_data(yourpath, target):
""" goes in, grabs the data for the target, gets the time index, intensity,and TIC
if connection error w/ MAST, skips it.
Also masks any flagged data points according to the QUALITY column.
parameters:
* yourpath, where you want the files saved to. must end in /
* targets, target list of all TICs
modified [lcg 07082020] - fixed handling no results, fixed deleting download folder"""
fitspath = yourpath + 'mastDownload/TESS/' # >> download directory
targ = "TIC " + str(int(target))
print(targ)
try:
#find and download data products for your target objectname='TIC '+str(int(target)),
obs_table = Observations.query_criteria(obs_collection='TESS',
dataproduct_type='timeseries',
target_name=str(int(target)),
objectname=targ)
data_products_by_obs = Observations.get_product_list(obs_table[0:8])
filter_products = Observations.filter_products(data_products_by_obs,
description = 'Light curves')
if len(filter_products) != 0:
manifest = Observations.download_products(filter_products, download_dir= yourpath, extension='fits')
else:
print("Query yielded no matching data produts for ", targ)
time1 = 0
i1 = 0
ticid = 0
#get all the paths to lc.fits files
filepaths = []
for root, dirs, files in os.walk(fitspath):
for name in files:
#print(name)
if name.endswith(("lc.fits")):
filepaths.append(root + "/" + name)
#print(len(filepaths))
#print(filepaths)
if len(filepaths) == 0: #if no lc.fits were downloaded, move on
print("No lc.fits files available for TIC ", targ)
time1 = 0
i1 = 0
ticid = 0
else: #if there are lc.fits files, open them and get the goods
#get the goods and then close it
f = fits.open(filepaths[0], memmap=False)
time1 = f[1].data['TIME']
i1 = f[1].data['PDCSAP_FLUX']
ticid = f[1].header["TICID"]
quality = f[1].data['QUALITY']
f.close()
# >> mask out any nonzero points
flagged_inds = np.nonzero(quality)
i1[flagged_inds] = np.nan # >> will be interpolated later
#then delete all downloads in the folder, no matter what type
if os.path.isdir(yourpath + "mastDownload") == True:
shutil.rmtree(yourpath + "mastDownload")
print("Download folder deleted.")
#corrects for connnection errors
except (ConnectionError, OSError, TimeoutError, RemoteServiceError):
print(targ, " could not be accessed due to an error.")
i1 = 0
time1 = 0
ticid = 0
return time1, i1, ticid
# PARAMETERS :
# - a (array_like) = obs_table
# RETURNS :
# - ndarray (index_array) = *finds specified images in obs_table**
#
# **astroquery.mast.download_products** [[documentation](https://astroquery.readthedocs.io/en/latest/api/astroquery.mast.ObservationsClass.html#astroquery.mast.ObservationsClass.download_products)]
#
# PARAMETERS :
# - products
# - productSubGroupDescription = ['RAW']
# - mrp_only = False (*default False*)
# In[ ]:
obs_table = Observations.query_criteria(instrument_name='ACS/WFC',
intentType='calibration',
target_name='BIAS',
calib_level=1)
#below are images listed in Table 01 - 07 from Grogin et al. 2011
#power-spectrum analysis [Table 05]
idx_ps = [
'ja8wa2e8q', 'ja8wa3fpq', 'ja8wa6jmq', 'ja8wa7jqq', 'ja8waaloq',
'ja8wablsq', 'ja8waedtq', 'ja8wafe2q', 'ja8wain1q', 'ja8wajnmq',
'ja8wambyq', 'ja8wanc2q', 'ja8wb0n3q', 'ja8wb1ndq', 'ja8wb4izq',
'ja8wb5j6q'
]
#striping mitigation testing [Table 06]
idx_sm = [
'ja8wc2gjq', 'ja8wc3gvq', 'ja8wc6xlq', 'ja8wc7yaq', 'ja8wcae8q',
'ja8wcbfbq', 'ja8wcetxq', 'ja8wcfugq', 'ja8wciipq', 'ja8wcjjdq',
'ja8wcmxiq', 'ja8wcnxmq', 'ja8wd0t3q', 'ja8wd1tnq', 'ja8wd4hhq',
def get_twomin_obs(ticlist):
"""
Given a list of ticids, return a table of all the data that is available on that target.
The table will be well formated to just be one one describing
the available sectors per target.
Input: list of ticids, int
Return: full table (astropy table of tic, obsid, filenames, camera, ccd)
for all relevant two minute data files.
summary table (one line per ticid), list of sectors, list of camera and ccd
"""
sectors = []
gis = []
obsid = []
tmin = []
tmax = []
exptime = []
target = []
dv_sectors = []
dv_obsid = []
dv_tmin = []
dv_tmax = []
dv_target = []
dv_exptime = []
for tic in ticlist:
observations = Observations.query_criteria(
obs_collection="TESS",
dataproduct_type=["timeseries"],
target_name=tic)
if len(observations) > 0:
observations.sort(keys=['sequence_number', 'target_name'])
for obs in observations:
#Match for multi-sector observations
match = re.search("s\d\d\d\d-s\d\d\d\d", obs['obs_id'])
if match == None:
if int(obs['target_name']) == int(tic):
obsid.append(obs['obs_id'])
gis.append(obs['proposal_id'])
sectors.append(obs['sequence_number'])
tmin.append(obs['t_min'])
tmax.append(obs['t_max'])
exptime.append(int(obs['t_exptime'] / 60.0))
target.append(obs['target_name'])
else:
dv_sectors.append(match.group(0))
dv_obsid.append(obs['obs_id'])
dv_tmin.append(obs['t_min'])
dv_tmax.append(obs['t_max'])
dv_target.append(obs['target_name'])
dv_exptime.append(int(obs['t_exptime'] / 60.0))
ts_table = QTable([target, sectors, gis, obsid, exptime, tmin, tmax],
names=('target_tic', 'sectors', 'GI_nums', 'obs_id',
'Exp_time_min', 't_min', 't_max'),
meta={'name': 'observation table'})
dv_table = QTable(
[dv_target, dv_sectors, dv_obsid, dv_exptime, dv_tmin, dv_tmax],
names=('target_tic', 'sector_range', 'obs_id', 'Exp_time_min', 't_min',
't_max'),
meta={'name': 'Multi-Sector Planet Search'})
return ts_table, dv_table
def retrieve_observation(obsid,
suffix=['FLC'],
archive=False,
clobber=False,
product_type=None):
"""Simple interface for retrieving an observation from the MAST archive
If the input obsid is for an association, it will request all members with
the specified suffixes.
Parameters
-----------
obsid : string or list of strings
ID or list of IDs for observations to be retrieved from the MAST archive.
Only the IPPSSOOT (rootname) of exposure or ASN needs to be provided; eg.,
ib6v06060. Additionally, a wild-carded `obsid` can be provided to
retrieve all products for a visit; e.g., "ib6v06*". Data from multiple
ASNs, exposures or visits can be retrieved by simply providing them as a list.
suffix : list, optional
List containing suffixes of files which should be requested from MAST.
Default value "['FLC']".
archive : Boolean, optional
Retain copies of the downloaded files in the astroquery created
sub-directories? Default is "False".
clobber : Boolean, optional
Download and Overwrite existing files? Default is "False".
product_type : str, optional
Specify what type of product you want from the archive, either 'pipeline'
or 'HAP' or 'both' (default). By default, all versions of the products
processed for the requested datasets will be returned. This would include:
- pipeline : files processed by `runastrodriz` to include the latest
distortion calibrations and the best possible alignment to GAIA
with `ipppssoot_fl[tc].fits` filenames for FLT/FLC files.
- HAP : files processed as a single visit and aligned (as possible) to GAIA
with `hst_<propid>_<visit>_<instr>_<det>_<filter>_<ipppssoo>_fl[tc].fits`
filenames.
Returns
-------
local_files : list
List of filenames
"""
local_files = []
if Observations is None:
log.warning(
"The astroquery package was not found. No files retrieved!")
return local_files
# Query MAST for the data with an observation type of either "science" or
# "calibration"
obs_table = Observations.query_criteria(obs_id=obsid)
# Catch the case where no files are found for download
if not obs_table:
log.info("WARNING: Query for {} returned NO RESULTS!".format(obsid))
return local_files
dpobs = Observations.get_product_list(obs_table)
if product_type:
ptypes = [
product_type_dict[product_type] in descr
for descr in dpobs['description']
]
dpobs = dpobs[ptypes]
data_products_by_id = Observations.filter_products(
dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# After the filtering has been done, ensure there is still data in the
# table for download. If the table is empty, look for FLT images in lieu
# of FLC images. Only want one or the other (not both!), so just do the
# filtering again.
if not data_products_by_id:
log.info("WARNING: No FLC files found for {} - will look for FLT "
"files instead.".format(obsid))
suffix = ['FLT']
data_products_by_id = Observations.filter_products(
dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# If still no data, then return. An exception will eventually be
# thrown in the higher level code.
if not data_products_by_id:
log.info(
"WARNING: No FLC or FLT files found for {}.".format(obsid))
return local_files
all_images = data_products_by_id['productFilename'].tolist()
log.info(all_images)
if not clobber:
rows_to_remove = []
for row_idx, row in enumerate(data_products_by_id):
fname = row['productFilename']
if os.path.isfile(fname):
log.info(fname + " already exists. File download skipped.")
rows_to_remove.append(row_idx)
data_products_by_id.remove_rows(rows_to_remove)
manifest = Observations.download_products(data_products_by_id,
mrp_only=False)
if not clobber:
for rownum in rows_to_remove[::-1]:
if manifest:
manifest.insert_row(
rownum, vals=[all_images[rownum], "LOCAL", "None", "None"])
else:
return all_images
download_dir = None
for file, file_status in zip(manifest['Local Path'], manifest['Status']):
if file_status != "LOCAL":
# Identify what sub-directory was created by astroquery for the
# download
if download_dir is None:
download_dir = os.path.dirname(os.path.abspath(file))
# Move or copy downloaded file to current directory
local_file = os.path.abspath(os.path.basename(file))
if archive:
shutil.copy(file, local_file)
else:
shutil.move(file, local_file)
# Record what files were downloaded and their current location
local_files.append(os.path.basename(local_file))
else:
local_files.append(file)
if not archive:
# Remove astroquery created sub-directories
shutil.rmtree('mastDownload')
return local_files
# Convert the duration to days from hours
duration_guess = np.array(tois.pl_trandurh, dtype=float) / 24.0
tois[["tid", "toi", "pl_orbper", "pl_trandep", "pl_trandurh"]]
# %% [markdown]
# Then we can search for and download the light curves.
# Note that this will fail if there is no 2-minute cadence light curve for this target.
# Typically this would be executed using [lightkurve](https://docs.lightkurve.org) directly, but we'll use the MAST API directly because the lightkurve light curve search is a little slow currently.
# %%
# Temporary workaround for slow MAST queries with lightkurve
observations = Observations.query_criteria(
target_name=f"{tic}",
radius=0.0001,
project=["TESS"],
obs_collection=["TESS"],
provenance_name="SPOC",
dataproduct_type="timeseries",
)
if not len(observations):
raise RuntimeError("no 2-minute cadence data")
products = Observations.get_product_list(observations)
products = products[products["productSubGroupDescription"] == "LC"]
files = Observations.download_products(
products, download_dir=tess_world.get_lightkurve_directory())
lcfs = lk.LightCurveCollection(
[lk.open(file).PDCSAP_FLUX for file in files["Local Path"]])
lc = lcfs.stitch().remove_nans()
# Extract the data in the correct format
x = np.ascontiguousarray(lc.time, dtype=np.float64)
#!/usr/bin/env python3
#
#
from astroquery.mast import Observations
import boto3
import json
import os
import IPython
wfc = Observations.query_criteria(
dataproduct_type=['image'],
project='HST',
instrument_name='ACS/WFC',
)
print("WFC: ", len(wfc))
hrc = Observations.query_criteria(
dataproduct_type=['image'],
project='HST',
instrument_name='ACS/HRC',
)
print("HRC: ", len(hrc))
IPython.embed()
