def astroquery_sci(data_start, data_end, proposal_id, paths, download):
"""Retreives raw science data for a given proposal ID using astroquery
To get the right files, astroquery requires the following inputs:
- intentType='science' – for the darks
- instrument_name="WFC3/UVIS" – or whichever instrument is needed
- proposal_id -- user-defined input string of program/proposal ID
"""
# Collecting the raw science files for a program ID using astroquery
if data_start:
# Round the data end date down so that the file from the new date isn't included
data_end_rnd = data_end // 0.001 * 0.001
sciobs = Observations.query_criteria(intentType='science',
t_min=[data_start, data_end_rnd],
instrument_name="WFC3/UVIS",
proposal_id=proposal_id)
else:
sciobs = Observations.query_criteria(intentType='science',
instrument_name="WFC3/UVIS",
proposal_id=proposal_id)
sciprod = Observations.get_product_list(sciobs)
rawsci = Observations.filter_products(sciprod,
productSubGroupDescription="RAW")
# If download option is set, move to the download directory, check if empty, then download
if download == True:
os.chdir(paths['PID_DIR'])
if os.path.exists(paths['DLD_DIR']):
print('Download directory exists!! Not downloading files')
else:
Observations.download_products(rawsci, mrp_only=False)
print('Download to {} complete'.format(paths['DLD_DIR']))
def filter_products(obsids, table):
""" filters the dataproducts for the given astroquery by obsids found in obs_id() and limits to only DRZ files
before downloading all filtered products to /Volumes/galaxies"""
## move indexing obsids to another function
obs = table['obsid']
dataProductsByID = Observations.get_product_list(obs)
dataProductsByID = Observations.filter_products(dataProductsByID, obs_id=obsids, productSubGroupDescription="DRZ")
Observations.download_products(dataProductsByID, "/Volumes/galaxies/mastDownload/HST") ## SPECIFY filepath!!!
def get_datafiles(self, ra, dec, verbose=True):
"""
Given an ra and dec create the correct output directory,
and download associated drizzle files
"""
coords = "{}_{}".format(ra, dec)
if verbose:
print(coords)
# Querying MAST and downloading all associated drz files
table_of_observations = Observations.query_criteria(
coordinates=coords,
radius=0.02,
obs_collection="HST",
dataproduct_type="image")
data_product_list = Observations.get_product_list(
table_of_observations)
manifest = Observations.download_products(
data_product_list,
extension="drz.fits",
obs_collection="HST",
download_dir=self.download_directory)
return list(
manifest["Local Path"]) # We just want the local file locations
def get_kepler_lcs(filenames):
"""
Purpose:
Downloads lightcurves
Args:
filenames - list of full filenames
returns:
paths_to_files - list containing the full paths to downloaded lightcurves.
"""
obj_ids = [i[:13] for i in filenames]
keplerObs = Observations.query_criteria(target_name=obj_ids,
obs_collection='Kepler')
keplerProds = Observations.get_product_list(keplerObs)
yourProd = Observations.filter_products(keplerProds, extension=filenames)
manifest = Observations.download_products(yourProd)
"""
The process of downloading the files sorts them in numerical order messing up the
order, so I'm making the manifest a dataframe and sampling it
one at a time from the filenames, not sure if there's a way to just sort the whole list
based on the original order.
"""
manifest = manifest.to_pandas(index='Local Path')
paths_to_files = []
for i, f in enumerate(
filenames): # forcing the order to match given filenames list
f_sampler = make_sampler([f]) # to find the right file from manifest
filename = f_sampler(manifest).index[0] # full local filepath
paths_to_files.append(filename)
return paths_to_files
def get_hlsp_lightcurves(tic_id,
hlsp_products=('CDIPS', 'TASOC'),
download_dir=None,
verbose=True):
"""This downloads TESS HLSP light curves for a given TIC ID.
Parameters
----------
tic_id : str
The TIC ID of the object as a string.
hlsp_products : sequence of str
List of desired HLSP products to search. For instance, ["CDIPS"].
download_dir : str
Path of directory to which light-curve will be downloaded.
Returns
-------
lcfiles : list or None
List of light-curve file paths. None if none are found and downloaded.
"""
if not astroquery_dependency:
LOGERROR(
"The astroquery package is required for this function to work.")
return None
lcfiles = []
for hlsp in hlsp_products:
obs_table = Observations.query_criteria(target_name=tic_id,
provenance_name=hlsp)
if verbose:
LOGINFO('Found {} {} light-curves.'.format(len(obs_table), hlsp))
# Get list of available products for this Observation.
cdips_products = Observations.get_product_list(obs_table)
# Download the products for this Observation.
manifest = Observations.download_products(cdips_products,
download_dir=download_dir)
if verbose:
LOGINFO("Done")
if len(manifest) >= 1:
lcfiles.append(list(manifest['Local Path']))
#
# flatten lcfiles list
#
if len(lcfiles) >= 1:
return_lcfiles = [item for sublist in lcfiles for item in sublist]
else:
return_lcfiles = None
return return_lcfiles
def download_target(targetname, radius='10 arcsec'):
download_dir = os.path.join(os.path.expanduser('~'), '.shadow-cache')
if not os.path.isdir(download_dir):
try:
os.mkdir(download_dir)
# downloads locally if OS error occurs
except OSError:
log.warning('Warning: unable to create {}. '
'Downloading MAST files to the current '
'working directory instead.'.format(download_dir))
download_dir = '.'
logging.getLogger('astropy').setLevel(log.getEffectiveLevel())
obsTable = Observations.query_criteria(target_name=targetname,
obs_collection='HST',
project='HST',
radius=radius)
obsTable = obsTable[(obsTable['instrument_name'] == 'WFC3/IR') &
(obsTable['dataRights'] == "PUBLIC")]
fnames = ['{}/mastDownload/'.format(download_dir) +
url.replace('mast:', '').replace('product', url.split('/')[-1].split('_')[0])[:-9] +
'_flt.fits' for url in obsTable['dataURL']]
log.info('Found {} files.'.format(len(obsTable)))
paths = []
for idx, t in enumerate(tqdm(obsTable, desc='Downloading files', total=len(obsTable))):
if os.path.isfile(fnames[idx]):
paths.append(fnames[idx])
else:
t1 = Observations.get_product_list(t)
t1 = t1[t1['productSubGroupDescription'] == 'FLT']
paths.append(Observations.download_products(t1, mrp_only=False, download_dir=download_dir)['Local Path'][0])
return paths
def get_all_TESS_data(object_name, radius = ".02 deg", get_PDC = True, get_all = False):
"""
Given a planet name, this function returns a dictionary of times, fluxes and
errors on fluxes in a juliet-friendly format for usage. The function does an
astroquery to MAST using a default radius of .02 deg around the target name. If get_PDC is True,
this function returns PDC fluxes. False returns SAP fluxes. If get_all is true, this function
returns a dictionary that in addition to the times, fluxes and errors, returns other
metadata.
"""
if not has_astroquery:
print("Error on using juliet function `get_all_TESS_data`: astroquery.mast not found.")
obs_table = Observations.query_object(object_name,radius=radius)
out_dict = {}
times = {}
fluxes = {}
fluxes_errors = {}
for i in range(len(obs_table['dataURL'])):
if 's_lc.fits' in obs_table['dataURL'][i]:
fname = obs_table['dataURL'][i].split('/')[-1]
metadata = fname.split('-')
if len(metadata) == 5:
# Extract metadata:
sector = np.int(metadata[1].split('s')[-1])
ticid = np.int(metadata[2])
# Download files:
data_products = Observations.get_product_list(obs_table[i])
manifest = Observations.download_products(data_products)
# Read lightcurve file:
d,h = fits.getdata('mastDownload/TESS/'+fname[:-8]+'/'+fname,header=True)
t,fs,fserr,f,ferr = d['TIME']+h['BJDREFI'],d['SAP_FLUX'],d['SAP_FLUX_ERR'],\
d['PDCSAP_FLUX'],d['PDCSAP_FLUX_ERR']
idx_goodpdc = np.where((f != 0.)&(~np.isnan(f)))[0]
idx_goodsap = np.where((fs != 0.)&(~np.isnan(fs)))[0]
# Save to output dictionary:
if 'TIC' not in out_dict.keys():
out_dict['TIC'] = ticid
out_dict[sector] = {}
out_dict[sector]['TIME_PDCSAP_FLUX'] = t[idx_goodpdc]
out_dict[sector]['PDCSAP_FLUX'] = f[idx_goodpdc]
out_dict[sector]['PDCSAP_FLUX_ERR'] = ferr[idx_goodpdc]
out_dict[sector]['TIME_SAP_FLUX'] = t[idx_goodsap]
out_dict[sector]['SAP_FLUX'] = fs[idx_goodsap]
out_dict[sector]['SAP_FLUX_ERR'] = fserr[idx_goodsap]
if get_PDC:
times['TESS'+str(sector)] = t[idx_goodpdc]
med = np.median(f[idx_goodpdc])
fluxes['TESS'+str(sector)] = f[idx_goodpdc]/med
fluxes_errors['TESS'+str(sector)] = ferr[idx_goodpdc]/med
else:
times['TESS'+str(sector)] = t[idx_goodsap]
med = np.median(fs[idx_goodsap])
fluxes['TESS'+str(sector)] = fs[idx_goodsap]/med
fluxes_errors['TESS'+str(sector)] = fserr[idx_goodsap]/med
# Remove downloaded folder:
os.system('rm -r mastDownload')
if get_all:
return out_dict, times, fluxes, fluxes_errors
else:
return times, fluxes, fluxes_errors
def download(self, quality_bitmask='default', download_dir=None):
"""Returns a single `KeplerTargetPixelFile` or `KeplerLightCurveFile` object.
If multiple files are present in `SearchResult.table`, only the first
will be downloaded.
Parameters
----------
quality_bitmask : str or int
Bitmask (integer) which identifies the quality flag bitmask that should
be used to mask out bad cadences. If a string is passed, it has the
following meaning:
* "none": no cadences will be ignored (`quality_bitmask=0`).
* "default": cadences with severe quality issues will be ignored
(`quality_bitmask=1130799`).
* "hard": more conservative choice of flags to ignore
(`quality_bitmask=1664431`). This is known to remove good data.
* "hardest": removes all data that has been flagged
(`quality_bitmask=2096639`). This mask is not recommended.
See the :class:`KeplerQualityFlags` class for details on the bitmasks.
download_dir : str
Location where the data files will be stored.
Defaults to "~/.lightkurve-cache" if `None` is passed.
Returns
-------
data : `TargetPixelFile` or `LightCurveFile` object
The first entry in the products table.
"""
if len(self.table) == 0:
warnings.warn("Cannot download from an empty search result.",
LightkurveWarning)
return None
if len(self.table) != 1:
warnings.warn('Warning: {} files available to download. '
'Only the first file has been downloaded. '
'Please use `download_all()` or specify a campaign, quarter, or '
'cadence to limit your search.'.format(len(self.table)),
LightkurveWarning)
# Make sure astroquery uses the same level of verbosity
logging.getLogger('astropy').setLevel(log.getEffectiveLevel())
# download first product in table
if download_dir is None:
download_dir = self._default_download_dir()
path = Observations.download_products(self.table[:1], mrp_only=False,
download_dir=download_dir)['Local Path']
# return single tpf or lcf
tpf_files = ['lpd-targ.fits', 'spd-targ.fits']
lcf_files = ['llc.fits', 'slc.fits']
if any(file in self.table['productFilename'][0] for file in tpf_files):
return KeplerTargetPixelFile(path[0], quality_bitmask=quality_bitmask)
elif any(file in self.table['productFilename'][0] for file in lcf_files):
return KeplerLightCurveFile(path[0], quality_bitmask=quality_bitmask)
def download_mast(obs_table,**kwargs):
"""Download data from the MAST archive.
This function gives you an easy way to download data from the MAST
archive, if you already have a table of observations (from Astroquery)
to hand. By default, this will download everything into your current
working directory.
Args:
obs_table (astropy.table.Table): The result of an astroquery.mast
query.
"""
Observations.download_products(obs_table,**kwargs
)
def dload_url(tic_id, sector): #Dowloads URL through query for general 2-minute cadence data given TIC ID and Sector
obsTable = Observations.query_criteria(
obs_collection = "TESS",
dataproduct_type = ["TIMESERIES"],
target_name = tic_id,
sequence_number=sector)
products = Observations.get_product_list(obsTable)
manifest = Observations.download_products(products, extension = ['fits'])
return manifest
def download_lightcurve(obsid):
"""Return a pandas dataframe of lightcurve data for one source."""
dl = Observations.download_products(obsid, productSubGroupDescription='LC')
fn = dl['Local Path'][0]
hdul = fits.open(fn)
lc = Table(hdul[1].data)
lc['NORM_PDCSAP_FLUX'] = lc['PDCSAP_FLUX'] / np.nanpercentile(
lc['PDCSAP_FLUX'], 50)
data = lc.to_pandas()
return data
def _download_one(self, table, quality_bitmask, download_dir, cutout_size):
"""Private method used by `download()` and `download_all()` to download
exactly one file from the MAST archive.
Always returns a `TargetPixelFile` or `LightCurveFile` object.
"""
# Make sure astroquery uses the same level of verbosity
logging.getLogger('astropy').setLevel(log.getEffectiveLevel())
if download_dir is None:
download_dir = self._default_download_dir()
# if the SearchResult row is a TESScut entry, then download cutout
if 'FFI Cutout' in table[0]['description']:
try:
log.debug("Started downloading TESSCut for '{}' sector {}."
"".format(table[0]['target_name'],
table[0]['sequence_number']))
path = self._fetch_tesscut_path(table[0]['target_name'],
table[0]['sequence_number'],
download_dir, cutout_size)
log.debug("Finished downloading.")
except Exception as exc:
msg = str(exc)
if "504" in msg:
# TESSCut will occasionally return a "504 Gateway Timeout
# error" when it is overloaded.
raise HTTPError(
'The TESS FFI cutout service at MAST appears '
'to be temporarily unavailable. It returned '
'the following error: {}'.format(exc))
else:
raise SearchError(
'Unable to download FFI cutout. Desired target '
'coordinates may be too near the edge of the FFI.'
'Error: {}'.format(exc))
return _open_downloaded_file(path,
quality_bitmask=quality_bitmask,
targetid=table[0]['targetid'])
else:
if cutout_size is not None:
warnings.warn(
'`cutout_size` can only be specified for TESS '
'Full Frame Image cutouts.', LightkurveWarning)
from astroquery.mast import Observations
log.debug("Started downloading {}.".format(
table[:1]['dataURL'][0]))
path = Observations.download_products(
table[:1], mrp_only=False,
download_dir=download_dir)['Local Path'][0]
log.debug("Finished downloading.")
# open() will determine filetype and return
return _open_downloaded_file(path, quality_bitmask=quality_bitmask)
def download_all(self, quality_bitmask='default', download_dir=None):
"""Returns a `TargetPixelFileCollection or `LightCurveFileCollection`.
Parameters
----------
quality_bitmask : str or int
Bitmask (integer) which identifies the quality flag bitmask that should
be used to mask out bad cadences. If a string is passed, it has the
following meaning:
* "none": no cadences will be ignored (`quality_bitmask=0`).
* "default": cadences with severe quality issues will be ignored
(`quality_bitmask=1130799`).
* "hard": more conservative choice of flags to ignore
(`quality_bitmask=1664431`). This is known to remove good data.
* "hardest": removes all data that has been flagged
(`quality_bitmask=2096639`). This mask is not recommended.
See the :class:`KeplerQualityFlags` class for details on the bitmasks.
download_dir : str
Location where the data files will be stored.
Defaults to "~/.lightkurve-cache" if `None` is passed.
Returns
-------
collection : `lightkurve.Collection` object
Returns a `LightCurveFileCollection` or `TargetPixelFileCollection`,
containing all entries in the products table
"""
if len(self.table) == 0:
warnings.warn("Cannot download from an empty search result.",
LightkurveWarning)
return None
# Make sure astroquery uses the same level of verbosity
logging.getLogger('astropy').setLevel(log.getEffectiveLevel())
# download all products listed in self.products
if download_dir is None:
download_dir = self._default_download_dir()
path = Observations.download_products(self.table, mrp_only=False,
download_dir=download_dir)['Local Path']
# return collection of tpf or lcf
tpf_files = ['lpd-targ.fits', 'spd-targ.fits']
lcf_files = ['llc.fits', 'slc.fits']
if any(file in self.table['productFilename'][0] for file in tpf_files):
tpfs = [KeplerTargetPixelFile(p, quality_bitmask=quality_bitmask)
for p in path]
return TargetPixelFileCollection(tpfs)
elif any(file in self.table['productFilename'][0] for file in lcf_files):
lcs = [KeplerLightCurveFile(p, quality_bitmask=quality_bitmask)
for p in path]
return LightCurveFileCollection(lcs)
def astroquery_darks(anneal_start, anneal_end, paths, download):
"""Retreives darks within an anneal cycle using astroquery
To get the right files, astroquery requires the following inputs:
- intentType='calibration' – for the darks
- instrument_name="WFC3/UVIS" – or whichever instrument is needed
- t_min – Anneal dates start and end defined above
- target_name=DARK* – all files identified as a type of dark
NO exposure time should be given (too stringent a cut). The astroquery
command returns more than the required files than would be collected
from MAST. However, the darks CTE correction code then selects only the
files it needs from these, so this is not a problem.
Astroquery takes the start and end dates of the anneal cycle. The end
anneal date in the astroquery command below is slightly reduced so that
the file at the start of the next anneal is not included.
e.g. [58170.7838 (start of Feb 21 2018 anneal), 58201.3608 (start of next Mar 24 2018 anneal]
--> [58170.7838, 58201.360]"""
# Round the next anneal start date down so that the file from the new date isn't included
anneal_end_rnd = anneal_end // 0.001 * 0.001
# Collecting the raw dark files for one anneal cycle using astroquery
darkobs = Observations.query_criteria(intentType='calibration',
instrument_name="WFC3/UVIS",
t_min=[anneal_start, anneal_end_rnd],
target_name="DARK*")
darkprod = Observations.get_product_list(darkobs)
rawdark = Observations.filter_products(darkprod,
productSubGroupDescription="RAW")
# If download option is set, move to the anneal directory, check if empty, then download
if download == True:
os.chdir(paths['ANN_DIR'])
if os.path.exists(paths['DLD_DIR']):
print('Download directory exists!! Not downloading files')
else:
Observations.download_products(rawdark, mrp_only=False)
print('Download to {} complete'.format(paths['DLD_DIR']))
def download_tess_data(target_name):
r"""Run a query to MAST to obtain the TESS data of an object.
Data is saved in the directories: `./mastDownload/TESS/<target_name>_*/*.fits`
Args:
target_name (str): name of the target: e.g., '231663901'
Returns:
bool: it returns `True` if data are retrieved and `False` otherwise.
"""
tess_obs = Observations.query_criteria(target_name=target_name, obs_collection='TESS')
if len(tess_obs) == 0:
msgs.warning('No TESS data for target {}'.format(target_name))
return False
for idx in np.arange(0, len(tess_obs)):
tess_prods = Observations.get_product_list(tess_obs[idx])
tess_fits_prods = Observations.filter_products(tess_prods, extension='fits', mrp_only=False)
Observations.download_products(tess_fits_prods, mrp_only=False, cache=False)
return True
def download_kepler_data(target_name):
r"""Run a query to MAST to obtain the kepler data of an object
The data is saved in the directories: `./mastDownload/Kepler/<target_name>_*/*.fits`
Args:
target_name (str): name of the target: e.g., 'kplr011446443'
Returns:
bool: it returns `True` if data are retrieved and `False` otherwise.
"""
kepler_obs = Observations.query_criteria(target_name=target_name, obs_collection='Kepler')
if len(kepler_obs) == 0:
msgs.warning('No Kepler data for target {}'.format(target_name))
return False
for idx in np.arange(0, len(kepler_obs)):
kepler_prods = Observations.get_product_list(kepler_obs[idx])
kepler_fits_prods = Observations.filter_products(kepler_prods, extension='fits', mrp_only=False)
Observations.download_products(kepler_fits_prods, mrp_only=False, cache=False)
return True
def download_products(data_products: pd.DataFrame, data_directory: Path) -> pd.DataFrame:
"""
A wrapper for MAST's `download_products`, allowing the use of Pandas DataFrames instead of AstroPy Tables.
Retries on error when communicating with the MAST server.
:param data_products: The data frame of data products to download. Will be converted from DataFrame to Table
for sending the request to MAST.
:param data_directory: The path to download the data to.
:return: The manifest of the download. Will be converted from Table to DataFrame for use.
"""
manifest = Observations.download_products(Table.from_pandas(data_products), download_dir=str(data_directory))
return manifest.to_pandas()
def get_galex_image(ra, dec, imsize):
"""
get galex image of a galaxy
Input:
* ra in deg
* dec in deg
* imsize in arcsec
Returns:
* image
"""
# following procedure outlined here:
# https://astroquery.readthedocs.io/en/latest/mast/mast.html
# get data products in region near ra,dec
obs_table = Observations.query_region("%12.8f %12.8f" % (c.ra, c.dec),
radius=.1 * u.arcmin)
# create a flag to select galex data
galexFlag = obs_table['obs_collection'] == 'GALEX'
# separate out galex data
data_products = Observations.get_product_list(obs_table[galexFlag])
# download the observations
manifest = Observations.download_products(data_products,
productType="SCIENCE")
for m in manifest:
# choose the first NUV image
if m['Local Path'].find('nd-int') > -1:
nuv_path = m['Local Path']
break
# should be able to construct path from the obs_id in data_products
# this will let us check if the image is already downloaded
#
# I can also save the cutout in a GALEX folder, and
# look for the image before calling this function
nuv, nuv_header = fits.getdata(nuv_path, header=True)
# this is a big image, so we need to get a cutout
nuv_wcs = WCS(nuv_header)
position = SkyCoord(ra, dec, unit="deg", frame='icrs')
cutout = Cutout2D(nuv,
position, (imsize * u.arcsec, imsize * u.arcsec),
wcs=nuv_wcs)
return cutout
def plot_lc(file, filepath='./fitsFiles/', local=True, c='blue', ax=False):
"""
kid should be full id including time information
Args:
file (str) - filename starting with kplr ending in .fits
filepath (str) - path to fits file
Returns:
None
"""
if local:
f = filepath + file
else:
filenames = [file]
obj_ids = [i[:13] for i in tmp.index[:n]]
keplerObs = Observations.query_criteria(target_name=obj_ids,
obs_collection='Kepler')
keplerProds = Observations.get_product_list(keplerObs)
yourProd = Observations.filter_products(keplerProds, extension=file)
manifest = Observations.download_products(yourProd)
manifest = manifest.to_pandas(index='Local Path')
f = manifest.index[0]
if not ax:
fig = plt.figure(figsize=(16, 4))
ax = fig.add_subplot(111)
t, nf, err = read_kepler_curve(f)
x = t
y = nf
axrange = 0.55 * (max(y) - min(y))
mid = (max(y) + min(y)) / 2
yaxmin = mid - axrange
yaxmax = mid + axrange
if yaxmin < .95:
if yaxmax > 1.05:
ax.set_ylim(yaxmin, yaxmax)
else:
ax.set_ylim(yaxmin, 1.05)
elif yaxmax > 1.05:
ax.set_ylim(.95, yaxmax)
else:
ax.set_ylim(.95, 1.05)
ax.set_xlim(min(x), max(x))
color = c
ax.plot(x, y, 'o', markeredgecolor='none', c=color, alpha=0.2)
ax.plot(x, y, '-', markeredgecolor='none', c=color, alpha=0.7)
#ax2.set_title(files[index][:13],fontsize = 20)
ax.set_ylabel(r'$\frac{\Delta F}{F}$', fontsize=25)
ax.tick_params(labelsize=20)
def drawData(index):
# Plots the lightcurve of the point chosen
ax2.cla()
### for fits files saved locally
#f = pathtofits+df.index[index]
#t,nf,err=read_kepler_curve(f)
### for fits files that need downloaded
f = df.index[index] # full file name ****-****_llc.fits
obj_id = f[:13] # observation id kplr********
keplerObs = Observations.query_criteria(target_name=obj_id,
obs_collection='Kepler')
keplerProds = Observations.get_product_list(keplerObs)
yourProd = Observations.filter_products(keplerProds, extension=f)
manifest = Observations.download_products(yourProd)
filename = manifest[0][0]
t, nf, err = qt.read_kepler_curve(filename)
shutil.rmtree('./mastDownload') # removing the downloaded data
x = t
y = nf
axrange = 0.55 * (max(y) - min(y))
mid = (max(y) + min(y)) / 2
yaxmin = mid - axrange
yaxmax = mid + axrange
if yaxmin < .95:
if yaxmax > 1.05:
ax2.set_ylim(yaxmin, yaxmax)
else:
ax2.set_ylim(yaxmin, 1.05)
elif yaxmax > 1.05:
ax2.set_ylim(.95, yaxmax)
else:
ax2.set_ylim(.95, 1.05)
ax2.plot(x,
y,
'o',
markeredgecolor='none',
c=colorVal[index],
alpha=0.2)
ax2.plot(x,
y,
'-',
markeredgecolor='none',
c=colorVal[index],
alpha=0.7)
#ax2.set_title(files[index][:13],fontsize = 20)
ax2.set_xlabel('Time (Days)', fontsize=22)
ax2.set_ylabel(r'$\frac{\Delta F}{F}$', fontsize=30)
fig.suptitle(files[index][:13], fontsize=30)
canvas.draw()
def download_products(query_products, output_dir=''):
""" Downloads all products in `query_products` to `output_dir`.
Parameters
----------
query_products : `astropy.table.Table`
Table of data products to download.
Notes
-----
Files are initially downladed temporary directory within `output_dir`
called 'temp', so if a subdirectory `temp` already exists within
`output_dir` an error is raised. ."""
# format path for output dir
if output_dir == '':
output_dir = os.getcwd()
output_dir = os.path.join(output_dir, '')
# make temp dir in `output_dir`. error if it exists.
print(output_dir + 'temp')
assert (os.path.isdir(output_dir + 'temp') is False)
os.makedirs(output_dir + 'temp')
print('Downloading {} files.'.format(len(query_products)))
Observations.download_products(query_products,
download_dir=output_dir + 'temp',
mrp_only=False)
# move files from mast download directories in `temp` to `output_dir`
files = glob.glob(output_dir + 'temp/mastDownload/HST/*/*.fits')
if len(files) > 0:
print('Cleaning up temp directory.')
for f in files:
shutil.copy(f, output_dir + os.path.basename(f))
# remove temp directory
shutil.rmtree(output_dir + 'temp') # remove mast download dir
def plot_top_n(Q_coo, n=10, sortby='sAverage', top_n_df=False):
"""
Purpose:
Plots the top n outlier lightcurves for a given quarter. This assumes data is saved in the standard
way for coo files, i.e. that feature data is saved as the attribute 'data' in a dataframe, that outlier scores are
saved as 'scores', and that the indices of each are the full file names.
Args:
Q_coo (cluster outlier object) - The cluster outlier object containing the feature data and outlier scores attributes
n (int) - number of plots to create of most outlying points
sortby (optional, str) - how to sort scores, default is the sampled average of k=4 to 13
top_n_df (optional, boolean) - whether or not to return a dataframe containing the features of the top outliers
Returns:
top_n_feats (optional, dataframe) - pandas dataframe containing the features of the top n outliers
"""
tmp = Q_coo.scores.sort_values(by=sortby, ascending=False)
filenames = list(tmp.index[:n])
obj_ids = [i[:13] for i in tmp.index[:n]]
keplerObs = Observations.query_criteria(target_name=obj_ids,
obs_collection='Kepler')
keplerProds = Observations.get_product_list(keplerObs)
yourProd = Observations.filter_products(keplerProds, extension=filenames)
manifest = Observations.download_products(yourProd)
"""
The process of downloading the files sorts them in numerical order messing up the
order of most outlying to leas, so I'm making the manifest a dataframe and sampling it
one at a time from the filenames, not sure if there's a way to just sort the whole list
based on the original order.
"""
manifest = manifest.to_pandas(index='Local Path')
for i, f in enumerate(
filenames): # forcing the order to match most to least outlying
f_sampler = make_sampler([f]) # to find the right file from manifest
filename = f_sampler(manifest).index[0] # full local filepath
fig = plt.figure(figsize=(15, 1))
ax = fig.add_subplot(111)
t, nf, err = read_kepler_curve(filename)
ax.errorbar(t, nf, err)
plt.title('KIC {}'.format(int(obj_ids[i][4:])))
shutil.rmtree('./mastDownload') # removing the downloaded data
if top_n_df:
top_n_sampler = make_sampler(tmp[:n].index)
top_n_feats = top_n_sampler(Q_coo.data)
return top_n_feats
else:
return
def download(self, key):
"""Download the data
Only download the observations contained in the interval specified by
the `key`. The `key` argument must correspond to one of the keys in the
:py:attr:`~download.Downloader.products` attribute. If it is not,
then a KeyError will be raised and the download will be skipped.
Parameters
----------
key : str
Date in ISO format (YYYY-MM-DD) of a given intervals start time
Returns
-------
None
Downloaded data will be stored in directory specified by the
:py:attr:`~download.Downloader.download_dir` attribute
"""
msg = ('Downloading data...\n '
'Download Directory: {}\n {}'.format(self.download_dir,
self._msg_div))
LOG.info(msg)
download_params = {
'download_dir': self.download_dir,
'mrp_only': False,
'dataproduct_type': self.product_type,
'productSubGroupDescription': self.SubGroupDescription
}
try:
download_list = self.products[key]['obsID'].tolist()
except KeyError as e:
LOG.error('{}\n{}'.format(e, self._msg_div))
else:
Observations.download_products(download_list, **download_params)
def get_cdips_product(star_num):
""" Given a list of numbers or a single number,
return the same number of FitsTable objects.
Given values can be TIC ids or indexes, the latter will be used to index
"""
global index_provided
if type(star_num) != list:
star_num = list(star_num)
num_ints = [int(n) for n in star_num]
tic_searches = []
mask = get_mask(table_data)
print("Filtered down to", np.count_nonzero(mask), "items to index from.")
temp_table = table_data[mask]
num_stars = len(table_data)
for n in num_ints:
if n > num_stars:
# Already have tics
tic_searches.append(str(n))
else:
if not index_provided: index_provided = True
tic_searches.append(str(int(temp_table['MatchID'][n])))
obs = Observations.query_criteria(target_name=tic_searches, provenance_name="CDIPS")
print(f"TIC {tic_searches} returned {len(obs)} object(s).")
# Get rid of duplicates across sectors
u, u_indexes = np.unique(obs['target_name'], return_index=True)
if len(u) != len(obs):
obs = obs[u_indexes]
print(f"After removing duplicates, {len(obs)} object(s) remain.")
if len(obs) > 0:
products = Observations.get_product_list(obs)
print("Downloading products...")
down = Observations.download_products(products)
print("Done.")
ft_list = []
for f in down:
ft = FitsTable(f['Local Path'])
ft_list.append(ft)
ft_list = sorted(ft_list, key=lambda x: tic_searches.index(x.ticid))
return ft_list
else:
return
def download_wcs_file(filtered, local_dir, n=10, cloud=False):
"""
filtered is results if id_wcs_file
local_dir is the location to put the file
cloud determines whether the file should first be pulled from the cloud.
For lambda local_dir should be /tmp
"""
if cloud:
Observations.enable_cloud_dataset(provider='AWS')
obsslice = slice(n, n + 1)
manifest = Observations.download_products(filtered[obsslice],
download_dir=local_dir,
mrp_only=False)
return manifest['Local Path'][0]
def download_products(products):
"""Download data products from MAST.
Downloads will be cached using astroquery.mast's caching system.
Parameters
----------
products : astropy.Table
A table detailing the products to be downloaded, i.e. the output
of `search_kepler_products` or `search_kepler_tpf_products`.
Returns
-------
paths : astropy.Table.Column
Detailing the paths of the downloaded or cached products.
"""
# Note: by default, MAST will only let us download "Minimum Recommended
# Products" (MRPs), which do not include e.g. Target Pixel Files.
# We need to set `mrp=False` to ensure MAST downloads whatever we want.
dl = Observations.download_products(products, mrp_only=False)
return dl['Local Path']
def getOneFileFromMastObs(self, ticid, sector, fileType, *args, **kwargs):
"""
Generic funtion to retrive a particular file type from a
CAOM observation at the MAST.
Private Function -- Only used by this class.
"""
if fileType not in ['LC', 'TP', 'DVT']:
raise ValueError("Requested file Type (%s) not in allowed list (LC,TP,DVT)."\
% fileType)
obsid = self.getPostageStampObservationId(sector, ticid)
manifest=Observations.download_products(obsid,\
productSubGroupDescription=[fileType],\
mrp_only=False,extension="fits",
download_dir=self.cachePath)
localUrl = manifest['Local Path'][0]
return self.parse(localUrl, *args, **kwargs)
def download_products(products):
"""Download data products from MAST.
Downloads will be cached using astroquery.mast's caching system.
Parameters
----------
products : astropy.Table
A table detailing the products to be downloaded, i.e. the output
of `search_products` or `search_kepler_tpf_products`.
Returns
-------
paths : astropy.Table.Column
Detailing the paths of the downloaded or cached products.
"""
# Note: by default, MAST will only let us download "Minimum Recommended
# Products" (MRPs), which do not include e.g. Target Pixel Files.
# We need to set `mrp=False` to ensure MAST downloads whatever we want.
# check if download directory exists (~/.lightkurve-cache)
cache_dir = os.path.join(os.path.expanduser('~'), '.lightkurve-cache')
if os.path.isdir(cache_dir):
download_dir = cache_dir
else:
# if it doesn't exist, make a new cache directory
try:
os.mkdir(cache_dir)
download_dir = cache_dir
# downloads locally if OS error occurs
except OSError:
log.warning(
'Warning: unable to create .lightkurve-cache directory. '
'Downloading MAST files to local directory.')
download_dir = '.'
dl = Observations.download_products(products,
mrp_only=False,
download_dir=download_dir)
return dl['Local Path']
def _download_one(self, table, quality_bitmask, download_dir, cutout_size):
"""Private method used by `download()` and `download_all()` to download
exactly one file from the MAST archive.
Always returns a `TargetPixelFile` or `LightCurveFile` object.
"""
# Make sure astroquery uses the same level of verbosity
logging.getLogger('astropy').setLevel(log.getEffectiveLevel())
if download_dir is None:
download_dir = self._default_download_dir()
# if the SearchResult row is a TESScut entry, then download cutout
if 'FFI Cutout' in table[0]['description']:
try:
path = self._fetch_tesscut_path(table[0]['target_name'],
table[0]['sequence_number'],
download_dir, cutout_size)
except:
raise SearchError(
'Unable to download FFI cutout. Desired target '
'coordinates may be too near the edge of the FFI.')
return _open_downloaded_file(path,
quality_bitmask=quality_bitmask,
targetid=table[0]['targetid'])
else:
if cutout_size is not None:
warnings.warn(
'`cutout_size` can only be specified for TESS '
'Full Frame Image cutouts.', LightkurveWarning)
from astroquery.mast import Observations
path = Observations.download_products(
table[:1], mrp_only=False,
download_dir=download_dir)['Local Path'][0]
# open() will determine filetype and return
return _open_downloaded_file(path, quality_bitmask=quality_bitmask)
def retrieve_observation(obsid, suffix=['FLC'], archive=False,clobber=False):
"""Simple interface for retrieving an observation from the MAST archive
If the input obsid is for an association, it will request all members with
the specified suffixes.
Parameters
-----------
obsid : string
ID for observation to be retrieved from the MAST archive. Only the
IPPSSOOT (rootname) of exposure or ASN needs to be provided; eg., ib6v06060.
suffix : list
List containing suffixes of files which should be requested from MAST.
path : string
Directory to use for writing out downloaded files. If `None` (default),
the current working directory will be used.
archive : Boolean
Retain copies of the downloaded files in the astroquery created sub-directories? Default is 'False'.
clobber : Boolean
Download and Overwrite existing files? Default is 'False'.
Returns
-------
local_files : list
List of filenames
"""
local_files = []
# Query MAST for the data with an observation type of either "science" or "calibration"
obsTable = Observations.query_criteria(obs_id=obsid, obstype='all')
# Catch the case where no files are found for download
if len(obsTable) == 0:
log.info("WARNING: Query for {} returned NO RESULTS!".format(obsid))
return local_files
dpobs = Observations.get_product_list(obsTable)
dataProductsByID = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# After the filtering has been done, ensure there is still data in the table for download.
# If the table is empty, look for FLT images in lieu of FLC images. Only want one
# or the other (not both!), so just do the filtering again.
if len(dataProductsByID) == 0:
log.info("WARNING: No FLC files found for {} - will look for FLT files instead.".format(obsid))
suffix = ['FLT']
dataProductsByID = Observations.filter_products(dpobs,
productSubGroupDescription=suffix,
extension='fits',
mrp_only=False)
# If still no data, then return. An exception will eventually be thrown in
# the higher level code.
if len(dataProductsByID) == 0:
log.info("WARNING: No FLC or FLT files found for {}.".format(obsid))
return local_files
allImages = []
for tableLine in dataProductsByID:
allImages.append(tableLine['productFilename'])
log.info(allImages)
if not clobber:
rowsToRemove = []
for rowCtr in range(0,len(dataProductsByID)):
if os.path.exists(dataProductsByID[rowCtr]['productFilename']):
log.info("{} already exists. File download skipped.".format(dataProductsByID[rowCtr]['productFilename']))
rowsToRemove.append(rowCtr)
if rowsToRemove:
rowsToRemove.reverse()
for rowNum in rowsToRemove:
dataProductsByID.remove_row(rowNum)
manifest = Observations.download_products(dataProductsByID, mrp_only=False)
if not clobber:
rowsToRemove.reverse()
for rownum in rowsToRemove:
if not manifest:
local_files = allImages
return local_files
else:
manifest.insert_row(rownum,vals=[allImages[rownum],"LOCAL","None","None"])
download_dir = None
for file,fileStatus in zip(manifest['Local Path'],manifest['Status']):
if fileStatus != "LOCAL":
# Identify what sub-directory was created by astroquery for the download
if download_dir is None:
file_path = file.split(os.sep)
file_path.remove('.')
download_dir = file_path[0]
# Move or copy downloaded file to current directory
local_file = os.path.abspath(os.path.basename(file))
if archive:
shutil.copy(file, local_file)
else:
shutil.move(file, local_file)
# Record what files were downloaded and their current location
local_files.append(os.path.basename(local_file))
else:
local_files.append(file)
if not archive:
# Remove astroquery created sub-directories
shutil.rmtree(download_dir)
return local_files
try:
print(i)
target_name = "TIC " + str(i)
#print(target_name)
data_table = Observations.query_criteria(provenance_name="SPOC",
objectname=target_name,
obs_collection="TESS",
dataproduct_type="timeseries",
radius=0)
#print(data_table)
#print(Observations.get_metadata("products"))
product_list = Observations.get_product_list(data_table)
#print(product_list)
download_data = Observations.download_products(
product_list,
extension="lc.fits",
download_dir="/data/wallaby/rmorris/SPOC/{}".format(tic_ids[j]))
#print(download_data[1][0])
print("Data downloaded correctly")
except:
print("Connection to MAST Not Established")
try:
time = []
norm_flux = []
###might not need background because already is PDCSAP flux?
ticker = 0
while ticker < len(download_data):
file = fits.open(str(download_data[ticker][0]))
from astroquery.mast import Mast, Observations
from astropy.visualization import make_lupton_rgb, ImageNormalize
import matplotlib.pyplot as plt
import reproject
result = Observations.query_object('M83')
selected_bands = result[(result['obs_collection'] == 'HST') &
(result['instrument_name'] == 'WFC3/UVIS') &
((result['filters'] == 'F657N') |
(result['filters'] == 'F487N') |
(result['filters'] == 'F336W')) &
(result['target_name'] == 'MESSIER-083')]
prodlist = Observations.get_product_list(selected_bands)
filtered_prodlist = Observations.filter_products(prodlist)
downloaded = Observations.download_products(filtered_prodlist)
blue = fits.open(downloaded['Local Path'][2])
red = fits.open(downloaded['Local Path'][5])
green = fits.open(downloaded['Local Path'][8])
target_header = red['SCI'].header
green_repr, _ = reproject.reproject_interp(green['SCI'], target_header)
blue_repr, _ = reproject.reproject_interp(blue['SCI'], target_header)
rgb_img = make_lupton_rgb(ImageNormalize(vmin=0, vmax=1)(red['SCI'].data),
ImageNormalize(vmin=0, vmax=0.3)(green_repr),
ImageNormalize(vmin=0, vmax=1)(blue_repr),
stretch=0.1,
minimum=0,
