def getObstable(self):
options = self.options
table = Observations.query_region(self.coord,
radius=self.options['radius'])
# HST-specific masks
filmask = [
table['filters'] == good for good in options['mask']['filters']
]
filmask = [any(l) for l in list(map(list, zip(*filmask)))]
expmask = table['t_exptime'] > options['mask']['t_exptime']
obsmask = [
table['obs_collection'] == good
for good in options['mask']['obs_collection']
]
obsmask = [any(l) for l in list(map(list, zip(*obsmask)))]
detmask = [
table['instrument_name'] == good
for good in options['mask']['instrument_name']
]
detmask = [any(l) for l in list(map(list, zip(*detmask)))]
# Construct and apply mask
mask = [all(l) for l in zip(filmask, expmask, obsmask, detmask)]
self.obstable = table[mask]
self.Nimages = len(self.obstable)
def check_exist_location(ra, dec, radius):
# This function takes the columns of RA and DEC to check if there are any observations
# of that region, and how many are there in MAST.
# RA and DEC must be in decimal format, NOT in DEG:MM:SS.
# print("\nChecking...\n")
results = []
location = str(ra) + ' ' + str(dec)
results = Observations.query_region(location, radius=radius)
final = results[(results['obs_collection'] == 'GALEX')
& (results['dataproduct_type'] == 'image')]
data_products_by_obs = Observations.get_product_list(final)
ObjIDs = data_products_by_obs[(
(data_products_by_obs['description'] == descript)
| (data_products_by_obs['description'] == skybkg)
| (data_products_by_obs['description'] == intmap)) & (
(data_products_by_obs['project'] == 'MIS')
| (data_products_by_obs['project'] == 'AIS'))]
counts = len(ObjIDs)
if counts == 0:
counts = -9999
ObjIDs = -9999
return counts, ObjIDs
def getObstable(self):
self.obstable = Observations.query_region(
str(self.ra) + " " + str(self.dec),
radius=str(self.radius) + " deg")
detmasks = [
self.obstable['instrument_name'] == name
for name in self.allowed_detector
]
filmasks = [self.obstable['filters'] != bad for bad in self.badfilter]
expmasks = [self.obstable['t_exptime'] >= self.minexp]
colmasks = [
self.obstable['obs_collection'] == coll for coll in self.collection
]
good=[all(l) for l in list(zip(\
[any(l) for l in list(map(list,zip(*detmasks)))],\
[any(l) for l in list(map(list,zip(*colmasks)))],\
[any(l) for l in list(map(list,zip(*expmasks)))],\
[any(l) for l in list(map(list,zip(*filmasks)))]))]
self.obstable = self.obstable[good]
self.Nimages = len(self.obstable)
self.filtlist = self.obstable['filters']
self.detlist = self.obstable['instrument_name']
self.obsid = self.obstable['obs_id']
self.jdlist = self.obstable['t_min'] + 2400000.5
def get_galex_image(ra, dec, imsize):
"""
get galex image of a galaxy
Input:
* ra in deg
* dec in deg
* imsize in arcsec
Returns:
* image
"""
# following procedure outlined here:
# https://astroquery.readthedocs.io/en/latest/mast/mast.html
# get data products in region near ra,dec
obs_table = Observations.query_region("%12.8f %12.8f" % (c.ra, c.dec),
radius=.1 * u.arcmin)
# create a flag to select galex data
galexFlag = obs_table['obs_collection'] == 'GALEX'
# separate out galex data
data_products = Observations.get_product_list(obs_table[galexFlag])
# download the observations
manifest = Observations.download_products(data_products,
productType="SCIENCE")
for m in manifest:
# choose the first NUV image
if m['Local Path'].find('nd-int') > -1:
nuv_path = m['Local Path']
break
# should be able to construct path from the obs_id in data_products
# this will let us check if the image is already downloaded
#
# I can also save the cutout in a GALEX folder, and
# look for the image before calling this function
nuv, nuv_header = fits.getdata(nuv_path, header=True)
# this is a big image, so we need to get a cutout
nuv_wcs = WCS(nuv_header)
position = SkyCoord(ra, dec, unit="deg", frame='icrs')
cutout = Cutout2D(nuv,
position, (imsize * u.arcsec, imsize * u.arcsec),
wcs=nuv_wcs)
return cutout
def find_object(catalog,
name,
raname='ra',
decname='dec',
namename='Name',
verbose=True):
m = catalog['Name'] == name
target = catalog[m][0]
ra, dec = target[raname, decname]
obsTable = Observations.query_region("%s %s " % (ra, dec),
radius=10 * u.arcsec)
return obsTable
def find_in_tess(catalog,
raname='ra',
decname='dec',
namename='Name',
verbose=True):
names = []
for j, name in enumerate(catalog['Name']):
ra, dec = catalog[j][raname, decname]
obsTable = Observations.query_region("%s %s " % (ra, dec),
radius=10 * u.arcsec)
m = [obsTable['obs_collection'] == 'TESS']
if np.sum(m) != 0:
names.append(name)
if verbose:
print(name)
in_tess = join(Table({namename: np.array(names)}), catalog, keys=namename)
return in_tess
def getObstable(self):
options = self.options
table = Observations.query_region(self.coord,
radius=self.options['radius'])
# HST-specific masks
filmask = [
table['filters'] == good for good in options['mask']['filters']
]
filmask = [any(l) for l in list(map(list, zip(*filmask)))]
expmask = table['t_exptime'] > options['mask']['t_exptime']
obsmask = [
table['obs_collection'] == good
for good in options['mask']['obs_collection']
]
obsmask = [any(l) for l in list(map(list, zip(*obsmask)))]
detmask = [
table['instrument_name'] == good
for good in options['mask']['instrument_name']
]
detmask = [any(l) for l in list(map(list, zip(*detmask)))]
# Construct and apply mask
mask = [all(l) for l in zip(filmask, expmask, obsmask, detmask)]
self.obstable = table[mask]
self.Nimages = 0
if self.obstable:
if len(self.obstable) > 0:
self.obstable['t_min'] = np.around(self.obstable['t_min'],
decimals=4)
self.obstable['t_max'] = np.around(self.obstable['t_max'],
decimals=4)
self.obstable.sort('obs_collection')
self.obstable = unique(self.obstable,
keys=['t_min'],
keep='last')
self.obstable = unique(self.obstable,
keys=['t_max'],
keep='last')
self.Nimages = len(self.obstable)
def downloadPS1(coord, filt, outdir='data/'):
# Check for filter
if not filt: filt='r'
obs = Observations.query_region(coord, radius=5*u.arcmin)
masks = []
masks.append([str(t).upper()=='PS1' for t in obs['obs_collection']])
masks.append([str(p).upper()=='IMAGE' for p in obs['dataproduct_type']])
masks.append([str(r).upper()=='PUBLIC' for r in obs['dataRights']])
masks.append([str(f).lower()==filt for f in obs['filters']])
masks = list(map(list, zip(*masks)))
mask = np.array([all(l) for l in masks])
obsTable = obs[mask]
if len(obsTable)==0:
error = 'ERROR: no matching PS1 images for ra={ra}, dec={dec}.'
print(error.format(ra=ra, dec=dec))
return(1)
else:
prodList = Observations.get_product_list(obsTable)
prodList = prodList[prodList['productType']=='SCIENCE']
prodList = prodList[prodList['description']=='stack data image']
message = 'We need to download {n} PS1 images.'
print(message.format(n=len(prodList)))
for product in prodList:
filename = product['productFilename']
outfile = outdir+filename
url = 'https://mast.stsci.edu/api/v0/download/file?uri='
url += product['dataURI']
if os.path.isfile(outfile):
message = '{filename} exists. Continuing...'
print(message.format(filename=outfile))
return(outfile)
else:
errflag = download_url_to_file(url, outfile)
return(outfile)
)
print(
'------------------------------------------------------------------------------------------------------------------'
)
for i in range(len(teff)):
#--------------------------------------
#--------Downloading Data--------------
#--------------------------------------
print('----------------------------------------------------------------')
print('---------------Working on ' + name[i] + ' -------')
print('----------------------------------------------------------------')
ra1 = utils.RA_to_deg(ra[i])
dec1 = utils.DEC_to_deg(dec[i])
coord = str(ra1) + ' ' + str(dec1)
obt = obs.query_region(coord, radius=0.001)
b = np.array([])
for j in range(len(obt['intentType'])):
if obt['obs_collection'][j] == 'TESS' and obt['dataproduct_type'][
j] == 'timeseries':
b = np.hstack((b, j))
if len(b) == 0:
print(
'******************************************************************************************'
)
print(
' '
)
print('Completed analysis for ' + str(i + 1) + ' system(s) / out of ' +
str(len(name)) + ' systems')
print(
def app_caom():
global blc
global trc
global im
if blc is None or trc is None or im is None: load_image()
searchString = '{} {}'.format(np.mean([blc[0], trc[0]]),
np.mean([blc[1], trc[1]]))
obsTable = Observations.query_region(searchString, radius=0.2)
obsDF = obsTable.to_pandas()
obsDF = obsDF[(obsDF['obs_collection'].isin(['HST', 'SWIFT'])) & (
obsDF['instrument_name'].isin(['WFC3/IR', 'ACS/WFC', 'STIS/CCD']))]
obsDF['coords'] = obsDF.apply(lambda x: parse_s_region(x['s_region']),
axis=1)
obsDF['coords'].head()
# get plot
p = make_base_bokeh()
# Loop over instruments, coloring each separately
for ins, color in zip(obsDF['instrument_name'].unique(), Spectral4):
ind = obsDF['instrument_name'] == ins
# Add patches with the observation footprings
patch_xs = [c['ra'] for c in obsDF['coords'][ind]]
patch_ys = [c['dec'] for c in obsDF['coords'][ind]]
data = {
'x': patch_xs,
'y': patch_ys,
'obs_collection': obsDF['obs_collection'][ind],
'instrument_name': obsDF['instrument_name'][ind],
'obs_id': obsDF['obs_id'][ind],
'target_name': obsDF['target_name'][ind],
'proposal_pi': obsDF['proposal_pi'][ind],
'obsID': obsDF['obsid'][ind]
}
p.patches('x',
'y',
source=data,
legend=ins,
fill_color=color,
fill_alpha=0.1,
line_color="white",
line_width=0.5)
# Add hover tooltip for MAST observations
tooltip = [("obs_collection", "@obs_collection"),
("instrument_name", "@instrument_name"), ("obs_id", "@obs_id"),
("target_name", "@target_name"),
('proposal_pi', '@proposal_pi')]
p.add_tools(HoverTool(tooltips=tooltip))
# Set up tap/click tool
# Only gets top one?
url = 'getproducts/@obsID'
p.add_tools(TapTool())
taptool = p.select(type=TapTool)
taptool.callback = OpenURL(url=url)
p.legend.click_policy = "hide"
script, div = components(p)
return render_template('base.html', script=script, plot=div)
quasar_list = pd.read_csv('/Users/lcgordon/Downloads/5000-hmq') #import list
n = 0 #counts through coordinate list
m = 0 #counter for coordinate pairs WITH DATA
#find all of the
while n < 5000:
#produce coordinate string from list
RA_float = quasar_list.iloc[n, 0]
RA = str(RA_float)
DEC_float = quasar_list.iloc[n, 1]
DEC = str(DEC_float)
coords = RA + "_" + DEC
#query MAST, produce table of all data products associated with those coordinates
table_of_observations = Observations.query_region(coords, radius=0.02)
product_list = Observations.get_product_list(
product_list) #makes list of all products
i = 0 #counter for obsTable rows
while i < len(table_of_observations):
obs_collection = str(table_of_observations[i]
[1]) #converts the type to a string (essential)
if obs_collection == "HST": #check to see if the observatory collection is HST, caNNOT USE 'and' comparison
m = m + 1 #add one to number of coordinate pairs that have data
print("Data at",
coords) #tells you which coordinate pairs, optional
i = len(
obsTable
) + 1 #makes i out of the range of obstable, ending this while loop
redshift_target = quasar_list.iloc[n][3] #redshift of the target.
r_mag_target = quasar_list.iloc[n][2] #magnitude of target
#calculate the absolute magnitude of the target from the red magnitude
dL = Distance(unit = u.Mpc, z=redshift_target, cosmology = cosmo) / u.Mpc
magnitude_target = r_mag_target - 5*np.log(dL) - 25
#if you have the absolute magnitude of your target, remove the above calculation and use:
#magnitude_target = quasar_list.iloc[n][2]
#instead
#print(r_magnitude_target, magnitude_target, redshift_target)
#search for and download all data products associated with those coordinates
obsTable = Observations.query_region(coords, radius = 0.02)
dataProductsByObservation = Observations.get_product_list(obsTable)
manifest = Observations.download_products(dataProductsByObservation, download_dir=coords,
obs_collection = "HST",
dataproduct_type = "image",
productType = "SCIENCE")
except TimeoutError:
n = n + 1
print("a timeout error occurred at", coords)
#isolate drizzle files
findQSOdrz = 'find . -type f \( -name "*drz.fits" \)'
out = subprocess.Popen(findQSOdrz, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def get_table(self, telescope):
options = self.options[telescope]
if (telescope.upper() == 'HST'):
from astroquery.mast import Observations
# Get table
table = Observations.query_region(self.coord,
radius=options['radius'])
# HST-specific masks
filmask = [
table['filters'] == good for good in options['mask']['filters']
]
filmask = [any(l) for l in list(map(list, zip(*filmask)))]
expmask = table['t_exptime'] > options['mask']['t_exptime']
obsmask = [
table['obs_collection'] == good
for good in options['mask']['obs_collection']
]
obsmask = [any(l) for l in list(map(list, zip(*obsmask)))]
detmask = [
table['instrument_name'] == good
for good in options['mask']['instrument_name']
]
detmask = [any(l) for l in list(map(list, zip(*detmask)))]
# Construct and apply mask
mask = [all(l) for l in zip(filmask, expmask, obsmask, detmask)]
table = table[mask]
if (telescope.upper() == 'SPITZER'):
import requests
from dateutil.parser import parse
params = {
'RA': self.coord.ra.degree,
'DEC': self.coord.dec.degree,
'DATASET': 'ivo://irsa.ipac.spitzer.level2',
'SIZE': options['radius'].to(u.degree).value,
'VERB': 3
}
url = options['url']
r = requests.get(url, params=params)
if 'No Matches' in r.text:
return None
if r.status_code != 200:
message = 'status message: {message}.'
print(message.format(message=r.text))
error = 'ERROR: could not get url {url}, status code {stat}.'
raise RuntimeError(error.format(url=url, stat=r.status_code))
return None
# Parse table from html text
# This is borrowed from astroquery.sha
raw_data = [line for line in r.text.split('\n')]
field_widths = [len(s) + 1 for s in raw_data[1].split('|')][1:-1]
col_names = [s.strip() for s in raw_data[1].split('|')][1:-1]
type_names = [s.strip() for s in raw_data[2].split('|')][1:-1]
cs = [0] + np.cumsum(field_widths).tolist()
def parse_line(line, cs=cs):
return [line[a:b] for a, b in zip(cs[:-1], cs[1:])]
data = [parse_line(row) for row in raw_data[4:-1]]
# Parse type names
dtypes = self.map_dtypes(type_names, field_widths)
# To table
# transpose data for appropriate table instance handling
table = Table(list(zip(*data)), names=col_names, dtype=dtypes)
# Split wavelength into instrument and filter
instrument = []
filt = []
for w in table['wavelength']:
dat = w.split()
inst = dat[0].strip()
if 'IRS' in inst:
f = dat[2]
elif 'IRAC' in inst:
f = dat[1]
elif 'MIPS' in inst:
if 'SED' in dat[1]:
f = dat[2]
else:
f = dat[1]
else:
print('Unknown instrument: ', inst)
instrument.append(inst)
filt.append(f)
table.add_column(
astropy.table.Column(name='instrument', data=instrument))
table.add_column(astropy.table.Column(name='filter', data=filt))
detmask = [
table['instrument'] == good
for good in options['mask']['instrument']
]
detmask = [any(l) for l in list(map(list, zip(*detmask)))]
promask = [[good in x for x in table['externalname']]
for good in options['mask']['externalname']]
promask = [any(l) for l in list(map(list, zip(*promask)))]
# Construct and apply mask
mask = [all(l) for l in zip(detmask, promask)]
table = table[mask]
table.add_column(
astropy.table.Column(name='jpg', data=[None] * len(table)))
# Estimate exposure time. This is only defined for Level 1 products
# since all other images are stacks. But we can guesstimate the
# exposure time by looking at the beginning and end time of the
# observation
exptime = [
(parse(end) - parse(start)).seconds
for start, end in zip(table['begintime'], table['endtime'])
]
table.add_column(astropy.table.Column(name='exptime',
data=exptime))
if (telescope.upper() == 'CHANDRA'):
import requests
from astropy.io.votable import parse
url = 'https://cxcfps.cfa.harvard.edu/cgi-bin/'+\
'cda/footprint/get_vo_table.pl'
ra = self.coord.ra.degree
dec = self.coord.dec.degree
params = {
'pos': '{0},{1}'.format(ra, dec),
'size': 0.2,
'grating': 'NONE',
'inst': 'ACIS-I,ACIS-S'
}
if True:
# Try to get a response from the URL
r = requests.get(url, params=params)
# Create a temporary file and then read it into a votable object.
# This is terrible, but best way I've found to complete this part
fname = 'test.xml'
f = open(fname, 'w')
f.write(r.text)
f.close()
votable = parse(fname)
os.remove(fname)
#except:
# return(None)
tbdata = votable.get_first_table().to_table()
# Although using a large search radius, check to make sure that the
# input coordinates are < 5 arcmin off axis
remove = []
for i, row in enumerate(copy.copy(tbdata)):
coord = SkyCoord(row['RA'],
row['Dec'],
unit=(u.deg, u.deg),
frame='icrs')
if self.coord.separation(coord).arcmin > 5:
remove.append(i)
tbdata.remove_rows(remove)
# Require that all observations have SI Mode = TE (timed exposure)
# rather than CC = continuous clocking (and thus not imaging)
remove = []
for i, row in enumerate(copy.copy(tbdata)):
if not row['SIMode'].decode('utf-8').startswith('TE'):
remove.append(i)
tbdata.remove_rows(remove)
table = tbdata
# Rename column names to standard names
for keypair in options['keymap']:
oldname = keypair[0]
newname = keypair[1]
table.rename_column(oldname, newname)
# Reorder table and make sure only standard keys are used
return table
