def get_quick_look():
mpl.rcParams['font.family'] = 'stixgeneral'
window, wc = get_default_windows()
#### Find the desired x1d's in the directory
# first, check if all_exposures.txt exists. if it does, read it in
if os.path.exists("all_exposures.txt"):
exposure_cat = ascii.read("all_exposures.txt")
else:
# if all_exposures does not exist, import scrape_headers and make it.
print "---> all_exposures.txt doesn't exist!!!!! so I'm going to make it now"
from scrape_headers import make_exposure_catalog as make_exposure_catalog
DATA_DIR = '.'
dataset_list = glob.glob(os.path.join(DATA_DIR, '*x1d.fits'))
if len(dataset_list) == 0:
return "there's nothing here to make all_exposures with, trying to exit gracefully :-("
exposure_cat = make_exposure_catalog(dataset_list)
# second, cull for anything with Flag = 1
mask = exposure_cat['Flag'] == 1
exposure_cat = exposure_cat[mask]
dataset_list = []
for root in exposure_cat['Rootname']:
dataset_list.append(root+"_x1d.fits")
if len(exposure_cat) == 0:
return "the flags in all_exposures.txt told me to not do anything :-("
# pathname = "file://"+os.getcwd()+"/"
pathname = ''
## does alias.txt exist?
alias_file = "alias.txt"
if (os.path.exists(alias_file)):
af = open(alias_file, 'r')
targname = af.readline()
af.close()
else:
print "---->>>> get_quick_look can't find "+alias_file+"!!!!! making one instead..... <<<<----------"
hdr0 = fitsio.read_header(dataset_list[0], 0)
targname = hdr0['TARGNAME'].strip()
af = open(alias_file, 'w')
af.write(targname)
af.close()
outfilename = targname + "_quicklook.html"
exists = os.path.isfile(outfilename)
if exists:
command = "rm -f "+outfilename
os.system(command)
outfile = open(outfilename, "w")
ra = exposure_cat['RA'][0]
dec = exposure_cat['DEC'][0]
tardescr = exposure_cat['Target Description'][0]
coord = SkyCoord(ra=ra*u.degree, dec=dec*u.degree)
info = """<html>
<head><h1 style="text-align:center;font-size:350%">"""+targname+"""</h1></head>
<body><p style="text-align:center;font-size=250%">"""+tardescr+"""<br>
α = """+str(ra)+""", δ = """+str(dec)+""" ("""+coord.to_string('hmsdms')+""")</font></p>
<hr />
"""
outfile.write(info)
#### Now loop through all the files to figure out the demographics
t, r = get_demographics(dataset_list)
LAMBDA_MIN = np.min(r['minwave']-2)
LAMBDA_MAX = np.max(r['maxwave']+2)
MAX_FLUX = 1.05*np.max(r['maxflux'])
print 'MAX_FLUX', r['maxflux']
t_pid = t.group_by(['PID'])
info="""<p style="text-align:center;font-size=200%">COS quick look of<br>
Programs: """
for k in t_pid.groups.keys:
info = info+"""<a href="http://www.stsci.edu/cgi-bin/get-proposal-info?id="""+str(k[0])+"""&observatory=HST">"""+str(k[0])+"""</a> """
info = info + """</p>"""
outfile.write(info)
width = 0.7
colors = get_default_pid_colors()
figname = "exptime_histogram.png"
plot_exptime_histogram(t, figname, colors=colors, t_pid=t_pid, width=width)
addfig = r"""<br><img src='"""+pathname+figname+r"""' style="width:600pix">"""
outfile.write(addfig)
figname = "exptime_fppos_histogram.png"
plot_exptime_fppos_histogram(t, figname, t_pid=t_pid, width=width)
addfig = r"""<img src='"""+pathname+figname+r"""' style="width:600pix">"""
outfile.write(addfig)
if (exposure_cat['Detector'] == 'FUV').any():
figname = "lifetime_position_histogram.png"
plot_lifetime_position_histogram(t, figname, t_pid=t_pid, width=width)
addfig = r"""<img src='"""+pathname+figname+r"""' style="width:600pix">"""
outfile.write(addfig)
add_coadd = find_and_plot_coadds(targname, pathname, LAMBDA_MIN, LAMBDA_MAX, 0, MAX_FLUX, window=window, wc=wc)
outfile.write(add_coadd)
#### add legend
info = """<p style="font-size=300%">Individual exposures<br>Legend: <b><font color="black">flux in black</font></b>, <b><font color="grey">errors in grey</font></b>, both smoothed over 7 pixels (~1 resel). S/N≡median(flux/error), per unsmoothed pixel, in shaded window.</p>"""
outfile.write(info)
#### Loop through files and make plots!
time_flux = []
for filename in dataset_list:
with f.open(filename) as hdulist: ##### want to change this to fitsio !!!!! #####
hdr = hdulist[0].header
data = hdulist[1].data
if (hdulist[1].data == None):
print "no data:",filename
continue
if (np.shape(data['flux'])[0] == 0):
print np.shape(data['flux']),filename
continue
output_name = targname+"_"+str(hdr['proposid'])+"_"+str(hdr['cenwave'])+"_"+str(hdr['rootname'])+".png"
print "FILENAME", output_name
labeltext = str(hdr['rootname'])+ """ PID """+str(hdr['proposid'])+""" Visit """+str(hdr['linenum'].split('.')[0])+"""
"""+str(hdr['primesi'])+"""/"""+str(hdr['opt_elem']) + """/"""+str(hdr['cenwave']) + """/FPPOS"""+str(hdr['fppos'])+"""
"""+ hdulist[1].header['date-obs']+""", Exptime = """ +str(int(data['exptime'][0]))+"""s"""
time = np.average([hdulist[1].header['expstart'], hdulist[1].header['expend']])
time_flux = plot_spectrum(output_name, data['wavelength'], data['flux'], LAMBDA_MIN, LAMBDA_MAX, 0, MAX_FLUX, \
window=window, wc=wc, labeltext=labeltext, error=data['error'], wgt=data['dq_wgt'], smooth=7, time=time, time_flux=time_flux)
addfig = r"""<br><img src='"""+pathname+output_name+r"""' style="width:1200pix">"""
print "writing out file ", addfig
outfile.write(addfig)
if np.size(time_flux) > 0:
tf = Table(rows=time_flux, names=('mjd','flux','error','window'))
plot_time_flux(tf, window=window, wc=wc)
addfig = r"""<br><img src='"""+pathname+r"""time_flux.png' style="width:1200pix">"""
outfile.write(addfig)
outfile.write("</body></html>")
outfile.close()
message = """
def get_demographics(dataset_list):
print "getting the demographics"
# first, check if all_exposures.txt exists. if it does, read it in
if os.path.exists("all_exposures.txt"):
exposure_cat = ascii.read("all_exposures.txt")
else:
# if all_exposures does not exist, import scrape_headers and make it.
print "---> all_exposures.txt doesn't exist!!!!! so I'm going to make it now"
from scrape_headers import make_exposure_catalog as make_exposure_catalog
exposure_cat = make_exposure_catalog(dataset_list)
mask = exposure_cat['Flag'] == 1
exposure_cat = exposure_cat[mask]
# if scrape_headers can't be found, do it the hard way.
# this can be done with a try except feature but things need re-arranging.
if (False):
LYA_MIN = 1206 ## should this depend on M vs L grating?
LYA_MAX = 1226
demographics = []
ranges = []
for filename in dataset_list:
hdr = fitsio.read_header(filename, 0)
data, hdr1 = fitsio.read(filename, ext=1, header=True)
if (data == None):
print "no data:", filename
continue
if (np.shape(data['FLUX'])[0] == 0):
print np.shape(data['FLUX']), filename
continue
demographics.append((hdr['PROPOSID'], hdr['LINENUM'].split('.')[0], hdr['LINENUM'].split('.')[1], hdr['CENWAVE'], hdr['FPPOS'], hdr['LIFE_ADJ'], hdr['PR_INV_L'], hdr['ROOTNAME'], data['EXPTIME'][0]))
indices = np.where((data["DQ_WGT"] > 0) & (data["DQ"] == 0) & ((data["WAVELENGTH"] > LYA_MAX) | (data["WAVELENGTH"] < LYA_MIN)))
minwave = 1100
maxwave = 1900
maxflux = 1e-14
if(np.shape(indices)[1] > 0):
if(hdr['OPT_ELEM'] == 'G140L'):
minwave = 900.
maxwave = 2160.
maxflux = 5.0 * np.mean(data["FLUX"][indices]) # changed from max of flux by JT 072015
if((hdr['OPT_ELEM'] == 'G130M') or (hdr['OPT_ELEM'] == 'G160M')):
minwave = 1100
maxwave = 1900
maxflux = 3.0 * np.median(data["FLUX"][indices]) # changed from max of flux by JT 072015
if hdr['DETECTOR'] == 'NUV':
minwave = 1700
maxwave = 3000
maxflux = 3.0 * np.mean(data["FLUX"][indices]) # changed from max of flux by JT 072015
ranges.append((minwave, maxwave, maxflux))
print "Incorporating minwave, maxwave, maxflux from: ", filename, hdr['DETECTOR'], hdr['OPT_ELEM'], hdr['CENWAVE'], minwave, maxwave, maxflux
t = Table(rows=demographics, names=('PID', 'Visit', 'expnum', 'cenwave', 'FPPOS', 'lp', 'PI_NAME', 'rootname', 'exptime'))
r = Table(rows=ranges, names=('minwave', 'maxwave', 'maxflux'), dtype=('f8', 'f8', 'f8'))
# now from the exposure_cat table, find the stuff needed for t and r to be returned
ranges = []
demographics = []
for i in range(len(exposure_cat)):
demographics.append((exposure_cat['PropID'][i], exposure_cat['Cenwave'][i], exposure_cat['FPPOS'][i], exposure_cat['LP'][i], exposure_cat['PI Name'][i], exposure_cat['Rootname'][i], exposure_cat['Exptime'][i]))
minwave = 1100
maxwave = 1900
maxflux = 1e-14
if(exposure_cat['Grating'][i] == 'G140L'):
minwave = 900.
maxwave = 2160.
maxflux = 5.0 * exposure_cat['Mean Flux'][i] # changed from max of flux by JT 072015
if((exposure_cat['Grating'][i] == 'G130M') or (exposure_cat['Grating'][i] == 'G160M')):
minwave = 1100
maxwave = 1900
maxflux = 3.0 * exposure_cat['Median Flux'][i] # changed from max of flux by JT 072015
if exposure_cat['Detector'][i] == 'NUV':
minwave = 1700
maxwave = 3000
maxflux = 3.0 * exposure_cat['Mean Flux'][i] # changed from max of flux by JT 072015
ranges.append((minwave, maxwave, maxflux))
t = Table(rows=demographics, names=('PID', 'cenwave', 'FPPOS', 'lp', 'PI_NAME', 'rootname', 'exptime'))
r = Table(rows=ranges, names=('minwave', 'maxwave', 'maxflux'), dtype=('f8', 'f8', 'f8'))
print "----------------------- THESE ARE THE DEMOGRAPHICS---------------------"
print t
print r
return t, r
def get_quick_look():
mpl.rcParams['font.family'] = 'stixgeneral'
window, wc, w_used = get_default_windows()
#### Find the desired x1d's in the directory
# first, check if all_exposures.txt exists. if it does, read it in
if os.path.exists("all_exposures.txt"):
exposure_cat = ascii.read("all_exposures.txt")
else:
# if all_exposures does not exist, import scrape_headers and make it.
print "---> all_exposures.txt doesn't exist!!!!! so I'm going to make it now"
from scrape_headers import make_exposure_catalog as make_exposure_catalog
DATA_DIR = '.'
dataset_list = glob.glob(os.path.join(DATA_DIR, '*x1d.fits.gz'))
if len(dataset_list) == 0:
print "did not find any x1d.fits.gz.gz files, trying for unzipped ones"
print "I'm also going to gzip any x1d.fits.gz files I do find..."
os.system("gzip *x1d.fits.gz")
dataset_list = glob.glob(os.path.join(DATA_DIR, '*x1d.fits.gz'))
if len(dataset_list) == 0:
return "there's nothing here to make all_exposures with, trying to exit gracefully :-("
exposure_cat = make_exposure_catalog(dataset_list)
# second, cull for anything with Flag = 1
mask = exposure_cat['Flag'] == 1
exposure_cat = exposure_cat[mask]
exposure_cat.sort('Cenwave')
dataset_list = []
for root in exposure_cat['Rootname']:
dataset_list.append(root+"_x1d.fits.gz")
if len(exposure_cat) == 0:
return "the flags in all_exposures.txt told me to not do anything :-("
# pathname = "file://"+os.getcwd()+"/"
pathname = ''
## does alias.txt exist?
alias_file = "alias.txt"
if (os.path.exists(alias_file)):
af = open(alias_file, 'r')
targname = af.readline()
targname = targname.strip()
af.close()
else:
print "---->>>> get_quick_look can't find "+alias_file+"!!!!! making one instead..... <<<<----------"
hdr0 = fitsio.read_header(dataset_list[0], 0)
targname = hdr0['TARGNAME'].strip()
af = open(alias_file, 'w')
af.write(targname)
af.close()
outfilename = targname + "_quicklook.html"
exists = os.path.isfile(outfilename)
if exists:
command = "rm -f "+outfilename
os.system(command)
outfile = open(outfilename, "w")
else:
outfile = open(outfilename, "w")
ra = exposure_cat['RA'][0]
dec = exposure_cat['DEC'][0]
mast_url = '"https://mast.stsci.edu/portal/Mashup/Clients/Mast/Portal.html?searchQuery='+str(ra)+','+str(dec)+'"'
tardescr = exposure_cat['Target Description'][0]
coord = SkyCoord(ra=ra*u.degree, dec=dec*u.degree)
targnamelink ="""<a href=".">"""+targname+"""</a>"""
info = """<html>
<head><title>"""+targname+"""</title>
<h1 style="text-align:center;font-size:350%">"""+targnamelink+"""</h1></head>
<body><p style="text-align:center;font-size=250%">"""+tardescr+"""<br>
α = """+str(ra)+""", δ = """+str(dec)+""" (<a href="""+mast_url+""">"""+coord.to_string('hmsdms')+"""</a>)</font></p>
<hr />
"""
outfile.write(info)
#### Now loop through all the files to figure out the demographics
t, r = get_demographics(dataset_list)
r_det = r.group_by(['detector'])
mask = r_det.groups.keys['detector'] == 'FUV'
r_fuv = r_det.groups[mask]
mask = r_det.groups.keys['detector'] == 'NUV'
r_nuv = r_det.groups[mask]
if (exposure_cat['Detector'] == 'FUV').any():
LAMBDA_MIN_FUV = np.min(r_fuv['minwave']-2)
LAMBDA_MAX_FUV = np.max(r_fuv['maxwave']+2)
MAX_FLUX_FUV = 1.05*np.max(r_fuv['maxflux'])
if (exposure_cat['Detector'] == 'NUV').any():
LAMBDA_MIN_NUV = np.min(r_nuv['minwave']-2)
LAMBDA_MAX_NUV = np.max(r_nuv['maxwave']+2)
MAX_FLUX_NUV = 1.05*np.max(r_nuv['maxflux'])
print 'MAX_FLUX', r['maxflux']
t_pid = t.group_by(['PID'])
info="""<p style="text-align:center;font-size=200%">Programs: """
for k in t_pid.groups.keys:
info = info+"""<a href="http://www.stsci.edu/cgi-bin/get-proposal-info?id="""+str(k[0])+"""&observatory=HST">"""+str(k[0])+"""</a> """
info = info + """</p>"""
outfile.write(info)
width = 0.7
colors = get_default_pid_colors(t_pid.groups.keys)
print colors
### first do demographics for just the FUV exposures, then repeat first two for any NUV observations.
if (exposure_cat['Detector'] == 'FUV').any():
t_det = t.group_by(['detector'])
mask = t_det.groups.keys['detector'] == 'FUV'
t_fuv = t_det.groups[mask]
t_fuv_pid = t_fuv.group_by(['PID'])
print t_det
print t_fuv
figname = "fuv_exptime_histogram.png"
title = "distribution of FUV exposure times by cenwave"
plot_exptime_histogram(t_fuv, figname, colors=colors, t_pid=t_fuv_pid, width=width, title=title)
addfig = r"""<br><img src='"""+pathname+figname+r"""' style="width:35%">"""
outfile.write(addfig)
figname = "fuv_exptime_fppos_histogram.png"
title = "distribution of FUV exposure times by cenwave and FP-POS"
plot_exptime_fppos_histogram(t_fuv, figname, colors=colors, t_pid=t_fuv_pid, width=width, title=title)
addfig = r"""<img src='"""+pathname+figname+r"""' style="width:35%">"""
outfile.write(addfig)
figname = "fuv_lifetime_position_histogram.png"
plot_lifetime_position_histogram(t_fuv, figname, colors=colors, t_pid=t_fuv_pid, width=width)
addfig = r"""<img src='"""+pathname+figname+r"""' style="width:25%"><br>"""
outfile.write(addfig)
if (exposure_cat['Detector'] == 'NUV').any():
t_det = t.group_by(['detector'])
mask = t_det.groups.keys['detector'] == 'NUV'
t_nuv = t_det.groups[mask]
t_nuv_pid = t_nuv.group_by(['PID'])
print t_det
print t_nuv
figname = "nuv_exptime_histogram.png"
title = "distribution of NUV exposure times by cenwave"
plot_exptime_histogram(t_nuv, figname, colors=colors, t_pid=t_nuv_pid, width=width, title=title)
addfig = r"""<br><img src='"""+pathname+figname+r"""' style="width:35%">"""
outfile.write(addfig)
figname = "nuv_exptime_fppos_histogram.png"
title = "distribution of NUV exposure times by cenwave and FP-POS"
plot_exptime_fppos_histogram(t_nuv, figname, colors=colors, t_pid=t_nuv_pid, width=width, title=title)
addfig = r"""<img src='"""+pathname+figname+r"""' style="width:35%">"""
outfile.write(addfig)
if (exposure_cat['Detector'] == 'FUV').any():
print ">>>>>> I AM ASSUMING YOU HAVE COADDS ONLY IF YOU HAVE FUV DATA !!!!!!! <<<<<<<<<<<<<<<<"
add_coadd = find_and_plot_coadds(targname, pathname, LAMBDA_MIN_FUV, LAMBDA_MAX_FUV, 0, MAX_FLUX_FUV, window=window, wc=wc)
outfile.write(add_coadd)
#### add legend
info = """<p style="font-size=300%">Individual exposures<br>Legend: <b><font color="black">flux in black</font></b>, <b><font color="grey">errors in grey</font></b>, both smoothed over 6 pixels (~1 resel). S/N≡median(flux/error), per ~1 resel, in shaded window.</p>"""
outfile.write(info)
#### Loop through files and make plots!
time_flux = []
for filename in dataset_list:
with f.open(filename) as hdulist: ##### want to change this to fitsio !!!!! #####
hdr = hdulist[0].header
data = hdulist[1].data
if (hdulist[1].data == None):
print "no data:",filename
continue
if (np.shape(data['flux'])[0] == 0):
print np.shape(data['flux']),filename
continue
output_name = targname+"_"+str(hdr['proposid'])+"_"+str(hdr['cenwave'])+"_"+str(hdr['rootname'])+".png"
print "FILENAME", output_name
labeltext = str(hdr['rootname'])+ """ PID """+str(hdr['proposid'])+""" Visit """+str(hdr['linenum'].split('.')[0])+"""
"""+str(hdr['primesi'])+"""/"""+str(hdr['opt_elem']) + """/"""+str(hdr['cenwave']) + """/FPPOS"""+str(hdr['fppos'])+"""
"""+ hdulist[1].header['date-obs']+""", Exptime = """ +str(int(data['exptime'][0]))+"""s"""
time = np.average([hdulist[1].header['expstart'], hdulist[1].header['expend']])
wavelength = data['wavelength']
flux = data['flux']
error = data['error']
wgt = data['dq_wgt']
if hdr['detector'] == 'FUV':
LAMBDA_MIN, LAMBDA_MAX, MAX_FLUX = LAMBDA_MIN_FUV, LAMBDA_MAX_FUV, MAX_FLUX_FUV
if hdr['detector'] == 'NUV':
LAMBDA_MIN, LAMBDA_MAX, MAX_FLUX = LAMBDA_MIN_NUV, LAMBDA_MAX_NUV, MAX_FLUX_NUV
if hdr['opt_elem'] == 'G230L':
### want to get rid of stripe c
wavelength = data['wavelength'][0:2]
flux = data['flux'][0:2]
error = data['error'][0:2]
wgt = data['dq_wgt'][0:2]
time_flux = plot_spectrum(output_name, wavelength, flux, LAMBDA_MIN, LAMBDA_MAX, 0, MAX_FLUX, \
window=window, wc=wc, labeltext=labeltext, error=error, wgt=wgt, smooth=6, time=time, time_flux=time_flux)
addfig = r"""<br><img src='"""+pathname+output_name+r"""' style="width:100%">"""
print "writing out file ", addfig
outfile.write(addfig)
if np.size(time_flux) > 0:
tf = Table(rows=time_flux, names=('mjd','flux','error','window'))
plot_time_flux(tf, window=window, wc=wc)
addfig = r"""<br><img src='"""+pathname+r"""time_flux.png' style="width:100%">"""
outfile.write(addfig)
outfile.write("</body></html>")
outfile.close()
message = """