def uvj_diagram(u='galaxy_u.fits',
v='galaxy_v.fits',
j='galaxy_j.fits',
target='galaxy',
file_name='galaxy*.fits',
size=200,
x0=1000,
y0=1000):
"""
Function for plotting a spatially resolved UVJ Diagram of an input target,
while comparing with EAZY templates.
INPUTS:
u: Fits file of the image in the U band.
v: Fits file of the image in the V band.
j: Fits file of the image in the J band.
target: The name of the target, to be used in the output files.
file_name: General form of the file names that contain the UVJ images.
size: Size of the slice to plot only part of the original image.
x0: Center of the x-coordinate of the slice.
y0: Center of the y-coordinate of the slice.
KEYWORDS:
PLOT: Set this keyword to plot the UVJ diagram.
OUTPUTS:
Plot of the UVJ diagram.
"""
from grizli import utils
import numpy as np
import astropy.io.fits as pyfits
import matplotlib.pyplot as plt
import glob
import pysynphot as S
slx = slice(x0 - size, x0 + size)
sly = slice(y0 - size, y0 + size)
u_im = pyfits.open(u)
v_im = pyfits.open(v)
j_im = pyfits.open(j)
for ext in [0, 1]:
if 'PHOTPLAM' in u_im[ext].header:
u_lam = u_im[ext].header['PHOTPLAM']
break
for ext in [0, 1]:
if 'PHOTPLAM' in v_im[ext].header:
v_lam = v_im[ext].header['PHOTPLAM']
break
for ext in [0, 1]:
if 'PHOTPLAM' in j_im[ext].header:
j_lam = j_im[ext].header['PHOTPLAM']
break
fu = u_im['SCI'].data[sly, slx] * u_im[1].header['IM2FLAM'] * (u_lam**2)
fv = v_im['SCI'].data[sly, slx] * v_im[1].header['IM2FLAM'] * (v_lam**2)
fj = j_im['SCI'].data[sly, slx] * j_im[1].header['IM2FLAM'] * (j_lam**2)
fu_error = u_im['ERR'].data[sly, slx] * u_im[1].header['IM2FLAM'] * (u_lam
**2)
fv_error = v_im['ERR'].data[sly, slx] * v_im[1].header['IM2FLAM'] * (v_lam
**2)
fj_error = j_im['ERR'].data[sly, slx] * j_im[1].header['IM2FLAM'] * (j_lam
**2)
u_v_err = (2.5 / np.log(10)) * np.sqrt((fu_error / fu)**2 +
(fv_error / fv)**2)
v_j_err = (2.5 / np.log(10)) * np.sqrt((fj_error / fj)**2 +
(fv_error / fv)**2)
u_v = -2.5 * np.log10(fu / fv)
v_j = -2.5 * np.log10(fv / fj)
mask = (u_v_err < 0.1) & (v_j_err < 0.1)
templ = utils.load_templates(full_line_list=[],
line_complexes=False,
fsps_templates=True,
alf_template=True)
files = glob.glob(file_name)
files.sort()
images = {}
bandpasses = {}
for file in files:
im = pyfits.open(file)
filt = utils.get_hst_filter(im[0].header)
for ext in [0, 1]:
if 'PHOTMODE' in im[ext].header:
bandpasses[filt.lower()] = S.ObsBandpass(
im[ext].header['PHOTMODE'].replace(' ', ','))
break
images[filt.lower()] = im['SCI'].data
template_fluxes = {}
for f in bandpasses:
template_fluxes[f] = np.zeros(len(templ))
# templates from EAZY
for i, t in enumerate(templ):
t_z = templ[t].zscale(0.03)
for f in bandpasses:
template_fluxes[f][i] = t_z.integrate_filter(bandpasses[f])
s = templ.keys()
uv = -2.5 * np.log10(template_fluxes[u] / template_fluxes[v])
vj = -2.5 * np.log10(template_fluxes[v] / template_fluxes[j])
fig, ax = plt.subplots(figsize=(12, 10))
plt.plot(v_j[mask], u_v[mask], 'kx', ms=1, zorder=-5)
for i, lab in enumerate(s):
plt.scatter(vj[i], uv[i], label=lab)
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.xlabel('V-J [observed] (AB mag)'.format(v, j))
plt.ylabel('U-V [observed] (AB mag)'.format(u, v))
plt.title('UVJ Diagram {0}'.format(target))
plt.show()
#-----------------------------------------------------------------------------
def run_grizli_fit(event):
import boto3
import json
import shutil
import gc
import matplotlib.pyplot as plt
import grizli
from grizli import fitting, utils, multifit
try:
from grizli.aws import db as grizli_db
dbFLAGS = grizli_db.FLAGS
except:
pass
utils.set_warnings()
#event = {'s3_object_path':'Pipeline/j001452+091221/Extractions/j001452+091221_00277.beams.fits'}
silent = False
if 'silent' in event:
silent = event['silent'] in TRUE_OPTIONS
###
### Parse event arguments
###
event_kwargs = {}
for k in event:
# Lists
if isinstance(event[k], str):
# Split lists
if ',' in event[k]:
try:
event_kwargs[k] = np.cast[float](event[k].split(','))
except:
event_kwargs[k] = event[k].split(',')
else:
event_kwargs[k] = event[k]
else:
try:
event_kwargs[k] = json.loads(event[k])
except:
event_kwargs[k] = event[k]
# Defaults
if 'skip_started' not in event_kwargs:
event_kwargs['skip_started'] = True
for k in ['quasar_fit', 'extract_from_flt', 'fit_stars', 'beam_info_only']:
if k not in event_kwargs:
event_kwargs[k] = False
if event_kwargs['beam_info_only'] in TRUE_OPTIONS:
dbtable = 'multibeam'
elif event_kwargs['quasar_fit'] in TRUE_OPTIONS:
dbtable = 'redshift_fit_quasar'
elif event_kwargs['fit_stars'] in TRUE_OPTIONS:
dbtable = 'stellar_fit'
else:
dbtable = 'redshift_fit'
if not silent:
print('Grizli version: ', grizli.__version__)
# Disk space
total, used, free = shutil.disk_usage("/")
if not silent:
print('Disk info: Total = {0:.2f} / Used = {1:.2f} / Free = {2:.2f}'.format(total // (2**20), used // (2**20), free // (2**20)))
## Output path
if 'output_path' in event:
output_path = event['output_path']
else:
output_path = None
if 'bucket' in event:
event_kwargs['bucket'] = event['bucket']
else:
event_kwargs['bucket'] = 'aws-grivam'
if 'working_directory' in event:
os.chdir(event['working_directory'])
else:
os.chdir('/tmp/')
if not silent:
print('Working directory: {0}'.format(os.getcwd()))
files = glob.glob('*')
files.sort()
# Filenames, etc.
beams_file = os.path.basename(event['s3_object_path'])
root = beams_file.split('_')[0]
id = int(beams_file.split('_')[1].split('.')[0])
try:
db_status = grizli_db.get_redshift_fit_status(root, id, table=dbtable)
except:
db_status = -1
# Initial log
start_log = '{0}_{1:05d}.start.log'.format(root, id)
full_start = 'Pipeline/{0}/Extractions/{1}'.format(root, start_log)
if ((start_log in files) | (db_status >= 0)) & event_kwargs['skip_started']:
print('Log file {0} found in {1} (db_status={2})'.format(start_log, os.getcwd(), db_status))
return True
if not silent:
for i, file in enumerate(files):
print('Initial file ({0}): {1}'.format(i+1, file))
if os.path.exists('{0}/matplotlibrc'.format(grizli.GRIZLI_PATH)):
os.system('cp {0}/matplotlibrc .'.format(grizli.GRIZLI_PATH))
s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
bkt = s3.Bucket(event_kwargs['bucket'])
if event_kwargs['skip_started']:
res = [r.key for r in bkt.objects.filter(Prefix=full_start)]
if res:
print('Already started ({0}), aborting.'.format(start_log))
return True
fp = open(start_log,'w')
fp.write(time.ctime()+'\n')
fp.close()
bkt.upload_file(start_log, full_start)
# Download fit arguments
if 'force_args' in event:
force_args = event['force_args'] in TRUE_OPTIONS
else:
force_args = False
args_files = ['{0}_fit_args.npy'.format(root), 'fit_args.npy']
for args_file in args_files:
if (not os.path.exists(args_file)) | force_args:
aws_file = 'Pipeline/{0}/Extractions/{1}'.format(root, args_file)
try:
bkt.download_file(aws_file, './{0}'.format(args_file),
ExtraArgs={"RequestPayer": "requester"})
print('Use args_file = {0}'.format(args_file))
break
except:
continue
# If no beams file in the bucket, try to generate it
put_beams=False
try:
if not os.path.exists(beams_file):
bkt.download_file(event['s3_object_path'], './{0}'.format(beams_file), ExtraArgs={"RequestPayer": "requester"})
put_beams = False
except:
print('Extract from GrismFLT object!')
if 'clean' in event:
if isinstance(event['clean'], str):
run_clean = event['clean'].lower() in ['true', 'y', '1']
else:
run_clean = event['clean']
else:
run_clean = True
try:
# Extracting beams
grizli_db.update_redshift_fit_status(root, id,
status=dbFLAGS['start_beams'],
table=dbtable)
except:
print('Set DB flag failed: start_beams')
pass
status = extract_beams_from_flt(root, event_kwargs['bucket'], id,
clean=run_clean, silent=silent)
# Garbage collector
gc.collect()
if status is False:
return False
else:
beams_file = status[0]
try:
# Beams are done
grizli_db.update_redshift_fit_status(root, id,
status=dbFLAGS['done_beams'],
table=dbtable)
except:
pass
put_beams = True
# upload it now
output_path = 'Pipeline/{0}/Extractions'.format(root)
for outfile in status:
aws_file = '{0}/{1}'.format(output_path, outfile)
print(aws_file)
bkt.upload_file(outfile, aws_file,
ExtraArgs={'ACL': 'public-read'})
if ('run_fit' in event) & (dbtable == 'redshift_fit'):
if event['run_fit'] in FALSE_OPTIONS:
res = bkt.delete_objects(Delete={'Objects':[{'Key':full_start}]})
try:
grizli_db.update_redshift_fit_status(root, id,
status=dbFLAGS['no_run_fit'],
table=dbtable)
except:
pass
return True
utils.fetch_acs_wcs_files(beams_file, bucket_name=event_kwargs['bucket'])
# Update the multibeam/beam_geometry tables
if os.path.exists(beams_file):
args = np.load(args_file, allow_pickle=True)[0]
for arg in event_kwargs:
if arg in args:
args[arg] = event_kwargs[arg]
grizli_db.multibeam_to_database(beams_file, Rspline=15, force=False,
**args)
if dbtable == 'multibeam':
### Done
res = bkt.delete_objects(Delete={'Objects':[{'Key':full_start}]})
return True
# Download WCS files
# if event_kwargs['check_wcs']:
# # WCS files for ACS
# files = [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/j'.format(root))]
# for file in files:
# if 'wcs.fits' in file:
# if os.path.exists(os.path.basename(file)):
# continue
#
# bkt.download_file(file, os.path.basename(file),
# ExtraArgs={"RequestPayer": "requester"})
# Is zr in the event dict?
# if 'zr' in event:
# zr = list(np.cast[float](event['zr']))
# else:
# try:
# zr = np.load('fit_args.npy')[0]['zr']
# except:
# zr = np.load('fit_args.npy', allow_pickle=True)[0]['zr']
# Directory listing
files = glob.glob('*')
files.sort()
for i, file in enumerate(files):
print('File ({0}): {1}'.format(i+1, file))
try:
files = glob.glob('{0}_{1:05d}*R30.fits'.format(root, id))
if (len(files) > 0) & (dbtable == 'redshift_fit'):
grizli_db.send_1D_to_database(files=files)
except:
print('Failed to send R30 to spec1d database')
pass
###
### Run the fit
try:
grizli_db.update_redshift_fit_status(root, id, table=dbtable,
status=dbFLAGS['start_redshift_fit'])
except:
print('Set DB flag failed: start_redshift_fit')
pass
if event_kwargs['quasar_fit'] in TRUE_OPTIONS:
# Don't recopy beams file
put_beams = False
# Don't make line maps
if 'min_line_sn' not in event_kwargs:
event_kwargs['min_line_sn'] = np.inf
# Don't make drizzled psfs
if 'get_ir_psfs' not in event_kwargs:
event_kwargs['get_ir_psfs'] = False
# Fit line widths
if 'get_line_width' not in event_kwargs:
event_kwargs['get_line_width'] = True
# sys_err
if 'sys_err' not in event_kwargs:
event_kwargs['sys_err'] = 0.05
# Don't use photometry
event_kwargs['phot_obj'] = None
event_kwargs['use_phot_obj'] = False
event_kwargs['fit_only_beams'] = True
event_kwargs['fit_beams'] = False
templ_args = {'uv_line_complex': True,
'broad_fwhm':2800,
'narrow_fwhm':1000,
'fixed_narrow_lines':True,
'Rspline':15,
'include_reddened_balmer_lines':False}
for k in templ_args:
if k in event_kwargs:
templ_args[k] = event_kwargs.pop(k)
if templ_args['broad_fwhm'] < 0:
use_simple_templates=True
templ_args['broad_fwhm'] *= -1
else:
use_simple_templates = False
print('load_quasar_templates(**{0})'.format(templ_args))
q0, q1 = utils.load_quasar_templates(**templ_args)
if use_simple_templates:
x0 = utils.load_templates(full_line_list=['highO32'], continuum_list=['quasar_lines.txt', 'red_blue_continuum.txt'], line_complexes=False, fwhm=1000)
for t in q0:
if 'bspl' in t:
x0[t] = q0[t]
q0 = x0
q1['red_blue_continuum.txt'] = x0['red_blue_continuum.txt']
# Quasar templates with fixed line ratios
# q0, q1 = utils.load_quasar_templates(uv_line_complex=True,
# broad_fwhm=2800, narrow_fwhm=1000,
# fixed_narrow_lines=True,
# Rspline=15)
if 'zr' not in event_kwargs:
event_kwargs['zr'] = [0.03, 6.8]
if 'fitter' not in event_kwargs:
event_kwargs['fitter'] = ['lstsq', 'lstsq']
print('run_all_parallel: {0}'.format(event_kwargs))
fitting.run_all_parallel(id, t0=q0, t1=q1, args_file=args_file,
**event_kwargs)
if output_path is None:
#output_path = 'Pipeline/QuasarFit'.format(root)
output_path = 'Pipeline/{0}/Extractions'.format(root)
elif event_kwargs['fit_stars'] in TRUE_OPTIONS:
args = np.load(args_file, allow_pickle=True)[0]
if 'psf' in event_kwargs:
args['psf'] = event_kwargs['psf'] in TRUE_OPTIONS
for k in ['fcontam', 'min_sens', 'sys_err']:
if k in event_kwargs:
print('Set arg {0}={1}'.format(k, event_kwargs[k]))
args[k] = event_kwargs[k]
# Load MultiBeam
mb = multifit.MultiBeam(beams_file, **args)
if 'fit_trace_shift' in args:
if args['fit_trace_shift']:
tr = mb.fit_trace_shift()
if 'spline_correction' in event_kwargs:
spline_correction = event_kwargs['spline_correction'] in TRUE_OPTIONS
else:
spline_correction = True
if 'fit_background' in event_kwargs:
fit_background = event_kwargs['fit_background'] in TRUE_OPTIONS
else:
fit_background = True
if 'fitter' in event_kwargs:
fitter = event_kwargs['fitter']
else:
fitter = 'lstsq'
if 'Rspline' in event_kwargs:
Rspline = event_kwargs['Rspline']
else:
Rspline = 15
if Rspline == 15:
logg_list = [4.5]
else:
logg_list = utils.PHOENIX_LOGG
if 'add_carbon_star' in event_kwargs:
add_carbon_star = event_kwargs['add_carbon_star']
else:
add_carbon_star = 25
if 'use_phoenix' in event_kwargs:
p = event_kwargs.pop('use_phoenix')
if p in TRUE_OPTIONS:
tstar = utils.load_phoenix_stars(logg_list=logg_list,
add_carbon_star=add_carbon_star)
else:
tstar = utils.load_templates(stars=True,
add_carbon_star=add_carbon_star)
else:
tstar = utils.load_phoenix_stars(logg_list=logg_list,
add_carbon_star=add_carbon_star)
kws = {'spline_correction':spline_correction,
'fit_background':fit_background,
'fitter':fitter,
'spline_args':{'Rspline':Rspline}}
print('kwargs: {0}'.format(kws))
# Fit the stellar templates
_res = mb.xfit_star(tstar=tstar, oned_args={}, **kws)
_res[0].savefig('{0}_{1:05d}.star.png'.format(root, id))
# Save log info
fp = open('{0}_{1:05d}.star.log'.format(root, id), 'w')
fp.write(_res[1])
fp.close()
if output_path is None:
#output_path = 'Pipeline/QuasarFit'.format(root)
output_path = 'Pipeline/{0}/Extractions'.format(root)
else:
# Normal galaxy redshift fit
fitting.run_all_parallel(id, fit_only_beams=True, fit_beams=False,
args_file=args_file, **event_kwargs)
if output_path is None:
output_path = 'Pipeline/{0}/Extractions'.format(root)
# Output files
files = glob.glob('{0}_{1:05d}*'.format(root, id))
for file in files:
if ('beams.fits' not in file) | put_beams:
aws_file = '{0}/{1}'.format(output_path, file)
if event_kwargs['quasar_fit'] in TRUE_OPTIONS:
# Don't copy stack
if 'stack' in file:
continue
# Add qso extension on outputs
aws_file = aws_file.replace('_{0:05d}.'.format(id),
'_{0:05d}.qso.'.format(id))
print('Upload {0} -> {1}'.format(file, aws_file))
bkt.upload_file(file, aws_file, ExtraArgs={'ACL': 'public-read'})
# Put data in the redshift_fit database table
try:
if dbtable == 'stellar_fit':
rowfile = '{0}_{1:05d}.star.log'.format(root, id)
else:
rowfile = '{0}_{1:05d}.row.fits'.format(root, id)
if os.path.exists(rowfile):
grizli_db.add_redshift_fit_row(rowfile, table=dbtable,
verbose=True)
# Add 1D spectra
files = glob.glob('{0}_{1:05d}*1D.fits'.format(root, id))
if (len(files) > 0) & (dbtable == 'redshift_fit'):
grizli_db.send_1D_to_database(files=files)
except:
print('Update row failed')
pass
# Remove start log now that done
res = bkt.delete_objects(Delete={'Objects':[{'Key':full_start}]})
# Garbage collector
gc.collect()
def subtract_continuum(line='f673n',
cont='f814w',
target='galaxy',
file_name='galaxy*.fits',
line_name='ha',
z=0.02,
plot=False):
"""
Function for subtracting the continuum in a line emission image
INPUTS:
line: Filter into which the emission line falls.
cont: Filter within which the emission line and broader continuum are contained.
target: The name of the target, to be used in the output files.
file_name: General form of the file names that contain the line and continuum images.
line_name: State 'ha' or 'pab' to subtract Balmer-alpha or Paschen-beta respectively.
z: Redshift of the target object.
KEYWORDS:
PLOT: Set this keyword to produce a plot of the two-dimensional
continuum subtracted image.
OUTPUTS:
Fits file with the continuum subtracted line emission image.
"""
import pysynphot as S
import numpy as np
import glob
from grizli import utils
import astropy.io.fits as pyfits
import matplotlib.pyplot as plt
# restframe wavelength of the emission lines to subtract
wave_pab = 1.2822e4
wave_ha = 6562.8
print('Target =', target)
files = glob.glob(file_name)
files.sort()
images = {}
headers = {}
bandpasses = {}
for file in files:
im = pyfits.open(file)
filt = utils.get_hst_filter(im[0].header).lower()
for ext in [0, 1]:
if 'PHOTMODE' in im[ext].header:
photflam = im[ext].header['PHOTFLAM']
headers[filt.lower()] = im[ext].header
bandpasses[filt.lower()] = S.ObsBandpass(
im[ext].header['PHOTMODE'].replace(' ', ','))
break
flat_flam = S.FlatSpectrum(1., fluxunits='flam')
obs = S.Observation(flat_flam, bandpasses[filt.lower()])
my_photflam = 1 / obs.countrate()
flat_ujy = S.FlatSpectrum(1, fluxunits='ujy')
obs = S.Observation(flat_ujy, bandpasses[filt.lower()])
my_photfnu = 1 / obs.countrate()
images[filt.lower()] = [im['SCI'].data, im['ERR'].data]
# Use PySynphot to compute flux calibration factors
if line_name == 'pab':
# Pa-beta
cont_filter, line_filter, line_wave, name = cont, line, wave_pab, 'pab'
elif line_name == 'ha':
# H-alpha
cont_filter, line_filter, line_wave, name = cont, line, wave_ha, 'ha'
################
# Continuum - flat spectrum
cont = S.FlatSpectrum(1.e-19, fluxunits='flam')
###############
# Continuum - slope spectrum
cont_wave = np.arange(1000, 2.e4)
slope = 0 # flat
slope = 1 # red slope, increasing toward longer wavelengths
cont_flux = (cont_wave / 1.e4)**slope
cont = S.ArraySpectrum(cont_wave, cont_flux, fluxunits='flam')
################
# Continuum, galaxy model
templ = utils.load_templates(full_line_list=[],
line_complexes=False,
alf_template=True)['alf_SSP.dat']
cont = S.ArraySpectrum(templ.wave * (1 + z), templ.flux, fluxunits='flam')
# Gaussian line model
ref_flux = 1.e-17
line_model = S.GaussianSource(ref_flux,
line_wave * (1 + z),
10,
waveunits='angstrom',
fluxunits='flam')
cont_contin_countrate = S.Observation(cont,
bandpasses[cont_filter]).countrate()
line_contin_countrate = S.Observation(cont,
bandpasses[line_filter]).countrate()
line_emline_countrate = S.Observation(line_model,
bandpasses[line_filter]).countrate()
# Continuum-subtracted, flux-calibrated
line_calib = (
images[line_filter][0] -
images[cont_filter][0] * line_contin_countrate / cont_contin_countrate)
line_calib /= line_emline_countrate
# Propagated error of the subtraction
err_sub = np.sqrt((images[line_filter][1]**2) +
(images[cont_filter][1] * line_contin_countrate /
cont_contin_countrate)**2)
err_sub /= line_emline_countrate
if plot:
print("Continuum subtracted image")
plt.figure()
plt.imshow(line_calib, vmin=-0.5, vmax=0.5)
plt.colorbar()
primary_extn = pyfits.PrimaryHDU()
sci_extn = pyfits.ImageHDU(data=line_calib, name='SCI')
err_extn = pyfits.ImageHDU(data=err_sub, name='ERR')
hdul = pyfits.HDUList([primary_extn, sci_extn, err_extn])
hdul.writeto('sub_{0}_{1}.fits'.format(line_name, target),
output_verify='fix',
overwrite=True)
print(line_name, ' Continuum Subtracted')