def dummy_setup_dict(file_list, setup):
"""
Generates a dummy setup_dict.
.. todo::
- Describe how this is used.
- Improve the description of setup and/or point to the function
used to generate it.
Args:
file_list (:obj:`list`):
List of files to set as science exposures.
setup (:obj:`str`):
String representation of the instrument setup.
Returns:
dict: Dictionary with the instrument setup.
"""
# setup_dict
setup_dict = {}
setup_dict[setup[0]] = {}
# Fill with dummy dicts
for ii in range(1,20): # Dummy detectors
setup_dict[setup[0]][parse.get_dnum(ii)] = dict(binning='1x1')
setup_dict[setup[0]][setup[-2:]] = {}
# Fill up filenames
setup_dict[setup[0]][setup[-2:]]['sci'] = file_list
# Write
# TODO: Why is this called here?
_ = write_calib(setup_dict)
return setup_dict
def init_hdus(update_det, outfile):
hdus, prihdu = None, None
if (update_det is not None) and os.path.isfile(outfile):
hdus = fits.open(outfile)
msgs.info("Using existing spec1d file, including the Header")
msgs.info(
"Will only update the data extension for {} detector(s)".format(
update_det))
prihdu = hdus[0]
# Names
hdu_names = [hdu.name for hdu in hdus]
# Remove the detector(s) being updated
if not isinstance(update_det, list):
update_det = [update_det]
popme = []
# Find em
for ss, hdu_name in enumerate(hdu_names):
for det in update_det:
sdet = parse.get_dnum(det, prefix=False)
idx = '{:s}{:s}'.format(specobjs.naming_model['det'], sdet)
if idx in hdu_name:
popme.append(ss)
# Remove em (and the bit in the Header too)
for popthis in reversed(popme):
hdus.pop(popthis)
keywd = 'EXT{:04d}'.format(popthis)
prihdu.header.remove(keywd)
# Return
return hdus, prihdu
def det_setup(isetup_dict, ddict):
""" Return detector setup on config dict or add to it if new
Parameters
----------
isetup_dict : dict
Selected setup_dict
ddict : dict
detector dict
Returns
-------
dkey : str
Name of the detector string associated to the input ddict
May be new or previously used
"""
det_str = [parse.get_dnum(i+1, prefix=False) for i in range(99)]
# Init
for dkey in det_str:
mtch = True
if dkey not in isetup_dict.keys():
isetup_dict[dkey] = ddict
break
for key in isetup_dict[dkey].keys():
mtch &= isetup_dict[dkey][key] == ddict[key]
if mtch:
break
# Return
return dkey
def load_std_trace(spec1dfile, det):
sdet = parse.get_dnum(det, prefix=False)
hdulist_1d = fits.open(spec1dfile)
det_nm = 'DET{:s}'.format(sdet)
for hdu in hdulist_1d:
if det_nm in hdu.name:
tbl = Table(hdu.data)
# TODO what is the data model for echelle standards? This routine needs to distinguish between echelle and longslit
trace = tbl['TRACE']
return trace
def main(args):
raise NotImplementedError('This script is currently out of date.')
# List only?
hdu = fits.open(args.file)
head0 = hdu[0].header
if args.list:
hdu.info()
return
# Init
sdet = get_dnum(args.det, prefix=False)
# One detector, sky sub for now
names = [hdu[i].name for i in range(len(hdu))]
try:
exten = names.index('DET{:s}-PROCESSED'.format(sdet))
except: # Backwards compatability
msgs.error('Requested detector {:s} was not processed.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
sciimg = hdu[exten].data
try:
exten = names.index('DET{:s}-SKY'.format(sdet))
except: # Backwards compatability
msgs.error('Requested detector {:s} has no sky model.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
skymodel = hdu[exten].data
try:
exten = names.index('DET{:s}-MASK'.format(sdet))
except ValueError: # Backwards compatability
msgs.error('Requested detector {:s} has no bit mask.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
mask = hdu[exten].data
frame = (sciimg - skymodel) * (mask == 0)
mdir = head0['PYPMFDIR']
mkey = head0['FRAMMKEY']
mast_key = '{0}_{1:02d}'.format(mkey, args.det)
if not os.path.exists(mdir):
mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir))
msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base))
mdir = mdir_base
# Assumes a MasterSlit file has been written
#slits = slittrace.SlitTraceSet.from_master('{0}_{1:02d}'.format(head0['TRACMKEY'], args.det),
mdir)
def set_idx(self):
"""
Generate a unique index for this spectrum
Sets self.idx internally
Returns:
str: :attr:`self.idx`
"""
# Detector string
sdet = parse.get_dnum(self.det, prefix=False)
if 'Echelle' in self.pypeline:
# ObjID
self.idx = naming_model['obj']
if self.ech_objid is None:
self.idx += '----'
else:
self.idx += '{:04d}'.format(self.ech_objid)
self.idx += '-' + naming_model['order']
# Order
if self.ech_orderindx is None:
self.idx += '----'
else:
self.idx += '{:04d}'.format(self.ech_order)
else:
# Spat
self.idx = naming_model['spat']
if self.spat_pixpos is None:
self.idx += '----'
else:
self.idx += '{:04d}'.format(int(np.rint(self.spat_pixpos)))
# Slit
self.idx += '-' + naming_model['slit']
if self.slitid is None:
self.idx += '----'
else:
self.idx += '{:04d}'.format(self.slitid)
self.idx += '-{:s}{:s}'.format(naming_model['det'], sdet)
# Return
return self.idx
def init_hdus(update_det, outfile):
"""
Load up existing header and HDUList
..todo:: Confirm this works when you are modifying an inner HDU
Args:
update_det (int or list):
outfile (str):
Returns:
fits.HDUList, fits.PrimaryHDU
"""
#
hdus = fits.open(outfile)
msgs.info("Using existing output file, including the Header")
msgs.info("Will only update the data extension for {} detector(s)".format(
update_det))
prihdu = hdus[0]
# Names
hdu_names = [hdu.name for hdu in hdus]
# Remove the detector(s) being updated
if not isinstance(update_det, list):
update_det = [update_det]
popme = []
# Find em
for ss, hdu_name in enumerate(hdu_names):
for det in update_det:
sdet = parse.get_dnum(det, prefix=False)
idx = '{:s}{:s}'.format(specobj.naming_model['det'], sdet)
if idx in hdu_name:
popme.append(ss)
# Remove em (and the bit in the Header too)
for popthis in reversed(popme):
hdus.pop(popthis)
keywd = 'EXT{:04d}'.format(popthis)
prihdu.header.remove(keywd)
# Return
return hdus, prihdu
def lacosmic(det,
sciframe,
saturation,
nonlinear,
varframe=None,
maxiter=1,
grow=1.5,
remove_compact_obj=True,
sigclip=5.0,
sigfrac=0.3,
objlim=5.0):
"""
Identify cosmic rays using the L.A.Cosmic algorithm
U{http://www.astro.yale.edu/dokkum/lacosmic/}
(article : U{http://arxiv.org/abs/astro-ph/0108003})
This routine is mostly courtesy of Malte Tewes
Args:
det:
sciframe:
saturation:
nonlinear:
varframe:
maxiter:
grow:
remove_compact_obj:
sigclip:
sigfrac:
objlim:
Returns:
ndarray: mask of cosmic rays (0=no CR, 1=CR)
"""
dnum = parse.get_dnum(det)
msgs.info("Detecting cosmic rays with the L.A.Cosmic algorithm")
# msgs.work("Include these parameters in the settings files to be adjusted by the user")
# Set the settings
scicopy = sciframe.copy()
crmask = np.cast['bool'](np.zeros(sciframe.shape))
sigcliplow = sigclip * sigfrac
# Determine if there are saturated pixels
satpix = np.zeros_like(sciframe)
# satlev = settings_det['saturation']*settings_det['nonlinear']
satlev = saturation * nonlinear
wsat = np.where(sciframe >= satlev)
if wsat[0].size == 0: satpix = None
else:
satpix[wsat] = 1.0
satpix = np.cast['bool'](satpix)
# Define the kernels
laplkernel = np.array([[0.0, -1.0, 0.0], [-1.0, 4.0, -1.0],
[0.0, -1.0, 0.0]]) # Laplacian kernal
growkernel = np.ones((3, 3))
for i in range(1, maxiter + 1):
msgs.info("Convolving image with Laplacian kernel")
# Subsample, convolve, clip negative values, and rebin to original size
subsam = utils.subsample(scicopy)
conved = signal.convolve2d(subsam,
laplkernel,
mode="same",
boundary="symm")
cliped = conved.clip(min=0.0)
lplus = utils.rebin_evlist(cliped, np.array(cliped.shape) / 2.0)
msgs.info("Creating noise model")
# Build a custom noise map, and compare this to the laplacian
m5 = ndimage.filters.median_filter(scicopy, size=5, mode='mirror')
if varframe is None:
noise = np.sqrt(np.abs(m5))
else:
noise = np.sqrt(varframe)
msgs.info("Calculating Laplacian signal to noise ratio")
# Laplacian S/N
s = lplus / (2.0 * noise
) # Note that the 2.0 is from the 2x2 subsampling
# Remove the large structures
sp = s - ndimage.filters.median_filter(s, size=5, mode='mirror')
msgs.info("Selecting candidate cosmic rays")
# Candidate cosmic rays (this will include HII regions)
candidates = sp > sigclip
nbcandidates = np.sum(candidates)
msgs.info("{0:5d} candidate pixels".format(nbcandidates))
# At this stage we use the saturated stars to mask the candidates, if available :
if satpix is not None:
msgs.info("Masking saturated pixels")
candidates = np.logical_and(np.logical_not(satpix), candidates)
nbcandidates = np.sum(candidates)
msgs.info(
"{0:5d} candidate pixels not part of saturated stars".format(
nbcandidates))
msgs.info("Building fine structure image")
# We build the fine structure image :
m3 = ndimage.filters.median_filter(scicopy, size=3, mode='mirror')
m37 = ndimage.filters.median_filter(m3, size=7, mode='mirror')
f = m3 - m37
f /= noise
f = f.clip(min=0.01)
msgs.info("Removing suspected compact bright objects")
# Now we have our better selection of cosmics :
if remove_compact_obj:
cosmics = np.logical_and(candidates, sp / f > objlim)
else:
cosmics = candidates
nbcosmics = np.sum(cosmics)
msgs.info("{0:5d} remaining candidate pixels".format(nbcosmics))
# What follows is a special treatment for neighbors, with more relaxed constains.
msgs.info("Finding neighboring pixels affected by cosmic rays")
# We grow these cosmics a first time to determine the immediate neighborhod :
growcosmics = np.cast['bool'](signal.convolve2d(
np.cast['float32'](cosmics),
growkernel,
mode="same",
boundary="symm"))
# From this grown set, we keep those that have sp > sigmalim
# so obviously not requiring sp/f > objlim, otherwise it would be pointless
growcosmics = np.logical_and(sp > sigclip, growcosmics)
# Now we repeat this procedure, but lower the detection limit to sigmalimlow :
finalsel = np.cast['bool'](signal.convolve2d(
np.cast['float32'](growcosmics),
growkernel,
mode="same",
boundary="symm"))
finalsel = np.logical_and(sp > sigcliplow, finalsel)
# Unmask saturated pixels:
if satpix is not None:
msgs.info("Masking saturated stars")
finalsel = np.logical_and(np.logical_not(satpix), finalsel)
ncrp = np.sum(finalsel)
msgs.info("{0:5d} pixels detected as cosmics".format(ncrp))
# We find how many cosmics are not yet known :
newmask = np.logical_and(np.logical_not(crmask), finalsel)
nnew = np.sum(newmask)
# We update the mask with the cosmics we have found :
crmask = np.logical_or(crmask, finalsel)
msgs.info(
"Iteration {0:d} -- {1:d} pixels identified as cosmic rays ({2:d} new)"
.format(i, ncrp, nnew))
if ncrp == 0: break
# Additional algorithms (not traditionally implemented by LA cosmic) to remove some false positives.
msgs.work(
"The following algorithm would be better on the rectified, tilts-corrected image"
)
filt = ndimage.sobel(sciframe, axis=1, mode='constant')
filty = ndimage.sobel(filt / np.sqrt(np.abs(sciframe)),
axis=0,
mode='constant')
filty[np.where(np.isnan(filty))] = 0.0
sigimg = cr_screen(filty)
sigsmth = ndimage.filters.gaussian_filter(sigimg, 1.5)
sigsmth[np.where(np.isnan(sigsmth))] = 0.0
sigmask = np.cast['bool'](np.zeros(sciframe.shape))
sigmask[np.where(sigsmth > sigclip)] = True
crmask = np.logical_and(crmask, sigmask)
msgs.info("Growing cosmic ray mask by 1 pixel")
crmask = grow_masked(crmask.astype(np.float), grow, 1.0)
return crmask.astype(bool)
def main(args):
# Build the fitstable since we currently need it for output. This should not be the case!
A_files = [os.path.join(args.full_rawpath, file) for file in args.Afiles]
B_files = [os.path.join(args.full_rawpath, file) for file in args.Bfiles]
data_files = A_files + B_files
ps = pypeitsetup.PypeItSetup(A_files,
path='./',
spectrograph_name='keck_mosfire')
ps.build_fitstbl()
fitstbl = ps.fitstbl
# Read in the spectrograph, config the parset
spectrograph = load_spectrograph('keck_mosfire')
spectrograph_def_par = spectrograph.default_pypeit_par()
parset = par.PypeItPar.from_cfg_lines(
cfg_lines=spectrograph_def_par.to_config(),
merge_with=config_lines(args))
science_path = os.path.join(parset['rdx']['redux_path'],
parset['rdx']['scidir'])
# Calibration Master directory
if args.master_dir is None:
msgs.error(
"You need to set an Environmental variable MOSFIRE_MASTERS that points at the Master Calibs"
)
# Define some hard wired master files here to be later parsed out of the directory
slit_masterframe_name = os.path.join(args.master_dir,
'MasterSlits_E_15_01.fits.gz')
tilts_masterframe_name = os.path.join(args.master_dir,
'MasterTilts_E_1_01.fits')
wvcalib_masterframe_name = os.path.join(args.master_dir,
'MasterWaveCalib_E_1_01.fits')
# For now don't require a standard
std_outfile = None
#std_outfile = os.path.join('/Users/joe/Dropbox/PypeIt_Redux/MOSFIRE/Nov19/quicklook/Science/',
# 'spec1d_m191118_0064-GD71_MOSFIRE_2019Nov18T104704.507.fits')
# make the get_std from pypeit a utility function or class method
det = 1 # MOSFIRE has a single detector
if std_outfile is not None:
# Get the standard trace if need be
sobjs = specobjs.SpecObjs.from_fitsfile(std_outfile)
this_det = sobjs.DET == det
if np.any(this_det):
sobjs_det = sobjs[this_det]
sobjs_std = sobjs_det.get_std()
std_trace = None if sobjs_std is None else sobjs_std.TRACE_SPAT.flatten(
)
else:
std_trace = None
else:
std_trace = None
# Read in the msbpm
sdet = get_dnum(det, prefix=False)
msbpm = spectrograph.bpm(A_files[0], det)
# Read in the slits
slits = slittrace.SlitTraceSet.from_file(slit_masterframe_name)
# Reset the bitmask
slits.mask = slits.mask_init.copy()
# Read in the wv_calib
wv_calib = wavecalib.WaveCalib.from_file(wvcalib_masterframe_name)
wv_calib.is_synced(slits)
slits.mask_wvcalib(wv_calib)
# Read in the tilts
tilts_obj = wavetilts.WaveTilts.from_file(tilts_masterframe_name)
tilts_obj.is_synced(slits)
slits.mask_wavetilts(tilts_obj)
# Build Science image
sciImg = buildimage.buildimage_fromlist(spectrograph,
det,
parset['scienceframe'],
A_files,
bpm=msbpm,
slits=slits,
ignore_saturation=False)
# Background Image?
sciImg = sciImg.sub(
buildimage.buildimage_fromlist(spectrograph,
det,
parset['scienceframe'],
B_files,
bpm=msbpm,
slits=slits,
ignore_saturation=False),
parset['scienceframe']['process'])
# Build the Calibrate object
caliBrate = calibrations.Calibrations(None, parset['calibrations'],
spectrograph, None)
caliBrate.slits = slits
caliBrate.wavetilts = tilts_obj
caliBrate.wv_calib = wv_calib
# Instantiate Reduce object
# Required for pypeline specific object
# At instantiaton, the fullmask in self.sciImg is modified
redux = reduce.Reduce.get_instance(sciImg,
spectrograph,
parset,
caliBrate,
'science',
ir_redux=True,
show=args.show,
det=det,
std_outfile=std_outfile)
manual_extract_dict = None
skymodel, objmodel, ivarmodel, outmask, sobjs, waveImg, tilts = redux.run(
std_trace=std_trace,
return_negative=True,
manual_extract_dict=manual_extract_dict,
show_peaks=args.show)
# TODO -- Do this upstream
# Tack on detector
for sobj in sobjs:
sobj.DETECTOR = sciImg.detector
# Construct the Spec2DObj with the positive image
spec2DObj_A = spec2dobj.Spec2DObj(det=det,
sciimg=sciImg.image,
ivarraw=sciImg.ivar,
skymodel=skymodel,
objmodel=objmodel,
ivarmodel=ivarmodel,
waveimg=waveImg,
bpmmask=outmask,
detector=sciImg.detector,
sci_spat_flexure=sciImg.spat_flexure,
tilts=tilts,
slits=copy.deepcopy(caliBrate.slits))
spec2DObj_A.process_steps = sciImg.process_steps
all_spec2d = spec2dobj.AllSpec2DObj()
all_spec2d['meta']['ir_redux'] = True
all_spec2d[det] = spec2DObj_A
# Save image A but with all the objects extracted, i.e. positive and negative
#outfile2d, outfile1d = save_exposure(fitstbl, 0, spectrograph, science_path, parset, caliBrate, all_spec2d, sobjs)
# Construct the Spec2DObj with the negative image
spec2DObj_B = spec2dobj.Spec2DObj(det=det,
sciimg=-sciImg.image,
ivarraw=sciImg.ivar,
skymodel=-skymodel,
objmodel=-objmodel,
ivarmodel=ivarmodel,
waveimg=waveImg,
bpmmask=outmask,
detector=sciImg.detector,
sci_spat_flexure=sciImg.spat_flexure,
tilts=tilts,
slits=copy.deepcopy(caliBrate.slits))
# Parse the offset information out of the headers. TODO in the future get this out of fitstable
dither_pattern_A, dither_id_A, offset_arcsec_A = parse_dither_pattern(
A_files, spectrograph.primary_hdrext)
dither_pattern_B, dither_id_B, offset_arcsec_B = parse_dither_pattern(
B_files, spectrograph.primary_hdrext)
# Print out a report on the offsets
msg_string = msgs.newline(
) + '****************************************************'
msg_string += msgs.newline(
) + ' Summary of offsets for dither pattern: {:s}'.format(
dither_pattern_A[0])
msg_string += msgs.newline(
) + '****************************************************'
msg_string += msgs.newline(
) + 'Position filename arcsec pixels '
msg_string += msgs.newline(
) + '----------------------------------------------------'
for iexp, file in enumerate(A_files):
msg_string += msgs.newline(
) + ' A {:s} {:6.2f} {:6.2f}'.format(
os.path.basename(file), offset_arcsec_A[iexp],
offset_arcsec_A[iexp] / sciImg.detector.platescale)
for iexp, file in enumerate(B_files):
msg_string += msgs.newline(
) + ' B {:s} {:6.2f} {:6.2f}'.format(
os.path.basename(file), offset_arcsec_B[iexp],
offset_arcsec_B[iexp] / sciImg.detector.platescale)
msg_string += msgs.newline(
) + '****************************************************'
msgs.info(msg_string)
#offset_dith_pix = offset_dith_pix = offset_arcsec_A[0]/sciImg.detector.platescale
offsets_dith_pix = (np.array([
0.0, np.mean(offset_arcsec_B) - np.mean(offset_arcsec_A)
])) / sciImg.detector.platescale
if args.offset is not None:
offsets_pixels = np.array([0.0, args.offset])
msgs.info('Using user specified offsets instead: {:5.2f}'.format(
args.offset))
else:
offsets_pixels = offsets_dith_pix
spec2d_list = [spec2DObj_A, spec2DObj_B]
# Instantiate Coadd2d
coadd = coadd2d.CoAdd2D.get_instance(spec2d_list,
spectrograph,
parset,
det=det,
offsets=offsets_pixels,
weights='uniform',
ir_redux=True,
debug=args.show,
samp_fact=args.samp_fact)
# Coadd the slits
coadd_dict_list = coadd.coadd(
only_slits=None, interp_dspat=False) # TODO implement only_slits later
# Create the pseudo images
pseudo_dict = coadd.create_pseudo_image(coadd_dict_list)
##########################
# Now display the images #
##########################
display.display.connect_to_ginga(raise_err=True, allow_new=True)
# Bug in ginga prevents me from using cuts here for some reason
#mean, med, sigma = sigma_clipped_stats(pseudo_dict['imgminsky'][pseudo_dict['inmask']], sigma_lower=5.0,sigma_upper=5.0)
#cut_min = mean - 4.0 * sigma
#cut_max = mean + 4.0 * sigma
chname_skysub = 'skysub-det{:s}'.format(sdet)
# Clear all channels at the beginning
# TODO: JFH For some reason Ginga crashes when I try to put cuts in here.
viewer, ch = ginga.show_image(pseudo_dict['imgminsky'],
chname=chname_skysub,
waveimg=pseudo_dict['waveimg'],
clear=True) # cuts=(cut_min, cut_max),
slit_left, slit_righ, _ = pseudo_dict['slits'].select_edges()
slit_id = slits.slitord_id[0]
ginga.show_slits(viewer, ch, slit_left, slit_righ, slit_ids=slit_id)
# SKRESIDS
chname_skyresids = 'sky_resid-det{:s}'.format(sdet)
image = pseudo_dict['imgminsky'] * np.sqrt(
pseudo_dict['sciivar']) * pseudo_dict['inmask'] # sky residual map
viewer, ch = ginga.show_image(
image,
chname_skyresids,
waveimg=pseudo_dict['waveimg'],
cuts=(-5.0, 5.0),
)
ginga.show_slits(viewer,
ch,
slit_left,
slit_righ,
slit_ids=slits.slitord_id[0])
shell = viewer.shell()
out = shell.start_global_plugin('WCSMatch')
out = shell.call_global_plugin_method('WCSMatch', 'set_reference_channel',
[chname_skyresids], {})
if args.embed:
embed()
return 0
def main(args):
raise NotImplementedError('This script is currently out of date.')
# List only?
hdu = fits.open(args.file)
head0 = hdu[0].header
if args.list:
hdu.info()
return
# Init
sdet = get_dnum(args.det, prefix=False)
# One detector, sky sub for now
names = [hdu[i].name for i in range(len(hdu))]
try:
exten = names.index('DET{:s}-PROCESSED'.format(sdet))
except: # Backwards compatability
msgs.error(
'Requested detector {:s} was not processed.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
sciimg = hdu[exten].data
try:
exten = names.index('DET{:s}-SKY'.format(sdet))
except: # Backwards compatability
msgs.error(
'Requested detector {:s} has no sky model.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
skymodel = hdu[exten].data
try:
exten = names.index('DET{:s}-MASK'.format(sdet))
except ValueError: # Backwards compatability
msgs.error(
'Requested detector {:s} has no bit mask.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
mask = hdu[exten].data
frame = (sciimg - skymodel) * (mask == 0)
mdir = head0['PYPMFDIR']
mkey = head0['FRAMMKEY']
mast_key = '{0}_{1:02d}'.format(mkey, args.det)
if not os.path.exists(mdir):
mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir))
msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(
mdir, mdir_base))
mdir = mdir_base
# Assumes a MasterSlit file has been written
#slits = slittrace.SlitTraceSet.from_master('{0}_{1:02d}'.format(head0['TRACMKEY'], args.det),
# mdir)
# Load the slits information
slits = slittrace.SlitTraceSet.from_master(mast_key, mdir)
# Object traces
left, right, mask = slits.select_edges()
msgs.error("You need to choose which slits you care about here")
# Get object traces
spec1d_file = args.file.replace('spec2d', 'spec1d')
if os.path.isfile(spec1d_file):
hdulist_1d = fits.open(spec1d_file)
else:
hdulist_1d = []
msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) +
msgs.newline() +
' No objects were extracted.')
msgs.error(
"This code needs to be refactored since tslits_dict was removed...")
import pdb
pdb.set_trace()
tslits_dict['objtrc'] = parse_traces(hdulist_1d, det_nm)
obj_trace = parse_traces(hdulist_1d, 'DET{:s}'.format(sdet))
# TODO :: Need to include standard star trace in the spec2d files
std_trace = None
# Extract some trace models
fwhm = 2 # Start with some default value
# TODO: Dictionaries like this are a pet peeve of mine. I'd prefer
# either individual objects or a class with a well-formed data model.
# TODO: Why do all of these dictionary elements need fwhm? Can they
# be different?
trace_models = dict()
# Brightest object on slit
trace_models['object'] = dict(trace_model=None, fwhm=fwhm)
if len(obj_trace['pkflux']) > 0:
smash_peakflux = obj_trace['pkflux']
ibri = smash_peakflux.argmax()
trace_models['object']['trace_model'] = obj_trace['traces'][ibri]
trace_models['object']['fwhm'] = obj_trace['fwhm'][ibri]
# Standard star trace
trace_models['std'] = dict(trace_model=std_trace,
fwhm=trace_models['object']['fwhm'])
# Trace of the slit edge
# TODO: Any particular reason to use the lefts?
trace_models['slit'] = dict(trace_model=left.copy(),
fwhm=trace_models['object']['fwhm'])
# Finally, initialise the GUI
gui.object_find.initialise(args.det,
frame,
left,
right,
obj_trace,
trace_models,
None,
printout=True,
slit_ids=slits.id)
ofgui = gui_object_find.initialise(args.det,
frame,
tslits_dict,
None,
printout=True,
slit_ids=slits.id)
def set_name(self):
"""
Generate a unique index for this spectrum based on the
slit/order, its position and for multi-slit the detector.
Multi-slit
Each object is named by its:
- spatial position (pixel number) on the reduced image [SPAT]
- the slit number based on SPAT center of the slit or SlitMask ID [SLIT]
- the detector number [DET]
For example::
SPAT0176-SLIT0185-DET01
Echelle
Returns:
str:
"""
if 'Echelle' in self.PYPELINE:
# ObjID
name = naming_model['obj']
ech_name = naming_model['obj']
if self['ECH_FRACPOS'] is None:
name += '----'
else:
# JFH TODO Why not just write it out with the decimal place. That is clearer than this??
name += '{:04d}'.format(int(np.rint(1000 * self.ECH_FRACPOS)))
ech_name += '{:04d}'.format(
int(np.rint(1000 * self.ECH_FRACPOS)))
sdet = parse.get_dnum(self.DET, prefix=False)
name += '-{:s}{:s}'.format(naming_model['det'], sdet)
ech_name += '-{:s}{:s}'.format(naming_model['det'], sdet)
# Order number
name += '-' + naming_model['order']
name += '{:04d}'.format(self.ECH_ORDER)
self.ECH_NAME = ech_name
self.NAME = name
elif 'MultiSlit' in self.PYPELINE:
# Spat
name = naming_model['spat']
if self['SPAT_PIXPOS'] is None:
name += '----'
else:
name += '{:04d}'.format(int(np.rint(self.SPAT_PIXPOS)))
# Slit
name += '-' + naming_model['slit']
name += '{:04d}'.format(self.SLITID)
sdet = parse.get_dnum(self.DET, prefix=False)
name += '-{:s}{:s}'.format(naming_model['det'], sdet)
self.NAME = name
elif 'IFU' in self.PYPELINE:
# Spat
name = naming_model['spat']
if self['SPAT_PIXPOS'] is None:
name += '----'
else:
name += '{:04d}'.format(int(np.rint(self.SPAT_PIXPOS)))
# Slit
name += '-' + naming_model['slit']
name += '{:04d}'.format(self.SLITID)
sdet = parse.get_dnum(self.DET, prefix=False)
name += '-{:s}{:s}'.format(naming_model['det'], sdet)
self.NAME = name
else:
msgs.error("Bad PYPELINE")
flat_model = np.ones_like(flat)
flat_model[thismask] = twod_model[thismask]*np.fmax(illumflat[thismask],0.05)*np.fmax(spec_model[thismask],1.0)
pixelflat[thismask] = flat[thismask]/flat_model[thismask]
# ToDo Add some code here to treat the edges and places where fits go bad?
return (pixelflat[thismask], illumflat[thismask], flat_model[thismask])
'''
type = 'ESI'
devpath = os.getenv('PYPEIT_DEV')
if type == 'LRIS_red':
det = 1
sdet = parse.get_dnum(det, prefix=False)
rawpath = devpath + '/RAW_DATA/Keck_LRIS_red/multi_400_8500_d560/'
masterpath = devpath + '/REDUX_OUT/Keck_LRIS_red/multi_400_8500_d560/MF_keck_lris_red/'
# Read in the msbias for bias subtraction
biasfile = masterpath + 'MasterBias_A_' + sdet + '_aa.fits'
msbias = fits.getdata(biasfile)
# Read in and process flat field images
pixflat_image_files = np.core.defchararray.add(rawpath, ['r170320_2057.fits','r170320_2058.fits','r170320_2059.fits']).tolist()
spectro_name = 'keck_lris_red'
spectrograph = load_spectrograph(spectrograph=spectro_name)
par = spectrograph.default_pypeit_par()
flatField = flatfield.FlatField(spectrograph, file_list=pixflat_image_files,det=det, par=par['calibrations']['pixelflatframe']
, msbias = msbias)
flatimg = flatField.build_pixflat()
# Read in the tilts
def main(args):
tstart = time.time()
# Parse the files sort by MJD
files = np.array([os.path.join(args.full_rawpath, file) for file in args.files])
nfiles = len(files)
# Read in the spectrograph, config the parset
spectrograph = load_spectrograph('vlt_fors2')
#spectrograph_def_par = spectrograph.default_pypeit_par()
spectrograph_cfg_lines = spectrograph.config_specific_par(files[0]).to_config()
parset = par.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
merge_with=config_lines(args))
science_path = os.path.join(parset['rdx']['redux_path'], parset['rdx']['scidir'])
target = spectrograph.get_meta_value(files[0], 'target')
mjds = np.zeros(nfiles)
for ifile, file in enumerate(files):
mjds[ifile] = spectrograph.get_meta_value(file, 'mjd', ignore_bad_header=True,
no_fussing=True)
files = files[np.argsort(mjds)]
# Calibration Master directory
#TODO hardwired for now
master_dir ='./'
#master_dir = resource_filename('pypeit', 'data/QL_MASTERS') \
# if args.master_dir is None else args.master_dir
if not os.path.isdir(master_dir):
msgs.error(f'{master_dir} does not exist! You must install the QL_MASTERS '
'directory; download the data from the PypeIt dev-suite Google Drive and '
'either define a QL_MASTERS environmental variable or use the '
'pypeit_install_ql_masters script.')
# Define some hard wired master files here to be later parsed out of the directory
fors2_grism = spectrograph.get_meta_value(files[0], 'dispname')
fors2_masters = os.path.join(master_dir, 'FORS2_MASTERS', fors2_grism)
bias_masterframe_name = \
utils.find_single_file(os.path.join(fors2_masters, "MasterBias*"))
slit_masterframe_name \
= utils.find_single_file(os.path.join(fors2_masters, "MasterSlits*"))
tilts_masterframe_name \
= utils.find_single_file(os.path.join(fors2_masters, "MasterTilts*"))
wvcalib_masterframe_name \
= utils.find_single_file(os.path.join(fors2_masters, 'MasterWaveCalib*'))
std_spec1d_file = utils.find_single_file(os.path.join(fors2_masters, 'spec1d_*'))
sensfunc_masterframe_name = utils.find_single_file(os.path.join(fors2_masters, 'sens_*'))
# TODO make and impelement sensfunc
if (bias_masterframe_name is None or not os.path.isfile(bias_masterframe_name)) or \
(slit_masterframe_name is None or not os.path.isfile(slit_masterframe_name)) or \
(tilts_masterframe_name is None or not os.path.isfile(tilts_masterframe_name)) or \
(std_spec1d_file is None or not os.path.isfile(std_spec1d_file)):
# or (sensfunc_masterframe_name is None or not os.path.isfile(sensfunc_masterframe_name)):
msgs.error('Master frames not found. Check that environment variable QL_MASTERS '
'points at the Master Calibs')
# We need the platescale
# Get detector (there's only one)
#det = 1 # MOSFIRE has a single detector
#detector = spectrograph.get_detector_par(det)
#detname = detector.name
# We need the platescale
det_container = spectrograph.get_detector_par(1, hdu=fits.open(files[0]))
binspectral, binspatial = parse_binning(det_container['binning'])
platescale = det_container['platescale']*binspatial
# Parse the offset information out of the headers.
_, _, offset_arcsec = spectrograph.parse_dither_pattern(files)
# Print out a report on the offsets
msg_string = msgs.newline() + '*******************************************************'
msg_string += msgs.newline() + ' Summary of offsets for target {:s}: '
msg_string += msgs.newline() + '*******************************************************'
msg_string += msgs.newline() + ' filename arcsec pixels '
msg_string += msgs.newline() + '----------------------------------------------------'
for iexp, file in enumerate(files):
msg_string += msgs.newline() + ' {:s} {:6.2f} {:6.2f}'.format(
os.path.basename(file), offset_arcsec[iexp], offset_arcsec[iexp] / platescale)
msg_string += msgs.newline() + '********************************************************'
msgs.info(msg_string)
## Read in the master frames that we need
##
det = 1 # Currently CHIP1 is supported
if std_spec1d_file is not None:
# Get the standard trace if need be
sobjs = specobjs.SpecObjs.from_fitsfile(std_spec1d_file)
this_det = sobjs.DET == det
if np.any(this_det):
sobjs_det = sobjs[this_det]
sobjs_std = sobjs_det.get_std()
std_trace = None if sobjs_std is None else sobjs_std.TRACE_SPAT.flatten()
else:
std_trace = None
else:
std_trace = None
# Read in the bias
msbias = buildimage.BiasImage.from_file(bias_masterframe_name)
# Read in the msbpm
sdet = get_dnum(det, prefix=False)
msbpm = spectrograph.bpm(files[0], det)
# Read in the slits
slits = slittrace.SlitTraceSet.from_file(slit_masterframe_name)
# Reset the bitmask
slits.mask = slits.mask_init.copy()
# Read in the wv_calib
wv_calib = wavecalib.WaveCalib.from_file(wvcalib_masterframe_name)
# wv_calib.is_synced(slits)
slits.mask_wvcalib(wv_calib)
# Read in the tilts
tilts_obj = wavetilts.WaveTilts.from_file(tilts_masterframe_name)
tilts_obj.is_synced(slits)
slits.mask_wavetilts(tilts_obj)
# Build the Calibrate object
caliBrate = calibrations.Calibrations(None, parset['calibrations'], spectrograph, None)
caliBrate.msbias = msbias
caliBrate.msbpm = msbpm
caliBrate.slits = slits
caliBrate.wavetilts = tilts_obj
caliBrate.wv_calib = wv_calib
# Find the unique offsets. This is a bit of a kludge, i.e. we are considering offsets within
# 0.1 arcsec of each other to be the same throw, but I should like to be able to specify a tolerance here,
# but then I need a version of unique that accepts a tolerance
uniq_offsets, uni_indx = np.unique(np.around(offset_arcsec), return_inverse=True)
nuniq = uniq_offsets.size
spec2d_list = []
offset_ref = offset_arcsec[0]
offsets_dith_pix = []
# Generalize to a multiple slits, doing one slit at a time?
islit = 0
# Loop over the unique throws and create a spec2d_A and spec2D_B for
# each, which are then fed into coadd2d with the correct offsets
# TODO Rework the logic here so that we can print out a unified report
# on what was actually reduced.
for iuniq in range(nuniq):
indx = uni_indx == iuniq
files_uni = files[indx]
offsets = offset_arcsec[indx]
msgs.info('Reducing images for offset = {:}'.format(offsets[0]))
spec2DObj = run(files_uni, caliBrate, spectrograph, det, parset, show=args.show, std_trace=std_trace)
spec2d_list += [spec2DObj]
offsets_dith_pix += [np.mean(offsets)/platescale]
offsets_dith_pix = np.array(offsets_dith_pix)
if args.offset is not None:
offsets_pixels = np.array([0.0, args.offset])
msgs.info('Using user specified offsets instead: {:5.2f}'.format(args.offset))
else:
offsets_pixels = offsets_dith_pix
# Instantiate Coadd2d
coadd = coadd2d.CoAdd2D.get_instance(spec2d_list, spectrograph, parset, det=det,
offsets=offsets_pixels, weights='uniform',
spec_samp_fact=args.spec_samp_fact,
spat_samp_fact=args.spat_samp_fact,
ir_redux=True, debug=args.show)
# Coadd the slits
# TODO implement only_slits later
coadd_dict_list = coadd.coadd(only_slits=None, interp_dspat=False)
# Create the pseudo images
pseudo_dict = coadd.create_pseudo_image(coadd_dict_list)
# Multiply in a sensitivity function to flux the 2d image
if args.flux:
# Load the sensitivity function
# wave_sens, sfunc, _, _, _ = sensfunc.SensFunc.load(sensfunc_masterframe_name)
sens = sensfunc.SensFunc.from_file(sensfunc_masterframe_name)
# Interpolate the sensitivity function onto the wavelength grid of
# the data. Since the image is rectified this is trivial and we
# don't need to do a 2d interpolation
exptime = spectrograph.get_meta_value(files[0], 'exptime')
sens_factor = flux_calib.get_sensfunc_factor(pseudo_dict['wave_mid'][:, islit],
sens.wave, sens.zeropoint, exptime,
extrap_sens=parset['fluxcalib']['extrap_sens'])
# Compute the median sensitivity and set the sensitivity to zero at
# locations 100 times the median. This prevents the 2d image from
# blowing up where the sens_factor explodes because there is no
# throughput
sens_gpm = sens_factor < 100.0 * np.median(sens_factor)
sens_factor_masked = sens_factor * sens_gpm
sens_factor_img = np.repeat(sens_factor_masked[:, np.newaxis], pseudo_dict['nspat'],
axis=1)
imgminsky = sens_factor_img * pseudo_dict['imgminsky']
imgminsky_gpm = sens_gpm[:, np.newaxis] & pseudo_dict['inmask']
else:
imgminsky = pseudo_dict['imgminsky']
imgminsky_gpm = pseudo_dict['inmask']
##########################
# Now display the images #
##########################
if not args.no_gui:
display.connect_to_ginga(raise_err=True, allow_new=True)
# TODO: Bug in ginga prevents me from using cuts here for some
# reason
mean, med, sigma = sigma_clipped_stats(imgminsky[imgminsky_gpm], sigma_lower=3.0,
sigma_upper=3.0)
chname_skysub = 'fluxed-skysub-det{:s}'.format(sdet) \
if args.flux else 'skysub-det{:s}'.format(sdet)
cuts_skysub = (med - 3.0 * sigma, med + 3.0 * sigma)
cuts_resid = (-5.0, 5.0)
# fits.writeto('/Users/joe/ginga_test.fits',imgminsky, overwrite=True)
# fits.writeto('/Users/joe/ginga_mask.fits',imgminsky_gpm.astype(float), overwrite=True)
# embed()
# Clear all channels at the beginning
# TODO: JFH For some reason Ginga crashes when I try to put cuts in here.
viewer, ch_skysub = display.show_image(imgminsky, chname=chname_skysub,
waveimg=pseudo_dict['waveimg'], clear=True,
cuts=cuts_skysub)
slit_left, slit_righ, _ = pseudo_dict['slits'].select_edges()
slit_id = slits.slitord_id[0]
display.show_slits(viewer, ch_skysub, slit_left, slit_righ, slit_ids=slit_id)
# SKRESIDS
chname_skyresids = 'sky_resid-det{:s}'.format(sdet)
# sky residual map
image = pseudo_dict['imgminsky'] * np.sqrt(pseudo_dict['sciivar']) * pseudo_dict['inmask']
viewer, ch_skyresids = display.show_image(image, chname_skyresids,
waveimg=pseudo_dict['waveimg'],
cuts=cuts_resid)
display.show_slits(viewer, ch_skyresids, slit_left, slit_righ,
slit_ids=slits.slitord_id[0])
shell = viewer.shell()
out = shell.start_global_plugin('WCSMatch')
out = shell.call_global_plugin_method('WCSMatch', 'set_reference_channel',
[chname_skysub], {})
# TODO extract along a spatial position
if args.writefits:
head0 = fits.getheader(files[0])
# TODO use meta tools for the object name in the future.
outfile = target + '_specXspat_{:3.2f}X{:3.2f}.fits'.format(args.spec_samp_fact,
args.spat_samp_fact)
hdu = fits.PrimaryHDU(imgminsky, header=head0)
hdu_resid = fits.ImageHDU(pseudo_dict['imgminsky'] \
* np.sqrt(pseudo_dict['sciivar']) * pseudo_dict['inmask'])
hdu_wave = fits.ImageHDU(pseudo_dict['waveimg'])
hdul = fits.HDUList([hdu, hdu_resid, hdu_wave])
msgs.info('Writing sky subtracted image to {:s}'.format(outfile))
hdul.writeto(outfile, overwrite=True)
msgs.info(utils.get_time_string(time.time()-tstart))
if args.embed:
embed()
return 0
def main(args):
# List only?
hdu = fits.open(args.file)
head0 = hdu[0].header
if args.list:
hdu.info()
return
# Init
sdet = get_dnum(args.det, prefix=False)
# One detector, sky sub for now
names = [hdu[i].name for i in range(len(hdu))]
try:
exten = names.index('DET{:s}-PROCESSED'.format(sdet))
except: # Backwards compatability
msgs.error('Requested detector {:s} was not processed.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
sciimg = hdu[exten].data
try:
exten = names.index('DET{:s}-SKY'.format(sdet))
except: # Backwards compatability
msgs.error('Requested detector {:s} has no sky model.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
skymodel = hdu[exten].data
try:
exten = names.index('DET{:s}-MASK'.format(sdet))
except ValueError: # Backwards compatability
msgs.error('Requested detector {:s} has no bit mask.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
mask = hdu[exten].data
frame = (sciimg - skymodel) * (mask == 0)
mdir = head0['PYPMFDIR']
if not os.path.exists(mdir):
mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir))
msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base))
mdir = mdir_base
trace_key = '{0}_{1:02d}'.format(head0['TRACMKEY'], args.det)
trc_file = os.path.join(mdir, MasterFrame.construct_file_name('Trace', trace_key))
# TODO -- Remove this once the move to Edges is complete
if args.old:
tslits_dict = TraceSlits.load_from_file(trc_file)[0]
else:
trc_file = trc_file.replace('Trace', 'Edges')+'.gz'
tslits_dict = edgetrace.EdgeTraceSet.from_file(trc_file).convert_to_tslits_dict()
shape = (tslits_dict['nspec'], tslits_dict['nspat'])
slit_ids = [trace_slits.get_slitid(shape, tslits_dict['slit_left'], tslits_dict['slit_righ'], ii)[0]
for ii in range(tslits_dict['slit_left'].shape[1])]
# Object traces
spec1d_file = args.file.replace('spec2d', 'spec1d')
det_nm = 'DET{:s}'.format(sdet)
if os.path.isfile(spec1d_file):
hdulist_1d = fits.open(spec1d_file)
else:
hdulist_1d = []
msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) + msgs.newline() +
' No objects were extracted.')
tslits_dict['objtrc'] = parse_traces(hdulist_1d, det_nm)
# TODO :: Need to include standard star trace in the spec2d files
std_trace = None
# Extract some trace models
fwhm = 2 # Start with some default value
# Brightest object on slit
trace_model_obj = None
trace_model_dict = dict()
if len(tslits_dict['objtrc']['pkflux']) > 0:
smash_peakflux = tslits_dict['objtrc']['pkflux']
ibri = smash_peakflux.argmax()
trace_model_obj = tslits_dict['objtrc']['traces'][ibri]
fwhm = tslits_dict['objtrc']['fwhm'][ibri]
trace_model_dict['object'] = dict(trace_model=trace_model_obj, fwhm=fwhm)
# Standard star trace
trace_model_dict['std'] = dict(trace_model=std_trace, fwhm=fwhm)
# Trace of the slit edge
trace_model_dict['slit'] = dict(trace_model=tslits_dict['slit_left'].copy(), fwhm=fwhm)
tslits_dict['trace_model'] = trace_model_dict
# Finally, initialise the GUI
gui_object_find.initialise(args.det, frame, tslits_dict, None, printout=True, slit_ids=slit_ids)
def load_coadd2d_stacks(self, spec2d):
"""
Routine to read in required images for 2d coadds given a list of spec2d files.
Args:
spec2d_files: list
List of spec2d filenames
det: int
detector in question
Returns:
dict: Dictionary containing all the images and keys required
for perfomring 2d coadds.
"""
# Get the detector string
sdet = parse.get_dnum(self.det, prefix=False)
# Get the master dir
redux_path = os.getcwd()
# Grab the files
#head2d_list = []
specobjs_list = []
slits_list = []
nfiles =len(spec2d)
detectors_list = []
for ifile, f in enumerate(spec2d):
if isinstance(f, spec2dobj.Spec2DObj):
# If spec2d is a list of objects
s2dobj = f
else:
# If spec2d is a list of files, option to also use spec1ds
s2dobj = spec2dobj.Spec2DObj.from_file(f, self.det)
spec1d_file = f.replace('spec2d', 'spec1d')
if os.path.isfile(spec1d_file):
sobjs = specobjs.SpecObjs.from_fitsfile(spec1d_file)
this_det = sobjs.DET == self.det
specobjs_list.append(sobjs[this_det])
# TODO the code should run without a spec1d file, but we need to implement that
slits_list.append(s2dobj.slits)
detectors_list.append(s2dobj.detector)
if ifile == 0:
sciimg_stack = np.zeros((nfiles,) + s2dobj.sciimg.shape, dtype=float)
waveimg_stack = np.zeros_like(sciimg_stack, dtype=float)
tilts_stack = np.zeros_like(sciimg_stack, dtype=float)
skymodel_stack = np.zeros_like(sciimg_stack, dtype=float)
sciivar_stack = np.zeros_like(sciimg_stack, dtype=float)
mask_stack = np.zeros_like(sciimg_stack, dtype=float)
slitmask_stack = np.zeros_like(sciimg_stack, dtype=int)
sciimg_stack[ifile, :, :] = s2dobj.sciimg
waveimg_stack[ifile, :, :] = s2dobj.waveimg
skymodel_stack[ifile, :, :] = s2dobj.skymodel
sciivar_stack[ifile, :, :] = s2dobj.ivarmodel
mask_stack[ifile, :, :] = s2dobj.bpmmask
# TODO -- Set back after done testing
slitmask_stack[ifile, :, :] = s2dobj.slits.slit_img(flexure=s2dobj.sci_spat_flexure)
#slitmask_stack[ifile, :, :] = spec2DObj.slits.slit_img(flexure=0.)
_spat_flexure = 0. if s2dobj.sci_spat_flexure is None else s2dobj.sci_spat_flexure
#_tilt_flexure_shift = _spat_flexure - spec2DObj.tilts.spat_flexure if spec2DObj.tilts.spat_flexure is not None else _spat_flexure
tilts_stack[ifile,:,:] = s2dobj.tilts #.fit2tiltimg(slitmask_stack[ifile, :, :], flexure=_tilt_flexure_shift)
return dict(specobjs_list=specobjs_list, slits_list=slits_list,
slitmask_stack=slitmask_stack,
sciimg_stack=sciimg_stack, sciivar_stack=sciivar_stack,
skymodel_stack=skymodel_stack, mask_stack=mask_stack,
tilts_stack=tilts_stack, waveimg_stack=waveimg_stack,
redux_path=redux_path,
detectors=detectors_list,
spectrograph=self.spectrograph.name,
pypeline=self.spectrograph.pypeline)
def main(args):
# List only?
if args.list:
hdu = fits.open(args.file)
hdu.info()
return
# Load it up -- NOTE WE ALLOW *OLD* VERSIONS TO GO FORTH
spec2DObj = spec2dobj.Spec2DObj.from_file(args.file,
args.det,
chk_version=False)
# Setup for PypeIt imports
msgs.reset(verbosity=2)
# Init
# TODO: get_dnum needs to be deprecated...
sdet = get_dnum(args.det, prefix=False)
# Grab the slit edges
slits = spec2DObj.slits
if spec2DObj.sci_spat_flexure is not None:
msgs.info("Offseting slits by {}".format(spec2DObj.sci_spat_flexure))
all_left, all_right, mask = slits.select_edges(
flexure=spec2DObj.sci_spat_flexure)
# TODO -- This may be too restrictive, i.e. ignore BADFLTCALIB??
gpm = mask == 0
left = all_left[:, gpm]
right = all_right[:, gpm]
slid_IDs = spec2DObj.slits.slitord_id[gpm]
bitMask = ImageBitMask()
# Object traces from spec1d file
spec1d_file = args.file.replace('spec2d', 'spec1d')
if args.file[-2:] == 'gz':
spec1d_file = spec1d_file[:-3]
if os.path.isfile(spec1d_file):
sobjs = specobjs.SpecObjs.from_fitsfile(spec1d_file)
else:
sobjs = None
msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) +
msgs.newline() +
' No objects were extracted.')
display.connect_to_ginga(raise_err=True, allow_new=True)
# Now show each image to a separate channel
# Show the bitmask?
mask_in = None
if args.showmask:
viewer, ch = display.show_image(spec2DObj.bpmmask,
chname="BPM",
waveimg=spec2DObj.waveimg,
clear=True)
#bpm, crmask, satmask, minmask, offslitmask, nanmask, ivar0mask, ivarnanmask, extractmask \
# SCIIMG
image = spec2DObj.sciimg # Processed science image
mean, med, sigma = sigma_clipped_stats(image[spec2DObj.bpmmask == 0],
sigma_lower=5.0,
sigma_upper=5.0)
cut_min = mean - 1.0 * sigma
cut_max = mean + 4.0 * sigma
chname_skysub = 'sciimg-det{:s}'.format(sdet)
# Clear all channels at the beginning
viewer, ch = display.show_image(image,
chname=chname_skysub,
waveimg=spec2DObj.waveimg,
clear=True)
if sobjs is not None:
show_trace(sobjs, args.det, viewer, ch)
display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs)
# SKYSUB
if args.ignore_extract_mask:
# TODO -- Is there a cleaner way to do this?
gpm = (spec2DObj.bpmmask == 0) | (spec2DObj.bpmmask == 2**
bitMask.bits['EXTRACT'])
else:
gpm = spec2DObj.bpmmask == 0
image = (spec2DObj.sciimg - spec2DObj.skymodel
) * gpm #(spec2DObj.mask == 0) # sky subtracted image
mean, med, sigma = sigma_clipped_stats(image[spec2DObj.bpmmask == 0],
sigma_lower=5.0,
sigma_upper=5.0)
cut_min = mean - 1.0 * sigma
cut_max = mean + 4.0 * sigma
chname_skysub = 'skysub-det{:s}'.format(sdet)
# Clear all channels at the beginning
# TODO: JFH For some reason Ginga crashes when I try to put cuts in here.
viewer, ch = display.show_image(
image,
chname=chname_skysub,
waveimg=spec2DObj.waveimg,
bitmask=bitMask,
mask=mask_in) #, cuts=(cut_min, cut_max),wcs_match=True)
if not args.removetrace and sobjs is not None:
show_trace(sobjs, args.det, viewer, ch)
display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs)
# SKRESIDS
chname_skyresids = 'sky_resid-det{:s}'.format(sdet)
image = (spec2DObj.sciimg - spec2DObj.skymodel) * np.sqrt(
spec2DObj.ivarmodel
) * gpm #(spec2DObj.bpmmask == 0) # sky residual map
viewer, ch = display.show_image(image,
chname_skyresids,
waveimg=spec2DObj.waveimg,
cuts=(-5.0, 5.0),
bitmask=bitMask,
mask=mask_in)
if not args.removetrace and sobjs is not None:
show_trace(sobjs, args.det, viewer, ch)
display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs)
# RESIDS
chname_resids = 'resid-det{:s}'.format(sdet)
# full model residual map
image = (spec2DObj.sciimg - spec2DObj.skymodel - spec2DObj.objmodel
) * np.sqrt(spec2DObj.ivarmodel) * (spec2DObj.bpmmask == 0)
viewer, ch = display.show_image(image,
chname=chname_resids,
waveimg=spec2DObj.waveimg,
cuts=(-5.0, 5.0),
bitmask=bitMask,
mask=mask_in)
if not args.removetrace and sobjs is not None:
show_trace(sobjs, args.det, viewer, ch)
display.show_slits(viewer, ch, left, right, slit_ids=slid_IDs)
# After displaying all the images sync up the images with WCS_MATCH
shell = viewer.shell()
shell.start_global_plugin('WCSMatch')
shell.call_global_plugin_method('WCSMatch', 'set_reference_channel',
[chname_resids], {})
if args.embed:
embed()
def load_coadd2d_stacks(spec2d_files, det):
"""
Args:
spec2d_files: list
List of spec2d filenames
det: int
detector in question
Returns:
stack_dict: dict
Dictionary containing all the images and keys required for perfomring 2d coadds.
"""
# Get the detector string
sdet = parse.get_dnum(det, prefix=False)
# Get the master dir
head0 = fits.getheader(spec2d_files[0])
master_dir = os.path.basename(head0['PYPMFDIR'])
redux_path = os.getcwd()
master_path = os.path.join(redux_path, master_dir)
# Grab the files
head2d_list=[]
tracefiles = []
waveimgfiles = []
tiltfiles = []
spec1d_files = []
for f in spec2d_files:
head = fits.getheader(f)
trace_key = '{0}_{1:02d}'.format(head['TRACMKEY'], det)
wave_key = '{0}_{1:02d}'.format(head['ARCMKEY'], det)
head2d_list.append(head)
spec1d_files.append(f.replace('spec2d', 'spec1d'))
tracefiles.append(os.path.join(master_path,
MasterFrame.construct_file_name('Trace', trace_key)))
waveimgfiles.append(os.path.join(master_path,
MasterFrame.construct_file_name('Wave', wave_key)))
tiltfiles.append(os.path.join(master_path,
MasterFrame.construct_file_name('Tilts', wave_key)))
nfiles = len(spec2d_files)
specobjs_list = []
head1d_list=[]
# TODO Sort this out with the correct detector extensions etc.
# Read in the image stacks
for ifile in range(nfiles):
waveimg = WaveImage.load_from_file(waveimgfiles[ifile])
tilts = WaveTilts.load_from_file(tiltfiles[ifile])
hdu = fits.open(spec2d_files[ifile])
# One detector, sky sub for now
names = [hdu[i].name for i in range(len(hdu))]
# science image
try:
exten = names.index('DET{:s}-PROCESSED'.format(sdet))
except: # Backwards compatability
det_error_msg(exten, sdet)
sciimg = hdu[exten].data
# skymodel
try:
exten = names.index('DET{:s}-SKY'.format(sdet))
except: # Backwards compatability
det_error_msg(exten, sdet)
skymodel = hdu[exten].data
# Inverse variance model
try:
exten = names.index('DET{:s}-IVARMODEL'.format(sdet))
except ValueError: # Backwards compatability
det_error_msg(exten, sdet)
sciivar = hdu[exten].data
# Mask
try:
exten = names.index('DET{:s}-MASK'.format(sdet))
except ValueError: # Backwards compatability
det_error_msg(exten, sdet)
mask = hdu[exten].data
if ifile == 0:
# the two shapes accomodate the possibility that waveimg and tilts are binned differently
shape_wave = (nfiles,waveimg.shape[0],waveimg.shape[1])
shape_sci = (nfiles,sciimg.shape[0],sciimg.shape[1])
waveimg_stack = np.zeros(shape_wave,dtype=float)
tilts_stack = np.zeros(shape_wave,dtype=float)
sciimg_stack = np.zeros(shape_sci,dtype=float)
skymodel_stack = np.zeros(shape_sci,dtype=float)
sciivar_stack = np.zeros(shape_sci,dtype=float)
mask_stack = np.zeros(shape_sci,dtype=float)
waveimg_stack[ifile,:,:] = waveimg
tilts_stack[ifile,:,:] = tilts['tilts']
sciimg_stack[ifile,:,:] = sciimg
sciivar_stack[ifile,:,:] = sciivar
mask_stack[ifile,:,:] = mask
skymodel_stack[ifile,:,:] = skymodel
sobjs, head = load.load_specobjs(spec1d_files[ifile])
head1d_list.append(head)
specobjs_list.append(sobjs)
# Right now we assume there is a single tslits_dict for all images and read in the first one
# TODO this needs to become a tslits_dict for each file to accomodate slits defined by flats taken on different
# nights
tslits_dict, _ = TraceSlits.load_from_file(tracefiles[0])
spectrograph = util.load_spectrograph(tslits_dict['spectrograph'])
slitmask = pixels.tslits2mask(tslits_dict)
slitmask_stack = np.einsum('i,jk->ijk', np.ones(nfiles), slitmask)
# Fill the master key dict
head2d = head2d_list[0]
master_key_dict = {}
master_key_dict['frame'] = head2d['FRAMMKEY'] + '_{:02d}'.format(det)
master_key_dict['bpm'] = head2d['BPMMKEY'] + '_{:02d}'.format(det)
master_key_dict['bias'] = head2d['BIASMKEY'] + '_{:02d}'.format(det)
master_key_dict['arc'] = head2d['ARCMKEY'] + '_{:02d}'.format(det)
master_key_dict['trace'] = head2d['TRACMKEY'] + '_{:02d}'.format(det)
master_key_dict['flat'] = head2d['FLATMKEY'] + '_{:02d}'.format(det)
stack_dict = dict(specobjs_list=specobjs_list, tslits_dict=tslits_dict,
slitmask_stack=slitmask_stack,
sciimg_stack=sciimg_stack, sciivar_stack=sciivar_stack,
skymodel_stack=skymodel_stack, mask_stack=mask_stack,
tilts_stack=tilts_stack, waveimg_stack=waveimg_stack,
head1d_list = head1d_list, head2d_list=head2d_list,
redux_path=redux_path, master_path=master_path, master_dir=master_dir,
master_key_dict=master_key_dict,
spectrograph = tslits_dict['spectrograph'])
return stack_dict
def save_2d_images(sci_output,
raw_header,
spectrograph,
master_key_dict,
mfdir,
outfile,
clobber=True,
update_det=None):
""" Write 2D images to the hard drive
Args:
sci_output (OrderedDict):
raw_header (astropy.fits.Header or dict):
master_key_dict (str):
mfdir (str):
outfile (str):
clobber: bool, optional
Returns:
"""
if os.path.isfile(outfile) and update_det is not None:
hdus, prihdu = init_hdus(update_det, outfile)
else:
# Primary HDU for output
prihdu = fits.PrimaryHDU()
# Update with original header, skipping a few keywords
hdus = [prihdu]
hdukeys = [
'BUNIT', 'COMMENT', '', 'BITPIX', 'NAXIS', 'NAXIS1', 'NAXIS2',
'HISTORY', 'EXTEND', 'DATASEC'
]
for key in raw_header.keys():
# Use new ones
if key in hdukeys:
continue
# Update unused ones
prihdu.header[key] = raw_header[key]
# History
if 'HISTORY' in raw_header.keys():
# Strip \n
tmp = str(raw_header['HISTORY']).replace('\n', ' ')
prihdu.header.add_history(str(tmp))
# PYPEIT
# TODO Should the spectrograph be written to the header?
prihdu.header['PIPELINE'] = str('PYPEIT')
prihdu.header['PYPELINE'] = spectrograph.pypeline
prihdu.header['SPECTROG'] = spectrograph.spectrograph
prihdu.header['DATE-RDX'] = str(
datetime.date.today().strftime('%Y-%b-%d'))
prihdu.header['FRAMMKEY'] = master_key_dict['frame'][:-3]
prihdu.header['BPMMKEY'] = master_key_dict['bpm'][:-3]
prihdu.header['BIASMKEY'] = master_key_dict['bias'][:-3]
prihdu.header['ARCMKEY'] = master_key_dict['arc'][:-3]
prihdu.header['TRACMKEY'] = master_key_dict['trace'][:-3]
prihdu.header['FLATMKEY'] = master_key_dict['flat'][:-3]
prihdu.header['PYPMFDIR'] = str(mfdir)
if sci_output['meta']['ir_redux']:
prihdu.header['SKYSUB'] = 'DIFF'
else:
prihdu.header['SKYSUB'] = 'MODEL'
# Fill in the images
ext = len(hdus) - 1
for key in sci_output.keys():
if key in ['meta']:
continue
else:
det = key
sdet = parse.get_dnum(det, caps=True) # e.g. DET02
if 'sciimg' not in sci_output[det]:
continue
# Specified detector number?
#if settings.argflag['reduce']['detnum'] is not None:
# if det not in map(int, settings.argflag['reduce']['detnum']):
# continue
# else:
# msgs.warn("Restricting the reduction to detector {:d}".format(det))
# Processed frame
ext += 1
keywd = 'EXT{:04d}'.format(ext)
prihdu.header[keywd] = '{:s}-Processed'.format(sdet)
hdu = fits.ImageHDU(sci_output[det]['sciimg']) #slf._sciframe[det-1])
hdu.name = prihdu.header[keywd]
hdus.append(hdu)
# Raw Inverse Variance
ext += 1
keywd = 'EXT{:04d}'.format(ext)
prihdu.header[keywd] = '{:s}-IVARRAW'.format(sdet)
hdu = fits.ImageHDU(
sci_output[det]['sciivar']) #slf._modelvarframe[det-1])
hdu.name = prihdu.header[keywd]
hdus.append(hdu)
# Background model
ext += 1
keywd = 'EXT{:04d}'.format(ext)
prihdu.header[keywd] = '{:s}-SKY'.format(sdet)
hdu = fits.ImageHDU(
sci_output[det]['skymodel']) #slf._modelvarframe[det-1])
hdu.name = prihdu.header[keywd]
hdus.append(hdu)
# Object model
ext += 1
keywd = 'EXT{:04d}'.format(ext)
prihdu.header[keywd] = '{:s}-OBJ'.format(sdet)
hdu = fits.ImageHDU(
sci_output[det]['objmodel']) #slf._modelvarframe[det-1])
hdu.name = prihdu.header[keywd]
hdus.append(hdu)
# Inverse Variance model
ext += 1
keywd = 'EXT{:04d}'.format(ext)
prihdu.header[keywd] = '{:s}-IVARMODEL'.format(sdet)
hdu = fits.ImageHDU(
sci_output[det]['ivarmodel']) # slf._modelvarframe[det-1])
hdu.name = prihdu.header[keywd]
hdus.append(hdu)
# Final mask
ext += 1
keywd = 'EXT{:04d}'.format(ext)
prihdu.header[keywd] = '{:s}-MASK'.format(sdet)
hdu = fits.ImageHDU(
sci_output[det]['outmask']) # slf._modelvarframe[det-1])
hdu.name = prihdu.header[keywd]
hdus.append(hdu)
# Finish
hdulist = fits.HDUList(hdus)
hdulist.writeto(outfile, overwrite=clobber)
msgs.info("Wrote: {:s}".format(outfile))