def test_all2dobj_update_image(init_dict):
# Build two
spec2DObj1 = spec2dobj.Spec2DObj(**init_dict)
spec2DObj2 = spec2dobj.Spec2DObj(**init_dict)
spec2DObj2.det = 2
#
allspec2D = spec2dobj.AllSpec2DObj()
allspec2D['meta']['ir_redux'] = False
allspec2D[1] = spec2DObj1
allspec2D[2] = spec2DObj2
# Write
ofile = data_path('tst_allspec2d.fits')
if os.path.isfile(ofile):
os.remove(ofile)
allspec2D.write_to_fits(ofile)
# Update
_allspec2D = spec2dobj.AllSpec2DObj()
spec2DObj1.sciimg = spec2DObj1.sciimg.copy() * 2.
_allspec2D['meta']['ir_redux'] = False
_allspec2D[1] = spec2DObj1
_allspec2D.write_to_fits(ofile, update_det=1, overwrite=True)
# Check
allspec2D_2 = spec2dobj.AllSpec2DObj.from_fits(ofile)
assert np.array_equal(allspec2D_2[1].sciimg, spec2DObj1.sciimg)
os.remove(ofile)
def test_all2dobj_update_image(init_dict):
allspec2D = spec2dobj.AllSpec2DObj()
allspec2D['meta']['bkg_redux'] = False
allspec2D['meta']['find_negative'] = False
for i in range(2):
d = load_spectrograph('keck_deimos').get_detector_par(i + 1)
allspec2D[d.name] = spec2dobj.Spec2DObj(detector=d, **init_dict)
# Write
ofile = tstutils.data_path('tst_allspec2d.fits')
if os.path.isfile(ofile):
os.remove(ofile)
allspec2D.write_to_fits(ofile)
_allspec2D = spec2dobj.AllSpec2DObj()
_allspec2D['meta']['bkg_redux'] = False
_allspec2D['meta']['find_negative'] = False
d = load_spectrograph('keck_deimos').get_detector_par(2)
detname = d.name
init_dict['sciimg'] = allspec2D[detname].sciimg.copy() * 2
_allspec2D[detname] = spec2dobj.Spec2DObj(detector=d, **init_dict)
_allspec2D.write_to_fits(ofile,
update_det=_allspec2D.detectors,
overwrite=True)
# Check
allspec2D_2 = spec2dobj.AllSpec2DObj.from_fits(ofile)
assert np.array_equal(allspec2D_2[detname].sciimg,
_allspec2D[detname].sciimg), 'Bad update'
assert np.array_equal(allspec2D_2[detname].sciimg,
allspec2D[detname].sciimg * 2), 'Bad update'
os.remove(ofile)
def test_all2dobj_write(init_dict):
# Build one
init_dict['detector'] = tstutils.get_kastb_detector()
spec2DObj = spec2dobj.Spec2DObj(**init_dict)
allspec2D = spec2dobj.AllSpec2DObj()
allspec2D['meta']['bkg_redux'] = False
allspec2D['meta']['find_negative'] = False
detname = spec2DObj.detname
allspec2D[detname] = spec2DObj
# Write
ofile = tstutils.data_path('tst_allspec2d.fits')
if os.path.isfile(ofile):
os.remove(ofile)
allspec2D.write_to_fits(ofile)
# Read
_allspec2D = spec2dobj.AllSpec2DObj.from_fits(ofile)
# Check
assert allspec2D.detectors == _allspec2D.detectors, 'Bad read: detector mismatch'
assert allspec2D['meta'] == _allspec2D['meta'], 'Bad read: meta mismatch'
# Try to update it
_allspec2D['meta']['bkg_redux'] = True
_allspec2D[detname].vel_corr = 2.
_allspec2D.write_to_fits(ofile, update_det='DET01')
__allspec2D = spec2dobj.AllSpec2DObj.from_fits(ofile)
assert __allspec2D['meta'] == _allspec2D['meta'], 'Bad read: meta mismatch'
assert __allspec2D['meta'] != allspec2D['meta'], 'Bad read: meta mismatch'
assert __allspec2D[detname].vel_corr == 2., 'Bad update'
os.remove(ofile)
def test_all2dobj_hdr(init_dict):
# Build one
spec2DObj = spec2dobj.Spec2DObj(**init_dict)
allspec2D = spec2dobj.AllSpec2DObj()
allspec2D['meta']['ir_redux'] = False
allspec2D[1] = spec2DObj
#
kast_file = data_path('b1.fits.gz')
header = fits.getheader(kast_file)
spectrograph = load_spectrograph('shane_kast_blue')
# Do it
hdr = allspec2D.build_primary_hdr(header, spectrograph, master_dir=data_path(''))
# Test it
assert hdr['SKYSUB'] == 'MODEL'
def test_all2dobj_hdr(init_dict):
# Build one
init_dict['detector'] = tstutils.get_kastb_detector()
spec2DObj = spec2dobj.Spec2DObj(**init_dict)
allspec2D = spec2dobj.AllSpec2DObj()
allspec2D['meta']['bkg_redux'] = False
allspec2D['meta']['find_negative'] = False
allspec2D[spec2DObj.detname] = spec2DObj
#
kast_file = tstutils.data_path('b1.fits.gz')
header = fits.getheader(kast_file)
spectrograph = load_spectrograph('shane_kast_blue')
# Do it
hdr = allspec2D.build_primary_hdr(header,
spectrograph,
master_dir=tstutils.data_path(''))
# Test it
assert hdr['SKYSUB'] == 'MODEL'
def test_all2dobj_write(init_dict):
# Build one
spec2DObj = spec2dobj.Spec2DObj(**init_dict)
allspec2D = spec2dobj.AllSpec2DObj()
allspec2D['meta']['ir_redux'] = False
allspec2D[1] = spec2DObj
allspec2D[1].detector = tstutils.get_kastb_detector()
# Write
ofile = data_path('tst_allspec2d.fits')
if os.path.isfile(ofile):
os.remove(ofile)
allspec2D.write_to_fits(ofile)
# Read
_allspec2D = spec2dobj.AllSpec2DObj.from_fits(ofile)
# Write again
os.remove(ofile)
_allspec2D.write_to_fits(ofile)
os.remove(ofile)
def main(args):
""" Executes 2d coadding
"""
msgs.info('PATH =' + os.getcwd())
# Load the file
if args.file is not None:
spectrograph_name, config_lines, spec2d_files \
= io.read_spec2d_file(args.file, filetype="coadd2d")
spectrograph = load_spectrograph(spectrograph_name)
# Parameters
# TODO: Shouldn't this reinstantiate the same parameters used in
# the PypeIt run that extracted the objects? Why are we not
# just passing the pypeit file?
# JFH: The reason is that the coadd2dfile may want different reduction parameters
# DP: I think config_specific_par() is more appropriate here. default_pypeit_par()
# is included in config_specific_par()
# NOTE `config_specific_par` works with the spec2d files because we construct the header
# of those files to include all the relevant keywords from the raw file.
spectrograph_cfg_lines = spectrograph.config_specific_par(
spec2d_files[0]).to_config()
parset = par.PypeItPar.from_cfg_lines(
cfg_lines=spectrograph_cfg_lines, merge_with=config_lines)
elif args.obj is not None:
# TODO: We should probably be reading the pypeit file and using
# those parameters here rather than using the default parset.
# TODO: This needs to define the science path
spec2d_files = glob.glob('./Science/spec2d_*' + args.obj + '*')
head0 = fits.getheader(spec2d_files[0])
spectrograph_name = head0['PYP_SPEC']
spectrograph = load_spectrograph(spectrograph_name)
# NOTE `config_specific_par` works with the spec2d files because we construct the header
# of those files to include all the relevant keywords from the raw file.
spectrograph_cfg_lines = spectrograph.config_specific_par(
spec2d_files[0]).to_config()
parset = par.PypeItPar.from_cfg_lines(
cfg_lines=spectrograph_cfg_lines)
else:
return msgs.error(
'You must provide either a coadd2d file (--file) or an object name (--obj)'
)
# If detector was passed as an argument override whatever was in the coadd2d_file
if args.det is not None:
msgs.info("Restricting reductions to detector={}".format(args.det))
# parset['rdx']['detnum'] = par.util.eval_tuple(args.det.split(','))
# TODO this needs to be adjusted if we want to pass (as inline command) mosaic detectors
parset['rdx']['detnum'] = [int(d) for d in args.det.split(',')]
# Get headers (if possible) and base names
spec1d_files = [
files.replace('spec2d', 'spec1d') for files in spec2d_files
]
head1d = None
for spec1d_file in spec1d_files:
if os.path.isfile(spec1d_file):
head1d = fits.getheader(spec1d_file)
break
if head1d is None:
msgs.warn(
"No 1D spectra so am generating a dummy header for output")
head1d = io.initialize_header()
head2d = fits.getheader(spec2d_files[0])
if args.basename is None:
#TODO Fix this, currently does not work if target names have - or _
filename_first = os.path.basename(spec2d_files[0])
filename_last = os.path.basename(spec2d_files[-1])
prefix_first = (filename_first.split('_')[1]).split('-')[0]
prefix_last = (filename_last.split('_')[1]).split('-')[0]
objname = (filename_first.split('-')[1]).split('_')[0]
basename = '{:s}-{:s}-{:s}'.format(prefix_first, prefix_last,
objname)
else:
basename = args.basename
# TODO Heliocentric for coadd2d needs to be thought through. Currently turning it off.
parset['calibrations']['wavelengths']['refframe'] = 'observed'
# TODO Flexure correction for coadd2d needs to be thought through. Currently turning it off.
parset['flexure']['spec_method'] = 'skip'
# Write the par to disk
par_outfile = basename + '_coadd2d.par'
print("Writing the parameters to {}".format(par_outfile))
parset.to_config(par_outfile,
exclude_defaults=True,
include_descr=False)
# Now run the coadds
skysub_mode = head2d['SKYSUB']
findobj_mode = head2d['FINDOBJ']
bkg_redux = True if 'DIFF' in skysub_mode else False
find_negative = True if 'NEG' in findobj_mode else False
# Print status message
msgs_string = 'Reducing target {:s}'.format(basename) + msgs.newline()
msgs_string += 'Coadding frame sky-subtraced with {:s}'.format(
skysub_mode)
msgs_string += 'Searching for objects that are {:s}'.format(
findobj_mode)
msgs_string += msgs.newline(
) + 'Combining frames in 2d coadd:' + msgs.newline()
for f, file in enumerate(spec2d_files):
msgs_string += 'Exp {0}: {1:s}'.format(
f, os.path.basename(file)) + msgs.newline()
msgs.info(msgs_string)
# TODO: This needs to be added to the parameter list for rdx
redux_path = os.getcwd()
master_dirname = os.path.basename(head2d['PYPMFDIR']) + '_coadd'
master_dir = os.path.join(redux_path, master_dirname)
# Make the new Master dir
if not os.path.isdir(master_dir):
msgs.info(
'Creating directory for Master output: {0}'.format(master_dir))
os.makedirs(master_dir)
# Instantiate the sci_dict
sci_dict = OrderedDict() # This needs to be ordered
sci_dict['meta'] = {}
sci_dict['meta']['vel_corr'] = 0.
sci_dict['meta']['bkg_redux'] = bkg_redux
sci_dict['meta']['find_negative'] = find_negative
# Make QA coadd directory
parset['rdx']['qadir'] += '_coadd'
qa_path = os.path.join(parset['rdx']['redux_path'],
parset['rdx']['qadir'], 'PNGs')
if not os.path.isdir(qa_path):
os.makedirs(qa_path)
# Find the detectors to reduce
# detectors = PypeIt.select_detectors(detnum=parset['rdx']['detnum'], ndet=spectrograph.ndet)
detectors = spectrograph.select_detectors(
subset=parset['rdx']['detnum'])
# if len(detectors) != spectrograph.ndet:
# msgs.warn('Not reducing detectors: {0}'.format(' '.join([str(d) for d in
# set(np.arange(spectrograph.ndet) + 1) - set(detectors)])))
msgs.info(f'Detectors to work on: {detectors}')
# Only_slits?
if args.only_slits:
parset['coadd2d']['only_slits'] = [
int(item) for item in args.only_slits.split(',')
]
# container for specobjs
all_specobjs = specobjs.SpecObjs()
# container for spec2dobj
all_spec2d = spec2dobj.AllSpec2DObj()
# set some meta
all_spec2d['meta']['bkg_redux'] = bkg_redux
all_spec2d['meta']['find_negative'] = find_negative
# Loop on detectors
for det in detectors:
msgs.info("Working on detector {0}".format(det))
# Instantiate Coadd2d
coadd = coadd2d.CoAdd2D.get_instance(
spec2d_files,
spectrograph,
parset,
det=det,
offsets=parset['coadd2d']['offsets'],
weights=parset['coadd2d']['weights'],
spec_samp_fact=args.spec_samp_fact,
spat_samp_fact=args.spat_samp_fact,
bkg_redux=bkg_redux,
find_negative=find_negative,
debug_offsets=args.debug_offsets,
debug=args.debug)
# TODO Add this stuff to a run method in coadd2d
# Coadd the slits
coadd_dict_list = coadd.coadd(
only_slits=parset['coadd2d']['only_slits'])
# Create the pseudo images
pseudo_dict = coadd.create_pseudo_image(coadd_dict_list)
# Reduce
msgs.info('Running the extraction')
# TODO -- This should mirror what is in pypeit.extract_one
# TODO -- JFH :: This ought to return a Spec2DObj and SpecObjs which
# would be slurped into AllSpec2DObj and all_specobsj, as below.
# TODO -- JFH -- Check that the slits we are using are correct
sci_dict[coadd.detname] = {}
sci_dict[coadd.detname]['sciimg'], sci_dict[coadd.detname]['sciivar'], \
sci_dict[coadd.detname]['skymodel'], sci_dict[coadd.detname]['objmodel'], \
sci_dict[coadd.detname]['ivarmodel'], sci_dict[coadd.detname]['outmask'], \
sci_dict[coadd.detname]['specobjs'], sci_dict[coadd.detname]['detector'], \
sci_dict[coadd.detname]['slits'], sci_dict[coadd.detname]['tilts'], \
sci_dict[coadd.detname]['waveimg'] \
= coadd.reduce(pseudo_dict, show=args.show, show_peaks=args.peaks, basename=basename)
# Tack on detector (similarly to pypeit.extract_one)
for sobj in sci_dict[coadd.detname]['specobjs']:
sobj.DETECTOR = sci_dict[coadd.detname]['detector']
# fill the specobjs container
all_specobjs.add_sobj(sci_dict[coadd.detname]['specobjs'])
# fill the spec2dobj container but first ...
# pull out maskdef_designtab from sci_dict[det]['slits']
maskdef_designtab = sci_dict[
coadd.detname]['slits'].maskdef_designtab
slits = copy.deepcopy(sci_dict[coadd.detname]['slits'])
slits.maskdef_designtab = None
# fill up
all_spec2d[coadd.detname] = spec2dobj.Spec2DObj(
sciimg=sci_dict[coadd.detname]['sciimg'],
ivarraw=sci_dict[coadd.detname]['sciivar'],
skymodel=sci_dict[coadd.detname]['skymodel'],
objmodel=sci_dict[coadd.detname]['objmodel'],
ivarmodel=sci_dict[coadd.detname]['ivarmodel'],
scaleimg=np.array([1.0], dtype=np.float),
bpmmask=sci_dict[coadd.detname]['outmask'],
detector=sci_dict[coadd.detname]['detector'],
slits=slits,
waveimg=sci_dict[coadd.detname]['waveimg'],
tilts=sci_dict[coadd.detname]['tilts'],
sci_spat_flexure=None,
sci_spec_flexure=None,
vel_corr=None,
vel_type=None,
maskdef_designtab=maskdef_designtab)
# Save pseudo image master files
#coadd.save_masters()
# SAVE TO DISK
# Make the new Science dir
# TODO: This needs to be defined by the user
scipath = os.path.join(redux_path, 'Science_coadd')
if not os.path.isdir(scipath):
msgs.info(
'Creating directory for Science output: {0}'.format(scipath))
os.makedirs(scipath)
# THE FOLLOWING MIMICS THE CODE IN pypeit.save_exposure()
subheader = spectrograph.subheader_for_spec(head2d, head2d)
# Write spec1D
if all_specobjs.nobj > 0:
outfile1d = os.path.join(scipath,
'spec1d_{:s}.fits'.format(basename))
all_specobjs.write_to_fits(subheader, outfile1d)
# Info
outfiletxt = os.path.join(scipath,
'spec1d_{:s}.txt'.format(basename))
sobjs = specobjs.SpecObjs.from_fitsfile(outfile1d,
chk_version=False)
sobjs.write_info(outfiletxt, spectrograph.pypeline)
# Build header for spec2d
outfile2d = os.path.join(scipath, 'spec2d_{:s}.fits'.format(basename))
pri_hdr = all_spec2d.build_primary_hdr(
head2d,
spectrograph,
subheader=subheader,
# TODO -- JFH :: Decide if we need any of these
redux_path=None,
master_key_dict=None,
master_dir=None)
# Write spec2d
all_spec2d.write_to_fits(outfile2d, pri_hdr=pri_hdr)
def main(args):
""" Executes 2d coadding
"""
msgs.warn('PATH =' + os.getcwd())
# Load the file
if args.file is not None:
spectrograph_name, config_lines, spec2d_files = io.read_spec2d_file(
args.file, filetype="coadd2d")
spectrograph = load_spectrograph(spectrograph_name)
# Parameters
# TODO: Shouldn't this reinstantiate the same parameters used in
# the PypeIt run that extracted the objects? Why are we not
# just passing the pypeit file?
# JFH: The reason is that the coadd2dfile may want different reduction parameters
spectrograph_def_par = spectrograph.default_pypeit_par()
parset = par.PypeItPar.from_cfg_lines(
cfg_lines=spectrograph_def_par.to_config(),
merge_with=config_lines)
elif args.obj is not None:
# TODO: We should probably be reading the pypeit file and using those parameters here rather than using the
# default parset.
# TODO: This needs to define the science path
spec2d_files = glob.glob('./Science/spec2d_*' + args.obj + '*')
head0 = fits.getheader(spec2d_files[0])
spectrograph_name = head0['PYP_SPEC']
spectrograph = load_spectrograph(spectrograph_name)
parset = spectrograph.default_pypeit_par()
else:
msgs.error(
'You must either input a coadd2d file with --file or an object name with --obj'
)
# Update with configuration specific parameters (which requires science file) and initialize spectrograph
spectrograph_cfg_lines = spectrograph.config_specific_par(
spec2d_files[0]).to_config()
parset = par.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
merge_with=parset.to_config())
# If detector was passed as an argument override whatever was in the coadd2d_file
if args.det is not None:
msgs.info("Restricting reductions to detector={}".format(args.det))
parset['rdx']['detnum'] = int(args.det)
# Get headers (if possible) and base names
spec1d_files = [
files.replace('spec2d', 'spec1d') for files in spec2d_files
]
head1d = None
for spec1d_file in spec1d_files:
if os.path.isfile(spec1d_file):
head1d = fits.getheader(spec1d_file)
break
if head1d is None:
msgs.warn("No 1D spectra so am generating a dummy header for output")
head1d = io.initialize_header()
head2d = fits.getheader(spec2d_files[0])
if args.basename is None:
filename = os.path.basename(spec2d_files[0])
basename = filename.split('_')[2]
else:
basename = args.basename
# Write the par to disk
par_outfile = basename + '_coadd2d.par'
print("Writing the parameters to {}".format(par_outfile))
parset.to_config(par_outfile)
# Now run the coadds
skysub_mode = head2d['SKYSUB']
ir_redux = True if 'DIFF' in skysub_mode else False
# Print status message
msgs_string = 'Reducing target {:s}'.format(basename) + msgs.newline()
msgs_string += 'Performing coadd of frames reduce with {:s} imaging'.format(
skysub_mode)
msgs_string += msgs.newline(
) + 'Combining frames in 2d coadd:' + msgs.newline()
for file in spec2d_files:
msgs_string += '{0:s}'.format(os.path.basename(file)) + msgs.newline()
msgs.info(msgs_string)
# TODO: This needs to be added to the parameter list for rdx
redux_path = os.getcwd()
master_dirname = os.path.basename(head2d['PYPMFDIR']) + '_coadd'
master_dir = os.path.join(redux_path, master_dirname)
# Make the new Master dir
if not os.path.isdir(master_dir):
msgs.info(
'Creating directory for Master output: {0}'.format(master_dir))
os.makedirs(master_dir)
# Instantiate the sci_dict
sci_dict = OrderedDict() # This needs to be ordered
sci_dict['meta'] = {}
sci_dict['meta']['vel_corr'] = 0.
sci_dict['meta']['ir_redux'] = ir_redux
# Find the detectors to reduce
detectors = PypeIt.select_detectors(detnum=parset['rdx']['detnum'],
ndet=spectrograph.ndet)
if len(detectors) != spectrograph.ndet:
msgs.warn('Not reducing detectors: {0}'.format(' '.join([
str(d)
for d in set(np.arange(spectrograph.ndet) + 1) - set(detectors)
])))
# Loop on detectors
for det in detectors:
msgs.info("Working on detector {0}".format(det))
sci_dict[det] = {}
# Instantiate Coadd2d
coadd = coadd2d.CoAdd2D.get_instance(
spec2d_files,
spectrograph,
parset,
det=det,
offsets=parset['coadd2d']['offsets'],
weights=parset['coadd2d']['weights'],
ir_redux=ir_redux,
debug_offsets=args.debug_offsets,
debug=args.debug,
samp_fact=args.samp_fact)
# Coadd the slits
coadd_dict_list = coadd.coadd(
only_slits=None) # TODO implement only_slits later
# Create the pseudo images
pseudo_dict = coadd.create_pseudo_image(coadd_dict_list)
# Reduce
msgs.info('Running the extraction')
# TODO -- This should mirror what is in pypeit.extract_one
# TODO -- JFH :: This ought to return a Spec2DObj and SpecObjs which would be slurped into
# AllSpec2DObj and all_specobsj, as below.
# TODO -- JFH -- Check that the slits we are using are correct
sci_dict[det]['sciimg'], sci_dict[det]['sciivar'], sci_dict[det]['skymodel'], sci_dict[det]['objmodel'], \
sci_dict[det]['ivarmodel'], sci_dict[det]['outmask'], sci_dict[det]['specobjs'], sci_dict[det]['detector'], \
sci_dict[det]['slits'], sci_dict[det]['tilts'], sci_dict[det]['waveimg'] = coadd.reduce(
pseudo_dict, show = args.show, show_peaks = args.peaks)
# Save pseudo image master files
#coadd.save_masters()
# Make the new Science dir
# TODO: This needs to be defined by the user
scipath = os.path.join(redux_path, 'Science_coadd')
if not os.path.isdir(scipath):
msgs.info('Creating directory for Science output: {0}'.format(scipath))
os.makedirs(scipath)
# THE FOLLOWING MIMICS THE CODE IN pypeit.save_exposure()
# TODO -- These lines should be above once reduce() passes back something sensible
all_specobjs = specobjs.SpecObjs()
for det in detectors:
all_specobjs.add_sobj(sci_dict[det]['specobjs'])
# Write
outfile1d = os.path.join(scipath, 'spec1d_{:s}.fits'.format(basename))
subheader = spectrograph.subheader_for_spec(head2d, head2d)
all_specobjs.write_to_fits(subheader, outfile1d)
# 2D spectra
# TODO -- These lines should be above once reduce() passes back something sensible
all_spec2d = spec2dobj.AllSpec2DObj()
all_spec2d['meta']['ir_redux'] = ir_redux
for det in detectors:
all_spec2d[det] = spec2dobj.Spec2DObj(
det=det,
sciimg=sci_dict[det]['sciimg'],
ivarraw=sci_dict[det]['sciivar'],
skymodel=sci_dict[det]['skymodel'],
objmodel=sci_dict[det]['objmodel'],
ivarmodel=sci_dict[det]['ivarmodel'],
scaleimg=np.array([1.0], dtype=np.float),
bpmmask=sci_dict[det]['outmask'],
detector=sci_dict[det]['detector'],
slits=sci_dict[det]['slits'],
waveimg=sci_dict[det]['waveimg'],
tilts=sci_dict[det]['tilts'],
sci_spat_flexure=None,
sci_spec_flexure=None,
vel_corr=None,
vel_type=None)
# Build header
outfile2d = os.path.join(scipath, 'spec2d_{:s}.fits'.format(basename))
pri_hdr = all_spec2d.build_primary_hdr(
head2d,
spectrograph,
subheader=subheader,
# TODO -- JFH :: Decide if we need any of these
redux_path=None,
master_key_dict=None,
master_dir=None)
# Write
all_spec2d.write_to_fits(outfile2d, pri_hdr=pri_hdr)
def main(args):
# Build the fitstable since we currently need it for output. This should not be the case!
A_files = [os.path.join(args.full_rawpath, file) for file in args.Afiles]
B_files = [os.path.join(args.full_rawpath, file) for file in args.Bfiles]
data_files = A_files + B_files
ps = pypeitsetup.PypeItSetup(A_files,
path='./',
spectrograph_name='keck_mosfire')
ps.build_fitstbl()
fitstbl = ps.fitstbl
# Read in the spectrograph, config the parset
spectrograph = load_spectrograph('keck_mosfire')
spectrograph_def_par = spectrograph.default_pypeit_par()
parset = par.PypeItPar.from_cfg_lines(
cfg_lines=spectrograph_def_par.to_config(),
merge_with=config_lines(args))
science_path = os.path.join(parset['rdx']['redux_path'],
parset['rdx']['scidir'])
# Calibration Master directory
if args.master_dir is None:
msgs.error(
"You need to set an Environmental variable MOSFIRE_MASTERS that points at the Master Calibs"
)
# Define some hard wired master files here to be later parsed out of the directory
slit_masterframe_name = os.path.join(args.master_dir,
'MasterSlits_E_15_01.fits.gz')
tilts_masterframe_name = os.path.join(args.master_dir,
'MasterTilts_E_1_01.fits')
wvcalib_masterframe_name = os.path.join(args.master_dir,
'MasterWaveCalib_E_1_01.fits')
# For now don't require a standard
std_outfile = None
#std_outfile = os.path.join('/Users/joe/Dropbox/PypeIt_Redux/MOSFIRE/Nov19/quicklook/Science/',
# 'spec1d_m191118_0064-GD71_MOSFIRE_2019Nov18T104704.507.fits')
# make the get_std from pypeit a utility function or class method
det = 1 # MOSFIRE has a single detector
if std_outfile is not None:
# Get the standard trace if need be
sobjs = specobjs.SpecObjs.from_fitsfile(std_outfile)
this_det = sobjs.DET == det
if np.any(this_det):
sobjs_det = sobjs[this_det]
sobjs_std = sobjs_det.get_std()
std_trace = None if sobjs_std is None else sobjs_std.TRACE_SPAT.flatten(
)
else:
std_trace = None
else:
std_trace = None
# Read in the msbpm
sdet = get_dnum(det, prefix=False)
msbpm = spectrograph.bpm(A_files[0], det)
# Read in the slits
slits = slittrace.SlitTraceSet.from_file(slit_masterframe_name)
# Reset the bitmask
slits.mask = slits.mask_init.copy()
# Read in the wv_calib
wv_calib = wavecalib.WaveCalib.from_file(wvcalib_masterframe_name)
wv_calib.is_synced(slits)
slits.mask_wvcalib(wv_calib)
# Read in the tilts
tilts_obj = wavetilts.WaveTilts.from_file(tilts_masterframe_name)
tilts_obj.is_synced(slits)
slits.mask_wavetilts(tilts_obj)
# Build Science image
sciImg = buildimage.buildimage_fromlist(spectrograph,
det,
parset['scienceframe'],
A_files,
bpm=msbpm,
slits=slits,
ignore_saturation=False)
# Background Image?
sciImg = sciImg.sub(
buildimage.buildimage_fromlist(spectrograph,
det,
parset['scienceframe'],
B_files,
bpm=msbpm,
slits=slits,
ignore_saturation=False),
parset['scienceframe']['process'])
# Build the Calibrate object
caliBrate = calibrations.Calibrations(None, parset['calibrations'],
spectrograph, None)
caliBrate.slits = slits
caliBrate.wavetilts = tilts_obj
caliBrate.wv_calib = wv_calib
# Instantiate Reduce object
# Required for pypeline specific object
# At instantiaton, the fullmask in self.sciImg is modified
redux = reduce.Reduce.get_instance(sciImg,
spectrograph,
parset,
caliBrate,
'science',
ir_redux=True,
show=args.show,
det=det,
std_outfile=std_outfile)
manual_extract_dict = None
skymodel, objmodel, ivarmodel, outmask, sobjs, waveImg, tilts = redux.run(
std_trace=std_trace,
return_negative=True,
manual_extract_dict=manual_extract_dict,
show_peaks=args.show)
# TODO -- Do this upstream
# Tack on detector
for sobj in sobjs:
sobj.DETECTOR = sciImg.detector
# Construct the Spec2DObj with the positive image
spec2DObj_A = spec2dobj.Spec2DObj(det=det,
sciimg=sciImg.image,
ivarraw=sciImg.ivar,
skymodel=skymodel,
objmodel=objmodel,
ivarmodel=ivarmodel,
waveimg=waveImg,
bpmmask=outmask,
detector=sciImg.detector,
sci_spat_flexure=sciImg.spat_flexure,
tilts=tilts,
slits=copy.deepcopy(caliBrate.slits))
spec2DObj_A.process_steps = sciImg.process_steps
all_spec2d = spec2dobj.AllSpec2DObj()
all_spec2d['meta']['ir_redux'] = True
all_spec2d[det] = spec2DObj_A
# Save image A but with all the objects extracted, i.e. positive and negative
#outfile2d, outfile1d = save_exposure(fitstbl, 0, spectrograph, science_path, parset, caliBrate, all_spec2d, sobjs)
# Construct the Spec2DObj with the negative image
spec2DObj_B = spec2dobj.Spec2DObj(det=det,
sciimg=-sciImg.image,
ivarraw=sciImg.ivar,
skymodel=-skymodel,
objmodel=-objmodel,
ivarmodel=ivarmodel,
waveimg=waveImg,
bpmmask=outmask,
detector=sciImg.detector,
sci_spat_flexure=sciImg.spat_flexure,
tilts=tilts,
slits=copy.deepcopy(caliBrate.slits))
# Parse the offset information out of the headers. TODO in the future get this out of fitstable
dither_pattern_A, dither_id_A, offset_arcsec_A = parse_dither_pattern(
A_files, spectrograph.primary_hdrext)
dither_pattern_B, dither_id_B, offset_arcsec_B = parse_dither_pattern(
B_files, spectrograph.primary_hdrext)
# Print out a report on the offsets
msg_string = msgs.newline(
) + '****************************************************'
msg_string += msgs.newline(
) + ' Summary of offsets for dither pattern: {:s}'.format(
dither_pattern_A[0])
msg_string += msgs.newline(
) + '****************************************************'
msg_string += msgs.newline(
) + 'Position filename arcsec pixels '
msg_string += msgs.newline(
) + '----------------------------------------------------'
for iexp, file in enumerate(A_files):
msg_string += msgs.newline(
) + ' A {:s} {:6.2f} {:6.2f}'.format(
os.path.basename(file), offset_arcsec_A[iexp],
offset_arcsec_A[iexp] / sciImg.detector.platescale)
for iexp, file in enumerate(B_files):
msg_string += msgs.newline(
) + ' B {:s} {:6.2f} {:6.2f}'.format(
os.path.basename(file), offset_arcsec_B[iexp],
offset_arcsec_B[iexp] / sciImg.detector.platescale)
msg_string += msgs.newline(
) + '****************************************************'
msgs.info(msg_string)
#offset_dith_pix = offset_dith_pix = offset_arcsec_A[0]/sciImg.detector.platescale
offsets_dith_pix = (np.array([
0.0, np.mean(offset_arcsec_B) - np.mean(offset_arcsec_A)
])) / sciImg.detector.platescale
if args.offset is not None:
offsets_pixels = np.array([0.0, args.offset])
msgs.info('Using user specified offsets instead: {:5.2f}'.format(
args.offset))
else:
offsets_pixels = offsets_dith_pix
spec2d_list = [spec2DObj_A, spec2DObj_B]
# Instantiate Coadd2d
coadd = coadd2d.CoAdd2D.get_instance(spec2d_list,
spectrograph,
parset,
det=det,
offsets=offsets_pixels,
weights='uniform',
ir_redux=True,
debug=args.show,
samp_fact=args.samp_fact)
# Coadd the slits
coadd_dict_list = coadd.coadd(
only_slits=None, interp_dspat=False) # TODO implement only_slits later
# Create the pseudo images
pseudo_dict = coadd.create_pseudo_image(coadd_dict_list)
##########################
# Now display the images #
##########################
display.display.connect_to_ginga(raise_err=True, allow_new=True)
# Bug in ginga prevents me from using cuts here for some reason
#mean, med, sigma = sigma_clipped_stats(pseudo_dict['imgminsky'][pseudo_dict['inmask']], sigma_lower=5.0,sigma_upper=5.0)
#cut_min = mean - 4.0 * sigma
#cut_max = mean + 4.0 * sigma
chname_skysub = 'skysub-det{:s}'.format(sdet)
# Clear all channels at the beginning
# TODO: JFH For some reason Ginga crashes when I try to put cuts in here.
viewer, ch = ginga.show_image(pseudo_dict['imgminsky'],
chname=chname_skysub,
waveimg=pseudo_dict['waveimg'],
clear=True) # cuts=(cut_min, cut_max),
slit_left, slit_righ, _ = pseudo_dict['slits'].select_edges()
slit_id = slits.slitord_id[0]
ginga.show_slits(viewer, ch, slit_left, slit_righ, slit_ids=slit_id)
# SKRESIDS
chname_skyresids = 'sky_resid-det{:s}'.format(sdet)
image = pseudo_dict['imgminsky'] * np.sqrt(
pseudo_dict['sciivar']) * pseudo_dict['inmask'] # sky residual map
viewer, ch = ginga.show_image(
image,
chname_skyresids,
waveimg=pseudo_dict['waveimg'],
cuts=(-5.0, 5.0),
)
ginga.show_slits(viewer,
ch,
slit_left,
slit_righ,
slit_ids=slits.slitord_id[0])
shell = viewer.shell()
out = shell.start_global_plugin('WCSMatch')
out = shell.call_global_plugin_method('WCSMatch', 'set_reference_channel',
[chname_skyresids], {})
if args.embed:
embed()
return 0
def reduce_exposure(self, frames, bg_frames=None, std_outfile=None):
"""
Reduce a single exposure
Args:
frames (:obj:`list`):
List of 0-indexed rows in :attr:`fitstbl` with the frames to
reduce.
bg_frames (:obj:`list`, optional):
List of frame indices for the background.
std_outfile (:obj:`str`, optional):
File with a previously reduced standard spectrum from
PypeIt.
Returns:
dict: The dictionary containing the primary outputs of
extraction.
"""
# if show is set, clear the ginga channels at the start of each new sci_ID
if self.show:
# TODO: Put this in a try/except block?
display.clear_all(allow_new=True)
has_bg = True if bg_frames is not None and len(
bg_frames) > 0 else False
# Is this an b/g subtraction reduction?
if has_bg:
self.bkg_redux = True
# The default is to find_negative objects if the bg_frames are classified as "science", and to not find_negative
# objects if the bg_frames are classified as "sky". This can be explicitly overridden if
# par['reduce']['findobj']['find_negative'] is set to something other than the default of None.
self.find_negative = (('science' in self.fitstbl['frametype'][bg_frames[0]]) |
('standard' in self.fitstbl['frametype'][bg_frames[0]]))\
if self.par['reduce']['findobj']['find_negative'] is None else self.par['reduce']['findobj']['find_negative']
else:
self.bkg_redux = False
self.find_negative = False
# Container for all the Spec2DObj
all_spec2d = spec2dobj.AllSpec2DObj()
all_spec2d['meta']['bkg_redux'] = self.bkg_redux
all_spec2d['meta']['find_negative'] = self.find_negative
# TODO -- Should we reset/regenerate self.slits.mask for a new exposure
# container for specobjs during first loop (objfind)
all_specobjs_objfind = specobjs.SpecObjs()
# container for specobjs during second loop (extraction)
all_specobjs_extract = specobjs.SpecObjs()
# list of global_sky obtained during objfind and used in extraction
initial_sky_list = []
# list of sciImg
sciImg_list = []
# List of detectors with successful calibration
calibrated_det = []
# list of successful MasterSlits calibrations to be used in the extraction loop
calib_slits = []
# List of objFind objects
objFind_list = []
# Print status message
msgs_string = 'Reducing target {:s}'.format(
self.fitstbl['target'][frames[0]]) + msgs.newline()
# TODO: Print these when the frames are actually combined,
# backgrounds are used, etc?
msgs_string += 'Combining frames:' + msgs.newline()
for iframe in frames:
msgs_string += '{0:s}'.format(
self.fitstbl['filename'][iframe]) + msgs.newline()
msgs.info(msgs_string)
if has_bg:
bg_msgs_string = ''
for iframe in bg_frames:
bg_msgs_string += '{0:s}'.format(
self.fitstbl['filename'][iframe]) + msgs.newline()
bg_msgs_string = msgs.newline(
) + 'Using background from frames:' + msgs.newline(
) + bg_msgs_string
msgs.info(bg_msgs_string)
# Find the detectors to reduce
subset = self.par['rdx']['slitspatnum'] if self.par['rdx']['slitspatnum'] is not None \
else self.par['rdx']['detnum']
detectors = self.spectrograph.select_detectors(subset=subset)
msgs.info(f'Detectors to work on: {detectors}')
# Loop on Detectors
# TODO: Attempt to put in a multiprocessing call here?
# objfind
for self.det in detectors:
msgs.info("Working on detector {0}".format(self.det))
# run calibration
self.caliBrate = self.calib_one(frames, self.det)
if not self.caliBrate.success:
msgs.warn(
f'Calibrations for detector {self.det} were unsuccessful! The step '
f'that failed was {self.caliBrate.failed_step}. Continuing by '
f'skipping this detector.')
continue
# we save only the detectors that had a successful calibration,
# and we use only those in the extract loop below
calibrated_det.append(self.det)
# we also save the successful MasterSlits calibrations because they are used and modified
# in the slitmask stuff in between the two loops
calib_slits.append(self.caliBrate.slits)
# global_sky, skymask and sciImg are needed in the extract loop
initial_sky, sobjs_obj, sciImg, objFind = self.objfind_one(
frames, self.det, bg_frames, std_outfile=std_outfile)
if len(sobjs_obj) > 0:
all_specobjs_objfind.add_sobj(sobjs_obj)
initial_sky_list.append(initial_sky)
sciImg_list.append(sciImg)
objFind_list.append(objFind)
# slitmask stuff
if self.par['reduce']['slitmask']['assign_obj']:
# get object positions from slitmask design and slitmask offsets for all the detectors
spat_flexure = np.array([ss.spat_flexure for ss in sciImg_list])
# Grab platescale with binning
bin_spec, bin_spat = parse.parse_binning(self.binning)
platescale = np.array(
[ss.detector.platescale * bin_spat for ss in sciImg_list])
# get the dither offset if available
if self.par['reduce']['slitmask']['use_dither_offset']:
dither = self.spectrograph.parse_dither_pattern(
[self.fitstbl.frame_paths(frames[0])])
dither_off = dither[2][0] if dither is not None else None
else:
dither_off = None
calib_slits = slittrace.get_maskdef_objpos_offset_alldets(
all_specobjs_objfind,
calib_slits,
spat_flexure,
platescale,
self.par['calibrations']['slitedges']['det_buffer'],
self.par['reduce']['slitmask'],
dither_off=dither_off)
# determine if slitmask offsets exist and compute an average offsets over all the detectors
calib_slits = slittrace.average_maskdef_offset(
calib_slits, platescale[0],
self.spectrograph.list_detectors(
mosaic='MSC' in calib_slits[0].detname))
# slitmask design matching and add undetected objects
all_specobjs_objfind = slittrace.assign_addobjs_alldets(
all_specobjs_objfind, calib_slits, spat_flexure, platescale,
self.par['reduce']['slitmask'],
self.par['reduce']['findobj']['find_fwhm'])
# Extract
for i, self.det in enumerate(calibrated_det):
# re-run (i.e., load) calibrations
self.caliBrate = self.calib_one(frames, self.det)
self.caliBrate.slits = calib_slits[i]
detname = sciImg_list[i].detector.name
# TODO: pass back the background frame, pass in background
# files as an argument. extract one takes a file list as an
# argument and instantiates science within
if all_specobjs_objfind.nobj > 0:
all_specobjs_on_det = all_specobjs_objfind[
all_specobjs_objfind.DET == detname]
else:
all_specobjs_on_det = all_specobjs_objfind
# Extract
all_spec2d[detname], tmp_sobjs \
= self.extract_one(frames, self.det, sciImg_list[i], objFind_list[i],
initial_sky_list[i], all_specobjs_on_det)
# Hold em
if tmp_sobjs.nobj > 0:
all_specobjs_extract.add_sobj(tmp_sobjs)
# JFH TODO write out the background frame?
# TODO -- Save here? Seems like we should. Would probably need to use update_det=True
# Return
return all_spec2d, all_specobjs_extract
def reduce_IR(A_files,
B_files,
caliBrate,
spectrograph,
det,
parset,
show=False,
std_trace=None):
"""
Peform 2d extraction for a set of files at the same unique A-B offset location.
Parameters
----------
A_files (list of strings):
Files at A position for this offset
B_files (list of strings)
Files at B position for this offeset
caliBrate (object):
CaliBrate object
spectrograph (object):
spectrograph object
det (int):
Detector number
parset (parsect object)
Parset
show (bool, optional):
Show 2d reduction outputs. Default=False
std_trace (string, optional)
Trace for standard star. Default=None
Returns
-------
spec2DObj_A, spec2DObj_B
spec2DObj_A (object, Spec2D):
Spec2d Object for extraction at A position
spec2DObj_B (object, Spec2D)
Spec2d Object for extraction at B position
"""
# Build Science image
sciImg = buildimage.buildimage_fromlist(spectrograph,
det,
parset['scienceframe'],
list(A_files),
bpm=caliBrate.msbpm,
slits=caliBrate.slits,
ignore_saturation=False)
# Background Image?
sciImg = sciImg.sub(
buildimage.buildimage_fromlist(spectrograph,
det,
parset['scienceframe'],
list(B_files),
bpm=caliBrate.msbpm,
slits=caliBrate.slits,
ignore_saturation=False),
parset['scienceframe']['process'])
# Instantiate FindObjects object
# Required for pypeline specific object
# At instantiaton, the fullmask in self.sciImg is modified
# DP: Should find_negative be True here? JFH: For quicklook yes!
objFind = find_objects.FindObjects.get_instance(sciImg,
spectrograph,
parset,
caliBrate,
'science',
bkg_redux=True,
find_negative=True,
show=show)
global_sky, sobjs_obj = objFind.run(std_trace=std_trace, show_peaks=show)
# Instantiate Extract object
extract = extraction.Extract.get_instance(sciImg,
sobjs_obj,
spectrograph,
parset,
caliBrate,
'science',
bkg_redux=True,
return_negative=True,
show=show)
skymodel, objmodel, ivarmodel, \
outmask, sobjs, waveimg, tilts = extract.run(global_sky, sobjs_obj)
scaleimg = np.array([1.0],
dtype=np.float) # np.array([1]) applies no scale
# TODO -- Do this upstream
# Tack on detector
for sobj in sobjs:
sobj.DETECTOR = sciImg.detector
# Construct table of spectral flexure
spec_flex_table = Table()
spec_flex_table['spat_id'] = caliBrate.slits.spat_id
spec_flex_table['sci_spec_flexure'] = extract.slitshift
# Construct the Spec2DObj with the positive image
spec2DObj_A = spec2dobj.Spec2DObj(
sciimg=sciImg.image,
ivarraw=sciImg.ivar,
skymodel=skymodel,
objmodel=objmodel,
ivarmodel=ivarmodel,
scaleimg=scaleimg,
waveimg=waveimg,
bpmmask=outmask,
detector=sciImg.detector,
sci_spat_flexure=sciImg.spat_flexure,
sci_spec_flexure=spec_flex_table,
vel_corr=None,
vel_type=parset['calibrations']['wavelengths']['refframe'],
tilts=tilts,
slits=copy.deepcopy(caliBrate.slits),
maskdef_designtab=None)
spec2DObj_A.process_steps = sciImg.process_steps
all_spec2d = spec2dobj.AllSpec2DObj()
all_spec2d['meta']['bkg_redux'] = True
all_spec2d[spec2DObj_A.detname] = spec2DObj_A
# Construct the Spec2DObj with the negative image
spec2DObj_B = spec2dobj.Spec2DObj(
sciimg=-sciImg.image,
ivarraw=sciImg.ivar,
skymodel=-skymodel,
objmodel=-objmodel,
ivarmodel=ivarmodel,
scaleimg=scaleimg,
waveimg=waveimg,
bpmmask=outmask,
detector=sciImg.detector,
sci_spat_flexure=sciImg.spat_flexure,
sci_spec_flexure=spec_flex_table,
vel_corr=None,
vel_type=parset['calibrations']['wavelengths']['refframe'],
tilts=tilts,
slits=copy.deepcopy(caliBrate.slits),
maskdef_designtab=None)
return spec2DObj_A, spec2DObj_B
def reduce_exposure(self, frames, bg_frames=None, std_outfile=None):
"""
Reduce a single exposure
Args:
frame (:obj:`int`):
0-indexed row in :attr:`fitstbl` with the frame to
reduce.
bg_frames (:obj:`list`, optional):
List of frame indices for the background.
std_outfile (:obj:`str`, optional):
File with a previously reduced standard spectrum from
PypeIt.
Returns:
dict: The dictionary containing the primary outputs of
extraction.
"""
# TODO:
# - change doc string to reflect that more than one frame can be
# provided
# if show is set, clear the ginga channels at the start of each new sci_ID
if self.show:
# TODO: Put this in a try/except block?
display.clear_all()
has_bg = True if bg_frames is not None and len(
bg_frames) > 0 else False
# Is this an IR reduction?
# TODO: Why specific to IR?
self.ir_redux = True if has_bg else False
# Container for all the Spec2DObj
all_spec2d = spec2dobj.AllSpec2DObj()
all_spec2d['meta']['ir_redux'] = self.ir_redux
# TODO -- Should we reset/regenerate self.slits.mask for a new exposure
all_specobjs = specobjs.SpecObjs()
# Print status message
msgs_string = 'Reducing target {:s}'.format(
self.fitstbl['target'][frames[0]]) + msgs.newline()
# TODO: Print these when the frames are actually combined,
# backgrounds are used, etc?
msgs_string += 'Combining frames:' + msgs.newline()
for iframe in frames:
msgs_string += '{0:s}'.format(
self.fitstbl['filename'][iframe]) + msgs.newline()
msgs.info(msgs_string)
if has_bg:
bg_msgs_string = ''
for iframe in bg_frames:
bg_msgs_string += '{0:s}'.format(
self.fitstbl['filename'][iframe]) + msgs.newline()
bg_msgs_string = msgs.newline(
) + 'Using background from frames:' + msgs.newline(
) + bg_msgs_string
msgs.info(bg_msgs_string)
# Find the detectors to reduce
detectors = PypeIt.select_detectors(
detnum=self.par['rdx']['detnum'],
slitspatnum=self.par['rdx']['slitspatnum'],
ndet=self.spectrograph.ndet)
if len(detectors) != self.spectrograph.ndet:
msgs.warn('Not reducing detectors: {0}'.format(' '.join([
str(d) for d in set(np.arange(self.spectrograph.ndet)) -
set(detectors)
])))
# Loop on Detectors
# TODO: Attempt to put in a multiprocessing call here?
for self.det in detectors:
msgs.info("Working on detector {0}".format(self.det))
# Instantiate Calibrations class
self.caliBrate = calibrations.Calibrations.get_instance(
self.fitstbl,
self.par['calibrations'],
self.spectrograph,
self.calibrations_path,
qadir=self.qa_path,
reuse_masters=self.reuse_masters,
show=self.show,
slitspat_num=self.par['rdx']['slitspatnum'])
# These need to be separate to accomodate COADD2D
self.caliBrate.set_config(frames[0], self.det,
self.par['calibrations'])
self.caliBrate.run_the_steps()
# Extract
# TODO: pass back the background frame, pass in background
# files as an argument. extract one takes a file list as an
# argument and instantiates science within
all_spec2d[self.det], tmp_sobjs \
= self.reduce_one(frames, self.det, bg_frames, std_outfile=std_outfile)
# Hold em
if tmp_sobjs.nobj > 0:
all_specobjs.add_sobj(tmp_sobjs)
# JFH TODO write out the background frame?
# TODO -- Save here? Seems like we should. Would probably need to use update_det=True
# Return
return all_spec2d, all_specobjs
