def MasterPinhole(self, fitsdict, det, msbias):
"""
Generate Master pinhole frame for a given detector
Parameters
----------
fitsdict : dict
Contains relevant information from fits header files
det : int
Index of the detector
Returns
-------
boolean : bool
Should other ScienceExposure classes be updated?
"""
dnum = settings.get_dnum(det)
# If the master pinhole is already made, use it
if self._mspinhole[det - 1] is not None:
msgs.info("An identical master pinhole frame already exists")
return False
if settings.argflag['reduce']['slitcen']['useframe'] in ['trace', 'pinhole']:
try:
mspinhole = armasters.get_master_frame(self, "pinhole")
except IOError:
msgs.info("Preparing a master pinhole frame with {0:s}".format(
settings.argflag['reduce']['slitcen']['useframe']))
ind = self._idx_cent
# Load the pinhole frames
frames = arload.load_frames(fitsdict, ind, det, frametype='pinhole', msbias=msbias, # self._msbias[det - 1],
trim=settings.argflag['reduce']['trim'])
if settings.argflag['pinhole']['combine']['match'] > 0.0:
sframes = arsort.match_frames(frames, settings.argflag['pinhole']['combine']['match'],
frametype='pinhole', satlevel=settings.spect[dnum]['saturation'] *
settings.spect['det'][det - 1]['nonlinear'])
subframes = np.zeros((frames.shape[0], frames.shape[1], len(sframes)))
numarr = np.array([])
for i in range(len(sframes)):
numarr = np.append(numarr, sframes[i].shape[2])
mspinhole = arcomb.comb_frames(sframes[i], det, 'pinhole')
subframes[:, :, i] = mspinhole.copy()
del sframes
# Combine all sub-frames
mspinhole = arcomb.comb_frames(subframes, det, 'pinhole', weights=numarr)
del subframes
else:
mspinhole = arcomb.comb_frames(frames, det, 'pinhole')
del frames
else: # It must be the name of a file the user wishes to load
mspinhole_name = settings.argflag['run']['directory']['master'] + '/' + \
settings.argflag['reduce']['slitcen']['useframe']
mspinhole, head = armasters.load_master(mspinhole_name, frametype=None)
debugger.set_trace() # NEED TO LOAD EXTRAS AS ABOVE
# Set and then delete the Master Trace frame
self.SetMasterFrame(mspinhole, "pinhole", det)
#armasters.save_masters(self, det, mftype='pinhole')
del mspinhole
return True
def argf_diff_and_dup():
""" Compares default argf values against those in the ARMS and AMRED files
Returns
-------
"""
# Load default argf file
baseargf = arparse.get_argflag_class(('BaseArgFlag', '.tmp'))
base_lines = baseargf.load_file()
baseargf.set_paramlist(base_lines)
# ARMS
msgs.info("===============================================")
msgs.info("Working on ARMS vs. Base")
arms = arparse.get_argflag_class(('ARMS', '.tmp'))
arms_lines = arms.load_file()
arms.set_paramlist(arms_lines)
# Look for duplicates and diffs
for key in baseargf._argflag.keys():
if key in arms._argflag.keys():
compare_dicts(key, baseargf._argflag[key], arms._argflag[key])
# ARMED
msgs.info("===============================================")
msgs.info("Working on ARMED vs. Base")
armed = arparse.get_argflag_class(('ARMED', '.tmp'))
armed_lines = armed.load_file()
armed.set_paramlist(armed_lines)
# Look for duplicates and diffs
for key in baseargf._argflag.keys():
if key in armed._argflag.keys():
compare_dicts(key, baseargf._argflag[key], armed._argflag[key])
def compare_dicts(top_key, dict1, dict2, skip_keys=()):
for key in dict1.keys():
if isinstance(dict1[key], dict):
if key in dict2.keys():
compare_dicts(top_key + ',' + key, dict1[key], dict2[key])
else:
try:
test = dict1[key] == dict2[key]
except KeyError:
pass
else:
if test:
if key not in skip_keys:
msgs.info("{:s},{:s} is a duplicate".format(
top_key, key))
else:
msgs.warn("{:s},{:s} is different".format(top_key, key))
def spect_diff_and_dup():
# Load default spect file
path = pypit.__path__[0] + '/data/settings/'
basespect = arparse.BaseSpect(path + 'settings.basespect', '.tmp')
base_lines = basespect.load_file()
basespect.set_paramlist(base_lines)
# ARMS instruments
for specname in [
'shane_kast_blue', 'shane_kast_red', 'keck_lris_blue',
'keck_lris_red', 'wht_isis_blue', 'keck_deimos'
]:
msgs.info("===============================================")
msgs.info("Working on {:s}".format(specname))
spect = arparse.get_spect_class(('ARMS', specname, ".tmp"))
spect_lines = spect.load_file()
spect.set_paramlist(spect_lines)
msgs.info("===============================================")
for key in basespect._spect.keys():
if key in ['set']:
continue
if key in spect._spect.keys():
compare_dicts(key,
basespect._spect[key],
spect._spect[key],
skip_keys=('index'))
def get_wscale(slf):
"""
This routine calculates the wavelength array based on the sampling size (in km/s) of each pixel.
It conveniently assumes a standard reference wavelength of 911.75348 A
"""
lam0 = 911.75348
step = 1.0 + settings.argflag['reduce']['pixelsize'] / 299792.458
# Determine the number of pixels from lam0 that need to be taken to reach the minimum wavelength of the spectrum
msgs.work(
"No orders should be masked -- remove this code when the auto wavelength ID routine is fixed, and properly extrapolates."
)
w = np.where(slf._waveids != -999999.9)
nmin = int(np.log10(np.min(slf._waveids[w]) / lam0) / np.log10(step))
nmax = int(1.0 + np.log10(np.max(slf._waveids[w]) / lam0) /
np.log10(step)) # 1.0+ is to round up
wave = np.min(slf._waveids[w]) * (step**np.arange(1 + nmax - nmin))
msgs.info("Extracted wavelength range will be: {0:.5f} - {1:.5f}".format(
wave.min(), wave.max()))
msgs.info(
"Total number of spectral pixels in the extracted spectrum will be: {0:d}"
.format(1 + nmax - nmin))
return wave
def bg_subtraction(slf, tilts, det, sciframe, varframe, bpix, crpix, **kwargs):
""" Wrapper to run the background subtraction on a series of slits
Parameters
----------
slf
det : int
sciframe : ndarray
varframe : ndarray
crpix : ndarray
kwargs
Passed to bg_subtraction_slit
Returns
-------
bgframe : ndarray
"""
# Setup
bgframe = np.zeros_like(sciframe)
gdslits = np.where(~slf._maskslits[det - 1])[0]
for slit in gdslits:
msgs.info("Working on slit: {:d}".format(slit))
# TODO -- Replace this try/except when a more stable b-spline is used..
try:
slit_bgframe = bg_subtraction_slit(slf, det, slit, tilts, sciframe,
varframe, bpix, crpix, **kwargs)
except ValueError: # Should have been bspline..
msgs.warn(
"B-spline sky subtraction failed. Slit {:d} will no longer be processed.."
.format(slit))
#msgs.warn("Continue if you wish..")
slf._maskslits[det - 1][slit] = True
else:
bgframe += slit_bgframe
# Return
return bgframe
def archive():
""" Generate archival file for the baseargf file or spect file
and instrument setting files
"""
settings_path = resource_filename('pypit', '/data/settings/')
archive_path = resource_filename('pypit', '/data/settings/archive/')
# Files
sett_files = glob.glob(settings_path + 'settings.*')
for sfile in sett_files:
# Extension
ext = sfile.split('.')[-1]
if ext in ['py', 'pyc']:
continue
msgs.info("===============================================")
sroot = sfile.split('/')[-1]
msgs.info("Working on settings file {:s}".format(sroot))
msgs.info("===============================================")
# Archive
arch_file = current_sett_file(archive_path, sfile)
if arch_file is None:
match = False
arch_root = ''
else: # Compare
match = filecmp.cmp(sfile, arch_file)
arch_root = arch_file.split('/')[-1]
if not match:
msgs.warn("Current archive {:s} does not match {:s}".format(
arch_root, sfile))
new_arch = archive_path + '/settings.{:s}.{:s}'.format(
time.strftime("%Y-%m-%d"), ext)
msgs.warn("Generating a new archive file: {:s}".format(
new_arch.split('/')[-1]))
copyfile(sfile, new_arch)
msgs.warn("Add it to the repository!")
else:
msgs.info(
"Current archive file {:s} matches current settings file".
format(arch_root))
# Get rid of peaks within 3% of slit edge which are almost always spurious
not_near_edge = (xcen > nsamp * 0.03) & (xcen < nsamp * 0.97)
xcen = xcen[not_near_edge]
ypeak = ypeak[not_near_edge]
npeak = len(xcen)
specobjs = []
# Choose which ones to keep and discard based on threshold params. Create SpecObj objects
if npeak > 0:
# Possible thresholds [significance, fraction of brightest, absolute]
threshvec = np.array(
[SIG_THRESH * sigma, PEAK_THRESH * ypeak.max(), ABS_THRESH])
threshold = threshvec.max()
msgs.info('Using object finding threshold of: {:5.2f}'.format(threshold))
# Trim to only objects above this threshold
ikeep = (ypeak >= threshold)
xcen = xcen[ikeep]
ypeak = ypeak[ikeep]
nobj_reg = len(xcen)
# Now create SpecObj objects for all of these
for iobj in range(nobj_reg):
specobj = SpecObj(frameshape,
slit_spat_pos,
slit_spec_pos,
det=det,
config=config,
slitid=slitid,
scidx=scidx,
objtype=objtype)
def reduce_frame(slf,
sciframe,
rawvarframe,
modelvarframe,
bpix,
datasec_img,
bgframe,
scidx,
fitsdict,
det,
crmask,
tilts,
mswave,
scitrace=None,
standard=False):
""" Run standard extraction steps on a frame
Parameters
----------
sciframe : image
Bias subtracted, trimmed, and flatfielded image
rawvarframe : ndarray
Variance array using the raw detector counts
modelvarframe : ndarray
Model variance array using the raw detector counts and an image of the sky background frame.
bgframe : ndarray
Sky background image
scidx : int
Index of the frame
fitsdict : dict
Contains relevant information from fits header files
det : int
Detector index
scitrace : list of dict
List containing dictionaries of the object trace parameters
standard : bool, optional
Standard star frame?
"""
dnum = settings.get_dnum(det)
###############
# Determine the final trace of the science objects
if scitrace is None:
msgs.info("Performing final object trace")
scitrace = artrace.trace_objects_in_slits(slf,
det,
sciframe - bgframe,
modelvarframe,
crmask,
bgreg=20,
doqa=(not standard),
standard=standard)
if standard:
# slf._msstd[det-1]['trace'] = scitrace
# specobjs = arspecobj.init_exp(slf, scidx, det, fitsdict, scitrace, objtype='standard')
# slf._msstd[det-1]['spobjs'] = specobjs
specobjs = arspecobj.init_exp(slf,
scidx,
det,
fitsdict,
scitrace,
objtype='standard')
else:
# Generate SpecObjExp list
specobjs = arspecobj.init_exp(slf,
scidx,
det,
fitsdict,
scitrace,
objtype='science')
slf._scitrace[det - 1] = scitrace
slf._specobjs[det - 1] = specobjs
###############
# Extract
noobj = True
for sl in range(len(scitrace)):
if 'nobj' in scitrace[sl].keys():
if scitrace[sl]['nobj'] != 0:
noobj = False
if noobj is True:
msgs.warn("No objects to extract for science frame" + msgs.newline() +
fitsdict['filename'][scidx])
return True
# Boxcar
msgs.info("Performing boxcar extraction")
bgcorr_box = arextract.boxcar(slf, det, specobjs, sciframe - bgframe,
rawvarframe, bpix, bgframe, crmask, scitrace,
mswave)
# Optimal
if not standard:
# KBW: Using variance_frame() in arextract leads to a circular
# import. I've changed the arextract.optimal_extract() function
# to return the object model, then the last step of generating
# the new variance image is done here.
msgs.info("Attempting optimal extraction with model profile")
arextract.obj_profiles(slf,
det,
specobjs,
sciframe - bgframe - bgcorr_box,
modelvarframe,
bgframe + bgcorr_box,
crmask,
scitrace,
tilts,
doqa=False)
# newvar = arextract.optimal_extract(slf, det, specobjs, sciframe-bgframe-bgcorr_box,
# modelvarframe, bgframe+bgcorr_box, crmask, scitrace)
obj_model = arextract.optimal_extract(slf, det, slf._specobjs[det - 1],
sciframe - bgframe - bgcorr_box,
modelvarframe,
bgframe + bgcorr_box, crmask,
scitrace, tilts, mswave)
newvar = arprocimg.variance_frame(datasec_img,
det,
sciframe - bgframe - bgcorr_box,
-1,
settings.spect[dnum],
skyframe=bgframe + bgcorr_box,
objframe=obj_model)
msgs.work("Should update variance image (and trace?) and repeat")
#
arextract.obj_profiles(slf, det, slf._specobjs[det - 1],
sciframe - bgframe - bgcorr_box, newvar,
bgframe + bgcorr_box, crmask, scitrace, tilts)
# finalvar = arextract.optimal_extract(slf, det, specobjs, sciframe-bgframe-bgcorr_box,
# newvar, bgframe+bgcorr_box, crmask, scitrace)
obj_model = arextract.optimal_extract(slf, det, specobjs,
sciframe - bgframe - bgcorr_box,
newvar, bgframe + bgcorr_box,
crmask, scitrace, tilts, mswave)
finalvar = arprocimg.variance_frame(datasec_img,
det,
sciframe - bgframe - bgcorr_box,
-1,
settings.spect[dnum],
skyframe=bgframe + bgcorr_box,
objframe=obj_model)
slf._modelvarframe[det - 1] = finalvar.copy()
# Flexure correction?
if settings.argflag['reduce']['flexure']['perform'] and (not standard):
if settings.argflag['reduce']['flexure']['method'] is not None:
flex_list = arwave.flexure_obj(slf, det)
arwave.flexure_qa(slf, det, flex_list)
# Correct Earth's motion
if (settings.argflag['reduce']['calibrate']['refframe'] in ['heliocentric', 'barycentric']) and \
(settings.argflag['reduce']['calibrate']['wavelength'] != "pixel"):
if settings.argflag['science']['extraction']['reuse']:
msgs.warn(
"{0:s} correction will not be applied if an extracted science frame exists, and is used"
.format(settings.argflag['reduce']['calibrate']['refframe']))
if slf._specobjs[det - 1] is not None:
msgs.info("Performing a {0:s} correction".format(
settings.argflag['reduce']['calibrate']['refframe']))
arwave.geomotion_correct(slf, det, fitsdict)
else:
msgs.info(
"There are no objects on detector {0:d} to perform a {1:s} correction"
.format(det,
settings.argflag['reduce']['calibrate']['refframe']))
else:
msgs.info("A heliocentric correction will not be performed")
# Final
if not standard:
slf._bgframe[det - 1] += bgcorr_box
# Return
return True
def reduce_multislit(slf,
tilts,
sciframe,
bpix,
datasec_img,
scidx,
fitsdict,
det,
mswave,
mspixelflatnrm=None,
standard=False,
slitprof=None,
debug=False):
""" Run standard extraction steps on an echelle frame
Parameters
----------
sciframe : image
Bias subtracted image (using arload.load_frame)
bpix : ndarray
Bad pixel mask
scidx : int
Index of the frame
fitsdict : dict
Contains relevant information from fits header files
det : int
Detector index
standard : bool, optional
Standard star frame?
"""
#
dnum = settings.get_dnum(det)
sciframe, rawvarframe, crmask = reduce_prepare(
slf,
sciframe,
bpix,
datasec_img,
scidx,
fitsdict,
det,
mspixelflatnrm=mspixelflatnrm,
slitprof=slitprof)
# Save sciframe
slf._sciframe[det - 1] = sciframe.copy()
###############
# Estimate Sky Background
if settings.argflag['reduce']['skysub']['perform']:
# Perform an iterative background/science extraction
if debug:
debugger.set_trace() # JXP says THIS MAY NOT WORK AS EXPECTED
msgs.warn("Reading background from 2D image on disk")
datfil = settings.argflag['run']['directory'][
'science'] + '/spec2d_{:s}.fits'.format(
slf._basename.replace(":", "_"))
hdu = fits.open(datfil)
bgframe = hdu[1].data - hdu[2].data
else:
msgs.info("First estimate of the sky background")
bgframe = bg_subtraction(slf, tilts, det, sciframe, rawvarframe,
bpix, crmask)
modelvarframe = arprocimg.variance_frame(datasec_img,
det,
sciframe,
scidx,
settings.spect[dnum],
fitsdict=fitsdict,
skyframe=bgframe)
else:
modelvarframe = rawvarframe.copy()
bgframe = np.zeros_like(sciframe)
if not standard: # Need to save
slf._modelvarframe[det - 1] = modelvarframe
slf._bgframe[det - 1] = bgframe
###############
# Find objects and estimate their traces
scitrace = artrace.trace_objects_in_slits(slf,
det,
sciframe - bgframe,
modelvarframe,
crmask,
bgreg=20,
doqa=False,
standard=standard)
if scitrace is None:
msgs.info("Not performing extraction for science frame" +
msgs.newline() + fitsdict['filename'][scidx[0]])
debugger.set_trace()
#continue
# Make sure that there are objects
noobj = True
for sl in range(len(scitrace)):
if 'nobj' in scitrace[sl].keys(
): # There can be empty dict's (skipped slits)
if scitrace[sl]['nobj'] != 0:
noobj = False
if noobj is True:
msgs.warn("No objects to extract for science frame" + msgs.newline() +
fitsdict['filename'][scidx])
return True
###############
# Finalize the Sky Background image
if settings.argflag['reduce']['skysub']['perform']:
# Perform an iterative background/science extraction
msgs.info("Finalizing the sky background image")
# Create a trace mask of the object
trcmask = np.zeros_like(sciframe)
for sl in range(len(scitrace)):
if 'nobj' in scitrace[sl].keys():
if scitrace[sl]['nobj'] > 0:
trcmask += scitrace[sl]['object'].sum(axis=2)
trcmask[np.where(trcmask > 0.0)] = 1.0
# Do it
bgframe = bg_subtraction(slf,
tilts,
det,
sciframe,
modelvarframe,
bpix,
crmask,
tracemask=trcmask)
# Redetermine the variance frame based on the new sky model
modelvarframe = arprocimg.variance_frame(datasec_img,
det,
sciframe,
scidx,
settings.spect[dnum],
fitsdict=fitsdict,
skyframe=bgframe)
# Save
if not standard:
slf._modelvarframe[det - 1] = modelvarframe
slf._bgframe[det - 1] = bgframe
###############
# Flexure down the slit? -- Not currently recommended
if settings.argflag['reduce']['flexure']['method'] == 'slitcen':
flex_dict = arwave.flexure_slit(slf, det)
arwave.flexure_qa(slf, det, flex_dict, slit_cen=True)
# Perform an optimal extraction
msgs.work(
"For now, perform extraction -- really should do this after the flexure+heliocentric correction"
)
return reduce_frame(slf,
sciframe,
rawvarframe,
modelvarframe,
bpix,
datasec_img,
bgframe,
scidx,
fitsdict,
det,
crmask,
tilts,
mswave,
standard=standard)
def reduce_echelle(slf,
sciframe,
scidx,
fitsdict,
det,
standard=False,
triml=1,
trimr=1,
mspixelflatnrm=None,
doqa=True):
""" Run standard extraction steps on an echelle frame
Parameters
----------
sciframe : image
Bias subtracted image (using arload.load_frame)
scidx : int
Index of the frame
fitsdict : dict
Contains relevant information from fits header files
det : int
Detector index
standard : bool, optional
Standard star frame?
triml : int (optional)
Number of pixels to trim from the left slit edge
trimr : int (optional)
Number of pixels to trim from the right slit edge
"""
msgs.work("Multiprocess this algorithm")
nspec = sciframe.shape[0]
nord = slf._lordloc[det - 1].shape[1]
# Prepare the frames for tracing and extraction
sciframe, rawvarframe, crmask = reduce_prepare(
slf,
sciframe,
scidx,
fitsdict,
det,
mspixelflatnrm=mspixelflatnrm,
standard=standard,
slitprof=slitprof)
bgframe = np.zeros_like(sciframe)
bgnl, bgnr = np.zeros(nord, dtype=np.int), np.zeros(nord, dtype=np.int)
skysub = True
if settings.argflag['reduce']['skysub']['perform']:
# Identify background pixels, and generate an image of the sky spectrum in each slit
for o in range(nord):
word = np.where((slf._slitpix[det - 1] == o + 1)
& (slf._scimask[det - 1] == 0))
if word[0].size == 0:
msgs.warn("There are no pixels in slit {0:d}".format(o + 1))
continue
tbgframe, nl, nr = background_subtraction(slf, sciframe,
rawvarframe, o, det)
bgnl[o], bgnr[o] = nl, nr
bgframe += tbgframe
if nl == 0 and nr == 0:
pass
# If just one slit cannot do sky subtraction, don't do sky subtraction
# msgs.warn("A sky subtraction will not be performed")
# skysub = False
# bgframe = np.zeros_like(sciframe)
# modelvarframe = rawvarframe.copy()
# break
if skysub:
# Provided the for loop above didn't break early, model the variance frame
dnum = settings.get_dnum(det)
modelvarframe = arprocimg.variance_frame(datasec_img,
det,
sciframe,
scidx,
settings.spect[dnum],
fitsdict=fitsdict,
skyframe=bgframe)
else:
modelvarframe = rawvarframe.copy()
bgframe = np.zeros_like(sciframe)
if not standard: # Need to save
slf._modelvarframe[det - 1] = modelvarframe
slf._bgframe[det - 1] = bgframe
# Obtain a first estimate of the object trace then
# fit the traces and perform a PCA for the refinements
trccoeff = np.zeros(
(settings.argflag['trace']['object']['order'] + 1, nord))
trcxfit = np.arange(nspec)
extrap_slit = np.zeros(nord)
for o in range(nord):
trace, error = artrace.trace_weighted(sciframe - bgframe,
slf._lordloc[det - 1][:, o],
slf._rordloc[det - 1][:, o],
mask=slf._scimask[det - 1],
wght="flux")
if trace is None:
extrap_slit[o] = 1
continue
# Find only the good pixels
w = np.where((error != 0.0) & (~np.isnan(error)))
if w[0].size <= 2 * settings.argflag['trace']['object']['order']:
extrap_slit[o] = 1
continue
# Convert the trace locations to be a fraction of the slit length,
# measured from the left slit edge.
trace -= slf._lordloc[det - 1][:, o]
trace /= (slf._rordloc[det - 1][:, o] - slf._lordloc[det - 1][:, o])
try:
msk, trccoeff[:, o] = arutils.robust_polyfit(
trcxfit[w],
trace[w],
settings.argflag['trace']['object']['order'],
function=settings.argflag['trace']['object']['function'],
weights=1.0 / error[w]**2,
minv=0.0,
maxv=nspec - 1.0)
except:
msgs.info("arproc.reduce_echelle")
debugger.set_trace()
refine = 0.0
if settings.argflag['trace']['object']['method'] == "pca":
# Identify the orders to be extrapolated during reconstruction
orders = 1.0 + np.arange(nord)
msgs.info("Performing a PCA on the object trace")
ofit = settings.argflag['trace']['object']['params']
lnpc = len(ofit) - 1
maskord = np.where(extrap_slit == 1)[0]
xcen = trcxfit[:, np.newaxis].repeat(nord, axis=1)
trccen = arutils.func_val(
trccoeff,
trcxfit,
settings.argflag['trace']['object']['function'],
minv=0.0,
maxv=nspec - 1.0).T
if np.sum(1.0 - extrap_slit) > ofit[0] + 1:
fitted, outpar = arpca.basis(
xcen,
trccen,
trccoeff,
lnpc,
ofit,
skipx0=False,
mask=maskord,
function=settings.argflag['trace']['object']['function'])
if doqa:
# arqa.pca_plot(slf, outpar, ofit, "Object_Trace", pcadesc="PCA of object trace")
arpca.pca_plot(slf.setup,
outpar,
ofit,
"Object_Trace",
pcadesc="PCA of object trace")
# Extrapolate the remaining orders requested
trccen, outpar = arpca.extrapolate(
outpar,
orders,
function=settings.argflag['trace']['object']['function'])
#refine = trccen-trccen[nspec//2, :].reshape((1, nord))
else:
msgs.warn("Could not perform a PCA on the object trace" +
msgs.newline() + "Not enough well-traced orders")
msgs.info("Using direct determination of the object trace instead")
pass
else:
msgs.error("Not ready for object trace method:" + msgs.newline() +
settings.argflag['trace']['object']['method'])
# Construct the left and right traces of the object profile
# The following code ensures that the fraction of the slit
# containing the object remains constant along the spectral
# direction
trcmean = np.mean(trccen, axis=0)
trobjl = (trcmean -
(1 + bgnl) / slf._pixwid[det - 1].astype(np.float)).reshape(
(1, nord)).repeat(nspec, axis=0)
trobjl = trccen - trobjl
trobjr = (-trcmean + (slf._pixwid[det - 1] - bgnr - 1) /
slf._pixwid[det - 1].astype(np.float)).reshape(
(1, nord)).repeat(nspec, axis=0)
trobjr = trccen + trobjr
# Convert trccen to the actual trace locations
trccen *= (slf._rordloc[det - 1] - slf._lordloc[det - 1])
trccen += slf._lordloc[det - 1]
trobjl *= (slf._rordloc[det - 1] - slf._lordloc[det - 1])
trobjl += slf._lordloc[det - 1]
trobjr *= (slf._rordloc[det - 1] - slf._lordloc[det - 1])
trobjr += slf._lordloc[det - 1]
# Generate an image of pixel weights for each object. Each weight can
# take any floating point value from 0 to 1 (inclusive). For the rec_obj_img,
# a weight of 1 means that the pixel is fully contained within the object
# region, and 0 means that the pixel is fully contained within the background
# region. The opposite is true for the rec_bg_img array. A pixel that is on
# the border of object/background is assigned a value between 0 and 1.
msgs.work(
"Eventually allow ARMED to find multiple objects in the one slit")
nobj = 1
rec_obj_img = np.zeros(sciframe.shape + (nobj, ))
rec_bg_img = np.zeros(sciframe.shape + (nobj, ))
for o in range(nord):
# Prepare object/background regions
objl = np.array([bgnl[o]])
objr = np.array([slf._pixwid[det - 1][o] - bgnr[o] - triml - trimr])
bckl = np.zeros((slf._pixwid[det - 1][o] - triml - trimr, 1))
bckr = np.zeros((slf._pixwid[det - 1][o] - triml - trimr, 1))
bckl[:bgnl[o]] = 1
if bgnr[o] != 0:
bckr[-bgnr[o]:] = 1
tobj_img, tbg_img = artrace.trace_objbg_image(slf,
det,
sciframe - bgframe,
o, [objl, objr],
[bckl, bckr],
triml=triml,
trimr=trimr)
rec_obj_img += tobj_img
rec_bg_img += tbg_img
# Create trace dict
scitrace = artrace.trace_object_dict(nobj,
trccen[:,
0].reshape(trccen.shape[0], 1),
object=rec_obj_img,
background=rec_bg_img)
for o in range(1, nord):
scitrace = artrace.trace_object_dict(nobj,
trccen[:, o].reshape(
trccen.shape[0], 1),
tracelist=scitrace)
# Save the quality control
if doqa:
artrace.obj_trace_qa(slf,
sciframe,
trobjl,
trobjr,
None,
det,
root="object_trace",
normalize=False)
# Finalize the Sky Background image
if settings.argflag['reduce']['skysub']['perform'] and (nobj >
0) and skysub:
msgs.info("Finalizing the sky background image")
# Identify background pixels, and generate an image of the sky spectrum in each slit
bgframe = np.zeros_like(sciframe)
for o in range(nord):
tbgframe, nl, nr = background_subtraction(slf,
sciframe,
rawvarframe,
o,
det,
refine=refine)
bgnl[o], bgnr[o] = nl, nr
bgframe += tbgframe
modelvarframe = arprocimg.variance_frame(datasec_img,
det,
sciframe,
scidx,
settings.spect[dnum],
fitsdict=fitsdict,
skyframe=bgframe)
# Perform an optimal extraction
return reduce_frame(slf,
sciframe,
rawvarframe,
modelvarframe,
bgframe,
scidx,
fitsdict,
det,
crmask,
scitrace=scitrace,
standard=standard)
def background_subtraction(slf,
sciframe,
varframe,
slitn,
det,
refine=0.0,
doqa=True):
""" Generate a frame containing the background sky spectrum
Parameters
----------
slf : Class
Science Exposure Class
sciframe : ndarray
science frame
varframe : ndarray
variance frame
slitn : int
Slit number
det : int
Detector index
refine : float or ndarray
refine the object traces. This should be a small value around 0.0.
If a float, a constant offset will be applied.
Otherwise, an array needs to be specified of the same length as
sciframe.shape[0] that contains the refinement of each pixel along
the spectral direction.
Returns
-------
bgframe : ndarray
An image, the same size as sciframe, that contains
the background spectrum within the specified slit.
nl : int
number of pixels from the left slit edge to use as background pixels
nr : int
number of pixels from the right slit edge to use as background pixels
"""
# Obtain all pixels that are within the slit edges, and are not masked
word = np.where((slf._slitpix[det - 1] == slitn + 1)
& (slf._scimask[det - 1] == 0))
if word[0].size == 0:
msgs.warn("There are no pixels in slit {0:d}".format(slitn))
debugger.set_trace()
nl, nr = 0, 0
return np.zeros_like(sciframe), nl, nr
# Calculate the oversampled object profiles
oversampling_factor = 3 # should be an integer according to the description in object_profile()
xedges, modvals = object_profile(slf,
sciframe,
slitn,
det,
refine=refine,
factor=oversampling_factor)
bincent = 0.5 * (xedges[1:] + xedges[:-1])
npix = slf._pixwid[det - 1][slitn]
tilts = slf._tilts[det - 1].copy()
lordloc = slf._lordloc[det - 1][:, slitn]
rordloc = slf._rordloc[det - 1][:, slitn]
# For each pixel, calculate the fraction along the slit's spatial direction
spatval = (word[1] - lordloc[word[0]] + refine) / (rordloc[word[0]] -
lordloc[word[0]])
# Cumulative sum and normalize
csum = np.cumsum(modvals)
csum -= csum[0]
csum /= csum[-1]
# Find a first guess of the edges of the object profile - assume this is the innermost 90 percent of the flux
argl = np.argmin(np.abs(csum - 0.05))
argr = np.argmin(np.abs(csum - 0.95))
# Considering the possible background pixels that are left of the object,
# find the first time where the object profile no longer decreases as you
# move toward the edge of the slit. This is the beginning of the noisy
# object profile, which is where the object can no longer be distinguished
# from the background.
wl = np.where((modvals[1:] < modvals[:-1]) & (bincent[1:] < bincent[argl]))
wr = np.where((modvals[1:] > modvals[:-1]) & (bincent[1:] > bincent[argr]))
nl, nr = 0, 0
if wl[0].size != 0:
# This is the index of the first time where the object profile
# no longer decreases as you move towards the slit edge
nl_index = np.max(wl[0])
# Calculate nl, defined as:
# "number of pixels from the left slit edge to use as background pixels",
# which is just nl_index with the sampling factor taken out
nl_index_origscale = int(nl_index / oversampling_factor + 0.5)
nl = nl_index_origscale
if wr[0].size != 0:
# This is the index of the first time where the object profile
# no longer decreases as you move towards the slit edge
nr_index = np.min(wr[0])
# Calculate nr, defined as:
# "number of pixels from the right slit edge to use as background pixels",
# which is npix minus nr_index with the sampling factor taken out
nr_index_origscale = int(nr_index / oversampling_factor + 0.5)
nr = npix - nr_index_origscale
if nl + nr < 5:
msgs.warn(
"The object profile appears to extrapolate to the edge of the slit"
)
msgs.info(
"A background subtraction will not be performed for slit {0:d}".
format(slitn + 1))
nl, nr = 0, 0
return np.zeros_like(sciframe), nl, nr
# Find background pixels and fit
wbgpix_spatval = np.where(
(spatval <= float(nl) / npix) |
(spatval >= float(npix - nr) /
npix)) # this cannot be used to index the 2D array tilts
wbgpix = (word[0][wbgpix_spatval], word[1][wbgpix_spatval]
) # this may be approproate for indexing the 2D array tilts
if settings.argflag['reduce']['skysub']['method'].lower() == 'bspline':
msgs.info("Using bspline sky subtraction")
srt = np.argsort(tilts[wbgpix])
ivar = arutils.calc_ivar(varframe)
# Perform a weighted b-spline fit to the sky background pixels
mask, bspl = arutils.robust_polyfit(
tilts[wbgpix][srt],
sciframe[wbgpix][srt],
3,
function='bspline',
weights=np.sqrt(ivar)[wbgpix][srt],
sigma=5.,
maxone=False,
**settings.argflag['reduce']['skysub']['bspline'])
bgf_flat = arutils.func_val(bspl, tilts.flatten(), 'bspline')
bgframe = bgf_flat.reshape(tilts.shape)
if doqa:
plt_bspline_sky(tilts, sciframe, bgf_flat, gdp)
debugger.set_trace()
else:
msgs.error('Not ready for this method for skysub {:s}'.format(
settings.argflag['reduce']['skysub']['method'].lower()))
if np.any(np.isnan(bgframe)):
msgs.warn("NAN in bgframe. Replacing with 0")
bad = np.isnan(bgframe)
bgframe[bad] = 0.
return bgframe, nl, nr
def reduce_prepare(slf,
sciframe,
bpix,
datasec_img,
scidx,
fitsdict,
det,
mspixelflatnrm=None,
standard=False,
slitprof=None):
""" Prepare the Run standard extraction steps on a frame
Parameters
----------
sciframe : image
Bias subtracted image (using arload.load_frame)
bpix : image
scidx : int
Index of the frame
fitsdict : dict
Contains relevant information from fits header files
det : int
Detector index
standard : bool, optional
Standard star frame?
"""
# Check inputs
if not isinstance(scidx, (int, np.integer)):
raise IOError("scidx needs to be an int")
# Convert ADUs to electrons
dnum = settings.get_dnum(det)
namp = settings.spect[dnum]['numamplifiers']
gain_list = settings.spect[dnum]['gain']
sciframe *= arprocimg.gain_frame(datasec_img, namp, gain_list)
# Mask
slf._scimask[det - 1] = np.zeros_like(sciframe).astype(int)
#msgs.info("Masking bad pixels")
#slf.update_sci_pixmask(det, bpix, 'BadPix')
# Variance
msgs.info(
"Generate raw variance frame (from detected counts [flat fielded])")
rawvarframe = arprocimg.variance_frame(datasec_img,
det,
sciframe,
scidx,
settings.spect[dnum],
fitsdict=fitsdict)
###############
# Subtract off the scattered light from the image
msgs.work("Scattered light subtraction is not yet implemented...")
###############
# Flat field the science frame (and variance)
if settings.argflag['reduce']['flatfield']['perform']:
msgs.info("Flat fielding the science frame")
# JXP -- I think it is a bad idea to modify the rawvarframe
#sciframe, rawvarframe = flatfield(slf, sciframe, slf._mspixelflatnrm[det-1], det, varframe=rawvarframe, slitprofile=slf._slitprof[det-1])
sciframe = arflat.flatfield(sciframe,
mspixelflatnrm,
bpix,
slitprofile=slitprof)
else:
msgs.info("Not performing a flat field calibration")
if not standard:
slf._sciframe[det - 1] = sciframe
slf._rawvarframe[det - 1] = rawvarframe
###############
# Identify cosmic rays
msgs.work("Include L.A.Cosmic arguments in the settings files")
if True:
crmask = arprocimg.lacosmic(datasec_img,
fitsdict,
det,
sciframe,
scidx,
settings.spect[dnum],
grow=1.5)
else:
crmask = np.zeros(sciframe.shape)
# Mask
slf.update_sci_pixmask(det, crmask, 'CR')
return sciframe, rawvarframe, crmask
maxcol = np.fmin(np.ceil(max(maxcols)),imax)
nc = int(maxcol - mincol + 1)
rows = np.arange(nspec, dtype=np.intp)
columns = np.arange(mincol, mincol + nc, dtype=np.intp)
ipix = np.ix_(rows,columns)
skymask = outmask & ~edgmask
if nc > 100:
npoly = 3
elif nc > 40:
npoly = 2
else:
npoly = 1
obj_profiles = np.zeros((nspec,nspat,objwork), dtype=float)
sigrej_eff = sigrej
for iiter in range(1,niter):
msgs.info("Iteration # " + "{:2d}".format(iiter) + " of " + "{:2d}".format(niter))
img_minsky = sciimg - skyimage
for ii in range(objwork):
iobj = group[ii]
if iiter == 1:
# If this is the first iteration, print status message. Initiate profile fitting with a simple
# boxcar extraction.
msgs.info("-------------------REDUCING-------------------")
msgs.info("Fitting profile for obj #: " + "{:d}".format(specobjs[iobj].objid) + " of {:d}".format(nobj))
msgs.info("At x = {:5.2f}".format(specobjs[iobj].spat_medpos) + " on slit # {:d}".format(specobjs[iobj].slitid))
msgs.info("----------------------------------------------")
flux = extract_boxcar(img_minsky*outmask, specobjs[iobj].trace_spat, box_rad, ycen = specobjs[iobj].trace_spec)
mvarimg = 1.0/(modelivar + (modelivar == 0))
mvar_box = extract_boxcar(mvarimg*outmask, specobjs[iobj].trace_spat, box_rad, ycen = specobjs[iobj].trace_spec)
pixtot = extract_boxcar(0*mvarimg + 1.0, specobjs[iobj].trace_spat, box_rad, ycen = specobjs[iobj].trace_spec)
mask_box = (extract_boxcar(~outmask, specobjs[iobj].trace_spat, box_rad, ycen=specobjs[iobj].trace_spec) != pixtot)
pass
# Get rid of peaks within 3% of slit edge which are almost always spurious
not_near_edge = (xcen > nsamp*0.03) & (xcen < nsamp*0.97)
xcen = xcen[not_near_edge]
ypeak = ypeak[not_near_edge]
npeak = len(xcen)
specobjs =[]
# Choose which ones to keep and discard based on threshold params. Create SpecObj objects
if npeak > 0:
# Possible thresholds [significance, fraction of brightest, absolute]
threshvec = np.array([SIG_THRESH*sigma, PEAK_THRESH*ypeak.max(), ABS_THRESH])
threshold = threshvec.max()
msgs.info('Using object finding threshold of: {:5.2f}'.format(threshold))
# Trim to only objects above this threshold
ikeep = (ypeak >= threshold)
xcen = xcen[ikeep]
ypeak = ypeak[ikeep]
nobj_reg = len(xcen)
# Now create SpecObj objects for all of these
for iobj in range(nobj_reg):
specobj = SpecObj(frameshape, slit_spat_pos, slit_spec_pos, det = det, config = config, slitid = slitid, scidx = scidx, objtype=objtype)
specobj.spat_fracpos = xcen[iobj]/nsamp
specobj.smash_peakflux = ypeak[iobj]
specobjs.append(specobj)
# Now deal with the hand apertures if a HAND_DICT was passed in. Add these to the SpecObj objects
if HAND_DICT is not None:
