def load_master(self, filename, force=False):
# Does the master file exist?
if not os.path.isfile(filename):
# msgs.warn("No Master frame found of type {:s}: {:s}".format(self.frametype, filename))
msgs.warn("No Master frame found of {:s}".format(filename))
if force:
msgs.error("Crashing out because reduce-masters-force=True:" + msgs.newline() + filename)
return None
else:
# msgs.info("Loading a pre-existing master calibration frame of type: {:}".format(self.frametype) + " from filename: {:}".format(filename))
msgs.info("Loading a pre-existing master calibration frame of SENSFUNC from filename: {:}".format(filename))
hdu = fits.open(filename)
norder = hdu[0].header['NORDER']
sens_dicts = {}
for iord in range(norder):
head = hdu[iord + 1].header
tbl = hdu['SENSFUNC-ORDER{0:04}'.format(iord)].data
sens_dict = {}
sens_dict['wave'] = tbl['WAVE']
sens_dict['sensfunc'] = tbl['SENSFUNC']
for key in ['wave_min', 'wave_max', 'exptime', 'airmass', 'std_file', 'std_ra', 'std_dec',
'std_name', 'cal_file', 'ech_orderindx']:
try:
sens_dict[key] = head[key.upper()]
except:
pass
sens_dicts[str(iord)] = sens_dict
sens_dicts['norder'] = norder
return sens_dicts
def write_science(sci_specobjs, sci_header, outfile):
"""
Write the flux-calibrated science spectra
Parameters
----------
outfile : str
Returns
-------
"""
if len(sci_specobjs) == 0:
msgs.warn("No science spectra to write to disk!")
#
if 'VEL-TYPE' in sci_header.keys():
helio_dict = dict(refframe=sci_header['VEL-TYPE'],
vel_correction=sci_header['VEL'])
else:
helio_dict = None
telescope=None
if 'LON-OBS' in sci_header.keys():
telescope = TelescopePar(longitude=sci_header['LON-OBS'],
latitude=sci_header['LAT-OBS'],
elevation=sci_header['ALT-OBS'])
# KLUDGE ME
if isinstance(sci_specobjs, list):
specObjs = specobjs.SpecObjs(sci_specobjs)
elif isinstance(sci_specobjs, specobjs.SpecObjs):
specObjs = sci_specobjs
else:
msgs.error("BAD INPUT")
save.save_1d_spectra_fits(specObjs, sci_header,'ECHELLE', outfile,
helio_dict=helio_dict,
telescope=telescope, overwrite=True)
def show_sensfunc(self):
"""
Plot the sensitivity function
"""
if self.sens_dict is None:
msgs.warn("You need to generate the sensfunc first!")
return None
plt.rcdefaults()
plt.rcParams["xtick.top"] = True
plt.rcParams["ytick.right"] = True
plt.rcParams["xtick.minor.visible"] = True
plt.rcParams["ytick.minor.visible"] = True
plt.rcParams["ytick.direction"] = 'in'
plt.rcParams["xtick.direction"] = 'in'
plt.rcParams["xtick.labelsize"] = 13
plt.rcParams["ytick.labelsize"] = 13
plt.rcParams['font.family'] = 'times new roman'
norder = self.sens_dict['norder']
for iord in range(norder):
sens_dict_iord = self.sens_dict[str(iord)]
plt.plot(sens_dict_iord['wave'], sens_dict_iord['sensfunc'])
plt.xlabel('Wavelength [ang]', fontsize=14)
plt.ylabel('Sensfunc', fontsize=14)
plt.ylim([0., 100.0])
plt.show()
def subtract_bias(self, bias_image, force=False):
"""
Perform bias subtraction
Args:
bias_image (np.ndarray):
Bias image
force (bool, optional):
Force the processing even if the image was already processed
"""
step = inspect.stack()[0][3]
# Check if already trimmed
if self.steps[step] and (not force):
msgs.warn("Image was already bias subtracted. Returning the current image")
return self.image.copy()
# Do it
self.image -= bias_image
self.steps[step] = True
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype in ['pinhole', 'bias']:
# No pinhole or bias frames
return np.zeros(len(fitstbl), dtype=bool)
if ftype in ['pixelflat', 'trace']:
return good_exp & (fitstbl['idname'] == 'PixFlat')
if ftype == 'standard':
return good_exp & (fitstbl['idname'] == 'Telluric')
if ftype == 'science':
return good_exp & (fitstbl['idname'] == 'Science')
if ftype in ['arc', 'tilt']:
return good_exp & (fitstbl['idname'] == 'Science')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype in ['science', 'standard']:
return good_exp & (fitstbl['idname'] == 'OBJECT')
if ftype == 'bias':
return good_exp & (fitstbl['idname'] == 'BIAS')
if ftype in ['pixelflat', 'trace', 'illumflat']:
return good_exp & (fitstbl['idname'] == 'FLAT,LAMP')
if ftype in ['pinhole', 'dark']:
# Don't type pinhole or dark frames
return np.zeros(len(fitstbl), dtype=bool)
if ftype in ['arc', 'tilt']:
return good_exp & (fitstbl['idname'] == 'WAVE,LAMP')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def subtract_dark(self, dark_image, force=False):
"""
Perform dark image subtraction
Args:
dark_image (PypeItImage):
Dark image
"""
step = inspect.stack()[0][3]
# Check if already bias subtracted
if self.steps[step] and not force:
msgs.warn(
"Image was already dark subtracted. Returning the current image"
)
return self.image.copy()
# Do it
self.image -= dark_image.image
self.steps[step] = True
def spec_from_array(wave,flux,sig,**kwargs):
"""
return spectrum from arrays of wave, flux and sigma
"""
ituple = (wave, flux, sig)
spectrum = XSpectrum1D.from_tuple(ituple, **kwargs)
# Polish a bit -- Deal with NAN, inf, and *very* large values that will exceed
# the floating point precision of float32 for var which is sig**2 (i.e. 1e38)
bad_flux = np.any([np.isnan(spectrum.flux), np.isinf(spectrum.flux),
np.abs(spectrum.flux) > 1e30,
spectrum.sig ** 2 > 1e10,
], axis=0)
if np.sum(bad_flux):
msgs.warn("There are some bad flux values in this spectrum. Will zero them out and mask them (not ideal)")
spectrum.data['flux'][spectrum.select][bad_flux] = 0.
spectrum.data['sig'][spectrum.select][bad_flux] = 0.
return spectrum
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype in ['science', 'standard']:
return good_exp & (fitstbl['lampstat01'] == 'Off') & (fitstbl['idname'] == 'object')
if ftype == 'bias':
return good_exp & (fitstbl['idname'] == 'zero')
if ftype in ['pixelflat', 'trace']:
return good_exp & (fitstbl['lampstat01'] == 'W') & (fitstbl['idname'] == 'flat')
if ftype in ['pinhole', 'dark']:
# Don't type pinhole or dark frames
return np.zeros(len(fitstbl), dtype=bool)
if ftype == 'arc':
return good_exp & (fitstbl['lampstat01'] == 'CuNe+CuAr') & (fitstbl['idname'] == 'arc')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def generate_sensfunc(self):
"""
Generate the senstivity function
Wrapper to flux.generate_sensfunc
Requires self.std has been set
Returns
-------
self.sensfunc : dict
"""
# Check internals
# if self.std is None:
# msgs.warn('First identify the star first (with find_standard).')
# return None
if self.std_header is None:
msgs.warn('First set std_header with a dict-like object holding RA, DEC, '
'AIRMASS, EXPTIME.')
return None
ext_final = fits.getheader(self.std_spec1d_file, -1)
norder = ext_final['ORDER'] + 1
self.sens_dict = {}
for iord in range(norder):
std_specobjs, std_header = load.ech_load_specobj(self.std_spec1d_file, order=iord)
std_idx = flux.find_standard(std_specobjs)
std = std_specobjs[std_idx]
wavemask = std.boxcar['WAVE'] > 1000.0 * units.AA
wave, counts, ivar = std.boxcar['WAVE'][wavemask], std.boxcar['COUNTS'][wavemask], \
std.boxcar['COUNTS_IVAR'][wavemask]
sens_dict_iord = flux.generate_sensfunc(wave, counts, ivar, std_header['AIRMASS'], std_header['EXPTIME'],
self.spectrograph, star_type=self.star_type, star_mag=self.star_mag,
telluric=self.telluric, ra=self.std_ra, dec=self.std_dec,
BALM_MASK_WID=self.BALM_MASK_WID,
nresln=self.nresln, std_file=self.std_file, debug=self.debug)
sens_dict_iord['ech_orderindx'] = iord
self.sens_dict[str(iord)] = sens_dict_iord
self.sens_dict['norder'] = norder
# Step
self.steps.append(inspect.stack()[0][3])
# Return
return self.sens_dict
def get_arc(self):
"""
Load or generate the Arc image
Requirements:
master_key, det, par
Args:
Returns:
`numpy.ndarray`_: :attr:`msarc` image
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
arc_files, self.master_key_dict['arc'] = self._prep_calibrations('arc')
masterframe_name = masterframe.construct_file_name(
buildimage.ArcImage,
self.master_key_dict['arc'],
master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.msarc = buildimage.ArcImage.from_file(masterframe_name)
elif len(arc_files) == 0:
msgs.warn("No frametype=arc files to build arc")
return
else: # Build it
msgs.info("Preparing a master {0:s} frame".format(
buildimage.ArcImage.master_type))
self.msarc = buildimage.buildimage_fromlist(self.spectrograph,
self.det,
self.par['arcframe'],
arc_files,
bias=self.msbias,
bpm=self.msbpm,
dark=self.msdark)
# Save
self.msarc.to_master_file(masterframe_name)
# Return
return self.msarc
def _chk_objs(self, items):
"""
Check that the input items exist internally as attributes
Args:
items (list):
Returns:
bool: True if all exist
"""
for obj in items:
if getattr(self, obj) is None:
msgs.warn("You need to generate {:s} prior to this calibration..".format(obj))
# Strip ms
iobj = obj[2:] if obj[0:2] == 'ms' else obj
msgs.warn("Use get_{:s}".format(iobj))
return False
return True
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
Args:
ftype (:obj:`str`):
Type of frame to check. Must be a valid frame type; see
frame-type :ref:`frame_type_defs`.
fitstbl (`astropy.table.Table`_):
The table with the metadata for one or more frames to check.
exprng (:obj:`list`, optional):
Range in the allowed exposure time for a frame of type
``ftype``. See
:func:`pypeit.core.framematch.check_frame_exptime`.
Returns:
`numpy.ndarray`_: Boolean array with the flags selecting the
exposures in ``fitstbl`` that are ``ftype`` type frames.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype in ['science', 'standard']:
return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 0) \
& (fitstbl['idname'] == 'object')
if ftype in ['bias', 'dark']:
return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 0) \
& (fitstbl['idname'] == 'dark')
if ftype in ['pixelflat', 'trace']:
# Flats and trace frames are typed together
return good_exp & self.lamps(fitstbl, 'dome') & (fitstbl['hatch'] == 1) \
& (fitstbl['idname'] == 'flatlamp')
if ftype == 'pinhole':
# Don't type pinhole frames
return np.zeros(len(fitstbl), dtype=bool)
if ftype in ['arc', 'tilt']:
# TODO: This is a kludge. Allow science frames to also be
# classified as arcs
is_arc = self.lamps(fitstbl, 'arcs') & (fitstbl['hatch'] == 1) \
& (fitstbl['idname'] == 'arclamp')
is_obj = self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 0) \
& (fitstbl['idname'] == 'object')
return good_exp & (is_arc | is_obj)
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def save(self, outfile=None, overwrite=True):
"""
Save the wavelength tilts data to a master frame
Args:
outfile (:obj:`str`, optional):
Name for the output file. Defaults to
:attr:`file_path`.
overwrite (:obj:`bool`, optional):
Overwrite any existing file.
"""
_outfile = self.file_path if outfile is None else outfile
# Check if it exists
if os.path.exists(_outfile) and not overwrite:
msgs.warn('Master file exists: {0}'.format(_outfile) +
msgs.newline() + 'Set overwrite=True to overwrite it.')
return
# Log
msgs.info('Saving master frame to {0}'.format(_outfile))
# Build the header
hdr = fits.Header()
# - Set the master frame type
hdr['FRAMETYP'] = (self.master_type,
'PypeIt: Master calibration frame type')
# - List the completed steps
hdr['STEPS'] = (','.join(self.steps), 'Completed reduction steps')
# - Tilts metadata
hdr['FUNC2D'] = self.tilts_dict['func2d']
hdr['NSLIT'] = self.tilts_dict['nslit']
# Write the fits file
fits.HDUList([
fits.PrimaryHDU(header=hdr),
fits.ImageHDU(data=self.tilts_dict['tilts'], name='TILTS'),
fits.ImageHDU(data=self.tilts_dict['coeffs'], name='COEFFS'),
fits.ImageHDU(data=self.tilts_dict['slitcen'], name='SLITCEN'),
fits.ImageHDU(data=self.tilts_dict['spat_order'],
name='SPAT_ORDER'),
fits.ImageHDU(data=self.tilts_dict['spec_order'],
name='SPEC_ORDER')
]).writeto(_outfile, overwrite=True)
def orient(self, force=False):
"""
Orient the image in the PypeIt format with spectra running blue (down)
to red (up).
Args:
force (bool, optional):
Force the processing even if the image was already processed
"""
step = inspect.stack()[0][3]
# Orient the image to have blue/red run bottom to top
# Check if already oriented
if self.steps[step] and not force:
msgs.warn("Image was already oriented. Returning current image")
return self.image.copy()
# Orient me
self.image = self.spectrograph.orient_image(self.image, self.det)
self.steps[step] = True
def connect_to_ginga(host='localhost',
port=9000,
raise_err=False,
allow_new=False):
"""
Connect to a RC Ginga.
Args:
host (:obj:`str`, optional):
Host name.
port (:obj:`int`, optional):
Probably should remain at 9000
raise_err (:obj:`bool`, optional):
Raise an error if no connection is made, otherwise just
raise a warning and continue
allow_new (:obj:`bool`, optional):
Allow a subprocess to be called to execute a new ginga
viewer if one is not already running.
Returns:
RemoteClient: connection to ginga viewer.
"""
# Start
viewer = grc.RemoteClient(host, port)
# Test
ginga = viewer.shell()
try:
tmp = ginga.get_current_workspace()
except:
if allow_new:
subprocess.Popen(['ginga', '--modules=RC'])
time.sleep(3)
return grc.RemoteClient(host, port)
if raise_err:
raise ValueError
else:
msgs.warn(
'Problem connecting to Ginga. Launch an RC Ginga viewer and '
'then continue: \n ginga --modules=RC')
# Return
return viewer
def load(self, ifile=None, return_header=False):
"""
Load the trace slits data.
Reads both the trace data and the trace image data. This is
largely a wrapper for
:func:`pypeit.traceslits.TraceSlits.load_from_file`.
Args:
ifile (:obj:`str`, optional):
Name of the master frame file. Defaults to
:attr:`file_path`.
return_header (:obj:`bool`, optional):
Return the header, which will include the TraceImage
metadata if available.
Returns:
tuple: Returns the trace slits dictionary and the trace slit
image. If return_header is true, the primary header is also
returned. If nothing is loaded, either because
:attr:`reuse_masters` is `False` or the file does not exist,
everything is returned as None (one per expected return
object).
"""
# Format the input and set the tuple for an empty return
_ifile = self.file_path if ifile is None else ifile
empty_return = (None, None, None) if return_header else (None, None)
if not self.reuse_masters:
# User does not want to load masters
msgs.warn('PypeIt will not reuse masters!')
return empty_return
if not os.path.isfile(_ifile):
# Master file doesn't exist
msgs.warn('No Master {0} frame found: {1}'.format(
self.master_type, self.file_path))
return empty_return
# Read and return
msgs.info('Loading Master {0} frame: {1}'.format(
self.master_type, _ifile))
return self.load_from_file(_ifile, return_header=return_header)
def save_master(self, tilts_dict, outfile=None, steps=None, overwrite=True):
"""
Save the tilts dict to a MasterFile
Args:
tilts_dict (dict): To be saved
outfile (str):
Returns:
"""
_outfile = self.ms_name if outfile is None else outfile
# Additional keywords for the Header
keywds = None if steps is None else dict(steps=','.join(steps))
# Check for existing
if os.path.exists(_outfile) and (not overwrite):
msgs.warn("This file already exists. Use overwrite=True to overwrite it")
return
#
msgs.info("Saving master {0:s} frame as:".format(self.frametype) + msgs.newline() + _outfile)
hdu0 = fits.PrimaryHDU(tilts_dict['tilts'])
hdu0.name='TILTS'
hdu0.header['FUNC2D'] = tilts_dict['func2d']
hdu0.header['NSLIT'] = tilts_dict['nslit']
hdul = [hdu0]
hdu_coeff = fits.ImageHDU(tilts_dict['coeffs'])
hdu_coeff.name='COEFFS'
hdul.append(hdu_coeff)
hdu_slitcen = fits.ImageHDU(tilts_dict['slitcen'])
hdu_slitcen.name = 'SLITCEN'
hdul.append(hdu_slitcen)
hdu_spat_order = fits.ImageHDU(tilts_dict['spat_order'])
hdu_spat_order.name = 'SPAT_ORDER'
hdul.append(hdu_spat_order)
hdu_spec_order = fits.ImageHDU(tilts_dict['spec_order'])
hdu_spec_order.name = 'SPEC_ORDER'
hdul.append(hdu_spec_order)
# Finish
hdulist = fits.HDUList(hdul)
hdulist.writeto(_outfile, clobber=True)
def get_tiltimg(self):
"""
Load or generate the Tilt image
Requirements:
master_key, det, par
Args:
Returns:
ndarray: :attr:`mstilt` image
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
tilt_files, self.master_key_dict['tilt'] = self._prep_calibrations('tilt')
masterframe_name = masterframe.construct_file_name(
buildimage.TiltImage, self.master_key_dict['tilt'], master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.mstilt = buildimage.TiltImage.from_file(masterframe_name)
elif len(tilt_files) == 0:
msgs.warn("No frametype=tilt files to build tiltimg")
return
else: # Build
msgs.info("Preparing a master {0:s} frame".format(buildimage.TiltImage.master_type))
self.mstilt = buildimage.buildimage_fromlist(self.spectrograph, self.det,
self.par['tiltframe'],
tilt_files, bias=self.msbias, bpm=self.msbpm,
slits=self.slits) # For flexure
# Save to Masters
self.mstilt.to_master_file(masterframe_name)
# TODO in the future add in a tilt_inmask
#self._update_cache('tilt', 'tilt_inmask', self.mstilt_inmask)
# Return
return self.mstilt
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype == 'science':
return good_exp & (fitstbl['idname'] == 'OBJECT')
if ftype == 'standard':
return good_exp & (fitstbl['idname'] == 'OBJECT')
if ftype == 'pixelflat' or ftype == 'trace':
# Flats and trace frames are typed together
return good_exp & (fitstbl['idname'] == 'FLAT')
if ftype == 'pinhole' or ftype == 'dark' or ftype == 'bias':
# Don't type pinhole, dark, or bias frames
return np.zeros(len(fitstbl), dtype=bool)
if ftype == 'arc':
return good_exp & (fitstbl['idname'] == 'ARC')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def get_result(self):
"""Save a mask containing the skysub regions, and print information
for what the user should include in their .pypeit file
"""
# Only do this if the user wishes to save the result
if self._use_updates:
# Generate the mask
inmask = skysub.generate_mask(self.pypeline, self._skyreg,
self.slits, self.slits_left,
self.slits_right)
# Save the mask
outfil = self._outname
if os.path.exists(self._outname) and not self._overwrite:
outfil = 'temp.fits'
msgs.warn("File exists:\n{0:s}\nSaving regions to 'temp.fits'")
self._overwrite = True
msskyreg = buildimage.SkyRegions(image=inmask.astype(np.float),
PYP_SPEC=self.spectrograph)
msskyreg.to_master_file(master_filename=outfil)
return
def subtract_overscan(self, force=False):
"""
Analyze and subtract the overscan from the image
Args:
force (bool, optional):
Force the processing even if the image was already processed
"""
step = inspect.stack()[0][3]
# Check if already trimmed
if self.steps[step] and (not force):
msgs.warn("Image was already trimmed")
temp = procimg.subtract_overscan(self.image, self.rawdatasec_img, self.oscansec_img,
method=self.par['overscan'],
params=self.par['overscan_par'])
# Fill
self.steps[step] = True
self.image = temp
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype == 'science':
return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'open')
if ftype == 'bias':
return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'closed')
if ftype in ['pixelflat', 'trace']:
# Flats and trace frames are typed together
return good_exp & self.lamps(fitstbl, 'dome') & (fitstbl['hatch'] == 'open')
if ftype in ['pinhole', 'dark']:
# Don't type pinhole or dark frames
return np.zeros(len(fitstbl), dtype=bool)
if ftype in ['arc', 'tilt']:
return good_exp & self.lamps(fitstbl, 'arcs') & (fitstbl['hatch'] == 'closed')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype in ['science', 'standard']:
return good_exp & (fitstbl['idname'] == 'OBJECT') & (fitstbl['lamps'] == 'Parking') \
& (fitstbl['dispname'] != 'OPEN')
if ftype == 'bias':
return good_exp & (fitstbl['dispname'] == 'OPEN')
if ftype in ['pixelflat', 'trace']:
return good_exp & (fitstbl['idname'] == 'CALIB') & (fitstbl['lamps'] == 'Halogen') \
& (fitstbl['dispname'] != 'OPEN')
if ftype in ['pinhole', 'dark']:
# Don't type pinhole or dark frames
return np.zeros(len(fitstbl), dtype=bool)
if ftype in ['arc', 'tilt']:
return good_exp & (fitstbl['idname'] == 'arc') & (fitstbl['lamps'] == 'Ne+Hg') \
& (fitstbl['dispname'] != 'OPEN')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype == 'science':
return good_exp & (fitstbl['lampstat01'] == 'off') & (
fitstbl['lampstat02']
== 'stowed') & (fitstbl['exptime'] > 100.0)
if ftype == 'standard':
return good_exp & (fitstbl['lampstat01'] == 'off') & (
fitstbl['lampstat02']
== 'stowed') & (fitstbl['exptime'] <= 100.0)
if ftype in ['arc', 'tilt']:
return good_exp & (fitstbl['lampstat01'] == 'on')
if ftype in ['pixelflat', 'trace']:
return good_exp & (fitstbl['lampstat01'] == 'off') & (
fitstbl['lampstat02'] == 'deployed')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype in ['science', 'standard']:
return good_exp & (fitstbl['idname'] == 'OBJECT') & (fitstbl['ra'] != 'none') \
& (fitstbl['dispname'] != 'Flat')
if ftype == 'bias':
return good_exp & (fitstbl['idname'] == 'BIAS')
if ftype in ['pixelflat', 'trace']:
# Flats and trace frames are typed together
return good_exp & (fitstbl['idname'] == 'FLAT') & (fitstbl['decker'] != 'Imaging')
if ftype in ['pinhole', 'dark']:
# Don't type pinhole or dark frames
return np.zeros(len(fitstbl), dtype=bool)
if ftype in ['arc', 'tilt']:
return good_exp & (fitstbl['idname'] == 'COMP') & (fitstbl['dispname'] != 'Flat')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def find_single_file(file_pattern):
"""Find a single file matching a wildcard pattern.
Args:
file_pattern (str): A filename pattern, see the python 'glob' module.
Returns:
str: A file name, or None if no filename was found. This will give a warning
if multiple files are found and return the first one.
"""
files = glob(file_pattern)
if len(files) == 1:
return files[0]
elif len(files) == 0:
return None
else:
msgs.warn(
f'Found multiple files matching {file_pattern}; using the first one.'
)
return files[0]
def _parse(cls,
hdu,
ext=None,
transpose_table_arrays=False,
debug=False,
hdu_prefix=None):
# Grab everything but the bspline's
_d, dm_version_passed, dm_type_passed, parsed_hdus = super(
WaveCalib, cls)._parse(hdu)
# Now the wave_fits
list_of_wave_fits = []
spat_ids = []
for ihdu in hdu:
if 'WAVEFIT' in ihdu.name:
# Allow for empty
if len(ihdu.data) == 0:
iwavefit = wv_fitting.WaveFit(ihdu.header['SPAT_ID'])
else:
iwavefit = wv_fitting.WaveFit.from_hdu(ihdu)
parsed_hdus += ihdu.name
if iwavefit.version != wv_fitting.WaveFit.version:
msgs.warn("Your WaveFit is out of date!!")
# Grab PypeItFit (if it exists)
hdname = ihdu.name.replace('WAVEFIT', 'PYPEITFIT')
if hdname in [khdu.name for khdu in hdu]:
iwavefit.pypeitfit = fitting.PypeItFit.from_hdu(
hdu[hdname])
parsed_hdus += hdname
list_of_wave_fits.append(iwavefit)
# Grab SPAT_ID for checking
spat_ids.append(iwavefit.spat_id)
elif ihdu.name == 'PYPEITFIT': # 2D fit
_d['wv_fit2d'] = fitting.PypeItFit.from_hdu(ihdu)
parsed_hdus += ihdu.name
# Check
if spat_ids != _d['spat_ids'].tolist():
msgs.error("Bad parsing of the MasterFlat")
# Finish
_d['wv_fits'] = np.asarray(list_of_wave_fits)
return _d, dm_version_passed, dm_type_passed, parsed_hdus
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
Args:
ftype (:obj:`str`):
Type of frame to check. Must be a valid frame type; see
frame-type :ref:`frame_type_defs`.
fitstbl (`astropy.table.Table`_):
The table with the metadata for one or more frames to check.
exprng (:obj:`list`, optional):
Range in the allowed exposure time for a frame of type
``ftype``. See
:func:`pypeit.core.framematch.check_frame_exptime`.
Returns:
`numpy.ndarray`_: Boolean array with the flags selecting the
exposures in ``fitstbl`` that are ``ftype`` type frames.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
# TODO: Allow for 'sky' frame type, for now include sky in
# 'science' category
if ftype == 'science':
return good_exp & (fitstbl['idname'] == 'Object')
if ftype == 'standard':
return good_exp & ((fitstbl['idname'] == 'Std') |
(fitstbl['idname'] == 'Object'))
if ftype == 'bias':
return good_exp & (fitstbl['idname'] == 'Bias')
if ftype == 'dark':
return good_exp & (fitstbl['idname'] == 'Dark')
if ftype in ['pixelflat', 'trace']:
# Flats and trace frames are typed together
return good_exp & ((fitstbl['idname'] == 'Flat') |
(fitstbl['idname'] == 'IntFlat'))
if ftype in ['arc', 'tilt']:
return good_exp & (fitstbl['idname'] == 'Line')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def load_extinction_data(longitude, latitude, toler=5. * units.deg):
"""
Find the best extinction file to use, based on longitude and latitude
Loads it and returns a Table
Parameters
----------
toler : Angle, optional
Tolerance for matching detector to site (5 deg)
Returns
-------
ext_file : Table
astropy Table containing the 'wavelength', 'extinct' data for AM=1.
"""
# Mosaic coord
mosaic_coord = coordinates.SkyCoord(longitude, latitude, frame='gcrs', unit=units.deg)
# Read list
extinct_path = resource_filename('pypeit', '/data/extinction/')
extinct_summ = extinct_path + 'README'
extinct_files = table.Table.read(extinct_summ, comment='#', format='ascii')
# Coords
ext_coord = coordinates.SkyCoord(extinct_files['Lon'], extinct_files['Lat'], frame='gcrs',
unit=units.deg)
# Match
idx, d2d, d3d = coordinates.match_coordinates_sky(mosaic_coord, ext_coord, nthneighbor=1)
if d2d < toler:
extinct_file = extinct_files[int(idx)]['File']
msgs.info("Using {:s} for extinction corrections.".format(extinct_file))
else:
msgs.warn("No file found for extinction corrections. Applying none")
msgs.warn("You should generate a site-specific file")
return None
# Read
extinct = table.Table.read(extinct_path + extinct_file, comment='#', format='ascii',
names=('iwave', 'mag_ext'))
wave = table.Column(np.array(extinct['iwave']) * units.AA, name='wave')
extinct.add_column(wave)
# Return
return extinct[['wave', 'mag_ext']]
def save(self, outfile=None, overwrite=True):
"""
Save the bias master data.
Args:
outfile (:obj:`str`, optional):
Name for the output file. Defaults to
:attr:`file_path`.
overwrite (:obj:`bool`, optional):
Overwrite any existing file.
"""
# Some checks
if self.pypeitImage is None:
msgs.warn('No MasterBias to save!')
return
if not self.pypeitImage.validate():
msgs.warn('MasterBias is not a proper image.')
return
# Proceed
_outfile = self.master_file_path if outfile is None else outfile
# Check if it exists
if os.path.exists(_outfile) and not overwrite:
msgs.warn('Master file exists: {0}'.format(_outfile) + msgs.newline()
+ 'Set overwrite=True to overwrite it.')
return
# Save
hdr = self.build_master_header(steps=self.process_steps, raw_files=self.file_list)
self.pypeitImage.write(_outfile, hdr=hdr, iext='BIAS')
msgs.info('Master frame written to {0}'.format(_outfile))
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
Args:
ftype (:obj:`str`):
Type of frame to check. Must be a valid frame type; see
frame-type :ref:`frame_type_defs`.
fitstbl (`astropy.table.Table`_):
The table with the metadata for one or more frames to check.
exprng (:obj:`list`, optional):
Range in the allowed exposure time for a frame of type
``ftype``. See
:func:`pypeit.core.framematch.check_frame_exptime`.
Returns:
`numpy.ndarray`_: Boolean array with the flags selecting the
exposures in ``fitstbl`` that are ``ftype`` type frames.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype == 'bias':
return fitstbl['idname'] == 'zero'
if ftype in ['science', 'standard']:
return good_exp & (fitstbl['lampstat01']
== 'off') & (fitstbl['idname'] == 'object')
if ftype in ['arc', 'tilt']:
# should flesh this out to include all valid arc lamp combos
return good_exp & (fitstbl['lampstat01'] != 'off') & (
fitstbl['idname'] == 'comp') & (fitstbl['decker'] != '5.0x180')
if ftype in ['trace', 'pixelflat']:
# i think the bright lamp, BC, is the only one ever used for this. imagetyp should always be set to flat.
return good_exp & (fitstbl['lampstat01']
== 'BC') & (fitstbl['idname'] == 'flat')
if ftype in ['illumflat']:
# these can be set to flat or object depending if they're twilight or dark sky
return good_exp & (fitstbl['idname'] in ['flat', 'object']) & (
fitstbl['lampstat01'] == 'off')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def check_frame_type(self, ftype, fitstbl, exprng=None):
"""
Check for frames of the provided type.
Args:
ftype (:obj:`str`):
Type of frame to check. Must be a valid frame type; see
frame-type :ref:`frame_type_defs`.
fitstbl (`astropy.table.Table`_):
The table with the metadata for one or more frames to check.
exprng (:obj:`list`, optional):
Range in the allowed exposure time for a frame of type
``ftype``. See
:func:`pypeit.core.framematch.check_frame_exptime`.
Returns:
`numpy.ndarray`_: Boolean array with the flags selecting the
exposures in ``fitstbl`` that are ``ftype`` type frames.
"""
good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng)
if ftype == 'science':
return good_exp & (fitstbl['idname'] == 'OBJECT')
if ftype == 'standard':
return good_exp & (fitstbl['idname'] == 'OBJECT')
if ftype in ['pixelflat', 'trace']:
# Flats and trace frames are typed together
return good_exp & (fitstbl['idname'] == 'FLAT')
if ftype in ['pinhole', 'dark', 'bias']:
# Don't type pinhole, dark, or bias frames
return np.zeros(len(fitstbl), dtype=bool)
if ftype in ['arc', 'tilt']:
## FW ToDo: self.dispname does not work yet. need to replace the following later.
if '32/mm' in fitstbl['dispname'][0]:
return good_exp & (fitstbl['idname'] == 'OBJECT')
elif '10/mmLBSX' in fitstbl['dispname'][0]:
return good_exp & (fitstbl['idname'] == 'ARC')
msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype))
return np.zeros(len(fitstbl), dtype=bool)
def parse_binning(binning):
"""
Parse input binning into binspectral, binspatial
Note that for some instruments, the meaning will be swapped if
parsed directly from the Header. The developer needs to react accordingly..
Args:
binning (str, ndarray or tuple):
Returns:
int,int: binspectral, binspatial
"""
# comma separated format
if isinstance(binning, str):
if ',' in binning:
binspectral, binspatial = [
int(item) for item in binning.split(',')
] # Keck standard, I think
elif 'x' in binning:
binspectral, binspatial = [
int(item) for item in binning.split('x')
] # LRIS
elif binning == 'None':
msgs.warn("Assuming unbinned, i.e. 1x1")
binspectral, binspatial = 1, 1
else:
binspectral, binspatial = [
int(item) for item in binning.strip().split(' ')
] # Gemini
elif isinstance(binning, tuple):
binspectral, binspatial = binning
elif isinstance(binning, np.ndarray):
binspectral, binspatial = binning
else:
msgs.error("Unable to parse input binning: {}".format(binning))
# Return
return binspectral, binspatial
def xcorr_shift_stretch(inspec1, inspec2, smooth = 5.0, shift_mnmx = (-0.05,0.05), stretch_mnmx = (0.9,1.1), debug = True):
nspec = inspec1.size
y1 = scipy.ndimage.filters.gaussian_filter(inspec1, smooth)
y2 = scipy.ndimage.filters.gaussian_filter(inspec2, smooth)
# Do the cross-correlation first and determine the
shift_cc, cc_val = xcorr_shift(y1, y2, debug = debug)
bounds = [(shift_cc + nspec*shift_mnmx[0],shift_cc + nspec*shift_mnmx[1]), stretch_mnmx]
#guess = np.array([shift_cc,1.0])
#result = scipy.optimize.minimize(xcorr_shift_stretch, guess, args=(y1,y2), bounds=bounds)
result = scipy.optimize.differential_evolution(zerolag_shift_stretch, args=(y1,y2), tol = 1e-4,
bounds=bounds, disp=False, polish=True)
#rranges = (slice(bounds[0][0], bounds[0][1], 0.1), slice(bounds[1][0], bounds[1][1], 1e-4))
#from IPython import embed
#embed()
#result = scipy.optimize.brute(xcorr_shift_stretch, rranges, args=(y1,y2), finish = scipy.optimize.fmin, disp=True)
#corr_val = xcorr_stretch((shift,stretch), y1,y2)
#corr = scipy.signal.correlate(y1, y2, mode='same')
if not result.success:
msgs.warn('Fit for shift and stretch did not converge!')
if debug:
x1 = np.arange(nspec)
inspec2_trans = shift_and_stretch(inspec2, result.x[0], result.x[1])
plt.plot(x1,inspec1, 'k-', drawstyle='steps', label ='inspec1')
plt.plot(x1,inspec2_trans, 'r-', drawstyle='steps', label = 'inspec2')
plt.title('shift= {:5.3f}'.format(result.x[0]) +
', stretch = {:7.5f}'.format(result.x[1]) + ', corr = {:5.3f}'.format(-result.fun))
plt.legend()
plt.show()
return result.success, result.x[0], result.x[1], -result.fun, shift_cc, cc_val
def ech_load_specobj(fname,order=None):
""" Load a spec1d file into a list of SpecObjExp objects
Parameters
----------
fname : str
Returns
-------
specObjs : list of SpecObjExp
head0
"""
#if order is None:
# msgs.warn('You did not specify an order. Return specObjs with all orders.')
# specObjs, head0 = load.load_specobj(fname)
# return specObjs, head0
speckeys = ['WAVE', 'SKY', 'MASK', 'FLAM', 'FLAM_IVAR', 'FLAM_SIG', 'COUNTS_IVAR', 'COUNTS']
#
specObjs = []
hdulist = fits.open(fname)
head0 = hdulist[0].header
for hdu in hdulist:
if hdu.name == 'PRIMARY':
continue
#elif hdu.name[8:17] != 'ORDER'+'{0:04}'.format(order):
# continue
# Parse name
idx = hdu.name
objp = idx.split('-')
if objp[-1][0:3] == 'DET':
det = int(objp[-1][3:])
else:
det = int(objp[-1][1:])
if objp[-2][:5] == 'ORDER':
iord = int(objp[-2][5:])
else:
msgs.warn('Loading longslit data ?')
iord = int(-1)
# if order is not None and iord !=order then do not return this extenction
# if order is None return all extensions
# if order is not None and iord ==order then only return the specific order you want.
if (order is not None) and (iord !=order):
continue
# Load data
spec = Table(hdu.data)
shape = (len(spec), 1024) # 2nd number is dummy
# New and wrong
try:
specobj = specobjs.SpecObj(shape, None, None, idx = idx)
except:
debugger.set_trace()
msgs.error("BUG ME")
# Add order number
specobj.ech_orderindx = iord
# ToDo: need to changed to the real order number?
specobj.ech_order = iord
# Add trace
try:
specobj.trace_spat = spec['TRACE']
except:
# KLUDGE!
specobj.trace_spat = np.arange(len(spec['BOX_WAVE']))
# Add spectrum
if 'BOX_COUNTS' in spec.keys():
for skey in speckeys:
try:
specobj.boxcar[skey] = spec['BOX_{:s}'.format(skey)].data
except KeyError:
pass
# Add units on wave
specobj.boxcar['WAVE'] = specobj.boxcar['WAVE'] * units.AA
if 'OPT_COUNTS' in spec.keys():
for skey in speckeys:
try:
specobj.optimal[skey] = spec['OPT_{:s}'.format(skey)].data
except KeyError:
pass
# Add units on wave
specobj.optimal['WAVE'] = specobj.optimal['WAVE'] * units.AA
# Append
specObjs.append(specobj)
# Return
return specObjs, head0
def pca_trace(xcen, usepca = None, npca = 2, npoly_cen = 3, debug=True):
nspec = xcen.shape[0]
norders = xcen.shape[1]
if usepca is None:
usepca = np.zeros(norders,dtype=bool)
# use_order = True orders used to predict the usepca = True bad orders
use_order = np.invert(usepca)
ngood = np.sum(use_order)
if ngood < npca:
msgs.warn('Not enough good traces for a PCA fit: ngood = {:d}'.format(ngood) + ' is < npca = {:d}'.format(npca))
msgs.warn('Using the input trace for now')
return xcen
pca = PCA(n_components=npca)
xcen_use = (xcen[:,use_order] - np.mean(xcen[:,use_order],0)).T
pca_coeffs_use = pca.fit_transform(xcen_use)
pca_vectors = pca.components_
# Fit first pca dimension (with largest variance) with a higher order npoly depending on number of good orders.
# Fit all higher dimensions (with lower variance) with a line
npoly = int(np.fmin(np.fmax(np.floor(3.3*ngood/norders),1.0),3.0))
npoly_vec = np.full(npca, npoly)
order_vec = np.arange(norders,dtype=float)
# pca_coeffs = np.zeros((norders, npca))
pca_coeffs_new = np.zeros((norders, npca))
# Now loop over the dimensionality of the compression and perform a polynomial fit to
for idim in range(npca):
# ToDO robust_polyfit is garbage remove it entirely from PypeIT!
xfit = order_vec[use_order]
yfit = pca_coeffs_use[:,idim]
norder = npoly_vec[idim]
# msk, poly_coeff = utils.robust_polyfit(xfit, yfit, norder, sigma = 3.0, function='polynomial')
# pca_coeffs[:,idim] = utils.func_val(poly_coeff, order_vec, 'polynomial')
# TESTING traceset fitting
xtemp = xfit.reshape(1, xfit.size)
ytemp = yfit.reshape(1, yfit.size)
tset = pydl.xy2traceset(xtemp, ytemp, ncoeff=norder,func='polynomial')
#tset_yfit = tset.yfit.reshape(tset.yfit.shape[1])
## Test new robust fitting with djs_reject
msk_new, poly_coeff_new = utils.robust_polyfit_djs(xfit, yfit, norder, \
function='polynomial', minv=None, maxv=None, bspline_par=None, \
guesses=None, maxiter=10, inmask=None, sigma=None, invvar=None, \
lower=5, upper=5, maxdev=None, maxrej=None, groupdim=None,
groupsize=None, \
groupbadpix=False, grow=0, sticky=False)
pca_coeffs_new[:,idim] = utils.func_val(poly_coeff_new, order_vec, 'polynomial')
if debug:
# Evaluate the fit
xvec = np.linspace(order_vec.min(),order_vec.max(),num=100)
(_,tset_fit) = tset.xy(xpos=xvec.reshape(1,xvec.size))
yfit_tset = tset_fit[0,:]
#robust_mask = msk == 0
robust_mask_new = msk_new == 1
tset_mask = tset.outmask[0,:]
plt.plot(xfit, yfit, 'ko', mfc='None', markersize=8.0, label='pca coeff')
#plt.plot(xfit[~robust_mask], yfit[~robust_mask], 'ms', mfc='None', markersize=10.0,label='robust_polyfit rejected')
#plt.plot(xfit[~robust_mask_new], yfit[~robust_mask_new], 'r+', markersize=20.0,label='robust_polyfit_djs rejected')
plt.plot(xfit[~tset_mask],yfit[~tset_mask], 'bo', markersize = 10.0, label = 'traceset rejected')
#plt.plot(xvec, utils.func_val(poly_coeff, xvec, 'polynomial'),ls='--', color='m', label='robust polyfit')
plt.plot(xvec, utils.func_val(poly_coeff_new, xvec, 'polynomial'),ls='-.', color='r', label='new robust polyfit')
plt.plot(xvec, yfit_tset,ls=':', color='b',label='traceset')
plt.legend()
plt.show()
#ToDo should we be masking the bad orders here and interpolating/extrapolating?
spat_mean = np.mean(xcen,0)
msk_spat, poly_coeff_spat = utils.robust_polyfit(order_vec, spat_mean, npoly_cen, sigma = 3.0, function = 'polynomial')
ibad = np.where(msk_spat == 1)
spat_mean[ibad] = utils.func_val(poly_coeff_spat,order_vec[ibad],'polynomial')
#pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(pca.mean_,np.ones(norders)) + (np.dot(pca_coeffs, pca_vectors)).T
pca_fit = np.outer(np.ones(nspec), spat_mean) + np.outer(pca.mean_,np.ones(norders)) + (np.dot(pca_coeffs_new, pca_vectors)).T
return pca_fit
def ech_objfind(image, ivar, ordermask, slit_left, slit_righ,inmask=None,plate_scale=0.2,npca=2,ncoeff = 5,min_snr=0.0,nabove_min_snr=0,
pca_percentile=20.0,snr_pca=3.0,box_radius=2.0,show_peaks=False,show_fits=False,show_trace=False):
if inmask is None:
inmask = (ordermask > 0)
frameshape = image.shape
nspec = frameshape[0]
norders = slit_left.shape[1]
if isinstance(plate_scale,(float, int)):
plate_scale_ord = np.full(norders, plate_scale) # 0.12 binned by 3 spatially for HIRES
elif isinstance(plate_scale,(np.ndarray, list, tuple)):
if len(plate_scale) == norders:
plate_scale_ord = plate_scale
elif len(plate_scale) == 1:
plate_scale_ord = np.full(norders, plate_scale[0])
else:
msgs.error('Invalid size for plate_scale. It must either have one element or norders elements')
else:
msgs.error('Invalid type for plate scale')
specmid = nspec // 2
slit_width = slit_righ - slit_left
spec_vec = np.arange(nspec)
slit_spec_pos = nspec/2.0
slit_spat_pos = np.zeros((norders, 2))
for iord in range(norders):
slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec, slit_left[:,iord]), np.interp(slit_spec_pos, spec_vec, slit_righ[:,iord]))
# Loop over orders and find objects
sobjs = specobjs.SpecObjs()
show_peaks=True
show_fits=True
# ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx!
for iord in range(norders):
msgs.info('Finding objects on slit # {:d}'.format(iord + 1))
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
specobj_dict = {'setup': 'HIRES', 'slitid': iord + 1, 'scidx': 0,'det': 1, 'objtype': 'science'}
sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \
extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks,
show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0)
# ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord
for spec in sobjs_slit:
spec.ech_orderindx = iord
sobjs.add_sobj(sobjs_slit)
nfound = len(sobjs)
# Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0"
FOFSEP = 1.0 # separation of FOF algorithm in arcseconds
FOF_frac = FOFSEP/(np.median(slit_width)*np.median(plate_scale_ord))
# Feige: made the code also works for only one object found in one order
# Run the FOF. We use fake coordinaes
fracpos = sobjs.spat_fracpos
ra_fake = fracpos/1000.0 # Divide all angles by 1000 to make geometry euclidian
dec_fake = 0.0*fracpos
if nfound>1:
(ingroup, multgroup, firstgroup, nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac/1000.0)
group = ingroup.copy()
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(uni_group)
msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects')
elif nfound==1:
group = np.zeros(1,dtype='int')
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(group)
msgs.warn('Only find one object no FOF matching is needed')
gfrac = np.zeros(nfound)
for jj in range(nobj):
this_group = group == uni_group[jj]
gfrac[this_group] = np.median(fracpos[this_group])
uni_frac = gfrac[uni_ind]
sobjs_align = sobjs.copy()
# Now fill in the missing objects and their traces
for iobj in range(nobj):
for iord in range(norders):
# Is there an object on this order that grouped into the current group in question?
on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx == iord)
if not np.any(on_slit):
# Add this to the sobjs_align, and assign required tags
thisobj = specobjs.SpecObj(frameshape, slit_spat_pos[iord,:], slit_spec_pos, det = sobjs_align[0].det,
setup = sobjs_align[0].setup, slitid = (iord + 1),
scidx = sobjs_align[0].scidx, objtype=sobjs_align[0].objtype)
thisobj.ech_orderindx = iord
thisobj.spat_fracpos = uni_frac[iobj]
thisobj.trace_spat = slit_left[:,iord] + slit_width[:,iord]*uni_frac[iobj] # new trace
thisobj.trace_spec = spec_vec
thisobj.spat_pixpos = thisobj.trace_spat[specmid]
thisobj.set_idx()
# Use the real detections of this objects for the FWHM
this_group = group == uni_group[iobj]
# Assign to the fwhm of the nearest detected order
imin = np.argmin(np.abs(sobjs_align[this_group].ech_orderindx - iord))
thisobj.fwhm = sobjs_align[imin].fwhm
thisobj.maskwidth = sobjs_align[imin].maskwidth
thisobj.ech_fracpos = uni_frac[iobj]
thisobj.ech_group = uni_group[iobj]
thisobj.ech_usepca = True
sobjs_align.add_sobj(thisobj)
group = np.append(group, uni_group[iobj])
gfrac = np.append(gfrac, uni_frac[iobj])
else:
# ToDo fix specobjs to get rid of these crappy loops!
for spec in sobjs_align[on_slit]:
spec.ech_fracpos = uni_frac[iobj]
spec.ech_group = uni_group[iobj]
spec.ech_usepca = False
# Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order
# respectively
sobjs_sort = specobjs.SpecObjs()
for iobj in range(nobj):
this_group = group == uni_group[iobj]
this_sobj = sobjs_align[this_group]
sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)])
# Loop over the objects and perform a quick and dirty extraction to assess S/N.
varimg = utils.calc_ivar(ivar)
flux_box = np.zeros((nspec, norders, nobj))
ivar_box = np.zeros((nspec, norders, nobj))
mask_box = np.zeros((nspec, norders, nobj))
SNR_arr = np.zeros((norders, nobj))
for iobj in range(nobj):
for iord in range(norders):
indx = (sobjs_sort.ech_group == uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord)
spec = sobjs_sort[indx]
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
box_rad_pix = box_radius/plate_scale_ord[iord]
flux_tmp = extract.extract_boxcar(image*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
var_tmp = extract.extract_boxcar(varimg*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
ivar_tmp = utils.calc_ivar(var_tmp)
pixtot = extract.extract_boxcar(ivar*0 + 1.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
mask_tmp = (extract.extract_boxcar(ivar*inmask_iord == 0.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) != pixtot)
flux_box[:,iord,iobj] = flux_tmp*mask_tmp
ivar_box[:,iord,iobj] = np.fmax(ivar_tmp*mask_tmp,0.0)
mask_box[:,iord,iobj] = mask_tmp
(mean, med_sn, stddev) = sigma_clipped_stats(flux_box[mask_tmp,iord,iobj]*np.sqrt(ivar_box[mask_tmp,iord,iobj]),
sigma_lower=5.0,sigma_upper=5.0)
SNR_arr[iord,iobj] = med_sn
# Purge objects with low SNR and that don't show up in enough orders
keep_obj = np.zeros(nobj,dtype=bool)
sobjs_trim = specobjs.SpecObjs()
uni_group_trim = np.array([],dtype=int)
uni_frac_trim = np.array([],dtype=float)
for iobj in range(nobj):
if (np.sum(SNR_arr[:,iobj] > min_snr) >= nabove_min_snr):
keep_obj[iobj] = True
ikeep = sobjs_sort.ech_group == uni_group[iobj]
sobjs_trim.add_sobj(sobjs_sort[ikeep])
uni_group_trim = np.append(uni_group_trim, uni_group[iobj])
uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj])
else:
msgs.info('Purging object #{:d}'.format(iobj) + ' which does not satisfy min_snr > {:5.2f}'.format(min_snr) +
' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) + ' orders')
nobj_trim = np.sum(keep_obj)
if nobj_trim == 0:
return specobjs.SpecObjs()
SNR_arr_trim = SNR_arr[:,keep_obj]
# Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA
for iobj in range(nobj_trim):
SNR_now = SNR_arr_trim[:,iobj]
indx = (sobjs_trim.ech_group == uni_group_trim[iobj])
# PCA interp/extrap if:
# (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca)
# OR
# (if this order was not originally traced by the object finding, see above)
usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) & (SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca
# ToDo fix specobjs to get rid of these crappy loops!
for iord, spec in enumerate(sobjs_trim[indx]):
spec.ech_usepca = usepca[iord]
if usepca[iord]:
msgs.info('Using PCA to predict trace for object #{:d}'.format(iobj) + ' on order #{:d}'.format(iord))
sobjs_final = sobjs_trim.copy()
# Loop over the objects one by one and adjust/predict the traces
npoly_cen = 3
pca_fits = np.zeros((nspec, norders, nobj_trim))
for iobj in range(nobj_trim):
igroup = sobjs_final.ech_group == uni_group_trim[iobj]
# PCA predict the masked orders which were not traced
pca_fits[:,:,iobj] = pca_trace((sobjs_final[igroup].trace_spat).T, usepca = None, npca = npca, npoly_cen = npoly_cen)
# usepca = sobjs_final[igroup].ech_usepca,
# Perform iterative flux weighted centroiding using new PCA predictions
xinit_fweight = pca_fits[:,:,iobj].copy()
inmask_now = inmask & (ordermask > 0)
xfit_fweight = extract.iter_tracefit(image, xinit_fweight, ncoeff, inmask = inmask_now, show_fits=show_fits)
# Perform iterative Gaussian weighted centroiding
xinit_gweight = xfit_fweight.copy()
xfit_gweight = extract.iter_tracefit(image, xinit_gweight, ncoeff, inmask = inmask_now, gweight=True,show_fits=show_fits)
# Assign the new traces
for iord, spec in enumerate(sobjs_final[igroup]):
spec.trace_spat = xfit_gweight[:,iord]
spec.spat_pixpos = spec.trace_spat[specmid]
# Set the IDs
sobjs_final.set_idx()
if show_trace:
viewer, ch = ginga.show_image(objminsky*(ordermask > 0))
for iobj in range(nobj_trim):
for iord in range(norders):
ginga.show_trace(viewer, ch, pca_fits[:,iord, iobj], str(uni_frac[iobj]), color='yellow')
for spec in sobjs_trim:
color = 'green' if spec.ech_usepca else 'magenta'
ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)
#for spec in sobjs_final:
# color = 'red' if spec.ech_usepca else 'green'
# ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)
return sobjs_final
