Esempio n. 1
0
    def MasterBias(self, fitsdict, det):
        """
        Generate Master Bias frame for a given detector

        Parameters
        ----------
        fitsdict : dict
          Contains relevant information from fits header files
        det : int
          Index of the detector

        Returns
        -------
        boolean : bool
          Should other ScienceExposure classes be updated?
        """

        # If the master bias is already made, use it
        if self._msbias[det - 1] is not None:
            msgs.info("An identical master {0:s} frame already exists".format(
                settings.argflag['bias']['useframe']))
            return False
        elif settings.argflag['bias']['useframe'] in ['bias', 'dark']:
            try:
                msbias = armasters.get_master_frame(self, "bias")
            except IOError:
                msgs.info("Preparing a master {0:s} frame".format(
                    settings.argflag['bias']['useframe']))
                # Get all of the bias frames for this science frame
                ind = self._idx_bias
                # Load the Bias/Dark frames
                frames = arload.load_frames(
                    fitsdict,
                    ind,
                    det,
                    frametype=settings.argflag['bias']['useframe'])
                msbias = arcomb.comb_frames(
                    frames,
                    det,
                    'bias',
                    printtype=settings.argflag['bias']['useframe'])
                del frames
        elif settings.argflag['bias']['useframe'] == 'overscan':
            self.SetMasterFrame('overscan', "bias", det, mkcopy=False)
            return False
        elif settings.argflag['bias']['useframe'] == 'none':
            msgs.info("Not performing a bias/dark subtraction")
            self.SetMasterFrame(None, "bias", det, mkcopy=False)
            return False
        else:  # It must be the name of a file the user wishes to load
            msbias_name = settings.argflag['run']['directory'][
                'master'] + u'/' + settings.argflag['bias']['useframe']
            msbias, head = arload.load_master(msbias_name, frametype="bias")
            settings.argflag['reduce']['masters']['loaded'].append('bias')
        # Set and then delete the Master Bias frame
        self.SetMasterFrame(msbias, "bias", det)
        armasters.save_masters(self, det, mftype='bias')

        del msbias
        return True
Esempio n. 2
0
    def MasterWaveCalib(self, fitsdict, sc, det):
        """
        Generate Master 1D Wave Solution (down slit center)

        Parameters
        ----------
        fitsdict : dict
          Contains relevant information from fits header files
        det : int
          Index of the detector

        Returns
        -------
        boolean : bool
          Should other ScienceExposure classes be updated?
        """
        from pypit import ararc

        if self._wvcalib[det - 1] is not None:
            msgs.info("An identical master wave calib frame already exists")
            return False
        else:
            wv_calib = None
        # Attempt to load the Master Frame
        try:
            wv_calib = armasters.get_master_frame(self, "wv_calib")
        except IOError:
            if settings.argflag["reduce"]["calibrate"][
                    "wavelength"] == "pixel":
                msgs.info("A wavelength calibration will not be performed")
            else:
                # Setup arc parameters (e.g. linelist)
                arcparam = ararc.setup_param(self, sc, det, fitsdict)
                self.SetFrame(self._arcparam, arcparam, det)
                ###############
                # Extract arc and identify lines
                if settings.argflag['arc']['calibrate']['method'] == 'simple':
                    wv_calib = ararc.simple_calib(self, det)
                elif settings.argflag['arc']['calibrate'][
                        'method'] == 'arclines':
                    wv_calib = ararc.calib_with_arclines(self, det)
        # Set
        if wv_calib is not None:
            self.SetFrame(self._wvcalib, wv_calib, det)
            armasters.save_masters(self, det, mftype='wv_calib')
        del wv_calib
        return True
Esempio n. 3
0
 def close(self):
     """
     Close the log file and QA PDFs before the code exits
     """
     # Close PDFs
     try:
         self.sciexp._qa.close()
     except AttributeError:
         pass
     else:
         if self._debug['develop']:
             from pypit import armasters
             armasters.save_masters(self.sciexp, self.sciexp.det,
                                self.sciexp._argflag['masters']['setup'])
     # Close log
     if self._log:
         self._log.close()
Esempio n. 4
0
    def BadPixelMask(self, fitsdict, det):
        """
        Generate Bad Pixel Mask for a given detector

        Parameters
        ----------
        fitsdict : dict
          Contains relevant information from fits header files
        det : int
          Index of the detector

        Returns
        -------
        boolean : bool
          Should other ScienceExposure classes be updated?
        """
        bpix = None
        if settings.argflag['reduce']['badpix'] == 'bias':
            try:
                bpix = armasters.get_master_frame(self, "badpix")
            except IOError:
                msgs.info("Preparing a bad pixel mask")
                # Get all of the bias frames for this science frame
                if len(self._idx_bias) == 0:
                    msgs.warn(
                        "No bias frames available to determine bad pixel mask")
                    msgs.info("Not preparing a bad pixel mask")
                    return False
                # Load the Bias frames
                bpix = arproc.badpix(self, det,
                                     self.GetMasterFrame('bias', det))
        else:
            # Instrument dependent
            if settings.argflag['run']['spectrograph'] in ['keck_lris_red']:
                bpix = arlris.bpm(self, 'red', fitsdict, det)
            else:
                msgs.info("Not preparing a bad pixel mask")
                return False
        # Save
        self.SetFrame(self._bpix, bpix, det)
        armasters.save_masters(self, det, mftype='badpix')
        del bpix
        return True
Esempio n. 5
0
    def MasterWave(self, fitsdict, sc, det):
        """
        Generate Master Wave frame for a given detector

        Parameters
        ----------
        fitsdict : dict
          Contains relevant information from fits header files
        det : int
          Index of the detector

        Returns
        -------
        boolean : bool
          Should other ScienceExposure classes be updated?
        """
        if self._mswave[det - 1] is not None:
            msgs.info("An identical master arc frame already exists")
            return False
        try:
            mswave = armasters.get_master_frame(self, "wave")
        except IOError:
            msgs.info("Preparing a master wave frame")
            if settings.argflag["reduce"]["calibrate"][
                    "wavelength"] == "pixel":
                mswave = self._tilts[det -
                                     1] * (self._tilts[det - 1].shape[0] - 1.0)
            else:
                wv_calib = self._wvcalib[det - 1]
                mswave = arutils.func_val(wv_calib['fitc'],
                                          self._tilts[det - 1],
                                          wv_calib['function'],
                                          minv=wv_calib['fmin'],
                                          maxv=wv_calib['fmax'])
        # Set and then delete the Master Arc frame
        self.SetMasterFrame(mswave, "wave", det)
        armasters.save_masters(self, det, mftype='wave')
        del mswave
        return True
Esempio n. 6
0
def ARMED(fitsdict, reuseMaster=False, reloadMaster=True):
    """
    Automatic Reduction and Modeling of Echelle Data

    Parameters
    ----------
    fitsdict : dict
      Contains relevant information from fits header files
    reuseMaster : bool
      If True, a master frame that will be used for another science frame
      will not be regenerated after it is first made.
      This setting comes with a price, and if a large number of science frames are
      being generated, it may be more efficient to simply regenerate the master
      calibrations on the fly.

    Returns
    -------
    status : int
      Status of the reduction procedure
      0 = Successful execution
      1 = ...
    """
    status = 0

    # Create a list of science exposure classes
    sciexp, setup_dict = armbase.SetupScience(fitsdict)
    if sciexp == 'setup':
        status = 1
        return status
    elif sciexp == 'calcheck':
        status = 2
        return status
    else:
        numsci = len(sciexp)

    # Create a list of master calibration frames
    #masters = armasters.MasterFrames(settings.spect['mosaic']['ndet'])

    # Masters
    #settings.argflag['reduce']['masters']['file'] = setup_file

    # Start reducing the data
    for sc in range(numsci):
        slf = sciexp[sc]
        scidx = slf._idx_sci[0]
        msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name))
        msgs.sciexp = slf  # For QA writing on exit, if nothing else.  Could write Masters too
        if reloadMaster and (sc > 0):
            settings.argflag['reduce']['masters']['reuse'] = True
        # Loop on Detectors
        for kk in range(settings.spect['mosaic']['ndet']):
            det = kk + 1  # Detectors indexed from 1
            if settings.argflag['reduce']['detnum'] is not None:
                if det != settings.argflag['reduce']['detnum']:
                    continue
                else:
                    msgs.warn("Restricting the reduction to detector {:d}".format(det))
            slf.det = det
            ###############
            # Get data sections
            arproc.get_datasec_trimmed(slf, fitsdict, det, scidx)
            # Setup
            setup = arsort.instr_setup(slf, det, fitsdict, setup_dict, must_exist=True)
            settings.argflag['reduce']['masters']['setup'] = setup
            slf.setup = setup
            ###############
            # Generate master bias frame
            update = slf.MasterBias(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
            ###############
            # Generate a bad pixel mask (should not repeat)
            update = slf.BadPixelMask(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
            ###############
            # Estimate gain and readout noise for the amplifiers
            msgs.work("Estimate Gain and Readout noise from the raw frames...")
            ###############
            # Generate a master arc frame
            update = slf.MasterArc(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
            ###############
            # Set the number of spectral and spatial pixels, and the bad pixel mask is it does not exist
            slf._nspec[det-1], slf._nspat[det-1] = slf._msarc[det-1].shape
            if slf._bpix[det-1] is None:
                slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det)
            ###############
            # Generate a master trace frame
            update = slf.MasterTrace(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
            ###############
            # Generate a master pinhole frame
            update = slf.MasterPinhole(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pinhole")
            ###############
            # Generate an array that provides the physical pixel locations on the detector
            slf.GetPixelLocations(det)
            ###############
            # Determine the edges of the spectrum (spatial)
            if ('trace'+settings.argflag['reduce']['masters']['setup'] not in settings.argflag['reduce']['masters']['loaded']):
                if True:#not msgs._debug['develop']:
                    msgs.info("Tracing slit edges with a {0:s} frame".format(settings.argflag['trace']['useframe']))
                    if settings.argflag['trace']['useframe'] == 'pinhole':
                        ###############
                        # Determine the centroid of the spectrum (spatial)
                        lordloc, rordloc, extord = artrace.trace_slits(slf, slf._mspinhole[det-1], det,
                                                                       pcadesc="PCA trace of the slit edges")

                        # Using the order centroid, expand the order edges until the edge of the science slit is found
                        if settings.argflag['trace']['slits']['expand']:
                            lordloc, rordloc = artrace.expand_slits(slf, slf._mstrace[det-1], det,
                                                                    0.5*(lordloc+rordloc), extord)
                    elif settings.argflag['trace']['useframe'] == 'trace':
                        ###############
                        # Determine the edges of the slit using a trace frame
                        lordloc, rordloc, extord = artrace.trace_slits(slf, slf._mstrace[det-1], det,
                                                                       pcadesc="PCA trace of the slit edges")
                    else:
                        msgs.error("Cannot trace slit edges using {0:s}".format(settings.argflag['trace']['useframe']))
                else:
                    lordloc, rordloc, extord = np.load("lordloc.npy"), np.load("rordloc.npy"), np.load("extord.npy")

                # Save the locations of the order edges
                slf.SetFrame(slf._lordloc, lordloc, det)
                slf.SetFrame(slf._rordloc, rordloc, det)

                # Convert physical trace into a pixel trace
                msgs.info("Converting physical trace locations to nearest pixel")
                pixcen = artrace.phys_to_pix(0.5 * (slf._lordloc[det - 1] + slf._rordloc[det - 1]), slf._pixlocn[det - 1], 1)
                pixwid = (slf._rordloc[det - 1] - slf._lordloc[det - 1]).mean(0).astype(np.int)
                lordpix = artrace.phys_to_pix(slf._lordloc[det - 1], slf._pixlocn[det - 1], 1)
                rordpix = artrace.phys_to_pix(slf._rordloc[det - 1], slf._pixlocn[det - 1], 1)
                slf.SetFrame(slf._pixcen, pixcen, det)
                slf.SetFrame(slf._pixwid, pixwid, det)
                slf.SetFrame(slf._lordpix, lordpix, det)
                slf.SetFrame(slf._rordpix, rordpix, det)
                msgs.info("Identifying the pixels belonging to each slit")
                slitpix = arproc.slit_pixels(slf, slf._mstrace[det-1].shape, det)
                slf.SetFrame(slf._slitpix, slitpix, det)
                # Save to disk
                armasters.save_masters(slf, det, mftype='trace')
                # Save QA for slit traces
                arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det - 1], extord,
                                       desc="Trace of the slit edges", normalize=False)
                armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")

            ###############
            # Generate the 1D wavelength solution
            update = slf.MasterWaveCalib(fitsdict, sc, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace")

            ###############
            # Derive the spectral tilt
            if slf._tilts[det-1] is None:
                try:
                    tilts = armasters.get_master_frame(slf, "tilts")
                except IOError:
                    # First time tilts are derived for this arc frame --> derive the order tilts
                    tilts, satmask, outpar = artrace.echelle_tilt(slf, slf._msarc[det - 1], det)
                    slf.SetFrame(slf._tilts, tilts, det)
                    slf.SetFrame(slf._satmask, satmask, det)
                    slf.SetFrame(slf._tiltpar, outpar, det)
                armasters.save_masters(slf, det, mftype='tilts')
            else:
                slf.SetFrame(slf._tilts, tilts, det)

            ###############
            # Prepare the pixel flat field frame
            update = slf.MasterFlatField(fitsdict, det)
            if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixelflat")

            ###############
            # Derive the spatial profile and blaze function
            if slf._slitprof[det-1] is None:
                if settings.argflag['reduce']['masters']['reuse']:
                    msslitprof_name = armasters.master_name('slitprof', settings.argflag['reduce']['masters']['setup'])
                    try:
                        slit_profiles, head = arload.load_master(msslitprof_name, frametype="slit profile")
                    except IOError:
                        pass
                    else:
                        slf.SetFrame(slf._slitprof, slit_profiles, det)
                        settings.argflag['reduce']['masters']['loaded'].append('slitprof'+settings.argflag['reduce']['masters']['setup'])
                if 'slitprof'+settings.argflag['reduce']['masters']['setup'] not in settings.argflag['reduce']['masters']['loaded']:
                    # First time slit profile is derived
                    msgs.info("Calculating slit profile from master trace frame")
                    slit_profiles, mstracenrm, msblaze, flat_ext1d, extrap_slit = arproc.slit_profile(slf, slf._mstrace[det - 1], det)
                    # If some slit profiles/blaze functions need to be extrapolated, do that now
                    if np.sum(extrap_slit) != 0.0:
                        slit_profiles, mstracenrm, msblaze = arproc.slit_profile_pca(slf, slf._mstrace[det - 1], det, msblaze, extrap_slit)
                    slf.SetFrame(slf._slitprof, slit_profiles, det)
                    slf.SetFrame(slf._msblaze, msblaze, det)
                    # Prepare some QA for the average slit profile along the slit
                    msgs.info("Preparing QA of each slit profile")
                    arqa.slit_profile(slf, mstracenrm, slit_profiles, slf._lordloc[det - 1], slf._rordloc[det - 1],
                                      slf._slitpix[det - 1], desc="Slit profile")
                    msgs.info("Saving blaze function QA")
                    arqa.plot_orderfits(slf, msblaze, flat_ext1d, desc="Blaze function", textplt="Order")

            ###############
            # Generate/load a master wave frame
            update = slf.MasterWave(fitsdict, sc, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave")

            ###############
            # Check if the user only wants to prepare the calibrations only
            msgs.info("All calibration frames have been prepared")
            if settings.argflag['run']['preponly']:
                msgs.info("If you would like to continue with the reduction, disable the command:" + msgs.newline() +
                          "run preponly False")
                continue

            ###############
            # Write setup
            #setup = arsort.calib_setup(sc, det, fitsdict, setup_dict, write=True)
            # Write MasterFrames (currently per detector)
            #armasters.save_masters(slf, det, setup)

            ###############
            # Load the science frame and from this generate a Poisson error frame
            msgs.info("Loading science frame")
            sciframe = arload.load_frames(fitsdict, [scidx], det,
                                          frametype='science',
                                          msbias=slf._msbias[det - 1])
            sciframe = sciframe[:, :, 0]
            # Extract
            msgs.info("Processing science frame")
            arproc.reduce_echelle(slf, sciframe, scidx, fitsdict, det)

        # Write 1D spectra
        save_format = 'fits'
        if save_format == 'fits':
            arsave.save_1d_spectra_fits(slf, fitsdict)
        elif save_format == 'hdf5':
            arsave.save_1d_spectra_hdf5(slf)
        else:
            msgs.error(save_format + ' is not a recognized output format!')
        # Write 2D images for the Science Frame
        arsave.save_2d_images(slf, fitsdict)
        # Free up some memory by replacing the reduced ScienceExposure class
        sciexp[sc] = None
    return status
Esempio n. 7
0
def ARMLSD(argflag, spect, fitsdict, reuseMaster=False):
    """
    Automatic Reduction and Modeling of Long Slit Data

    Parameters
    ----------
    argflag : dict
      Arguments and flags used for reduction
    spect : dict
      Properties of the spectrograph.
    fitsdict : dict
      Contains relevant information from fits header files
    reuseMaster : bool
      If True, a master frame that will be used for another science frame
      will not be regenerated after it is first made.
      This setting comes with a price, and if a large number of science frames are
      being generated, it may be more efficient to simply regenerate the master
      calibrations on the fly.

    Returns
    -------
    status : int
      Status of the reduction procedure
      0 = Successful execution
      1 = ...
    """
    status = 0

    # Create a list of science exposure classes
    sciexp = armbase.SetupScience(argflag, spect, fitsdict)
    numsci = len(sciexp)

    # Create a list of master calibration frames
    masters = armasters.MasterFrames(spect['mosaic']['ndet'])

    # Use Masters?  Requires setup file
    setup_file = argflag['out']['sorted'] + '.setup'
    try:
        calib_dict = ltu.loadjson(setup_file)
    except:
        msgs.info("No setup file {:s} for MasterFrames".format(setup_file))
        calib_dict = {}
    else:
        argflag['masters']['setup_file'] = setup_file

    # Start reducing the data
    for sc in range(numsci):
        slf = sciexp[sc]
        scidx = slf._idx_sci[0]
        msgs.info("Reducing file {0:s}, target {1:s}".format(
            fitsdict['filename'][scidx], slf._target_name))
        msgs.sciexp = slf  # For QA writing on exit, if nothing else.  Could write Masters too
        # Loop on Detectors
        for kk in xrange(slf._spect['mosaic']['ndet']):
            det = kk + 1  # Detectors indexed from 1
            slf.det = det
            ###############
            # Get amplifier sections
            arproc.get_ampsec_trimmed(slf, fitsdict, det, scidx)
            # Setup
            setup = arsort.calib_setup(slf,
                                       sc,
                                       det,
                                       fitsdict,
                                       calib_dict,
                                       write=False)
            slf._argflag['masters']['setup'] = setup
            ###############
            # Generate master bias frame
            update = slf.MasterBias(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
            ###############
            # Generate a bad pixel mask (should not repeat)
            update = slf.BadPixelMask(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
            ###############
            # Generate a master arc frame
            update = slf.MasterArc(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
            ###############
            # Determine the dispersion direction (and transpose if necessary)
            slf.GetDispersionDirection(fitsdict, det, scidx)
            if slf._bpix[
                    det -
                    1] is None:  # Needs to be done here after nspec is set
                slf.SetFrame(
                    slf._bpix,
                    np.zeros((slf._nspec[det - 1], slf._nspat[det - 1])), det)
            '''
            ###############
            # Estimate gain and readout noise for the amplifiers
            msgs.work("Estimate Gain and Readout noise from the raw frames...")
            update = slf.MasterRN(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
            '''
            ###############
            # Generate a master trace frame
            update = slf.MasterTrace(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp,
                                      sc,
                                      det,
                                      ftype="flat",
                                      chktype="trace")
            ###############
            # Generate an array that provides the physical pixel locations on the detector
            slf.GetPixelLocations(det)
            # Determine the edges of the spectrum (spatial)
            if 'trace' + slf._argflag['masters']['setup'] not in slf._argflag[
                    'masters']['loaded']:
                ###############
                # Determine the edges of the spectrum (spatial)
                lordloc, rordloc, extord = artrace.trace_orders(
                    slf,
                    slf._mstrace[det - 1],
                    det,
                    singleSlit=True,
                    pcadesc="PCA trace of the slit edges")
                slf.SetFrame(slf._lordloc, lordloc, det)
                slf.SetFrame(slf._rordloc, rordloc, det)

                # Convert physical trace into a pixel trace
                msgs.info(
                    "Converting physical trace locations to nearest pixel")
                pixcen = artrace.phys_to_pix(
                    0.5 * (slf._lordloc[det - 1] + slf._rordloc[det - 1]),
                    slf._pixlocn[det - 1], 1)
                pixwid = (slf._rordloc[det - 1] -
                          slf._lordloc[det - 1]).mean(0).astype(np.int)
                lordpix = artrace.phys_to_pix(slf._lordloc[det - 1],
                                              slf._pixlocn[det - 1], 1)
                rordpix = artrace.phys_to_pix(slf._rordloc[det - 1],
                                              slf._pixlocn[det - 1], 1)
                slf.SetFrame(slf._pixcen, pixcen, det)
                slf.SetFrame(slf._pixwid, pixwid, det)
                slf.SetFrame(slf._lordpix, lordpix, det)
                slf.SetFrame(slf._rordpix, rordpix, det)
                # Save QA for slit traces
                arqa.slit_trace_qa(slf,
                                   slf._mstrace[det - 1],
                                   slf._lordpix[det - 1],
                                   slf._rordpix[det - 1],
                                   extord,
                                   desc="Trace of the slit edges")

            ###############
            # Prepare the pixel flat field frame
            update = slf.MasterFlatField(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp,
                                      sc,
                                      det,
                                      ftype="flat",
                                      chktype="pixflat")
            ###############
            # Generate the 1D wavelength solution
            update = slf.MasterWaveCalib(fitsdict, sc, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp,
                                      sc,
                                      det,
                                      ftype="arc",
                                      chktype="trace")
            ###############
            # Derive the spectral tilt
            if slf._tilts[det - 1] is None:
                if slf._argflag['masters']['use']:
                    mstilt_name = armasters.master_name(
                        slf._argflag['run']['masterdir'], 'tilts',
                        slf._argflag['masters']['setup'])
                    try:
                        tilts, head = arload.load_master(mstilt_name,
                                                         frametype="tilts")
                    except IOError:
                        pass
                    else:
                        slf.SetFrame(slf._tilts, tilts, det)
                        slf._argflag['masters']['loaded'].append(
                            'tilts' + slf._argflag['masters']['setup'])
                if 'tilts' + slf._argflag['masters'][
                        'setup'] not in slf._argflag['masters']['loaded']:
                    # First time tilts are derived for this arc frame --> derive the order tilts
                    tilts, satmask, outpar = artrace.model_tilt(
                        slf, det, slf._msarc[det - 1])
                    slf.SetFrame(slf._tilts, tilts, det)
                    slf.SetFrame(slf._satmask, satmask, det)
                    slf.SetFrame(slf._tiltpar, outpar, det)

            ###############
            # Generate/load a master wave frame
            update = slf.MasterWave(fitsdict, sc, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp,
                                      sc,
                                      det,
                                      ftype="arc",
                                      chktype="wave")

            # Check if the user only wants to prepare the calibrations only
            msgs.info("All calibration frames have been prepared")
            if slf._argflag['run']['preponly']:
                msgs.info("If you would like to continue with the reduction," +
                          msgs.newline() + "disable the run+preponly command")
                continue

            # Write setup
            setup = arsort.calib_setup(slf,
                                       sc,
                                       det,
                                       fitsdict,
                                       calib_dict,
                                       write=True)
            # Write MasterFrames (currently per detector)
            armasters.save_masters(slf, det, setup)

            ###############
            # Load the science frame and from this generate a Poisson error frame
            msgs.info("Loading science frame")
            sciframe = arload.load_frames(slf,
                                          fitsdict, [scidx],
                                          det,
                                          frametype='science',
                                          msbias=slf._msbias[det - 1],
                                          transpose=slf._transpose)
            sciframe = sciframe[:, :, 0]
            # Extract
            msgs.info("Processing science frame")
            arproc.reduce_frame(slf, sciframe, scidx, fitsdict, det)

            #continue
            #msgs.error("UP TO HERE")
            ###############
            # Perform a velocity correction
            if (slf._argflag['reduce']['heliocorr'] == True) & False:
                if slf._argflag['science']['load']['extracted'] == True:
                    msgs.warn(
                        "Heliocentric correction will not be applied if an extracted science frame exists, and is used"
                    )
                msgs.work("Perform a full barycentric correction")
                msgs.work(
                    "Include the facility to correct for gravitational redshifts and time delays (see Pulsar timing work)"
                )
                msgs.info("Performing a heliocentric correction")
                # Load the header for the science frame
                slf._waveids = arvcorr.helio_corr(slf, scidx[0])
            else:
                msgs.info("A heliocentric correction will not be performed")

            ###############
            # Using model sky, calculate a flexure correction

        # Close the QA for this object
        slf._qa.close()

        ###############
        # Flux
        ###############
        # Standard star (is this a calibration, e.g. goes above?)
        msgs.info("Processing standard star")
        msgs.info("Assuming one star per detector mosaic")
        msgs.info("Waited until last detector to process")

        msgs.work("Need to check for existing sensfunc")
        update = slf.MasterStandard(scidx, fitsdict)
        if update and reuseMaster:
            armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
        #
        msgs.work("Consider using archived sensitivity if not found")
        msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
        for kk in xrange(slf._spect['mosaic']['ndet']):
            det = kk + 1  # Detectors indexed from 1
            arflux.apply_sensfunc(slf, det, scidx, fitsdict)

        # Write 1D spectra
        arsave.save_1d_spectra(slf)
        # Write 2D images for the Science Frame
        arsave.save_2d_images(slf)
        # Free up some memory by replacing the reduced ScienceExposure class
        sciexp[sc] = None
    return status
Esempio n. 8
0
def ARMLSD(argflag, spect, fitsdict, reuseMaster=False):
    """
    Automatic Reduction and Modeling of Long Slit Data

    Parameters
    ----------
    argflag : dict
      Arguments and flags used for reduction
    spect : dict
      Properties of the spectrograph.
    fitsdict : dict
      Contains relevant information from fits header files
    reuseMaster : bool
      If True, a master frame that will be used for another science frame
      will not be regenerated after it is first made.
      This setting comes with a price, and if a large number of science frames are
      being generated, it may be more efficient to simply regenerate the master
      calibrations on the fly.

    Returns
    -------
    status : int
      Status of the reduction procedure
      0 = Successful execution
      1 = ...
    """
    status = 0

    # Create a list of science exposure classes
    sciexp = armbase.SetupScience(argflag, spect, fitsdict)
    numsci = len(sciexp)

    # Create a list of master calibration frames
    masters = armasters.MasterFrames(spect['mosaic']['ndet'])

    # Use Masters?  Requires setup file
    setup_file = argflag['out']['sorted']+'.setup'
    try:
        calib_dict = ltu.loadjson(setup_file)
    except:
        msgs.info("No setup file {:s} for MasterFrames".format(setup_file))
        calib_dict = {}
    else:
        argflag['masters']['setup_file'] = setup_file

    # Start reducing the data
    for sc in range(numsci):
        slf = sciexp[sc]
        scidx = slf._idx_sci[0]
        msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name))
        msgs.sciexp = slf  # For QA writing on exit, if nothing else.  Could write Masters too
        # Loop on Detectors
        for kk in xrange(slf._spect['mosaic']['ndet']):
            det = kk + 1  # Detectors indexed from 1
            slf.det = det
            ###############
            # Get amplifier sections
            arproc.get_ampsec_trimmed(slf, fitsdict, det, scidx)
            # Setup
            setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=False)
            slf._argflag['masters']['setup'] = setup
            ###############
            # Generate master bias frame
            update = slf.MasterBias(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
            ###############
            # Generate a bad pixel mask (should not repeat)
            update = slf.BadPixelMask(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
            ###############
            # Generate a master arc frame
            update = slf.MasterArc(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
            ###############
            # Determine the dispersion direction (and transpose if necessary)
            slf.GetDispersionDirection(fitsdict, det, scidx)
            if slf._bpix[det-1] is None:  # Needs to be done here after nspec is set
                slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det)
            '''
            ###############
            # Estimate gain and readout noise for the amplifiers
            msgs.work("Estimate Gain and Readout noise from the raw frames...")
            update = slf.MasterRN(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
            '''
            ###############
            # Generate a master trace frame
            update = slf.MasterTrace(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
            ###############
            # Generate an array that provides the physical pixel locations on the detector
            slf.GetPixelLocations(det)
            # Determine the edges of the spectrum (spatial)
            if 'trace'+slf._argflag['masters']['setup'] not in slf._argflag['masters']['loaded']:
                ###############
                # Determine the edges of the spectrum (spatial)
                lordloc, rordloc, extord = artrace.trace_orders(slf, slf._mstrace[det-1], det, singleSlit=True, pcadesc="PCA trace of the slit edges")
                slf.SetFrame(slf._lordloc, lordloc, det)
                slf.SetFrame(slf._rordloc, rordloc, det)

                # Convert physical trace into a pixel trace
                msgs.info("Converting physical trace locations to nearest pixel")
                pixcen = artrace.phys_to_pix(0.5*(slf._lordloc[det-1]+slf._rordloc[det-1]), slf._pixlocn[det-1], 1)
                pixwid = (slf._rordloc[det-1]-slf._lordloc[det-1]).mean(0).astype(np.int)
                lordpix = artrace.phys_to_pix(slf._lordloc[det-1], slf._pixlocn[det-1], 1)
                rordpix = artrace.phys_to_pix(slf._rordloc[det-1], slf._pixlocn[det-1], 1)
                slf.SetFrame(slf._pixcen, pixcen, det)
                slf.SetFrame(slf._pixwid, pixwid, det)
                slf.SetFrame(slf._lordpix, lordpix, det)
                slf.SetFrame(slf._rordpix, rordpix, det)
                # Save QA for slit traces
                arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1], slf._rordpix[det-1], extord, desc="Trace of the slit edges")

            ###############
            # Prepare the pixel flat field frame
            update = slf.MasterFlatField(fitsdict, det)
            if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixflat")
            ###############
            # Generate the 1D wavelength solution
            update = slf.MasterWaveCalib(fitsdict, sc, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace")
            ###############
            # Derive the spectral tilt
            if slf._tilts[det-1] is None:
                if slf._argflag['masters']['use']:
                    mstilt_name = armasters.master_name(slf._argflag['run']['masterdir'],
                                                        'tilts', slf._argflag['masters']['setup'])
                    try:
                        tilts, head = arload.load_master(mstilt_name, frametype="tilts")
                    except IOError:
                        pass
                    else:
                        slf.SetFrame(slf._tilts, tilts, det)
                        slf._argflag['masters']['loaded'].append('tilts'+slf._argflag['masters']['setup'])
                if 'tilts'+slf._argflag['masters']['setup'] not in slf._argflag['masters']['loaded']:
                    # First time tilts are derived for this arc frame --> derive the order tilts
                    tilts, satmask, outpar = artrace.model_tilt(slf, det, slf._msarc[det-1])
                    slf.SetFrame(slf._tilts, tilts, det)
                    slf.SetFrame(slf._satmask, satmask, det)
                    slf.SetFrame(slf._tiltpar, outpar, det)

            ###############
            # Generate/load a master wave frame
            update = slf.MasterWave(fitsdict, sc, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave")

            # Check if the user only wants to prepare the calibrations only
            msgs.info("All calibration frames have been prepared")
            if slf._argflag['run']['preponly']:
                msgs.info("If you would like to continue with the reduction,"
                          +msgs.newline()+"disable the run+preponly command")
                continue

            # Write setup
            setup = arsort.calib_setup(slf, sc, det, fitsdict, calib_dict, write=True)
            # Write MasterFrames (currently per detector)
            armasters.save_masters(slf, det, setup)

            ###############
            # Load the science frame and from this generate a Poisson error frame
            msgs.info("Loading science frame")
            sciframe = arload.load_frames(slf, fitsdict, [scidx], det,
                                          frametype='science',
                                          msbias=slf._msbias[det-1],
                                          transpose=slf._transpose)
            sciframe = sciframe[:, :, 0]
            # Extract
            msgs.info("Processing science frame")
            arproc.reduce_frame(slf, sciframe, scidx, fitsdict, det)

            #continue
            #msgs.error("UP TO HERE")
            ###############
            # Perform a velocity correction
            if (slf._argflag['reduce']['heliocorr'] == True) & False:
                if slf._argflag['science']['load']['extracted'] == True:
                    msgs.warn("Heliocentric correction will not be applied if an extracted science frame exists, and is used")
                msgs.work("Perform a full barycentric correction")
                msgs.work("Include the facility to correct for gravitational redshifts and time delays (see Pulsar timing work)")
                msgs.info("Performing a heliocentric correction")
                # Load the header for the science frame
                slf._waveids = arvcorr.helio_corr(slf, scidx[0])
            else:
                msgs.info("A heliocentric correction will not be performed")

            ###############
            # Using model sky, calculate a flexure correction

        # Close the QA for this object
        slf._qa.close()

        ###############
        # Flux
        ###############
        # Standard star (is this a calibration, e.g. goes above?)
        msgs.info("Processing standard star")
        msgs.info("Assuming one star per detector mosaic")
        msgs.info("Waited until last detector to process")

        msgs.work("Need to check for existing sensfunc")
        update = slf.MasterStandard(scidx, fitsdict)
        if update and reuseMaster:
            armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
        #
        msgs.work("Consider using archived sensitivity if not found")
        msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
        for kk in xrange(slf._spect['mosaic']['ndet']):
            det = kk + 1  # Detectors indexed from 1
            arflux.apply_sensfunc(slf, det, scidx, fitsdict)

        # Write 1D spectra
        arsave.save_1d_spectra(slf)
        # Write 2D images for the Science Frame
        arsave.save_2d_images(slf)
        # Free up some memory by replacing the reduced ScienceExposure class
        sciexp[sc] = None
    return status
Esempio n. 9
0
def ARMLSD(fitsdict, reuseMaster=False, reloadMaster=True):
    """
    Automatic Reduction and Modeling of Long Slit Data

    Parameters
    ----------
    fitsdict : dict
      Contains relevant information from fits header files
    reuseMaster : bool
      If True, a master frame that will be used for another science frame
      will not be regenerated after it is first made.
      This setting comes with a price, and if a large number of science frames are
      being generated, it may be more efficient to simply regenerate the master
      calibrations on the fly.

    Returns
    -------
    status : int
      Status of the reduction procedure
      0 = Successful full execution
      1 = Successful processing of setup or calcheck
    """
    status = 0

    # Create a list of science exposure classes
    sciexp, setup_dict = armbase.SetupScience(fitsdict)
    if sciexp == 'setup':
        status = 1
        return status
    elif sciexp == 'calcheck':
        status = 2
        return status
    else:
        numsci = len(sciexp)

    # Create a list of master calibration frames
    #masters = armasters.MasterFrames(settings.spect['mosaic']['ndet'])

    # Masters
    #settings.argflag['reduce']['masters']['file'] = setup_file

    # Start reducing the data
    for sc in range(numsci):
        slf = sciexp[sc]
        scidx = slf._idx_sci[0]
        msgs.info("Reducing file {0:s}, target {1:s}".format(fitsdict['filename'][scidx], slf._target_name))
        msgs.sciexp = slf  # For QA writing on exit, if nothing else.  Could write Masters too
        if reloadMaster and (sc > 0):
            settings.argflag['reduce']['masters']['reuse'] = True
        # Loop on Detectors
        for kk in range(settings.spect['mosaic']['ndet']):
            det = kk + 1  # Detectors indexed from 1
            if settings.argflag['reduce']['detnum'] is not None:
                if det != settings.argflag['reduce']['detnum']:
                    continue
                else:
                    msgs.warn("Restricting the reduction to detector {:d}".format(det))
            slf.det = det
            ###############
            # Get data sections
            arproc.get_datasec_trimmed(slf, fitsdict, det, scidx)
            # Setup
            setup = arsort.instr_setup(slf, det, fitsdict, setup_dict, must_exist=True)
            settings.argflag['reduce']['masters']['setup'] = setup
            slf.setup = setup
            ###############
            # Generate master bias frame
            update = slf.MasterBias(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="bias")
            ###############
            # Generate a bad pixel mask (should not repeat)
            update = slf.BadPixelMask(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
            ###############
            # Generate a master arc frame
            update = slf.MasterArc(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc")
            ###############
            # Set the number of spectral and spatial pixels, and the bad pixel mask is it does not exist
            slf._nspec[det-1], slf._nspat[det-1] = slf._msarc[det-1].shape
            if slf._bpix[det-1] is None:
                slf.SetFrame(slf._bpix, np.zeros((slf._nspec[det-1], slf._nspat[det-1])), det)
            '''
            ###############
            # Estimate gain and readout noise for the amplifiers
            msgs.work("Estimate Gain and Readout noise from the raw frames...")
            update = slf.MasterRN(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="readnoise")
            '''
            ###############
            # Generate a master trace frame
            update = slf.MasterTrace(fitsdict, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")
            ###############
            # Generate an array that provides the physical pixel locations on the detector
            slf.GetPixelLocations(det)
            # Determine the edges of the spectrum (spatial)
            if ('trace'+settings.argflag['reduce']['masters']['setup'] not in settings.argflag['reduce']['masters']['loaded']):
                ###############
                # Determine the edges of the spectrum (spatial)
                lordloc, rordloc, extord = artrace.trace_slits(slf, slf._mstrace[det-1], det, pcadesc="PCA trace of the slit edges")
                slf.SetFrame(slf._lordloc, lordloc, det)
                slf.SetFrame(slf._rordloc, rordloc, det)

                # Convert physical trace into a pixel trace
                msgs.info("Converting physical trace locations to nearest pixel")
                pixcen = artrace.phys_to_pix(0.5*(slf._lordloc[det-1]+slf._rordloc[det-1]), slf._pixlocn[det-1], 1)
                pixwid = (slf._rordloc[det-1]-slf._lordloc[det-1]).mean(0).astype(np.int)
                lordpix = artrace.phys_to_pix(slf._lordloc[det-1], slf._pixlocn[det-1], 1)
                rordpix = artrace.phys_to_pix(slf._rordloc[det-1], slf._pixlocn[det-1], 1)
                slf.SetFrame(slf._pixcen, pixcen, det)
                slf.SetFrame(slf._pixwid, pixwid, det)
                slf.SetFrame(slf._lordpix, lordpix, det)
                slf.SetFrame(slf._rordpix, rordpix, det)
                msgs.info("Identifying the pixels belonging to each slit")
                slitpix = arproc.slit_pixels(slf, slf._mstrace[det-1].shape, det)
                slf.SetFrame(slf._slitpix, slitpix, det)
                # Save to disk
                armasters.save_masters(slf, det, mftype='trace')
                # Save QA for slit traces
                arqa.slit_trace_qa(slf, slf._mstrace[det-1], slf._lordpix[det-1],
                                       slf._rordpix[det-1], extord,
                                       desc="Trace of the slit edges D{:02d}".format(det), use_slitid=det)
                armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="trace")

            ###############
            # Generate the 1D wavelength solution
            update = slf.MasterWaveCalib(fitsdict, sc, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="trace")

            ###############
            # Derive the spectral tilt
            if slf._tilts[det-1] is None:
                try:
                    tilts = armasters.get_master_frame(slf, "tilts")
                except IOError:
                    # First time tilts are derived for this arc frame --> derive the order tilts
                    tilts, satmask, outpar = artrace.multislit_tilt(slf, slf._msarc[det-1], det)
                    slf.SetFrame(slf._tilts, tilts, det)
                    slf.SetFrame(slf._satmask, satmask, det)
                    slf.SetFrame(slf._tiltpar, outpar, det)
                    armasters.save_masters(slf, det, mftype='tilts')
                else:
                    slf.SetFrame(slf._tilts, tilts, det)

            ###############
            # Prepare the pixel flat field frame
            update = slf.MasterFlatField(fitsdict, det)
            if update and reuseMaster: armbase.UpdateMasters(sciexp, sc, det, ftype="flat", chktype="pixelflat")

            ###############
            # Generate/load a master wave frame
            update = slf.MasterWave(fitsdict, sc, det)
            if update and reuseMaster:
                armbase.UpdateMasters(sciexp, sc, det, ftype="arc", chktype="wave")

            ###############
            # Check if the user only wants to prepare the calibrations only
            msgs.info("All calibration frames have been prepared")
            if settings.argflag['run']['preponly']:
                msgs.info("If you would like to continue with the reduction, disable the command:" + msgs.newline() +
                          "run preponly False")
                continue

            ###############
            # Write setup
            #setup = arsort.calib_setup(sc, det, fitsdict, setup_dict, write=True)
            # Write MasterFrames (currently per detector)
            #armasters.save_masters(slf, det, setup)

            ###############
            # Load the science frame and from this generate a Poisson error frame
            msgs.info("Loading science frame")
            sciframe = arload.load_frames(fitsdict, [scidx], det,
                                          frametype='science',
                                          msbias=slf._msbias[det-1])
            sciframe = sciframe[:, :, 0]
            # Extract
            msgs.info("Processing science frame")
            arproc.reduce_multislit(slf, sciframe, scidx, fitsdict, det)

            ###############
            # Using model sky, calculate a flexure correction

        ###############
        # Flux
        ###############
        # Standard star (is this a calibration, e.g. goes above?)
        msgs.info("Processing standard star")
        msgs.info("Assuming one star per detector mosaic")
        msgs.info("Waited until last detector to process")

        update = slf.MasterStandard(fitsdict)
        if update and reuseMaster:
            armbase.UpdateMasters(sciexp, sc, 0, ftype="standard")
        #
        msgs.work("Consider using archived sensitivity if not found")
        msgs.info("Fluxing with {:s}".format(slf._sensfunc['std']['name']))
        for kk in range(settings.spect['mosaic']['ndet']):
            det = kk + 1  # Detectors indexed from 1
            if slf._specobjs[det-1] is not None:
                arflux.apply_sensfunc(slf, det, scidx, fitsdict)
            else:
                msgs.info("There are no objects on detector {0:d} to apply a flux calibration".format(det))

        # Write 1D spectra
        save_format = 'fits'
        if save_format == 'fits':
            arsave.save_1d_spectra_fits(slf, fitsdict)
        elif save_format == 'hdf5':
            arsave.save_1d_spectra_hdf5(slf)
        else:
            msgs.error(save_format + ' is not a recognized output format!')
        arsave.save_obj_info(slf, fitsdict)
        # Write 2D images for the Science Frame
        arsave.save_2d_images(slf, fitsdict)
        # Free up some memory by replacing the reduced ScienceExposure class
        sciexp[sc] = None
    return status
Esempio n. 10
0
    def MasterFlatField(self, fitsdict, det):
        """
        Generate Master Flat-field frame for a given detector

        Parameters
        ----------
        fitsdict : dict
          Contains relevant information from fits header files
        det : int
          Index of the detector

        Returns
        -------
        boolean : bool
          Should other ScienceExposure classes be updated?
        """
        from pypit import arqa

        if settings.argflag['reduce']['flatfield'][
                'perform']:  # Only do it if the user wants to flat field
            # If the master pixelflat is already made, use it
            if self._mspixelflat[det - 1] is not None:
                msgs.info("An identical master pixelflat frame already exists")
                if self._mspixelflatnrm[det - 1] is None:
                    # Normalize the flat field
                    msgs.info("Normalizing the pixel flat")
                    slit_profiles, mstracenrm, msblaze, flat_ext1d, extrap_slit = \
                        arproc.slit_profile(self, self.GetMasterFrame("pixelflat", det),
                                            det, ntcky=settings.argflag['reduce']['flatfield']['params'][0])
                    # If some slit profiles/blaze functions need to be extrapolated, do that now
                    if np.sum(extrap_slit) != 0.0:
                        slit_profiles, mstracenrm, msblaze = arproc.slit_profile_pca(
                            self, self.GetMasterFrame("pixelflat", det), det,
                            msblaze, extrap_slit, slit_profiles)
                    mspixelflatnrm = mstracenrm.copy()
                    winpp = np.where(slit_profiles != 0.0)
                    mspixelflatnrm[winpp] /= slit_profiles[winpp]
                    self.SetMasterFrame(mspixelflatnrm, "normpixelflat", det)
                    armasters.save_masters(self, det, mftype='normpixelflat')
                    if np.array_equal(self._idx_flat, self._idx_trace):
                        # The flat field frame is also being used to trace the slit edges and determine the slit
                        # profile. Avoid recalculating the slit profile and blaze function and save them here.
                        self.SetFrame(self._msblaze, msblaze, det)
                        self.SetFrame(self._slitprof, slit_profiles, det)
                        armasters.save_masters(self, det, mftype='slitprof')
                        if settings.argflag["reduce"]["slitprofile"][
                                "perform"]:
                            msgs.info("Preparing QA of each slit profile")
                            arqa.slit_profile(self,
                                              mstracenrm,
                                              slit_profiles,
                                              self._lordloc[det - 1],
                                              self._rordloc[det - 1],
                                              self._slitpix[det - 1],
                                              desc="Slit profile")
                        msgs.info("Saving blaze function QA")
                        arqa.plot_orderfits(self,
                                            msblaze,
                                            flat_ext1d,
                                            desc="Blaze function")
                return False
            ###############
            # Generate/load a master pixel flat frame
            if settings.argflag['reduce']['flatfield']['useframe'] in [
                    'pixelflat', 'trace'
            ]:
                try:
                    mspixelflatnrm = armasters.get_master_frame(
                        self, "normpixelflat")
                except IOError:
                    msgs.info("Preparing a master pixel flat frame with {0:s}".
                              format(settings.argflag['reduce']['flatfield']
                                     ['useframe']))
                    # Get all of the pixel flat frames for this science frame
                    ind = self._idx_flat
                    # Load the frames for tracing
                    frames = arload.load_frames(fitsdict,
                                                ind,
                                                det,
                                                frametype='pixel flat',
                                                msbias=self._msbias[det - 1])
                    if settings.argflag['pixelflat']['combine']['match'] > 0.0:
                        sframes = arsort.match_frames(
                            frames,
                            settings.argflag['pixelflat']['combine']['match'],
                            frametype='pixel flat',
                            satlevel=self._nonlinear)
                        subframes = np.zeros(
                            (frames.shape[0], frames.shape[1], len(sframes)))
                        numarr = np.array([])
                        for i in range(len(sframes)):
                            numarr = np.append(numarr, sframes[i].shape[2])
                            mspixelflat = arcomb.comb_frames(
                                sframes[i],
                                det,
                                'pixelflat',
                                printtype='pixel flat')
                            subframes[:, :, i] = mspixelflat.copy()
                        del sframes
                        # Combine all sub-frames
                        mspixelflat = arcomb.comb_frames(
                            subframes,
                            det,
                            'pixelflat',
                            weights=numarr,
                            printtype='pixel flat')
                        del subframes
                    else:
                        mspixelflat = arcomb.comb_frames(
                            frames, det, 'pixelflat', printtype='pixel flat')
                    del frames
                    # Apply gain (instead of ampsec scale)
                    mspixelflat *= arproc.gain_frame(self, det)
                    # Normalize the flat field
                    msgs.info("Normalizing the pixel flat")
                    slit_profiles, mstracenrm, msblaze, flat_ext1d, extrap_slit = \
                        arproc.slit_profile(self, mspixelflat, det, ntcky=settings.argflag['reduce']['flatfield']['params'][0])
                    # If some slit profiles/blaze functions need to be extrapolated, do that now
                    if np.sum(extrap_slit) != 0.0:
                        slit_profiles, mstracenrm, msblaze = arproc.slit_profile_pca(
                            self, mspixelflat, det, msblaze, extrap_slit,
                            slit_profiles)
                    mspixelflatnrm = mstracenrm.copy()
                    winpp = np.where(slit_profiles != 0.0)
                    mspixelflatnrm[winpp] /= slit_profiles[winpp]
                    if np.array_equal(self._idx_flat, self._idx_trace):
                        # The flat field frame is also being used to trace the slit edges and determine the slit
                        # profile. Avoid recalculating the slit profile and blaze function and save them here.
                        self.SetFrame(self._msblaze, msblaze, det)
                        self.SetFrame(self._slitprof, slit_profiles, det)
                        armasters.save_masters(self, det, mftype='slitprof')
                        if settings.argflag["reduce"]["slitprofile"][
                                "perform"]:
                            msgs.info("Preparing QA of each slit profile")
                            arqa.slit_profile(self,
                                              mstracenrm,
                                              slit_profiles,
                                              self._lordloc[det - 1],
                                              self._rordloc[det - 1],
                                              self._slitpix[det - 1],
                                              desc="Slit profile")
                        msgs.info("Saving blaze function QA")
                        arqa.plot_orderfits(self,
                                            msblaze,
                                            flat_ext1d,
                                            desc="Blaze function")
                else:
                    mspixelflat = mspixelflatnrm
            else:  # It must be the name of a file the user wishes to load
                mspixelflat_name = armasters.user_master_name(
                    settings.argflag['run']['directory']['master'],
                    settings.argflag['reduce']['flatfield']['useframe'])
                mspixelflatnrm, head = arload.load_master(mspixelflat_name,
                                                          exten=det,
                                                          frametype=None)
                mspixelflat = mspixelflatnrm
            # Now that the combined, master flat field frame is loaded...
        else:
            msgs.work(
                "Pixel Flat arrays need to be generated when not flat fielding"
            )
            msgs.bug("Blaze is currently undefined")
            mspixelflat = np.ones_like(self._msarc)
            mspixelflatnrm = np.ones_like(self._msarc)
        # Set Master Frames
        self.SetMasterFrame(mspixelflat, "pixelflat", det)
        self.SetMasterFrame(mspixelflatnrm, "normpixelflat", det)
        armasters.save_masters(self, det, mftype='normpixelflat')
        return True
Esempio n. 11
0
    def MasterArc(self, fitsdict, det):
        """
        Generate Master Arc frame for a given detector

        Parameters
        ----------
        fitsdict : dict
          Contains relevant information from fits header files
        det : int
          Index of the detector

        Returns
        -------
        boolean : bool
          Should other ScienceExposure classes be updated?
        """
        dnum = settings.get_dnum(det)

        if self._msarc[det - 1] is not None:
            msgs.info("A master arc frame already exists for this frame")
            return False
        if settings.argflag['arc']['useframe'] in ['arc']:
            # Master Frame
            try:
                msarc = armasters.get_master_frame(self, "arc")
            except IOError:
                msgs.info("Preparing a master arc frame")
                ind = self._idx_arcs
                # Load the arc frames
                frames = arload.load_frames(fitsdict,
                                            ind,
                                            det,
                                            frametype='arc',
                                            msbias=self._msbias[det - 1])
                if settings.argflag['arc']['combine']['match'] > 0.0:
                    sframes = arsort.match_frames(
                        frames,
                        settings.argflag['arc']['combine']['match'],
                        frametype='arc',
                        satlevel=settings.spect[dnum]['saturation'] *
                        settings.spect[dnum]['nonlinear'])
                    subframes = np.zeros(
                        (frames.shape[0], frames.shape[1], len(sframes)))
                    numarr = np.array([])
                    for i in range(len(sframes)):
                        numarr = np.append(numarr, sframes[i].shape[2])
                        msarc = arcomb.comb_frames(sframes[i], det, 'arc')
                        # Send the data away to be saved
                        subframes[:, :, i] = msarc.copy()
                    del sframes
                    # Combine all sub-frames
                    msarc = arcomb.comb_frames(subframes,
                                               det,
                                               'arc',
                                               weights=numarr)
                    del subframes
                else:
                    msarc = arcomb.comb_frames(frames, det, 'arc')
                del frames
        else:  # Use input frame name located in MasterFrame directory
            msarc_name = settings.argflag['run']['directory'][
                'master'] + '/' + settings.argflag['arc']['useframe']
            msarc, _ = arload.load_master(msarc_name, frametype=None)

        # Set and then delete the Master Arc frame
        self.SetMasterFrame(msarc, "arc", det)
        armasters.save_masters(self, det, mftype='arc')
        del msarc
        return True