def load_master(self, filename, force=False):

        # Does the master file exist?
        if not os.path.isfile(filename):
            # msgs.warn("No Master frame found of type {:s}: {:s}".format(self.frametype, filename))
            msgs.warn("No Master frame found of {:s}".format(filename))
            if force:
                msgs.error("Crashing out because reduce-masters-force=True:" + msgs.newline() + filename)
            return None
        else:
            # msgs.info("Loading a pre-existing master calibration frame of type: {:}".format(self.frametype) + " from filename: {:}".format(filename))
            msgs.info("Loading a pre-existing master calibration frame of SENSFUNC from filename: {:}".format(filename))

            hdu = fits.open(filename)
            norder = hdu[0].header['NORDER']
            sens_dicts = {}
            for iord in range(norder):
                head = hdu[iord + 1].header
                tbl = hdu['SENSFUNC-ORDER{0:04}'.format(iord)].data
                sens_dict = {}
                sens_dict['wave'] = tbl['WAVE']
                sens_dict['sensfunc'] = tbl['SENSFUNC']
                for key in ['wave_min', 'wave_max', 'exptime', 'airmass', 'std_file', 'std_ra', 'std_dec',
                            'std_name', 'cal_file', 'ech_orderindx']:
                    try:
                        sens_dict[key] = head[key.upper()]
                    except:
                        pass
                sens_dicts[str(iord)] = sens_dict
            sens_dicts['norder'] = norder
            return sens_dicts
Пример #2
0
def write_science(sci_specobjs, sci_header, outfile):
    """
    Write the flux-calibrated science spectra

    Parameters
    ----------
    outfile : str

    Returns
    -------

    """
    if len(sci_specobjs) == 0:
        msgs.warn("No science spectra to write to disk!")
    #
    if 'VEL-TYPE' in sci_header.keys():
        helio_dict = dict(refframe=sci_header['VEL-TYPE'],
                          vel_correction=sci_header['VEL'])
    else:
        helio_dict = None
    telescope=None
    if 'LON-OBS' in sci_header.keys():
        telescope = TelescopePar(longitude=sci_header['LON-OBS'],
                                 latitude=sci_header['LAT-OBS'],
                                 elevation=sci_header['ALT-OBS'])
    # KLUDGE ME
    if isinstance(sci_specobjs, list):
        specObjs = specobjs.SpecObjs(sci_specobjs)
    elif isinstance(sci_specobjs, specobjs.SpecObjs):
        specObjs = sci_specobjs
    else:
        msgs.error("BAD INPUT")
    save.save_1d_spectra_fits(specObjs, sci_header,'ECHELLE', outfile,
                              helio_dict=helio_dict,
                              telescope=telescope, overwrite=True)
Пример #3
0
def rebin(a, newshape):
    '''Rebin an array to a new shape using slicing. This routine is taken from:
    https://scipy-cookbook.readthedocs.io/items/Rebinning.html
    '''

    if not len(a.shape) == len(newshape):
        msgs.error('Dimension of a image does not match dimension of new requested image shape')

    slices = [slice(0, old, float(old) / new) for old, new in zip(a.shape, newshape)]
    coordinates = np.mgrid[slices]
    indices = coordinates.astype('i')  # choose the biggest smaller integer index
    return a[tuple(indices)]
Пример #4
0
def load_spec_order(fname,objid=None,order=None,extract='OPT',flux=True):
    """
    Loading single order spectrum from a PypeIt 1D specctrum fits file
    :param file:
    :param objid:
    :param order:
    :param extract:
    :param flux:
    :return:
    """
    if objid is None:
        objid = 0
    if order is None:
        msgs.error('Please specify which order you want to load')

    # read extension name into a list
    primary_header = fits.getheader(fname, 0)
    nspec = primary_header['NSPEC']
    extnames = [primary_header['EXT0001']] * nspec
    for kk in range(nspec):
        extnames[kk] = primary_header['EXT' + '{0:04}'.format(kk + 1)]
    extnameroot = extnames[0]

    # Figure out which extension is the required data
    ordername = '{0:04}'.format(order)
    extname = extnameroot.replace('OBJ0000', objid)
    extname = extname.replace('ORDER0000', 'ORDER' + ordername)
    try:
        exten = extnames.index(extname) + 1
        msgs.info("Loading extension {:s} of spectrum {:s}".format(extname, fname))
    except:
        msgs.error("Spectrum {:s} does not contain {:s} extension".format(fname, extname))

    spectrum = load.load_1dspec(fname, exten=exten, extract=extract, flux=flux)
    # Polish a bit -- Deal with NAN, inf, and *very* large values that will exceed
    #   the floating point precision of float32 for var which is sig**2 (i.e. 1e38)
    bad_flux = np.any([np.isnan(spectrum.flux), np.isinf(spectrum.flux),
                       np.abs(spectrum.flux) > 1e30,
                       spectrum.sig ** 2 > 1e10,
                       ], axis=0)
    # Sometimes Echelle spectra have zero wavelength
    bad_wave = spectrum.wavelength < 1000.0*units.AA
    bad_all = bad_flux + bad_wave
    ## trim bad part
    wave_out,flux_out,sig_out = spectrum.wavelength[~bad_all],spectrum.flux[~bad_all],spectrum.sig[~bad_all]
    spectrum_out = XSpectrum1D.from_tuple((wave_out,flux_out,sig_out), verbose=False)
    #if np.sum(bad_flux):
    #    msgs.warn("There are some bad flux values in this spectrum.  Will zero them out and mask them (not ideal)")
    #    spectrum.data['flux'][spectrum.select][bad_flux] = 0.
    #    spectrum.data['sig'][spectrum.select][bad_flux] = 0.

    return spectrum_out
Пример #5
0
def ech_load_spec(files,objid=None,order=None,extract='OPT',flux=True):
    """
    files: A list of file names
    objid:
    extract:
    flux:
    """

    nfiles = len(files)
    if objid is None:
        objid = ['OBJ0000'] * nfiles
    elif len(objid) == 1:
        objid = objid * nfiles
    elif len(objid) != nfiles:
        msgs.error('The length of objid should be either 1 or equal to the number of spectra files.')

    fname = files[0]
    ext_final = fits.getheader(fname, -1)
    norder = ext_final['ORDER'] + 1
    msgs.info('spectrum {:s} has {:d} orders'.format(fname, norder))
    if norder <= 1:
        msgs.error('The number of orders have to be greater than one for echelle. Longslit data?')

    # Load spectra
    spectra_list = []
    for ii, fname in enumerate(files):

        if order is None:
            msgs.info('Loading all orders into a gaint spectra')
            for iord in range(norder):
                spectrum = load_spec_order(fname,objid=objid[ii],order=iord,extract=extract,flux=flux)
                # Append
                spectra_list.append(spectrum)
        elif order >= norder:
            msgs.error('order number cannot greater than the total number of orders')
        else:
            spectrum = load_spec_order(fname, objid=objid[ii], order=order, extract=extract, flux=flux)
            # Append
            spectra_list.append(spectrum)
    # Join into one XSpectrum1D object
    spectra = collate(spectra_list)
    # Return
    return spectra
def ech_objfind(image,
                ivar,
                ordermask,
                slit_left,
                slit_righ,
                inmask=None,
                plate_scale=0.2,
                npca=2,
                ncoeff=5,
                min_snr=0.0,
                nabove_min_snr=0,
                pca_percentile=20.0,
                snr_pca=3.0,
                box_radius=2.0,
                show_peaks=False,
                show_fits=False,
                show_trace=False):

    if inmask is None:
        inmask = (ordermask > 0)

    frameshape = image.shape
    nspec = frameshape[0]
    norders = slit_left.shape[1]

    if isinstance(plate_scale, (float, int)):
        plate_scale_ord = np.full(
            norders, plate_scale)  # 0.12 binned by 3 spatially for HIRES
    elif isinstance(plate_scale, (np.ndarray, list, tuple)):
        if len(plate_scale) == norders:
            plate_scale_ord = plate_scale
        elif len(plate_scale) == 1:
            plate_scale_ord = np.full(norders, plate_scale[0])
        else:
            msgs.error(
                'Invalid size for plate_scale. It must either have one element or norders elements'
            )
    else:
        msgs.error('Invalid type for plate scale')

    specmid = nspec // 2
    slit_width = slit_righ - slit_left
    spec_vec = np.arange(nspec)
    slit_spec_pos = nspec / 2.0
    slit_spat_pos = np.zeros((norders, 2))
    for iord in range(norders):
        slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec,
                                            slit_left[:, iord]),
                                  np.interp(slit_spec_pos, spec_vec,
                                            slit_righ[:, iord]))

    # Loop over orders and find objects
    sobjs = specobjs.SpecObjs()
    show_peaks = True
    show_fits = True
    # ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx!
    for iord in range(norders):
        msgs.info('Finding objects on slit # {:d}'.format(iord + 1))
        thismask = ordermask == (iord + 1)
        inmask_iord = inmask & thismask
        specobj_dict = {
            'setup': 'HIRES',
            'slitid': iord + 1,
            'scidx': 0,
            'det': 1,
            'objtype': 'science'
        }
        sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \
            extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks,
                            show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0)
        # ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord
        for spec in sobjs_slit:
            spec.ech_orderindx = iord
        sobjs.add_sobj(sobjs_slit)

    nfound = len(sobjs)

    # Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0"
    FOFSEP = 1.0  # separation of FOF algorithm in arcseconds
    FOF_frac = FOFSEP / (np.median(slit_width) * np.median(plate_scale_ord))

    # Feige: made the code also works for only one object found in one order
    # Run the FOF. We use fake coordinaes
    fracpos = sobjs.spat_fracpos
    ra_fake = fracpos / 1000.0  # Divide all angles by 1000 to make geometry euclidian
    dec_fake = 0.0 * fracpos
    if nfound > 1:
        (ingroup, multgroup, firstgroup,
         nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac / 1000.0)
        group = ingroup.copy()
        uni_group, uni_ind = np.unique(group, return_index=True)
        nobj = len(uni_group)
        msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects')
    elif nfound == 1:
        group = np.zeros(1, dtype='int')
        uni_group, uni_ind = np.unique(group, return_index=True)
        nobj = len(group)
        msgs.warn('Only find one object no FOF matching is needed')

    gfrac = np.zeros(nfound)
    for jj in range(nobj):
        this_group = group == uni_group[jj]
        gfrac[this_group] = np.median(fracpos[this_group])

    uni_frac = gfrac[uni_ind]

    sobjs_align = sobjs.copy()
    # Now fill in the missing objects and their traces
    for iobj in range(nobj):
        for iord in range(norders):
            # Is there an object on this order that grouped into the current group in question?
            on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx
                                                    == iord)
            if not np.any(on_slit):
                # Add this to the sobjs_align, and assign required tags
                thisobj = specobjs.SpecObj(frameshape,
                                           slit_spat_pos[iord, :],
                                           slit_spec_pos,
                                           det=sobjs_align[0].det,
                                           setup=sobjs_align[0].setup,
                                           slitid=(iord + 1),
                                           scidx=sobjs_align[0].scidx,
                                           objtype=sobjs_align[0].objtype)
                thisobj.ech_orderindx = iord
                thisobj.spat_fracpos = uni_frac[iobj]
                thisobj.trace_spat = slit_left[:,
                                               iord] + slit_width[:, iord] * uni_frac[
                                                   iobj]  # new trace
                thisobj.trace_spec = spec_vec
                thisobj.spat_pixpos = thisobj.trace_spat[specmid]
                thisobj.set_idx()
                # Use the real detections of this objects for the FWHM
                this_group = group == uni_group[iobj]
                # Assign to the fwhm of the nearest detected order
                imin = np.argmin(
                    np.abs(sobjs_align[this_group].ech_orderindx - iord))
                thisobj.fwhm = sobjs_align[imin].fwhm
                thisobj.maskwidth = sobjs_align[imin].maskwidth
                thisobj.ech_fracpos = uni_frac[iobj]
                thisobj.ech_group = uni_group[iobj]
                thisobj.ech_usepca = True
                sobjs_align.add_sobj(thisobj)
                group = np.append(group, uni_group[iobj])
                gfrac = np.append(gfrac, uni_frac[iobj])
            else:
                # ToDo fix specobjs to get rid of these crappy loops!
                for spec in sobjs_align[on_slit]:
                    spec.ech_fracpos = uni_frac[iobj]
                    spec.ech_group = uni_group[iobj]
                    spec.ech_usepca = False

    # Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order
    # respectively
    sobjs_sort = specobjs.SpecObjs()
    for iobj in range(nobj):
        this_group = group == uni_group[iobj]
        this_sobj = sobjs_align[this_group]
        sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)])

    # Loop over the objects and perform a quick and dirty extraction to assess S/N.
    varimg = utils.calc_ivar(ivar)
    flux_box = np.zeros((nspec, norders, nobj))
    ivar_box = np.zeros((nspec, norders, nobj))
    mask_box = np.zeros((nspec, norders, nobj))
    SNR_arr = np.zeros((norders, nobj))
    for iobj in range(nobj):
        for iord in range(norders):
            indx = (sobjs_sort.ech_group
                    == uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord)
            spec = sobjs_sort[indx]
            thismask = ordermask == (iord + 1)
            inmask_iord = inmask & thismask
            box_rad_pix = box_radius / plate_scale_ord[iord]
            flux_tmp = extract.extract_boxcar(image * inmask_iord,
                                              spec.trace_spat,
                                              box_rad_pix,
                                              ycen=spec.trace_spec)
            var_tmp = extract.extract_boxcar(varimg * inmask_iord,
                                             spec.trace_spat,
                                             box_rad_pix,
                                             ycen=spec.trace_spec)
            ivar_tmp = utils.calc_ivar(var_tmp)
            pixtot = extract.extract_boxcar(ivar * 0 + 1.0,
                                            spec.trace_spat,
                                            box_rad_pix,
                                            ycen=spec.trace_spec)
            mask_tmp = (extract.extract_boxcar(ivar * inmask_iord == 0.0,
                                               spec.trace_spat,
                                               box_rad_pix,
                                               ycen=spec.trace_spec) != pixtot)
            flux_box[:, iord, iobj] = flux_tmp * mask_tmp
            ivar_box[:, iord, iobj] = np.fmax(ivar_tmp * mask_tmp, 0.0)
            mask_box[:, iord, iobj] = mask_tmp
            (mean, med_sn, stddev) = sigma_clipped_stats(
                flux_box[mask_tmp, iord, iobj] *
                np.sqrt(ivar_box[mask_tmp, iord, iobj]),
                sigma_lower=5.0,
                sigma_upper=5.0)
            SNR_arr[iord, iobj] = med_sn

    # Purge objects with low SNR and that don't show up in enough orders
    keep_obj = np.zeros(nobj, dtype=bool)
    sobjs_trim = specobjs.SpecObjs()
    uni_group_trim = np.array([], dtype=int)
    uni_frac_trim = np.array([], dtype=float)
    for iobj in range(nobj):
        if (np.sum(SNR_arr[:, iobj] > min_snr) >= nabove_min_snr):
            keep_obj[iobj] = True
            ikeep = sobjs_sort.ech_group == uni_group[iobj]
            sobjs_trim.add_sobj(sobjs_sort[ikeep])
            uni_group_trim = np.append(uni_group_trim, uni_group[iobj])
            uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj])
        else:
            msgs.info(
                'Purging object #{:d}'.format(iobj) +
                ' which does not satisfy min_snr > {:5.2f}'.format(min_snr) +
                ' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) +
                ' orders')

    nobj_trim = np.sum(keep_obj)
    if nobj_trim == 0:
        return specobjs.SpecObjs()

    SNR_arr_trim = SNR_arr[:, keep_obj]

    # Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA
    for iobj in range(nobj_trim):
        SNR_now = SNR_arr_trim[:, iobj]
        indx = (sobjs_trim.ech_group == uni_group_trim[iobj])
        # PCA interp/extrap if:
        #      (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca)
        #                                 OR
        #      (if this order was not originally traced by the object finding, see above)
        usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) &
                  (SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca
        # ToDo fix specobjs to get rid of these crappy loops!
        for iord, spec in enumerate(sobjs_trim[indx]):
            spec.ech_usepca = usepca[iord]
            if usepca[iord]:
                msgs.info('Using PCA to predict trace for object #{:d}'.format(
                    iobj) + ' on order #{:d}'.format(iord))

    sobjs_final = sobjs_trim.copy()
    # Loop over the objects one by one and adjust/predict the traces
    npoly_cen = 3
    pca_fits = np.zeros((nspec, norders, nobj_trim))
    for iobj in range(nobj_trim):
        igroup = sobjs_final.ech_group == uni_group_trim[iobj]
        # PCA predict the masked orders which were not traced
        pca_fits[:, :, iobj] = pca_trace((sobjs_final[igroup].trace_spat).T,
                                         usepca=None,
                                         npca=npca,
                                         npoly_cen=npoly_cen)
        # usepca = sobjs_final[igroup].ech_usepca,
        # Perform iterative flux weighted centroiding using new PCA predictions
        xinit_fweight = pca_fits[:, :, iobj].copy()
        inmask_now = inmask & (ordermask > 0)
        xfit_fweight = extract.iter_tracefit(image,
                                             xinit_fweight,
                                             ncoeff,
                                             inmask=inmask_now,
                                             show_fits=show_fits)
        # Perform iterative Gaussian weighted centroiding
        xinit_gweight = xfit_fweight.copy()
        xfit_gweight = extract.iter_tracefit(image,
                                             xinit_gweight,
                                             ncoeff,
                                             inmask=inmask_now,
                                             gweight=True,
                                             show_fits=show_fits)
        # Assign the new traces
        for iord, spec in enumerate(sobjs_final[igroup]):
            spec.trace_spat = xfit_gweight[:, iord]
            spec.spat_pixpos = spec.trace_spat[specmid]

    # Set the IDs
    sobjs_final.set_idx()
    if show_trace:
        viewer, ch = ginga.show_image(objminsky * (ordermask > 0))
        for iobj in range(nobj_trim):
            for iord in range(norders):
                ginga.show_trace(viewer,
                                 ch,
                                 pca_fits[:, iord, iobj],
                                 str(uni_frac[iobj]),
                                 color='yellow')

        for spec in sobjs_trim:
            color = 'green' if spec.ech_usepca else 'magenta'
            ginga.show_trace(viewer,
                             ch,
                             spec.trace_spat,
                             spec.idx,
                             color=color)

        #for spec in sobjs_final:
        #    color = 'red' if spec.ech_usepca else 'green'
        #    ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)

    return sobjs_final
Пример #7
0
def main(args):

    import os
    import numpy as np

    from IPython import embed

    from pypeit import pypeit
    from pypeit import pypeitsetup
    from pypeit.core import framematch
    from pypeit import msgs

    spec = args.spectrograph

    # Config the run
    cfg_lines = ['[rdx]']
    cfg_lines += ['    spectrograph = {0}'.format(spec)]
    cfg_lines += [
        '    redux_path = {0}_A'.format(os.path.join(os.getcwd(), spec))
    ]
    if args.slit_spat is not None:
        msgs.info("--slit_spat provided.  Ignoring --det")
    else:
        cfg_lines += ['    detnum = {0}'.format(args.det)]
    # Restrict on slit
    if args.slit_spat is not None:
        cfg_lines += ['    slitspatnum = {0}'.format(args.slit_spat)]
    # Allow for bad headers
    if args.ignore_headers:
        cfg_lines += ['    ignore_bad_headers = True']
    cfg_lines += ['[scienceframe]']
    cfg_lines += ['    [[process]]']
    cfg_lines += ['        mask_cr = False']
    # Calibrations
    cfg_lines += ['[baseprocess]']
    cfg_lines += ['    use_biasimage = False']
    cfg_lines += ['[calibrations]']
    # Input pixel flat?
    if args.user_pixflat is not None:
        cfg_lines += ['    [[flatfield]]']
        cfg_lines += ['        pixelflat_file = {0}'.format(args.user_pixflat)]
    # Reduction restrictions
    cfg_lines += ['[reduce]']
    cfg_lines += ['    [[extraction]]']
    cfg_lines += ['         skip_optimal = True']
    # Set boxcar radius
    if args.box_radius is not None:
        cfg_lines += ['    boxcar_radius = {0}'.format(args.box_radius)]
    cfg_lines += ['    [[findobj]]']
    cfg_lines += ['         skip_second_find = True']

    # Data files
    data_files = [
        os.path.join(args.full_rawpath, args.arc),
        os.path.join(args.full_rawpath, args.flat),
        os.path.join(args.full_rawpath, args.science)
    ]

    # Setup
    ps = pypeitsetup.PypeItSetup(data_files,
                                 path='./',
                                 spectrograph_name=spec,
                                 cfg_lines=cfg_lines)
    ps.build_fitstbl()
    # TODO -- Get the type_bits from  'science'
    bm = framematch.FrameTypeBitMask()
    file_bits = np.zeros(3, dtype=bm.minimum_dtype())
    file_bits[0] = bm.turn_on(file_bits[0], ['arc', 'tilt'])
    file_bits[1] = bm.turn_on(
        file_bits[1], ['pixelflat', 'trace', 'illumflat']
        if args.user_pixflat is None else ['trace', 'illumflat'])
    file_bits[2] = bm.turn_on(file_bits[2], 'science')

    # PypeItSetup sorts according to MJD
    #   Deal with this
    asrt = []
    for ifile in data_files:
        bfile = os.path.basename(ifile)
        idx = ps.fitstbl['filename'].data.tolist().index(bfile)
        asrt.append(idx)
    asrt = np.array(asrt)

    # Set bits
    ps.fitstbl.set_frame_types(file_bits[asrt])
    ps.fitstbl.set_combination_groups()
    # Extras
    ps.fitstbl['setup'] = 'A'

    # Write
    ofiles = ps.fitstbl.write_pypeit(configs='A',
                                     write_bkg_pairs=True,
                                     cfg_lines=cfg_lines)
    if len(ofiles) > 1:
        msgs.error("Bad things happened..")

    # Instantiate the main pipeline reduction object
    pypeIt = pypeit.PypeIt(ofiles[0],
                           verbosity=2,
                           reuse_masters=True,
                           overwrite=True,
                           logname='mos.log',
                           show=False)
    # Run
    pypeIt.reduce_all()
    msgs.info('Data reduction complete')
    # QA HTML
    msgs.info('Generating QA HTML')
    pypeIt.build_qa()

    return 0
Пример #8
0
    def compound_meta(self, headarr, meta_key):
        """
        Methods to generate metadata requiring interpretation of the header
        data, instead of simply reading the value of a header card.

        Args:
            headarr (:obj:`list`):
                List of `astropy.io.fits.Header`_ objects.
            meta_key (:obj:`str`):
                Metadata keyword to construct.

        Returns:
            object: Metadata value read from the header(s).
        """
        # Populate the idname based on the header information of LUCI
        # This is an implicit way of pre-typing without adding too many
        # variables to the self.meta.
        if meta_key == 'idname':
            targetname = (headarr[0].get('OBJECT'))
            dispname = (headarr[0].get('GRATNAME'))
            calib_unit = (headarr[0].get('CALIB'))
            filter1 = (headarr[0].get('FILTER1'))
            filter2 = (headarr[0].get('FILTER2'))
            lamp1 = (headarr[0].get('LAMP1'))
            lamp2 = (headarr[0].get('LAMP2'))
            lamp3 = (headarr[0].get('LAMP3'))
            lamp4 = (headarr[0].get('LAMP4'))
            lamp5 = (headarr[0].get('LAMP5'))
            lamp6 = (headarr[0].get('LAMP6'))

            # object frame -> will be typed as science
            # This currently includes sky flats, science and standard images
            # We will guess standards using the beginning of their names.
            if ((dispname != 'Mirror') and (calib_unit == False)
                    and (lamp1 == False) and (lamp2 == False)
                    and (lamp3 == False) and (lamp4 == False)
                    and (lamp5 == False) and (lamp6 == False)):
                if (targetname[:3] == 'HIP' or targetname[:2] == 'HD'
                        or targetname[:5] == 'Feige'):
                    return 'standard'
                else:
                    return 'object'
            # flat frames -> will be typed as pixelflat, trace
            elif ((calib_unit == True)
                  and ((lamp4 == True) or (lamp5 == True) or (lamp6 == True))):
                return 'flat'
            # arcs -> will be typed as arc, tilt
            elif ((dispname != 'Mirror') and (calib_unit == True)
                  and ((lamp1 == True) or (lamp2 == True) or (lamp3 == True))):
                return 'arc'
            # pixelflat off -> will be typed as bias
            elif ((dispname != 'Mirror') and (calib_unit == True)
                  and (lamp1 == False) and (lamp2 == False)
                  and (lamp3 == False) and (lamp4 == False)
                  and (lamp5 == False) and (lamp6 == False)
                  and (filter1 != 'blind') and (filter2 != 'blind')):
                return 'flat_off'
            # darks -> will not be typed currently
            elif ((filter1 == 'blind') or (filter2 == 'blind')):
                return 'dark'
        msgs.error("Not ready for this compound meta")
Пример #9
0
def read_gmos(raw_file, det=1):
    """
    Read the GMOS data file

    Parameters
    ----------
    raw_file : str
      Filename
    detector_par : ParSet
      Needed for numamplifiers if not other things
    det : int, optional
      Detector number; Default = 1

    Returns
    -------
    array : ndarray
      Combined image 
    header : FITS header
    sections : list
      List of datasec, oscansec, ampsec sections
    """

    # Check for file; allow for extra .gz, etc. suffix
    fil = glob.glob(raw_file + '*')
    if len(fil) != 1:
        msgs.error("Found {:d} files matching {:s}".format(len(fil)))

    # Read
    msgs.info("Reading GMOS file: {:s}".format(fil[0]))
    hdu = fits.open(fil[0])
    head0 = hdu[0].header
    head1 = hdu[1].header

    # Number of amplifiers (could pull from DetectorPar but this avoids needing the spectrograph, e.g. view_fits)
    numamp = (len(hdu) - 1) // 3

    # Setup for datasec, oscansec
    dsec = []
    osec = []

    # get the x and y binning factors...
    binning = head1['CCDSUM']
    xbin, ybin = [int(ibin) for ibin in binning.split(' ')]

    # First read over the header info to determine the size of the output array...
    datasec = head1['DATASEC']
    x1, x2, y1, y2 = np.array(parse.load_sections(datasec,
                                                  fmt_iraf=False)).flatten()
    biassec = head1['BIASSEC']
    b1, b2, b3, b4 = np.array(parse.load_sections(biassec,
                                                  fmt_iraf=False)).flatten()
    nxb = b2 - b1 + 1

    # determine the output array size...
    nx = (x2 - x1 + 1) * numamp + nxb * numamp
    ny = y2 - y1 + 1

    # allocate output array...
    array = np.zeros((nx, ny))

    if numamp == 2:
        if det == 1:  # BLUEST DETECTOR
            order = range(6, 4, -1)
        elif det == 2:  # BLUEST DETECTOR
            order = range(3, 5)
        elif det == 3:  # BLUEST DETECTOR
            order = range(1, 3)
    elif numamp == 4:
        if det == 1:  # BLUEST DETECTOR
            order = range(12, 8, -1)
        elif det == 2:  # BLUEST DETECTOR
            order = range(8, 4, -1)
        elif det == 3:  # BLUEST DETECTOR
            order = range(4, 0, -1)
    else:
        debugger.set_trace()

    # insert extensions into master image...
    for kk, jj in enumerate(order):

        # grab complete extension...
        data, overscan, datasec, biassec, x1, x2 = gemini_read_amp(hdu, jj)
        #, linebias=linebias, nobias=nobias, $
        #x1=x1, x2=x2, y1=y1, y2=y2, gaindata=gaindata)
        # insert components into output array...
        inx = data.shape[0]
        xs = inx * kk
        xe = xs + inx

        # insert data...
        # Data section
        #section = '[:,{:d}:{:d}]'.format(xs, xe)  # Eliminate lines
        section = '[{:d}:{:d},:]'.format(xs, xe)  # Eliminate lines
        dsec.append(section)
        array[xs:xe, :] = np.flipud(data)

        #; insert postdata...
        xs = nx - numamp * nxb + kk * nxb
        xe = xs + nxb
        #debugger.set_trace()
        #section = '[:,{:d}:{:d}]'.format(xs, xe)
        osection = '[{:d}:{:d},:]'.format(xs, xe)  # TRANSPOSED FOR WHAT COMES
        osec.append(osection)
        array[xs:xe, :] = overscan

    # make sure BZERO is a valid integer for IRAF
    obzero = head1['BZERO']
    #head0['O_BZERO'] = obzero
    head0['BZERO'] = 32768 - obzero

    # Return, transposing array back to goofy Python indexing
    return array, head0, (dsec, osec)
Пример #10
0
    def get_std(self, multi_spec_det=None):
        """
        Return the standard star from this Specobjs. For MultiSlit this
        will be a single specobj in SpecObjs container, for Echelle it
        will be the standard for all the orders.

        Args:
            multi_spec_det (list):
                If there are multiple detectors arranged in the spectral direction, return the sobjs for
                the standard on each detector.

        Returns:
            SpecObj or SpecObjs or None

        """
        # Is this MultiSlit or Echelle
        pypeline = (self.PYPELINE)[0]
        if 'MultiSlit' in pypeline or 'IFU' in pypeline:
            # Have to do a loop to extract the counts for all objects
            if self.OPT_COUNTS[0] is not None:
                SNR = np.median(self.OPT_COUNTS *
                                np.sqrt(self.OPT_COUNTS_IVAR),
                                axis=1)
            elif self.BOX_COUNTS[0] is not None:
                SNR = np.median(self.BOX_COUNTS *
                                np.sqrt(self.BOX_COUNTS_IVAR),
                                axis=1)
            else:
                return None
            # For multiple detectors grab the requested detectors
            if multi_spec_det is not None:
                sobjs_std = SpecObjs(header=self.header)
                # Now append the maximum S/N object on each detector
                for idet in multi_spec_det:
                    this_det = self.DET == idet
                    istd = SNR[this_det].argmax()
                    sobjs_std.add_sobj(self[this_det][istd])
            else:  # For normal multislit take the brightest object
                istd = SNR.argmax()
                # Return
                sobjs_std = SpecObjs(specobjs=[self[istd]], header=self.header)
            sobjs_std.header = self.header
            return sobjs_std
        elif 'Echelle' in pypeline:
            uni_objid = np.unique(
                self.ECH_FRACPOS)  # A little risky using floats
            uni_order = np.unique(self.ECH_ORDER)
            nobj = len(uni_objid)
            norders = len(uni_order)
            # Build up S/N
            SNR = np.zeros((norders, nobj))
            for iobj in range(nobj):
                for iord in range(norders):
                    ind = (self.ECH_FRACPOS == uni_objid[iobj]) & (
                        self.ECH_ORDER == uni_order[iord])
                    spec = self[ind]
                    # Grab SNR
                    if self.OPT_COUNTS[0] is not None:
                        SNR[iord,
                            iobj] = np.median(spec[0].OPT_COUNTS *
                                              np.sqrt(spec[0].OPT_COUNTS_IVAR))
                    elif self.BOX_COUNTS[0] is not None:
                        SNR[iord,
                            iobj] = np.median(spec[0].BOX_COUNTS *
                                              np.sqrt(spec[0].BOX_COUNTS_IVAR))
                    else:
                        return None
            # Maximize S/N
            SNR_all = np.sqrt(np.sum(SNR**2, axis=0))
            objid_std = uni_objid[SNR_all.argmax()]
            # Finish
            indx = self.ECH_FRACPOS == objid_std
            # Return
            sobjs_std = SpecObjs(specobjs=self[indx], header=self.header)
            sobjs_std.header = self.header
            return sobjs_std
        else:
            msgs.error('Unknown pypeline')
Пример #11
0
    def echelle_2dfit(self, wv_calib, debug=False, skip_QA=False):
        """
        Fit a two-dimensional wavelength solution for echelle data.

        Primarily a wrapper for :func:`pypeit.core.arc.fit2darc`,
        using data unpacked from the ``wv_calib`` dictionary.

        Args:
            wv_calib (:class:`pypeit.wavecalib.WaveCalib`):
                Wavelength calibration object
            debug (:obj:`bool`, optional):
                Show debugging info
            skip_QA (:obj:`bool`, optional):
                Flag to skip construction of the nominal QA plots.

        Returns:
            :class:`pypeit.fitting.PypeItFit`: object containing information from 2-d fit.
        """
        if self.spectrograph.pypeline != 'Echelle':
            msgs.error(
                'Cannot execute echelle_2dfit for a non-echelle spectrograph.')

        msgs.info('Fitting 2-d wavelength solution for echelle....')
        all_wave = np.array([], dtype=float)
        all_pixel = np.array([], dtype=float)
        all_order = np.array([], dtype=float)

        # Obtain a list of good slits
        ok_mask_idx = np.where(np.invert(self.wvc_bpm))[0]
        ok_mask_order = self.slits.slitord_id[ok_mask_idx]
        nspec = self.msarc.image.shape[0]
        # Loop
        for ii in range(wv_calib.nslits):
            iorder = self.slits.ech_order[ii]
            if iorder not in ok_mask_order:
                continue
            # Slurp
            mask_now = wv_calib.wv_fits[ii].pypeitfit.bool_gpm
            all_wave = np.append(all_wave,
                                 wv_calib.wv_fits[ii]['wave_fit'][mask_now])
            all_pixel = np.append(all_pixel,
                                  wv_calib.wv_fits[ii]['pixel_fit'][mask_now])
            all_order = np.append(
                all_order,
                np.full_like(wv_calib.wv_fits[ii]['pixel_fit'][mask_now],
                             float(iorder)))

        # Fit
        # THIS NEEDS TO BE DEVELOPED
        fit2d = arc.fit2darc(all_wave,
                             all_pixel,
                             all_order,
                             nspec,
                             nspec_coeff=self.par['ech_nspec_coeff'],
                             norder_coeff=self.par['ech_norder_coeff'],
                             sigrej=self.par['ech_sigrej'],
                             debug=debug)

        self.steps.append(inspect.stack()[0][3])

        # QA
        # TODO -- TURN QA BACK ON!
        #skip_QA = True
        if not skip_QA:
            outfile_global = qa.set_qa_filename(self.master_key,
                                                'arc_fit2d_global_qa',
                                                out_dir=self.qa_path)
            arc.fit2darc_global_qa(fit2d, nspec, outfile=outfile_global)
            outfile_orders = qa.set_qa_filename(self.master_key,
                                                'arc_fit2d_orders_qa',
                                                out_dir=self.qa_path)
            arc.fit2darc_orders_qa(fit2d, nspec, outfile=outfile_orders)

        return fit2d
Пример #12
0
def main(args):
    """
    Executes telluric correction.
    """

    # Determine the spectrograph
    header = fits.getheader(args.spec1dfile)
    spectrograph = load_spectrograph(header['PYP_SPEC'])
    spectrograph_def_par = spectrograph.default_pypeit_par()

    # If the .tell file was passed in read it and overwrite default parameters
    if args.tell_file is not None:
        cfg_lines = read_tellfile(args.tell_file)
        par = pypeitpar.PypeItPar.from_cfg_lines(
            cfg_lines=spectrograph_def_par.to_config(), merge_with=cfg_lines)
    else:
        par = spectrograph_def_par

    # If args was provided override defaults. Note this does undo .tell file
    if args.objmodel is not None:
        par['tellfit']['objmodel'] = args.objmodel
    if args.pca_file is not None:
        par['tellfit']['pca_file'] = args.pca_file
    if args.redshift is not None:
        par['tellfit']['redshift'] = args.redshift

    if args.tell_grid is not None:
        par['tellfit']['tell_grid'] = args.tell_grid
    elif par['sensfunc']['IR']['telgridfile'] is not None:
        par['tellfit']['tell_grid'] = par['sensfunc']['IR']['telgridfile']
    else:
        msgs.warn('No telluric grid file given. Using {:}'.format(
            'TelFit_MaunaKea_3100_26100_R20000.fits'))
        par['tellfit']['tell_grid'] = resource_filename(
            'pypeit', '/data/telluric/TelFit_MaunaKea_3100_26100_R20000.fits')

    # Write the par to disk
    print("Writing the parameters to {}".format(args.par_outfile))
    par['tellfit'].to_config('telluric.par',
                             section_name='tellfit',
                             include_descr=False)

    # Parse the output filename
    outfile = (os.path.basename(args.spec1dfile)).replace(
        '.fits', '_tellcorr.fits')
    modelfile = (os.path.basename(args.spec1dfile)).replace(
        '.fits', '_tellmodel.fits')

    # Run the telluric fitting procedure.
    if par['tellfit']['objmodel'] == 'qso':
        # run telluric.qso_telluric to get the final results
        TelQSO = telluric.qso_telluric(
            args.spec1dfile,
            par['tellfit']['tell_grid'],
            par['tellfit']['pca_file'],
            par['tellfit']['redshift'],
            modelfile,
            outfile,
            npca=par['tellfit']['npca'],
            pca_lower=par['tellfit']['pca_lower'],
            pca_upper=par['tellfit']['pca_upper'],
            bounds_norm=par['tellfit']['bounds_norm'],
            tell_norm_thresh=par['tellfit']['tell_norm_thresh'],
            only_orders=par['tellfit']['only_orders'],
            bal_wv_min_max=par['tellfit']['bal_wv_min_max'],
            debug_init=args.debug,
            disp=args.debug,
            debug=args.debug,
            show=args.plot)
    elif par['tellfit']['objmodel'] == 'star':
        TelStar = telluric.star_telluric(
            args.spec1dfile,
            par['tellfit']['tell_grid'],
            modelfile,
            outfile,
            star_type=par['tellfit']['star_type'],
            star_mag=par['tellfit']['star_mag'],
            star_ra=par['tellfit']['star_ra'],
            star_dec=par['tellfit']['star_dec'],
            func=par['tellfit']['func'],
            model=par['tellfit']['model'],
            polyorder=par['tellfit']['polyorder'],
            only_orders=par['tellfit']['only_orders'],
            mask_abs_lines=par['tellfit']['mask_abs_lines'],
            delta_coeff_bounds=par['tellfit']['delta_coeff_bounds'],
            minmax_coeff_bounds=par['tellfit']['minmax_coeff_bounds'],
            debug_init=args.debug,
            disp=args.debug,
            debug=args.debug,
            show=args.plot)
    elif par['tellfit']['objmodel'] == 'poly':
        TelPoly = telluric.poly_telluric(
            args.spec1dfile,
            par['tellfit']['tell_grid'],
            modelfile,
            outfile,
            z_obj=par['tellfit']['redshift'],
            func=par['tellfit']['func'],
            model=par['tellfit']['model'],
            polyorder=par['tellfit']['polyorder'],
            fit_wv_min_max=par['tellfit']['fit_wv_min_max'],
            mask_lyman_a=par['tellfit']['mask_lyman_a'],
            delta_coeff_bounds=par['tellfit']['delta_coeff_bounds'],
            minmax_coeff_bounds=par['tellfit']['minmax_coeff_bounds'],
            only_orders=par['tellfit']['only_orders'],
            debug_init=args.debug,
            disp=args.debug,
            debug=args.debug,
            show=args.plot)
    else:
        msgs.error(
            "Object model is not supported yet. Please choose one of 'qso', 'star', 'poly'."
        )
def ech_load_specobj(fname, order=None):
    """ Load a spec1d file into a list of SpecObjExp objects
    Parameters
    ----------
    fname : str

    Returns
    -------
    specObjs : list of SpecObjExp
    head0
    """
    #if order is None:
    #    msgs.warn('You did not specify an order. Return specObjs with all orders.')
    #    specObjs, head0 = load.load_specobj(fname)
    #    return specObjs, head0

    speckeys = [
        'WAVE', 'SKY', 'MASK', 'FLAM', 'FLAM_IVAR', 'FLAM_SIG', 'COUNTS_IVAR',
        'COUNTS'
    ]
    #
    specObjs = []
    hdulist = fits.open(fname)
    head0 = hdulist[0].header
    for hdu in hdulist:
        if hdu.name == 'PRIMARY':
            continue
        #elif hdu.name[8:17] != 'ORDER'+'{0:04}'.format(order):
        #    continue
        # Parse name
        idx = hdu.name
        objp = idx.split('-')
        if objp[-1][0:3] == 'DET':
            det = int(objp[-1][3:])
        else:
            det = int(objp[-1][1:])
        if objp[-2][:5] == 'ORDER':
            iord = int(objp[-2][5:])
        else:
            msgs.warn('Loading longslit data ?')
            iord = int(-1)
        # if order is not None and iord !=order then do not return this extenction
        # if order is None return all extensions
        # if order is not None and iord ==order then only return the specific order you want.
        if (order is not None) and (iord != order):
            continue
        # Load data
        spec = Table(hdu.data)
        shape = (len(spec), 1024)  # 2nd number is dummy
        # New and wrong
        try:
            specobj = specobjs.SpecObj(shape, None, None, idx=idx)
        except:
            debugger.set_trace()
            msgs.error("BUG ME")
        # Add order number
        specobj.ech_orderindx = iord
        # ToDo: need to changed to the real order number?
        specobj.ech_order = iord
        # Add trace
        try:
            specobj.trace_spat = spec['TRACE']
        except:
            # KLUDGE!
            specobj.trace_spat = np.arange(len(spec['BOX_WAVE']))
        # Add spectrum
        if 'BOX_COUNTS' in spec.keys():
            for skey in speckeys:
                try:
                    specobj.boxcar[skey] = spec['BOX_{:s}'.format(skey)].data
                except KeyError:
                    pass
            # Add units on wave
            specobj.boxcar['WAVE'] = specobj.boxcar['WAVE'] * units.AA

        if 'OPT_COUNTS' in spec.keys():
            for skey in speckeys:
                try:
                    specobj.optimal[skey] = spec['OPT_{:s}'.format(skey)].data
                except KeyError:
                    pass
            # Add units on wave
            specobj.optimal['WAVE'] = specobj.optimal['WAVE'] * units.AA
        # Append
        specObjs.append(specobj)
    # Return
    return specObjs, head0
Пример #14
0
    def build_wv_calib(self, arccen, method, skip_QA=False):
        """
        Main routine to generate the wavelength solutions in a loop over slits
        Wrapper to arc.simple_calib or arc.calib_with_arclines

        self.maskslits is updated for slits that fail

        Args:
            method : str
              'simple' -- arc.simple_calib
              'arclines' -- arc.calib_with_arclines
              'holy-grail' -- wavecal.autoid.HolyGrail
              'reidentify' -- wavecal.auotid.ArchiveReid
              'full_template' -- wavecal.auotid.full_template
            skip_QA (bool, optional)

        Returns:
            dict:  self.wv_calib
        """
        # Obtain a list of good slits
        ok_mask = np.where(~self.maskslits)[0]

        # Obtain calibration for all slits
        if method == 'simple':
            # Should only run this on 1 slit
            #self.par['n_first'] = 2
            #self.par['n_final'] = 3
            #self.par['func'] = 'legendre'
            #self.par['sigrej_first'] = 2.
            #self.par['sigrej_final'] = 3.
            #self.par['match_toler'] = 3.

            #CuI = wavecal.waveio.load_line_list('CuI', use_ion=True, NIST=True)
            #ArI = wavecal.waveio.load_line_list('ArI', use_ion=True, NIST=True)
            #ArII = wavecal.waveio.load_line_list('ArII', use_ion=True, NIST=True)
            #llist = vstack([CuI, ArI, ArII])
            lines = self.par['lamps']
            line_lists = waveio.load_line_lists(lines)

            self.wv_calib = arc.simple_calib_driver(self.msarc, line_lists, arccen, ok_mask,
                                                    nfitpix=self.par['nfitpix'],

                                                    IDpixels=self.par['IDpixels'],
                                                    IDwaves=self.par['IDwaves'])
        elif method == 'semi-brute':
            debugger.set_trace()  # THIS IS BROKEN
            final_fit = {}
            for slit in ok_mask:
                # HACKS BY JXP
                self.par['wv_cen'] = 8670.
                self.par['disp'] = 1.524
                # ToDO remove these hacks and use the parset in semi_brute
                best_dict, ifinal_fit = autoid.semi_brute(arccen[:, slit],
                                                                  self.par['lamps'], self.par['wv_cen'],
                                                                  (self)['disp'],match_toler=self.par['match_toler'],
                                                                  func=self.par['func'],n_first=self.par['n_first'],
                                                                  sigrej_first=self.par['n_first'],
                                                                  n_final=self.par['n_final'],
                                                                  sigrej_final=self.par['sigrej_final'],
                                                                  sigdetect=self.par['sigdetect'],
                                                                  nonlinear_counts= self.nonlinear_counts)
                final_fit[str(slit)] = ifinal_fit.copy()
        elif method == 'basic':
            final_fit = {}
            for slit in ok_mask:
                status, ngd_match, match_idx, scores, ifinal_fit = \
                    autoid.basic(arccen[:, slit], self.par['lamps'], self.par['wv_cen'], self.par['disp'],
                                 nonlinear_counts = self.nonlinear_counts)
                final_fit[str(slit)] = ifinal_fit.copy()
                if status != 1:
                    self.maskslits[slit] = True
        elif method == 'holy-grail':
            # Sometimes works, sometimes fails
            arcfitter = autoid.HolyGrail(arccen, par=self.par, ok_mask=ok_mask)
            patt_dict, final_fit = arcfitter.get_results()
        elif method == 'reidentify':
            # Now preferred
            arcfitter = autoid.ArchiveReid(arccen, par=self.par, ok_mask=ok_mask)
            patt_dict, final_fit = arcfitter.get_results()
        elif method == 'full_template':
            # Now preferred
            if self.binspectral is None:
                msgs.error("You must specify binspectral for the full_template method!")
            final_fit = autoid.full_template(arccen, self.par, ok_mask, self.det,
                                                     self.binspectral,
                                                     nsnippet=self.par['nsnippet'])

        else:
            msgs.error('Unrecognized wavelength calibration method: {:}'.format(method))

        self.wv_calib = final_fit

        # Remake mask (*mainly for the QA that follows*)
        self.maskslits = self.make_maskslits(len(self.maskslits))
        ok_mask = np.where(~self.maskslits)[0]

        # QA
        if not skip_QA:
            for slit in ok_mask:
                outfile = qa.set_qa_filename(self.master_key, 'arc_fit_qa', slit=slit, out_dir=self.redux_path)
                autoid.arc_fit_qa(self.wv_calib[str(slit)], outfile = outfile)
        # Step
        self.steps.append(inspect.stack()[0][3])
        # Return
        return self.wv_calib
Пример #15
0
    def get_rawimage(self, raw_file, det):
        """
        Load up the raw image and generate a few other bits and pieces
        that are key for image processing

        Args:
            raw_file (str):
            det (int):

        Returns:
            tuple:
                raw_img (np.ndarray) -- Raw image for this detector
                hdu (astropy.io.fits.HDUList)
                exptime (float)
                rawdatasec_img (np.ndarray)
                oscansec_img (np.ndarray)

        """
        # Check for file; allow for extra .gz, etc. suffix
        fil = glob.glob(raw_file + '*')
        if len(fil) != 1:
            msgs.error("Found {:d} files matching {:s}".format(len(fil)))

        # Read
        msgs.info("Reading LBT/MODS file: {:s}".format(fil[0]))
        hdu = fits.open(fil[0])
        head = hdu[0].header

        # TODO These parameters should probably be stored in the detector par

        # Number of amplifiers (could pull from DetectorPar but this avoids needing the spectrograph, e.g. view_fits)
        numamp = 4

        # get the x and y binning factors...
        xbin, ybin = head['CCDXBIN'], head['CCDYBIN']

        datasize = head['DETSIZE']  # Unbinned size of detector full array
        _, nx_full, _, ny_full = np.array(
            parse.load_sections(datasize, fmt_iraf=False)).flatten()

        # Determine the size of the output array...
        nx, ny = int(nx_full / xbin), int(ny_full / ybin)
        nbias1 = 48
        nbias2 = 8240

        # allocate output array...
        array = hdu[
            0].data.T * 1.0  ## Convert to float in order to get it processed with procimg.py
        rawdatasec_img = np.zeros_like(array, dtype=int)
        oscansec_img = np.zeros_like(array, dtype=int)

        ## allocate datasec and oscansec to the image
        # apm 1
        rawdatasec_img[int(nbias1 / xbin):int(nx / 2), :int(ny / 2)] = 1
        oscansec_img[1:int(nbias1 / xbin), :int(
            ny / 2)] = 1  # exclude the first pixel since it always has problem

        # apm 2
        rawdatasec_img[int(nx / 2):int(nbias2 / xbin), :int(ny / 2)] = 2
        oscansec_img[int(nbias2 / xbin):nx - 1, :int(
            ny / 2)] = 2  # exclude the last pixel since it always has problem

        # apm 3
        rawdatasec_img[int(nbias1 / xbin):int(nx / 2), int(ny / 2):] = 3
        oscansec_img[
            1:int(nbias1 / xbin),
            int(ny /
                2):] = 3  # exclude the first pixel since it always has problem

        # apm 4
        rawdatasec_img[int(nx / 2):int(nbias2 / xbin), int(ny / 2):] = 4
        oscansec_img[
            int(nbias2 / xbin):nx - 1,
            int(ny /
                2):] = 4  # exclude the last pixel since it always has problem

        # Need the exposure time
        exptime = hdu[self.meta['exptime']['ext']].header[self.meta['exptime']
                                                          ['card']]
        # Return, transposing array back to orient the overscan properly
        return np.flipud(array), hdu, exptime, np.flipud(
            rawdatasec_img), np.flipud(oscansec_img)
def ech_objfind(image, ivar, ordermask, slit_left, slit_righ,inmask=None,plate_scale=0.2,npca=2,ncoeff = 5,min_snr=0.0,nabove_min_snr=0,
                pca_percentile=20.0,snr_pca=3.0,box_radius=2.0,show_peaks=False,show_fits=False,show_trace=False):


    if inmask is None:
        inmask = (ordermask > 0)


    frameshape = image.shape
    nspec = frameshape[0]
    norders = slit_left.shape[1]

    if isinstance(plate_scale,(float, int)):
        plate_scale_ord = np.full(norders, plate_scale)  # 0.12 binned by 3 spatially for HIRES
    elif isinstance(plate_scale,(np.ndarray, list, tuple)):
        if len(plate_scale) == norders:
            plate_scale_ord = plate_scale
        elif len(plate_scale) == 1:
            plate_scale_ord = np.full(norders, plate_scale[0])
        else:
            msgs.error('Invalid size for plate_scale. It must either have one element or norders elements')
    else:
        msgs.error('Invalid type for plate scale')

    specmid = nspec // 2
    slit_width = slit_righ - slit_left
    spec_vec = np.arange(nspec)
    slit_spec_pos = nspec/2.0
    slit_spat_pos = np.zeros((norders, 2))
    for iord in range(norders):
        slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec, slit_left[:,iord]), np.interp(slit_spec_pos, spec_vec, slit_righ[:,iord]))

    # Loop over orders and find objects
    sobjs = specobjs.SpecObjs()
    show_peaks=True
    show_fits=True
    # ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx!
    for iord  in range(norders):
        msgs.info('Finding objects on slit # {:d}'.format(iord + 1))
        thismask = ordermask == (iord + 1)
        inmask_iord = inmask & thismask
        specobj_dict = {'setup': 'HIRES', 'slitid': iord + 1, 'scidx': 0,'det': 1, 'objtype': 'science'}
        sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \
            extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks,
                            show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0)
        # ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord
        for spec in sobjs_slit:
            spec.ech_orderindx = iord
        sobjs.add_sobj(sobjs_slit)


    nfound = len(sobjs)

    # Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0"
    FOFSEP = 1.0 # separation of FOF algorithm in arcseconds
    FOF_frac = FOFSEP/(np.median(slit_width)*np.median(plate_scale_ord))

    # Feige: made the code also works for only one object found in one order
    # Run the FOF. We use fake coordinaes
    fracpos = sobjs.spat_fracpos
    ra_fake = fracpos/1000.0 # Divide all angles by 1000 to make geometry euclidian
    dec_fake = 0.0*fracpos
    if nfound>1:
        (ingroup, multgroup, firstgroup, nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac/1000.0)
        group = ingroup.copy()
        uni_group, uni_ind = np.unique(group, return_index=True)
        nobj = len(uni_group)
        msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects')
    elif nfound==1:
        group = np.zeros(1,dtype='int')
        uni_group, uni_ind = np.unique(group, return_index=True)
        nobj = len(group)
        msgs.warn('Only find one object no FOF matching is needed')

    gfrac = np.zeros(nfound)
    for jj in range(nobj):
        this_group = group == uni_group[jj]
        gfrac[this_group] = np.median(fracpos[this_group])

    uni_frac = gfrac[uni_ind]

    sobjs_align = sobjs.copy()
    # Now fill in the missing objects and their traces
    for iobj in range(nobj):
        for iord in range(norders):
            # Is there an object on this order that grouped into the current group in question?
            on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx == iord)
            if not np.any(on_slit):
                # Add this to the sobjs_align, and assign required tags
                thisobj = specobjs.SpecObj(frameshape, slit_spat_pos[iord,:], slit_spec_pos, det = sobjs_align[0].det,
                                           setup = sobjs_align[0].setup, slitid = (iord + 1),
                                           scidx = sobjs_align[0].scidx, objtype=sobjs_align[0].objtype)
                thisobj.ech_orderindx = iord
                thisobj.spat_fracpos = uni_frac[iobj]
                thisobj.trace_spat = slit_left[:,iord] + slit_width[:,iord]*uni_frac[iobj] # new trace
                thisobj.trace_spec = spec_vec
                thisobj.spat_pixpos = thisobj.trace_spat[specmid]
                thisobj.set_idx()
                # Use the real detections of this objects for the FWHM
                this_group = group == uni_group[iobj]
                # Assign to the fwhm of the nearest detected order
                imin = np.argmin(np.abs(sobjs_align[this_group].ech_orderindx - iord))
                thisobj.fwhm = sobjs_align[imin].fwhm
                thisobj.maskwidth = sobjs_align[imin].maskwidth
                thisobj.ech_fracpos = uni_frac[iobj]
                thisobj.ech_group = uni_group[iobj]
                thisobj.ech_usepca = True
                sobjs_align.add_sobj(thisobj)
                group = np.append(group, uni_group[iobj])
                gfrac = np.append(gfrac, uni_frac[iobj])
            else:
                # ToDo fix specobjs to get rid of these crappy loops!
                for spec in sobjs_align[on_slit]:
                    spec.ech_fracpos = uni_frac[iobj]
                    spec.ech_group = uni_group[iobj]
                    spec.ech_usepca = False

    # Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order
    # respectively
    sobjs_sort = specobjs.SpecObjs()
    for iobj in range(nobj):
        this_group = group == uni_group[iobj]
        this_sobj = sobjs_align[this_group]
        sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)])

    # Loop over the objects and perform a quick and dirty extraction to assess S/N.
    varimg = utils.calc_ivar(ivar)
    flux_box = np.zeros((nspec, norders, nobj))
    ivar_box = np.zeros((nspec, norders, nobj))
    mask_box = np.zeros((nspec, norders, nobj))
    SNR_arr = np.zeros((norders, nobj))
    for iobj in range(nobj):
        for iord in range(norders):
            indx = (sobjs_sort.ech_group == uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord)
            spec = sobjs_sort[indx]
            thismask = ordermask == (iord + 1)
            inmask_iord = inmask & thismask
            box_rad_pix = box_radius/plate_scale_ord[iord]
            flux_tmp  = extract.extract_boxcar(image*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
            var_tmp  = extract.extract_boxcar(varimg*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
            ivar_tmp = utils.calc_ivar(var_tmp)
            pixtot  = extract.extract_boxcar(ivar*0 + 1.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
            mask_tmp = (extract.extract_boxcar(ivar*inmask_iord == 0.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) != pixtot)
            flux_box[:,iord,iobj] = flux_tmp*mask_tmp
            ivar_box[:,iord,iobj] = np.fmax(ivar_tmp*mask_tmp,0.0)
            mask_box[:,iord,iobj] = mask_tmp
            (mean, med_sn, stddev) = sigma_clipped_stats(flux_box[mask_tmp,iord,iobj]*np.sqrt(ivar_box[mask_tmp,iord,iobj]),
                                                         sigma_lower=5.0,sigma_upper=5.0)
            SNR_arr[iord,iobj] = med_sn



    # Purge objects with low SNR and that don't show up in enough orders
    keep_obj = np.zeros(nobj,dtype=bool)
    sobjs_trim = specobjs.SpecObjs()
    uni_group_trim = np.array([],dtype=int)
    uni_frac_trim =  np.array([],dtype=float)
    for iobj in range(nobj):
        if (np.sum(SNR_arr[:,iobj] > min_snr) >= nabove_min_snr):
            keep_obj[iobj] = True
            ikeep = sobjs_sort.ech_group == uni_group[iobj]
            sobjs_trim.add_sobj(sobjs_sort[ikeep])
            uni_group_trim = np.append(uni_group_trim, uni_group[iobj])
            uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj])
        else:
            msgs.info('Purging object #{:d}'.format(iobj) + ' which does not satisfy min_snr > {:5.2f}'.format(min_snr) +
                      ' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) + ' orders')

    nobj_trim = np.sum(keep_obj)
    if nobj_trim == 0:
        return specobjs.SpecObjs()

    SNR_arr_trim = SNR_arr[:,keep_obj]

    # Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA
    for iobj in range(nobj_trim):
        SNR_now = SNR_arr_trim[:,iobj]
        indx = (sobjs_trim.ech_group == uni_group_trim[iobj])
        # PCA interp/extrap if:
        #      (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca)
        #                                 OR
        #      (if this order was not originally traced by the object finding, see above)
        usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) & (SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca
        # ToDo fix specobjs to get rid of these crappy loops!
        for iord, spec in enumerate(sobjs_trim[indx]):
            spec.ech_usepca = usepca[iord]
            if usepca[iord]:
                msgs.info('Using PCA to predict trace for object #{:d}'.format(iobj) + ' on order #{:d}'.format(iord))

    sobjs_final = sobjs_trim.copy()
    # Loop over the objects one by one and adjust/predict the traces
    npoly_cen = 3
    pca_fits = np.zeros((nspec, norders, nobj_trim))
    for iobj in range(nobj_trim):
        igroup = sobjs_final.ech_group == uni_group_trim[iobj]
        # PCA predict the masked orders which were not traced
        pca_fits[:,:,iobj] = pca_trace((sobjs_final[igroup].trace_spat).T, usepca = None, npca = npca, npoly_cen = npoly_cen)
        # usepca = sobjs_final[igroup].ech_usepca,
        # Perform iterative flux weighted centroiding using new PCA predictions
        xinit_fweight = pca_fits[:,:,iobj].copy()
        inmask_now = inmask & (ordermask > 0)
        xfit_fweight = extract.iter_tracefit(image, xinit_fweight, ncoeff, inmask = inmask_now, show_fits=show_fits)
        # Perform iterative Gaussian weighted centroiding
        xinit_gweight = xfit_fweight.copy()
        xfit_gweight = extract.iter_tracefit(image, xinit_gweight, ncoeff, inmask = inmask_now, gweight=True,show_fits=show_fits)
        # Assign the new traces
        for iord, spec in enumerate(sobjs_final[igroup]):
            spec.trace_spat = xfit_gweight[:,iord]
            spec.spat_pixpos = spec.trace_spat[specmid]


    # Set the IDs
    sobjs_final.set_idx()
    if show_trace:
        viewer, ch = ginga.show_image(objminsky*(ordermask > 0))
        for iobj in range(nobj_trim):
            for iord in range(norders):
                ginga.show_trace(viewer, ch, pca_fits[:,iord, iobj], str(uni_frac[iobj]), color='yellow')

        for spec in sobjs_trim:
            color = 'green' if spec.ech_usepca else 'magenta'
            ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)

        #for spec in sobjs_final:
        #    color = 'red' if spec.ech_usepca else 'green'
        #    ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)

    return sobjs_final
    def __init__(self, std_spec1d_file=None, sci_spec1d_file=None, sens_file=None,
                 std_specobjs=None, std_header=None, spectrograph=None,
                 telluric=False, setup=None, master_dir=None, mode=None,
                 star_type=None, star_mag=None, BALM_MASK_WID=5.0, nresln=None, debug=False):

        # Load standard files
        std_spectro = None
        self.std_spec1d_file = std_spec1d_file
        # Need to unwrap these (sometimes)..
        self.std_specobjs = std_specobjs
        self.std_header = std_header
        if self.std_spec1d_file is not None:
            self.std_specobjs, self.std_header = load.ech_load_specobj(self.std_spec1d_file)
            msgs.info('Loaded {0} spectra from the spec1d standard star file: {1}'.format(
                len(self.std_specobjs), self.std_spec1d_file))
            std_spectro = self.std_header['INSTRUME']

        try:
            self.std_ra = self.std_header['RA']
        except:
            self.std_ra = None
        try:
            self.std_dec = self.std_header['DEC']
        except:
            self.std_dec = None
        try:
            self.std_file = self.std_header['FILENAME']
        except:
            self.std_file = None

        # Load the science files
        sci_spectro = None
        self.sci_spec1d_file = sci_spec1d_file
        self.sci_specobjs = []
        self.sci_header = None
        if self.sci_spec1d_file is not None:
            self.sci_specobjs, self.sci_header = load.ech_load_specobj(self.sci_spec1d_file)
            msgs.info('Loaded {0} spectra from the spec1d science file: {1}'.format(
                len(self.sci_specobjs), self.sci_spec1d_file))
            sci_spectro = self.sci_header['INSTRUME']

        # Compare instruments if they exist
        if std_spectro is not None and sci_spectro is not None and std_spectro != sci_spectro:
            msgs.error('Standard spectra are not the same instrument as science!!')

        # Instantiate the spectrograph
        _spectrograph = spectrograph
        if _spectrograph is None:
            _spectrograph = std_spectro
            if _spectrograph is not None:
                msgs.info("Spectrograph set to {0} from standard file".format(_spectrograph))
        if _spectrograph is None:
            _spectrograph = sci_spectro
            if _spectrograph is not None:
                msgs.info("Spectrograph set to {0} from science file".format(_spectrograph))
        self.spectrograph = load_spectrograph(_spectrograph)

        # MasterFrame
        masterframe.MasterFrame.__init__(self, self.frametype, setup,
                                         master_dir=master_dir, mode=mode)
        # Get the extinction data
        self.extinction_data = None
        if self.spectrograph is not None:
            self.extinction_data \
                = flux.load_extinction_data(self.spectrograph.telescope['longitude'],
                                            self.spectrograph.telescope['latitude'])
        elif self.sci_header is not None and 'LON-OBS' in self.sci_header.keys():
            self.extinction_data \
                = flux.load_extinction_data(self.sci_header['LON-OBS'],
                                            self.sci_header['LAT-OBS'])

        # Once the spectrograph is instantiated, can also set the
        # extinction data
        # Parameters
        self.sens_file = sens_file

        # Set telluric option
        self.telluric = telluric

        # Main outputs
        self.sens_dict = None if self.sens_file is None \
            else self.load_master(self.sens_file)

        # Attributes
        self.steps = []

        # Key Internals
        self.std = None  # Standard star spectrum (SpecObj object)
        self.std_idx = None  # Nested indices for the std_specobjs list that corresponds
        # to the star!
        # Echelle key
        self.star_type = star_type
        self.star_mag = star_mag
        self.BALM_MASK_WID = BALM_MASK_WID
        self.nresln = nresln
        self.debug = debug
Пример #18
0
 def compound_meta(self, headarr, meta_key):
     if meta_key == 'mjd':
         time = headarr[0]['DATE']
         ttime = Time(time, format='isot')
         return ttime.mjd
     msgs.error("Not ready for this compound meta")
Пример #19
0
    def echelle_2dfit(self, wv_calib, debug=False, skip_QA=False):
        """
        Fit a two-dimensional wavelength solution for echelle data.

        Primarily a wrapper for :func:`pypeit.core.arc.fit2darc`,
        using data unpacked from the ``wv_calib`` dictionary.
        
        Args:
            wv_calib (:obj:`dict`):
                Wavelength calibration dictionary.  See ??
            debug (:obj:`bool`, optional):
                Show debugging info
            skip_QA (:obj:`bool`, optional):
                Flag to skip construction of the nominal QA plots.

        Returns:
            :obj:`dict`: Dictionary containing information from 2-d
            fit.
        """
        if self.spectrograph.pypeline != 'Echelle':
            msgs.error(
                'Cannot execute echelle_2dfit for a non-echelle spectrograph.')

        msgs.info('Fitting 2-d wavelength solution for echelle....')
        all_wave = np.array([], dtype=float)
        all_pixel = np.array([], dtype=float)
        all_order = np.array([], dtype=float)

        # Obtain a list of good slits
        ok_mask_idx = np.where(np.invert(self.wvc_bpm))[0]
        ok_mask_order = self.slits.slitord_id[ok_mask_idx]
        nspec = self.msarc.image.shape[0]
        for iorder in wv_calib.keys():  # Spatial based
            if int(iorder) not in ok_mask_order:
                continue
            #try:
            #    iorder, iindx = self.spectrograph.slit2order(self.spat_coo[self.slits.spatid_to_zero(int(islit))])
            #except:
            #    embed()
            mask_now = wv_calib[iorder]['mask']
            all_wave = np.append(all_wave,
                                 wv_calib[iorder]['wave_fit'][mask_now])
            all_pixel = np.append(all_pixel,
                                  wv_calib[iorder]['pixel_fit'][mask_now])
            all_order = np.append(
                all_order,
                np.full_like(wv_calib[iorder]['pixel_fit'][mask_now],
                             float(iorder)))

        # Fit
        fit2d_dict = arc.fit2darc(all_wave,
                                  all_pixel,
                                  all_order,
                                  nspec,
                                  nspec_coeff=self.par['ech_nspec_coeff'],
                                  norder_coeff=self.par['ech_norder_coeff'],
                                  sigrej=self.par['ech_sigrej'],
                                  debug=debug)

        self.steps.append(inspect.stack()[0][3])

        # QA
        if not skip_QA:
            outfile_global = qa.set_qa_filename(self.master_key,
                                                'arc_fit2d_global_qa',
                                                out_dir=self.qa_path)
            arc.fit2darc_global_qa(fit2d_dict, outfile=outfile_global)
            outfile_orders = qa.set_qa_filename(self.master_key,
                                                'arc_fit2d_orders_qa',
                                                out_dir=self.qa_path)
            arc.fit2darc_orders_qa(fit2d_dict, outfile=outfile_orders)

        return fit2d_dict
Пример #20
0
    def get_rawimage(self, raw_file, det):
        """
        Read a raw KCWI data frame

        NOTE: The amplifiers are arranged as follows:

        |   (0,ny)  --------- (nx,ny)
        |           | 3 | 4 |
        |           ---------
        |           | 1 | 2 |
        |     (0,0) --------- (nx, 0)

        Parameters
        ----------
        raw_file : str
            Filename
        det (int or None):
            Detector number

        Returns
        -------
        array : ndarray
            Combined image
        hdu : HDUList
            Opened fits file.
        sections : list
            List of datasec, oscansec, ampsec sections. datasec,
            oscansec needs to be for an *unbinned* image as per
            standard convention
        """
        # Check for file; allow for extra .gz, etc. suffix
        fil = glob.glob(raw_file + '*')
        if len(fil) != 1:
            msgs.error("Found {:d} files matching {:s}".format(
                len(fil), raw_file))

        # Read
        msgs.info("Reading KCWI file: {:s}".format(fil[0]))
        hdu = fits.open(fil[0])
        detpar = self.get_detector_par(hdu, det if det is None else 1)
        head0 = hdu[0].header
        raw_img = hdu[detpar['dataext']].data.astype(float)

        # Some properties of the image
        numamps = head0['NVIDINP']
        # Exposure time (used by ProcessRawImage)
        headarr = self.get_headarr(hdu)
        exptime = self.get_meta_value(headarr, 'exptime')

        # get the x and y binning factors...
        binning = head0['BINNING']
        xbin, ybin = [int(ibin) for ibin in binning.split(',')]
        binning_raw = binning

        # Always assume normal FITS header formatting
        one_indexed = True
        include_last = True
        for section in ['DSEC', 'BSEC']:

            # Initialize the image (0 means no amplifier)
            pix_img = np.zeros(raw_img.shape, dtype=int)
            for i in range(numamps):
                # Get the data section
                sec = head0[section + "{0:1d}".format(i + 1)]

                # Convert the data section from a string to a slice
                datasec = parse.sec2slice(sec,
                                          one_indexed=one_indexed,
                                          include_end=include_last,
                                          require_dim=2,
                                          binning=binning_raw)
                # Flip the datasec
                datasec = datasec[::-1]

                # Assign the amplifier
                pix_img[datasec] = i + 1

            # Finish
            if section == 'DSEC':
                rawdatasec_img = pix_img.copy()
            elif section == 'BSEC':
                oscansec_img = pix_img.copy()

        # Return
        return detpar, raw_img, hdu, exptime, rawdatasec_img, oscansec_img
Пример #21
0
def build_waveimg(spectrograph, tilts, slits, wv_calib, spat_flexure=None):
    """
    Main algorithm to build the wavelength image.

    The wavelength image is only constructed for good slits based on
    ``slits.mask`` and selected using
    :func:`pypeit.slittrace.SlitTraceSetBitMask.exclude_for_reducing`.

    Args:
        spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`):
            Spectrograph object
        tilts (`numpy.ndarray`_):
            Image holding tilts
        slits (:class:`~pypeit.slittrace.SlitTraceSet`):
            Object holding the slit left and right edge traces.
        wv_calib (:obj:`dict`):
            Object holding the wavelength calibration
        spat_flexure (:obj:`float`, optional):
            Spatial offset to apply for flexure.

    Returns:
        `numpy.ndarray`_: The wavelength image.
    """
    # Setup
    #ok_slits = slits.mask == 0
    bpm = slits.mask.astype(bool)
    bpm &= np.invert(
        slits.bitmask.flagged(slits.mask,
                              flag=slits.bitmask.exclude_for_reducing))
    ok_slits = np.invert(bpm)
    #
    image = np.zeros_like(tilts)
    slitmask = slits.slit_img(flexure=spat_flexure,
                              exclude_flag=slits.bitmask.exclude_for_reducing)

    par = wv_calib['par']
    if slits.ech_order is None and par['echelle']:
        msgs.error('Echelle orders must be provided by the slits object!')

#    slit_spat_pos = slits.spatial_coordinates(flexure=spat_flexure)

# If this is echelle print out a status message and do some error checking
    if par['echelle']:
        msgs.info('Evaluating 2-d wavelength solution for echelle....')
        if len(wv_calib['fit2d']['orders']) != np.sum(ok_slits):
            msgs.error(
                'wv_calib and ok_slits do not line up. Something is very wrong!'
            )

    # Unpack some 2-d fit parameters if this is echelle
#    for slit_spat in slits.spat_id[ok_slits]:
    for i in range(slits.nslits):
        if not ok_slits[i]:
            continue
        slit_spat = slits.spat_id[i]
        thismask = (slitmask == slit_spat)
        if not np.any(thismask):
            msgs.error("Something failed in wavelengths or masking..")
        if par['echelle']:
            #            # TODO: Put this in `SlitTraceSet`?
            #            order, indx = spectrograph.slit2order(slit_spat_pos[slits.spatid_to_zero(slit_spat)])
            # evaluate solution
            image[thismask] = utils.func_val(
                wv_calib['fit2d']['coeffs'],
                tilts[thismask],
                wv_calib['fit2d']['func2d'],
                #                                             x2=np.ones_like(tilts[thismask])*order,
                x2=np.full_like(tilts[thismask], slits.ech_order[i]),
                minx=wv_calib['fit2d']['min_spec'],
                maxx=wv_calib['fit2d']['max_spec'],
                minx2=wv_calib['fit2d']['min_order'],
                maxx2=wv_calib['fit2d']['max_order'])
            image[thismask] /= slits.ech_order[i]
        else:
            #iwv_calib = wv_calib[str(slit)]
            iwv_calib = wv_calib[str(slit_spat)]
            image[thismask] = utils.func_val(iwv_calib['fitc'],
                                             tilts[thismask],
                                             iwv_calib['function'],
                                             minx=iwv_calib['fmin'],
                                             maxx=iwv_calib['fmax'])
    # Return
    return image
Пример #22
0
    def build_wv_calib(self, arccen, method, skip_QA=False):
        """
        Main routine to generate the wavelength solutions in a loop over slits
        Wrapper to arc.simple_calib or arc.calib_with_arclines

        self.maskslits is updated for slits that fail

        Args:
            method : str
              'simple' -- arc.simple_calib
              'arclines' -- arc.calib_with_arclines
              'holy-grail' -- wavecal.autoid.HolyGrail
              'reidentify' -- wavecal.auotid.ArchiveReid
              'identify' -- wavecal.identify.Identify
              'full_template' -- wavecal.auotid.full_template
            skip_QA (bool, optional)

        Returns:
            dict:  self.wv_calib
        """
        # Obtain a list of good slits
        ok_mask_idx = np.where(np.invert(self.wvc_bpm))[0]

        # Obtain calibration for all slits
        if method == 'simple':
            lines = self.par['lamps']
            line_lists = waveio.load_line_lists(lines)

            final_fit = arc.simple_calib_driver(
                line_lists,
                arccen,
                ok_mask_idx,
                n_final=self.par['n_final'],
                sigdetect=self.par['sigdetect'],
                IDpixels=self.par['IDpixels'],
                IDwaves=self.par['IDwaves'])
        elif method == 'holy-grail':
            # Sometimes works, sometimes fails
            arcfitter = autoid.HolyGrail(
                arccen,
                par=self.par,
                ok_mask=ok_mask_idx,
                nonlinear_counts=self.nonlinear_counts)
            patt_dict, final_fit = arcfitter.get_results()
        elif method == 'identify':
            final_fit = {}
            # Manually identify lines
            msgs.info("Initializing the wavelength calibration tool")
            embed(header='line 222 wavecalib.py')
            for slit_idx in ok_mask_idx:
                arcfitter = Identify.initialise(arccen,
                                                self.slits,
                                                slit=slit_idx,
                                                par=self.par)
                final_fit[str(slit_idx)] = arcfitter.get_results()
                arcfitter.store_solution(final_fit[str(slit_idx)],
                                         "",
                                         self.binspectral,
                                         specname=self.spectrograph.name,
                                         gratname="UNKNOWN",
                                         dispangl="UNKNOWN")
        elif method == 'reidentify':
            # Now preferred
            # Slit positions
            arcfitter = autoid.ArchiveReid(
                arccen,
                self.spectrograph,
                self.par,
                ok_mask=ok_mask_idx,
                #slit_spat_pos=self.spat_coo,
                orders=self.orders,
                nonlinear_counts=self.nonlinear_counts)
            patt_dict, final_fit = arcfitter.get_results()
        elif method == 'full_template':
            # Now preferred
            if self.binspectral is None:
                msgs.error(
                    "You must specify binspectral for the full_template method!"
                )
            final_fit = autoid.full_template(
                arccen,
                self.par,
                ok_mask_idx,
                self.det,
                self.binspectral,
                nonlinear_counts=self.nonlinear_counts,
                nsnippet=self.par['nsnippet'])
        else:
            msgs.error(
                'Unrecognized wavelength calibration method: {:}'.format(
                    method))

        # Build the DataContainer
        # Loop on WaveFit items
        tmp = []
        for idx in range(self.slits.nslits):
            item = final_fit.pop(str(idx))
            if item is None:  # Add an empty WaveFit
                tmp.append(wv_fitting.WaveFit(self.slits.spat_id[idx]))
            else:
                # This is for I/O naming
                item.spat_id = self.slits.spat_id[idx]
                tmp.append(item)
        self.wv_calib = WaveCalib(
            wv_fits=np.asarray(tmp),
            arc_spectra=arccen,
            nslits=self.slits.nslits,
            spat_ids=self.slits.spat_id,
            PYP_SPEC=self.spectrograph.name,
        )

        # Update mask
        self.update_wvmask()

        #TODO For generalized echelle (not hard wired) assign order number here before, i.e. slits.ech_order

        # QA
        if not skip_QA:
            ok_mask_idx = np.where(np.invert(self.wvc_bpm))[0]
            for slit_idx in ok_mask_idx:
                outfile = qa.set_qa_filename(
                    self.master_key,
                    'arc_fit_qa',
                    slit=self.slits.slitord_id[slit_idx],
                    out_dir=self.qa_path)
                #
                autoid.arc_fit_qa(self.wv_calib.wv_fits[slit_idx],
                                  outfile=outfile)

        # Return
        self.steps.append(inspect.stack()[0][3])
        return self.wv_calib
Пример #23
0
    def main(args):

        # Parse the detector name
        try:
            det = int(args.det)
        except:
            detname = args.det
        else:
            detname = DetectorContainer.get_name(det)

        # Load em
        line_names, line_wav = list_of_spectral_lines()

        files = np.array(args.files)

        if args.z is not None:
            zs = np.array(args.z)

        # Loop on the files
        for i in range(files.size):
            # reinitialize lines wave
            line_wav_z = line_wav.copy()

            # Load 2D object
            file = files[i]
            # List only?
            if args.list:
                io.fits_open(file).info()
                continue
            spec2DObj = spec2dobj.Spec2DObj.from_file(file,
                                                      detname,
                                                      chk_version=False)

            # Deal with redshifts
            if args.z is not None:
                z = zs[i] if zs.size == files.size else zs[0]
                line_wav_z *= (1 + z)  #redshift linelist
            else:
                z = None

            # Save?
            folder = None
            if args.mode == 'save':
                folder = '{}_noisecheck'.format(file.split('.fits')[0])
                if not os.path.exists(folder): os.makedirs(folder)
            elif args.mode == 'print':
                # Generate a Table for pretty printing
                tbl = Table()
                tbl['Slit'] = spec2DObj.slits.slitord_id
                tbl['med_chis'] = spec2DObj.med_chis
                tbl['std_chis'] = spec2DObj.std_chis
                print('')
                print(tbl)
                print('-----------------------------------------------------')
                return

            # Find the slit of interest
            all_maskdef_ids = spec2DObj.slits.maskdef_id
            all_pypeit_ids = spec2DObj.slits.slitord_id
            if args.maskdef_id is not None and all_maskdef_ids is None:
                msgs.error(
                    'This spec2d does not have maskdef_id. Choose a pypeit_id insteed.'
                )

            # Build the mask
            input_mask = spec2DObj.bpmmask == 0
            if args.wavemin is not None:
                input_mask *= spec2DObj.waveimg > args.wavemin
            if args.wavemax is not None:
                input_mask *= spec2DObj.waveimg < args.wavemax

            # Decide on slits to show
            show_slits = range(all_pypeit_ids.size)
            if args.pypeit_id is not None or args.maskdef_id is not None:
                if args.maskdef_id is not None and args.maskdef_id in all_maskdef_ids:
                    slitidx = np.where(
                        all_maskdef_ids == args.maskdef_id)[0][0]
                elif args.pypeit_id is not None and args.pypeit_id in all_pypeit_ids:
                    slitidx = np.where(all_pypeit_ids == args.pypeit_id)[0][0]
                show_slits = range(slitidx, slitidx + 1)

            # loop on em
            for i in show_slits:
                pypeit_id = all_pypeit_ids[i]
                if all_maskdef_ids is not None:
                    basename = '{}_{}_maskdefID{}_pypeitID{}'.format(
                        spec2DObj.head0['DECKER'], detname, all_maskdef_ids[i],
                        pypeit_id)
                else:
                    basename = '{}_{}_pypeitID{}'.format(
                        spec2DObj.head0['DECKER'], detname, pypeit_id)

                # Chi
                chi_slit, _, _ = spec2DObj.calc_chi_slit(i, pad=args.pad)

                if chi_slit is None:
                    continue

                # Cut down
                chi_select = chi_slit * input_mask
                if np.all(chi_select == 0):
                    msgs.warn(
                        f"All of the chi values are masked in slit {pypeit_id} of {basename}!"
                    )
                    continue

                # Flux to show
                # get flux and err from in this slit
                flux_slit, err_slit = get_flux_slit(spec2DObj, i, pad=args.pad)
                # flux in the wavelength range
                flux_select = flux_slit * input_mask
                # Error in the wavelength range
                err_select = err_slit * input_mask

                # get edges of the slit to plot
                left, right, _ = spec2DObj.slits.select_edges()
                spat_start = int(left[:, i].min())
                spat_end = int(right[:, i].max())
                mid_spat = int((spat_end + spat_start) / 2.)

                # Wavelengths
                if spec2DObj.waveimg[input_mask].size == 0:
                    msgs.warn(
                        f"None of the wavelength values work in slit {pypeit_id} of {basename}!"
                    )
                    continue
                lbda_1darray = spec2DObj.waveimg[:, mid_spat]

                line_wav_plt = np.array([])
                line_names_plt = np.array([])
                if z is not None:
                    for i in range(line_wav_z.shape[0]):
                        if lbda_1darray[lbda_1darray != 0].min() < line_wav_z[
                                i] < lbda_1darray[lbda_1darray != 0].max():
                            line_wav_plt = np.append(
                                line_wav_plt,
                                lbda_1darray.searchsorted(line_wav_z[i]))
                            line_names_plt = np.append(line_names_plt,
                                                       line_names[i])

                plot(chi_slit[:, spat_start:spat_end],
                     chi_select,
                     flux_select,
                     err_select,
                     basename,
                     line_wav_plt,
                     line_names_plt,
                     lbda_1darray,
                     lbda_min=args.wavemin,
                     lbda_max=args.wavemax,
                     aspect_ratio=args.aspect_ratio)
                if args.mode == 'plot':
                    plt.show()
                if args.mode == 'save':
                    plt.savefig('{}/noisecheck_{}.png'.format(
                        folder, basename),
                                bbox_inches='tight',
                                dpi=400)
                plt.close()
Пример #24
0
    def unpack_object(self, ret_flam=False, extract_type='OPT'):
        """
        Utility function to unpack the sobjs for one object and
        return various numpy arrays describing the spectrum and meta
        data. The user needs to already have trimmed the Specobjs to
        the relevant indices for the object.

        Args:
           ret_flam (:obj:`bool`, optional):
              If True return the FLAM, otherwise return COUNTS.

        Returns:
            tuple: Returns the following where all numpy arrays
            returned have shape (nspec, norders) for Echelle data and
            (nspec,) for Multislit data.

                - wave (`numpy.ndarray`_): Wavelength grids
                - flux (`numpy.ndarray`_): Flambda or counts
                - flux_ivar (`numpy.ndarray`_): Inverse variance (of
                  Flambda or counts)
                - flux_gpm (`numpy.ndarray`_): Good pixel mask.
                  True=Good
                - meta_spec (dict:) Dictionary containing meta data.
                  The keys are defined by
                  spectrograph.header_cards_from_spec()
                - header (astropy.io.header object): header from
                  spec1d file
        """
        # Prep
        norddet = self.nobj
        flux_attr = 'FLAM' if ret_flam else 'COUNTS'
        flux_key = '{}_{}'.format(extract_type, flux_attr)
        wave_key = '{}_WAVE'.format(extract_type)
        # Test
        if getattr(self, flux_key)[0] is None:
            msgs.error(
                "Flux not available for {}.  Try the other ".format(flux_key))
        #
        nspec = getattr(self, flux_key)[0].size
        # Allocate arrays and unpack spectrum
        wave = np.zeros((nspec, norddet))
        flux = np.zeros((nspec, norddet))
        flux_ivar = np.zeros((nspec, norddet))
        flux_gpm = np.zeros((nspec, norddet), dtype=bool)
        detector = np.zeros(norddet, dtype=int)
        ech_orders = np.zeros(norddet, dtype=int)

        # TODO make the extraction that is desired OPT vs BOX an optional input variable.
        for iorddet in range(norddet):
            wave[:, iorddet] = getattr(self, wave_key)[iorddet]
            flux_gpm[:, iorddet] = getattr(
                self, '{}_MASK'.format(extract_type))[iorddet]
            detector[iorddet] = self[iorddet].DET
            if self[0].PYPELINE == 'Echelle':
                ech_orders[iorddet] = self[iorddet].ECH_ORDER
            flux[:, iorddet] = getattr(self, flux_key)[iorddet]
            flux_ivar[:, iorddet] = getattr(self, flux_key +
                                            '_IVAR')[iorddet]  #OPT_FLAM_IVAR

        # Populate meta data
        spectrograph = load_spectrograph(self.header['PYP_SPEC'])

        meta_spec = spectrograph.parse_spec_header(self.header)
        # Add the pyp spec.
        # TODO JFH: Make this an atribute of the specobj by default.
        meta_spec['PYP_SPEC'] = self.header['PYP_SPEC']
        meta_spec['PYPELINE'] = self[0].PYPELINE
        meta_spec['DET'] = detector
        # Return
        if self[0].PYPELINE in ['MultiSlit', 'IFU'] and self.nobj == 1:
            meta_spec['ECH_ORDERS'] = None
            return wave.reshape(nspec), flux.reshape(nspec), flux_ivar.reshape(nspec), \
                   flux_gpm.reshape(nspec), meta_spec, self.header
        else:
            meta_spec['ECH_ORDERS'] = ech_orders
            return wave, flux, flux_ivar, flux_gpm, meta_spec, self.header
Пример #25
0
def ech_coadd(files,objids=None,extract='OPT',flux=True,giantcoadd=False,orderscale='median',mergeorder=True,
              wave_grid_method='velocity', niter=5,wave_grid_min=None, wave_grid_max=None,v_pix=None,
              scale_method='auto', do_offset=False, sigrej_final=3.,do_var_corr=False,
              SN_MIN_MEDSCALE = 0.5, overlapfrac = 0.01, num_min_pixels=10,phot_scale_dicts=None,
              qafile=None, outfile=None,do_cr=True, debug=False,**kwargs):
    """
    routines for coadding spectra observed with echelle spectrograph.
    parameters:
        files (list): file names
        objids (str): objid
        extract (str): 'OPT' or 'BOX'
        flux (bool): fluxed or not
        giantcoadd (bool): coadding order by order or do it at once?
        wave_grid_method (str): default velocity
        niter (int): number of iteration for rejections
        wave_grid_min (float): min wavelength, None means it will find the min value from your spectra
        wave_grid_max (float): max wavelength, None means it will find the max value from your spectra
        v_pix (float): delta velocity, see coadd.py
        scale_method (str): see coadd.py
        do_offset (str): see coadd.py, not implemented yet.
        sigrej_final (float): see coadd.py
        do_var_corr (bool): see coadd.py, default False. It seems True will results in a large error
        SN_MIN_MEDSCALE (float): minimum SNR for scaling different orders
        overlapfrac (float): minimum overlap fraction for scaling different orders.
        qafile (str): name of qafile
        outfile (str): name of coadded spectrum
        do_cr (bool): remove cosmic rays?
        debug (bool): show debug plots?
        kwargs: see coadd.py
    returns:
        spec1d: coadded XSpectrum1D
    """

    nfile = len(files)
    if nfile <=1:
        msgs.info('Only one spectrum exits coadding...')
        return

    fname = files[0]
    ext_final = fits.getheader(fname, -1)
    norder = ext_final['ECHORDER'] + 1
    msgs.info('spectrum {:s} has {:d} orders'.format(fname, norder))
    if norder <= 1:
        msgs.error('The number of orders have to be greater than one for echelle. Longslit data?')

    if giantcoadd:
        msgs.info('Coadding all orders and exposures at once')
        spectra = load.ech_load_spec(files, objid=objids,order=None, extract=extract, flux=flux)
        wave_grid = np.zeros((2,spectra.nspec))
        for i in range(spectra.nspec):
            wave_grid[0, i] = spectra[i].wvmin.value
            wave_grid[1, i] = spectra[i].wvmax.value
        ech_kwargs = {'echelle': True, 'wave_grid_min': np.min(wave_grid), 'wave_grid_max': np.max(wave_grid),
                      'v_pix': v_pix}
        kwargs.update(ech_kwargs)
        # Coadding
        spec1d = coadd.coadd_spectra(spectra, wave_grid_method=wave_grid_method, niter=niter,
                                          scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
                                          do_var_corr=do_var_corr, qafile=qafile, outfile=outfile,
                                          do_cr=do_cr, debug=debug,**kwargs)
    else:
        msgs.info('Coadding individual orders first and then merge order')
        spectra_list = []
        # Keywords for Table
        rsp_kwargs = {}
        rsp_kwargs['wave_tag'] = '{:s}_WAVE'.format(extract)
        rsp_kwargs['flux_tag'] = '{:s}_FLAM'.format(extract)
        rsp_kwargs['sig_tag'] = '{:s}_FLAM_SIG'.format(extract)
        #wave_grid = np.zeros((2,norder))
        for iord in range(norder):
            spectra = load.ech_load_spec(files, objid=objids, order=iord, extract=extract, flux=flux)
            ech_kwargs = {'echelle': False, 'wave_grid_min': spectra.wvmin.value, 'wave_grid_max': spectra.wvmax.value, 'v_pix': v_pix}
            #wave_grid[0,iord] = spectra.wvmin.value
            #wave_grid[1,iord] = spectra.wvmax.value
            kwargs.update(ech_kwargs)
            # Coadding the individual orders
            if qafile is not None:
                qafile_iord = qafile+'_%s'%str(iord)
            else:
                qafile_iord =  None
            spec1d_iord = coadd.coadd_spectra(spectra, wave_grid_method=wave_grid_method, niter=niter,
                                       scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
                                       do_var_corr=do_var_corr, qafile=qafile_iord, outfile=None,
                                       do_cr=do_cr, debug=debug, **kwargs)
            spectrum = spec_from_array(spec1d_iord.wavelength, spec1d_iord.flux, spec1d_iord.sig,**rsp_kwargs)
            spectra_list.append(spectrum)

        spectra_coadd = collate(spectra_list)

        # Rebin the spectra
        # ToDo: we should read in JFH's wavelength grid here.
        # Join into one XSpectrum1D object
        # Final wavelength array
        kwargs['wave_grid_min'] = np.min(spectra_coadd.data['wave'][spectra_coadd.data['wave'] > 0])
        kwargs['wave_grid_max'] = np.max(spectra_coadd.data['wave'][spectra_coadd.data['wave'] > 0])
        wave_final = coadd.new_wave_grid(spectra_coadd.data['wave'], wave_method=wave_grid_method, **kwargs)
        # The rebin function in linetools can not work on collated spectra (i.e. filled 0).
        # Thus I have to rebin the spectra first and then collate again.
        spectra_list_new = []
        for i in range(spectra_coadd.nspec):
            speci = spectra_list[i].rebin(wave_final * units.AA, all=True, do_sig=True, grow_bad_sig=True,
                                          masking='none')
            spectra_list_new.append(speci)
        spectra_coadd_rebin = collate(spectra_list_new)

        ## Note
        if orderscale == 'photometry':
            # Only tested on NIRES.
            if phot_scale_dicts is not None:
                spectra_coadd_rebin = order_phot_scale(spectra_coadd_rebin, phot_scale_dicts, debug=debug)
            else:
                msgs.warn('No photometric information is provided. Will use median scale.')
                orderscale = 'median'
        elif orderscale == 'median':
            #rmask = spectra_coadd_rebin.data['sig'].filled(0.) > 0.
            #sn2, weights = coadd.sn_weights(fluxes, sigs, rmask, wave)
            ## scaling different orders
            order_median_scale(spectra_coadd_rebin, nsig=sigrej_final, niter=niter, overlapfrac=overlapfrac,
                               num_min_pixels=num_min_pixels, SN_MIN_MEDSCALE=SN_MIN_MEDSCALE, debug=debug)
        else:
            msgs.warn('No any scaling is performed between different orders.')


        if mergeorder:
            fluxes, sigs, wave = coadd.unpack_spec(spectra_coadd_rebin, all_wave=False)
            ## Megering orders
            msgs.info('Merging different orders')
            ## ToDo: Joe claimed not to use pixel depedent weighting.
            weights = 1.0 / sigs**2
            weights[~np.isfinite(weights)] = 0.0
            weight_combine = np.sum(weights, axis=0)
            weight_norm = weights / weight_combine
            weight_norm[np.isnan(weight_norm)] = 1.0
            flux_final = np.sum(fluxes * weight_norm, axis=0)
            sig_final = np.sqrt(np.sum((weight_norm * sigs) ** 2, axis=0))
            spec1d_final = spec_from_array(wave_final * units.AA,flux_final,sig_final,**rsp_kwargs)

            if outfile is not None:
                msgs.info('Saving the final calibrated spectrum as {:s}'.format(outfile))
                coadd.write_to_disk(spec1d_final, outfile)

            if (qafile is not None) or (debug):
                # plot and save qa
                plt.figure(figsize=(12, 6))
                ax1 = plt.axes([0.07, 0.13, 0.9, 0.4])
                ax2 = plt.axes([0.07, 0.55, 0.9, 0.4])
                plt.setp(ax2.get_xticklabels(), visible=False)

                medf = np.median(spec1d_final.flux)
                ylim = (np.sort([0. - 0.3 * medf, 5 * medf]))
                cmap = plt.get_cmap('RdYlBu_r')
                for idx in range(spectra_coadd_rebin.nspec):
                    spectra_coadd_rebin.select = idx
                    color = cmap(float(idx) / spectra_coadd_rebin.nspec)
                    ind_good = spectra_coadd_rebin.sig > 0
                    ax1.plot(spectra_coadd_rebin.wavelength[ind_good], spectra_coadd_rebin.flux[ind_good], color=color)

                if (np.max(spec1d_final.wavelength) > (9000.0 * units.AA)):
                    skytrans_file = resource_filename('pypeit', '/data/skisim/atm_transmission_secz1.5_1.6mm.dat')
                    skycat = np.genfromtxt(skytrans_file, dtype='float')
                    scale = 0.85 * ylim[1]
                    ax2.plot(skycat[:, 0] * 1e4, skycat[:, 1] * scale, 'm-', alpha=0.5)

                ax2.plot(spec1d_final.wavelength, spec1d_final.sig, ls='steps-', color='0.7')
                ax2.plot(spec1d_final.wavelength, spec1d_final.flux, ls='steps-', color='b')

                ax1.set_xlim([np.min(spec1d_final.wavelength.value), np.max(spec1d_final.wavelength.value)])
                ax2.set_xlim([np.min(spec1d_final.wavelength.value), np.max(spec1d_final.wavelength.value)])
                ax1.set_ylim(ylim)
                ax2.set_ylim(ylim)
                ax1.set_xlabel('Wavelength (Angstrom)')
                ax1.set_ylabel('Flux')
                ax2.set_ylabel('Flux')

                plt.tight_layout(pad=0.2, h_pad=0., w_pad=0.2)

                if len(qafile.split('.')) == 1:
                    msgs.info("No fomat given for the qafile, save to PDF format.")
                    qafile = qafile + '.pdf'
                if qafile:
                    plt.savefig(qafile)
                    msgs.info("Wrote coadd QA: {:s}".format(qafile))
                if debug:
                    plt.show()
                plt.close()

            ### Do NOT remove this part althoug it is deprecated.
            # we may need back to using this pieces of code after fixing the coadd.coadd_spectra problem on first order.
            #kwargs['echelle'] = True
            #kwargs['wave_grid_min'] = np.min(wave_grid)
            #kwargs['wave_grid_max'] = np.max(wave_grid)
            #spec1d_final = coadd.coadd_spectra(spectra_coadd_rebin, wave_grid_method=wave_grid_method, niter=niter,
            #                                  scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
            #                                  do_var_corr=do_var_corr, qafile=qafile, outfile=outfile,
            #                                  do_cr=do_cr, debug=debug, **kwargs)
            return spec1d_final
        else:
            msgs.warn('Skipped merging orders')
            if outfile is not None:
                for iord in range(len(spectra_list)):
                    outfile_iord = outfile.replace('.fits','_ORDER{:04d}.fits'.format(iord))
                    msgs.info('Saving the final calibrated spectrum of order {:d} as {:s}'.format(iord,outfile))
                    spectra_list[iord].write_to_fits(outfile_iord)
            return spectra_list
Пример #26
0
    def write_to_fits(self,
                      subheader,
                      outfile,
                      overwrite=True,
                      update_det=None,
                      slitspatnum=None,
                      debug=False):
        """
        Write the set of SpecObj objects to one multi-extension FITS file

        Args:
            outfile (str):
            subheader (:obj:`dict`):
            overwrite (bool, optional):
            slitspatnum (:obj:`str` or :obj:`list`, optional):
                Restricted set of slits for reduction
            update_det (int or list, optional):
              If provided, do not clobber the existing file but only update
              the indicated detectors.  Useful for re-running on a subset of detectors

        """
        if os.path.isfile(outfile) and (not overwrite):
            msgs.warn("Outfile exists.  Set overwrite=True to clobber it")
            return

        # If the file exists and update_det (and slit_spat_num) is provided, use the existing header
        #   and load up all the other hdus so that we only over-write the ones
        #   we are updating
        if os.path.isfile(outfile) and (update_det is not None
                                        or slitspatnum is not None):
            _specobjs = SpecObjs.from_fitsfile(outfile)
            mask = np.ones(_specobjs.nobj, dtype=bool)
            # Update_det
            if update_det is not None:
                # Pop out those with this detector (and slit if slit_spat_num is provided)
                for det in np.atleast_1d(update_det):
                    mask[_specobjs.DET == det] = False
            elif slitspatnum is not None:  # slitspatnum
                dets, spat_ids = slittrace.parse_slitspatnum(slitspatnum)
                for det, spat_id in zip(dets, spat_ids):
                    mask[(_specobjs.DET == det)
                         & (_specobjs.SLITID == spat_id)] = False
            _specobjs = _specobjs[mask]
            # Add in the new
            for sobj in self.specobjs:
                _specobjs.add_sobj(sobj)
        else:
            _specobjs = self.specobjs

        # Build up the Header
        header = initialize_header(primary=True)
        for key in subheader.keys():
            header[key.upper()] = subheader[key]

        # Init
        prihdu = fits.PrimaryHDU()
        hdus = [prihdu]
        prihdu.header = header

        # Add class info
        prihdu.header['DMODCLS'] = (self.__class__.__name__, 'Datamodel class')
        prihdu.header['DMODVER'] = (self.version, 'Datamodel version')

        detector_hdus = {}
        nspec, ext = 0, 0
        # Loop on the SpecObj objects
        for sobj in _specobjs:
            if sobj is None:
                continue
            # HDUs
            if debug:
                import pdb
                pdb.set_trace()
            shdul = sobj.to_hdu()
            if len(shdul) == 2:  # Detector?
                detector_hdus[sobj['DET']] = shdul[1]
                shdu = [shdul[0]]
            elif len(shdul) == 1:  # Detector?
                shdu = shdul
            else:
                msgs.error("Should not get here...")
            # Check -- If sobj had only 1 array, the BinTableHDU test will fail
            assert len(shdu) == 1, 'Bad data model!!'
            assert isinstance(shdu[0],
                              fits.hdu.table.BinTableHDU), 'Bad data model2'
            #shdu[0].header['DMODCLS'] = (self.__class__.__name__, 'Datamodel class')
            #shdu[0].header['DMODVER'] = (self.version, 'Datamodel version')
            # Name
            shdu[0].name = sobj.NAME
            # Extension
            keywd = 'EXT{:04d}'.format(ext)
            prihdu.header[keywd] = sobj.NAME
            ext += 1
            nspec += 1
            # Append
            hdus += shdu

        # Deal with Detectors
        for key, item in detector_hdus.items():
            # TODO - Add EXT to the primary header for these??
            prefix = specobj.det_hdu_prefix(key)
            # Name
            if prefix not in item.name:  # In case we are re-loading
                item.name = specobj.det_hdu_prefix(key) + item.name
            # Append
            hdus += [item]

        # A few more for the header
        prihdu.header['NSPEC'] = nspec

        # Code versions
        initialize_header(hdr=prihdu.header)

        # Finish
        hdulist = fits.HDUList(hdus)
        if debug:
            import pdb
            pdb.set_trace()
        hdulist.writeto(outfile, overwrite=overwrite)
        msgs.info("Wrote 1D spectra to {:s}".format(outfile))
        return
Пример #27
0
def subtract_overscan(rawframe, datasec_img, oscansec_img,
                          method='savgol', params=[5, 65]):
    """
    Subtract overscan

    Args:
        rawframe (:obj:`numpy.ndarray`):
            Frame from which to subtract overscan
        numamplifiers (int):
            Number of amplifiers for this detector.
        datasec_img (:obj:`numpy.ndarray`):
            An array the same shape as rawframe that identifies
            the pixels associated with the data on each amplifier.
            0 for not data, 1 for amplifier 1, 2 for amplifier 2, etc.
        oscansec_img (:obj:`numpy.ndarray`):
            An array the same shape as rawframe that identifies
            the pixels associated with the overscan region on each
            amplifier.
            0 for not data, 1 for amplifier 1, 2 for amplifier 2, etc.
        method (:obj:`str`, optional):
            The method used to fit the overscan region.  Options are
            polynomial, savgol, median.
        params (:obj:`list`, optional):
            Parameters for the overscan subtraction.  For
            method=polynomial, set params = order, number of pixels,
            number of repeats ; for method=savgol, set params = order,
            window size ; for method=median, params are ignored.

    Returns:
        :obj:`numpy.ndarray`: The input frame with the overscan region
        subtracted
    """
    # Copy the data so that the subtraction is not done in place
    no_overscan = rawframe.copy()

    # Amplifiers
    amps = np.unique(datasec_img[datasec_img > 0]).tolist()

    # Perform the overscan subtraction for each amplifier
    for amp in amps:
        # Pull out the overscan data
        overscan, _ = rect_slice_with_mask(rawframe, oscansec_img, amp)
        # Pull out the real data
        data, data_slice = rect_slice_with_mask(rawframe, datasec_img, amp)

        # Shape along at least one axis must match
        data_shape = data.shape
        if not np.any([dd == do for dd, do in zip(data_shape, overscan.shape)]):
            msgs.error('Overscan sections do not match amplifier sections for'
                       'amplifier {0}'.format(amp))
        compress_axis = 1 if data_shape[0] == overscan.shape[0] else 0

        # Fit/Model the overscan region
        osfit = np.median(overscan) if method.lower() == 'median' \
            else np.median(overscan, axis=compress_axis)
        if method.lower() == 'polynomial':
            # TODO: Use np.polynomial.polynomial.polyfit instead?
            c = np.polyfit(np.arange(osfit.size), osfit, params[0])
            ossub = np.polyval(c, np.arange(osfit.size))
        elif method.lower() == 'savgol':
            ossub = signal.savgol_filter(osfit, params[1], params[0])
        elif method.lower() == 'median':
            # Subtract scalar and continue
            no_overscan[data_slice] -= osfit
            continue
        else:
            raise ValueError('Unrecognized overscan subtraction method: {0}'.format(method))

        # Subtract along the appropriate axis
        no_overscan[tuple(data_slice)] -= (ossub[:, None] if compress_axis == 1 else ossub[None, :])

    return no_overscan
Пример #28
0
def read_deimos(raw_file, det=None):
    """
    Read a raw DEIMOS data frame (one or more detectors)
    Packed in a multi-extension HDU
    Based on pypeit.arlris.read_lris...
       Based on readmhdufits.pro

    Parameters
    ----------
    raw_file : str
      Filename

    Returns
    -------
    array : ndarray
      Combined image
    header : FITS header
    sections : tuple
      List of datasec, oscansec sections
    """

    # Check for file; allow for extra .gz, etc. suffix
    fil = glob.glob(raw_file + '*')
    if len(fil) != 1:
        msgs.error('Found {0} files matching {1}'.format(
            len(fil), raw_file + '*'))
    # Read
    try:
        msgs.info("Reading DEIMOS file: {:s}".format(fil[0]))
    except AttributeError:
        print("Reading DEIMOS file: {:s}".format(fil[0]))

    hdu = fits.open(fil[0])
    head0 = hdu[0].header

    # Get post, pre-pix values
    precol = head0['PRECOL']
    postpix = head0['POSTPIX']
    preline = head0['PRELINE']
    postline = head0['POSTLINE']
    detlsize = head0['DETLSIZE']
    x0, x_npix, y0, y_npix = np.array(parse.load_sections(detlsize)).flatten()

    # Create final image
    if det is None:
        image = np.zeros((x_npix, y_npix + 4 * postpix))

    # Setup for datasec, oscansec
    dsec = []
    osec = []

    # get the x and y binning factors...
    binning = head0['BINNING']
    if binning != '1,1':
        msgs.error("This binning for DEIMOS might not work.  But it might..")

    xbin, ybin = [int(ibin) for ibin in binning.split(',')]

    # DEIMOS detectors
    nchip = 8

    if det is None:
        chips = range(nchip)
    else:
        chips = [det - 1]  # Indexing starts at 0 here
    # Loop
    for tt in chips:
        data, oscan = deimos_read_1chip(hdu, tt + 1)

        # if n_elements(nobias) eq 0 then nobias = 0

        # One detector??
        if det is not None:
            image = np.zeros((data.shape[0], data.shape[1] + oscan.shape[1]))

        # Indexing
        x1, x2, y1, y2, o_x1, o_x2, o_y1, o_y2 = indexing(tt, postpix, det=det)

        # Fill
        image[y1:y2, x1:x2] = data
        image[o_y1:o_y2, o_x1:o_x2] = oscan

        # Sections
        idsec = '[{:d}:{:d},{:d}:{:d}]'.format(y1, y2, x1, x2)
        iosec = '[{:d}:{:d},{:d}:{:d}]'.format(o_y1, o_y2, o_x1, o_x2)
        dsec.append(idsec)
        osec.append(iosec)
    # Return
    return image, head0, (dsec, osec)
Пример #29
0
def pattern_frequency(frame, axis=1):
    """
    Using the supplied 2D array, calculate the pattern frequency
    along the specified axis.

    Args:
        frame (:obj:`numpy.ndarray`):
            2D array to measure the pattern frequency
        axis (int, optional):
            Which axis should the pattern frequency be measured?

    Returns:
        :obj:`float`: The frequency of the sinusoidal pattern.
    """
    # For axis=0, transpose
    arr = frame.copy()
    if axis == 0:
        arr = frame.T
    elif axis != 1:
        msgs.error("frame must be a 2D image, and axis must be 0 or 1")

    # Calculate the output image dimensions of the model signal
    # Subtract the DC offset
    arr -= np.median(arr, axis=1)[:, np.newaxis]
    # Find significant deviations and ignore those rows
    mad = 1.4826*np.median(np.abs(arr))
    ww = np.where(arr > 10*mad)
    # Create a mask of these rows
    msk = np.sort(np.unique(ww[0]))

    # Compute the Fourier transform to obtain an estimate of the dominant frequency component
    amp = np.fft.rfft(arr, axis=1)
    idx = (np.arange(arr.shape[0]), np.argmax(np.abs(amp), axis=1))

    # Construct the variables of the sinusoidal waveform
    amps = (np.abs(amp))[idx] * (2.0 / arr.shape[1])
    phss = np.arctan2(amp.imag, amp.real)[idx]
    frqs = idx[1]

    # Use the above to as initial guess parameters in chi-squared minimisation
    cosfunc = lambda xarr, *p: p[0] * np.cos(2.0 * np.pi * p[1] * xarr + p[2])
    xdata = np.linspace(0.0, 1.0, arr.shape[1])
    # Calculate the amplitude distribution
    amp_dist = np.zeros(arr.shape[0])
    frq_dist = np.zeros(arr.shape[0])
    # Loop over all rows to new independent values that can be averaged
    for ii in range(arr.shape[0]):
        if ii in msk:
            continue
        try:
            popt, pcov = curve_fit(cosfunc, xdata, arr[ii, :], p0=[amps[ii], frqs[ii], phss[ii]],
                                   bounds=([-np.inf, frqs[ii]-1, -np.inf],
                                           [+np.inf, frqs[ii]+1, +np.inf]))
        except ValueError:
            msgs.warn("Input data invalid for pattern frequency fit of row {0:d}/{1:d}".format(ii+1, arr.shape[0]))
            continue
        except RuntimeError:
            msgs.warn("Pattern frequency fit failed for row {0:d}/{1:d}".format(ii+1, arr.shape[0]))
            continue
        amp_dist[ii] = popt[0]
        frq_dist[ii] = popt[1]
    ww = np.where(amp_dist > 0.0)
    use_amp = np.median(amp_dist[ww])
    use_frq = np.median(frq_dist[ww])
    # Calculate the frequency distribution with a prior on the amplitude
    frq_dist = np.zeros(arr.shape[0])
    for ii in range(arr.shape[0]):
        if ii in msk:
            continue
        try:
            popt, pcov = curve_fit(cosfunc, xdata, arr[ii, :], p0=[use_amp, use_frq, phss[ii]],
                                   bounds=([use_amp * 0.99999999, use_frq-1, -np.inf],
                                           [use_amp * 1.00000001, use_frq+1, +np.inf]))
        except ValueError:
            msgs.warn("Input data invalid for patern frequency fit of row {0:d}/{1:d}".format(ii+1, arr.shape[0]))
            continue
        except RuntimeError:
            msgs.warn("Pattern frequency fit failed for row {0:d}/{1:d}".format(ii+1, arr.shape[0]))
            continue
        frq_dist[ii] = popt[1]
    # Ignore masked values, and return the best estimate of the frequency
    ww = np.where(frq_dist > 0.0)
    medfrq = np.median(frq_dist[ww])
    return medfrq/(arr.shape[1]-1)
Пример #30
0
def main(args):
    """ Executes 2d coadding
    """
    msgs.warn('PATH =' + os.getcwd())
    # Load the file
    if args.file is not None:
        spectrograph, config_lines, spec2d_files = read_coadd2d_file(args.file)
        # Parameters
        # TODO: Shouldn't this reinstantiate the same parameters used in
        # the PypeIt run that extracted the objects?  Why are we not
        # just passing the pypeit file?
        # JFH: The reason is that the coadd2dfile may want different reduction parameters
        spectrograph_def_par = spectrograph.default_pypeit_par()
        parset = par.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_def_par.to_config(),
                                                 merge_with=config_lines)
    elif args.obj is not None:
        # TODO: We should probably be reading the pypeit file and using those parameters here rather than using the
        # default parset.
        # TODO: This needs to define the science path
        spec2d_files = glob.glob('./Science/spec2d_*' + args.obj + '*')
        head0 = fits.getheader(spec2d_files[0])
        spectrograph_name = head0['SPECTROG']
        spectrograph = load_spectrograph(spectrograph_name)
        parset = spectrograph.default_pypeit_par()
    else:
        msgs.error('You must either input a coadd2d file with --file or an object name with --obj')

    # Update with configuration specific parameters (which requires science file) and initialize spectrograph
    spectrograph_cfg_lines = spectrograph.config_specific_par(spec2d_files[0]).to_config()
    parset = par.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=parset.to_config())

    # If detector was passed as an argument override whatever was in the coadd2d_file
    if args.det is not None:
        msgs.info("Restricting reductions to detector={}".format(args.det))
        parset['rdx']['detnum'] = int(args.det)

    # Get headers (if possible) and base names
    spec1d_files = [files.replace('spec2d', 'spec1d') for files in spec2d_files]
    head1d = None
    for spec1d_file in spec1d_files:
        if os.path.isfile(spec1d_file):
            head1d = fits.getheader(spec1d_file)
            break
    if head1d is None:
        msgs.warn("No 1D spectra so am generating a dummy header for output")
        head1d = io.initialize_header()

    head2d = fits.getheader(spec2d_files[0])
    if args.basename is None:
        filename = os.path.basename(spec2d_files[0])
        basename = filename.split('_')[2]
    else:
        basename = args.basename

    # Write the par to disk
    par_outfile = basename+'_coadd2d.par'
    print("Writing the parameters to {}".format(par_outfile))
    parset.to_config(par_outfile)

    # Now run the coadds

    skysub_mode = head2d['SKYSUB']
    ir_redux = True if 'DIFF' in skysub_mode else False

    # Print status message
    msgs_string = 'Reducing target {:s}'.format(basename) + msgs.newline()
    msgs_string += 'Performing coadd of frames reduce with {:s} imaging'.format(skysub_mode)
    msgs_string += msgs.newline() + 'Combining frames in 2d coadd:' + msgs.newline()
    for file in spec2d_files:
        msgs_string += '{0:s}'.format(os.path.basename(file)) + msgs.newline()
    msgs.info(msgs_string)

    # TODO: This needs to be added to the parameter list for rdx
    redux_path = os.getcwd()
    master_dirname = os.path.basename(head2d['PYPMFDIR']) + '_coadd'
    master_dir = os.path.join(redux_path, master_dirname)

    # Make the new Master dir
    if not os.path.isdir(master_dir):
        msgs.info('Creating directory for Master output: {0}'.format(master_dir))
        os.makedirs(master_dir)

    # Instantiate the sci_dict
    sci_dict = OrderedDict()  # This needs to be ordered
    sci_dict['meta'] = {}
    sci_dict['meta']['vel_corr'] = 0.
    sci_dict['meta']['ir_redux'] = ir_redux

    # Find the detectors to reduce
    detectors = PypeIt.select_detectors(detnum=parset['rdx']['detnum'], ndet=spectrograph.ndet)
    if len(detectors) != spectrograph.ndet:
        msgs.warn('Not reducing detectors: {0}'.format(' '.join([str(d) for d in
        set(np.arange(spectrograph.ndet) + 1) - set(detectors)])))

    # Loop on detectors
    for det in detectors:
        msgs.info("Working on detector {0}".format(det))
        sci_dict[det] = {}

        # Instantiate Coadd2d
        coadd = coadd2d.CoAdd2D.get_instance(spec2d_files, spectrograph, parset, det=det,
                                             offsets=parset['coadd2d']['offsets'],
                                             weights=parset['coadd2d']['weights'],
                                             ir_redux=ir_redux,
                                             debug_offsets=args.debug_offsets, debug=args.debug,
                                             samp_fact=args.samp_fact)

        # Coadd the slits
        coadd_dict_list = coadd.coadd(only_slits=None) # TODO implement only_slits later
        # Create the psuedo images
        psuedo_dict = coadd.create_psuedo_image(coadd_dict_list)
        # Reduce
        msgs.info('Running the extraction')
        sci_dict[det]['sciimg'], sci_dict[det]['sciivar'], sci_dict[det]['skymodel'], sci_dict[det]['objmodel'], \
        sci_dict[det]['ivarmodel'], sci_dict[det]['outmask'], sci_dict[det]['specobjs'] = coadd.reduce(
            psuedo_dict, show = args.show, show_peaks = args.peaks)
        # Save psuedo image master files
        coadd.save_masters(master_dir)

    # Make the new Science dir
    # TODO: This needs to be defined by the user
    scipath = os.path.join(redux_path, 'Science_coadd')
    if not os.path.isdir(scipath):
        msgs.info('Creating directory for Science output: {0}'.format(scipath))
        os.makedirs(scipath)

    # Save the results
    save.save_all(sci_dict, coadd.stack_dict['master_key_dict'], master_dir, spectrograph, head1d,
                  head2d, scipath, basename)#, binning=coadd.binning)
Пример #31
0
def replace_columns(img, bad_cols, replace_with='mean', copy=False):
    """
    Replace bad image columns.

    Args:
        img (`numpy.ndarray`_):
            A 2D array with image values to replace.
        bad_cols (`numpy.ndarray`_):
            Boolean array selecting bad columns in `img`.  Must have the
            correct shape.
        replace_with (:obj:`str`, optional):
            Method to use for the replacements.  Can be 'mean' (see
            :func:`replace_column_mean`) or 'linear' (see
            :func:`replace_column_linear`).
        copy (:obj:`bool`, optional):
            Copy `img` to a new array before making any
            modifications.  Otherwise, `img` is modified in-place.

    Returns:
        `numpy.ndarray`_: The modified image, which is either a new
        array or points to the in-place modification of `img` according
        to the value of `copy`.
    """
    # Check
    if img.ndim != 2:
        msgs.error('Images must be 2D!')
    if bad_cols.size != img.shape[1]:
        msgs.error('Bad column array has incorrect length!')
    if np.all(bad_cols):
        msgs.error('All columns are bad!')

    _img = img.copy() if copy else img

    if np.sum(bad_cols) == 0:
        # No bad columns
        return _img

    # Find the starting/ending indices of adjacent bad columns
    borders = np.zeros(img.shape[1], dtype=int)
    borders[bad_cols] = 1
    borders = borders - np.roll(borders,1)
    if borders[0] == -1:
        borders[0] = 0

    # Get edge indices and deal with edge cases
    lindx = borders == 1
    ledges = np.where(lindx)[0] if np.any(lindx) else [0]
    rindx = borders == -1
    redges = np.where(rindx)[0] if np.any(rindx) else [img.shape[1]]
    if ledges[0] > redges[0]:
        ledges = np.append([0], ledges)
    if ledges[-1] > redges[-1]:
        redges = np.append(redges, [img.shape[1]])
    # If this is tripped, there's a coding error
    assert len(ledges) == len(redges), 'Problem in edge setup'

    # Replace the image values
    if replace_with == 'mean':
        for l,r in zip(ledges, redges):
            replace_column_mean(_img, l, r)
    elif replace_with == 'linear':
        for l,r in zip(ledges, redges):
            replace_column_linear(_img, l, r)
    else:
        msgs.error('Unknown replace_columns method.  Must be mean or linear.')
    return _img
Пример #32
0
def read_coaddfile(ifile):
    """
    Read a PypeIt .coadd1d file, akin to a standard PypeIt file

    The top is a config block that sets ParSet parameters
      The spectrograph is required

    Args:
        ifile (str):
          Name of the flux file

    Returns:
        cfg_lines (list):
          Config lines to modify ParSet values
        spec1dfiles (list):
          Contains spec1dfiles to be coadded
        objids (list):
          Object ids aligned with each of the spec1dfiles


    """
    # Read in the pypeit reduction file
    msgs.info('Loading the coadd1d file')
    lines = par.util._read_pypeit_file_lines(ifile)
    is_config = np.ones(len(lines), dtype=bool)


    # Parse the fluxing block
    spec1dfiles = []
    objids_in = []
    s, e = par.util._find_pypeit_block(lines, 'coadd1d')
    if s >= 0 and e < 0:
        msgs.error("Missing 'coadd1d end' in {0}".format(ifile))
    elif (s < 0) or (s==e):
        msgs.error("Missing coadd1d block in in {0}. Check the input format for the .coadd1d file".format(ifile))
    else:
        for ctr, line in enumerate(lines[s:e]):
            prs = line.split(' ')
            spec1dfiles.append(prs[0])
            if ctr == 0 and len(prs) != 2:
                msgs.error('Invalid format for .coadd1d file.' + msgs.newline() +
                           'You must have specify a spec1dfile and objid on the first line of the coadd1d block')
            if len(prs) > 1:
                objids_in.append(prs[1])
        is_config[s-1:e+1] = False

    # Chck the sizes of the inputs
    nspec = len(spec1dfiles)
    if len(objids_in) == 1:
        objids = nspec*objids_in
    elif len(objids_in) == nspec:
        objids = objids_in
    else:
        msgs.error('Invalid format for .flux file.' + msgs.newline() +
                   'You must specify a single objid on the first line of the coadd1d block,' + msgs.newline() +
                   'or specify am objid for every spec1dfile in the coadd1d block.' + msgs.newline() +
                   'Run pypeit_coadd_1dspec --help for information on the format')
    # Construct config to get spectrograph
    cfg_lines = list(lines[is_config])

    # Return
    return cfg_lines, spec1dfiles, objids
Пример #33
0
def ech_load_specobj(fname,order=None):

    """ Load a spec1d file into a list of SpecObjExp objects
    Parameters
    ----------
    fname : str

    Returns
    -------
    specObjs : list of SpecObjExp
    head0
    """
    #if order is None:
    #    msgs.warn('You did not specify an order. Return specObjs with all orders.')
    #    specObjs, head0 = load.load_specobj(fname)
    #    return specObjs, head0

    speckeys = ['WAVE', 'SKY', 'MASK', 'FLAM', 'FLAM_IVAR', 'FLAM_SIG', 'COUNTS_IVAR', 'COUNTS']
    #
    specObjs = []
    hdulist = fits.open(fname)
    head0 = hdulist[0].header
    for hdu in hdulist:
        if hdu.name == 'PRIMARY':
            continue
        #elif hdu.name[8:17] != 'ORDER'+'{0:04}'.format(order):
        #    continue
        # Parse name
        idx = hdu.name
        objp = idx.split('-')
        if objp[-1][0:3] == 'DET':
            det = int(objp[-1][3:])
        else:
            det = int(objp[-1][1:])
        if objp[-2][:5] == 'ORDER':
            iord = int(objp[-2][5:])
        else:
            msgs.warn('Loading longslit data ?')
            iord = int(-1)
        # if order is not None and iord !=order then do not return this extenction
        # if order is None return all extensions
        # if order is not None and iord ==order then only return the specific order you want.
        if (order is not None) and (iord !=order):
            continue
        # Load data
        spec = Table(hdu.data)
        shape = (len(spec), 1024)  # 2nd number is dummy
        # New and wrong
        try:
            specobj = specobjs.SpecObj(shape, None, None, idx = idx)
        except:
            debugger.set_trace()
            msgs.error("BUG ME")
        # Add order number
        specobj.ech_orderindx = iord
        # ToDo: need to changed to the real order number?
        specobj.ech_order = iord
        # Add trace
        try:
            specobj.trace_spat = spec['TRACE']
        except:
            # KLUDGE!
            specobj.trace_spat = np.arange(len(spec['BOX_WAVE']))
        # Add spectrum
        if 'BOX_COUNTS' in spec.keys():
            for skey in speckeys:
                try:
                    specobj.boxcar[skey] = spec['BOX_{:s}'.format(skey)].data
                except KeyError:
                    pass
            # Add units on wave
            specobj.boxcar['WAVE'] = specobj.boxcar['WAVE'] * units.AA

        if 'OPT_COUNTS' in spec.keys():
            for skey in speckeys:
                try:
                    specobj.optimal[skey] = spec['OPT_{:s}'.format(skey)].data
                except KeyError:
                    pass
            # Add units on wave
            specobj.optimal['WAVE'] = specobj.optimal['WAVE'] * units.AA
        # Append
        specObjs.append(specobj)
    # Return
    return specObjs, head0
Пример #34
0
    def get_meta_value(self, inp, meta_key, required=False, ignore_bad_header=False, usr_row=None):
        """
        Return meta data from a given file (or its array of headers)

        Args:
            inp (str or list):
              Input filename or headarr list
            meta_key: str or list of str
            headarr: list, optional
              List of headers
            required: bool, optional
              Require the meta key to be returnable
            ignore_bad_header: bool, optional
              Over-ride required;  not recommended
            usr_row: Row
              Provides user supplied frametype (and other things not used)

        Returns:
            value: value or list of values

        """
        if isinstance(inp, str):
            headarr = self.get_headarr(inp)
        else:
            headarr = inp

        # Loop?
        if isinstance(meta_key, list):
            values = []
            for mdict in meta_key:
                values.append(self.get_meta_value(headarr, mdict, required=required))
            #
            return values

        # Are we prepared to provide this meta data?
        if meta_key not in self.meta.keys():
            if required:
                msgs.error("Need to allow for meta_key={} in your meta data".format(meta_key))
            else:
                msgs.warn("Requested meta data does not exist...")
                return None
        # Is this not derivable?  If so, use the default
        #   or search for it as a compound method
        value = None
        if self.meta[meta_key]['card'] is None:
            if 'default' in self.meta[meta_key].keys():
                value = self.meta[meta_key]['default']
            elif 'compound' in self.meta[meta_key].keys():
                value = self.compound_meta(headarr, meta_key)
            else:
                msgs.error("Failed to load spectrograph value for meta: {}".format(meta_key))
        else:
            # Grab from the header, if we can
            try:
                value = headarr[self.meta[meta_key]['ext']][self.meta[meta_key]['card']]
            except (KeyError, TypeError):
                value = None

        if value is None:
            # Was this required?
            if required:
                kerror = True
                if not ignore_bad_header:
                    # Is this meta required for this frame type (Spectrograph specific)
                    if ('required_ftypes' in self.meta[meta_key]) and (usr_row is not None):
                        kerror = False
                        # Is it required?
                        for ftype in usr_row['frametype'].split(','):
                            if ftype in self.meta[meta_key]['required_ftypes']:
                                kerror = True
                    # Bomb out?
                    if kerror:
                        msgs.error('Required meta "{:s}" did not load!  You may have a corrupt header'.format(meta_key))
                else:
                    msgs.warn("Required card {:s} missing from your header.  Proceeding with risk..".format(
                        self.meta[meta_key]['card']))
            return None

        # Deal with dtype (DO THIS HERE OR IN METADATA?  I'M TORN)
        if self.meta_data_model[meta_key]['dtype'] == str:
            value = str(value).strip()
        elif self.meta_data_model[meta_key]['dtype'] == int:
            value = int(value)
        elif self.meta_data_model[meta_key]['dtype'] == float:
            value = float(value)
        elif self.meta_data_model[meta_key]['dtype'] == tuple:
            assert isinstance(value, tuple)
        else:
            debugger.set_trace()
        # Return
        return value
Пример #35
0
def flexure_obj_oldbuggyversion(specobjs, maskslits, method, sky_spectrum, sky_file=None, mxshft=None):
    """Correct wavelengths for flexure, object by object

    Parameters:
    ----------
    method : str
      'boxcar' -- Recommneded
      'slitpix' --

    Returns:
    ----------
    flex_list: list
      list of dicts containing flexure results
        Aligned with specobjs
        Filled with a basically empty dict if the slit is skipped or there is no object

    """
    msgs.work("Consider doing 2 passes in flexure as in LowRedux")
    # Load Archive
#    skyspec_fil, arx_sky = flexure_archive(spectrograph=spectrograph, skyspec_fil=skyspec_fil)

    # Loop on objects
    flex_list = []

    gdslits = np.where(~maskslits)[0]
    for sl in range(len(specobjs)):
        # Reset
        flex_dict = dict(polyfit=[], shift=[], subpix=[], corr=[],
                         corr_cen=[], spec_file=sky_file, smooth=[],
                         arx_spec=[], sky_spec=[])
        if sl not in gdslits:
            flex_list.append(flex_dict.copy())
            continue
        msgs.info("Working on flexure in slit (if an object was detected): {:d}".format(sl))
        for specobj in specobjs[sl]:  # for convenience
            if specobj is None:
                continue

            # Using boxcar
            if method in ['boxcar', 'slitcen']:
                sky_wave = specobj.boxcar['WAVE'] #.to('AA').value
                sky_flux = specobj.boxcar['COUNTS_SKY']
            else:
                msgs.error("Not ready for this flexure method: {}".format(method))

            # Generate 1D spectrum for object
            obj_sky = xspectrum1d.XSpectrum1D.from_tuple((sky_wave, sky_flux))

            # Calculate the shift
            fdict = flex_shift(obj_sky, sky_spectrum, mxshft=mxshft)

            # Simple interpolation to apply
            npix = len(sky_wave)
            x = np.linspace(0., 1., npix)
            # Apply
            for attr in ['boxcar', 'optimal']:
                if not hasattr(specobj, attr):
                    continue
                if 'WAVE' in getattr(specobj, attr).keys():
                    msgs.info("Applying flexure correction to {0:s} extraction for object:".format(attr) +
                              msgs.newline() + "{0:s}".format(str(specobj)))
                    f = interpolate.interp1d(x, sky_wave, bounds_error=False, fill_value="extrapolate")
                    getattr(specobj, attr)['WAVE'] = f(x+fdict['shift']/(npix-1))*units.AA
            # Shift sky spec too
            cut_sky = fdict['sky_spec']
            x = np.linspace(0., 1., cut_sky.npix)
            f = interpolate.interp1d(x, cut_sky.wavelength.value, bounds_error=False, fill_value="extrapolate")
            twave = f(x + fdict['shift']/(cut_sky.npix-1))*units.AA
            new_sky = xspectrum1d.XSpectrum1D.from_tuple((twave, cut_sky.flux))

            # Update dict
            for key in ['polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth', 'arx_spec']:
                flex_dict[key].append(fdict[key])
            flex_dict['sky_spec'].append(new_sky)
        flex_list.append(flex_dict.copy())
    return flex_list
Пример #36
0
    def get_rawimage(self, raw_file, det):
        """
        Read raw images and generate a few other bits and pieces
        that are key for image processing.

        Parameters
        ----------
        raw_file : :obj:`str`
            File to read
        det : :obj:`int`
            1-indexed detector to read

        Returns
        -------
        detector_par : :class:`pypeit.images.detector_container.DetectorContainer`
            Detector metadata parameters.
        raw_img : `numpy.ndarray`_
            Raw image for this detector.
        hdu : `astropy.io.fits.HDUList`_
            Opened fits file
        exptime : :obj:`float`
            Exposure time read from the file header
        rawdatasec_img : `numpy.ndarray`_
            Data (Science) section of the detector as provided by setting the
            (1-indexed) number of the amplifier used to read each detector
            pixel. Pixels unassociated with any amplifier are set to 0.
        oscansec_img : `numpy.ndarray`_
            Overscan section of the detector as provided by setting the
            (1-indexed) number of the amplifier used to read each detector
            pixel. Pixels unassociated with any amplifier are set to 0.
        """
        # Check for file; allow for extra .gz, etc. suffix
        fil = glob.glob(raw_file + '*')
        if len(fil) != 1:
            msgs.error("Found {:d} files matching {:s}".format(len(fil)))

        # Read FITS image
        msgs.info("Reading MMT Blue Channel file: {:s}".format(fil[0]))
        hdu = fits.open(fil[0])
        hdr = hdu[0].header

        # we're flipping FITS x/y to pypeit y/x here. pypeit wants blue on the
        # bottom, slit bottom on the right...
        rawdata = np.fliplr(hdu[0].data.astype(float).transpose())

        exptime = hdr['EXPTIME']

        # TODO Store these parameters in the DetectorPar.
        # Number of amplifiers
        detector_par = self.get_detector_par(hdu, det if det is None else 1)
        numamp = detector_par['numamplifiers']

        # First read over the header info to determine the size of the output array...
        datasec = hdr['DATASEC']
        xdata1, xdata2, ydata1, ydata2 = np.array(
            parse.load_sections(datasec, fmt_iraf=False)).flatten()

        # Get the overscan section
        biassec = hdr['BIASSEC']
        xbias1, xbias2, ybias1, ybias2 = np.array(
            parse.load_sections(biassec, fmt_iraf=False)).flatten()

        # allocate output arrays and fill in with mask values
        rawdatasec_img = np.zeros_like(rawdata, dtype=int)
        oscansec_img = np.zeros_like(rawdata, dtype=int)

        # trim bad sections at beginning of data and bias sections
        rawdatasec_img[xdata1 + 2:xdata2, ydata1:ydata2 - 1] = 1
        oscansec_img[xbias1 + 2:xbias2, ybias1:ybias2 - 1] = 1

        return detector_par, rawdata, hdu, exptime, rawdatasec_img, oscansec_img
Пример #37
0
    def build_waveimg(self,
                      tilts,
                      slits,
                      spat_flexure=None,
                      spec_flexure=None):
        """
        Main algorithm to build the wavelength image

        Only applied to good slits, which means any non-flagged or flagged
         in the exclude_for_reducing list

        Args:
            tilts (`numpy.ndarray`_):
                Image holding tilts
            slits (:class:`pypeit.slittrace.SlitTraceSet`):
            spat_flexure (float, optional):
                Spatial flexure correction in pixels.
            spec_flexure (float, `numpy.ndarray`_, optional):
                Spectral flexure correction in pixels. If a float,
                the same spectral flexure correction will be applied
                to all slits. If a numpy array, the length of the
                array should be the same as the number of slits. The
                value of each element is the spectral shift in pixels
                to be applied to each slit.

        Returns:
            `numpy.ndarray`_: The wavelength image.
        """
        # Check spatial flexure type
        if (spat_flexure
                is not None) and (not isinstance(spat_flexure, float)):
            msgs.error("Spatial flexure must be None or float")
        # Check spectral flexure type
        if spec_flexure is None: spec_flex = np.zeros(slits.nslits)
        elif isinstance(spec_flexure, float):
            spec_flex = spec_flexure * np.ones(slits.nslits)
        elif isinstance(spec_flexure, np.ndarray):
            spec_flex = spec_flexure.copy()
            assert (spec_flexure.size == slits.nslits)
        spec_flex /= (slits.nspec - 1)

        # Setup
        #ok_slits = slits.mask == 0
        bpm = slits.mask.astype(bool)
        bpm &= np.logical_not(
            slits.bitmask.flagged(slits.mask,
                                  flag=slits.bitmask.exclude_for_reducing))
        ok_slits = np.logical_not(bpm)
        #
        image = np.zeros_like(tilts)
        slitmask = slits.slit_img(
            flexure=spat_flexure,
            exclude_flag=slits.bitmask.exclude_for_reducing)

        # If this is echelle print out a status message and do some error checking
        if self.par['echelle']:
            msgs.info('Evaluating 2-d wavelength solution for echelle....')
            # TODO UPDATE THIS!!
            #if len(wv_calib['fit2d']['orders']) != np.sum(ok_slits):
            #    msgs.error('wv_calib and ok_slits do not line up. Something is very wrong!')

        # Unpack some 2-d fit parameters if this is echelle
        for islit in np.where(ok_slits)[0]:
            slit_spat = slits.spat_id[islit]
            thismask = (slitmask == slit_spat)
            if not np.any(thismask):
                msgs.error("Something failed in wavelengths or masking..")
            if self.par['echelle']:
                # # TODO: Put this in `SlitTraceSet`?
                # evaluate solution --
                image[thismask] = self.wv_fit2d.eval(
                    tilts[thismask] + spec_flex[islit],
                    x2=np.full_like(tilts[thismask], slits.ech_order[islit]))
                image[thismask] /= slits.ech_order[islit]
            else:
                iwv_fits = self.wv_fits[islit]
                image[thismask] = iwv_fits.pypeitfit.eval(tilts[thismask] +
                                                          spec_flex[islit])
        # Return
        return image
Пример #38
0
def flexure_obj(specobjs, maskslits, method, sky_file, mxshft=None):
    """Correct wavelengths for flexure, object by object

    Parameters:
    ----------
    method : str
      'boxcar' -- Recommneded
      'slitpix' --
    sky_file: str

    Returns:
    ----------
    flex_list: list
      list of dicts containing flexure results
        Aligned with specobjs
        Filled with a basically empty dict if the slit is skipped or there is no object

    """
    sv_fdict = None
    msgs.work("Consider doing 2 passes in flexure as in LowRedux")
    # Load Archive
    sky_spectrum = load_sky_spectrum(sky_file)

    nslits = len(maskslits)
    gdslits = np.where(~maskslits)[0]

    # Loop on objects
    flex_list = []

    # Slit/objects to come back to
    return_later_sobjs = []

    # Loop over slits, and then over objects here
    for slit in range(nslits):
        msgs.info("Working on flexure in slit (if an object was detected): {:d}".format(slit))
        indx = specobjs.slitid == slit
        this_specobjs = specobjs[indx]
        # Reset
        flex_dict = dict(polyfit=[], shift=[], subpix=[], corr=[],
                         corr_cen=[], spec_file=sky_file, smooth=[],
                         arx_spec=[], sky_spec=[])
        # If no objects on this slit append an empty dictionary
        if slit not in gdslits:
            flex_list.append(flex_dict.copy())
            continue
        for ss, specobj in enumerate(this_specobjs):
            if specobj is None:
                continue
            msgs.info("Working on flexure for object # {:d}".format(specobj.objid) + "in slit # {:d}".format(specobj.slitid))
            # Using boxcar
            if method in ['boxcar', 'slitcen']:
                sky_wave = specobj.boxcar['WAVE'] #.to('AA').value
                sky_flux = specobj.boxcar['COUNTS_SKY']
            else:
                msgs.error("Not ready for this flexure method: {}".format(method))

            # Generate 1D spectrum for object
            obj_sky = xspectrum1d.XSpectrum1D.from_tuple((sky_wave, sky_flux))

            # Calculate the shift
            fdict = flex_shift(obj_sky, sky_spectrum, mxshft=mxshft)
            punt = False
            if fdict is None:
                msgs.warn("Flexure shift calculation failed for this spectrum.")
                if sv_fdict is not None:
                    msgs.warn("Will used saved estimate from a previous slit/object")
                    fdict = copy.deepcopy(sv_fdict)
                else:
                    # One does not exist yet
                    # Save it for later
                    return_later_sobjs.append([slit, ss])
                    punt = True
            else:
                sv_fdict = copy.deepcopy(fdict)

            # Punt?
            if punt:
                break

            # Interpolate
            new_sky = specobj.flexure_interp(sky_wave, fdict)
            # Update dict
            for key in ['polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth', 'arx_spec']:
                flex_dict[key].append(fdict[key])
            flex_dict['sky_spec'].append(new_sky)

        flex_list.append(flex_dict.copy())

        # Do we need to go back?
        for items in return_later_sobjs:
            if sv_fdict is None:
                msgs.info("No flexure corrections could be made")
                break
            # Setup
            slit, ss = items
            flex_dict = flex_list[slit]
            specobj = specobjs[ss]
            sky_wave = specobj.boxcar['WAVE'] #.to('AA').value
            # Copy me
            fdict = copy.deepcopy(sv_fdict)
            # Interpolate
            new_sky = specobj.flexure_interp(sky_wave, fdict)
            # Update dict
            for key in ['polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth', 'arx_spec']:
                flex_dict[key].append(fdict[key])
            flex_dict['sky_spec'].append(new_sky)

    return flex_list
Пример #39
0
def ech_coadd(files,objids=None,extract='OPT',flux=True,giantcoadd=False,
              wave_grid_method='velocity', niter=5,wave_grid_min=None, wave_grid_max=None,v_pix=None,
              scale_method='auto', do_offset=False, sigrej_final=3.,do_var_corr=False,
              qafile=None, outfile=None,do_cr=True, debug=False,**kwargs):

    nfile = len(files)
    if nfile <=1:
        msgs.info('Only one spectrum exits coadding...')
        return

    fname = files[0]
    ext_final = fits.getheader(fname, -1)
    norder = ext_final['ORDER'] + 1
    msgs.info('spectrum {:s} has {:d} orders'.format(fname, norder))
    if norder <= 1:
        msgs.error('The number of orders have to be greater than one for echelle. Longslit data?')

    if giantcoadd:
        msgs.info('Coadding all orders and exposures at once')
        spectra = ech_load_spec(files, objid=objids,order=None, extract=extract, flux=flux)
        wave_grid = np.zeros((2,spectra.nspec))
        for i in range(spectra.nspec):
            wave_grid[0, i] = spectra[i].wvmin.value
            wave_grid[1, i] = spectra[i].wvmax.value
        ech_kwargs = {'echelle': True, 'wave_grid_min': np.min(wave_grid), 'wave_grid_max': np.max(wave_grid),
                      'v_pix': v_pix}
        kwargs.update(ech_kwargs)
        # Coadding
        spec1d = coadd.coadd_spectra(spectra, wave_grid_method=wave_grid_method, niter=niter,
                                          scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
                                          do_var_corr=do_var_corr, qafile=qafile, outfile=outfile,
                                          do_cr=do_cr, debug=debug,**kwargs)
    else:
        msgs.info('Coadding individual orders first and then merge order')
        spectra_list = []
        # Keywords for Table
        rsp_kwargs = {}
        rsp_kwargs['wave_tag'] = '{:s}_WAVE'.format(extract)
        rsp_kwargs['flux_tag'] = '{:s}_FLAM'.format(extract)
        rsp_kwargs['sig_tag'] = '{:s}_FLAM_SIG'.format(extract)
        wave_grid = np.zeros((2,norder))
        for iord in range(norder):
            spectra = ech_load_spec(files, objid=objids, order=iord, extract=extract, flux=flux)
            ech_kwargs = {'echelle': False, 'wave_grid_min': spectra.wvmin.value, 'wave_grid_max': spectra.wvmax.value, 'v_pix': v_pix}
            wave_grid[0,iord] = spectra.wvmin.value
            wave_grid[1,iord] = spectra.wvmax.value
            kwargs.update(ech_kwargs)
            # Coadding the individual orders
            if qafile is not None:
                qafile_iord = qafile+'_%s'%str(iord)
            else:
                qafile_iord =  None
            spec1d_iord = coadd.coadd_spectra(spectra, wave_grid_method=wave_grid_method, niter=niter,
                                       scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
                                       do_var_corr=do_var_corr, qafile=qafile_iord, outfile=outfile,
                                       do_cr=do_cr, debug=debug, **kwargs)
            spectrum = spec_from_array(spec1d_iord.wavelength, spec1d_iord.flux, spec1d_iord.sig,**rsp_kwargs)
            spectra_list.append(spectrum)
        # Join into one XSpectrum1D object
        spectra_coadd = collate(spectra_list)
        kwargs['echelle'] = True
        kwargs['wave_grid_min'] = np.min(wave_grid)
        kwargs['wave_grid_max'] = np.max(wave_grid)
        # ToDo: Currently I'm not using the first order due to some problem. Need to add it back after fix the problem.
        spec1d = coadd.coadd_spectra(spectra_coadd[1:], wave_grid_method=wave_grid_method, niter=niter,
                                          scale_method=scale_method, do_offset=do_offset, sigrej_final=sigrej_final,
                                          do_var_corr=do_var_corr, qafile=qafile, outfile=outfile,
                                          do_cr=do_cr, debug=debug, **kwargs)

    return spec1d