Exemple #1
0
def flexure_slit(slf, det):
    """Correct wavelength down slit center for flexure

    Parameters:
    ----------
    slf :
    det : int
    """
    # Load Archive
    skyspec_fil, arx_sky = flexure_archive(slf, det)

    # Extract
    censpec_wv = arextract.boxcar_cen(slf, det, slf._mswave[det-1])
    censpec_fx = arextract.boxcar_cen(slf, det, slf._bgframe[det-1])
    cen_sky = xspectrum1d.XSpectrum1D.from_tuple((censpec_wv, censpec_fx))
    # Find shift
    fdict = flex_shift(slf, det, cen_sky, arx_sky)
    msgs.work("Flexure shift = {:g} down slit center".format(fdict['shift']))
    # Refit
    #  What if xfit shifts outside of 0-1?
    xshift = fdict['shift']/(slf._msarc[det-1].shape[0]-1)
    mask, fit = arutils.robust_polyfit(np.array(slf._wvcalib[det-1]['xfit'])+xshift,
                                       np.array(slf._wvcalib[det-1]['yfit']),
                                       len(slf._wvcalib[det-1]['fitc']),
                                       function=slf._wvcalib[det-1]['function'], sigma=slf._wvcalib[det-1]['nrej'], minv=slf._wvcalib[det-1]['fmin'], maxv=slf._wvcalib[det-1]['fmax'])
    # Update wvcalib
    slf._wvcalib[det-1]['shift'] = fdict['shift']  # pixels
    slf._wvcalib[det-1]['fitc'] = fit
    msgs.work("Add another QA for wavelengths?")
    # Update mswave
    wv_calib = slf._wvcalib[det-1]
    slf._mswave[det-1] = arutils.func_val(wv_calib['fitc'], slf._tilts[det-1], wv_calib['function'], minv=wv_calib['fmin'], maxv=wv_calib['fmax'])
    # Write to Masters?  Not for now
    # For QA (kludgy..)
    censpec_wv = arextract.boxcar_cen(slf, det, slf._mswave[det-1])
    fdict['sky_spec'] = xspectrum1d.XSpectrum1D.from_tuple((censpec_wv, censpec_fx))
    flex_dict = dict(polyfit=[], shift=[], subpix=[], corr=[],
                     corr_cen=[], spec_file=skyspec_fil, smooth=[],
                     arx_spec=[], sky_spec=[])
    #debugger.set_trace()
    #debugger.xplot(censpec_wv, censpec_fx, xtwo=fdict['arx_spec'].wavelength, ytwo=fdict['arx_spec'].flux*50)
    for key in ['polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth', 'sky_spec', 'arx_spec']:
        flex_dict[key].append(fdict[key])
    return flex_dict
Exemple #2
0
def flexure_slit(slf, det):
    """Correct wavelength down slit center for flexure

    Parameters:
    ----------
    slf :
    det : int
    """
    # Load Archive
    skyspec_fil, arx_sky = flexure_archive(slf, det)

    # Extract
    censpec_wv = arextract.boxcar_cen(slf, det, slf._mswave[det-1])
    censpec_fx = arextract.boxcar_cen(slf, det, slf._bgframe[det-1])
    cen_sky = xspectrum1d.XSpectrum1D.from_tuple((censpec_wv, censpec_fx))
    # Find shift
    fdict = flex_shift(slf, det, cen_sky, arx_sky)
    msgs.work("Flexure shift = {:g} down slit center".format(fdict['shift']))
    # Refit
    #  What if xfit shifts outside of 0-1?
    xshift = fdict['shift']/(slf._msarc[det-1].shape[0]-1)
    mask, fit = arutils.robust_polyfit(np.array(slf._wvcalib[det-1]['xfit'])+xshift,
                                       np.array(slf._wvcalib[det-1]['yfit']),
                                       len(slf._wvcalib[det-1]['fitc']),
                                       function=slf._wvcalib[det-1]['function'], sigma=slf._wvcalib[det-1]['nrej'], minv=slf._wvcalib[det-1]['fmin'], maxv=slf._wvcalib[det-1]['fmax'])
    # Update wvcalib
    slf._wvcalib[det-1]['shift'] = fdict['shift']  # pixels
    slf._wvcalib[det-1]['fitc'] = fit
    msgs.work("Add another QA for wavelengths?")
    # Update mswave
    wv_calib = slf._wvcalib[det-1]
    slf._mswave[det-1] = arutils.func_val(wv_calib['fitc'], slf._tilts[det-1], wv_calib['function'], minv=wv_calib['fmin'], maxv=wv_calib['fmax'])
    # Write to Masters?  Not for now
    # For QA (kludgy..)
    censpec_wv = arextract.boxcar_cen(slf, det, slf._mswave[det-1])
    fdict['sky_spec'] = xspectrum1d.XSpectrum1D.from_tuple((censpec_wv, censpec_fx))
    flex_dict = dict(polyfit=[], shift=[], subpix=[], corr=[],
                     corr_cen=[], spec_file=skyspec_fil, smooth=[],
                     arx_spec=[], sky_spec=[])
    #debugger.set_trace()
    #debugger.xplot(censpec_wv, censpec_fx, xtwo=fdict['arx_spec'].wavelength, ytwo=fdict['arx_spec'].flux*50)
    for key in ['polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth', 'sky_spec', 'arx_spec']:
        flex_dict[key].append(fdict[key])
    return flex_dict
Exemple #3
0
def refine_iter(outpar, orders, mask, irshft, relshift, fitord, function='polynomial'):
    fail = False
    x0ex = arutils.func_val(outpar['x0res'], orders, function,  minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
    # Make the refinement
    x0ex[irshft] += relshift
    # Refit the data to improve the refinement
    good = np.where(mask != 0.0)[0]
#	x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
    null, x0res = arutils.robust_polyfit(orders[good], x0ex[good], fitord, sigma=2.0, function=function,
                                         minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
    #x0res = arutils.func_fit(orders[good], x0ex[good], function, fitord, min=outpar['x0in'][0], max=outpar['x0in'][-1])
    x0fit = arutils.func_val(x0res, orders, function, minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
    chisq = (x0ex[good]-x0fit[good])**2.0
    chisqnu = np.sum(chisq)/np.sum(mask)
    msgs.prindent("  Reduced chi-squared = {0:E}".format(chisqnu))
    if chisqnu > 0.5: # The refinement went wrong, ignore the refinement
        fail = True
    outpar['x0res'] = x0res
    extfit = np.dot(outpar['eigv'], outpar['high_matr'].T) + np.outer(x0fit, np.ones(outpar['eigv'].shape[0])).T
    return extfit, outpar, fail
Exemple #4
0
def refine_iter(outpar,
                orders,
                mask,
                irshft,
                relshift,
                fitord,
                function='polynomial'):
    fail = False
    x0ex = arutils.func_val(outpar['x0res'],
                            orders,
                            function,
                            minv=outpar['x0in'][0],
                            maxv=outpar['x0in'][-1])
    # Make the refinement
    x0ex[irshft] += relshift
    # Refit the data to improve the refinement
    good = np.where(mask != 0.0)[0]
    #	x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
    null, x0res = arutils.robust_polyfit(orders[good],
                                         x0ex[good],
                                         fitord,
                                         sigma=2.0,
                                         function=function,
                                         minv=outpar['x0in'][0],
                                         maxv=outpar['x0in'][-1])
    #x0res = arutils.func_fit(orders[good], x0ex[good], function, fitord, min=outpar['x0in'][0], max=outpar['x0in'][-1])
    x0fit = arutils.func_val(x0res,
                             orders,
                             function,
                             minv=outpar['x0in'][0],
                             maxv=outpar['x0in'][-1])
    chisq = (x0ex[good] - x0fit[good])**2.0
    chisqnu = np.sum(chisq) / np.sum(mask)
    msgs.prindent("  Reduced chi-squared = {0:E}".format(chisqnu))
    if chisqnu > 0.5:  # The refinement went wrong, ignore the refinement
        fail = True
    outpar['x0res'] = x0res
    extfit = np.dot(outpar['eigv'], outpar['high_matr'].T) + np.outer(
        x0fit, np.ones(outpar['eigv'].shape[0])).T
    return extfit, outpar, fail
Exemple #5
0
def background_subtraction(slf,
                           sciframe,
                           varframe,
                           slitn,
                           det,
                           refine=0.0,
                           doqa=True):
    """ Generate a frame containing the background sky spectrum

    Parameters
    ----------
    slf : Class
      Science Exposure Class
    sciframe : ndarray
      science frame
    varframe : ndarray
      variance frame
    slitn : int
      Slit number
    det : int
      Detector index
    refine : float or ndarray
      refine the object traces. This should be a small value around 0.0.
      If a float, a constant offset will be applied.
      Otherwise, an array needs to be specified of the same length as
      sciframe.shape[0] that contains the refinement of each pixel along
      the spectral direction.

    Returns
    -------
    bgframe : ndarray
      An image, the same size as sciframe, that contains
      the background spectrum within the specified slit.
    nl : int
      number of pixels from the left slit edge to use as background pixels
    nr : int
      number of pixels from the right slit edge to use as background pixels
    """
    # Obtain all pixels that are within the slit edges, and are not masked
    word = np.where((slf._slitpix[det - 1] == slitn + 1)
                    & (slf._scimask[det - 1] == 0))
    if word[0].size == 0:
        msgs.warn("There are no pixels in slit {0:d}".format(slitn))
        debugger.set_trace()
        nl, nr = 0, 0
        return np.zeros_like(sciframe), nl, nr
    # Calculate the oversampled object profiles
    oversampling_factor = 3  # should be an integer according to the description in object_profile()
    xedges, modvals = object_profile(slf,
                                     sciframe,
                                     slitn,
                                     det,
                                     refine=refine,
                                     factor=oversampling_factor)
    bincent = 0.5 * (xedges[1:] + xedges[:-1])
    npix = slf._pixwid[det - 1][slitn]
    tilts = slf._tilts[det - 1].copy()
    lordloc = slf._lordloc[det - 1][:, slitn]
    rordloc = slf._rordloc[det - 1][:, slitn]
    # For each pixel, calculate the fraction along the slit's spatial direction
    spatval = (word[1] - lordloc[word[0]] + refine) / (rordloc[word[0]] -
                                                       lordloc[word[0]])
    # Cumulative sum and normalize
    csum = np.cumsum(modvals)
    csum -= csum[0]
    csum /= csum[-1]
    # Find a first guess of the edges of the object profile - assume this is the innermost 90 percent of the flux
    argl = np.argmin(np.abs(csum - 0.05))
    argr = np.argmin(np.abs(csum - 0.95))
    # Considering the possible background pixels that are left of the object,
    # find the first time where the object profile no longer decreases as you
    # move toward the edge of the slit. This is the beginning of the noisy
    # object profile, which is where the object can no longer be distinguished
    # from the background.
    wl = np.where((modvals[1:] < modvals[:-1]) & (bincent[1:] < bincent[argl]))
    wr = np.where((modvals[1:] > modvals[:-1]) & (bincent[1:] > bincent[argr]))
    nl, nr = 0, 0
    if wl[0].size != 0:
        # This is the index of the first time where the object profile
        # no longer decreases as you move towards the slit edge
        nl_index = np.max(wl[0])
        # Calculate nl, defined as:
        # "number of pixels from the left slit edge to use as background pixels",
        # which is just nl_index with the sampling factor taken out
        nl_index_origscale = int(nl_index / oversampling_factor + 0.5)
        nl = nl_index_origscale
    if wr[0].size != 0:
        # This is the index of the first time where the object profile
        # no longer decreases as you move towards the slit edge
        nr_index = np.min(wr[0])
        # Calculate nr, defined as:
        # "number of pixels from the right slit edge to use as background pixels",
        # which is npix minus nr_index with the sampling factor taken out
        nr_index_origscale = int(nr_index / oversampling_factor + 0.5)
        nr = npix - nr_index_origscale
    if nl + nr < 5:
        msgs.warn(
            "The object profile appears to extrapolate to the edge of the slit"
        )
        msgs.info(
            "A background subtraction will not be performed for slit {0:d}".
            format(slitn + 1))
        nl, nr = 0, 0
        return np.zeros_like(sciframe), nl, nr
    # Find background pixels and fit
    wbgpix_spatval = np.where(
        (spatval <= float(nl) / npix) |
        (spatval >= float(npix - nr) /
         npix))  # this cannot be used to index the 2D array tilts
    wbgpix = (word[0][wbgpix_spatval], word[1][wbgpix_spatval]
              )  # this may be approproate for indexing the 2D array tilts
    if settings.argflag['reduce']['skysub']['method'].lower() == 'bspline':
        msgs.info("Using bspline sky subtraction")
        srt = np.argsort(tilts[wbgpix])
        ivar = arutils.calc_ivar(varframe)
        # Perform a weighted b-spline fit to the sky background pixels
        mask, bspl = arutils.robust_polyfit(
            tilts[wbgpix][srt],
            sciframe[wbgpix][srt],
            3,
            function='bspline',
            weights=np.sqrt(ivar)[wbgpix][srt],
            sigma=5.,
            maxone=False,
            **settings.argflag['reduce']['skysub']['bspline'])
        bgf_flat = arutils.func_val(bspl, tilts.flatten(), 'bspline')
        bgframe = bgf_flat.reshape(tilts.shape)
        if doqa:
            plt_bspline_sky(tilts, sciframe, bgf_flat, gdp)
            debugger.set_trace()
    else:
        msgs.error('Not ready for this method for skysub {:s}'.format(
            settings.argflag['reduce']['skysub']['method'].lower()))
    if np.any(np.isnan(bgframe)):
        msgs.warn("NAN in bgframe.  Replacing with 0")
        bad = np.isnan(bgframe)
        bgframe[bad] = 0.
    return bgframe, nl, nr
Exemple #6
0
def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None, function='polynomial', retmask=False):
    nrow = xfit.shape[0]
    ntrace = xfit.shape[1]
    if x0in is None:
        x0in = np.arange(float(ntrace))

    # Mask out some orders if they are bad
    if mask is None or mask.size == 0:
        usetrace = np.arange(ntrace)
        outmask = np.ones((nrow, ntrace))
    else:
        usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0]
        outmask = np.ones((nrow, ntrace))
        outmask[:,mask] = 0.0

    # Do the PCA analysis
    eigc, hidden = get_pc(coeff[1:npc+1, usetrace], npc)

    modl = arutils.func_vander(xfit[:,0], function, npc)
    eigv = np.dot(modl[:,1:], eigc)

    med_hidden = np.median(hidden, axis=1)
    med_highorder = med_hidden.copy()
    med_highorder[0] = 0

    high_order_matrix = med_highorder.T[np.newaxis,:].repeat(ntrace, axis=0)

    # y = hidden[0,:]
    # coeff0 = arutils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function)

    # y = hidden[1,:]
    # coeff1 = arutils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function)

    coeffstr = []
    for i in xrange(1, npc+1):
        # if pnpc[i] == 0:
        #     coeffstr.append([-9.99E9])
        #     continue
        # coeff0 = arutils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1])
        if weights is not None:
            tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace], hidden[i-1,:], pnpc[i],
                                                   weights=weights[usetrace], sigma=2.0, function=function,
                                                   minv=x0in[0], maxv=x0in[-1])
        else:
            tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace], hidden[i-1,:], pnpc[i],
                                                   sigma=2.0, function=function, minv=x0in[0], maxv=x0in[-1])
        coeffstr.append(coeff0)
        high_order_matrix[:,i-1] = arutils.func_val(coeff0, x0in, function, minv=x0in[0], maxv=x0in[-1])
    # high_order_matrix[:,1] = arutils.func_val(coeff1, x0in, function)
    high_fit = high_order_matrix.copy()

    high_order_fit = np.dot(eigv, high_order_matrix.T)
    sub = (yfit - high_order_fit) * outmask

    numer = np.sum(sub, axis=0)
    denom = np.sum(outmask, axis=0)
    x0 = np.zeros(float(ntrace))
    fitmask = np.zeros(float(ntrace))
    #fitmask[mask] = 1
    x0fit = np.zeros(float(ntrace))
    chisqnu = 0.0
    chisqold = 0.0
    robust = True
    #svx0 = numer/(denom+(denom == 0).astype(np.int))
    if not skipx0:
        fitmask = (np.abs(denom) > 10).astype(np.int)
        if robust:
            good = np.where(fitmask != 0)[0]
            bad = np.where(fitmask == 0)[0]
            x0[good] = numer[good]/denom[good]
            imask = np.zeros(float(ntrace))
            imask[bad] = 1.0
            ttmask, x0res = arutils.robust_polyfit(x0in, x0, pnpc[0], weights=weights, sigma=2.0,
                                                   function=function, minv=x0in[0], maxv=x0in[-1], initialmask=imask)
            x0fit = arutils.func_val(x0res, x0in, function, minv=x0in[0], maxv=x0in[-1])
            good = np.where(ttmask == 0)[0]
            xstd = 1.0  # This should represent the dispersion in the fit
            chisq = ((x0[good]-x0fit[good])/xstd)**2.0
            chisqnu = np.sum(chisq)/np.sum(ttmask)
            fitmask = 1.0-ttmask
            msgs.prindent("  Reduced chi-squared = {0:E}".format(chisqnu))
        else:
            for i in xrange(1, 5):
                good = np.where(fitmask != 0)[0]
                x0[good] = numer[good]/denom[good]
#				x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
                x0res = arutils.func_fit(x0in[good], x0[good], function, pnpc[0],
                                         weights=weights, minv=x0in[0], maxv=x0in[-1])
                x0fit = arutils.func_val(x0res, x0in, function, minv=x0in[0], maxv=x0in[-1])
                chisq = (x0[good]-x0fit[good])**2.0
                fitmask[good] *= (chisq < np.sum(chisq)/2.0).astype(np.int)
                chisqnu = np.sum(chisq)/np.sum(fitmask)
                msgs.prindent("  Reduced chi-squared = {0:E}".format(chisqnu))
                if chisqnu == chisqold:
                    break
                else:
                    chisqold = chisqnu
        if chisqnu > 2.0:
            msgs.warn("PCA has very large residuals")
        elif chisqnu > 0.5:
            msgs.warn("PCA has fairly large residuals")
        #bad = np.where(fitmask==0)[0]
        #x0[bad] = x0fit[bad]
    else:
        x0res = 0.0
    x3fit = np.dot(eigv,high_order_matrix.T) + np.outer(x0fit,np.ones(nrow)).T
    outpar = dict({'high_fit': high_fit, 'x0': x0, 'x0in': x0in, 'x0fit': x0fit, 'x0res': x0res, 'x0mask': fitmask,
                   'hidden': hidden, 'usetrc': usetrace, 'eigv': eigv, 'npc': npc, 'coeffstr': coeffstr})
    if retmask:
        return x3fit, outpar, tmask
    else:
        return x3fit, outpar
Exemple #7
0
def bspline_magfit(wave, flux, var, flux_std, nointerp=False, **kwargs):
    """
    Perform a bspline fit to the flux ratio of standard to
    observed counts.  Used to generate a sensitivity function.

    Parameters
    ----------
    wave : ndarray
    flux : ndarray
      counts/s as observed
    var : ndarray
      variance
    flux_std : Quantity array
      standard star true flux (erg/s/cm^2/A)
    nointer : bool, optional [False]
      Skip interpolation over bad points (not recommended)?
    **kwargs : keywords for robust_polyfit

    Returns
    -------
    """
    from pypit import arutils
    invvar = (var > 0.)/(var + (var <= 0.))
    nx = wave.size
    pos_error = 1./np.sqrt(np.maximum(invvar,0.) + (invvar == 0))
    pos_mask = (flux > pos_error/10.0) & (invvar > 0) & (flux_std > 0.0)
    #pos = pos_mask==1 npos)

    fluxlog = 2.5*np.log10(np.maximum(flux,pos_error/10))
    logivar = invvar * flux**2 * pos_mask*1.08574

    # cap the magfunc so that sensfunc < 1.0e10
    magfunc = 2.5*np.log10(np.maximum(flux_std,1.0e-2)) - fluxlog
    magfunc = np.minimum(magfunc,25.0)
    sensfunc = 10.0**(0.4*magfunc)*pos_mask

    # Interpolate over masked pixels
    if not nointerp:
        bad = logivar <= 0. 
        if np.sum(bad) > 0:
            f = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], magfunc[~bad], k=2)
            magfunc[bad] = f(wave[bad])
            fi = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], logivar[~bad], k=2)
            logivar[bad] = fi(wave[bad])

    #  First iteration
    mask, tck = arutils.robust_polyfit(wave, magfunc, 3, function='bspline', weights=np.sqrt(logivar), **kwargs)
    logfit1 = arutils.func_val(tck,wave,'bspline')
    modelfit1 = 10.0**(0.4*(logfit1))

    residual = sensfunc/(modelfit1 + (modelfit1 == 0)) - 1.
    new_mask = pos_mask & (sensfunc > 0)
    residual_ivar = (modelfit1*flux/(sensfunc + (sensfunc == 0.0)))**2*invvar
    residual_ivar = residual_ivar*new_mask

    # Interpolate over masked pixels
    if not nointerp:
        if np.sum(bad) > 0:
            f = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], residual[~bad], k=2)
            residual[bad] = f(wave[bad])
            fi = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], residual_ivar[~bad], k=2)
            residual_ivar[bad] = fi(wave[bad])

    #  Now do one more fit to the ratio of data/model - 1.
    # Fuss with the knots first ()
    inner_knots = arutils.bspline_inner_knots(tck[0])
    #
    mask, tck_residual = arutils.robust_polyfit(wave, residual, 3, function='bspline', weights=np.sqrt(residual_ivar), knots=inner_knots, **kwargs)
    if tck_residual[1].size != tck[1].size:
        msgs.error('Problem with bspline knots in bspline_magfit')
    #bset_residual = bspline_iterfit(wave, residual, weights=np.sqrt(residual_ivar), knots = tck[0])

    tck_log1 = list(tck)
    tck_log1[1] = tck[1] + tck_residual[1]

    sensfit = 10.0**(0.4*(arutils.func_val(tck_log1,wave, 'bspline')))

    absdev = np.median(np.abs(sensfit/modelfit1-1))
    msgs.info('Difference between fits is {:g}'.format(absdev))

    # QA
    msgs.work("Add QA for sensitivity function")

    return tck_log1
Exemple #8
0
def simple_calib(slf, det, get_poly=False):
    """Simple calibration algorithm for longslit wavelengths

    Uses slf._arcparam to guide the analysis

    Parameters
    ----------
    get_poly : bool, optional
      Pause to record the polynomial pix = b0 + b1*lambda + b2*lambda**2

    Returns
    -------
    final_fit : dict
      Dict of fit info
    """

    # Extract the arc
    msgs.work("Detecting lines..")
    tampl, tcent, twid, w, satsnd, yprep = detect_lines(
        slf, det, slf._msarc[det - 1])

    # Cut down to the good ones
    tcent = tcent[w]
    tampl = tampl[w]
    msgs.info('Detected {:d} lines in the arc spectrum.'.format(len(w[0])))

    # Parameters (just for convenience)
    aparm = slf._arcparam[det - 1]

    # Read Arc linelist
    llist = aparm['llist']

    # IDs were input by hand
    if len(settings.argflag['arc']['calibrate']['IDpixels']) > 0:
        # Check that there are at least 5 values
        pixels = np.array(settings.argflag['arc']['calibrate']['IDpixels'])
        if np.sum(pixels > 0.) < 5:
            msgs.error("Need to give at least 5 pixel values!")
        #
        msgs.info("Using input lines to seed the wavelength solution")
        # Calculate median offset
        mdiff = [
            np.min(np.abs(tcent - pix))
            for pix in settings.argflag['arc']['calibrate']['IDpixels']
        ]
        med_poff = np.median(np.array(mdiff))
        msgs.info("Will apply a median offset of {:g} pixels".format(med_poff))

        # Match input lines to observed spectrum
        nid = len(settings.argflag['arc']['calibrate']['IDpixels'])
        idx_str = np.ones(nid).astype(int)
        ids = np.zeros(nid)
        idsion = np.array(['     '] * nid)
        gd_str = np.arange(nid).astype(int)
        for jj, pix in enumerate(
                settings.argflag['arc']['calibrate']['IDpixels']):
            diff = np.abs(tcent - pix - med_poff)
            if np.min(diff) > 2.:
                debugger.set_trace()
                msgs.error("No match with input pixel {:g}!".format(pix))
            else:
                imn = np.argmin(diff)
            # Set
            idx_str[jj] = imn
            # Take wavelength from linelist instead of input value
            wdiff = np.abs(llist['wave'] -
                           settings.argflag['arc']['calibrate']['IDwaves'][jj])
            imnw = np.argmin(wdiff)
            if wdiff[imnw] > 0.015:  # Arbitrary tolerance
                msgs.error(
                    "Input IDwaves={:g} is not in the linelist.  Fix".format(
                        settings.argflag['arc']['calibrate']['IDwaves'][jj]))
            else:
                ids[jj] = llist['wave'][imnw]
                idsion[jj] = llist['Ion'][imnw]
                msgs.info("Identifying arc line: {:s} {:g}".format(
                    idsion[jj], ids[jj]))
    else:
        # Generate dpix pairs
        msgs.info("Using pair algorithm for wavelength solution")
        nlist = len(llist)
        dpix_list = np.zeros((nlist, nlist))
        for kk, row in enumerate(llist):
            #dpix_list[kk,:] = (np.array(row['wave'] - llist['wave']))/disp
            dpix_list[kk, :] = slf._msarc[det - 1].shape[0] * (
                aparm['b1'] * (np.array(row['wave'] - llist['wave'])) +
                aparm['b2'] * np.array(row['wave']**2 - llist['wave']**2))

        # Lambda pairs for the strongest N lines
        srt = np.argsort(tampl)
        idx_str = srt[-aparm['Nstrong']:]
        idx_str.sort()
        dpix_obs = np.zeros((aparm['Nstrong'], aparm['Nstrong']))
        for kk, idx in enumerate(idx_str):
            dpix_obs[kk, :] = np.array(tcent[idx] - tcent[idx_str])

        # Match up (ugly loops)
        ids = np.zeros(aparm['Nstrong'])
        idsion = np.array(['     '] * aparm['Nstrong'])
        for kk in range(aparm['Nstrong']):
            med_off = np.zeros(nlist)
            for ss in range(nlist):
                dpix = dpix_list[ss]
                min_off = []
                for jj in range(aparm['Nstrong']):
                    min_off.append(np.min(np.abs(dpix_obs[kk, jj] - dpix)))
                med_off[ss] = np.median(min_off)
            # Set by minimum
            idm = np.argmin(med_off)
            ids[kk] = llist['wave'][idm]
            idsion[kk] = llist['Ion'][idm]

        # Calculate disp of the strong lines
        disp_str = np.zeros(aparm['Nstrong'])
        for kk in range(aparm['Nstrong']):
            disp_val = (ids[kk] - ids) / (tcent[idx_str[kk]] - tcent[idx_str])
            isf = np.isfinite(disp_val)
            disp_str[kk] = np.median(disp_val[isf])
        # Consider calculating the RMS with clipping
        gd_str = np.where(
            np.abs(disp_str - aparm['disp']) /
            aparm['disp'] < aparm['disp_toler'])[0]
        msgs.info('Found {:d} lines within the dispersion threshold'.format(
            len(gd_str)))
        if len(gd_str) < 5:
            if msgs._debug['arc']:
                msgs.warn('You should probably try your best to ID lines now.')
                debugger.set_trace()
                debugger.plot1d(yprep)
            else:
                msgs.error('Insufficient lines to auto-fit.')

    # Debug
    #debug=True
    if msgs._debug['arc']:
        #tmp = list(gd_str)
        #tmp.pop(1)
        #gd_str = np.array(tmp)
        #xdb.xpcol(tcent[idx_str[gd_str]],ids[gd_str])
        #xdb.xplot(tcent[idx_str[gd_str]],ids[gd_str],scatter=True)
        # debugger.xplot(yprep)
        debugger.set_trace()

    msgs.work('Cross correlate here?')

    # Setup for fitting
    ifit = idx_str[gd_str]
    sv_ifit = list(ifit)  # Keep the originals
    all_ids = -999. * np.ones(len(tcent))
    all_idsion = np.array(['12345'] * len(tcent))
    all_ids[ifit] = ids[gd_str]
    all_idsion[ifit] = idsion[gd_str]
    # Fit
    n_order = aparm['n_first']
    flg_quit = False
    fmin, fmax = -1., 1.
    msgs.info('Iterative wavelength fitting..')
    while (n_order <= aparm['n_final']) and (flg_quit is False):
        #msgs.info('n_order={:d}'.format(n_order))
        # Fit with rejection
        xfit, yfit = tcent[ifit], all_ids[ifit]
        mask, fit = arutils.robust_polyfit(xfit,
                                           yfit,
                                           n_order,
                                           function=aparm['func'],
                                           sigma=aparm['nsig_rej'],
                                           minv=fmin,
                                           maxv=fmax)
        # Reject but keep originals (until final fit)
        ifit = list(ifit[mask == 0]) + sv_ifit
        # Find new points (should we allow removal of the originals?)
        twave = arutils.func_val(fit,
                                 tcent,
                                 aparm['func'],
                                 minv=fmin,
                                 maxv=fmax)
        for ss, iwave in enumerate(twave):
            mn = np.min(np.abs(iwave - llist['wave']))
            if mn / aparm['disp'] < aparm['match_toler']:
                imn = np.argmin(np.abs(iwave - llist['wave']))
                #if msgs._debug['arc']:
                #    print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss]))
                # Update and append
                all_ids[ss] = llist['wave'][imn]
                all_idsion[ss] = llist['Ion'][imn]
                ifit.append(ss)
        # Keep unique ones
        ifit = np.unique(np.array(ifit, dtype=int))
        #if msgs._debug['arc']:
        #    debugger.set_trace()
        # Increment order
        if n_order < aparm['n_final']:
            n_order += 1
        else:
            # This does 2 iterations at the final order
            flg_quit = True

    # Final fit (originals can now be rejected)
    fmin, fmax = 0., 1.
    xfit, yfit = tcent[ifit] / (slf._msarc[det - 1].shape[0] -
                                1), all_ids[ifit]
    mask, fit = arutils.robust_polyfit(xfit,
                                       yfit,
                                       n_order,
                                       function=aparm['func'],
                                       sigma=aparm['nsig_rej_final'],
                                       minv=fmin,
                                       maxv=fmax)  #, debug=True)
    irej = np.where(mask == 1)[0]
    if len(irej) > 0:
        xrej = xfit[irej]
        yrej = yfit[irej]
        for imask in irej:
            msgs.info('Rejecting arc line {:g}'.format(yfit[imask]))
    else:
        xrej = []
        yrej = []
    xfit = xfit[mask == 0]
    yfit = yfit[mask == 0]
    ions = all_idsion[ifit][mask == 0]
    #
    if msgs._debug['arc']:
        msarc = slf._msarc[det - 1]
        wave = arutils.func_val(fit,
                                np.arange(msarc.shape[0]) /
                                float(msarc.shape[0]),
                                'legendre',
                                minv=fmin,
                                maxv=fmax)
        debugger.set_trace()

        #debugger.xplot(xfit, np.ones(len(xfit)), scatter=True,
        #    xtwo=np.arange(msarc.shape[0]),ytwo=yprep)
        #debugger.xplot(xfit,yfit, scatter=True, xtwo=np.arange(msarc.shape[0]),
        #    ytwo=wave)
        #debugger.set_trace()
        #wave = arutils.func_val(fit, np.arange(msarc.shape[0])/float(msarc.shape[0]),
        #    'legendre', min=fmin, max=fmax)

    # 2nd order Poly fit for archival
    #get_poly=True
    if get_poly:
        poly_fit = arutils.func_fit(yfit,
                                    xfit,
                                    'polynomial',
                                    2,
                                    minv=fmin,
                                    maxv=fmax)
        print(' Most likely you with to record these values:')
        print(poly_fit)
        debugger.set_trace()
    # Pack up fit
    final_fit = dict(fitc=fit,
                     function=aparm['func'],
                     xfit=xfit,
                     yfit=yfit,
                     ions=ions,
                     fmin=fmin,
                     fmax=fmax,
                     xnorm=float(slf._msarc[det - 1].shape[0]),
                     xrej=xrej,
                     yrej=yrej,
                     mask=mask,
                     spec=yprep,
                     nrej=aparm['nsig_rej_final'],
                     shift=0.,
                     tcent=tcent)
    # QA
    arqa.arc_fit_qa(slf, final_fit)
    # RMS
    rms_ang = arutils.calc_fit_rms(xfit,
                                   yfit,
                                   fit,
                                   aparm['func'],
                                   minv=fmin,
                                   maxv=fmax)
    wave = arutils.func_val(fit,
                            np.arange(slf._msarc[det - 1].shape[0]) /
                            float(slf._msarc[det - 1].shape[0]),
                            aparm['func'],
                            minv=fmin,
                            maxv=fmax)
    rms_pix = rms_ang / np.median(np.abs(wave - np.roll(wave, 1)))
    msgs.info("Fit RMS = {} pix".format(rms_pix))
    # Return
    return final_fit
Exemple #9
0
def clean_cr(spectra,
             smask,
             n_grow_mask=1,
             cr_nsig=7.,
             nrej_low=5.,
             debug=False,
             cr_everyn=6,
             cr_bsigma=5.,
             cr_two_alg='bspline',
             **kwargs):
    """ Sigma-clips the flux arrays to remove obvious CR

    Parameters
    ----------
    spectra :
    smask : ndarray
      Data mask
    n_grow_mask : int, optional
        Number of pixels to grow the initial mask by
        on each side. Defaults to 1 pixel
    cr_nsig : float, optional
      Number of sigma for rejection for CRs

    Returns
    -------
    """
    # Init
    fluxes, sigs, wave = unpack_spec(spectra)
    npix = wave.size

    if spectra.nspec == 2:
        msgs.info("Only 2 exposures.  Using custom procedure")
        if cr_two_alg == 'diff':
            diff = fluxes[0, :] - fluxes[1, :]
            # Robust mean/median
            med, mad = arutils.robust_meanstd(diff)
            # Spec0?
            cr0 = (diff - med) > cr_nsig * mad
            if n_grow_mask > 0:
                cr0 = grow_mask(cr0, n_grow=n_grow_mask)
            msgs.info("Rejecting {:d} CRs in exposure 0".format(np.sum(cr0)))
            smask[0, cr0] = True
            if debug:
                debugger.plot1d(wave,
                                fluxes[0, :],
                                xtwo=wave[cr0],
                                ytwo=fluxes[0, cr0],
                                mtwo='s')
            # Spec1?
            cr1 = (-1 * (diff - med)) > cr_nsig * mad
            if n_grow_mask > 0:
                cr1 = grow_mask(cr1, n_grow=n_grow_mask)
            smask[1, cr1] = True
            if debug:
                debugger.plot1d(wave,
                                fluxes[1, :],
                                xtwo=wave[cr1],
                                ytwo=fluxes[1, cr1],
                                mtwo='s')
            msgs.info("Rejecting {:d} CRs in exposure 1".format(np.sum(cr1)))
        elif cr_two_alg == 'ratio':
            diff = fluxes[0, :] - fluxes[1, :]
            rtio = fluxes[0, :] / fluxes[1, :]
            # Robust mean/median
            rmed, rmad = arutils.robust_meanstd(rtio)
            dmed, dmad = arutils.robust_meanstd(diff)
            # Spec0?            med, mad = arutils.robust_meanstd(diff)
            cr0 = ((rtio - rmed) > cr_nsig * rmad) & (
                (diff - dmed) > cr_nsig * dmad)
            if n_grow_mask > 0:
                cr0 = grow_mask(cr0, n_grow=n_grow_mask)
            msgs.info("Rejecting {:d} CRs in exposure 0".format(np.sum(cr0)))
            smask[0, cr0] = True
            if debug:
                debugger.plot1d(wave,
                                fluxes[0, :],
                                xtwo=wave[cr0],
                                ytwo=fluxes[0, cr0],
                                mtwo='s')
            # Spec1?
            cr1 = (-1 * (rtio - rmed) > cr_nsig * rmad) & (-1 * (diff - dmed) >
                                                           cr_nsig * dmad)
            if n_grow_mask > 0:
                cr1 = grow_mask(cr1, n_grow=n_grow_mask)
            smask[1, cr1] = True
            if debug:
                debugger.plot1d(wave,
                                fluxes[1, :],
                                xtwo=wave[cr1],
                                ytwo=fluxes[1, cr1],
                                mtwo='s')
            msgs.info("Rejecting {:d} CRs in exposure 1".format(np.sum(cr1)))
        elif cr_two_alg == 'bspline':
            # Package Data for convenience
            waves = spectra.data['wave'].flatten()  # Packed 0,1
            flux = fluxes.flatten()
            sig = sigs.flatten()
            #
            gd = np.where(sig > 0.)[0]
            srt = np.argsort(waves[gd])
            idx = gd[srt]
            # The following may eliminate bright, narrow emission lines
            mask, spl = arutils.robust_polyfit(waves[idx],
                                               flux[idx],
                                               3,
                                               function='bspline',
                                               weights=1. / sig[gd][srt],
                                               sigma=cr_bsigma,
                                               maxone=False,
                                               everyn=cr_everyn)
            # Reject CR (with grow)
            spec_fit = arutils.func_val(spl, wave, 'bspline')
            for ii in range(2):
                diff = fluxes[ii, :] - spec_fit
                cr = (diff > cr_nsig * sigs[ii, :]) & (sigs[ii, :] > 0.)
                if debug:
                    debugger.plot1d(spectra.data['wave'][0, :],
                                    spectra.data['flux'][ii, :],
                                    spec_fit,
                                    xtwo=spectra.data['wave'][0, cr],
                                    ytwo=spectra.data['flux'][ii, cr],
                                    mtwo='s')
                if n_grow_mask > 0:
                    cr = grow_mask(cr, n_grow=n_grow_mask)
                # Mask
                smask[ii, cr] = True
                msgs.info("Cleaning {:d} CRs in exposure {:d}".format(
                    np.sum(cr), ii))
            # Reject Low
            if nrej_low > 0.:
                for ii in range(2):
                    diff = spec_fit - fluxes[ii, :]
                    rej_low = (diff > nrej_low * sigs[ii, :]) & (sigs[ii, :] >
                                                                 0.)
                    if False:
                        debugger.plot1d(spectra.data['wave'][0, :],
                                        spectra.data['flux'][ii, :],
                                        spec_fit,
                                        xtwo=spectra.data['wave'][0, rej_low],
                                        ytwo=spectra.data['flux'][ii, rej_low],
                                        mtwo='s')
                    msgs.info(
                        "Removing {:d} low values in exposure {:d}".format(
                            np.sum(rej_low), ii))
                    smask[ii, rej_low] = True
            else:
                msgs.error("Bad algorithm for combining two spectra!")
        # Check
        if debug:
            gd0 = ~smask[0, :]
            gd1 = ~smask[1, :]
            debugger.plot1d(wave[gd0],
                            fluxes[0, gd0],
                            xtwo=wave[gd1],
                            ytwo=fluxes[1, gd1])
            debugger.set_trace()

    else:
        # Median of the masked array -- Best for 3 or more spectra
        mflux = np.ma.array(fluxes, mask=smask)
        refflux = np.ma.median(mflux, axis=0)
        diff = fluxes - refflux.filled(0.)

        # Loop on spectra
        for ispec in range(spectra.nspec):
            # Generate ivar
            gds = (~smask[ispec, :]) & (sigs[ispec, :] > 0.)
            ivar = np.zeros(npix)
            ivar[gds] = 1. / sigs[ispec, gds]**2
            #
            chi2 = diff[ispec]**2 * ivar
            badchi = (ivar > 0.0) & (chi2 > cr_nsig**2)
            nbad = np.sum(badchi)
            if nbad > 0:
                # Grow?
                if n_grow_mask > 0:
                    badchi = grow_mask(badchi, n_grow=n_grow_mask)
                # Mask
                smask[ispec, badchi] = True
                msgs.info("Rejecting {:d} CRs in exposure {:d}".format(
                    nbad, ispec))
    # Return
    return
Exemple #10
0
def iterative_fitting(spec,
                      tcent,
                      ifit,
                      IDs,
                      llist,
                      disp,
                      plot_fil=None,
                      verbose=False,
                      load_pypit=False,
                      aparm=None):

    if aparm is None:
        aparm = dict(
            llist='',
            disp=disp,  # Ang/unbinned pixel
            disp_toler=0.1,  # 10% tolerance
            match_toler=3.,  # Matcing tolerance (pixels)
            func='legendre',  # Function for fitting
            n_first=3,  # Order of polynomial for first fit
            n_final=4,  # Order of polynomial for final fit
            nsig_rej=2.,  # Number of sigma for rejection
            nsig_rej_final=3.0)  # Number of sigma for rejection (final fit)
    # PYPIT
    if load_pypit:
        from pypit import pyputils
        msgs = pyputils.get_dummy_logger()
    from pypit import arutils
    from pypit import arqa
    if load_pypit:
        arutils.dummy_settings()

    npix = spec.size

    # Setup for fitting
    sv_ifit = list(ifit)  # Keep the originals
    all_ids = -999. * np.ones(len(tcent))
    all_idsion = np.array(['UNKNWN'] * len(tcent))
    all_ids[ifit] = IDs

    # Fit
    n_order = aparm['n_first']
    flg_quit = False
    fmin, fmax = -1., 1.
    while (n_order <= aparm['n_final']) and (flg_quit is False):
        # Fit with rejection
        xfit, yfit = tcent[ifit], all_ids[ifit]
        mask, fit = arutils.robust_polyfit(xfit,
                                           yfit,
                                           n_order,
                                           function=aparm['func'],
                                           sigma=aparm['nsig_rej'],
                                           minv=fmin,
                                           maxv=fmax)

        rms_ang = arutils.calc_fit_rms(xfit[mask == 0],
                                       yfit[mask == 0],
                                       fit,
                                       aparm['func'],
                                       minv=fmin,
                                       maxv=fmax)
        rms_pix = rms_ang / disp
        if verbose:
            print("RMS = {:g}".format(rms_pix))
        # DEBUG
        # Reject but keep originals (until final fit)
        ifit = list(ifit[mask == 0]) + sv_ifit
        # Find new points (should we allow removal of the originals?)
        twave = arutils.func_val(fit,
                                 tcent,
                                 aparm['func'],
                                 minv=fmin,
                                 maxv=fmax)
        for ss, iwave in enumerate(twave):
            mn = np.min(np.abs(iwave - llist['wave']))
            if mn / aparm['disp'] < aparm['match_toler']:
                imn = np.argmin(np.abs(iwave - llist['wave']))
                #if verbose:
                #    print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss]))
                # Update and append
                all_ids[ss] = llist['wave'][imn]
                all_idsion[ss] = llist['ion'][imn]
                ifit.append(ss)
        # Keep unique ones
        ifit = np.unique(np.array(ifit, dtype=int))
        # Increment order
        if n_order < (aparm['n_final'] + 2):
            n_order += 1
        else:
            # This does 2 iterations at the final order
            flg_quit = True

    # Final fit (originals can now be rejected)
    fmin, fmax = 0., 1.
    xfit, yfit = tcent[ifit] / (npix - 1), all_ids[ifit]
    mask, fit = arutils.robust_polyfit(xfit,
                                       yfit,
                                       n_order,
                                       function=aparm['func'],
                                       sigma=aparm['nsig_rej_final'],
                                       minv=fmin,
                                       maxv=fmax)  #, debug=True)
    irej = np.where(mask == 1)[0]
    if len(irej) > 0:
        xrej = xfit[irej]
        yrej = yfit[irej]
        if verbose:
            for kk, imask in enumerate(irej):
                wave = arutils.func_val(fit,
                                        xrej[kk],
                                        aparm['func'],
                                        minv=fmin,
                                        maxv=fmax)
                print('Rejecting arc line {:g}; {:g}'.format(
                    yfit[imask], wave))
    else:
        xrej = []
        yrej = []
    xfit = xfit[mask == 0]
    yfit = yfit[mask == 0]
    ions = all_idsion[ifit][mask == 0]
    # Final RMS
    rms_ang = arutils.calc_fit_rms(xfit,
                                   yfit,
                                   fit,
                                   aparm['func'],
                                   minv=fmin,
                                   maxv=fmax)
    rms_pix = rms_ang / disp
    #
    '''
    if msgs._debug['arc']:
        msarc = slf._msarc[det-1]
        wave = arutils.func_val(fit, np.arange(msarc.shape[0])/float(msarc.shape[0]),
            'legendre', minv=fmin, maxv=fmax)
        debugger.xplot(xfit,yfit, scatter=True,
            xtwo=np.arange(msarc.shape[0])/float(msarc.shape[0]),
            ytwo=wave)
        debugger.xpcol(xfit*msarc.shape[0], yfit)
        debugger.set_trace()
    '''

    #debugger.xplot(xfit, np.ones(len(xfit)), scatter=True,
    #    xtwo=np.arange(msarc.shape[0]),ytwo=yprep)
    #debugger.xplot(xfit,yfit, scatter=True, xtwo=np.arange(msarc.shape[0]),
    #    ytwo=wave)
    #debugger.set_trace()
    #wave = arutils.func_val(fit, np.arange(msarc.shape[0])/float(msarc.shape[0]),
    #    'legendre', min=fmin, max=fmax)

    # Pack up fit
    final_fit = dict(fitc=fit,
                     function=aparm['func'],
                     xfit=xfit,
                     yfit=yfit,
                     ions=ions,
                     fmin=fmin,
                     fmax=fmax,
                     xnorm=float(npix),
                     xrej=xrej,
                     yrej=yrej,
                     mask=mask,
                     spec=spec,
                     nrej=aparm['nsig_rej_final'],
                     shift=0.,
                     tcent=tcent,
                     rms=rms_pix)
    # QA
    if plot_fil is not None:
        arqa.arc_fit_qa(None, final_fit, outfil=plot_fil)
    # Return
    return final_fit
ireg = HAND_FLAG == False
for iiter in range(niter):
    xpos1, xerr1 = trace_fweight(image*mask,xfit1[:,ireg], radius = fwhm_vec[iiter])
    # Get the indices of the current trace
    tracemask = np.zeros_like(xpos0,dtype=int)
    xfit1[:,ireg] = 0.0
    # Mask out anything that left the image.
    off_image = (xpos1 < -0.2*nspat) | (xpos1 > 1.2*nspat)
    tracemask[off_image] = 1
    xind = (np.fmax(np.fmin(np.rint(xpos1),nspat-1),0)).astype(int)
    for iobj in range(nobj):
        if specobjs[iobj].HAND_FLAG==False:
            # Mask out anything that has left the slit/order.
            tracemask[:,iobj] = (thismask[yind, xind[:,iobj]] == False).astype(int)
            # ToDO add maxdev functionality?
            polymask, coeff_fit1 = robust_polyfit(spec_vec,xpos1[:,iobj], ncoeff, function = 'legendre',
                                                  initialmask= tracemask[:,iobj],forceimask=True)
            xfit1[:,iobj] = func_val(coeff_fit1, spec_vec, 'legendre')
            plt.plot(spec_vec,xpos1[:,iobj],c='k',marker='+',markersize=1.5,linestyle='None')
            if (tracemask[:,iobj] == 1).any():
                plt.plot(spec_vec[(tracemask[:,iobj] == 1)],xpos1[(tracemask[:,iobj] == 1),iobj],c='r',marker='+',markersize=1.5,linestyle='None')
            plt.plot(spec_vec,xfit1[:,iobj],c='orange')
            plt.show()
        else:
            pass


#    pos_set1 = xy2traceset(ypos,xpos1, ncoeff=ncoeff,maxdev = 5.0,invvar = xinvvar)



Exemple #12
0
def flex_shift(slf, det, obj_skyspec, arx_skyspec):
    """ Calculate shift between object sky spectrum and archive sky spectrum

    Parameters
    ----------
    slf
    det
    obj_skyspec
    arx_skyspec

    Returns
    -------
    flex_dict
    """
    #Determine the brightest emission lines
    msgs.warn("If we use Paranal, cut down on wavelength early on")
    arx_amp, arx_cent, arx_wid, arx_w, arx_satsnd, arx_yprep = ararc.detect_lines(
        slf, det, msarc=None, censpec=arx_skyspec.flux.value, MK_SATMASK=False)
    obj_amp, obj_cent, obj_wid, obj_w, obj_satsnd, obj_yprep = ararc.detect_lines(
        slf, det, msarc=None, censpec=obj_skyspec.flux.value, MK_SATMASK=False)

    #Keep only 5 brightest amplitude lines (xxx_keep is array of indices within arx_w of the 5 brightest)
    arx_keep = np.argsort(arx_amp[arx_w])[-5:]
    obj_keep = np.argsort(obj_amp[obj_w])[-5:]

    #Calculate wavelength (Angstrom per pixel)
    arx_disp = np.append(
        arx_skyspec.wavelength.value[1] - arx_skyspec.wavelength.value[0],
        arx_skyspec.wavelength.value[1:] - arx_skyspec.wavelength.value[:-1])
    #arx_disp = (np.amax(arx_sky.wavelength.value)-np.amin(arx_sky.wavelength.value))/arx_sky.wavelength.size
    obj_disp = np.append(
        obj_skyspec.wavelength.value[1] - obj_skyspec.wavelength.value[0],
        obj_skyspec.wavelength.value[1:] - obj_skyspec.wavelength.value[:-1])
    #obj_disp = (np.amax(obj_sky.wavelength.value)-np.amin(obj_sky.wavelength.value))/obj_sky.wavelength.size

    #Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need this? can just use sigmas
    arx_idx = (arx_cent + 0.5).astype(
        np.int)[arx_w][arx_keep]  # The +0.5 is for rounding
    arx_res = arx_skyspec.wavelength.value[arx_idx]/\
              (arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep])
    obj_idx = (obj_cent + 0.5).astype(
        np.int)[obj_w][obj_keep]  # The +0.5 is for rounding
    obj_res = obj_skyspec.wavelength.value[obj_idx]/ \
              (obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
    #obj_res = (obj_sky.wavelength.value[0]+(obj_disp*obj_cent[obj_w][obj_keep]))/(
    #    obj_disp*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
    msgs.info("Resolution of Archive={:g} and Observation={:g}".format(
        np.median(arx_res), np.median(obj_res)))

    #Determine sigma of gaussian for smoothing
    arx_sig2 = (arx_disp[arx_idx] * arx_wid[arx_w][arx_keep])**2.
    obj_sig2 = (obj_disp[obj_idx] * obj_wid[obj_w][obj_keep])**2.

    arx_med_sig2 = np.median(arx_sig2)
    obj_med_sig2 = np.median(obj_sig2)

    if obj_med_sig2 >= arx_med_sig2:
        smooth_sig = np.sqrt(obj_med_sig2 - arx_med_sig2)  # Ang
        smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx])
        arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix * 2 *
                                               np.sqrt(2 * np.log(2)))
    else:
        msgs.warn("Prefer archival sky spectrum to have higher resolution")
        smooth_sig_pix = 0.
        msgs.warn("New Sky has higher resolution than Archive.  Not smoothing")
        #smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2)

    #Determine region of wavelength overlap
    min_wave = max(np.amin(arx_skyspec.wavelength.value),
                   np.amin(obj_skyspec.wavelength.value))
    max_wave = min(np.amax(arx_skyspec.wavelength.value),
                   np.amax(obj_skyspec.wavelength.value))

    #Smooth higher resolution spectrum by smooth_sig (flux is conserved!)
    #    if np.median(obj_res) >= np.median(arx_res):
    #        msgs.warn("New Sky has higher resolution than Archive.  Not smoothing")
    #obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig)
    #    else:
    #tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
    #        arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2)))
    #arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)

    # Define wavelengths of overlapping spectra
    keep_idx = np.where((obj_skyspec.wavelength.value >= min_wave)
                        & (obj_skyspec.wavelength.value <= max_wave))[0]
    #keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave]

    #Rebin both spectra onto overlapped wavelength range
    if len(keep_idx) <= 50:
        msgs.error("Not enough overlap between sky spectra")
    else:  #rebin onto object ALWAYS
        keep_wave = obj_skyspec.wavelength[keep_idx]
        arx_skyspec = arx_skyspec.rebin(keep_wave)
        obj_skyspec = obj_skyspec.rebin(keep_wave)
        # Trim edges (rebinning is junk there)
        arx_skyspec.data['flux'][0, :2] = 0.
        arx_skyspec.data['flux'][0, -2:] = 0.
        obj_skyspec.data['flux'][0, :2] = 0.
        obj_skyspec.data['flux'][0, -2:] = 0.

    #   Normalize spectra to unit average sky count
    norm = np.sum(obj_skyspec.flux.value) / obj_skyspec.npix
    obj_skyspec.flux = obj_skyspec.flux / norm
    norm2 = np.sum(arx_skyspec.flux.value) / arx_skyspec.npix
    arx_skyspec.flux = arx_skyspec.flux / norm2
    if (norm < 0.):
        msgs.warn("Bad normalization of object in flexure algorithm")
        msgs.warn("Will try the median")
        norm = np.median(obj_skyspec.flux.value)
        if (norm < 0.):
            msgs.error("Improper sky spectrum for flexure.  Is it too faint??")
    if (norm2 < 0.):
        msgs.error(
            "Bad normalization of archive in flexure. You are probably using wavelengths well beyond the archive."
        )

    #deal with bad pixels
    msgs.work("Need to mask bad pixels")

    #deal with underlying continuum
    msgs.work("Consider taking median first [5 pixel]")
    everyn = obj_skyspec.npix // 20
    mask, ct = arutils.robust_polyfit(obj_skyspec.wavelength.value,
                                      obj_skyspec.flux.value,
                                      3,
                                      function='bspline',
                                      sigma=3.,
                                      everyn=everyn)
    obj_sky_cont = arutils.func_val(ct, obj_skyspec.wavelength.value,
                                    'bspline')
    obj_sky_flux = obj_skyspec.flux.value - obj_sky_cont
    mask, ct_arx = arutils.robust_polyfit(arx_skyspec.wavelength.value,
                                          arx_skyspec.flux.value,
                                          3,
                                          function='bspline',
                                          sigma=3.,
                                          everyn=everyn)
    arx_sky_cont = arutils.func_val(ct_arx, arx_skyspec.wavelength.value,
                                    'bspline')
    arx_sky_flux = arx_skyspec.flux.value - arx_sky_cont

    # Consider shaprness filtering (e.g. LowRedux)
    msgs.work("Consider taking median first [5 pixel]")

    #Cross correlation of spectra
    #corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same")
    corr = np.correlate(arx_sky_flux, obj_sky_flux, "same")

    #Create array around the max of the correlation function for fitting for subpixel max
    # Restrict to pixels within maxshift of zero lag
    lag0 = corr.size // 2
    mxshft = settings.argflag['reduce']['flexure']['maxshift']
    max_corr = np.argmax(corr[lag0 - mxshft:lag0 + mxshft]) + lag0 - mxshft
    subpix_grid = np.linspace(max_corr - 3., max_corr + 3., 7.)

    #Fit a 2-degree polynomial to peak of correlation function
    fit = arutils.func_fit(subpix_grid, corr[subpix_grid.astype(np.int)],
                           'polynomial', 2)
    max_fit = -0.5 * fit[1] / fit[2]

    #Calculate and apply shift in wavelength
    shift = float(max_fit) - lag0
    msgs.info("Flexure correction of {:g} pixels".format(shift))
    #model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0]

    if msgs._debug['flexure']:
        debugger.plot1d(arx_skyspec.wavelength,
                        arx_sky_flux,
                        xtwo=np.roll(obj_skyspec.wavelength, int(-1 * shift)),
                        ytwo=obj_sky_flux)
        #debugger.xplot(arx_sky.wavelength, arx_sky.flux, xtwo=np.roll(obj_sky.wavelength.value,9), ytwo=obj_sky.flux*100)
        debugger.set_trace()

    flex_dict = dict(polyfit=fit,
                     shift=shift,
                     subpix=subpix_grid,
                     corr=corr[subpix_grid.astype(np.int)],
                     sky_spec=obj_skyspec,
                     arx_spec=arx_skyspec,
                     corr_cen=corr.size / 2,
                     smooth=smooth_sig_pix)
    # Return
    return flex_dict
Exemple #13
0
def basis(xfit,
          yfit,
          coeff,
          npc,
          pnpc,
          weights=None,
          skipx0=True,
          x0in=None,
          mask=None,
          function='polynomial',
          retmask=False):
    nrow = xfit.shape[0]
    ntrace = xfit.shape[1]
    if x0in is None:
        x0in = np.arange(float(ntrace))

    # Mask out some orders if they are bad
    if mask is None or mask.size == 0:
        usetrace = np.arange(ntrace)
        outmask = np.ones((nrow, ntrace))
    else:
        usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0]
        outmask = np.ones((nrow, ntrace))
        outmask[:, mask] = 0.0

    # Do the PCA analysis
    eigc, hidden = get_pc(coeff[1:npc + 1, usetrace], npc)

    modl = arutils.func_vander(xfit[:, 0], function, npc)
    eigv = np.dot(modl[:, 1:], eigc)

    med_hidden = np.median(hidden, axis=1)
    med_highorder = med_hidden.copy()
    med_highorder[0] = 0

    high_order_matrix = med_highorder.T[np.newaxis, :].repeat(ntrace, axis=0)

    # y = hidden[0,:]
    # coeff0 = arutils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function)

    # y = hidden[1,:]
    # coeff1 = arutils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function)

    coeffstr = []
    for i in xrange(1, npc + 1):
        # if pnpc[i] == 0:
        #     coeffstr.append([-9.99E9])
        #     continue
        # coeff0 = arutils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1])
        if weights is not None:
            tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace],
                                                   hidden[i - 1, :],
                                                   pnpc[i],
                                                   weights=weights[usetrace],
                                                   sigma=2.0,
                                                   function=function,
                                                   minv=x0in[0],
                                                   maxv=x0in[-1])
        else:
            tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace],
                                                   hidden[i - 1, :],
                                                   pnpc[i],
                                                   sigma=2.0,
                                                   function=function,
                                                   minv=x0in[0],
                                                   maxv=x0in[-1])
        coeffstr.append(coeff0)
        high_order_matrix[:, i - 1] = arutils.func_val(coeff0,
                                                       x0in,
                                                       function,
                                                       minv=x0in[0],
                                                       maxv=x0in[-1])
    # high_order_matrix[:,1] = arutils.func_val(coeff1, x0in, function)
    high_fit = high_order_matrix.copy()

    high_order_fit = np.dot(eigv, high_order_matrix.T)
    sub = (yfit - high_order_fit) * outmask

    numer = np.sum(sub, axis=0)
    denom = np.sum(outmask, axis=0)
    x0 = np.zeros(float(ntrace))
    fitmask = np.zeros(float(ntrace))
    #fitmask[mask] = 1
    x0fit = np.zeros(float(ntrace))
    chisqnu = 0.0
    chisqold = 0.0
    robust = True
    #svx0 = numer/(denom+(denom == 0).astype(np.int))
    if not skipx0:
        fitmask = (np.abs(denom) > 10).astype(np.int)
        if robust:
            good = np.where(fitmask != 0)[0]
            bad = np.where(fitmask == 0)[0]
            x0[good] = numer[good] / denom[good]
            imask = np.zeros(float(ntrace))
            imask[bad] = 1.0
            ttmask, x0res = arutils.robust_polyfit(x0in,
                                                   x0,
                                                   pnpc[0],
                                                   weights=weights,
                                                   sigma=2.0,
                                                   function=function,
                                                   minv=x0in[0],
                                                   maxv=x0in[-1],
                                                   initialmask=imask)
            x0fit = arutils.func_val(x0res,
                                     x0in,
                                     function,
                                     minv=x0in[0],
                                     maxv=x0in[-1])
            good = np.where(ttmask == 0)[0]
            xstd = 1.0  # This should represent the dispersion in the fit
            chisq = ((x0[good] - x0fit[good]) / xstd)**2.0
            chisqnu = np.sum(chisq) / np.sum(ttmask)
            fitmask = 1.0 - ttmask
            msgs.prindent("  Reduced chi-squared = {0:E}".format(chisqnu))
        else:
            for i in xrange(1, 5):
                good = np.where(fitmask != 0)[0]
                x0[good] = numer[good] / denom[good]
                #				x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
                x0res = arutils.func_fit(x0in[good],
                                         x0[good],
                                         function,
                                         pnpc[0],
                                         weights=weights,
                                         minv=x0in[0],
                                         maxv=x0in[-1])
                x0fit = arutils.func_val(x0res,
                                         x0in,
                                         function,
                                         minv=x0in[0],
                                         maxv=x0in[-1])
                chisq = (x0[good] - x0fit[good])**2.0
                fitmask[good] *= (chisq < np.sum(chisq) / 2.0).astype(np.int)
                chisqnu = np.sum(chisq) / np.sum(fitmask)
                msgs.prindent("  Reduced chi-squared = {0:E}".format(chisqnu))
                if chisqnu == chisqold:
                    break
                else:
                    chisqold = chisqnu
        if chisqnu > 2.0:
            msgs.warn("PCA has very large residuals")
        elif chisqnu > 0.5:
            msgs.warn("PCA has fairly large residuals")
        #bad = np.where(fitmask==0)[0]
        #x0[bad] = x0fit[bad]
    else:
        x0res = 0.0
    x3fit = np.dot(eigv, high_order_matrix.T) + np.outer(x0fit,
                                                         np.ones(nrow)).T
    outpar = dict({
        'high_fit': high_fit,
        'x0': x0,
        'x0in': x0in,
        'x0fit': x0fit,
        'x0res': x0res,
        'x0mask': fitmask,
        'hidden': hidden,
        'usetrc': usetrace,
        'eigv': eigv,
        'npc': npc,
        'coeffstr': coeffstr
    })
    if retmask:
        return x3fit, outpar, tmask
    else:
        return x3fit, outpar
Exemple #14
0
def reduce_echelle(slf,
                   sciframe,
                   scidx,
                   fitsdict,
                   det,
                   standard=False,
                   triml=1,
                   trimr=1,
                   mspixelflatnrm=None,
                   doqa=True):
    """ Run standard extraction steps on an echelle frame

    Parameters
    ----------
    sciframe : image
      Bias subtracted image (using arload.load_frame)
    scidx : int
      Index of the frame
    fitsdict : dict
      Contains relevant information from fits header files
    det : int
      Detector index
    standard : bool, optional
      Standard star frame?
    triml : int (optional)
      Number of pixels to trim from the left slit edge
    trimr : int (optional)
      Number of pixels to trim from the right slit edge
    """
    msgs.work("Multiprocess this algorithm")
    nspec = sciframe.shape[0]
    nord = slf._lordloc[det - 1].shape[1]
    # Prepare the frames for tracing and extraction
    sciframe, rawvarframe, crmask = reduce_prepare(
        slf,
        sciframe,
        scidx,
        fitsdict,
        det,
        mspixelflatnrm=mspixelflatnrm,
        standard=standard,
        slitprof=slitprof)
    bgframe = np.zeros_like(sciframe)
    bgnl, bgnr = np.zeros(nord, dtype=np.int), np.zeros(nord, dtype=np.int)
    skysub = True
    if settings.argflag['reduce']['skysub']['perform']:
        # Identify background pixels, and generate an image of the sky spectrum in each slit
        for o in range(nord):
            word = np.where((slf._slitpix[det - 1] == o + 1)
                            & (slf._scimask[det - 1] == 0))
            if word[0].size == 0:
                msgs.warn("There are no pixels in slit {0:d}".format(o + 1))
                continue
            tbgframe, nl, nr = background_subtraction(slf, sciframe,
                                                      rawvarframe, o, det)
            bgnl[o], bgnr[o] = nl, nr
            bgframe += tbgframe
            if nl == 0 and nr == 0:
                pass
                # If just one slit cannot do sky subtraction, don't do sky subtraction
                # msgs.warn("A sky subtraction will not be performed")
                # skysub = False
                # bgframe = np.zeros_like(sciframe)
                # modelvarframe = rawvarframe.copy()
                # break
        if skysub:
            # Provided the for loop above didn't break early, model the variance frame
            dnum = settings.get_dnum(det)
            modelvarframe = arprocimg.variance_frame(datasec_img,
                                                     det,
                                                     sciframe,
                                                     scidx,
                                                     settings.spect[dnum],
                                                     fitsdict=fitsdict,
                                                     skyframe=bgframe)
    else:
        modelvarframe = rawvarframe.copy()
        bgframe = np.zeros_like(sciframe)
    if not standard:  # Need to save
        slf._modelvarframe[det - 1] = modelvarframe
        slf._bgframe[det - 1] = bgframe
    # Obtain a first estimate of the object trace then
    # fit the traces and perform a PCA for the refinements
    trccoeff = np.zeros(
        (settings.argflag['trace']['object']['order'] + 1, nord))
    trcxfit = np.arange(nspec)
    extrap_slit = np.zeros(nord)
    for o in range(nord):
        trace, error = artrace.trace_weighted(sciframe - bgframe,
                                              slf._lordloc[det - 1][:, o],
                                              slf._rordloc[det - 1][:, o],
                                              mask=slf._scimask[det - 1],
                                              wght="flux")
        if trace is None:
            extrap_slit[o] = 1
            continue
        # Find only the good pixels
        w = np.where((error != 0.0) & (~np.isnan(error)))
        if w[0].size <= 2 * settings.argflag['trace']['object']['order']:
            extrap_slit[o] = 1
            continue
        # Convert the trace locations to be a fraction of the slit length,
        # measured from the left slit edge.
        trace -= slf._lordloc[det - 1][:, o]
        trace /= (slf._rordloc[det - 1][:, o] - slf._lordloc[det - 1][:, o])
        try:
            msk, trccoeff[:, o] = arutils.robust_polyfit(
                trcxfit[w],
                trace[w],
                settings.argflag['trace']['object']['order'],
                function=settings.argflag['trace']['object']['function'],
                weights=1.0 / error[w]**2,
                minv=0.0,
                maxv=nspec - 1.0)
        except:
            msgs.info("arproc.reduce_echelle")
            debugger.set_trace()
    refine = 0.0
    if settings.argflag['trace']['object']['method'] == "pca":
        # Identify the orders to be extrapolated during reconstruction
        orders = 1.0 + np.arange(nord)
        msgs.info("Performing a PCA on the object trace")
        ofit = settings.argflag['trace']['object']['params']
        lnpc = len(ofit) - 1
        maskord = np.where(extrap_slit == 1)[0]

        xcen = trcxfit[:, np.newaxis].repeat(nord, axis=1)
        trccen = arutils.func_val(
            trccoeff,
            trcxfit,
            settings.argflag['trace']['object']['function'],
            minv=0.0,
            maxv=nspec - 1.0).T
        if np.sum(1.0 - extrap_slit) > ofit[0] + 1:
            fitted, outpar = arpca.basis(
                xcen,
                trccen,
                trccoeff,
                lnpc,
                ofit,
                skipx0=False,
                mask=maskord,
                function=settings.argflag['trace']['object']['function'])
            if doqa:
                #                arqa.pca_plot(slf, outpar, ofit, "Object_Trace", pcadesc="PCA of object trace")
                arpca.pca_plot(slf.setup,
                               outpar,
                               ofit,
                               "Object_Trace",
                               pcadesc="PCA of object trace")
            # Extrapolate the remaining orders requested
            trccen, outpar = arpca.extrapolate(
                outpar,
                orders,
                function=settings.argflag['trace']['object']['function'])
            #refine = trccen-trccen[nspec//2, :].reshape((1, nord))
        else:
            msgs.warn("Could not perform a PCA on the object trace" +
                      msgs.newline() + "Not enough well-traced orders")
            msgs.info("Using direct determination of the object trace instead")
            pass
    else:
        msgs.error("Not ready for object trace method:" + msgs.newline() +
                   settings.argflag['trace']['object']['method'])
    # Construct the left and right traces of the object profile
    # The following code ensures that the fraction of the slit
    # containing the object remains constant along the spectral
    # direction
    trcmean = np.mean(trccen, axis=0)
    trobjl = (trcmean -
              (1 + bgnl) / slf._pixwid[det - 1].astype(np.float)).reshape(
                  (1, nord)).repeat(nspec, axis=0)
    trobjl = trccen - trobjl
    trobjr = (-trcmean + (slf._pixwid[det - 1] - bgnr - 1) /
              slf._pixwid[det - 1].astype(np.float)).reshape(
                  (1, nord)).repeat(nspec, axis=0)
    trobjr = trccen + trobjr
    # Convert trccen to the actual trace locations
    trccen *= (slf._rordloc[det - 1] - slf._lordloc[det - 1])
    trccen += slf._lordloc[det - 1]
    trobjl *= (slf._rordloc[det - 1] - slf._lordloc[det - 1])
    trobjl += slf._lordloc[det - 1]
    trobjr *= (slf._rordloc[det - 1] - slf._lordloc[det - 1])
    trobjr += slf._lordloc[det - 1]

    # Generate an image of pixel weights for each object. Each weight can
    # take any floating point value from 0 to 1 (inclusive). For the rec_obj_img,
    # a weight of 1 means that the pixel is fully contained within the object
    # region, and 0 means that the pixel is fully contained within the background
    # region. The opposite is true for the rec_bg_img array. A pixel that is on
    # the border of object/background is assigned a value between 0 and 1.
    msgs.work(
        "Eventually allow ARMED to find multiple objects in the one slit")
    nobj = 1
    rec_obj_img = np.zeros(sciframe.shape + (nobj, ))
    rec_bg_img = np.zeros(sciframe.shape + (nobj, ))
    for o in range(nord):
        # Prepare object/background regions
        objl = np.array([bgnl[o]])
        objr = np.array([slf._pixwid[det - 1][o] - bgnr[o] - triml - trimr])
        bckl = np.zeros((slf._pixwid[det - 1][o] - triml - trimr, 1))
        bckr = np.zeros((slf._pixwid[det - 1][o] - triml - trimr, 1))
        bckl[:bgnl[o]] = 1
        if bgnr[o] != 0:
            bckr[-bgnr[o]:] = 1
        tobj_img, tbg_img = artrace.trace_objbg_image(slf,
                                                      det,
                                                      sciframe - bgframe,
                                                      o, [objl, objr],
                                                      [bckl, bckr],
                                                      triml=triml,
                                                      trimr=trimr)
        rec_obj_img += tobj_img
        rec_bg_img += tbg_img

    # Create trace dict
    scitrace = artrace.trace_object_dict(nobj,
                                         trccen[:,
                                                0].reshape(trccen.shape[0], 1),
                                         object=rec_obj_img,
                                         background=rec_bg_img)
    for o in range(1, nord):
        scitrace = artrace.trace_object_dict(nobj,
                                             trccen[:, o].reshape(
                                                 trccen.shape[0], 1),
                                             tracelist=scitrace)

    # Save the quality control
    if doqa:
        artrace.obj_trace_qa(slf,
                             sciframe,
                             trobjl,
                             trobjr,
                             None,
                             det,
                             root="object_trace",
                             normalize=False)

    # Finalize the Sky Background image
    if settings.argflag['reduce']['skysub']['perform'] and (nobj >
                                                            0) and skysub:
        msgs.info("Finalizing the sky background image")
        # Identify background pixels, and generate an image of the sky spectrum in each slit
        bgframe = np.zeros_like(sciframe)
        for o in range(nord):
            tbgframe, nl, nr = background_subtraction(slf,
                                                      sciframe,
                                                      rawvarframe,
                                                      o,
                                                      det,
                                                      refine=refine)
            bgnl[o], bgnr[o] = nl, nr
            bgframe += tbgframe
        modelvarframe = arprocimg.variance_frame(datasec_img,
                                                 det,
                                                 sciframe,
                                                 scidx,
                                                 settings.spect[dnum],
                                                 fitsdict=fitsdict,
                                                 skyframe=bgframe)

    # Perform an optimal extraction
    return reduce_frame(slf,
                        sciframe,
                        rawvarframe,
                        modelvarframe,
                        bgframe,
                        scidx,
                        fitsdict,
                        det,
                        crmask,
                        scitrace=scitrace,
                        standard=standard)
Exemple #15
0
def obj_profiles(slf,
                 det,
                 specobjs,
                 sciframe,
                 varframe,
                 skyframe,
                 crmask,
                 scitrace,
                 COUNT_LIM=25.,
                 doqa=True,
                 pickle_file=None):
    """ Derive spatial profiles for each object
    Parameters
    ----------
    slf
    det
    specobjs
    sciframe
    varframe
    skyframe
    crmask
    scitrace
    Returns
    -------
    """
    '''  FOR DEVELOPING
    import pickle
    if False:
        tilts = slf._tilts[det-1]
        args = [det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts]
        msgs.warn("Pickling in the profile code")
        with open("trc_pickle.p",'wb') as f:
            pickle.dump(args,f)
        debugger.set_trace()
    if pickle_file is not None:
        f = open(pickle_file,'r')
        args = pickle.load(f)
        f.close()
        det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts = args
        slf = None
    else:
        tilts = slf._tilts[det-1]
    '''
    # Init QA
    #
    sigframe = np.sqrt(varframe)
    # Loop on slits
    for sl in range(len(specobjs)):
        # Loop on objects
        nobj = scitrace[sl]['traces'].shape[1]
        scitrace[sl]['opt_profile'] = []
        msgs.work("Should probably loop on S/N")
        for o in range(nobj):
            msgs.info(
                "Deriving spatial profile of object {0:d}/{1:d} in slit {2:d}/{3:d}"
                .format(o + 1, nobj, sl + 1, len(specobjs)))
            # Get object pixels
            if scitrace[sl]['background'] is None:
                # The object for all slits is provided in the first extension
                objreg = np.copy(scitrace[0]['object'][:, :, o])
                wzro = np.where(slf._slitpix[det - 1] != sl + 1)
                objreg[wzro] = 0.0
            else:
                objreg = scitrace[sl]['object'][:, :, o]
            # Calculate slit image
            slit_img = artrace.slit_image(slf, det, scitrace[sl],
                                          o)  #, tilts=tilts)
            # Object pixels
            weight = objreg.copy()
            # Identify good rows
            gdrow = np.where(specobjs[sl][o].boxcar['counts'] > COUNT_LIM)[0]
            # Normalized image
            norm_img = sciframe / np.outer(specobjs[sl][o].boxcar['counts'],
                                           np.ones(sciframe.shape[1]))
            # Eliminate rows with CRs (wipes out boxcar)
            crspec = np.sum(crmask * weight, axis=1)
            cr_rows = np.where(crspec > 0)[0]
            weight[cr_rows, :] = 0.
            #
            if len(gdrow) > 100:  # Good S/N regime
                msgs.info("Good S/N for profile")
                # Eliminate low count regions
                badrow = np.where(
                    specobjs[sl][o].boxcar['counts'] < COUNT_LIM)[0]
                weight[badrow, :] = 0.
                # Extract profile
                gdprof = (weight > 0) & (sigframe > 0.) & (
                    ~np.isnan(slit_img)
                )  # slit_img=nan if the slit is partially on the chip
                slit_val = slit_img[gdprof]
                flux_val = norm_img[gdprof]
                #weight_val = sciframe[gdprof]/sigframe[gdprof]  # S/N
                weight_val = 1. / sigframe[gdprof]  # 1/N
                msgs.work(
                    "Weight by S/N in boxcar extraction? [avoid CRs; smooth?]")
                # Fit
                fdict = dict(
                    func=settings.argflag['science']['extraction']['profile'],
                    deg=3,
                    extrap=False)
                if fdict['func'] == 'gaussian':
                    fdict['deg'] = 2
                elif fdict['func'] == 'moffat':
                    fdict['deg'] = 3
                else:
                    msgs.error("Not ready for this type of object profile")
                msgs.work(
                    "Might give our own guess here instead of using default")
                guess = None
                # Check if there are enough pixels in the slit to perform fit
                if slit_val.size <= fdict['deg'] + 1:
                    msgs.warn(
                        "Not enough pixels to determine profile of object={0:s} in slit {1:d}"
                        .format(specobjs[sl][o].idx, sl + 1) + msgs.newline() +
                        "Skipping Optimal")
                    fdict['extrap'] = True
                    scitrace[sl]['opt_profile'].append(copy.deepcopy(fdict))
                    continue
                # Fit the profile
                try:
                    mask, gfit = arutils.robust_polyfit(slit_val,
                                                        flux_val,
                                                        fdict['deg'],
                                                        function=fdict['func'],
                                                        weights=weight_val,
                                                        maxone=False,
                                                        guesses=guess)
                except RuntimeError:
                    msgs.warn("Bad Profile fit for object={:s}." +
                              msgs.newline() +
                              "Skipping Optimal".format(specobjs[sl][o].idx))
                    fdict['extrap'] = True
                    scitrace[sl]['opt_profile'].append(copy.deepcopy(fdict))
                    continue
                except ValueError:
                    debugger.set_trace()  # NaNs in the values?  Check
                msgs.work("Consider flagging/removing CRs here")
                # Record
                fdict['param'] = gfit.copy()
                fdict['mask'] = mask
                fdict['slit_val'] = slit_val
                fdict['flux_val'] = flux_val
                scitrace[sl]['opt_profile'].append(copy.deepcopy(fdict))
                specobjs[sl][o].optimal['fwhm'] = fdict['param'][1]  # Pixels
                if msgs._debug['obj_profile']:
                    gdp = mask == 0
                    mn = np.min(slit_val[gdp])
                    mx = np.max(slit_val[gdp])
                    xval = np.linspace(mn, mx, 1000)
                    model = arutils.func_val(gfit, xval, fdict['func'])
                    import matplotlib.pyplot as plt
                    plt.clf()
                    ax = plt.gca()
                    ax.scatter(slit_val[gdp],
                               flux_val[gdp],
                               marker='.',
                               s=0.7,
                               edgecolor='none',
                               facecolor='black')
                    ax.plot(xval, model, 'b')
                    # Gaussian too?
                    if False:
                        fdictg = dict(func='gaussian', deg=2)
                        maskg, gfitg = arutils.robust_polyfit(
                            slit_val,
                            flux_val,
                            fdict['deg'],
                            function=fdictg['func'],
                            weights=weight_val,
                            maxone=False)
                        modelg = arutils.func_val(gfitg, xval, fdictg['func'])
                        ax.plot(xval, modelg, 'r')
                    plt.show()
                    debugger.set_trace()
            elif len(gdrow) > 10:  #
                msgs.warn(
                    "Low extracted flux for obj={:s} in slit {:d}.  Not ready for Optimal"
                    .format(specobjs[sl][o].idx, sl + 1))
                scitrace[sl]['opt_profile'].append({})
                continue
            elif len(gdrow) >= 0:  # limit is ">= 0" to avoid crash for gdrow=0
                msgs.warn(
                    "Low extracted flux for obj={:s} in slit {:d}.  Not ready for Optimal"
                    .format(specobjs[sl][o].idx, sl + 1))
                scitrace[sl]['opt_profile'].append({})
                continue
    # QA
    if doqa:  #not msgs._debug['no_qa'] and doqa:
        msgs.info("Preparing QA for spatial object profiles")
        arqa.obj_profile_qa(slf, specobjs, scitrace, det)
    return
Exemple #16
0
 tracemask = np.zeros_like(xpos0, dtype=int)
 xfit1[:, ireg] = 0.0
 # Mask out anything that left the image.
 off_image = (xpos1 < -0.2 * nspat) | (xpos1 > 1.2 * nspat)
 tracemask[off_image] = 1
 xind = (np.fmax(np.fmin(np.rint(xpos1), nspat - 1), 0)).astype(int)
 for iobj in range(nobj):
     if specobjs[iobj].HAND_FLAG == False:
         # Mask out anything that has left the slit/order.
         tracemask[:, iobj] = (thismask[yind,
                                        xind[:, iobj]] == False).astype(int)
         # ToDO add maxdev functionality?
         polymask, coeff_fit1 = robust_polyfit(spec_vec,
                                               xpos1[:, iobj],
                                               ncoeff,
                                               function='legendre',
                                               initialmask=tracemask[:,
                                                                     iobj],
                                               forceimask=True)
         xfit1[:, iobj] = func_val(coeff_fit1, spec_vec, 'legendre')
         plt.plot(spec_vec,
                  xpos1[:, iobj],
                  c='k',
                  marker='+',
                  markersize=1.5,
                  linestyle='None')
         if (tracemask[:, iobj] == 1).any():
             plt.plot(spec_vec[(tracemask[:, iobj] == 1)],
                      xpos1[(tracemask[:, iobj] == 1), iobj],
                      c='r',
                      marker='+',
Exemple #17
0
def obj_profiles(slf, det, specobjs, sciframe, varframe, skyframe, crmask,
                 scitrace, COUNT_LIM=25., pickle_file=None):
    """ Derive spatial profiles for each object
    Parameters
    ----------
    slf
    det
    specobjs
    sciframe
    varframe
    skyframe
    crmask
    scitrace
    Returns
    -------
    """
    '''  FOR DEVELOPING
    import pickle
    if False:
        tilts = slf._tilts[det-1]
        args = [det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts]
        msgs.warn("Pickling in the profile code")
        with open("trc_pickle.p",'wb') as f:
            pickle.dump(args,f)
        debugger.set_trace()
    if pickle_file is not None:
        f = open(pickle_file,'r')
        args = pickle.load(f)
        f.close()
        det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts = args
        slf = None
    else:
        tilts = slf._tilts[det-1]
    '''
    # Init QA
    #
    sigframe = np.sqrt(varframe)
    # Loop
    nobj = scitrace['traces'].shape[1]
    scitrace['opt_profile'] = []
    msgs.work("Should probably loop on S/N")
    for o in range(nobj):
        # Calculate slit image
        slit_img = artrace.slit_image(slf, det, scitrace, o)#, tilts=tilts)
        # Object pixels
        weight = scitrace['object'][:,:,o].copy()
        # Identify good rows
        gdrow = np.where(specobjs[o].boxcar['counts'] > COUNT_LIM)[0]
        # Normalized image
        norm_img = sciframe / np.outer(specobjs[o].boxcar['counts'], np.ones(sciframe.shape[1]))
        # Eliminate rows with CRs (wipes out boxcar)
        crspec = np.sum(crmask*weight,axis=1)
        cr_rows = np.where(crspec > 0)[0]
        weight[cr_rows,:] = 0.
        #
        if len(gdrow) > 100:  # Good S/N regime
            msgs.info("Good S/N for profile")
            # Eliminate low count regions
            badrow = np.where(specobjs[o].boxcar['counts'] < COUNT_LIM)[0]
            weight[badrow,:] = 0.
            # Extract profile
            gdprof = (weight > 0) & (sigframe > 0.)
            slit_val = slit_img[gdprof]
            flux_val = norm_img[gdprof]
            #weight_val = sciframe[gdprof]/sigframe[gdprof]  # S/N
            weight_val = 1./sigframe[gdprof]  # 1/N
            msgs.work("Weight by S/N in boxcar extraction? [avoid CRs; smooth?]")
            # Fit
            fdict = dict(func=slf._argflag['science']['extraction']['profile'], deg=3)
            if fdict['func'] == 'gaussian':
                fdict['deg'] = 2
            elif fdict['func'] == 'moffat':
                fdict['deg'] = 3
            else:
                msgs.error("Not ready for this type of object profile")
            msgs.work("Might give our own guess here instead of using default")
            guess = None
            try:
                mask, gfit = arutils.robust_polyfit(slit_val, flux_val, fdict['deg'], function=fdict['func'], weights=weight_val, maxone=False, guesses=guess)
            except RuntimeError:
                msgs.warn("Bad Profile fit for object={:s}.  Skipping Optimal".format(specobjs[o].idx))
                scitrace['opt_profile'].append(fdict)
                continue
            except ValueError:
                debugger.set_trace()  # NaNs in the values?  Check
            msgs.work("Consider flagging/removing CRs here")
            # Record
            fdict['param'] = gfit
            fdict['mask'] = mask
            fdict['slit_val'] = slit_val
            fdict['flux_val'] = flux_val
            scitrace['opt_profile'].append(fdict)
            if msgs._debug['obj_profile']: #
                gdp = mask == 0
                mn = np.min(slit_val[gdp])
                mx = np.max(slit_val[gdp])
                xval = np.linspace(mn,mx,1000)
                model = arutils.func_val(gfit, xval, fdict['func'])
                import matplotlib.pyplot as plt
                plt.clf()
                ax = plt.gca()
                ax.scatter(slit_val[gdp], flux_val[gdp], marker='.', s=0.7, edgecolor='none', facecolor='black')
                ax.plot(xval, model, 'b')
                # Gaussian too?
                if False:
                    fdictg = dict(func='gaussian', deg=2)
                    maskg, gfitg = arutils.robust_polyfit(slit_val, flux_val, fdict['deg'], function=fdictg['func'], weights=weight_val, maxone=False)
                    modelg = arutils.func_val(gfitg, xval, fdictg['func'])
                    ax.plot(xval, modelg, 'r')
                plt.show()
                debugger.set_trace()
        elif len(gdrow) > 10:  #
            msgs.warn("Low extracted flux for obj={:s}.  Not ready for Optimal".format(specobjs[o].idx))
            scitrace['opt_profile'].append({})
            continue
    # QA
    arqa.obj_profile_qa(slf, specobjs, scitrace)
    return scitrace['opt_profile']