def boxcar_extraction_from_filenames(image_filename, psf_filename, fibers=None, width=7): """ Fast boxcar extraction of spectra from a preprocessed image and a trace set Args: image_filename : input preprocessed fits filename psf_filename : input PSF fits filename Optional: fibers : 1D np.array of int (default is all fibers, the first fiber is always = 0) width : extraction boxcar width, default is 7 Returns: flux : 2D np.array of shape (nfibers,n0=image.shape[0]), sum of pixel values per row of length=width per fiber ivar : 2D np.array of shape (nfibers,n0), ivar[f,j] = 1/( sum_[j,b:e] (1/image.ivar) ), ivar=0 if at least 1 pixel in the row has image.ivar=0 or image.mask!=0 wave : 2D np.array of shape (nfibers,n0), determined from the traces """ tset = read_xytraceset(psf_filename) image = read_image(image_filename) qframe = qproc_boxcar_extraction(xytraceset, image, fibers=fibers, width=width) return qframe.flux, qframe.ivar, qframe.wave
def compute_dy_using_boxcar_extraction(xytraceset, image, fibers, width=7, degyy=2): """ Measures y offsets (internal wavelength calibration) from a preprocessed image and a trace set using a cross-correlation of boxcar extracted spectra. Uses boxcar_extraction , resample_boxcar_frame , compute_dy_from_spectral_cross_correlations_of_frame Args: xytraceset : XYTraceset object image : DESI preprocessed image object Optional: fibers : 1D np.array of int (default is all fibers, the first fiber is always = 0) width : int, extraction boxcar width, default is 7 degyy : int, degree of polynomial fit of shifts as a function of y, used to reject outliers. Returns: x : 1D array of x coordinates on CCD (axis=1 in numpy image array, AXIS=0 in FITS, cross-dispersion axis = fiber number direction) y : 1D array of y coordinates on CCD (axis=0 in numpy image array, AXIS=1 in FITS, wavelength dispersion axis) dy : 1D array of shifts along y coordinates on CCD ey : 1D array of uncertainties on dy fiber : 1D array of fiber ID (first fiber = 0) wave : 1D array of wavelength """ log = get_logger() # boxcar extraction qframe = qproc_boxcar_extraction(xytraceset, image, fibers=fibers, width=7) # resampling on common finer wavelength grid flux, ivar, wave = resample_boxcar_frame(qframe.flux, qframe.ivar, qframe.wave, oversampling=4) # median flux used as internal spectral reference mflux = np.median(flux, axis=0) # measure y shifts wavemin = xytraceset.wavemin wavemax = xytraceset.wavemax xcoef = xytraceset.x_vs_wave_traceset._coeff ycoef = xytraceset.y_vs_wave_traceset._coeff return compute_dy_from_spectral_cross_correlations_of_frame( flux=flux, ivar=ivar, wave=wave, xcoef=xcoef, ycoef=ycoef, wavemin=wavemin, wavemax=wavemax, reference_flux=mflux, n_wavelength_bins=degyy + 4)
def compute_dy_using_boxcar_extraction(xytraceset, image, fibers, width=7, degyy=2) : """ Measures y offsets (internal wavelength calibration) from a preprocessed image and a trace set using a cross-correlation of boxcar extracted spectra. Uses boxcar_extraction , resample_boxcar_frame , compute_dy_from_spectral_cross_correlations_of_frame Args: xytraceset : XYTraceset object image : DESI preprocessed image object Optional: fibers : 1D np.array of int (default is all fibers, the first fiber is always = 0) width : int, extraction boxcar width, default is 7 degyy : int, degree of polynomial fit of shifts as a function of y, used to reject outliers. Returns: x : 1D array of x coordinates on CCD (axis=1 in numpy image array, AXIS=0 in FITS, cross-dispersion axis = fiber number direction) y : 1D array of y coordinates on CCD (axis=0 in numpy image array, AXIS=1 in FITS, wavelength dispersion axis) dy : 1D array of shifts along y coordinates on CCD ey : 1D array of uncertainties on dy fiber : 1D array of fiber ID (first fiber = 0) wave : 1D array of wavelength """ log=get_logger() # boxcar extraction qframe = qproc_boxcar_extraction(xytraceset, image, fibers=fibers, width=7) # resampling on common finer wavelength grid flux, ivar, wave = resample_boxcar_frame(qframe.flux, qframe.ivar, qframe.wave, oversampling=4) # median flux used as internal spectral reference mflux=np.median(flux,axis=0) # measure y shifts wavemin = xytraceset.wavemin wavemax = xytraceset.wavemax xcoef = xytraceset.x_vs_wave_traceset._coeff ycoef = xytraceset.y_vs_wave_traceset._coeff return compute_dy_from_spectral_cross_correlations_of_frame(flux=flux, ivar=ivar, wave=wave, xcoef=xcoef, ycoef=ycoef, wavemin=wavemin, wavemax=wavemax, reference_flux = mflux , n_wavelength_bins = degyy+4)
def boxcar_extraction_from_filenames(image_filename,psf_filename,fibers=None, width=7) : """ Fast boxcar extraction of spectra from a preprocessed image and a trace set Args: image_filename : input preprocessed fits filename psf_filename : input PSF fits filename Optional: fibers : 1D np.array of int (default is all fibers, the first fiber is always = 0) width : extraction boxcar width, default is 7 Returns: flux : 2D np.array of shape (nfibers,n0=image.shape[0]), sum of pixel values per row of length=width per fiber ivar : 2D np.array of shape (nfibers,n0), ivar[f,j] = 1/( sum_[j,b:e] (1/image.ivar) ), ivar=0 if at least 1 pixel in the row has image.ivar=0 or image.mask!=0 wave : 2D np.array of shape (nfibers,n0), determined from the traces """ tset = read_xytraceset(psf_filename) image = read_image(image_filename) qframe = qproc_boxcar_extraction(xytraceset,image,fibers=fibers,width=width) return qframe.flux, qframe.ivar, qframe.wave
def shift_ycoef_using_external_spectrum(psf, xytraceset, image, fibers, spectrum_filename, degyy=2, width=7): """ Measure y offsets (external wavelength calibration) from a preprocessed image , a PSF + trace set using a cross-correlation of boxcar extracted spectra and an external well-calibrated spectrum. The PSF shape is used to convolve the input spectrum. It could also be used to correct for the PSF asymetry (disabled for now). A relative flux calibration of the spectra is performed internally. Args: psf : specter PSF xytraceset : XYTraceset object image : DESI preprocessed image object fibers : 1D np.array of fiber indices spectrum_filename : path to input spectral file ( read with np.loadtxt , first column is wavelength (in vacuum and Angstrom) , second column in flux (arb. units) Optional: width : int, extraction boxcar width, default is 7 degyy : int, degree of polynomial fit of shifts as a function of y, used to reject outliers. Returns: ycoef : 2D np.array of same shape as input, with modified Legendre coefficents for each fiber to convert wavelenght to YCCD """ log = get_logger() wavemin = xytraceset.wavemin wavemax = xytraceset.wavemax xcoef = xytraceset.x_vs_wave_traceset._coeff ycoef = xytraceset.y_vs_wave_traceset._coeff tmp = np.loadtxt(spectrum_filename).T ref_wave = tmp[0] ref_spectrum = tmp[1] log.info("read reference spectrum in %s with %d entries" % (spectrum_filename, ref_wave.size)) log.info("rextract spectra with boxcar") # boxcar extraction qframe = qproc_boxcar_extraction(xytraceset, image, fibers=fibers, width=7) # resampling on common finer wavelength grid flux, ivar, wave = resample_boxcar_frame(qframe.flux, qframe.ivar, qframe.wave, oversampling=2) # median flux used as internal spectral reference mflux = np.median(flux, axis=0) mivar = np.median(ivar, axis=0) * flux.shape[0] * (2. / np.pi ) # very appoximate ! # trim ref_spectrum i = (ref_wave >= wave[0]) & (ref_wave <= wave[-1]) ref_wave = ref_wave[i] ref_spectrum = ref_spectrum[i] # check wave is linear or make it linear if np.abs( (ref_wave[1] - ref_wave[0]) - (ref_wave[-1] - ref_wave[-2])) > 0.0001 * (ref_wave[1] - ref_wave[0]): log.info( "reference spectrum wavelength is not on a linear grid, resample it" ) dwave = np.min(np.gradient(ref_wave)) tmp_wave = np.linspace(ref_wave[0], ref_wave[-1], int((ref_wave[-1] - ref_wave[0]) / dwave)) ref_spectrum = resample_flux(tmp_wave, ref_wave, ref_spectrum) ref_wave = tmp_wave i = np.argmax(ref_spectrum) central_wave_for_psf_evaluation = ref_wave[i] fiber_for_psf_evaluation = (flux.shape[0] // 2) try: # compute psf at most significant line of ref_spectrum dwave = ref_wave[i + 1] - ref_wave[i] hw = int(3. / dwave) + 1 # 3A half width wave_range = ref_wave[i - hw:i + hw + 1] x, y = psf.xy(fiber_for_psf_evaluation, wave_range) x = np.tile( x[hw] + np.arange(-hw, hw + 1) * (y[-1] - y[0]) / (2 * hw + 1), (y.size, 1)) y = np.tile(y, (2 * hw + 1, 1)).T kernel2d = psf._value(x, y, fiber_for_psf_evaluation, central_wave_for_psf_evaluation) kernel1d = np.sum(kernel2d, axis=1) log.info( "convolve reference spectrum using PSF at fiber %d and wavelength %dA" % (fiber_for_psf_evaluation, central_wave_for_psf_evaluation)) ref_spectrum = fftconvolve(ref_spectrum, kernel1d, mode='same') except: log.warning("couldn't convolve reference spectrum: %s %s" % (sys.exc_info()[0], sys.exc_info()[1])) # resample input spectrum log.info("resample convolved reference spectrum") ref_spectrum = resample_flux(wave, ref_wave, ref_spectrum) log.info("absorb difference of calibration") x = (wave - wave[wave.size // 2]) / 50. kernel = np.exp(-x**2 / 2) f1 = fftconvolve(mflux, kernel, mode='same') f2 = fftconvolve(ref_spectrum, kernel, mode='same') if np.all(f2 > 0): scale = f1 / f2 ref_spectrum *= scale log.info("fit shifts on wavelength bins") # define bins n_wavelength_bins = degyy + 4 y_for_dy = np.array([]) dy = np.array([]) ey = np.array([]) wave_for_dy = np.array([]) for b in range(n_wavelength_bins): wmin = wave[0] + ((wave[-1] - wave[0]) / n_wavelength_bins) * b if b < n_wavelength_bins - 1: wmax = wave[0] + ( (wave[-1] - wave[0]) / n_wavelength_bins) * (b + 1) else: wmax = wave[-1] ok = (wave >= wmin) & (wave <= wmax) sw = np.sum(mflux[ok] * (mflux[ok] > 0)) if sw == 0: continue dwave, err = compute_dy_from_spectral_cross_correlation( mflux[ok], wave[ok], ref_spectrum[ok], ivar=mivar[ok], hw=10.) bin_wave = np.sum(mflux[ok] * (mflux[ok] > 0) * wave[ok]) / sw x, y = psf.xy(fiber_for_psf_evaluation, bin_wave) eps = 0.1 x, yp = psf.xy(fiber_for_psf_evaluation, bin_wave + eps) dydw = (yp - y) / eps if err * dydw < 1: dy = np.append(dy, -dwave * dydw) ey = np.append(ey, err * dydw) wave_for_dy = np.append(wave_for_dy, bin_wave) y_for_dy = np.append(y_for_dy, y) log.info("wave = %fA , y=%d, measured dwave = %f +- %f A" % (bin_wave, y, dwave, err)) if False: # we don't need this for now try: log.info("correcting bias due to asymmetry of PSF") hw = 5 oversampling = 4 xx = np.tile( np.arange(2 * hw * oversampling + 1) - hw * oversampling, (2 * hw * oversampling + 1, 1)) / float(oversampling) yy = xx.T x, y = psf.xy(fiber_for_psf_evaluation, central_wave_for_psf_evaluation) prof = psf._value(xx + x, yy + y, fiber_for_psf_evaluation, central_wave_for_psf_evaluation) dy_asym_central = np.sum(yy * prof) / np.sum(prof) for i in range(dy.size): x, y = psf.xy(fiber_for_psf_evaluation, wave_for_dy[i]) prof = psf._value(xx + x, yy + y, fiber_for_psf_evaluation, wave_for_dy[i]) dy_asym = np.sum(yy * prof) / np.sum(prof) log.info( "y=%f, measured dy=%f , bias due to PSF asymetry = %f" % (y, dy[i], dy_asym - dy_asym_central)) dy[i] -= (dy_asym - dy_asym_central) except: log.warning("couldn't correct for asymmetry of PSF: %s %s" % (sys.exc_info()[0], sys.exc_info()[1])) log.info("polynomial fit of shifts and modification of PSF ycoef") # pol fit coef = np.polyfit(wave_for_dy, dy, degyy, w=1. / ey**2) pol = np.poly1d(coef) for i in range(dy.size): log.info( "wave=%fA y=%f, measured dy=%f+-%f , pol(wave) = %f" % (wave_for_dy[i], y_for_dy[i], dy[i], ey[i], pol(wave_for_dy[i]))) log.info("apply this to the PSF ycoef") wave = np.linspace(wavemin, wavemax, 100) dy = pol(wave) dycoef = legfit(legx(wave, wavemin, wavemax), dy, deg=ycoef.shape[1] - 1) for fiber in range(ycoef.shape[0]): ycoef[fiber] += dycoef return ycoef
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) t0 = time.time() log = get_logger() # guess if it is a preprocessed or a raw image hdulist = fits.open(args.image) is_input_preprocessed = ("IMAGE" in hdulist) & ("IVAR" in hdulist) primary_header = hdulist[0].header hdulist.close() if is_input_preprocessed: image = read_image(args.image) else: if args.camera is None: print( "ERROR: Need to specify camera to open a raw fits image (with all cameras in different fits HDUs)" ) print( "Try adding the option '--camera xx', with xx in {brz}{0-9}, like r7, or type 'desi_qproc --help' for more options" ) sys.exit(12) image = read_raw(args.image, args.camera, fill_header=[ 1, ]) if args.auto: log.debug("AUTOMATIC MODE") try: night = image.meta['NIGHT'] if not 'EXPID' in image.meta: if 'EXPNUM' in image.meta: log.warning('using EXPNUM {} for EXPID'.format( image.meta['EXPNUM'])) image.meta['EXPID'] = image.meta['EXPNUM'] expid = image.meta['EXPID'] except KeyError as e: log.error( "Need at least NIGHT and EXPID (or EXPNUM) to run in auto mode. Retry without the --auto option." ) log.error(str(e)) sys.exit(12) indir = os.path.dirname(args.image) if args.fibermap is None: filename = '{}/fibermap-{:08d}.fits'.format(indir, expid) if os.path.isfile(filename): log.debug("auto-mode: found a fibermap, {}, using it!".format( filename)) args.fibermap = filename if args.output_preproc is None: if not is_input_preprocessed: args.output_preproc = '{}/preproc-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera.lower(), expid) log.debug("auto-mode: will write preproc in " + args.output_preproc) else: log.debug( "auto-mode: will not write preproc because input is a preprocessed image" ) if args.auto_output_dir != '.': if not os.path.isdir(args.auto_output_dir): log.debug("auto-mode: creating directory " + args.auto_output_dir) os.makedirs(args.auto_output_dir) if args.output_preproc is not None: write_image(args.output_preproc, image) cfinder = None if args.psf is None: if cfinder is None: cfinder = CalibFinder([image.meta, primary_header]) args.psf = cfinder.findfile("PSF") log.info(" Using PSF {}".format(args.psf)) tset = read_xytraceset(args.psf) # add fibermap if args.fibermap: if os.path.isfile(args.fibermap): fibermap = read_fibermap(args.fibermap) else: log.error("no fibermap file {}".format(args.fibermap)) fibermap = None else: fibermap = None if "OBSTYPE" in image.meta: obstype = image.meta["OBSTYPE"].upper() image.meta["OBSTYPE"] = obstype # make sure it's upper case qframe = None else: log.warning("No OBSTYPE keyword, trying to guess ...") qframe = qproc_boxcar_extraction(tset, image, width=args.width, fibermap=fibermap) obstype = check_qframe_flavor( qframe, input_flavor=image.meta["FLAVOR"]).upper() image.meta["OBSTYPE"] = obstype log.info("OBSTYPE = '{}'".format(obstype)) if args.auto: # now set the things to do if obstype == "SKY" or obstype == "TWILIGHT" or obstype == "SCIENCE": args.shift_psf = True args.output_psf = '{}/psf-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.apply_fiberflat = True args.skysub = True args.output_skyframe = '{}/qsky-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.fluxcalib = True args.outframe = '{}/qcframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) elif obstype == "ARC" or obstype == "TESTARC": args.shift_psf = True args.output_psf = '{}/psf-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.compute_lsf_sigma = True elif obstype == "FLAT" or obstype == "TESTFLAT": args.shift_psf = True args.output_psf = '{}/psf-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.compute_fiberflat = '{}/qfiberflat-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) if args.shift_psf: # using the trace shift script if args.auto: options = option_list({ "psf": args.psf, "image": "dummy", "outpsf": "dummy", "continuum": ((obstype == "FLAT") | (obstype == "TESTFLAT")), "sky": ((obstype == "SCIENCE") | (obstype == "SKY")) }) else: options = option_list({ "psf": args.psf, "image": "dummy", "outpsf": "dummy" }) tmp_args = trace_shifts_script.parse(options=options) tset = trace_shifts_script.fit_trace_shifts(image=image, args=tmp_args) qframe = qproc_boxcar_extraction(tset, image, width=args.width, fibermap=fibermap) if tset.meta is not None: # add traceshift info in the qframe, this will be saved in the qframe header if qframe.meta is None: qframe.meta = dict() for k in tset.meta.keys(): qframe.meta[k] = tset.meta[k] if args.output_rawframe is not None: write_qframe(args.output_rawframe, qframe) log.info("wrote raw extracted frame in {}".format( args.output_rawframe)) if args.compute_lsf_sigma: tset = process_arc(qframe, tset, linelist=None, npoly=2, nbins=2) if args.output_psf is not None: for k in qframe.meta: if k not in tset.meta: tset.meta[k] = qframe.meta[k] write_xytraceset(args.output_psf, tset) if args.compute_fiberflat is not None: fiberflat = qproc_compute_fiberflat(qframe) #write_qframe(args.compute_fiberflat,qflat) write_fiberflat(args.compute_fiberflat, fiberflat, header=qframe.meta) log.info("wrote fiberflat in {}".format(args.compute_fiberflat)) if args.apply_fiberflat or args.input_fiberflat: if args.input_fiberflat is None: if cfinder is None: cfinder = CalibFinder([image.meta, primary_header]) try: args.input_fiberflat = cfinder.findfile("FIBERFLAT") except KeyError as e: log.error("no FIBERFLAT for this spectro config") sys.exit(12) log.info("applying fiber flat {}".format(args.input_fiberflat)) flat = read_fiberflat(args.input_fiberflat) qproc_apply_fiberflat(qframe, flat) if args.skysub: log.info("sky subtraction") if args.output_skyframe is not None: skyflux = qproc_sky_subtraction(qframe, return_skymodel=True) sqframe = QFrame(qframe.wave, skyflux, np.ones(skyflux.shape)) write_qframe(args.output_skyframe, sqframe) log.info("wrote sky model in {}".format(args.output_skyframe)) else: qproc_sky_subtraction(qframe) if args.fluxcalib: if cfinder is None: cfinder = CalibFinder([image.meta, primary_header]) # check for flux calib if cfinder.haskey("FLUXCALIB"): fluxcalib_filename = cfinder.findfile("FLUXCALIB") fluxcalib = read_average_flux_calibration(fluxcalib_filename) log.info("read average calib in {}".format(fluxcalib_filename)) seeing = qframe.meta["SEEING"] airmass = qframe.meta["AIRMASS"] exptime = qframe.meta["EXPTIME"] exposure_calib = fluxcalib.value(seeing=seeing, airmass=airmass) for q in range(qframe.nspec): fiber_calib = np.interp(qframe.wave[q], fluxcalib.wave, exposure_calib) * exptime inv_calib = (fiber_calib > 0) / (fiber_calib + (fiber_calib == 0)) qframe.flux[q] *= inv_calib qframe.ivar[q] *= fiber_calib**2 * (fiber_calib > 0) # add keyword in header giving the calibration factor applied at a reference wavelength band = qframe.meta["CAMERA"].upper()[0] if band == "B": refwave = 4500 elif band == "R": refwave = 6500 else: refwave = 8500 calvalue = np.interp(refwave, fluxcalib.wave, exposure_calib) * exptime qframe.meta["CALWAVE"] = refwave qframe.meta["CALVALUE"] = calvalue else: log.error( "Cannot calibrate fluxes because no FLUXCALIB keywork in calibration files" ) fibers = parse_fibers(args.fibers) if fibers is None: fibers = qframe.flux.shape[0] else: ii = np.arange(qframe.fibers.size)[np.in1d(qframe.fibers, fibers)] if ii.size == 0: log.error("no such fibers in frame,") log.error("fibers are in range [{}:{}]".format( qframe.fibers[0], qframe.fibers[-1] + 1)) sys.exit(12) qframe = qframe[ii] if args.outframe is not None: write_qframe(args.outframe, qframe) log.info("wrote {}".format(args.outframe)) t1 = time.time() log.info("all done in {:3.1f} sec".format(t1 - t0)) if args.plot: log.info("plotting {} spectra".format(qframe.wave.shape[0])) import matplotlib.pyplot as plt fig = plt.figure() for i in range(qframe.wave.shape[0]): j = (qframe.ivar[i] > 0) plt.plot(qframe.wave[i, j], qframe.flux[i, j]) plt.grid() plt.xlabel("wavelength") plt.ylabel("flux") plt.show()
def shift_ycoef_using_external_spectrum(psf,xytraceset,image,fibers,spectrum_filename,degyy=2,width=7) : """ Measure y offsets (external wavelength calibration) from a preprocessed image , a PSF + trace set using a cross-correlation of boxcar extracted spectra and an external well-calibrated spectrum. The PSF shape is used to convolve the input spectrum. It could also be used to correct for the PSF asymetry (disabled for now). A relative flux calibration of the spectra is performed internally. Args: psf : specter PSF xytraceset : XYTraceset object image : DESI preprocessed image object fibers : 1D np.array of fiber indices spectrum_filename : path to input spectral file ( read with np.loadtxt , first column is wavelength (in vacuum and Angstrom) , second column in flux (arb. units) Optional: width : int, extraction boxcar width, default is 7 degyy : int, degree of polynomial fit of shifts as a function of y, used to reject outliers. Returns: ycoef : 2D np.array of same shape as input, with modified Legendre coefficents for each fiber to convert wavelenght to YCCD """ log = get_logger() wavemin = xytraceset.wavemin wavemax = xytraceset.wavemax xcoef = xytraceset.x_vs_wave_traceset._coeff ycoef = xytraceset.y_vs_wave_traceset._coeff tmp=np.loadtxt(spectrum_filename).T ref_wave=tmp[0] ref_spectrum=tmp[1] log.info("read reference spectrum in %s with %d entries"%(spectrum_filename,ref_wave.size)) log.info("rextract spectra with boxcar") # boxcar extraction qframe = qproc_boxcar_extraction(xytraceset, image, fibers=fibers, width=7) # resampling on common finer wavelength grid flux, ivar, wave = resample_boxcar_frame(qframe.flux, qframe.ivar, qframe.wave, oversampling=2) # median flux used as internal spectral reference mflux=np.median(flux,axis=0) mivar=np.median(ivar,axis=0)*flux.shape[0]*(2./np.pi) # very appoximate ! # trim ref_spectrum i=(ref_wave>=wave[0])&(ref_wave<=wave[-1]) ref_wave=ref_wave[i] ref_spectrum=ref_spectrum[i] # check wave is linear or make it linear if np.abs((ref_wave[1]-ref_wave[0])-(ref_wave[-1]-ref_wave[-2]))>0.0001*(ref_wave[1]-ref_wave[0]) : log.info("reference spectrum wavelength is not on a linear grid, resample it") dwave = np.min(np.gradient(ref_wave)) tmp_wave = np.linspace(ref_wave[0],ref_wave[-1],int((ref_wave[-1]-ref_wave[0])/dwave)) ref_spectrum = resample_flux(tmp_wave, ref_wave , ref_spectrum) ref_wave = tmp_wave i=np.argmax(ref_spectrum) central_wave_for_psf_evaluation = ref_wave[i] fiber_for_psf_evaluation = (flux.shape[0]//2) try : # compute psf at most significant line of ref_spectrum dwave=ref_wave[i+1]-ref_wave[i] hw=int(3./dwave)+1 # 3A half width wave_range = ref_wave[i-hw:i+hw+1] x,y=psf.xy(fiber_for_psf_evaluation,wave_range) x=np.tile(x[hw]+np.arange(-hw,hw+1)*(y[-1]-y[0])/(2*hw+1),(y.size,1)) y=np.tile(y,(2*hw+1,1)).T kernel2d=psf._value(x,y,fiber_for_psf_evaluation,central_wave_for_psf_evaluation) kernel1d=np.sum(kernel2d,axis=1) log.info("convolve reference spectrum using PSF at fiber %d and wavelength %dA"%(fiber_for_psf_evaluation,central_wave_for_psf_evaluation)) ref_spectrum=fftconvolve(ref_spectrum,kernel1d, mode='same') except : log.warning("couldn't convolve reference spectrum: %s %s"%(sys.exc_info()[0],sys.exc_info()[1])) # resample input spectrum log.info("resample convolved reference spectrum") ref_spectrum = resample_flux(wave, ref_wave , ref_spectrum) log.info("absorb difference of calibration") x=(wave-wave[wave.size//2])/50. kernel=np.exp(-x**2/2) f1=fftconvolve(mflux,kernel,mode='same') f2=fftconvolve(ref_spectrum,kernel,mode='same') if np.all(f2>0) : scale=f1/f2 ref_spectrum *= scale log.info("fit shifts on wavelength bins") # define bins n_wavelength_bins = degyy+4 y_for_dy=np.array([]) dy=np.array([]) ey=np.array([]) wave_for_dy=np.array([]) for b in range(n_wavelength_bins) : wmin=wave[0]+((wave[-1]-wave[0])/n_wavelength_bins)*b if b<n_wavelength_bins-1 : wmax=wave[0]+((wave[-1]-wave[0])/n_wavelength_bins)*(b+1) else : wmax=wave[-1] ok=(wave>=wmin)&(wave<=wmax) sw= np.sum(mflux[ok]*(mflux[ok]>0)) if sw==0 : continue dwave,err = compute_dy_from_spectral_cross_correlation(mflux[ok],wave[ok],ref_spectrum[ok],ivar=mivar[ok],hw=3.) bin_wave = np.sum(mflux[ok]*(mflux[ok]>0)*wave[ok])/sw x,y=psf.xy(fiber_for_psf_evaluation,bin_wave) eps=0.1 x,yp=psf.xy(fiber_for_psf_evaluation,bin_wave+eps) dydw=(yp-y)/eps if err*dydw<1 : dy=np.append(dy,-dwave*dydw) ey=np.append(ey,err*dydw) wave_for_dy=np.append(wave_for_dy,bin_wave) y_for_dy=np.append(y_for_dy,y) log.info("wave = %fA , y=%d, measured dwave = %f +- %f A"%(bin_wave,y,dwave,err)) if False : # we don't need this for now try : log.info("correcting bias due to asymmetry of PSF") hw=5 oversampling=4 xx=np.tile(np.arange(2*hw*oversampling+1)-hw*oversampling,(2*hw*oversampling+1,1))/float(oversampling) yy=xx.T x,y=psf.xy(fiber_for_psf_evaluation,central_wave_for_psf_evaluation) prof=psf._value(xx+x,yy+y,fiber_for_psf_evaluation,central_wave_for_psf_evaluation) dy_asym_central = np.sum(yy*prof)/np.sum(prof) for i in range(dy.size) : x,y=psf.xy(fiber_for_psf_evaluation,wave_for_dy[i]) prof=psf._value(xx+x,yy+y,fiber_for_psf_evaluation,wave_for_dy[i]) dy_asym = np.sum(yy*prof)/np.sum(prof) log.info("y=%f, measured dy=%f , bias due to PSF asymetry = %f"%(y,dy[i],dy_asym-dy_asym_central)) dy[i] -= (dy_asym-dy_asym_central) except : log.warning("couldn't correct for asymmetry of PSF: %s %s"%(sys.exc_info()[0],sys.exc_info()[1])) log.info("polynomial fit of shifts and modification of PSF ycoef") # pol fit coef = np.polyfit(wave_for_dy,dy,degyy,w=1./ey**2) pol = np.poly1d(coef) for i in range(dy.size) : log.info("wave=%fA y=%f, measured dy=%f+-%f , pol(wave) = %f"%(wave_for_dy[i],y_for_dy[i],dy[i],ey[i],pol(wave_for_dy[i]))) log.info("apply this to the PSF ycoef") wave = np.linspace(wavemin,wavemax,100) dy = pol(wave) dycoef = legfit(legx(wave,wavemin,wavemax),dy,deg=ycoef.shape[1]-1) for fiber in range(ycoef.shape[0]) : ycoef[fiber] += dycoef return ycoef
def compute_image_model(image,xytraceset,fiberflat=None,fibermap=None,with_spectral_smoothing=True,with_sky_model=True,\ spectral_smoothing_sigma_length=10.,spectral_smoothing_nsig=4.,psf=None): ''' Returns a model of the input image, using a fast extraction, a processing of spectra with a common sky model and a smoothing, followed by a reprojection on the CCD image. Inputs: image: a preprocessed image in the form of a desispec.image.Image object xytraceset: a desispec.xytraceset.XYTraceSet object with trace coordinates Optional: fiberflat: a desispec.fiberflat.FiberFlat object with_spectral_smoothing: try and smooth the spectra to reduce noise (and eventualy reduce variance correlation) with_sky_model: use a sky model as part of the spectral modeling to reduce the noise (requires a fiberflat) spectral_smoothing_sigma_length: sigma of Gaussian smoothing along wavelength in A spectral_smoothing_nsig: number of sigma rejection threshold to fall back to the original extracted spectrum instead of the smooth one psf: specter.psf.GaussHermitePSF object to be used for the 1D projection (slow, by default=None, in which case a Gaussian profile is used) returns: a 2D np.array of same shape as image.pix ''' log = get_logger() # first perform a fast boxcar extraction log.info("extract spectra") image.mask = None qframe = qproc_boxcar_extraction(xytraceset, image) fqframe = None sqframe = None ratio = None if fiberflat is not None: log.info("fiberflat") fqframe = copy.deepcopy(qframe) flat = qproc_apply_fiberflat(fqframe, fiberflat=fiberflat, return_flat=True) if with_sky_model: if fiberflat is None: log.warning( "cannot compute and use a sky model without a fiberflat") else: # crude model of the sky, accounting for fiber throughput variation log.info("sky") sqframe = copy.deepcopy(fqframe) sky = qproc_sky_subtraction(sqframe, return_skymodel=True) if with_spectral_smoothing: log.info("spectral smoothing") sigma = spectral_smoothing_sigma_length / np.mean( np.gradient(qframe.wave[qframe.nspec // 2])) log.debug("smoothing sigma in flux bin units = {}".format(sigma)) hw = int(3 * sigma) u = (np.arange(2 * hw + 1) - hw) kernel = np.exp(-u**2 / sigma**2 / 2.) kernel /= np.sum(kernel) nsig = 5 y = np.arange(qframe.flux.shape[1]) for s in range(qframe.nspec): if sqframe is not None: fflux = sqframe.flux[s] fivar = sqframe.ivar[s] elif fqframe is not None: fflux = fqframe.flux[s] fivar = fqframe.ivar[s] else: fflux = qframe.flux[s] fivar = qframe.ivar[s] sfflux = scipy.signal.fftconvolve(fflux, kernel, "same") good = ((fivar * (fflux - sfflux)**2) < (spectral_smoothing_nsig**2)) out = ~good nout = np.sum(out) if nout > 0: # recompute sflux while masking outliers tflux = fflux.copy() tflux[out] = np.interp(y[out], y[good], fflux[good]) sfflux = scipy.signal.fftconvolve(tflux, kernel, "same") # and replace the 'out' region by the original data # because we want to keep it to have a fair variance sfflux[out] = fflux[out] # replace by smooth version (+ possibly average sky) if sqframe is not None: qframe.flux[s] = (sky[s] + sfflux) * flat[s] elif fqframe is not None: qframe.flux[s] = sfflux * flat[s] else: qframe.flux[s] = sfflux log.info("project back spectra on image") y = np.arange(image.pix.shape[0]) # cross dispersion profile xsig = 1. * np.ones((qframe.nspec, image.pix.shape[0])) if xytraceset.xsig_vs_wave_traceset: log.info("use traceset in PSF for xsig") for s in range(xytraceset.nspec): wave = xytraceset.wave_vs_y(s, y) xsig[s] = xytraceset.xsig_vs_wave(s, wave) else: log.info("use default xsig={}".format(np.mean(xsig))) # keep only positive flux qframe.flux *= (qframe.flux > 0.) # convert counts per A to counts per pixel dwave = np.gradient(qframe.wave, axis=1) qframe.flux *= dwave # because true profile is not Gaussian and we do not integrate the # profile in pixels, we have to apply an adjustment empirical_scale = 1.1 log.info("empirical adjustment of xsig by {:4.2f}".format(empirical_scale)) xsig *= empirical_scale model = np.zeros(image.pix.shape) if psf is None: # use simple Gaussian log.info("use Gaussian sigma of average = {:4.2f}".format( np.mean(xsig))) for s in range(qframe.nspec): x = xytraceset.x_vs_y(s, y) numba_proj(model, x, xsig[s], qframe.flux[s]) else: # this takes a lot of time log.debug("Use PSF for projection, but in 1D") psf._polyparams['HSIZEY'] = 0 psf._polyparams['GHDEGY'] = 0 model = psf.project(qframe.wave, qframe.flux, xyrange=(0, image.pix.shape[1], 0, image.pix.shape[0])) log.debug("done projecting") log.info("done") return model