def boxcar_extraction_from_filenames(image_filename,
psf_filename,
fibers=None,
width=7):
"""
Fast boxcar extraction of spectra from a preprocessed image and a trace set
Args:
image_filename : input preprocessed fits filename
psf_filename : input PSF fits filename
Optional:
fibers : 1D np.array of int (default is all fibers, the first fiber is always = 0)
width : extraction boxcar width, default is 7
Returns:
flux : 2D np.array of shape (nfibers,n0=image.shape[0]), sum of pixel values per row of length=width per fiber
ivar : 2D np.array of shape (nfibers,n0), ivar[f,j] = 1/( sum_[j,b:e] (1/image.ivar) ), ivar=0 if at least 1 pixel in the row has image.ivar=0 or image.mask!=0
wave : 2D np.array of shape (nfibers,n0), determined from the traces
"""
tset = read_xytraceset(psf_filename)
image = read_image(image_filename)
qframe = qproc_boxcar_extraction(xytraceset,
image,
fibers=fibers,
width=width)
return qframe.flux, qframe.ivar, qframe.wave
def compute_dy_using_boxcar_extraction(xytraceset,
image,
fibers,
width=7,
degyy=2):
"""
Measures y offsets (internal wavelength calibration) from a preprocessed image and a trace set using a cross-correlation of boxcar extracted spectra.
Uses boxcar_extraction , resample_boxcar_frame , compute_dy_from_spectral_cross_correlations_of_frame
Args:
xytraceset : XYTraceset object
image : DESI preprocessed image object
Optional:
fibers : 1D np.array of int (default is all fibers, the first fiber is always = 0)
width : int, extraction boxcar width, default is 7
degyy : int, degree of polynomial fit of shifts as a function of y, used to reject outliers.
Returns:
x : 1D array of x coordinates on CCD (axis=1 in numpy image array, AXIS=0 in FITS, cross-dispersion axis = fiber number direction)
y : 1D array of y coordinates on CCD (axis=0 in numpy image array, AXIS=1 in FITS, wavelength dispersion axis)
dy : 1D array of shifts along y coordinates on CCD
ey : 1D array of uncertainties on dy
fiber : 1D array of fiber ID (first fiber = 0)
wave : 1D array of wavelength
"""
log = get_logger()
# boxcar extraction
qframe = qproc_boxcar_extraction(xytraceset, image, fibers=fibers, width=7)
# resampling on common finer wavelength grid
flux, ivar, wave = resample_boxcar_frame(qframe.flux,
qframe.ivar,
qframe.wave,
oversampling=4)
# median flux used as internal spectral reference
mflux = np.median(flux, axis=0)
# measure y shifts
wavemin = xytraceset.wavemin
wavemax = xytraceset.wavemax
xcoef = xytraceset.x_vs_wave_traceset._coeff
ycoef = xytraceset.y_vs_wave_traceset._coeff
return compute_dy_from_spectral_cross_correlations_of_frame(
flux=flux,
ivar=ivar,
wave=wave,
xcoef=xcoef,
ycoef=ycoef,
wavemin=wavemin,
wavemax=wavemax,
reference_flux=mflux,
n_wavelength_bins=degyy + 4)
def compute_dy_using_boxcar_extraction(xytraceset, image, fibers, width=7, degyy=2) :
"""
Measures y offsets (internal wavelength calibration) from a preprocessed image and a trace set using a cross-correlation of boxcar extracted spectra.
Uses boxcar_extraction , resample_boxcar_frame , compute_dy_from_spectral_cross_correlations_of_frame
Args:
xytraceset : XYTraceset object
image : DESI preprocessed image object
Optional:
fibers : 1D np.array of int (default is all fibers, the first fiber is always = 0)
width : int, extraction boxcar width, default is 7
degyy : int, degree of polynomial fit of shifts as a function of y, used to reject outliers.
Returns:
x : 1D array of x coordinates on CCD (axis=1 in numpy image array, AXIS=0 in FITS, cross-dispersion axis = fiber number direction)
y : 1D array of y coordinates on CCD (axis=0 in numpy image array, AXIS=1 in FITS, wavelength dispersion axis)
dy : 1D array of shifts along y coordinates on CCD
ey : 1D array of uncertainties on dy
fiber : 1D array of fiber ID (first fiber = 0)
wave : 1D array of wavelength
"""
log=get_logger()
# boxcar extraction
qframe = qproc_boxcar_extraction(xytraceset, image, fibers=fibers, width=7)
# resampling on common finer wavelength grid
flux, ivar, wave = resample_boxcar_frame(qframe.flux, qframe.ivar, qframe.wave, oversampling=4)
# median flux used as internal spectral reference
mflux=np.median(flux,axis=0)
# measure y shifts
wavemin = xytraceset.wavemin
wavemax = xytraceset.wavemax
xcoef = xytraceset.x_vs_wave_traceset._coeff
ycoef = xytraceset.y_vs_wave_traceset._coeff
return compute_dy_from_spectral_cross_correlations_of_frame(flux=flux, ivar=ivar, wave=wave, xcoef=xcoef, ycoef=ycoef, wavemin=wavemin, wavemax=wavemax, reference_flux = mflux , n_wavelength_bins = degyy+4)
def boxcar_extraction_from_filenames(image_filename,psf_filename,fibers=None, width=7) :
"""
Fast boxcar extraction of spectra from a preprocessed image and a trace set
Args:
image_filename : input preprocessed fits filename
psf_filename : input PSF fits filename
Optional:
fibers : 1D np.array of int (default is all fibers, the first fiber is always = 0)
width : extraction boxcar width, default is 7
Returns:
flux : 2D np.array of shape (nfibers,n0=image.shape[0]), sum of pixel values per row of length=width per fiber
ivar : 2D np.array of shape (nfibers,n0), ivar[f,j] = 1/( sum_[j,b:e] (1/image.ivar) ), ivar=0 if at least 1 pixel in the row has image.ivar=0 or image.mask!=0
wave : 2D np.array of shape (nfibers,n0), determined from the traces
"""
tset = read_xytraceset(psf_filename)
image = read_image(image_filename)
qframe = qproc_boxcar_extraction(xytraceset,image,fibers=fibers,width=width)
return qframe.flux, qframe.ivar, qframe.wave
def shift_ycoef_using_external_spectrum(psf,
xytraceset,
image,
fibers,
spectrum_filename,
degyy=2,
width=7):
"""
Measure y offsets (external wavelength calibration) from a preprocessed image , a PSF + trace set using a cross-correlation of boxcar extracted spectra
and an external well-calibrated spectrum.
The PSF shape is used to convolve the input spectrum. It could also be used to correct for the PSF asymetry (disabled for now).
A relative flux calibration of the spectra is performed internally.
Args:
psf : specter PSF
xytraceset : XYTraceset object
image : DESI preprocessed image object
fibers : 1D np.array of fiber indices
spectrum_filename : path to input spectral file ( read with np.loadtxt , first column is wavelength (in vacuum and Angstrom) , second column in flux (arb. units)
Optional:
width : int, extraction boxcar width, default is 7
degyy : int, degree of polynomial fit of shifts as a function of y, used to reject outliers.
Returns:
ycoef : 2D np.array of same shape as input, with modified Legendre coefficents for each fiber to convert wavelenght to YCCD
"""
log = get_logger()
wavemin = xytraceset.wavemin
wavemax = xytraceset.wavemax
xcoef = xytraceset.x_vs_wave_traceset._coeff
ycoef = xytraceset.y_vs_wave_traceset._coeff
tmp = np.loadtxt(spectrum_filename).T
ref_wave = tmp[0]
ref_spectrum = tmp[1]
log.info("read reference spectrum in %s with %d entries" %
(spectrum_filename, ref_wave.size))
log.info("rextract spectra with boxcar")
# boxcar extraction
qframe = qproc_boxcar_extraction(xytraceset, image, fibers=fibers, width=7)
# resampling on common finer wavelength grid
flux, ivar, wave = resample_boxcar_frame(qframe.flux,
qframe.ivar,
qframe.wave,
oversampling=2)
# median flux used as internal spectral reference
mflux = np.median(flux, axis=0)
mivar = np.median(ivar, axis=0) * flux.shape[0] * (2. / np.pi
) # very appoximate !
# trim ref_spectrum
i = (ref_wave >= wave[0]) & (ref_wave <= wave[-1])
ref_wave = ref_wave[i]
ref_spectrum = ref_spectrum[i]
# check wave is linear or make it linear
if np.abs(
(ref_wave[1] - ref_wave[0]) -
(ref_wave[-1] - ref_wave[-2])) > 0.0001 * (ref_wave[1] - ref_wave[0]):
log.info(
"reference spectrum wavelength is not on a linear grid, resample it"
)
dwave = np.min(np.gradient(ref_wave))
tmp_wave = np.linspace(ref_wave[0], ref_wave[-1],
int((ref_wave[-1] - ref_wave[0]) / dwave))
ref_spectrum = resample_flux(tmp_wave, ref_wave, ref_spectrum)
ref_wave = tmp_wave
i = np.argmax(ref_spectrum)
central_wave_for_psf_evaluation = ref_wave[i]
fiber_for_psf_evaluation = (flux.shape[0] // 2)
try:
# compute psf at most significant line of ref_spectrum
dwave = ref_wave[i + 1] - ref_wave[i]
hw = int(3. / dwave) + 1 # 3A half width
wave_range = ref_wave[i - hw:i + hw + 1]
x, y = psf.xy(fiber_for_psf_evaluation, wave_range)
x = np.tile(
x[hw] + np.arange(-hw, hw + 1) * (y[-1] - y[0]) / (2 * hw + 1),
(y.size, 1))
y = np.tile(y, (2 * hw + 1, 1)).T
kernel2d = psf._value(x, y, fiber_for_psf_evaluation,
central_wave_for_psf_evaluation)
kernel1d = np.sum(kernel2d, axis=1)
log.info(
"convolve reference spectrum using PSF at fiber %d and wavelength %dA"
% (fiber_for_psf_evaluation, central_wave_for_psf_evaluation))
ref_spectrum = fftconvolve(ref_spectrum, kernel1d, mode='same')
except:
log.warning("couldn't convolve reference spectrum: %s %s" %
(sys.exc_info()[0], sys.exc_info()[1]))
# resample input spectrum
log.info("resample convolved reference spectrum")
ref_spectrum = resample_flux(wave, ref_wave, ref_spectrum)
log.info("absorb difference of calibration")
x = (wave - wave[wave.size // 2]) / 50.
kernel = np.exp(-x**2 / 2)
f1 = fftconvolve(mflux, kernel, mode='same')
f2 = fftconvolve(ref_spectrum, kernel, mode='same')
if np.all(f2 > 0):
scale = f1 / f2
ref_spectrum *= scale
log.info("fit shifts on wavelength bins")
# define bins
n_wavelength_bins = degyy + 4
y_for_dy = np.array([])
dy = np.array([])
ey = np.array([])
wave_for_dy = np.array([])
for b in range(n_wavelength_bins):
wmin = wave[0] + ((wave[-1] - wave[0]) / n_wavelength_bins) * b
if b < n_wavelength_bins - 1:
wmax = wave[0] + (
(wave[-1] - wave[0]) / n_wavelength_bins) * (b + 1)
else:
wmax = wave[-1]
ok = (wave >= wmin) & (wave <= wmax)
sw = np.sum(mflux[ok] * (mflux[ok] > 0))
if sw == 0:
continue
dwave, err = compute_dy_from_spectral_cross_correlation(
mflux[ok], wave[ok], ref_spectrum[ok], ivar=mivar[ok], hw=10.)
bin_wave = np.sum(mflux[ok] * (mflux[ok] > 0) * wave[ok]) / sw
x, y = psf.xy(fiber_for_psf_evaluation, bin_wave)
eps = 0.1
x, yp = psf.xy(fiber_for_psf_evaluation, bin_wave + eps)
dydw = (yp - y) / eps
if err * dydw < 1:
dy = np.append(dy, -dwave * dydw)
ey = np.append(ey, err * dydw)
wave_for_dy = np.append(wave_for_dy, bin_wave)
y_for_dy = np.append(y_for_dy, y)
log.info("wave = %fA , y=%d, measured dwave = %f +- %f A" %
(bin_wave, y, dwave, err))
if False: # we don't need this for now
try:
log.info("correcting bias due to asymmetry of PSF")
hw = 5
oversampling = 4
xx = np.tile(
np.arange(2 * hw * oversampling + 1) - hw * oversampling,
(2 * hw * oversampling + 1, 1)) / float(oversampling)
yy = xx.T
x, y = psf.xy(fiber_for_psf_evaluation,
central_wave_for_psf_evaluation)
prof = psf._value(xx + x, yy + y, fiber_for_psf_evaluation,
central_wave_for_psf_evaluation)
dy_asym_central = np.sum(yy * prof) / np.sum(prof)
for i in range(dy.size):
x, y = psf.xy(fiber_for_psf_evaluation, wave_for_dy[i])
prof = psf._value(xx + x, yy + y, fiber_for_psf_evaluation,
wave_for_dy[i])
dy_asym = np.sum(yy * prof) / np.sum(prof)
log.info(
"y=%f, measured dy=%f , bias due to PSF asymetry = %f" %
(y, dy[i], dy_asym - dy_asym_central))
dy[i] -= (dy_asym - dy_asym_central)
except:
log.warning("couldn't correct for asymmetry of PSF: %s %s" %
(sys.exc_info()[0], sys.exc_info()[1]))
log.info("polynomial fit of shifts and modification of PSF ycoef")
# pol fit
coef = np.polyfit(wave_for_dy, dy, degyy, w=1. / ey**2)
pol = np.poly1d(coef)
for i in range(dy.size):
log.info(
"wave=%fA y=%f, measured dy=%f+-%f , pol(wave) = %f" %
(wave_for_dy[i], y_for_dy[i], dy[i], ey[i], pol(wave_for_dy[i])))
log.info("apply this to the PSF ycoef")
wave = np.linspace(wavemin, wavemax, 100)
dy = pol(wave)
dycoef = legfit(legx(wave, wavemin, wavemax), dy, deg=ycoef.shape[1] - 1)
for fiber in range(ycoef.shape[0]):
ycoef[fiber] += dycoef
return ycoef
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
t0 = time.time()
log = get_logger()
# guess if it is a preprocessed or a raw image
hdulist = fits.open(args.image)
is_input_preprocessed = ("IMAGE" in hdulist) & ("IVAR" in hdulist)
primary_header = hdulist[0].header
hdulist.close()
if is_input_preprocessed:
image = read_image(args.image)
else:
if args.camera is None:
print(
"ERROR: Need to specify camera to open a raw fits image (with all cameras in different fits HDUs)"
)
print(
"Try adding the option '--camera xx', with xx in {brz}{0-9}, like r7, or type 'desi_qproc --help' for more options"
)
sys.exit(12)
image = read_raw(args.image, args.camera, fill_header=[
1,
])
if args.auto:
log.debug("AUTOMATIC MODE")
try:
night = image.meta['NIGHT']
if not 'EXPID' in image.meta:
if 'EXPNUM' in image.meta:
log.warning('using EXPNUM {} for EXPID'.format(
image.meta['EXPNUM']))
image.meta['EXPID'] = image.meta['EXPNUM']
expid = image.meta['EXPID']
except KeyError as e:
log.error(
"Need at least NIGHT and EXPID (or EXPNUM) to run in auto mode. Retry without the --auto option."
)
log.error(str(e))
sys.exit(12)
indir = os.path.dirname(args.image)
if args.fibermap is None:
filename = '{}/fibermap-{:08d}.fits'.format(indir, expid)
if os.path.isfile(filename):
log.debug("auto-mode: found a fibermap, {}, using it!".format(
filename))
args.fibermap = filename
if args.output_preproc is None:
if not is_input_preprocessed:
args.output_preproc = '{}/preproc-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera.lower(), expid)
log.debug("auto-mode: will write preproc in " +
args.output_preproc)
else:
log.debug(
"auto-mode: will not write preproc because input is a preprocessed image"
)
if args.auto_output_dir != '.':
if not os.path.isdir(args.auto_output_dir):
log.debug("auto-mode: creating directory " +
args.auto_output_dir)
os.makedirs(args.auto_output_dir)
if args.output_preproc is not None:
write_image(args.output_preproc, image)
cfinder = None
if args.psf is None:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
args.psf = cfinder.findfile("PSF")
log.info(" Using PSF {}".format(args.psf))
tset = read_xytraceset(args.psf)
# add fibermap
if args.fibermap:
if os.path.isfile(args.fibermap):
fibermap = read_fibermap(args.fibermap)
else:
log.error("no fibermap file {}".format(args.fibermap))
fibermap = None
else:
fibermap = None
if "OBSTYPE" in image.meta:
obstype = image.meta["OBSTYPE"].upper()
image.meta["OBSTYPE"] = obstype # make sure it's upper case
qframe = None
else:
log.warning("No OBSTYPE keyword, trying to guess ...")
qframe = qproc_boxcar_extraction(tset,
image,
width=args.width,
fibermap=fibermap)
obstype = check_qframe_flavor(
qframe, input_flavor=image.meta["FLAVOR"]).upper()
image.meta["OBSTYPE"] = obstype
log.info("OBSTYPE = '{}'".format(obstype))
if args.auto:
# now set the things to do
if obstype == "SKY" or obstype == "TWILIGHT" or obstype == "SCIENCE":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.apply_fiberflat = True
args.skysub = True
args.output_skyframe = '{}/qsky-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.fluxcalib = True
args.outframe = '{}/qcframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
elif obstype == "ARC" or obstype == "TESTARC":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.compute_lsf_sigma = True
elif obstype == "FLAT" or obstype == "TESTFLAT":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.compute_fiberflat = '{}/qfiberflat-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
if args.shift_psf:
# using the trace shift script
if args.auto:
options = option_list({
"psf":
args.psf,
"image":
"dummy",
"outpsf":
"dummy",
"continuum": ((obstype == "FLAT") | (obstype == "TESTFLAT")),
"sky": ((obstype == "SCIENCE") | (obstype == "SKY"))
})
else:
options = option_list({
"psf": args.psf,
"image": "dummy",
"outpsf": "dummy"
})
tmp_args = trace_shifts_script.parse(options=options)
tset = trace_shifts_script.fit_trace_shifts(image=image, args=tmp_args)
qframe = qproc_boxcar_extraction(tset,
image,
width=args.width,
fibermap=fibermap)
if tset.meta is not None:
# add traceshift info in the qframe, this will be saved in the qframe header
if qframe.meta is None:
qframe.meta = dict()
for k in tset.meta.keys():
qframe.meta[k] = tset.meta[k]
if args.output_rawframe is not None:
write_qframe(args.output_rawframe, qframe)
log.info("wrote raw extracted frame in {}".format(
args.output_rawframe))
if args.compute_lsf_sigma:
tset = process_arc(qframe, tset, linelist=None, npoly=2, nbins=2)
if args.output_psf is not None:
for k in qframe.meta:
if k not in tset.meta:
tset.meta[k] = qframe.meta[k]
write_xytraceset(args.output_psf, tset)
if args.compute_fiberflat is not None:
fiberflat = qproc_compute_fiberflat(qframe)
#write_qframe(args.compute_fiberflat,qflat)
write_fiberflat(args.compute_fiberflat, fiberflat, header=qframe.meta)
log.info("wrote fiberflat in {}".format(args.compute_fiberflat))
if args.apply_fiberflat or args.input_fiberflat:
if args.input_fiberflat is None:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
try:
args.input_fiberflat = cfinder.findfile("FIBERFLAT")
except KeyError as e:
log.error("no FIBERFLAT for this spectro config")
sys.exit(12)
log.info("applying fiber flat {}".format(args.input_fiberflat))
flat = read_fiberflat(args.input_fiberflat)
qproc_apply_fiberflat(qframe, flat)
if args.skysub:
log.info("sky subtraction")
if args.output_skyframe is not None:
skyflux = qproc_sky_subtraction(qframe, return_skymodel=True)
sqframe = QFrame(qframe.wave, skyflux, np.ones(skyflux.shape))
write_qframe(args.output_skyframe, sqframe)
log.info("wrote sky model in {}".format(args.output_skyframe))
else:
qproc_sky_subtraction(qframe)
if args.fluxcalib:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
# check for flux calib
if cfinder.haskey("FLUXCALIB"):
fluxcalib_filename = cfinder.findfile("FLUXCALIB")
fluxcalib = read_average_flux_calibration(fluxcalib_filename)
log.info("read average calib in {}".format(fluxcalib_filename))
seeing = qframe.meta["SEEING"]
airmass = qframe.meta["AIRMASS"]
exptime = qframe.meta["EXPTIME"]
exposure_calib = fluxcalib.value(seeing=seeing, airmass=airmass)
for q in range(qframe.nspec):
fiber_calib = np.interp(qframe.wave[q], fluxcalib.wave,
exposure_calib) * exptime
inv_calib = (fiber_calib > 0) / (fiber_calib +
(fiber_calib == 0))
qframe.flux[q] *= inv_calib
qframe.ivar[q] *= fiber_calib**2 * (fiber_calib > 0)
# add keyword in header giving the calibration factor applied at a reference wavelength
band = qframe.meta["CAMERA"].upper()[0]
if band == "B":
refwave = 4500
elif band == "R":
refwave = 6500
else:
refwave = 8500
calvalue = np.interp(refwave, fluxcalib.wave,
exposure_calib) * exptime
qframe.meta["CALWAVE"] = refwave
qframe.meta["CALVALUE"] = calvalue
else:
log.error(
"Cannot calibrate fluxes because no FLUXCALIB keywork in calibration files"
)
fibers = parse_fibers(args.fibers)
if fibers is None:
fibers = qframe.flux.shape[0]
else:
ii = np.arange(qframe.fibers.size)[np.in1d(qframe.fibers, fibers)]
if ii.size == 0:
log.error("no such fibers in frame,")
log.error("fibers are in range [{}:{}]".format(
qframe.fibers[0], qframe.fibers[-1] + 1))
sys.exit(12)
qframe = qframe[ii]
if args.outframe is not None:
write_qframe(args.outframe, qframe)
log.info("wrote {}".format(args.outframe))
t1 = time.time()
log.info("all done in {:3.1f} sec".format(t1 - t0))
if args.plot:
log.info("plotting {} spectra".format(qframe.wave.shape[0]))
import matplotlib.pyplot as plt
fig = plt.figure()
for i in range(qframe.wave.shape[0]):
j = (qframe.ivar[i] > 0)
plt.plot(qframe.wave[i, j], qframe.flux[i, j])
plt.grid()
plt.xlabel("wavelength")
plt.ylabel("flux")
plt.show()
def shift_ycoef_using_external_spectrum(psf,xytraceset,image,fibers,spectrum_filename,degyy=2,width=7) :
"""
Measure y offsets (external wavelength calibration) from a preprocessed image , a PSF + trace set using a cross-correlation of boxcar extracted spectra
and an external well-calibrated spectrum.
The PSF shape is used to convolve the input spectrum. It could also be used to correct for the PSF asymetry (disabled for now).
A relative flux calibration of the spectra is performed internally.
Args:
psf : specter PSF
xytraceset : XYTraceset object
image : DESI preprocessed image object
fibers : 1D np.array of fiber indices
spectrum_filename : path to input spectral file ( read with np.loadtxt , first column is wavelength (in vacuum and Angstrom) , second column in flux (arb. units)
Optional:
width : int, extraction boxcar width, default is 7
degyy : int, degree of polynomial fit of shifts as a function of y, used to reject outliers.
Returns:
ycoef : 2D np.array of same shape as input, with modified Legendre coefficents for each fiber to convert wavelenght to YCCD
"""
log = get_logger()
wavemin = xytraceset.wavemin
wavemax = xytraceset.wavemax
xcoef = xytraceset.x_vs_wave_traceset._coeff
ycoef = xytraceset.y_vs_wave_traceset._coeff
tmp=np.loadtxt(spectrum_filename).T
ref_wave=tmp[0]
ref_spectrum=tmp[1]
log.info("read reference spectrum in %s with %d entries"%(spectrum_filename,ref_wave.size))
log.info("rextract spectra with boxcar")
# boxcar extraction
qframe = qproc_boxcar_extraction(xytraceset, image, fibers=fibers, width=7)
# resampling on common finer wavelength grid
flux, ivar, wave = resample_boxcar_frame(qframe.flux, qframe.ivar, qframe.wave, oversampling=2)
# median flux used as internal spectral reference
mflux=np.median(flux,axis=0)
mivar=np.median(ivar,axis=0)*flux.shape[0]*(2./np.pi) # very appoximate !
# trim ref_spectrum
i=(ref_wave>=wave[0])&(ref_wave<=wave[-1])
ref_wave=ref_wave[i]
ref_spectrum=ref_spectrum[i]
# check wave is linear or make it linear
if np.abs((ref_wave[1]-ref_wave[0])-(ref_wave[-1]-ref_wave[-2]))>0.0001*(ref_wave[1]-ref_wave[0]) :
log.info("reference spectrum wavelength is not on a linear grid, resample it")
dwave = np.min(np.gradient(ref_wave))
tmp_wave = np.linspace(ref_wave[0],ref_wave[-1],int((ref_wave[-1]-ref_wave[0])/dwave))
ref_spectrum = resample_flux(tmp_wave, ref_wave , ref_spectrum)
ref_wave = tmp_wave
i=np.argmax(ref_spectrum)
central_wave_for_psf_evaluation = ref_wave[i]
fiber_for_psf_evaluation = (flux.shape[0]//2)
try :
# compute psf at most significant line of ref_spectrum
dwave=ref_wave[i+1]-ref_wave[i]
hw=int(3./dwave)+1 # 3A half width
wave_range = ref_wave[i-hw:i+hw+1]
x,y=psf.xy(fiber_for_psf_evaluation,wave_range)
x=np.tile(x[hw]+np.arange(-hw,hw+1)*(y[-1]-y[0])/(2*hw+1),(y.size,1))
y=np.tile(y,(2*hw+1,1)).T
kernel2d=psf._value(x,y,fiber_for_psf_evaluation,central_wave_for_psf_evaluation)
kernel1d=np.sum(kernel2d,axis=1)
log.info("convolve reference spectrum using PSF at fiber %d and wavelength %dA"%(fiber_for_psf_evaluation,central_wave_for_psf_evaluation))
ref_spectrum=fftconvolve(ref_spectrum,kernel1d, mode='same')
except :
log.warning("couldn't convolve reference spectrum: %s %s"%(sys.exc_info()[0],sys.exc_info()[1]))
# resample input spectrum
log.info("resample convolved reference spectrum")
ref_spectrum = resample_flux(wave, ref_wave , ref_spectrum)
log.info("absorb difference of calibration")
x=(wave-wave[wave.size//2])/50.
kernel=np.exp(-x**2/2)
f1=fftconvolve(mflux,kernel,mode='same')
f2=fftconvolve(ref_spectrum,kernel,mode='same')
if np.all(f2>0) :
scale=f1/f2
ref_spectrum *= scale
log.info("fit shifts on wavelength bins")
# define bins
n_wavelength_bins = degyy+4
y_for_dy=np.array([])
dy=np.array([])
ey=np.array([])
wave_for_dy=np.array([])
for b in range(n_wavelength_bins) :
wmin=wave[0]+((wave[-1]-wave[0])/n_wavelength_bins)*b
if b<n_wavelength_bins-1 :
wmax=wave[0]+((wave[-1]-wave[0])/n_wavelength_bins)*(b+1)
else :
wmax=wave[-1]
ok=(wave>=wmin)&(wave<=wmax)
sw= np.sum(mflux[ok]*(mflux[ok]>0))
if sw==0 :
continue
dwave,err = compute_dy_from_spectral_cross_correlation(mflux[ok],wave[ok],ref_spectrum[ok],ivar=mivar[ok],hw=3.)
bin_wave = np.sum(mflux[ok]*(mflux[ok]>0)*wave[ok])/sw
x,y=psf.xy(fiber_for_psf_evaluation,bin_wave)
eps=0.1
x,yp=psf.xy(fiber_for_psf_evaluation,bin_wave+eps)
dydw=(yp-y)/eps
if err*dydw<1 :
dy=np.append(dy,-dwave*dydw)
ey=np.append(ey,err*dydw)
wave_for_dy=np.append(wave_for_dy,bin_wave)
y_for_dy=np.append(y_for_dy,y)
log.info("wave = %fA , y=%d, measured dwave = %f +- %f A"%(bin_wave,y,dwave,err))
if False : # we don't need this for now
try :
log.info("correcting bias due to asymmetry of PSF")
hw=5
oversampling=4
xx=np.tile(np.arange(2*hw*oversampling+1)-hw*oversampling,(2*hw*oversampling+1,1))/float(oversampling)
yy=xx.T
x,y=psf.xy(fiber_for_psf_evaluation,central_wave_for_psf_evaluation)
prof=psf._value(xx+x,yy+y,fiber_for_psf_evaluation,central_wave_for_psf_evaluation)
dy_asym_central = np.sum(yy*prof)/np.sum(prof)
for i in range(dy.size) :
x,y=psf.xy(fiber_for_psf_evaluation,wave_for_dy[i])
prof=psf._value(xx+x,yy+y,fiber_for_psf_evaluation,wave_for_dy[i])
dy_asym = np.sum(yy*prof)/np.sum(prof)
log.info("y=%f, measured dy=%f , bias due to PSF asymetry = %f"%(y,dy[i],dy_asym-dy_asym_central))
dy[i] -= (dy_asym-dy_asym_central)
except :
log.warning("couldn't correct for asymmetry of PSF: %s %s"%(sys.exc_info()[0],sys.exc_info()[1]))
log.info("polynomial fit of shifts and modification of PSF ycoef")
# pol fit
coef = np.polyfit(wave_for_dy,dy,degyy,w=1./ey**2)
pol = np.poly1d(coef)
for i in range(dy.size) :
log.info("wave=%fA y=%f, measured dy=%f+-%f , pol(wave) = %f"%(wave_for_dy[i],y_for_dy[i],dy[i],ey[i],pol(wave_for_dy[i])))
log.info("apply this to the PSF ycoef")
wave = np.linspace(wavemin,wavemax,100)
dy = pol(wave)
dycoef = legfit(legx(wave,wavemin,wavemax),dy,deg=ycoef.shape[1]-1)
for fiber in range(ycoef.shape[0]) :
ycoef[fiber] += dycoef
return ycoef
def compute_image_model(image,xytraceset,fiberflat=None,fibermap=None,with_spectral_smoothing=True,with_sky_model=True,\
spectral_smoothing_sigma_length=10.,spectral_smoothing_nsig=4.,psf=None):
'''
Returns a model of the input image, using a fast extraction, a processing of
spectra with a common sky model and a smoothing, followed by a reprojection
on the CCD image.
Inputs:
image: a preprocessed image in the form of a desispec.image.Image object
xytraceset: a desispec.xytraceset.XYTraceSet object with trace coordinates
Optional:
fiberflat: a desispec.fiberflat.FiberFlat object
with_spectral_smoothing: try and smooth the spectra to reduce noise (and eventualy reduce variance correlation)
with_sky_model: use a sky model as part of the spectral modeling to reduce the noise (requires a fiberflat)
spectral_smoothing_sigma_length: sigma of Gaussian smoothing along wavelength in A
spectral_smoothing_nsig: number of sigma rejection threshold to fall back to the original extracted spectrum instead of the smooth one
psf: specter.psf.GaussHermitePSF object to be used for the 1D projection (slow, by default=None, in which case a Gaussian profile is used)
returns:
a 2D np.array of same shape as image.pix
'''
log = get_logger()
# first perform a fast boxcar extraction
log.info("extract spectra")
image.mask = None
qframe = qproc_boxcar_extraction(xytraceset, image)
fqframe = None
sqframe = None
ratio = None
if fiberflat is not None:
log.info("fiberflat")
fqframe = copy.deepcopy(qframe)
flat = qproc_apply_fiberflat(fqframe,
fiberflat=fiberflat,
return_flat=True)
if with_sky_model:
if fiberflat is None:
log.warning(
"cannot compute and use a sky model without a fiberflat")
else:
# crude model of the sky, accounting for fiber throughput variation
log.info("sky")
sqframe = copy.deepcopy(fqframe)
sky = qproc_sky_subtraction(sqframe, return_skymodel=True)
if with_spectral_smoothing:
log.info("spectral smoothing")
sigma = spectral_smoothing_sigma_length / np.mean(
np.gradient(qframe.wave[qframe.nspec // 2]))
log.debug("smoothing sigma in flux bin units = {}".format(sigma))
hw = int(3 * sigma)
u = (np.arange(2 * hw + 1) - hw)
kernel = np.exp(-u**2 / sigma**2 / 2.)
kernel /= np.sum(kernel)
nsig = 5
y = np.arange(qframe.flux.shape[1])
for s in range(qframe.nspec):
if sqframe is not None:
fflux = sqframe.flux[s]
fivar = sqframe.ivar[s]
elif fqframe is not None:
fflux = fqframe.flux[s]
fivar = fqframe.ivar[s]
else:
fflux = qframe.flux[s]
fivar = qframe.ivar[s]
sfflux = scipy.signal.fftconvolve(fflux, kernel, "same")
good = ((fivar * (fflux - sfflux)**2) <
(spectral_smoothing_nsig**2))
out = ~good
nout = np.sum(out)
if nout > 0:
# recompute sflux while masking outliers
tflux = fflux.copy()
tflux[out] = np.interp(y[out], y[good], fflux[good])
sfflux = scipy.signal.fftconvolve(tflux, kernel, "same")
# and replace the 'out' region by the original data
# because we want to keep it to have a fair variance
sfflux[out] = fflux[out]
# replace by smooth version (+ possibly average sky)
if sqframe is not None:
qframe.flux[s] = (sky[s] + sfflux) * flat[s]
elif fqframe is not None:
qframe.flux[s] = sfflux * flat[s]
else:
qframe.flux[s] = sfflux
log.info("project back spectra on image")
y = np.arange(image.pix.shape[0])
# cross dispersion profile
xsig = 1. * np.ones((qframe.nspec, image.pix.shape[0]))
if xytraceset.xsig_vs_wave_traceset:
log.info("use traceset in PSF for xsig")
for s in range(xytraceset.nspec):
wave = xytraceset.wave_vs_y(s, y)
xsig[s] = xytraceset.xsig_vs_wave(s, wave)
else:
log.info("use default xsig={}".format(np.mean(xsig)))
# keep only positive flux
qframe.flux *= (qframe.flux > 0.)
# convert counts per A to counts per pixel
dwave = np.gradient(qframe.wave, axis=1)
qframe.flux *= dwave
# because true profile is not Gaussian and we do not integrate the
# profile in pixels, we have to apply an adjustment
empirical_scale = 1.1
log.info("empirical adjustment of xsig by {:4.2f}".format(empirical_scale))
xsig *= empirical_scale
model = np.zeros(image.pix.shape)
if psf is None: # use simple Gaussian
log.info("use Gaussian sigma of average = {:4.2f}".format(
np.mean(xsig)))
for s in range(qframe.nspec):
x = xytraceset.x_vs_y(s, y)
numba_proj(model, x, xsig[s], qframe.flux[s])
else: # this takes a lot of time
log.debug("Use PSF for projection, but in 1D")
psf._polyparams['HSIZEY'] = 0
psf._polyparams['GHDEGY'] = 0
model = psf.project(qframe.wave,
qframe.flux,
xyrange=(0, image.pix.shape[1], 0,
image.pix.shape[0]))
log.debug("done projecting")
log.info("done")
return model