def test_rejection_ala_sdss(self):
"""
Very basic test of reject_cosmic_rays_ala_sdss
"""
#- Does it reject a diagonal cosmic ray?
image = Image(self.pix, self.ivar, camera="r0")
rejected = reject_cosmic_rays_ala_sdss(image, dilate=False)
diff = np.sum(np.abs((self.pix > 0).astype(int) -
rejected.astype(int)))
self.assertTrue(diff == 0)
#- Does it not find a PSF-like object?
image = Image(self.psfpix, self.ivar, camera="r0")
rejected = reject_cosmic_rays_ala_sdss(image, dilate=False)
diff = np.sum(np.abs((self.pix > 0).astype(int) -
rejected.astype(int)))
self.assertTrue(np.all(self.psfpix * (rejected == 0) == self.psfpix))
#- Can it find one and not the other?
image = Image(self.pix + self.psfpix, self.ivar, camera="r0")
rejected = reject_cosmic_rays_ala_sdss(image, dilate=False)
diff = np.sum(np.abs((self.pix > 0).astype(int) -
rejected.astype(int)))
self.assertTrue(np.all(self.psfpix * (rejected == 0) == self.psfpix))
diff = np.sum(np.abs((self.pix > 0).astype(int) -
rejected.astype(int)))
self.assertTrue(diff == 0)
def test_assertions(self):
with self.assertRaises(ValueError):
Image(self.pix[0], self.ivar[0]) #- pix not 2D
with self.assertRaises(ValueError):
Image(self.pix, self.ivar[0]) #- pix.shape != ivar.shape
with self.assertRaises(ValueError):
Image(self.pix, self.ivar,
self.mask[:, 0:1]) #- pix.shape != mask.shape
def test_readnoise(self):
image = Image(self.pix, self.ivar)
self.assertEqual(image.readnoise, 0.0)
image = Image(self.pix, self.ivar, readnoise=1.0)
self.assertEqual(image.readnoise, 1.0)
readnoise = np.random.uniform(size=self.pix.shape)
image = Image(self.pix, self.ivar, readnoise=readnoise)
self.assertTrue(np.all(image.readnoise == readnoise))
def test_mask(self):
image = Image(self.pix, self.ivar)
self.assertTrue(np.all(image.mask == (self.ivar == 0) * ccdmask.BAD))
self.assertEqual(image.mask.dtype, np.uint32)
image = Image(self.pix, self.ivar, self.mask)
self.assertTrue(np.all(image.mask == self.mask))
self.assertEqual(image.mask.dtype, np.uint32)
image = Image(self.pix, self.ivar, self.mask.astype(np.int16))
self.assertEqual(image.mask.dtype, np.uint32)
def read_image(filename):
"""
Returns desispec.image.Image object from input file
"""
fx = fits.open(filename, uint=True, memmap=False)
image = native_endian(fx['IMAGE'].data).astype(np.float64)
ivar = native_endian(fx['IVAR'].data).astype(np.float64)
mask = native_endian(fx['MASK'].data).astype(np.uint16)
camera = fx['IMAGE'].header['CAMERA'].lower()
meta = fx['IMAGE'].header
if 'READNOISE' in fx:
readnoise = native_endian(fx['READNOISE'].data).astype(np.float64)
else:
readnoise = fx['IMAGE'].header['RDNOISE']
return Image(image, ivar, mask=mask, readnoise=readnoise,
camera=camera, meta=meta)
def test_camera(self):
image = Image(self.pix, self.ivar)
self.assertEqual(image.camera, 'unknown')
image = Image(self.pix, self.ivar, camera='b0')
self.assertEqual(image.camera, 'b0')
def test_init(self):
image = Image(self.pix, self.ivar)
self.assertTrue(np.all(image.pix == self.pix))
self.assertTrue(np.all(image.ivar == self.ivar))
def simulate(night,
expid,
camera,
nspec=None,
verbose=False,
ncpu=None,
trimxy=False,
cosmics=None):
"""
Run pixel-level simulation of input spectra
Args:
night (string) : YEARMMDD
expid (integer) : exposure id
camera (str) : e.g. b0, r1, z9
nspec (int) : number of spectra to simulate
verbose (boolean) : if True, print status messages
ncpu (int) : number of CPU cores to use in parallel
trimxy (boolean) : trim image to just pixels with input signal
cosmics (str) : filename with dark images with cosmics to add
Reads:
$DESI_SPECTRO_SIM/$PIXPROD/{night}/simspec-{expid}.fits
Writes:
$DESI_SPECTRO_SIM/$PIXPROD/{night}/simpix-{camera}-{expid}.fits
$DESI_SPECTRO_SIM/$PIXPROD/{night}/pix-{camera}-{expid}.fits
"""
if verbose:
print "Reading input files"
channel = camera[0].lower()
ispec = int(camera[1])
assert channel in 'brz'
assert 0 <= ispec < 10
#- Load DESI parameters
params = desimodel.io.load_desiparams()
nfibers = params['spectro']['nfibers']
#- Load simspec file
simfile = io.findfile('simspec', night=night, expid=expid)
simspec = io.read_simspec(simfile)
wave = simspec.wave[channel]
if simspec.skyphot is not None:
phot = simspec.phot[channel] + simspec.skyphot[channel]
else:
phot = simspec.phot[channel]
if ispec * nfibers >= simspec.nspec:
print "ERROR: camera {} not in the {} spectra in {}/{}".format(
camera, simspec.nspec, night, os.path.basename(simfile))
return
#- Load PSF
psf = desimodel.io.load_psf(channel)
#- Trim to just the spectra for this spectrograph
if nspec is None:
ii = slice(nfibers * ispec, nfibers * (ispec + 1))
else:
ii = slice(nfibers * ispec, nfibers * ispec + nspec)
phot = phot[ii]
#- check if simulation has less than 500 input spectra
if phot.shape[0] < nspec:
nspec = phot.shape[0]
#- Project to image and append that to file
if verbose:
print "Projecting photons onto CCD"
img = parallel_project(psf, wave, phot, ncpu=ncpu)
if trimxy:
xmin, xmax, ymin, ymax = psf.xyrange((0, nspec), wave)
img = img[0:ymax, 0:xmax]
# img = img[ymin:ymax, xmin:xmax]
# hdr['CRVAL1'] = xmin+1
# hdr['CRVAL2'] = ymin+1
#- Prepare header
hdr = simspec.header
tmp = '/'.join(simfile.split('/')[-3:]) #- last 3 elements of path
hdr['SIMFILE'] = (tmp, 'Input simulation file')
#- Strip unnecessary keywords
for key in ('EXTNAME', 'LOGLAM', 'AIRORVAC', 'CRVAL1', 'CDELT1'):
if key in hdr:
del hdr[key]
#- Write noiseless output
simpixfile = io.findfile('simpix', night=night, expid=expid, camera=camera)
io.write_simpix(simpixfile, img, meta=hdr)
#- Add cosmics from library of dark images
#- in this case, don't add readnoise since the dark image already has it
if cosmics is not None:
cosmics = io.read_cosmics(cosmics, expid, shape=img.shape)
pix = np.random.poisson(img) + cosmics.pix
readnoise = cosmics.meta['RDNOISE']
ivar = 1.0 / (pix.clip(0) + readnoise**2)
mask = cosmics.mask
#- Or just add noise
else:
params = desimodel.io.load_desiparams()
channel = camera[0].lower()
readnoise = params['ccd'][channel]['readnoise']
pix = np.random.poisson(img) + np.random.normal(scale=readnoise,
size=img.shape)
ivar = 1.0 / (pix.clip(0) + readnoise**2)
mask = np.zeros(img.shape, dtype=np.uint16)
#- Metadata to be included in pix file header is in the fibermap header
#- TODO: this is fragile; consider updating fibermap to use astropy Table
#- that includes the header rather than directly assuming FITS as the
#- underlying format.
fibermapfile = desispec.io.findfile('fibermap', night=night, expid=expid)
fm, fmhdr = desispec.io.read_fibermap(fibermapfile, header=True)
meta = dict()
try:
meta['TELRA'] = simspec.header['TELRA']
meta['TELDEC'] = simspec.header['TELDEC']
except KeyError: #- temporary backwards compatibilty
meta['TELRA'] = fmhdr['TELERA']
meta['TELDEC'] = fmhdr['TELEDEC']
meta['TILEID'] = simspec.header['TILEID']
meta['DATE-OBS'] = simspec.header['DATE-OBS']
meta['FLAVOR'] = simspec.header['FLAVOR']
meta['EXPTIME'] = simspec.header['EXPTIME']
meta['AIRMASS'] = simspec.header['AIRMASS']
image = Image(pix,
ivar,
mask,
readnoise=readnoise,
camera=camera,
meta=meta)
pixfile = desispec.io.findfile('pix',
night=night,
camera=camera,
expid=expid)
desispec.io.write_image(pixfile, image)
if verbose:
print "Wrote " + pixfile
def preproc(rawimage, header, bias=False, pixflat=False, mask=False):
'''
preprocess image using metadata in header
image = ((rawimage-bias-overscan)*gain)/pixflat
Args:
rawimage : 2D numpy array directly from raw data file
header : dict-like metadata, e.g. from FITS header, with keywords
CAMERA, BIASSECx, DATASECx, CCDSECx
where x = 1, 2, 3, 4 for each of the 4 amplifiers.
Optional bias, pixflat, and mask can each be:
False: don't apply that step
True: use default calibration data for that night
ndarray: use that array
filename (str or unicode): read HDU 0 and use that
DATE-OBS is required in header if bias, pixflat, or mask=True
Returns Image object with member variables:
image : 2D preprocessed image in units of electrons per pixel
ivar : 2D inverse variance of image
mask : 2D mask of image (0=good)
readnoise : 2D per-pixel readnoise of image
meta : metadata dictionary
TODO: define what keywords are included
preprocessing includes the following steps:
- bias image subtraction
- overscan subtraction (from BIASSEC* keyword defined regions)
- readnoise estimation (from BIASSEC* keyword defined regions)
- gain correction (from GAIN* keywords)
- pixel flat correction
- cosmic ray masking
- propagation of input known bad pixel mask
- inverse variance estimation
Notes:
The bias image is subtracted before any other calculation to remove any
non-uniformities in the overscan regions prior to calculating overscan
levels and readnoise.
The readnoise is an image not just one number per amp, because the pixflat
image also affects the interpreted readnoise.
The inverse variance is estimated from the readnoise and the image itself,
and thus is biased.
'''
#- TODO: Check for required keywords first
#- Subtract bias image
camera = header['CAMERA'].lower()
if bias is not False and bias is not None:
if bias is True:
#- use default bias file for this camera/night
dateobs = header['DATE-OBS']
bias = read_bias(camera=camera, dateobs=dateobs)
elif isinstance(bias, (str, unicode)):
#- treat as filename
bias = read_bias(filename=bias)
if bias.shape == rawimage.shape:
rawimage = rawimage - bias
else:
raise ValueError('shape mismatch bias {} != rawimage {}'.format(
bias.shape, rawimage.shape))
#- Output arrays
yy, xx = _parse_sec_keyword(header['CCDSEC4']) #- 4 = upper right
image = np.zeros((yy.stop, xx.stop))
readnoise = np.zeros_like(image)
for amp in ['1', '2', '3', '4']:
ii = _parse_sec_keyword(header['BIASSEC' + amp])
#- Initial teststand data may be missing GAIN* keywords; don't crash
if 'GAIN' + amp in header:
gain = header['GAIN' + amp] #- gain = electrons / ADU
else:
log.error('Missing keyword GAIN{}; using 1.0'.format(amp))
gain = 1.0
overscan, rdnoise = _overscan(rawimage[ii])
rdnoise *= gain
kk = _parse_sec_keyword(header['CCDSEC' + amp])
readnoise[kk] = rdnoise
header['OVERSCN' + amp] = overscan
header['OBSRDN' + amp] = rdnoise
#- Warn/error if measured readnoise is very different from expected
if 'RDNOISE' + amp in header:
expected_readnoise = header['RDNOISE' + amp]
if rdnoise < 0.5 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'
.format(amp, rdnoise, expected_readnoise))
elif rdnoise < 0.9 * expected_readnoise:
log.warn(
'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'
.format(amp, rdnoise, expected_readnoise))
elif rdnoise > 2.0 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'
.format(amp, rdnoise, expected_readnoise))
elif rdnoise > 1.2 * expected_readnoise:
log.warn(
'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'
.format(amp, rdnoise, expected_readnoise))
else:
log.warn('Expected readnoise keyword {} missing'.format('RDNOISE' +
amp))
#- subtract overscan from data region and apply gain
jj = _parse_sec_keyword(header['DATASEC' + amp])
data = rawimage[jj] - overscan
image[kk] = data * gain
#- Load mask
if mask is not False and mask is not None:
if mask is True:
dateobs = header['DATE-OBS']
mask = read_mask(camera=camera, dateobs=dateobs)
elif isinstance(mask, (str, unicode)):
mask = read_mask(filename=mask)
else:
mask = np.zeros(image.shape, dtype=np.int32)
if mask.shape != image.shape:
raise ValueError('shape mismatch mask {} != image {}'.format(
mask.shape, image.shape))
#- Divide by pixflat image
if pixflat is not False and pixflat is not None:
if pixflat is True:
dateobs = header['DATE-OBS']
pixflat = read_pixflat(camera=camera, dateobs=dateobs)
elif isinstance(pixflat, (str, unicode)):
pixflat = read_pixflat(filename=pixflat)
if pixflat.shape != image.shape:
raise ValueError('shape mismatch pixflat {} != image {}'.format(
pixflat.shape, image.shape))
if np.all(pixflat != 0.0):
image /= pixflat
readnoise /= pixflat
else:
good = (pixflat != 0.0)
image[good] /= pixflat[good]
readnoise[good] /= pixflat[good]
mask[~good] |= ccdmask.PIXFLATZERO
lowpixflat = (0 < pixflat) & (pixflat < 0.1)
if np.any(lowpixflat):
mask[lowpixflat] |= ccdmask.PIXFLATLOW
#- Inverse variance, estimated directly from the data (BEWARE: biased!)
var = image.clip(0) + readnoise**2
ivar = 1.0 / var
img = Image(image,
ivar=ivar,
mask=mask,
meta=header,
readnoise=readnoise,
camera=camera)
#- update img.mask to mask cosmic rays
cosmics.reject_cosmic_rays(img)
return img
