def test_rejection_ala_sdss(self): """ Very basic test of reject_cosmic_rays_ala_sdss """ #- Does it reject a diagonal cosmic ray? image = Image(self.pix, self.ivar, camera="r0") rejected = reject_cosmic_rays_ala_sdss(image, dilate=False) diff = np.sum(np.abs((self.pix > 0).astype(int) - rejected.astype(int))) self.assertTrue(diff == 0) #- Does it not find a PSF-like object? image = Image(self.psfpix, self.ivar, camera="r0") rejected = reject_cosmic_rays_ala_sdss(image, dilate=False) diff = np.sum(np.abs((self.pix > 0).astype(int) - rejected.astype(int))) self.assertTrue(np.all(self.psfpix * (rejected == 0) == self.psfpix)) #- Can it find one and not the other? image = Image(self.pix + self.psfpix, self.ivar, camera="r0") rejected = reject_cosmic_rays_ala_sdss(image, dilate=False) diff = np.sum(np.abs((self.pix > 0).astype(int) - rejected.astype(int))) self.assertTrue(np.all(self.psfpix * (rejected == 0) == self.psfpix)) diff = np.sum(np.abs((self.pix > 0).astype(int) - rejected.astype(int))) self.assertTrue(diff == 0)
def test_assertions(self): with self.assertRaises(ValueError): Image(self.pix[0], self.ivar[0]) #- pix not 2D with self.assertRaises(ValueError): Image(self.pix, self.ivar[0]) #- pix.shape != ivar.shape with self.assertRaises(ValueError): Image(self.pix, self.ivar, self.mask[:, 0:1]) #- pix.shape != mask.shape
def test_readnoise(self): image = Image(self.pix, self.ivar) self.assertEqual(image.readnoise, 0.0) image = Image(self.pix, self.ivar, readnoise=1.0) self.assertEqual(image.readnoise, 1.0) readnoise = np.random.uniform(size=self.pix.shape) image = Image(self.pix, self.ivar, readnoise=readnoise) self.assertTrue(np.all(image.readnoise == readnoise))
def test_mask(self): image = Image(self.pix, self.ivar) self.assertTrue(np.all(image.mask == (self.ivar == 0) * ccdmask.BAD)) self.assertEqual(image.mask.dtype, np.uint32) image = Image(self.pix, self.ivar, self.mask) self.assertTrue(np.all(image.mask == self.mask)) self.assertEqual(image.mask.dtype, np.uint32) image = Image(self.pix, self.ivar, self.mask.astype(np.int16)) self.assertEqual(image.mask.dtype, np.uint32)
def read_image(filename): """ Returns desispec.image.Image object from input file """ fx = fits.open(filename, uint=True, memmap=False) image = native_endian(fx['IMAGE'].data).astype(np.float64) ivar = native_endian(fx['IVAR'].data).astype(np.float64) mask = native_endian(fx['MASK'].data).astype(np.uint16) camera = fx['IMAGE'].header['CAMERA'].lower() meta = fx['IMAGE'].header if 'READNOISE' in fx: readnoise = native_endian(fx['READNOISE'].data).astype(np.float64) else: readnoise = fx['IMAGE'].header['RDNOISE'] return Image(image, ivar, mask=mask, readnoise=readnoise, camera=camera, meta=meta)
def test_camera(self): image = Image(self.pix, self.ivar) self.assertEqual(image.camera, 'unknown') image = Image(self.pix, self.ivar, camera='b0') self.assertEqual(image.camera, 'b0')
def test_init(self): image = Image(self.pix, self.ivar) self.assertTrue(np.all(image.pix == self.pix)) self.assertTrue(np.all(image.ivar == self.ivar))
def simulate(night, expid, camera, nspec=None, verbose=False, ncpu=None, trimxy=False, cosmics=None): """ Run pixel-level simulation of input spectra Args: night (string) : YEARMMDD expid (integer) : exposure id camera (str) : e.g. b0, r1, z9 nspec (int) : number of spectra to simulate verbose (boolean) : if True, print status messages ncpu (int) : number of CPU cores to use in parallel trimxy (boolean) : trim image to just pixels with input signal cosmics (str) : filename with dark images with cosmics to add Reads: $DESI_SPECTRO_SIM/$PIXPROD/{night}/simspec-{expid}.fits Writes: $DESI_SPECTRO_SIM/$PIXPROD/{night}/simpix-{camera}-{expid}.fits $DESI_SPECTRO_SIM/$PIXPROD/{night}/pix-{camera}-{expid}.fits """ if verbose: print "Reading input files" channel = camera[0].lower() ispec = int(camera[1]) assert channel in 'brz' assert 0 <= ispec < 10 #- Load DESI parameters params = desimodel.io.load_desiparams() nfibers = params['spectro']['nfibers'] #- Load simspec file simfile = io.findfile('simspec', night=night, expid=expid) simspec = io.read_simspec(simfile) wave = simspec.wave[channel] if simspec.skyphot is not None: phot = simspec.phot[channel] + simspec.skyphot[channel] else: phot = simspec.phot[channel] if ispec * nfibers >= simspec.nspec: print "ERROR: camera {} not in the {} spectra in {}/{}".format( camera, simspec.nspec, night, os.path.basename(simfile)) return #- Load PSF psf = desimodel.io.load_psf(channel) #- Trim to just the spectra for this spectrograph if nspec is None: ii = slice(nfibers * ispec, nfibers * (ispec + 1)) else: ii = slice(nfibers * ispec, nfibers * ispec + nspec) phot = phot[ii] #- check if simulation has less than 500 input spectra if phot.shape[0] < nspec: nspec = phot.shape[0] #- Project to image and append that to file if verbose: print "Projecting photons onto CCD" img = parallel_project(psf, wave, phot, ncpu=ncpu) if trimxy: xmin, xmax, ymin, ymax = psf.xyrange((0, nspec), wave) img = img[0:ymax, 0:xmax] # img = img[ymin:ymax, xmin:xmax] # hdr['CRVAL1'] = xmin+1 # hdr['CRVAL2'] = ymin+1 #- Prepare header hdr = simspec.header tmp = '/'.join(simfile.split('/')[-3:]) #- last 3 elements of path hdr['SIMFILE'] = (tmp, 'Input simulation file') #- Strip unnecessary keywords for key in ('EXTNAME', 'LOGLAM', 'AIRORVAC', 'CRVAL1', 'CDELT1'): if key in hdr: del hdr[key] #- Write noiseless output simpixfile = io.findfile('simpix', night=night, expid=expid, camera=camera) io.write_simpix(simpixfile, img, meta=hdr) #- Add cosmics from library of dark images #- in this case, don't add readnoise since the dark image already has it if cosmics is not None: cosmics = io.read_cosmics(cosmics, expid, shape=img.shape) pix = np.random.poisson(img) + cosmics.pix readnoise = cosmics.meta['RDNOISE'] ivar = 1.0 / (pix.clip(0) + readnoise**2) mask = cosmics.mask #- Or just add noise else: params = desimodel.io.load_desiparams() channel = camera[0].lower() readnoise = params['ccd'][channel]['readnoise'] pix = np.random.poisson(img) + np.random.normal(scale=readnoise, size=img.shape) ivar = 1.0 / (pix.clip(0) + readnoise**2) mask = np.zeros(img.shape, dtype=np.uint16) #- Metadata to be included in pix file header is in the fibermap header #- TODO: this is fragile; consider updating fibermap to use astropy Table #- that includes the header rather than directly assuming FITS as the #- underlying format. fibermapfile = desispec.io.findfile('fibermap', night=night, expid=expid) fm, fmhdr = desispec.io.read_fibermap(fibermapfile, header=True) meta = dict() try: meta['TELRA'] = simspec.header['TELRA'] meta['TELDEC'] = simspec.header['TELDEC'] except KeyError: #- temporary backwards compatibilty meta['TELRA'] = fmhdr['TELERA'] meta['TELDEC'] = fmhdr['TELEDEC'] meta['TILEID'] = simspec.header['TILEID'] meta['DATE-OBS'] = simspec.header['DATE-OBS'] meta['FLAVOR'] = simspec.header['FLAVOR'] meta['EXPTIME'] = simspec.header['EXPTIME'] meta['AIRMASS'] = simspec.header['AIRMASS'] image = Image(pix, ivar, mask, readnoise=readnoise, camera=camera, meta=meta) pixfile = desispec.io.findfile('pix', night=night, camera=camera, expid=expid) desispec.io.write_image(pixfile, image) if verbose: print "Wrote " + pixfile
def preproc(rawimage, header, bias=False, pixflat=False, mask=False): ''' preprocess image using metadata in header image = ((rawimage-bias-overscan)*gain)/pixflat Args: rawimage : 2D numpy array directly from raw data file header : dict-like metadata, e.g. from FITS header, with keywords CAMERA, BIASSECx, DATASECx, CCDSECx where x = 1, 2, 3, 4 for each of the 4 amplifiers. Optional bias, pixflat, and mask can each be: False: don't apply that step True: use default calibration data for that night ndarray: use that array filename (str or unicode): read HDU 0 and use that DATE-OBS is required in header if bias, pixflat, or mask=True Returns Image object with member variables: image : 2D preprocessed image in units of electrons per pixel ivar : 2D inverse variance of image mask : 2D mask of image (0=good) readnoise : 2D per-pixel readnoise of image meta : metadata dictionary TODO: define what keywords are included preprocessing includes the following steps: - bias image subtraction - overscan subtraction (from BIASSEC* keyword defined regions) - readnoise estimation (from BIASSEC* keyword defined regions) - gain correction (from GAIN* keywords) - pixel flat correction - cosmic ray masking - propagation of input known bad pixel mask - inverse variance estimation Notes: The bias image is subtracted before any other calculation to remove any non-uniformities in the overscan regions prior to calculating overscan levels and readnoise. The readnoise is an image not just one number per amp, because the pixflat image also affects the interpreted readnoise. The inverse variance is estimated from the readnoise and the image itself, and thus is biased. ''' #- TODO: Check for required keywords first #- Subtract bias image camera = header['CAMERA'].lower() if bias is not False and bias is not None: if bias is True: #- use default bias file for this camera/night dateobs = header['DATE-OBS'] bias = read_bias(camera=camera, dateobs=dateobs) elif isinstance(bias, (str, unicode)): #- treat as filename bias = read_bias(filename=bias) if bias.shape == rawimage.shape: rawimage = rawimage - bias else: raise ValueError('shape mismatch bias {} != rawimage {}'.format( bias.shape, rawimage.shape)) #- Output arrays yy, xx = _parse_sec_keyword(header['CCDSEC4']) #- 4 = upper right image = np.zeros((yy.stop, xx.stop)) readnoise = np.zeros_like(image) for amp in ['1', '2', '3', '4']: ii = _parse_sec_keyword(header['BIASSEC' + amp]) #- Initial teststand data may be missing GAIN* keywords; don't crash if 'GAIN' + amp in header: gain = header['GAIN' + amp] #- gain = electrons / ADU else: log.error('Missing keyword GAIN{}; using 1.0'.format(amp)) gain = 1.0 overscan, rdnoise = _overscan(rawimage[ii]) rdnoise *= gain kk = _parse_sec_keyword(header['CCDSEC' + amp]) readnoise[kk] = rdnoise header['OVERSCN' + amp] = overscan header['OBSRDN' + amp] = rdnoise #- Warn/error if measured readnoise is very different from expected if 'RDNOISE' + amp in header: expected_readnoise = header['RDNOISE' + amp] if rdnoise < 0.5 * expected_readnoise: log.error( 'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}' .format(amp, rdnoise, expected_readnoise)) elif rdnoise < 0.9 * expected_readnoise: log.warn( 'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}' .format(amp, rdnoise, expected_readnoise)) elif rdnoise > 2.0 * expected_readnoise: log.error( 'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}' .format(amp, rdnoise, expected_readnoise)) elif rdnoise > 1.2 * expected_readnoise: log.warn( 'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}' .format(amp, rdnoise, expected_readnoise)) else: log.warn('Expected readnoise keyword {} missing'.format('RDNOISE' + amp)) #- subtract overscan from data region and apply gain jj = _parse_sec_keyword(header['DATASEC' + amp]) data = rawimage[jj] - overscan image[kk] = data * gain #- Load mask if mask is not False and mask is not None: if mask is True: dateobs = header['DATE-OBS'] mask = read_mask(camera=camera, dateobs=dateobs) elif isinstance(mask, (str, unicode)): mask = read_mask(filename=mask) else: mask = np.zeros(image.shape, dtype=np.int32) if mask.shape != image.shape: raise ValueError('shape mismatch mask {} != image {}'.format( mask.shape, image.shape)) #- Divide by pixflat image if pixflat is not False and pixflat is not None: if pixflat is True: dateobs = header['DATE-OBS'] pixflat = read_pixflat(camera=camera, dateobs=dateobs) elif isinstance(pixflat, (str, unicode)): pixflat = read_pixflat(filename=pixflat) if pixflat.shape != image.shape: raise ValueError('shape mismatch pixflat {} != image {}'.format( pixflat.shape, image.shape)) if np.all(pixflat != 0.0): image /= pixflat readnoise /= pixflat else: good = (pixflat != 0.0) image[good] /= pixflat[good] readnoise[good] /= pixflat[good] mask[~good] |= ccdmask.PIXFLATZERO lowpixflat = (0 < pixflat) & (pixflat < 0.1) if np.any(lowpixflat): mask[lowpixflat] |= ccdmask.PIXFLATLOW #- Inverse variance, estimated directly from the data (BEWARE: biased!) var = image.clip(0) + readnoise**2 ivar = 1.0 / var img = Image(image, ivar=ivar, mask=mask, meta=header, readnoise=readnoise, camera=camera) #- update img.mask to mask cosmic rays cosmics.reject_cosmic_rays(img) return img