def test_rejection_ala_sdss(self): """ Very basic test of reject_cosmic_rays_ala_sdss """ #- Does it reject a diagonal cosmic ray? image = Image(self.pix, self.ivar, camera="r0") rejected = reject_cosmic_rays_ala_sdss(image, dilate=False) diff = np.sum(np.abs((self.pix > 0).astype(int) - rejected.astype(int))) self.assertTrue(diff == 0) #- Does it not find a PSF-like object? image = Image(self.psfpix, self.ivar, camera="r0") rejected = reject_cosmic_rays_ala_sdss(image, dilate=False) diff = np.sum(np.abs((self.pix > 0).astype(int) - rejected.astype(int))) self.assertTrue(np.all(self.psfpix * (rejected == 0) == self.psfpix)) #- Can it find one and not the other? image = Image(self.pix + self.psfpix, self.ivar, camera="r0") rejected = reject_cosmic_rays_ala_sdss(image, dilate=False) diff = np.sum(np.abs((self.pix > 0).astype(int) - rejected.astype(int))) self.assertTrue(np.all(self.psfpix * (rejected == 0) == self.psfpix)) diff = np.sum(np.abs((self.pix > 0).astype(int) - rejected.astype(int))) self.assertTrue(diff == 0)
def test_bootcalib(self): from desispec.bootcalib import bootcalib from desispec.image import Image arc = fits.getdata(self.testarc) flat = fits.getdata(self.testflat) arcimage = Image(arc, np.ones_like(arc), camera='b0') flatimage = Image(flat, np.ones_like(flat), camera='b0') results = bootcalib(3, flatimage, arcimage)
def test_assertions(self): with self.assertRaises(ValueError): Image(self.pix[0], self.ivar[0]) #- pix not 2D with self.assertRaises(ValueError): Image(self.pix, self.ivar[0]) #- pix.shape != ivar.shape with self.assertRaises(ValueError): Image(self.pix, self.ivar, self.mask[:, 0:1]) #- pix.shape != mask.shape
def test_readnoise(self): image = Image(self.pix, self.ivar) self.assertEqual(image.readnoise, 0.0) image = Image(self.pix, self.ivar, readnoise=1.0) self.assertEqual(image.readnoise, 1.0) readnoise = np.random.uniform(size=self.pix.shape) image = Image(self.pix, self.ivar, readnoise=readnoise) self.assertTrue(np.all(image.readnoise == readnoise))
def test_mask(self): image = Image(self.pix, self.ivar) self.assertTrue(np.all(image.mask == (self.ivar == 0) * ccdmask.BAD)) self.assertEqual(image.mask.dtype, np.uint32) image = Image(self.pix, self.ivar, self.mask) self.assertTrue(np.all(image.mask == self.mask)) self.assertEqual(image.mask.dtype, np.uint32) image = Image(self.pix, self.ivar, self.mask.astype(np.int16)) self.assertEqual(image.mask.dtype, np.uint32)
def test_different_cameras(self): '''test a PSF-like spot with a masked column going through it''' for camera in ('b0', 'r1', 'z2'): image = Image(self.pix, self.ivar, mask=self.badmask, camera=camera) rejected = reject_cosmic_rays_ala_sdss(image,dilate=False) cosmic = image.pix > 0 self.assertTrue(np.all(rejected[cosmic])) #- camera must be valid with self.assertRaises(KeyError): image = Image(self.pix, self.ivar, mask=self.badmask, camera='a0') rejected = reject_cosmic_rays_ala_sdss(image,dilate=False)
def read_image(filename): """ Returns desispec.image.Image object from input file """ log = get_logger() t0 = time.time() with fits.open(filename, uint=True, memmap=False) as fx: image = native_endian(fx['IMAGE'].data).astype(np.float64) ivar = native_endian(fx['IVAR'].data).astype(np.float64) mask = native_endian(fx['MASK'].data).astype(np.uint16) camera = fx['IMAGE'].header['CAMERA'].lower() meta = fx['IMAGE'].header if 'READNOISE' in fx: readnoise = native_endian(fx['READNOISE'].data).astype(np.float64) else: readnoise = fx['IMAGE'].header['RDNOISE'] duration = time.time() - t0 log.info(iotime.format('read', filename, duration)) return Image(image, ivar, mask=mask, readnoise=readnoise, camera=camera, meta=meta)
def test_image_rw(self): shape = (5, 5) pix = np.random.uniform(size=shape) ivar = np.random.uniform(size=shape) mask = np.random.randint(0, 3, size=shape) img1 = Image(pix, ivar, mask, readnoise=1.0, camera='b0') desispec.io.write_image(self.testfile, img1) img2 = desispec.io.read_image(self.testfile) #- Check output datatypes self.assertEqual(img2.pix.dtype, np.float64) self.assertEqual(img2.ivar.dtype, np.float64) self.assertEqual(img2.mask.dtype, np.uint32) #- Rounding from keeping np.float32 on disk means they aren't equal self.assertFalse(np.all(img1.pix == img2.pix)) self.assertFalse(np.all(img1.ivar == img2.ivar)) #- But they should be close, and identical after float64->float32 self.assertTrue(np.allclose(img1.pix, img2.pix)) self.assertTrue(np.all(img1.pix.astype(np.float32) == img2.pix)) self.assertTrue(np.allclose(img1.ivar, img2.ivar)) self.assertTrue(np.all(img1.ivar.astype(np.float32) == img2.ivar)) #- masks should agree self.assertTrue(np.all(img1.mask == img2.mask)) self.assertEqual(img1.readnoise, img2.readnoise) self.assertEqual(img1.camera, img2.camera) self.assertEqual(img2.mask.dtype, np.uint32) #- should work with various kinds of metadata header input meta = dict(BLAT='foo', BAR='quat', BIZ=1.0) img1 = Image(pix, ivar, mask, readnoise=1.0, camera='b0', meta=meta) desispec.io.write_image(self.testfile, img1) img2 = desispec.io.read_image(self.testfile) for key in meta: self.assertEqual(meta[key], img2.meta[key], 'meta[{}] not propagated'.format(key)) #- img2 has meta as a FITS header instead of a dictionary; #- confirm that works too desispec.io.write_image(self.testfile, img2) img3 = desispec.io.read_image(self.testfile) for key in meta: self.assertEqual(meta[key], img3.meta[key], 'meta[{}] not propagated'.format(key))
def test_reject_cosmics(self): """ Test that the generic cosmics interface updates the mask """ image = Image(self.pix, self.ivar, camera="r0") reject_cosmic_rays(image) cosmic = (image.pix > 0) self.assertTrue(np.all(image.mask[cosmic] & ccdmask.COSMIC))
def read_image(filename): """ Returns desispec.image.Image object from input file """ fx = fits.open(filename, uint=True, memmap=False) image = native_endian(fx['IMAGE'].data).astype(np.float64) ivar = native_endian(fx['IVAR'].data).astype(np.float64) mask = native_endian(fx['MASK'].data).astype(np.uint16) camera = fx['IMAGE'].header['CAMERA'].lower() meta = fx['IMAGE'].header if 'READNOISE' in fx: readnoise = native_endian(fx['READNOISE'].data).astype(np.float64) else: readnoise = fx['IMAGE'].header['RDNOISE'] return Image(image, ivar, mask=mask, readnoise=readnoise, camera=camera, meta=meta)
def preproc(rawimage, header, primary_header, bias=True, dark=True, pixflat=True, mask=True, bkgsub=False, nocosmic=False, cosmics_nsig=6, cosmics_cfudge=3., cosmics_c2fudge=0.8, ccd_calibration_filename=None, nocrosstalk=False): ''' preprocess image using metadata in header image = ((rawimage-bias-overscan)*gain)/pixflat Args: rawimage : 2D numpy array directly from raw data file header : dict-like metadata, e.g. from FITS header, with keywords CAMERA, BIASSECx, DATASECx, CCDSECx where x = A, B, C, D for each of the 4 amplifiers (also supports old naming convention 1, 2, 3, 4). primary_header: dict-like metadata fit keywords EXPTIME, DOSVER DATE-OBS is also required if bias, pixflat, or mask=True Optional bias, pixflat, and mask can each be: False: don't apply that step True: use default calibration data for that night ndarray: use that array filename (str or unicode): read HDU 0 and use that Optional background subtraction with median filtering if bkgsub=True Optional disabling of cosmic ray rejection if nocosmic=True Optional tuning of cosmic ray rejection parameters: cosmics_nsig: number of sigma above background required cosmics_cfudge: number of sigma inconsistent with PSF required cosmics_c2fudge: fudge factor applied to PSF Returns Image object with member variables: image : 2D preprocessed image in units of electrons per pixel ivar : 2D inverse variance of image mask : 2D mask of image (0=good) readnoise : 2D per-pixel readnoise of image meta : metadata dictionary TODO: define what keywords are included preprocessing includes the following steps: - bias image subtraction - overscan subtraction (from BIASSEC* keyword defined regions) - readnoise estimation (from BIASSEC* keyword defined regions) - gain correction (from GAIN* keywords) - pixel flat correction - cosmic ray masking - propagation of input known bad pixel mask - inverse variance estimation Notes: The bias image is subtracted before any other calculation to remove any non-uniformities in the overscan regions prior to calculating overscan levels and readnoise. The readnoise is an image not just one number per amp, because the pixflat image also affects the interpreted readnoise. The inverse variance is estimated from the readnoise and the image itself, and thus is biased. ''' log = get_logger() calibration_data = None if ccd_calibration_filename is None: srch_file = "data/ccd/ccd_calibration.yaml" if not resource_exists('desispec', srch_file): log.error( "Cannot find CCD calibration file {:s}".format(srch_file)) else: ccd_calibration_filename = resource_filename('desispec', srch_file) if ccd_calibration_filename is not None and ccd_calibration_filename is not False: calibration_data = read_ccd_calibration(header, primary_header, ccd_calibration_filename) #- Get path to calibration data if "DESI_CCD_CALIBRATION_DATA" in os.environ: calibration_data_path = os.environ["DESI_CCD_CALIBRATION_DATA"] else: calibration_data_path = None #- TODO: Check for required keywords first #- Subtract bias image camera = header['CAMERA'].lower() #- convert rawimage to float64 : this is the output format of read_image rawimage = rawimage.astype(np.float64) bias = get_calibration_image(calibration_data, calibration_data_path, "BIAS", bias) if bias is not False: #- it's an array if bias.shape == rawimage.shape: log.info("subtracting bias") rawimage = rawimage - bias else: raise ValueError('shape mismatch bias {} != rawimage {}'.format( bias.shape, rawimage.shape)) if calibration_data and "AMPLIFIERS" in calibration_data: amp_ids = list(calibration_data["AMPLIFIERS"]) else: amp_ids = ['A', 'B', 'C', 'D'] #- check whether it's indeed CCDSECx with x in ['A','B','C','D'] # or older version with x in ['1','2','3','4'] # we can remove this piece of code at later times has_valid_keywords = True for amp in amp_ids: if not 'CCDSEC%s' % amp in header: log.warning( "No CCDSEC%s keyword in header , will look for alternative naming CCDSEC{1,2,3,4} ..." % amp) has_valid_keywords = False break if not has_valid_keywords: amp_ids = ['1', '2', '3', '4'] for amp in ['1', '2', '3', '4']: if not 'CCDSEC%s' % amp in header: log.error("No CCDSEC%s keyword, exit" % amp) raise KeyError("No CCDSEC%s keyword" % amp) #- Output arrays ny = 0 nx = 0 for amp in amp_ids: yy, xx = _parse_sec_keyword(header['CCDSEC%s' % amp]) ny = max(ny, yy.stop) nx = max(nx, xx.stop) image = np.zeros((ny, nx)) readnoise = np.zeros_like(image) #- Load mask mask = get_calibration_image(calibration_data, calibration_data_path, "MASK", mask) if mask is False: mask = np.zeros(image.shape, dtype=np.int32) else: if mask.shape != image.shape: raise ValueError('shape mismatch mask {} != image {}'.format( mask.shape, image.shape)) #- Load dark dark = get_calibration_image(calibration_data, calibration_data_path, "DARK", dark) if dark is not False: if dark.shape != image.shape: log.error('shape mismatch dark {} != image {}'.format( dark.shape, image.shape)) raise ValueError('shape mismatch dark {} != image {}'.format( dark.shape, image.shape)) if calibration_data and "EXPTIMEKEY" in calibration_data: exptime_key = calibration_data["EXPTIMEKEY"] log.info("Using exposure time keyword %s for dark normalization" % exptime_key) else: exptime_key = "EXPTIME" exptime = primary_header[exptime_key] log.info("Multiplying dark by exptime %f" % (exptime)) dark *= exptime for amp in amp_ids: ii = _parse_sec_keyword(header['BIASSEC' + amp]) #- Initial teststand data may be missing GAIN* keywords; don't crash if 'GAIN' + amp in header: gain = header['GAIN' + amp] #- gain = electrons / ADU else: if calibration_data and 'GAIN' + amp in calibration_data: gain = float(calibration_data['GAIN' + amp]) log.info('Using GAIN{}={} from calibration data'.format( amp, gain)) else: gain = 1.0 log.warning( 'Missing keyword GAIN{} in header and nothing in calib data; using {}' .format(amp, gain)) #- Add saturation level if 'SATURLEV' + amp in header: saturlev = header['SATURLEV' + amp] # in electrons else: if calibration_data and 'SATURLEV' + amp in calibration_data: saturlev = float(calibration_data['SATURLEV' + amp]) log.info('Using SATURLEV{}={} from calibration data'.format( amp, saturlev)) else: saturlev = 200000 log.warning( 'Missing keyword SATURLEV{} in header and nothing in calib data; using 200000' .format(amp, saturlev)) overscan_image = rawimage[ii].copy() nrows = overscan_image.shape[0] log.info("nrows in overscan=%d" % nrows) overscan = np.zeros(nrows) rdnoise = np.zeros(nrows) overscan_per_row = True if calibration_data and 'OVERSCAN' + amp in calibration_data and calibration_data[ "OVERSCAN" + amp].upper() == "PER_ROW": log.info( "Subtracting overscan per row for amplifier %s of camera %s" % (amp, camera)) for j in range(nrows): if np.isnan(np.sum(overscan_image[j])): log.warning( "NaN values in row %d of overscan of amplifier %s of camera %s" % (j, amp, camera)) continue o, r = _overscan(overscan_image[j]) #log.info("%d %f %f"%(j,o,r)) overscan[j] = o rdnoise[j] = r else: log.info( "Subtracting average overscan for amplifier %s of camera %s" % (amp, camera)) o, r = _overscan(overscan_image) overscan += o rdnoise += r rdnoise *= gain median_rdnoise = np.median(rdnoise) median_overscan = np.median(overscan) log.info("Median rdnoise and overscan= %f %f" % (median_rdnoise, median_overscan)) kk = _parse_sec_keyword(header['CCDSEC' + amp]) for j in range(nrows): readnoise[kk][j] = rdnoise[j] header['OVERSCN' + amp] = median_overscan header['OBSRDN' + amp] = median_rdnoise #- Warn/error if measured readnoise is very different from expected if exists if 'RDNOISE' + amp in header: expected_readnoise = header['RDNOISE' + amp] if median_rdnoise < 0.5 * expected_readnoise: log.error( 'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}' .format(amp, median_rdnoise, expected_readnoise)) elif median_rdnoise < 0.9 * expected_readnoise: log.warning( 'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}' .format(amp, median_rdnoise, expected_readnoise)) elif median_rdnoise > 2.0 * expected_readnoise: log.error( 'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}' .format(amp, median_rdnoise, expected_readnoise)) elif median_rdnoise > 1.2 * expected_readnoise: log.warning( 'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}' .format(amp, median_rdnoise, expected_readnoise)) #else: # log.warning('Expected readnoise keyword {} missing'.format('RDNOISE'+amp)) log.info("Measured readnoise for AMP %s = %f" % (amp, median_rdnoise)) #- subtract overscan from data region and apply gain jj = _parse_sec_keyword(header['DATASEC' + amp]) data = rawimage[jj].copy() for k in range(nrows): data[k] -= overscan[k] #- apply saturlev (defined in ADU), prior to multiplication by gain saturated = (rawimage[jj] >= saturlev) mask[kk][saturated] |= ccdmask.SATURATED #- subtract dark prior to multiplication by gain if dark is not False: log.info("subtracting dark for amp %s" % amp) data -= dark[kk] image[kk] = data * gain if not nocrosstalk: #- apply cross-talk # the ccd looks like : # C D # A B # for cross talk, we need a symmetric 4x4 flip_matrix # of coordinates ABCD giving flip of both axis # when computing crosstalk of # A B C D # # A AA AB AC AD # B BA BB BC BD # C CA CB CC CD # D DA DB DC BB # orientation_matrix_defines change of orientation # fip_axis_0 = np.array([[1, 1, -1, -1], [1, 1, -1, -1], [-1, -1, 1, 1], [-1, -1, 1, 1]]) fip_axis_1 = np.array([[1, -1, 1, -1], [-1, 1, -1, 1], [1, -1, 1, -1], [-1, 1, -1, 1]]) for a1 in range(len(amp_ids)): amp1 = amp_ids[a1] ii1 = _parse_sec_keyword(header['CCDSEC' + amp1]) a1flux = image[ii1] #a1mask=mask[ii1] for a2 in range(len(amp_ids)): if a1 == a2: continue amp2 = amp_ids[a2] if calibration_data is None: continue if not "CROSSTALK%s%s" % (amp1, amp2) in calibration_data: continue crosstalk = calibration_data["CROSSTALK%s%s" % (amp1, amp2)] if crosstalk == 0.: continue log.info("Correct for crosstalk=%f from AMP %s into %s" % (crosstalk, amp1, amp2)) a12flux = crosstalk * a1flux.copy() #a12mask=a1mask.copy() if fip_axis_0[a1, a2] == -1: a12flux = a12flux[::-1] #a12mask=a12mask[::-1] if fip_axis_1[a1, a2] == -1: a12flux = a12flux[:, ::-1] #a12mask=a12mask[:,::-1] ii2 = _parse_sec_keyword(header['CCDSEC' + amp2]) image[ii2] -= a12flux # mask[ii2] |= a12mask (not sure we really need to propagate the mask) #- Divide by pixflat image pixflat = get_calibration_image(calibration_data, calibration_data_path, "PIXFLAT", pixflat) if pixflat is not False: if pixflat.shape != image.shape: raise ValueError('shape mismatch pixflat {} != image {}'.format( pixflat.shape, image.shape)) if np.all(pixflat != 0.0): image /= pixflat readnoise /= pixflat else: good = (pixflat != 0.0) image[good] /= pixflat[good] readnoise[good] /= pixflat[good] mask[~good] |= ccdmask.PIXFLATZERO lowpixflat = (0 < pixflat) & (pixflat < 0.1) if np.any(lowpixflat): mask[lowpixflat] |= ccdmask.PIXFLATLOW #- Inverse variance, estimated directly from the data (BEWARE: biased!) var = image.clip(0) + readnoise**2 ivar = np.zeros(var.shape) ivar[var > 0] = 1.0 / var[var > 0] if bkgsub: bkg = _background(image, header) image -= bkg img = Image(image, ivar=ivar, mask=mask, meta=header, readnoise=readnoise, camera=camera) #- update img.mask to mask cosmic rays if not nocosmic: cosmics.reject_cosmic_rays(img, nsig=cosmics_nsig, cfudge=cosmics_cfudge, c2fudge=cosmics_c2fudge) return img
def simulate(night, expid, camera, nspec=None, verbose=False, ncpu=None, trimxy=False, cosmics=None): """ Run pixel-level simulation of input spectra Args: night (string) : YEARMMDD expid (integer) : exposure id camera (str) : e.g. b0, r1, z9 nspec (int) : number of spectra to simulate verbose (boolean) : if True, print status messages ncpu (int) : number of CPU cores to use in parallel trimxy (boolean) : trim image to just pixels with input signal cosmics (str) : filename with dark images with cosmics to add Reads: $DESI_SPECTRO_SIM/$PIXPROD/{night}/simspec-{expid}.fits Writes: $DESI_SPECTRO_SIM/$PIXPROD/{night}/simpix-{camera}-{expid}.fits $DESI_SPECTRO_SIM/$PIXPROD/{night}/pix-{camera}-{expid}.fits """ if verbose: print "Reading input files" channel = camera[0].lower() ispec = int(camera[1]) assert channel in 'brz' assert 0 <= ispec < 10 #- Load DESI parameters params = desimodel.io.load_desiparams() nfibers = params['spectro']['nfibers'] #- Load simspec file simfile = io.findfile('simspec', night=night, expid=expid) simspec = io.read_simspec(simfile) wave = simspec.wave[channel] if simspec.skyphot is not None: phot = simspec.phot[channel] + simspec.skyphot[channel] else: phot = simspec.phot[channel] if ispec * nfibers >= simspec.nspec: print "ERROR: camera {} not in the {} spectra in {}/{}".format( camera, simspec.nspec, night, os.path.basename(simfile)) return #- Load PSF psf = desimodel.io.load_psf(channel) #- Trim to just the spectra for this spectrograph if nspec is None: ii = slice(nfibers * ispec, nfibers * (ispec + 1)) else: ii = slice(nfibers * ispec, nfibers * ispec + nspec) phot = phot[ii] #- check if simulation has less than 500 input spectra if phot.shape[0] < nspec: nspec = phot.shape[0] #- Project to image and append that to file if verbose: print "Projecting photons onto CCD" img = parallel_project(psf, wave, phot, ncpu=ncpu) if trimxy: xmin, xmax, ymin, ymax = psf.xyrange((0, nspec), wave) img = img[0:ymax, 0:xmax] # img = img[ymin:ymax, xmin:xmax] # hdr['CRVAL1'] = xmin+1 # hdr['CRVAL2'] = ymin+1 #- Prepare header hdr = simspec.header tmp = '/'.join(simfile.split('/')[-3:]) #- last 3 elements of path hdr['SIMFILE'] = (tmp, 'Input simulation file') #- Strip unnecessary keywords for key in ('EXTNAME', 'LOGLAM', 'AIRORVAC', 'CRVAL1', 'CDELT1'): if key in hdr: del hdr[key] #- Write noiseless output simpixfile = io.findfile('simpix', night=night, expid=expid, camera=camera) io.write_simpix(simpixfile, img, meta=hdr) #- Add cosmics from library of dark images #- in this case, don't add readnoise since the dark image already has it if cosmics is not None: cosmics = io.read_cosmics(cosmics, expid, shape=img.shape) pix = np.random.poisson(img) + cosmics.pix readnoise = cosmics.meta['RDNOISE'] ivar = 1.0 / (pix.clip(0) + readnoise**2) mask = cosmics.mask #- Or just add noise else: params = desimodel.io.load_desiparams() channel = camera[0].lower() readnoise = params['ccd'][channel]['readnoise'] pix = np.random.poisson(img) + np.random.normal(scale=readnoise, size=img.shape) ivar = 1.0 / (pix.clip(0) + readnoise**2) mask = np.zeros(img.shape, dtype=np.uint16) #- Metadata to be included in pix file header is in the fibermap header #- TODO: this is fragile; consider updating fibermap to use astropy Table #- that includes the header rather than directly assuming FITS as the #- underlying format. fibermapfile = desispec.io.findfile('fibermap', night=night, expid=expid) fm, fmhdr = desispec.io.read_fibermap(fibermapfile, header=True) meta = dict() try: meta['TELRA'] = simspec.header['TELRA'] meta['TELDEC'] = simspec.header['TELDEC'] except KeyError: #- temporary backwards compatibilty meta['TELRA'] = fmhdr['TELERA'] meta['TELDEC'] = fmhdr['TELEDEC'] meta['TILEID'] = simspec.header['TILEID'] meta['DATE-OBS'] = simspec.header['DATE-OBS'] meta['FLAVOR'] = simspec.header['FLAVOR'] meta['EXPTIME'] = simspec.header['EXPTIME'] meta['AIRMASS'] = simspec.header['AIRMASS'] image = Image(pix, ivar, mask, readnoise=readnoise, camera=camera, meta=meta) pixfile = desispec.io.findfile('pix', night=night, camera=camera, expid=expid) desispec.io.write_image(pixfile, image) if verbose: print "Wrote " + pixfile
def simulate(camera, simspec, psf, nspec=None, ncpu=None, cosmics=None, wavemin=None, wavemax=None, preproc=True, comm=None): """Run pixel-level simulation of input spectra Args: camera (string) : b0, r1, .. z9 simspec : desispec.io.SimSpec object from desispec.io.read_simspec() psf : subclass of specter.psf.psf.PSF, e.g. from desimodel.io.load_psf() Options: nspec (int): number of spectra to simulate ncpu (int): number of CPU cores to use in parallel cosmics (desispec.image.Image): e.g. from desisim.io.read_cosmics() wavemin (float): minimum wavelength range to simulate wavemax (float): maximum wavelength range to simulate preproc (boolean, optional) : also preprocess raw data (default True) Returns: (image, rawpix, truepix) tuple, where image is the preproc Image object (only header is meaningful if preproc=False), rawpix is a 2D ndarray of unprocessed raw pixel data, and truepix is a 2D ndarray of truth for image.pix """ if (comm is None) or (comm.rank == 0): log.info('Starting pixsim.simulate camera {} at {}'.format(camera, asctime())) #- parse camera name into channel and spectrograph number channel = camera[0].lower() ispec = int(camera[1]) assert channel in 'brz', \ 'unrecognized channel {} camera {}'.format(channel, camera) assert 0 <= ispec < 10, \ 'unrecognized spectrograph {} camera {}'.format(ispec, camera) assert len(camera) == 2, \ 'unrecognized camera {}'.format(camera) #- Load DESI parameters params = desimodel.io.load_desiparams() #- this is not necessarily true, the truth in is the fibermap nfibers = params['spectro']['nfibers'] phot = simspec.cameras[camera].phot if simspec.cameras[camera].skyphot is not None: phot += simspec.cameras[camera].skyphot if nspec is not None: phot = phot[0:nspec] else: nspec = phot.shape[0] #- Trim wavelengths if needed wave = simspec.cameras[camera].wave if wavemin is not None: ii = (wave >= wavemin) phot = phot[:, ii] wave = wave[ii] if wavemax is not None: ii = (wave <= wavemax) phot = phot[:, ii] wave = wave[ii] #- Project to image and append that to file if (comm is None) or (comm.rank == 0): log.info('Starting {} projection at {}'.format(camera, asctime())) # The returned true pixel values will only exist on rank 0 in the # MPI case. Otherwise it will be None. truepix = parallel_project(psf, wave, phot, ncpu=ncpu, comm=comm) if (comm is None) or (comm.rank == 0): log.info('Finished {} projection at {}'.format(camera, asctime())) image = None rawpix = None if (comm is None) or (comm.rank == 0): #- Start metadata header header = simspec.header.copy() header['CAMERA'] = camera header['DOSVER'] = 'SIM' header['FEEVER'] = 'SIM' header['DETECTOR'] = 'SIM' #- Add cosmics from library of dark images ny = truepix.shape[0] // 2 nx = truepix.shape[1] // 2 if cosmics is not None: # set to zeros values with mask bit 0 (= dead column or hot pixels) cosmics_pix = cosmics.pix*((cosmics.mask&1)==0) pix = np.random.poisson(truepix) + cosmics_pix try: #- cosmics templates >= v0.3 rdnoiseA = cosmics.meta['OBSRDNA'] rdnoiseB = cosmics.meta['OBSRDNB'] rdnoiseC = cosmics.meta['OBSRDNC'] rdnoiseD = cosmics.meta['OBSRDND'] except KeyError: #- cosmics templates <= v0.2 print(cosmic.meta) rdnoiseA = cosmics.meta['RDNOISE0'] rdnoiseB = cosmics.meta['RDNOISE1'] rdnoiseC = cosmics.meta['RDNOISE2'] rdnoiseD = cosmics.meta['RDNOISE3'] else: pix = truepix readnoise = params['ccd'][channel]['readnoise'] rdnoiseA = rdnoiseB = rdnoiseC = rdnoiseD = readnoise #- data already has noise if cosmics were added noisydata = (cosmics is not None) #- Split by amplifier and expand into raw data nprescan = params['ccd'][channel]['prescanpixels'] if 'overscanpixels' in params['ccd'][channel]: noverscan = params['ccd'][channel]['overscanpixels'] else: noverscan = 50 #- Reproducibly random overscan bias level offsets across diff exp assert channel in 'brz' if channel == 'b': irand = ispec elif channel == 'r': irand = 10 + ispec elif channel == 'z': irand = 20 + ispec seeds = np.random.RandomState(0).randint(2**32-1, size=30) rand = np.random.RandomState(seeds[irand]) nyraw = ny nxraw = nx + nprescan + noverscan rawpix = np.empty( (nyraw*2, nxraw*2), dtype=np.int32 ) gain = params['ccd'][channel]['gain'] #- Amp A/1 Lower Left rawpix[0:nyraw, 0:nxraw] = \ photpix2raw(pix[0:ny, 0:nx], gain, rdnoiseA, readorder='lr', nprescan=nprescan, noverscan=noverscan, offset=rand.uniform(100, 200), noisydata=noisydata) #- Amp B/2 Lower Right rawpix[0:nyraw, nxraw:nxraw+nxraw] = \ photpix2raw(pix[0:ny, nx:nx+nx], gain, rdnoiseB, readorder='rl', nprescan=nprescan, noverscan=noverscan, offset=rand.uniform(100, 200), noisydata=noisydata) #- Amp C/3 Upper Left rawpix[nyraw:nyraw+nyraw, 0:nxraw] = \ photpix2raw(pix[ny:ny+ny, 0:nx], gain, rdnoiseC, readorder='lr', nprescan=nprescan, noverscan=noverscan, offset=rand.uniform(100, 200), noisydata=noisydata) #- Amp D/4 Upper Right rawpix[nyraw:nyraw+nyraw, nxraw:nxraw+nxraw] = \ photpix2raw(pix[ny:ny+ny, nx:nx+nx], gain, rdnoiseD, readorder='rl', nprescan=nprescan, noverscan=noverscan, offset=rand.uniform(100, 200), noisydata=noisydata) def xyslice2header(xyslice): ''' convert 2D slice into IRAF style [a:b,c:d] header value e.g. xyslice2header(np.s_[0:10, 5:20]) -> '[6:20,1:10]' ''' yy, xx = xyslice value = '[{}:{},{}:{}]'.format(xx.start+1, xx.stop, yy.start+1, yy.stop) return value #- Amp order from DESI-1964 (previously 1-4 instead of A-D) #- C D #- A B xoffset = nprescan+nx+noverscan header['PRESECA'] = xyslice2header(np.s_[0:nyraw, 0:0+nprescan]) header['DATASECA'] = xyslice2header(np.s_[0:nyraw, nprescan:nprescan+nx]) header['BIASSECA'] = xyslice2header(np.s_[0:nyraw, nprescan+nx:nprescan+nx+noverscan]) header['CCDSECA'] = xyslice2header(np.s_[0:ny, 0:nx]) header['PRESECB'] = xyslice2header(np.s_[0:nyraw, xoffset+noverscan+nx:xoffset+noverscan+nx+nprescan]) header['DATASECB'] = xyslice2header(np.s_[0:nyraw, xoffset+noverscan:xoffset+noverscan+nx]) header['BIASSECB'] = xyslice2header(np.s_[0:nyraw, xoffset:xoffset+noverscan]) header['CCDSECB'] = xyslice2header(np.s_[0:ny, nx:2*nx]) header['PRESECC'] = xyslice2header(np.s_[nyraw:2*nyraw, 0:0+nprescan]) header['DATASECC'] = xyslice2header(np.s_[nyraw:2*nyraw, nprescan:nprescan+nx]) header['BIASSECC'] = xyslice2header(np.s_[nyraw:2*nyraw, nprescan+nx:nprescan+nx+noverscan]) header['CCDSECC'] = xyslice2header(np.s_[ny:2*ny, 0:nx]) header['PRESECD'] = xyslice2header(np.s_[nyraw:2*nyraw, xoffset+noverscan+nx:xoffset+noverscan+nx+nprescan]) header['DATASECD'] = xyslice2header(np.s_[nyraw:2*nyraw, xoffset+noverscan:xoffset+noverscan+nx]) header['BIASSECD'] = xyslice2header(np.s_[nyraw:2*nyraw, xoffset:xoffset+noverscan]) header['CCDSECD'] = xyslice2header(np.s_[ny:2*ny, nx:2*nx]) #- Add additional keywords to mimic real raw data header['INSTRUME'] = 'DESI' header['PROCTYPE'] = 'RAW' header['PRODTYPE'] = 'image' header['EXPFRAME'] = 0 header['REQTIME'] = simspec.header['EXPTIME'] header['TIMESYS'] = 'UTC' #- DATE-OBS format YEAR-MM-DDThh:mm:ss.sss -> OBSID kpnoYEARMMDDthhmmss header['OBSID']='kp4m'+header['DATE-OBS'][0:19].replace('-','').replace(':','').lower() header['TIME-OBS'] = header['DATE-OBS'].split('T')[1] header['DELTARA'] = 0.0 header['DELTADEC'] = 0.0 header['SPECGRPH'] = ispec header['CCDNAME'] = 'CCDS' + str(ispec) + str(channel).upper() header['CCDPREP'] = 'purge,clear' header['CCDSIZE'] = str(rawpix.shape) header['CCDTEMP'] = 850.0 header['CPUTEMP'] = 63.7 header['CASETEMP'] = 62.8 header['CCDTMING'] = 'sim_timing.txt' header['CCDCFG'] = 'sim.cfg' header['SETTINGS'] = 'sim_detectors.json' header['VESSEL'] = 7 #- I don't know what this is header['FEEBOX'] = 'sim097' header['PGAGAIN'] = 5 header['OCSVER'] = 'SIM' header['CONSTVER'] = 'SIM' header['BLDTIME'] = 0.35 header['DIGITIME'] = 61.9 #- Remove some spurious header keywords from upstream if 'BUNIT' in header and header['BUNIT'] == 'Angstrom': del header['BUNIT'] if 'MJD' in header and 'MJD-OBS' not in header: header['MJD-OBS'] = header['MJD'] del header['MJD'] for key in ['RA', 'DEC']: if key in header: del header[key] #- Drive MJD-OBS from DATE-OBS if needed if 'MJD-OBS' not in header: header['MJD-OBS'] = Time(header['DATE-OBS']).mjd #- from http://www-kpno.kpno.noao.edu/kpno-misc/mayall_params.html kpno_longitude = -(111. + 35/60. + 59.6/3600) * u.deg #- Convert DATE-OBS to sexigesimal (sigh) Local Sidereal Time #- Use mean ST as close enough for sims to avoid nutation calc t = Time(header['DATE-OBS']) st = t.sidereal_time('mean', kpno_longitude).to('deg').value hour = st/15 minute = (hour % 1)*60 second = (minute % 1)*60 header['ST'] = '{:02d}:{:02d}:{:0.3f}'.format( int(hour), int(minute), second) if preproc: log.debug('Running preprocessing at {}'.format(asctime())) image = desispec.preproc.preproc(rawpix, header, primary_header=simspec.header) else: log.debug('Skipping preprocessing') image = Image(np.zeros(truepix.shape), np.zeros(truepix.shape), meta=header) if (comm is None) or (comm.rank == 0): log.info('Finished pixsim.simulate for camera {} at {}'.format(camera, asctime())) return image, rawpix, truepix
def test_camera(self): image = Image(self.pix, self.ivar) self.assertEqual(image.camera, 'unknown') image = Image(self.pix, self.ivar, camera='b0') self.assertEqual(image.camera, 'b0')
def test_init(self): image = Image(self.pix, self.ivar) self.assertTrue(np.all(image.pix == self.pix)) self.assertTrue(np.all(image.ivar == self.ivar))
def read_cosmics(filename, expid=1, shape=None, jitter=True): """ Reads a dark image with cosmics from the input filename. The input might have multiple dark images; use the `expid%n` image where `n` is the number of images in the input cosmics file. Args: filename : FITS filename with EXTNAME=IMAGE-*, IVAR-*, MASK-* HDUs expid : integer, use `expid % n` image where `n` is number of images shape : (ny, nx, optional) tuple for output image shape jitter (bool, optional): If True (default), apply random flips and rolls so you don't get the exact same cosmics every time Returns: `desisim.image.Image` object with attributes pix, ivar, mask """ fx = fits.open(filename) imagekeys = list() for i in range(len(fx)): if fx[i].name.startswith('IMAGE-'): imagekeys.append(fx[i].name.split('-', 1)[1]) assert len(imagekeys) > 0, 'No IMAGE-* extensions found in ' + filename i = expid % len(imagekeys) pix = native_endian(fx['IMAGE-' + imagekeys[i]].data.astype(np.float64)) ivar = native_endian(fx['IVAR-' + imagekeys[i]].data.astype(np.float64)) mask = native_endian(fx['MASK-' + imagekeys[i]].data) meta = fx['IMAGE-' + imagekeys[i]].header meta['CRIMAGE'] = (imagekeys[i], 'input cosmic ray image') #- De-trend each amplifier nx = pix.shape[1] // 2 ny = pix.shape[0] // 2 kernel_size = min(201, ny // 3, nx // 3) pix[0:ny, 0:nx] -= spline_medfilt2d(pix[0:ny, 0:nx], kernel_size) pix[0:ny, nx:2 * nx] -= spline_medfilt2d(pix[0:ny, nx:2 * nx], kernel_size) pix[ny:2 * ny, 0:nx] -= spline_medfilt2d(pix[ny:2 * ny, 0:nx], kernel_size) pix[ny:2 * ny, nx:2 * nx] -= spline_medfilt2d(pix[ny:2 * ny, nx:2 * nx], kernel_size) if shape is not None: if len(shape) != 2: raise ValueError('Invalid shape {}'.format(shape)) pix = _resize(pix, shape) ivar = _resize(ivar, shape) mask = _resize(mask, shape) if jitter: #- Randomly flip left-right and/or up-down if np.random.uniform(0, 1) > 0.5: pix = np.fliplr(pix) ivar = np.fliplr(ivar) mask = np.fliplr(mask) meta['CRFLIPLR'] = (True, 'Input cosmics image flipped Left/Right') else: meta['CRFLIPLR'] = (False, 'Input cosmics image NOT flipped Left/Right') if np.random.uniform(0, 1) > 0.5: pix = np.flipud(pix) ivar = np.flipud(ivar) mask = np.flipud(mask) meta['CRFLIPUD'] = (True, 'Input cosmics image flipped Up/Down') else: meta['CRFLIPUD'] = (False, 'Input cosmics image NOT flipped Up/Down') #- Randomly roll image a bit nx, ny = np.random.randint(-100, 100, size=2) pix = np.roll(np.roll(pix, ny, axis=0), nx, axis=1) ivar = np.roll(np.roll(ivar, ny, axis=0), nx, axis=1) mask = np.roll(np.roll(mask, ny, axis=0), nx, axis=1) meta['CRSHIFTX'] = (nx, 'Input cosmics image shift in x') meta['CRSHIFTY'] = (nx, 'Input cosmics image shift in y') else: meta['CRFLIPLR'] = (False, 'Input cosmics image NOT flipped Left/Right') meta['CRFLIPUD'] = (False, 'Input cosmics image NOT flipped Up/Down') meta['CRSHIFTX'] = (0, 'Input cosmics image shift in x') meta['CRSHIFTY'] = (0, 'Input cosmics image shift in y') del meta['RDNOISE0'] #- Amp 1 lower left nx = pix.shape[1] // 2 ny = pix.shape[0] // 2 iixy = np.s_[0:ny, 0:nx] cx = pix[iixy][mask[iixy] == 0] mean, median, std = sigma_clipped_stats(cx, sigma=3, iters=5) meta['RDNOISE1'] = std #- Amp 2 lower right iixy = np.s_[0:ny, nx:2 * nx] cx = pix[iixy][mask[iixy] == 0] mean, median, std = sigma_clipped_stats(cx, sigma=3, iters=5) meta['RDNOISE2'] = std #- Amp 3 upper left iixy = np.s_[ny:2 * ny, 0:nx] mean, median, std = sigma_clipped_stats(pix[iixy], sigma=3, iters=5) meta['RDNOISE3'] = std #- Amp 4 upper right iixy = np.s_[ny:2 * ny, nx:2 * nx] mean, median, std = sigma_clipped_stats(pix[iixy], sigma=3, iters=5) meta['RDNOISE4'] = std fx.close() return Image(pix, ivar, mask, meta=meta)
def preproc(rawimage, header, primary_header, bias=True, dark=True, pixflat=True, mask=True, bkgsub=False, nocosmic=False, cosmics_nsig=6, cosmics_cfudge=3., cosmics_c2fudge=0.5, ccd_calibration_filename=None, nocrosstalk=False, nogain=False, overscan_per_row=False, use_overscan_row=False, use_savgol=None, nodarktrail=False, remove_scattered_light=False, psf_filename=None, bias_img=None): ''' preprocess image using metadata in header image = ((rawimage-bias-overscan)*gain)/pixflat Args: rawimage : 2D numpy array directly from raw data file header : dict-like metadata, e.g. from FITS header, with keywords CAMERA, BIASSECx, DATASECx, CCDSECx where x = A, B, C, D for each of the 4 amplifiers (also supports old naming convention 1, 2, 3, 4). primary_header: dict-like metadata fit keywords EXPTIME, DOSVER DATE-OBS is also required if bias, pixflat, or mask=True Optional bias, pixflat, and mask can each be: False: don't apply that step True: use default calibration data for that night ndarray: use that array filename (str or unicode): read HDU 0 and use that Optional overscan features: overscan_per_row : bool, Subtract the overscan_col values row by row from the data. use_overscan_row : bool, Subtract off the overscan_row from the data (default: False). Requires ORSEC in the Header use_savgol : bool, Specify whether to use Savitsky-Golay filter for the overscan. (default: False). Requires use_overscan_row=True to have any effect. Optional background subtraction with median filtering if bkgsub=True Optional disabling of cosmic ray rejection if nocosmic=True Optional disabling of dark trail correction if nodarktrail=True Optional bias image (testing only) may be provided by bias_img= Optional tuning of cosmic ray rejection parameters: cosmics_nsig: number of sigma above background required cosmics_cfudge: number of sigma inconsistent with PSF required cosmics_c2fudge: fudge factor applied to PSF Optional fit and subtraction of scattered light Returns Image object with member variables: pix : 2D preprocessed image in units of electrons per pixel ivar : 2D inverse variance of image mask : 2D mask of image (0=good) readnoise : 2D per-pixel readnoise of image meta : metadata dictionary TODO: define what keywords are included preprocessing includes the following steps: - bias image subtraction - overscan subtraction (from BIASSEC* keyword defined regions) - readnoise estimation (from BIASSEC* keyword defined regions) - gain correction (from GAIN* keywords) - pixel flat correction - cosmic ray masking - propagation of input known bad pixel mask - inverse variance estimation Notes: The bias image is subtracted before any other calculation to remove any non-uniformities in the overscan regions prior to calculating overscan levels and readnoise. The readnoise is an image not just one number per amp, because the pixflat image also affects the interpreted readnoise. The inverse variance is estimated from the readnoise and the image itself, and thus is biased. ''' log = get_logger() header = header.copy() cfinder = None if ccd_calibration_filename is not False: cfinder = CalibFinder([header, primary_header], yaml_file=ccd_calibration_filename) #- TODO: Check for required keywords first #- Subtract bias image camera = header['CAMERA'].lower() #- convert rawimage to float64 : this is the output format of read_image rawimage = rawimage.astype(np.float64) # Savgol if cfinder and cfinder.haskey("USE_ORSEC"): use_overscan_row = cfinder.value("USE_ORSEC") if cfinder and cfinder.haskey("SAVGOL"): use_savgol = cfinder.value("SAVGOL") # Set bias image, as desired if bias_img is None: bias = get_calibration_image(cfinder, "BIAS", bias) else: bias = bias_img if bias is not False: #- it's an array if bias.shape == rawimage.shape: log.info("subtracting bias") rawimage = rawimage - bias else: raise ValueError('shape mismatch bias {} != rawimage {}'.format( bias.shape, rawimage.shape)) #- Check if this file uses amp names 1,2,3,4 (old) or A,B,C,D (new) amp_ids = get_amp_ids(header) #- Double check that we have the necessary keywords missing_keywords = list() for prefix in ['CCDSEC', 'BIASSEC']: for amp in amp_ids: key = prefix + amp if not key in header: log.error('No {} keyword in header'.format(key)) missing_keywords.append(key) if len(missing_keywords) > 0: raise KeyError("Missing keywords {}".format( ' '.join(missing_keywords))) #- Output arrays ny = 0 nx = 0 for amp in amp_ids: yy, xx = parse_sec_keyword(header['CCDSEC%s' % amp]) ny = max(ny, yy.stop) nx = max(nx, xx.stop) image = np.zeros((ny, nx)) readnoise = np.zeros_like(image) #- Load mask mask = get_calibration_image(cfinder, "MASK", mask) if mask is False: mask = np.zeros(image.shape, dtype=np.int32) else: if mask.shape != image.shape: raise ValueError('shape mismatch mask {} != image {}'.format( mask.shape, image.shape)) #- Load dark dark = get_calibration_image(cfinder, "DARK", dark) if dark is not False: if dark.shape != image.shape: log.error('shape mismatch dark {} != image {}'.format( dark.shape, image.shape)) raise ValueError('shape mismatch dark {} != image {}'.format( dark.shape, image.shape)) if cfinder and cfinder.haskey("EXPTIMEKEY"): exptime_key = cfinder.value("EXPTIMEKEY") log.info("Using exposure time keyword %s for dark normalization" % exptime_key) else: exptime_key = "EXPTIME" exptime = primary_header[exptime_key] log.info("Multiplying dark by exptime %f" % (exptime)) dark *= exptime for amp in amp_ids: # Grab the sections ov_col = parse_sec_keyword(header['BIASSEC' + amp]) if 'ORSEC' + amp in header.keys(): ov_row = parse_sec_keyword(header['ORSEC' + amp]) elif use_overscan_row: log.error('No ORSEC{} keyword; not using overscan_row'.format(amp)) use_overscan_row = False if nogain: gain = 1. else: #- Initial teststand data may be missing GAIN* keywords; don't crash if 'GAIN' + amp in header: gain = header['GAIN' + amp] #- gain = electrons / ADU else: if cfinder and cfinder.haskey('GAIN' + amp): gain = float(cfinder.value('GAIN' + amp)) log.info('Using GAIN{}={} from calibration data'.format( amp, gain)) else: gain = 1.0 log.warning( 'Missing keyword GAIN{} in header and nothing in calib data; using {}' .format(amp, gain)) #- Add saturation level if 'SATURLEV' + amp in header: saturlev = header['SATURLEV' + amp] # in electrons else: if cfinder and cfinder.haskey('SATURLEV' + amp): saturlev = float(cfinder.value('SATURLEV' + amp)) log.info('Using SATURLEV{}={} from calibration data'.format( amp, saturlev)) else: saturlev = 200000 log.warning( 'Missing keyword SATURLEV{} in header and nothing in calib data; using 200000' .format(amp, saturlev)) # Generate the overscan images raw_overscan_col = rawimage[ov_col].copy() if use_overscan_row: raw_overscan_row = rawimage[ov_row].copy() overscan_row = np.zeros_like(raw_overscan_row) # Remove overscan_col from overscan_row raw_overscan_squared = rawimage[ov_row[0], ov_col[1]].copy() for row in range(raw_overscan_row.shape[0]): o, r = _overscan(raw_overscan_squared[row]) overscan_row[row] = raw_overscan_row[row] - o # Now remove the overscan_col nrows = raw_overscan_col.shape[0] log.info("nrows in overscan=%d" % nrows) overscan_col = np.zeros(nrows) rdnoise = np.zeros(nrows) if (cfinder and cfinder.haskey('OVERSCAN' + amp) and cfinder.value("OVERSCAN" + amp).upper() == "PER_ROW") or overscan_per_row: log.info( "Subtracting overscan per row for amplifier %s of camera %s" % (amp, camera)) for j in range(nrows): if np.isnan(np.sum(overscan_col[j])): log.warning( "NaN values in row %d of overscan of amplifier %s of camera %s" % (j, amp, camera)) continue o, r = _overscan(raw_overscan_col[j]) #log.info("%d %f %f"%(j,o,r)) overscan_col[j] = o rdnoise[j] = r else: log.info( "Subtracting average overscan for amplifier %s of camera %s" % (amp, camera)) o, r = _overscan(raw_overscan_col) overscan_col += o rdnoise += r rdnoise *= gain median_rdnoise = np.median(rdnoise) median_overscan = np.median(overscan_col) log.info("Median rdnoise and overscan= %f %f" % (median_rdnoise, median_overscan)) kk = parse_sec_keyword(header['CCDSEC' + amp]) for j in range(nrows): readnoise[kk][j] = rdnoise[j] header['OVERSCN' + amp] = (median_overscan, 'ADUs (gain not applied)') if gain != 1: rdnoise_message = 'electrons (gain is applied)' gain_message = 'e/ADU (gain applied to image)' else: rdnoise_message = 'ADUs (gain not applied)' gain_message = 'gain not applied to image' header['OBSRDN' + amp] = (median_rdnoise, rdnoise_message) header['GAIN' + amp] = (gain, gain_message) #- Warn/error if measured readnoise is very different from expected if exists if 'RDNOISE' + amp in header: expected_readnoise = header['RDNOISE' + amp] if median_rdnoise < 0.5 * expected_readnoise: log.error( 'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}' .format(amp, median_rdnoise, expected_readnoise)) elif median_rdnoise < 0.9 * expected_readnoise: log.warning( 'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}' .format(amp, median_rdnoise, expected_readnoise)) elif median_rdnoise > 2.0 * expected_readnoise: log.error( 'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}' .format(amp, median_rdnoise, expected_readnoise)) elif median_rdnoise > 1.2 * expected_readnoise: log.warning( 'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}' .format(amp, median_rdnoise, expected_readnoise)) #else: # log.warning('Expected readnoise keyword {} missing'.format('RDNOISE'+amp)) log.info("Measured readnoise for AMP %s = %f" % (amp, median_rdnoise)) #- subtract overscan from data region and apply gain jj = parse_sec_keyword(header['DATASEC' + amp]) data = rawimage[jj].copy() # Subtract columns for k in range(nrows): data[k] -= overscan_col[k] # And now the rows if use_overscan_row: # Savgol? if use_savgol: log.info("Using savgol") collapse_oscan_row = np.zeros(overscan_row.shape[1]) for col in range(overscan_row.shape[1]): o, _ = _overscan(overscan_row[:, col]) collapse_oscan_row[col] = o oscan_row = _savgol_clipped(collapse_oscan_row, niter=0) oimg_row = np.outer(np.ones(data.shape[0]), oscan_row) data -= oimg_row else: o, r = _overscan(overscan_row) data -= o #- apply saturlev (defined in ADU), prior to multiplication by gain saturated = (rawimage[jj] >= saturlev) mask[kk][saturated] |= ccdmask.SATURATED #- ADC to electrons image[kk] = data * gain if not nocrosstalk: #- apply cross-talk # the ccd looks like : # C D # A B # for cross talk, we need a symmetric 4x4 flip_matrix # of coordinates ABCD giving flip of both axis # when computing crosstalk of # A B C D # # A AA AB AC AD # B BA BB BC BD # C CA CB CC CD # D DA DB DC BB # orientation_matrix_defines change of orientation # fip_axis_0 = np.array([[1, 1, -1, -1], [1, 1, -1, -1], [-1, -1, 1, 1], [-1, -1, 1, 1]]) fip_axis_1 = np.array([[1, -1, 1, -1], [-1, 1, -1, 1], [1, -1, 1, -1], [-1, 1, -1, 1]]) for a1 in range(len(amp_ids)): amp1 = amp_ids[a1] ii1 = parse_sec_keyword(header['CCDSEC' + amp1]) a1flux = image[ii1] #a1mask=mask[ii1] for a2 in range(len(amp_ids)): if a1 == a2: continue amp2 = amp_ids[a2] if cfinder is None: continue if not cfinder.haskey("CROSSTALK%s%s" % (amp1, amp2)): continue crosstalk = cfinder.value("CROSSTALK%s%s" % (amp1, amp2)) if crosstalk == 0.: continue log.info("Correct for crosstalk=%f from AMP %s into %s" % (crosstalk, amp1, amp2)) a12flux = crosstalk * a1flux.copy() #a12mask=a1mask.copy() if fip_axis_0[a1, a2] == -1: a12flux = a12flux[::-1] #a12mask=a12mask[::-1] if fip_axis_1[a1, a2] == -1: a12flux = a12flux[:, ::-1] #a12mask=a12mask[:,::-1] ii2 = parse_sec_keyword(header['CCDSEC' + amp2]) image[ii2] -= a12flux # mask[ii2] |= a12mask (not sure we really need to propagate the mask) #- Poisson noise variance (prior to dark subtraction and prior to pixel flat field) #- This is biasing, but that's what we have for now poisson_var = image.clip(0) #- subtract dark after multiplication by gain if dark is not False: log.info("subtracting dark for amp %s" % amp) image -= dark #- Correct for dark trails if any if not nodarktrail and cfinder is not None: for amp in amp_ids: if cfinder.haskey("DARKTRAILAMP%s" % amp): amplitude = cfinder.value("DARKTRAILAMP%s" % amp) width = cfinder.value("DARKTRAILWIDTH%s" % amp) ii = _parse_sec_keyword(header["CCDSEC" + amp]) log.info( "Removing dark trails for amplifier %s with width=%3.1f and amplitude=%5.4f" % (amp, width, amplitude)) correct_dark_trail(image, ii, left=((amp == "B") | (amp == "D")), width=width, amplitude=amplitude) #- Divide by pixflat image pixflat = get_calibration_image(cfinder, "PIXFLAT", pixflat) if pixflat is not False: if pixflat.shape != image.shape: raise ValueError('shape mismatch pixflat {} != image {}'.format( pixflat.shape, image.shape)) almost_zero = 0.001 if np.all(pixflat > almost_zero): image /= pixflat readnoise /= pixflat poisson_var /= pixflat**2 else: good = (pixflat > almost_zero) image[good] /= pixflat[good] readnoise[good] /= pixflat[good] poisson_var[good] /= pixflat[good]**2 mask[~good] |= ccdmask.PIXFLATZERO lowpixflat = (0 < pixflat) & (pixflat < 0.1) if np.any(lowpixflat): mask[lowpixflat] |= ccdmask.PIXFLATLOW #- Inverse variance, estimated directly from the data (BEWARE: biased!) var = poisson_var + readnoise**2 ivar = np.zeros(var.shape) ivar[var > 0] = 1.0 / var[var > 0] #- Ridiculously high readnoise is bad mask[readnoise > 100] |= ccdmask.BADREADNOISE if bkgsub: bkg = _background(image, header) image -= bkg img = Image(image, ivar=ivar, mask=mask, meta=header, readnoise=readnoise, camera=camera) #- update img.mask to mask cosmic rays if not nocosmic: cosmics.reject_cosmic_rays(img, nsig=cosmics_nsig, cfudge=cosmics_cfudge, c2fudge=cosmics_c2fudge) if remove_scattered_light: if psf_filename is None: psf_filename = cfinder.findfile("PSF") xyset = read_xytraceset(psf_filename) img.pix -= model_scattered_light(img, xyset) return img
def test_slicing(self): meta = dict(blat='foo', NAXIS1=self.pix.shape[1], NAXIS2=self.pix.shape[0]) img1 = Image(self.pix, self.ivar, meta=meta) nx, ny = 2, 3 img2 = img1[0:ny, 0:nx] self.assertEqual(img2.pix.shape[0], ny) self.assertEqual(img2.pix.shape[1], nx) self.assertTrue(img2.pix.shape == img2.ivar.shape) self.assertTrue(img2.pix.shape == img2.mask.shape) self.assertEqual(img1.camera, img2.camera) self.assertEqual(img1.readnoise, img2.readnoise) for key in img1.meta: if key != 'NAXIS1' and key != 'NAXIS2': self.assertEqual(img1.meta[key], img2.meta[key]) self.assertFalse(img1.meta is img2.meta) self.assertEqual(img2.meta['NAXIS1'], nx) self.assertEqual(img2.meta['NAXIS2'], ny) #- also works for non-None mask and meta=None img1 = Image(self.pix, self.ivar, mask=(self.ivar == 0)) nx, ny = 2, 3 img2 = img1[0:ny, 0:nx] self.assertEqual(img2.pix.shape[0], ny) self.assertEqual(img2.pix.shape[1], nx) self.assertTrue(img2.pix.shape == img2.ivar.shape) self.assertTrue(img2.pix.shape == img2.mask.shape) #- Slice and dice multiple ways, getting meta NAXIS1/NAXIS2 correct img1 = Image(self.pix, self.ivar, meta=meta) img2 = img1[0:ny] self.assertEqual(img2.pix.shape[0], ny) self.assertEqual(img2.pix.shape[1], img1.pix.shape[1]) self.assertEqual(img2.pix.shape[0], img2.meta['NAXIS2']) self.assertEqual(img2.pix.shape[1], img2.meta['NAXIS1']) img2 = img1[0:ny, :] self.assertEqual(img2.pix.shape[0], ny) self.assertEqual(img2.pix.shape[1], img1.pix.shape[1]) self.assertEqual(img2.pix.shape[0], img2.meta['NAXIS2']) self.assertEqual(img2.pix.shape[1], img2.meta['NAXIS1']) img2 = img1[:, 0:nx] self.assertEqual(img2.pix.shape[0], img1.pix.shape[0]) self.assertEqual(img2.pix.shape[1], nx) self.assertEqual(img2.pix.shape[0], img2.meta['NAXIS2']) self.assertEqual(img2.pix.shape[1], img2.meta['NAXIS1']) #- test slicing readnoise image readnoise = np.random.uniform(size=self.pix.shape) img1 = Image(self.pix, self.ivar, meta=meta, readnoise=readnoise) xy = np.s_[1:1 + ny, 2:2 + nx] img2 = img1[xy] self.assertTrue(np.all(img1.pix[xy] == img2.pix)) self.assertTrue(np.all(img1.ivar[xy] == img2.ivar)) self.assertTrue(np.all(img1.mask[xy] == img2.mask)) self.assertTrue(np.all(img1.readnoise[xy] == img2.readnoise)) #- Test bad slicing with self.assertRaises(ValueError): img1['blat'] with self.assertRaises(ValueError): img1[1:2, 3:4, 5:6] with self.assertRaises(ValueError): img1[1:2, 'blat'] with self.assertRaises(ValueError): img1[None, 1:2]
def simulate(camera, simspec, psf, nspec=None, ncpu=None, cosmics=None, wavemin=None, wavemax=None, preproc=True, comm=None): """Run pixel-level simulation of input spectra Args: camera (string) : b0, r1, .. z9 simspec : desispec.io.SimSpec object from desispec.io.read_simspec() psf : subclass of specter.psf.psf.PSF, e.g. from desimodel.io.load_psf() Options: nspec (int): number of spectra to simulate ncpu (int): number of CPU cores to use in parallel cosmics (desispec.image.Image): e.g. from desisim.io.read_cosmics() wavemin (float): minimum wavelength range to simulate wavemax (float): maximum wavelength range to simulate preproc (boolean, optional) : also preprocess raw data (default True) Returns: (image, rawpix, truepix) tuple, where image is the preproc Image object (only header is meaningful if preproc=False), rawpix is a 2D ndarray of unprocessed raw pixel data, and truepix is a 2D ndarray of truth for image.pix """ if (comm is None) or (comm.rank == 0): log.info('Starting pixsim.simulate camera {} at {}'.format( camera, asctime())) #- parse camera name into channel and spectrograph number channel = camera[0].lower() ispec = int(camera[1]) assert channel in 'brz', \ 'unrecognized channel {} camera {}'.format(channel, camera) assert 0 <= ispec < 10, \ 'unrecognized spectrograph {} camera {}'.format(ispec, camera) assert len(camera) == 2, \ 'unrecognized camera {}'.format(camera) #- Load DESI parameters params = desimodel.io.load_desiparams() #- this is not necessarily true, the truth in is the fibermap nfibers = params['spectro']['nfibers'] phot = simspec.cameras[camera].phot if simspec.cameras[camera].skyphot is not None: phot += simspec.cameras[camera].skyphot if nspec is not None: phot = phot[0:nspec] else: nspec = phot.shape[0] #- Trim wavelengths if needed wave = simspec.cameras[camera].wave if wavemin is not None: ii = (wave >= wavemin) phot = phot[:, ii] wave = wave[ii] if wavemax is not None: ii = (wave <= wavemax) phot = phot[:, ii] wave = wave[ii] #- Project to image and append that to file if (comm is None) or (comm.rank == 0): log.info('Starting {} projection at {}'.format(camera, asctime())) # The returned true pixel values will only exist on rank 0 in the # MPI case. Otherwise it will be None. truepix = parallel_project(psf, wave, phot, ncpu=ncpu, comm=comm) if (comm is None) or (comm.rank == 0): log.info('Finished {} projection at {}'.format(camera, asctime())) image = None rawpix = None if (comm is None) or (comm.rank == 0): #- Start metadata header header = simspec.header.copy() header['CAMERA'] = camera header['DOSVER'] = 'SIM' header['FEEVER'] = 'SIM' header['DETECTOR'] = 'SIM' gain = params['ccd'][channel]['gain'] for amp in ('1', '2', '3', '4'): header['GAIN' + amp] = gain #- Add cosmics from library of dark images ny = truepix.shape[0] // 2 nx = truepix.shape[1] // 2 if cosmics is not None: # set to zeros values with mask bit 0 (= dead column or hot pixels) cosmics_pix = cosmics.pix * ((cosmics.mask & 1) == 0) pix = np.random.poisson(truepix) + cosmics_pix header['RDNOISE1'] = cosmics.meta['RDNOISE1'] header['RDNOISE2'] = cosmics.meta['RDNOISE2'] header['RDNOISE3'] = cosmics.meta['RDNOISE3'] header['RDNOISE4'] = cosmics.meta['RDNOISE4'] else: pix = truepix readnoise = params['ccd'][channel]['readnoise'] header['RDNOISE1'] = readnoise header['RDNOISE2'] = readnoise header['RDNOISE3'] = readnoise header['RDNOISE4'] = readnoise # if (comm is None) or (comm.rank == 0): # log.info('RDNOISE1 {}'.format(header['RDNOISE1'])) # log.info('RDNOISE2 {}'.format(header['RDNOISE2'])) # log.info('RDNOISE3 {}'.format(header['RDNOISE3'])) # log.info('RDNOISE4 {}'.format(header['RDNOISE4'])) #- data already has noise if cosmics were added noisydata = (cosmics is not None) #- Split by amplifier and expand into raw data nprescan = params['ccd'][channel]['prescanpixels'] if 'overscanpixels' in params['ccd'][channel]: noverscan = params['ccd'][channel]['overscanpixels'] else: noverscan = 50 nyraw = ny nxraw = nx + nprescan + noverscan rawpix = np.empty((nyraw * 2, nxraw * 2), dtype=np.int32) #- Amp 0 Lower Left rawpix[0:nyraw, 0:nxraw] = \ photpix2raw(pix[0:ny, 0:nx], gain, header['RDNOISE1'], readorder='lr', nprescan=nprescan, noverscan=noverscan, noisydata=noisydata) #- Amp 2 Lower Right rawpix[0:nyraw, nxraw:nxraw+nxraw] = \ photpix2raw(pix[0:ny, nx:nx+nx], gain, header['RDNOISE2'], readorder='rl', nprescan=nprescan, noverscan=noverscan, noisydata=noisydata) #- Amp 3 Upper Left rawpix[nyraw:nyraw+nyraw, 0:nxraw] = \ photpix2raw(pix[ny:ny+ny, 0:nx], gain, header['RDNOISE3'], readorder='lr', nprescan=nprescan, noverscan=noverscan, noisydata=noisydata) #- Amp 4 Upper Right rawpix[nyraw:nyraw+nyraw, nxraw:nxraw+nxraw] = \ photpix2raw(pix[ny:ny+ny, nx:nx+nx], gain, header['RDNOISE4'], readorder='rl', nprescan=nprescan, noverscan=noverscan, noisydata=noisydata) def xyslice2header(xyslice): ''' convert 2D slice into IRAF style [a:b,c:d] header value e.g. xyslice2header(np.s_[0:10, 5:20]) -> '[6:20,1:10]' ''' yy, xx = xyslice value = '[{}:{},{}:{}]'.format(xx.start + 1, xx.stop, yy.start + 1, yy.stop) return value #- Amp order from DESI-1964 #- 3 4 #- 1 2 xoffset = nprescan + nx + noverscan header['PRESEC1'] = xyslice2header(np.s_[0:nyraw, 0:0 + nprescan]) header['DATASEC1'] = xyslice2header(np.s_[0:nyraw, nprescan:nprescan + nx]) header['BIASSEC1'] = xyslice2header( np.s_[0:nyraw, nprescan + nx:nprescan + nx + noverscan]) header['CCDSEC1'] = xyslice2header(np.s_[0:ny, 0:nx]) header['PRESEC2'] = xyslice2header( np.s_[0:nyraw, xoffset + noverscan + nx:xoffset + noverscan + nx + nprescan]) header['DATASEC2'] = xyslice2header( np.s_[0:nyraw, xoffset + noverscan:xoffset + noverscan + nx]) header['BIASSEC2'] = xyslice2header(np.s_[0:nyraw, xoffset:xoffset + noverscan]) header['CCDSEC2'] = xyslice2header(np.s_[0:ny, nx:2 * nx]) header['PRESEC3'] = xyslice2header(np.s_[nyraw:2 * nyraw, 0:0 + nprescan]) header['DATASEC3'] = xyslice2header(np.s_[nyraw:2 * nyraw, nprescan:nprescan + nx]) header['BIASSEC3'] = xyslice2header( np.s_[nyraw:2 * nyraw, nprescan + nx:nprescan + nx + noverscan]) header['CCDSEC3'] = xyslice2header(np.s_[ny:2 * ny, 0:nx]) header['PRESEC4'] = xyslice2header( np.s_[nyraw:2 * nyraw, xoffset + noverscan + nx:xoffset + noverscan + nx + nprescan]) header['DATASEC4'] = xyslice2header( np.s_[nyraw:2 * nyraw, xoffset + noverscan:xoffset + noverscan + nx]) header['BIASSEC4'] = xyslice2header(np.s_[nyraw:2 * nyraw, xoffset:xoffset + noverscan]) header['CCDSEC4'] = xyslice2header(np.s_[ny:2 * ny, nx:2 * nx]) if preproc: log.debug('Running preprocessing at {}'.format(asctime())) image = desispec.preproc.preproc(rawpix, header, primary_header=simspec.header) else: log.debug('Skipping preprocessing') image = Image(np.zeros(rawpix.shape), np.zeros(rawpix.shape), meta=header) if (comm is None) or (comm.rank == 0): log.info('Finished pixsim.simulate for camera {} at {}'.format( camera, asctime())) return image, rawpix, truepix
def test_psf_with_bad_column(self): '''test a PSF-like spot with a masked column going through it''' image = Image(self.psfpix, self.ivar, mask=self.badmask, camera="r0") image.pix[self.badmask>0] = 0 rejected=reject_cosmic_rays_ala_sdss(image,dilate=False) self.assertTrue(not np.any(rejected))
def read_cosmics(filename, expid=1, shape=None, jitter=True): """ Reads a dark image with cosmics from the input filename. The input might have multiple dark images; use the `expid%n` image where `n` is the number of images in the input cosmics file. Args: filename : FITS filename with EXTNAME=IMAGE-*, IVAR-*, MASK-* HDUs expid : integer, use `expid % n` image where `n` is number of images Optional: shape : (ny, nx) tuple for output image shape jitter : If True (default), apply random flips and rolls so you don't get the exact same cosmics every time Returns: `desisim.image.Image` object with attributes pix, ivar, mask """ fx = fits.open(filename) imagekeys = list() for i in range(len(fx)): if fx[i].name.startswith('IMAGE-'): imagekeys.append(fx[i].name.split('-', 1)[1]) assert len(imagekeys) > 0, 'No IMAGE-* extensions found in ' + filename i = expid % len(imagekeys) pix = native_endian(fx['IMAGE-' + imagekeys[i]].data.astype(np.float64)) ivar = native_endian(fx['IVAR-' + imagekeys[i]].data.astype(np.float64)) mask = native_endian(fx['MASK-' + imagekeys[i]].data) meta = fx['IMAGE-' + imagekeys[i]].header meta['CRIMAGE'] = (imagekeys[i], 'input cosmic ray image') if shape is not None: if len(shape) != 2: raise ValueError('Invalid shape {}'.format(shape)) newpix = np.empty(shape, dtype=np.float64) ny = min(shape[0], pix.shape[0]) nx = min(shape[1], pix.shape[1]) newpix[0:ny, 0:nx] = pix[0:ny, 0:nx] pix = _resize(pix, shape) ivar = _resize(ivar, shape) mask = _resize(mask, shape) if jitter: #- Randomly flip left-right and/or up-down if np.random.uniform(0, 1) > 0.5: pix = np.fliplr(pix) ivar = np.fliplr(ivar) mask = np.fliplr(mask) meta['CRFLIPLR'] = (True, 'Input cosmics image flipped Left/Right') else: meta['CRFLIPLR'] = (False, 'Input cosmics image NOT flipped Left/Right') if np.random.uniform(0, 1) > 0.5: pix = np.flipud(pix) ivar = np.flipud(ivar) mask = np.flipud(mask) meta['CRFLIPUD'] = (True, 'Input cosmics image flipped Up/Down') else: meta['CRFLIPUD'] = (True, 'Input cosmics image NOT flipped Up/Down') #- Randomly roll image a bit nx, ny = np.random.randint(-200, 200, size=2) pix = np.roll(np.roll(pix, ny, axis=0), nx, axis=1) ivar = np.roll(np.roll(ivar, ny, axis=0), nx, axis=1) mask = np.roll(np.roll(mask, ny, axis=0), nx, axis=1) meta['CRSHIFTX'] = (nx, 'Input cosmics image shift in x') meta['CRSHIFTY'] = (nx, 'Input cosmics image shift in y') else: meta['CRFLIPLR'] = (False, 'Input cosmics image NOT flipped Left/Right') meta['CRFLIPUD'] = (True, 'Input cosmics image NOT flipped Up/Down') meta['CRSHIFTX'] = (0, 'Input cosmics image shift in x') meta['CRSHIFTY'] = (0, 'Input cosmics image shift in y') #- RDNOISEn -> average RDNOISE if 'RDNOISE' not in meta: x = meta['RDNOISE0'] + meta['RDNOISE1'] + meta['RDNOISE2'] + meta[ 'RDNOISE3'] meta['RDNOISE'] = x / 4.0 return Image(pix, ivar, mask, meta=meta)
def preproc(rawimage, header, bias=False, pixflat=False, mask=False): ''' preprocess image using metadata in header image = ((rawimage-bias-overscan)*gain)/pixflat Args: rawimage : 2D numpy array directly from raw data file header : dict-like metadata, e.g. from FITS header, with keywords CAMERA, BIASSECx, DATASECx, CCDSECx where x = 1, 2, 3, 4 for each of the 4 amplifiers. Optional bias, pixflat, and mask can each be: False: don't apply that step True: use default calibration data for that night ndarray: use that array filename (str or unicode): read HDU 0 and use that DATE-OBS is required in header if bias, pixflat, or mask=True Returns Image object with member variables: image : 2D preprocessed image in units of electrons per pixel ivar : 2D inverse variance of image mask : 2D mask of image (0=good) readnoise : 2D per-pixel readnoise of image meta : metadata dictionary TODO: define what keywords are included preprocessing includes the following steps: - bias image subtraction - overscan subtraction (from BIASSEC* keyword defined regions) - readnoise estimation (from BIASSEC* keyword defined regions) - gain correction (from GAIN* keywords) - pixel flat correction - cosmic ray masking - propagation of input known bad pixel mask - inverse variance estimation Notes: The bias image is subtracted before any other calculation to remove any non-uniformities in the overscan regions prior to calculating overscan levels and readnoise. The readnoise is an image not just one number per amp, because the pixflat image also affects the interpreted readnoise. The inverse variance is estimated from the readnoise and the image itself, and thus is biased. ''' #- TODO: Check for required keywords first #- Subtract bias image camera = header['CAMERA'].lower() if bias is not False and bias is not None: if bias is True: #- use default bias file for this camera/night dateobs = header['DATE-OBS'] bias = read_bias(camera=camera, dateobs=dateobs) elif isinstance(bias, (str, unicode)): #- treat as filename bias = read_bias(filename=bias) if bias.shape == rawimage.shape: rawimage = rawimage - bias else: raise ValueError('shape mismatch bias {} != rawimage {}'.format( bias.shape, rawimage.shape)) #- Output arrays yy, xx = _parse_sec_keyword(header['CCDSEC4']) #- 4 = upper right image = np.zeros((yy.stop, xx.stop)) readnoise = np.zeros_like(image) for amp in ['1', '2', '3', '4']: ii = _parse_sec_keyword(header['BIASSEC' + amp]) #- Initial teststand data may be missing GAIN* keywords; don't crash if 'GAIN' + amp in header: gain = header['GAIN' + amp] #- gain = electrons / ADU else: log.error('Missing keyword GAIN{}; using 1.0'.format(amp)) gain = 1.0 overscan, rdnoise = _overscan(rawimage[ii]) rdnoise *= gain kk = _parse_sec_keyword(header['CCDSEC' + amp]) readnoise[kk] = rdnoise header['OVERSCN' + amp] = overscan header['OBSRDN' + amp] = rdnoise #- Warn/error if measured readnoise is very different from expected if 'RDNOISE' + amp in header: expected_readnoise = header['RDNOISE' + amp] if rdnoise < 0.5 * expected_readnoise: log.error( 'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}' .format(amp, rdnoise, expected_readnoise)) elif rdnoise < 0.9 * expected_readnoise: log.warn( 'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}' .format(amp, rdnoise, expected_readnoise)) elif rdnoise > 2.0 * expected_readnoise: log.error( 'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}' .format(amp, rdnoise, expected_readnoise)) elif rdnoise > 1.2 * expected_readnoise: log.warn( 'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}' .format(amp, rdnoise, expected_readnoise)) else: log.warn('Expected readnoise keyword {} missing'.format('RDNOISE' + amp)) #- subtract overscan from data region and apply gain jj = _parse_sec_keyword(header['DATASEC' + amp]) data = rawimage[jj] - overscan image[kk] = data * gain #- Load mask if mask is not False and mask is not None: if mask is True: dateobs = header['DATE-OBS'] mask = read_mask(camera=camera, dateobs=dateobs) elif isinstance(mask, (str, unicode)): mask = read_mask(filename=mask) else: mask = np.zeros(image.shape, dtype=np.int32) if mask.shape != image.shape: raise ValueError('shape mismatch mask {} != image {}'.format( mask.shape, image.shape)) #- Divide by pixflat image if pixflat is not False and pixflat is not None: if pixflat is True: dateobs = header['DATE-OBS'] pixflat = read_pixflat(camera=camera, dateobs=dateobs) elif isinstance(pixflat, (str, unicode)): pixflat = read_pixflat(filename=pixflat) if pixflat.shape != image.shape: raise ValueError('shape mismatch pixflat {} != image {}'.format( pixflat.shape, image.shape)) if np.all(pixflat != 0.0): image /= pixflat readnoise /= pixflat else: good = (pixflat != 0.0) image[good] /= pixflat[good] readnoise[good] /= pixflat[good] mask[~good] |= ccdmask.PIXFLATZERO lowpixflat = (0 < pixflat) & (pixflat < 0.1) if np.any(lowpixflat): mask[lowpixflat] |= ccdmask.PIXFLATLOW #- Inverse variance, estimated directly from the data (BEWARE: biased!) var = image.clip(0) + readnoise**2 ivar = 1.0 / var img = Image(image, ivar=ivar, mask=mask, meta=header, readnoise=readnoise, camera=camera) #- update img.mask to mask cosmic rays cosmics.reject_cosmic_rays(img) return img