def test_io(self): io.write_raw(self.rawfile, self.rawimage, self.header, primary_header=self.primary_header, camera='b0') io.write_raw(self.rawfile, self.rawimage, self.header, primary_header=self.primary_header, camera='R1') io.write_raw(self.rawfile, self.rawimage, self.header, primary_header=self.primary_header, camera='z9') self.header['CAMERA'] = 'B1' io.write_raw(self.rawfile, self.rawimage, self.header, primary_header=self.primary_header) b0 = io.read_raw(self.rawfile, 'b0') b1 = io.read_raw(self.rawfile, 'b1') r1 = io.read_raw(self.rawfile, 'r1') z9 = io.read_raw(self.rawfile, 'Z9') self.assertEqual(b0.meta['CAMERA'], 'b0') self.assertEqual(b1.meta['CAMERA'], 'b1') self.assertEqual(r1.meta['CAMERA'], 'r1') self.assertEqual(z9.meta['CAMERA'], 'z9')
def test_io(self): io.write_raw(self.rawfile, self.rawimage, self.header, camera='b0') io.write_raw(self.rawfile, self.rawimage, self.header, camera='R1') io.write_raw(self.rawfile, self.rawimage, self.header, camera='z9') self.header['CAMERA'] = 'B1' io.write_raw(self.rawfile, self.rawimage, self.header) b0 = io.read_raw(self.rawfile, 'b0') b1 = io.read_raw(self.rawfile, 'b1') r1 = io.read_raw(self.rawfile, 'r1') z9 = io.read_raw(self.rawfile, 'Z9') self.assertEqual(b0.meta['CAMERA'], 'b0') self.assertEqual(b1.meta['CAMERA'], 'b1') self.assertEqual(r1.meta['CAMERA'], 'r1') self.assertEqual(z9.meta['CAMERA'], 'z9')
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) if args.cameras is None: args.cameras = [c + str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None): if len(args.cameras) > 1: raise ValueError( 'must use only one camera with --bias, --pixflat, --mask options' ) if (args.pixfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --pixfile option') if args.outdir is None: args.outdir = os.getcwd() for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=args.bias, pixflat=args.pixflat, mask=args.mask) except IOError: log.error('Camera {} not in {}'.format(camera, args.infile)) continue if args.pixfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] pixfile = io.findfile('pix', night=night, expid=expid, camera=camera, outdir=args.outdir) else: pixfile = args.pixfile io.write_image(pixfile, img)
def run(self): file_dir = '/global/cscratch1/sd/zhangkai/desi/test/' if use_mpi: comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() test_file = file_dir + 'test' + str(rank).zfill(2) + '.fits.fz' test_file_output = file_dir + 'test' + str(rank).zfill( 2) + '_out.fits.fz' ##### Read ###### print('Read ' + test_file) rawdata = read_raw(test_file, camera='R1') h = fitsio.read_header(test_file, 1) ##### Write ###### print('Write ' + test_file_output) write_image(test_file_output, rawdata) #write_raw(test_file_output,rawdata,None) else: pass
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) if args.cameras is None: args.cameras = [c+str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None): if len(args.cameras) > 1: raise ValueError('must use only one camera with --bias, --pixflat, --mask options') if (args.pixfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --pixfile option') if args.outdir is None: args.outdir = os.getcwd() for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=args.bias, pixflat=args.pixflat, mask=args.mask) except IOError: log.error('Camera {} not in {}'.format(camera, args.infile)) continue if args.pixfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] pixfile = io.findfile('pix', night=night, expid=expid, camera=camera, outdir=args.outdir) else: pixfile = args.pixfile io.write_image(pixfile, img)
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) bias=True if args.bias : bias=args.bias if args.nobias : bias=False dark=True if args.dark : dark=args.dark if args.nodark : dark=False pixflat=True if args.pixflat : pixflat=args.pixflat if args.nopixflat : pixflat=False mask=True if args.mask : mask=args.mask if args.nomask : mask=False if args.cameras is None: args.cameras = [c+str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None): if len(args.cameras) > 1: raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options') if (args.pixfile is not None): log.warning('--pixfile is deprecated; please use --outfile instead') if args.outfile is None: args.outfile = args.pixfile else: log.critical("Set --outfile not --pixfile and certainly not both") sys.exit(1) if (args.outfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --outfile option') if args.outdir is None: args.outdir = os.getcwd() log.warning('--outdir not specified; using {}'.format(args.outdir)) ccd_calibration_filename = None if args.no_ccd_calib_filename : ccd_calibration_filename = False elif args.ccd_calib_filename is not None : ccd_calibration_filename = args.ccd_calib_filename for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub, nocosmic=args.nocosmic, cosmics_nsig=args.cosmics_nsig, cosmics_cfudge=args.cosmics_cfudge, cosmics_c2fudge=args.cosmics_c2fudge, ccd_calibration_filename=ccd_calibration_filename, nocrosstalk=args.nocrosstalk, nogain=args.nogain, fill_header=args.fill_header ) except IOError: log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile)) continue if(args.zero_masked) : img.pix *= (img.mask==0) if args.outfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] outfile = io.findfile('preproc', night=night, expid=expid, camera=camera, outdir=args.outdir) else: outfile = args.outfile io.write_image(outfile, img)
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) bias=True if args.bias : bias=args.bias if args.nobias : bias=False dark=True if args.dark : dark=args.dark if args.nodark : dark=False pixflat=True if args.pixflat : pixflat=args.pixflat if args.nopixflat : pixflat=False mask=True if args.mask : mask=args.mask if args.nomask : mask=False if args.cameras is None: args.cameras = [c+str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None): if len(args.cameras) > 1: raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options') if (args.outfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --outfile option') if args.outdir is None: args.outdir = os.getcwd() log.warning('--outdir not specified; using {}'.format(args.outdir)) ccd_calibration_filename = None if args.no_ccd_calib_filename : ccd_calibration_filename = False elif args.ccd_calib_filename is not None : ccd_calibration_filename = args.ccd_calib_filename if args.fibermap and not os.path.exists(fibermap): raise ValueError('--fibermap {} not found'.format(args.fibermap)) if args.fibermap is None: datadir, infile = os.path.split(os.path.abspath(args.infile)) fibermapfile = infile.replace('desi-', 'fibermap-').replace('.fits.fz', '.fits') args.fibermap = os.path.join(datadir, fibermapfile) if args.nofibermap: fibermap = None elif os.path.exists(args.fibermap): fibermap = io.read_fibermap(args.fibermap) else: log.warning('fibermap file not found; creating blank fibermap') fibermap = io.empty_fibermap(5000) for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub, nocosmic=args.nocosmic, cosmics_nsig=args.cosmics_nsig, cosmics_cfudge=args.cosmics_cfudge, cosmics_c2fudge=args.cosmics_c2fudge, ccd_calibration_filename=ccd_calibration_filename, nocrosstalk=args.nocrosstalk, nogain=args.nogain, nodarktrail=args.nodarktrail, fill_header=args.fill_header, ) except IOError: log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile)) continue if(args.zero_masked) : img.pix *= (img.mask==0) if args.outfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] outfile = io.findfile('preproc', night=night, expid=expid, camera=camera, outdir=args.outdir) else: outfile = args.outfile if fibermap: petal_loc = int(img.camera[1]) ii = (fibermap['PETAL_LOC'] == petal_loc) img.fibermap = fibermap[ii] io.write_image(outfile, img) log.info("Wrote {}".format(outfile))
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) bias = True if args.bias: bias = args.bias if args.nobias: bias = False dark = True if args.dark: dark = args.dark if args.nodark: dark = False pixflat = True if args.pixflat: pixflat = args.pixflat if args.nopixflat: pixflat = False mask = True if args.mask: mask = args.mask if args.nomask: mask = False if args.cameras is None: args.cameras = [c + str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or ( args.mask is not None) or (args.dark is not None): if len(args.cameras) > 1: raise ValueError( 'must use only one camera with --bias, --dark, --pixflat, --mask options' ) if (args.pixfile is not None): log.warning('--pixfile is deprecated; please use --outfile instead') if args.outfile is None: args.outfile = args.pixfile else: log.critical("Set --outfile not --pixfile and certainly not both") sys.exit(1) if (args.outfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --outfile option') if args.outdir is None: args.outdir = os.getcwd() log.warning('--outdir not specified; using {}'.format(args.outdir)) ccd_calibration_filename = None if args.no_ccd_calib_filename: ccd_calibration_filename = False elif args.ccd_calib_filename is not None: ccd_calibration_filename = args.ccd_calib_filename for camera in args.cameras: try: img = io.read_raw( args.infile, camera, bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub, nocosmic=args.nocosmic, cosmics_nsig=args.cosmics_nsig, cosmics_cfudge=args.cosmics_cfudge, cosmics_c2fudge=args.cosmics_c2fudge, ccd_calibration_filename=ccd_calibration_filename, nocrosstalk=args.nocrosstalk, nogain=args.nogain, fill_header=args.fill_header) except IOError: log.error( 'Error while reading or preprocessing camera {} in {}'.format( camera, args.infile)) continue if (args.zero_masked): img.pix *= (img.mask == 0) if args.outfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] outfile = io.findfile('preproc', night=night, expid=expid, camera=camera, outdir=args.outdir) else: outfile = args.outfile io.write_image(outfile, img)
# Get the bias if args.bias is not None: biasfile = pyfits.open(args.bias) bias_img = biasfile[0].data log.info("read given bias image file") else: log.info("No bias image given, will try to find one automatically") # read raw data and preprocess them img = io.read_raw(filename, args.camera, bias_img=bias_img, bias=True, nogain=False, nocosmic=True, mask=False, dark=False, pixflat=False, nocrosstalk=True, ccd_calibration_filename=False) shape = img.pix.shape log.info("adding dark %s divided by exposure time %f s" % (filename, exptime)) images.append(img.pix / exptime) images = np.array(images) log.info("compute median image ...") med_image = np.median(images, axis=0)
def preproc_file(infile, camera, outfile=None, outdir=None, fibermap=None, zero_masked=False, **preproc_opts): """ Preprocess a single camera from a single input file Args: infile : input raw data file camera : camera, e.g. 'b0', 'r1', 'z9' Options: outfile: output preprocessed image file to write outdir: output directory; derive filename from infile NIGHT and EXPID fibermap: fibermap filename to include in output zero_masked (bool): set masked pixels to 0 preproc_opts: dictionary to pass to preproc Returns error code (1=error, 0=success) but will not raise exception if there is an I/O or preprocessing failure (allows other parallel procs to proceed). Note: either `outfile` or `outdir` must be provided """ try: img = io.read_raw(infile, camera, fibermapfile=fibermap, **preproc_opts) except IOError as e: #- print error and return error code, but don't raise exception so #- that this won't block other cameras for multiprocessing log.error( 'Error while reading or preprocessing camera {} in {}'.format( camera, infile)) log.error(e) return 1 if zero_masked: log.info("Setting masked pixels values to zero") img.pix *= (img.mask == 0) if outfile is None: night = img.meta['NIGHT'] if not 'EXPID' in img.meta.keys(): if 'EXPNUM' in img.meta.keys(): img.meta['EXPID'] = img.meta['EXPNUM'] else: mess = "no EXPID nor EXPNUM in img.meta, cannot create output filename" log.error(mess) raise KeyError(mess) expid = img.meta['EXPID'] outfile = io.findfile('preproc', night=night, expid=expid, camera=camera, outdir=outdir) io.write_image(outfile, img) log.info("Wrote {}".format(outfile)) return 0
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) t0 = time.time() log = get_logger() # guess if it is a preprocessed or a raw image hdulist = fits.open(args.image) is_input_preprocessed = ("IMAGE" in hdulist) & ("IVAR" in hdulist) primary_header = hdulist[0].header hdulist.close() if is_input_preprocessed: image = read_image(args.image) else: if args.camera is None: print( "ERROR: Need to specify camera to open a raw fits image (with all cameras in different fits HDUs)" ) print( "Try adding the option '--camera xx', with xx in {brz}{0-9}, like r7, or type 'desi_qproc --help' for more options" ) sys.exit(12) image = read_raw(args.image, args.camera, fill_header=[ 1, ]) if args.auto: log.debug("AUTOMATIC MODE") try: night = image.meta['NIGHT'] if not 'EXPID' in image.meta: if 'EXPNUM' in image.meta: log.warning('using EXPNUM {} for EXPID'.format( image.meta['EXPNUM'])) image.meta['EXPID'] = image.meta['EXPNUM'] expid = image.meta['EXPID'] except KeyError as e: log.error( "Need at least NIGHT and EXPID (or EXPNUM) to run in auto mode. Retry without the --auto option." ) log.error(str(e)) sys.exit(12) indir = os.path.dirname(args.image) if args.fibermap is None: filename = '{}/fibermap-{:08d}.fits'.format(indir, expid) if os.path.isfile(filename): log.debug("auto-mode: found a fibermap, {}, using it!".format( filename)) args.fibermap = filename if args.output_preproc is None: if not is_input_preprocessed: args.output_preproc = '{}/preproc-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera.lower(), expid) log.debug("auto-mode: will write preproc in " + args.output_preproc) else: log.debug( "auto-mode: will not write preproc because input is a preprocessed image" ) if args.auto_output_dir != '.': if not os.path.isdir(args.auto_output_dir): log.debug("auto-mode: creating directory " + args.auto_output_dir) os.makedirs(args.auto_output_dir) if args.output_preproc is not None: write_image(args.output_preproc, image) cfinder = None if args.psf is None: if cfinder is None: cfinder = CalibFinder([image.meta, primary_header]) args.psf = cfinder.findfile("PSF") log.info(" Using PSF {}".format(args.psf)) tset = read_xytraceset(args.psf) # add fibermap if args.fibermap: if os.path.isfile(args.fibermap): fibermap = read_fibermap(args.fibermap) else: log.error("no fibermap file {}".format(args.fibermap)) fibermap = None else: fibermap = None if "OBSTYPE" in image.meta: obstype = image.meta["OBSTYPE"].upper() image.meta["OBSTYPE"] = obstype # make sure it's upper case qframe = None else: log.warning("No OBSTYPE keyword, trying to guess ...") qframe = qproc_boxcar_extraction(tset, image, width=args.width, fibermap=fibermap) obstype = check_qframe_flavor( qframe, input_flavor=image.meta["FLAVOR"]).upper() image.meta["OBSTYPE"] = obstype log.info("OBSTYPE = '{}'".format(obstype)) if args.auto: # now set the things to do if obstype == "SKY" or obstype == "TWILIGHT" or obstype == "SCIENCE": args.shift_psf = True args.output_psf = '{}/psf-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.apply_fiberflat = True args.skysub = True args.output_skyframe = '{}/qsky-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.fluxcalib = True args.outframe = '{}/qcframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) elif obstype == "ARC" or obstype == "TESTARC": args.shift_psf = True args.output_psf = '{}/psf-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.compute_lsf_sigma = True elif obstype == "FLAT" or obstype == "TESTFLAT": args.shift_psf = True args.output_psf = '{}/psf-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) args.compute_fiberflat = '{}/qfiberflat-{}-{:08d}.fits'.format( args.auto_output_dir, args.camera, expid) if args.shift_psf: # using the trace shift script if args.auto: options = option_list({ "psf": args.psf, "image": "dummy", "outpsf": "dummy", "continuum": ((obstype == "FLAT") | (obstype == "TESTFLAT")), "sky": ((obstype == "SCIENCE") | (obstype == "SKY")) }) else: options = option_list({ "psf": args.psf, "image": "dummy", "outpsf": "dummy" }) tmp_args = trace_shifts_script.parse(options=options) tset = trace_shifts_script.fit_trace_shifts(image=image, args=tmp_args) qframe = qproc_boxcar_extraction(tset, image, width=args.width, fibermap=fibermap) if tset.meta is not None: # add traceshift info in the qframe, this will be saved in the qframe header if qframe.meta is None: qframe.meta = dict() for k in tset.meta.keys(): qframe.meta[k] = tset.meta[k] if args.output_rawframe is not None: write_qframe(args.output_rawframe, qframe) log.info("wrote raw extracted frame in {}".format( args.output_rawframe)) if args.compute_lsf_sigma: tset = process_arc(qframe, tset, linelist=None, npoly=2, nbins=2) if args.output_psf is not None: for k in qframe.meta: if k not in tset.meta: tset.meta[k] = qframe.meta[k] write_xytraceset(args.output_psf, tset) if args.compute_fiberflat is not None: fiberflat = qproc_compute_fiberflat(qframe) #write_qframe(args.compute_fiberflat,qflat) write_fiberflat(args.compute_fiberflat, fiberflat, header=qframe.meta) log.info("wrote fiberflat in {}".format(args.compute_fiberflat)) if args.apply_fiberflat or args.input_fiberflat: if args.input_fiberflat is None: if cfinder is None: cfinder = CalibFinder([image.meta, primary_header]) try: args.input_fiberflat = cfinder.findfile("FIBERFLAT") except KeyError as e: log.error("no FIBERFLAT for this spectro config") sys.exit(12) log.info("applying fiber flat {}".format(args.input_fiberflat)) flat = read_fiberflat(args.input_fiberflat) qproc_apply_fiberflat(qframe, flat) if args.skysub: log.info("sky subtraction") if args.output_skyframe is not None: skyflux = qproc_sky_subtraction(qframe, return_skymodel=True) sqframe = QFrame(qframe.wave, skyflux, np.ones(skyflux.shape)) write_qframe(args.output_skyframe, sqframe) log.info("wrote sky model in {}".format(args.output_skyframe)) else: qproc_sky_subtraction(qframe) if args.fluxcalib: if cfinder is None: cfinder = CalibFinder([image.meta, primary_header]) # check for flux calib if cfinder.haskey("FLUXCALIB"): fluxcalib_filename = cfinder.findfile("FLUXCALIB") fluxcalib = read_average_flux_calibration(fluxcalib_filename) log.info("read average calib in {}".format(fluxcalib_filename)) seeing = qframe.meta["SEEING"] airmass = qframe.meta["AIRMASS"] exptime = qframe.meta["EXPTIME"] exposure_calib = fluxcalib.value(seeing=seeing, airmass=airmass) for q in range(qframe.nspec): fiber_calib = np.interp(qframe.wave[q], fluxcalib.wave, exposure_calib) * exptime inv_calib = (fiber_calib > 0) / (fiber_calib + (fiber_calib == 0)) qframe.flux[q] *= inv_calib qframe.ivar[q] *= fiber_calib**2 * (fiber_calib > 0) # add keyword in header giving the calibration factor applied at a reference wavelength band = qframe.meta["CAMERA"].upper()[0] if band == "B": refwave = 4500 elif band == "R": refwave = 6500 else: refwave = 8500 calvalue = np.interp(refwave, fluxcalib.wave, exposure_calib) * exptime qframe.meta["CALWAVE"] = refwave qframe.meta["CALVALUE"] = calvalue else: log.error( "Cannot calibrate fluxes because no FLUXCALIB keywork in calibration files" ) fibers = parse_fibers(args.fibers) if fibers is None: fibers = qframe.flux.shape[0] else: ii = np.arange(qframe.fibers.size)[np.in1d(qframe.fibers, fibers)] if ii.size == 0: log.error("no such fibers in frame,") log.error("fibers are in range [{}:{}]".format( qframe.fibers[0], qframe.fibers[-1] + 1)) sys.exit(12) qframe = qframe[ii] if args.outframe is not None: write_qframe(args.outframe, qframe) log.info("wrote {}".format(args.outframe)) t1 = time.time() log.info("all done in {:3.1f} sec".format(t1 - t0)) if args.plot: log.info("plotting {} spectra".format(qframe.wave.shape[0])) import matplotlib.pyplot as plt fig = plt.figure() for i in range(qframe.wave.shape[0]): j = (qframe.ivar[i] > 0) plt.plot(qframe.wave[i, j], qframe.flux[i, j]) plt.grid() plt.xlabel("wavelength") plt.ylabel("flux") plt.show()
def compute_dark_file(rawfiles, outfile, camera, bias=None, nocosmic=False, scale=False, exptime=None): """ Compute classic dark model from input dark images Args: rawfiles (list of str): list of input raw data files (desi-*.fits.fz) outfile (str): output file with dark model to write camera (str): camera to process, e.g. b0, r1, z9 Options: bias (str or list): bias file to use, or list of bias files nocosmic (bool): use medians instead of cosmic identification scale (bool): apply scale correction for EM0 teststand data exptime (float): write EXPTIME header keyword; all inputs must match Note: if bias is None, no bias correction is applied. If it is a single file, then use that bias for all darks. If it is a list, it must have len(rawfiles) and gives the per-file bias to use. Note: this computes a classic dark model without any non-linear terms. see bin/compute_dark_nonlinear for current DESI dark model. TODO: separate algorithm from I/O """ log = get_logger() log.info("read images ...") shape = None images = [] first_image_header = None if nocosmic: masks = None else: masks = [] for ifile, filename in enumerate(rawfiles): log.info(f'Reading {filename} camera {camera}') # collect exposure times fitsfile = pyfits.open(filename) primary_header = fitsfile[0].header if not "EXPTIME" in primary_header: primary_header = fitsfile[1].header if "EXPREQ" in primary_header: thisexptime = primary_header["EXPREQ"] log.warning( "Using EXPREQ and not EXPTIME, because a more accurate quantity on teststand" ) else: thisexptime = primary_header["EXPTIME"] flavor = primary_header['FLAVOR'].upper() if flavor != 'DARK': message = f'Input {filename} flavor {flavor} != DARK' log.error(message) raise ValueError(message) if exptime is not None: if round(exptime) != round(thisexptime): message = f'Input {filename} exptime {thisexptime} != requested exptime {exptime}' log.error(message) raise ValueError(message) if first_image_header is None: first_image_header = fitsfile[camera].header fitsfile.close() if bias is not None: if isinstance(bias, str): thisbias = bias elif isinstance(bias, (list, tuple, np.array)): thisbias = bias[ifile] else: message = 'bias should be None, str, list, or tuple, not {}'.format( type(bias)) log.error(message) raise RuntimeError(message) else: thisbias = False # read raw data and preprocess them img = io.read_raw(filename, camera, bias=thisbias, nocosmic=nocosmic, mask=False, dark=False, pixflat=False) # propagate gains to first_image_header if 'GAINA' in img.meta and 'GAINA' not in first_image_header: first_image_header['GAINA'] = img.meta['GAINA'] first_image_header['GAINB'] = img.meta['GAINB'] first_image_header['GAINC'] = img.meta['GAINC'] first_image_header['GAIND'] = img.meta['GAIND'] if shape is None: shape = img.pix.shape log.info("adding dark %s divided by exposure time %f s" % (filename, thisexptime)) images.append(img.pix.ravel() / thisexptime) if masks is not None: masks.append(img.mask.ravel()) images = np.array(images) if masks is not None: masks = np.array(masks) smask = np.sum(masks, axis=0) else: smask = np.zeros(images[0].shape) log.info("compute median image ...") medimage = masked_median(images, masks) if scale: log.info("compute a scale per image ...") sm2 = np.sum((smask == 0) * medimage**2) ok = (medimage > 0.6 * np.median(medimage)) * (smask == 0) for i, image in enumerate(rawfiles): s = np.sum((smask == 0) * medimage * image) / sm2 #s=np.median(image[ok]/medimage[ok]) log.info("image %d scale = %f" % (i, s)) images[i] /= s log.info("recompute median image after scaling ...") medimage = masked_median(images, masks) if True: log.info("compute mask ...") ares = np.abs(images - medimage) nsig = 4. mask = (ares < nsig * 1.4826 * np.median(ares, axis=0)) # average (not median) log.info("compute average ...") meanimage = np.sum(images * mask, axis=0) / np.sum(mask, axis=0) meanimage = meanimage.reshape(shape) else: meanimage = medimage.reshape(shape) log.info("write result in %s ..." % outfile) hdulist = pyfits.HDUList([pyfits.PrimaryHDU(meanimage.astype('float32'))]) # copy some keywords for key in [ "TELESCOP", "INSTRUME", "SPECGRPH", "SPECID", "DETECTOR", "CAMERA", "CCDNAME", "CCDPREP", "CCDSIZE", "CCDTEMP", "CPUTEMP", "CASETEMP", "CCDTMING", "CCDCFG", "SETTINGS", "VESSEL", "FEEVER", "FEEBOX", "PRESECA", "PRRSECA", "DATASECA", "TRIMSECA", "BIASSECA", "ORSECA", "CCDSECA", "DETSECA", "AMPSECA", "PRESECB", "PRRSECB", "DATASECB", "TRIMSECB", "BIASSECB", "ORSECB", "CCDSECB", "DETSECB", "AMPSECB", "PRESECC", "PRRSECC", "DATASECC", "TRIMSECC", "BIASSECC", "ORSECC", "CCDSECC", "DETSECC", "AMPSECC", "PRESECD", "PRRSECD", "DATASECD", "TRIMSECD", "BIASSECD", "ORSECD", "CCDSECD", "DETSECD", "AMPSECD", "DAC0", "DAC1", "DAC2", "DAC3", "DAC4", "DAC5", "DAC6", "DAC7", "DAC8", "DAC9", "DAC10", "DAC11", "DAC12", "DAC13", "DAC14", "DAC15", "DAC16", "DAC17", "CLOCK0", "CLOCK1", "CLOCK2", "CLOCK3", "CLOCK4", "CLOCK5", "CLOCK6", "CLOCK7", "CLOCK8", "CLOCK9", "CLOCK10", "CLOCK11", "CLOCK12", "CLOCK13", "CLOCK14", "CLOCK15", "CLOCK16", "CLOCK17", "CLOCK18", "OFFSET0", "OFFSET1", "OFFSET2", "OFFSET3", "OFFSET4", "OFFSET5", "OFFSET6", "OFFSET7", "DELAYS", "CDSPARMS", "PGAGAIN", "OCSVER", "DOSVER", "CONSTVER", "GAINA", "GAINB", "GAINC", "GAIND", ]: if key in first_image_header: hdulist[0].header[key] = (first_image_header[key], first_image_header.comments[key]) if exptime is not None: hdulist[0].header['EXPTIME'] = exptime hdulist[0].header["BUNIT"] = "electron/s" hdulist[0].header["EXTNAME"] = "DARK" for i, filename in enumerate(rawfiles): hdulist[0].header["INPUT%03d" % i] = os.path.basename(filename) hdulist.writeto(outfile, overwrite=True) log.info(f"Wrote {outfile}") log.info(f"done")