def test_io(self):
io.write_raw(self.rawfile,
self.rawimage,
self.header,
primary_header=self.primary_header,
camera='b0')
io.write_raw(self.rawfile,
self.rawimage,
self.header,
primary_header=self.primary_header,
camera='R1')
io.write_raw(self.rawfile,
self.rawimage,
self.header,
primary_header=self.primary_header,
camera='z9')
self.header['CAMERA'] = 'B1'
io.write_raw(self.rawfile,
self.rawimage,
self.header,
primary_header=self.primary_header)
b0 = io.read_raw(self.rawfile, 'b0')
b1 = io.read_raw(self.rawfile, 'b1')
r1 = io.read_raw(self.rawfile, 'r1')
z9 = io.read_raw(self.rawfile, 'Z9')
self.assertEqual(b0.meta['CAMERA'], 'b0')
self.assertEqual(b1.meta['CAMERA'], 'b1')
self.assertEqual(r1.meta['CAMERA'], 'r1')
self.assertEqual(z9.meta['CAMERA'], 'z9')
def test_io(self):
io.write_raw(self.rawfile, self.rawimage, self.header, camera='b0')
io.write_raw(self.rawfile, self.rawimage, self.header, camera='R1')
io.write_raw(self.rawfile, self.rawimage, self.header, camera='z9')
self.header['CAMERA'] = 'B1'
io.write_raw(self.rawfile, self.rawimage, self.header)
b0 = io.read_raw(self.rawfile, 'b0')
b1 = io.read_raw(self.rawfile, 'b1')
r1 = io.read_raw(self.rawfile, 'r1')
z9 = io.read_raw(self.rawfile, 'Z9')
self.assertEqual(b0.meta['CAMERA'], 'b0')
self.assertEqual(b1.meta['CAMERA'], 'b1')
self.assertEqual(r1.meta['CAMERA'], 'r1')
self.assertEqual(z9.meta['CAMERA'], 'z9')
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
if args.cameras is None:
args.cameras = [c + str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask
is not None):
if len(args.cameras) > 1:
raise ValueError(
'must use only one camera with --bias, --pixflat, --mask options'
)
if (args.pixfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --pixfile option')
if args.outdir is None:
args.outdir = os.getcwd()
for camera in args.cameras:
try:
img = io.read_raw(args.infile,
camera,
bias=args.bias,
pixflat=args.pixflat,
mask=args.mask)
except IOError:
log.error('Camera {} not in {}'.format(camera, args.infile))
continue
if args.pixfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
pixfile = io.findfile('pix',
night=night,
expid=expid,
camera=camera,
outdir=args.outdir)
else:
pixfile = args.pixfile
io.write_image(pixfile, img)
def run(self):
file_dir = '/global/cscratch1/sd/zhangkai/desi/test/'
if use_mpi:
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
test_file = file_dir + 'test' + str(rank).zfill(2) + '.fits.fz'
test_file_output = file_dir + 'test' + str(rank).zfill(
2) + '_out.fits.fz'
##### Read ######
print('Read ' + test_file)
rawdata = read_raw(test_file, camera='R1')
h = fitsio.read_header(test_file, 1)
##### Write ######
print('Write ' + test_file_output)
write_image(test_file_output,
rawdata) #write_raw(test_file_output,rawdata,None)
else:
pass
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
if args.cameras is None:
args.cameras = [c+str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None):
if len(args.cameras) > 1:
raise ValueError('must use only one camera with --bias, --pixflat, --mask options')
if (args.pixfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --pixfile option')
if args.outdir is None:
args.outdir = os.getcwd()
for camera in args.cameras:
try:
img = io.read_raw(args.infile, camera,
bias=args.bias, pixflat=args.pixflat, mask=args.mask)
except IOError:
log.error('Camera {} not in {}'.format(camera, args.infile))
continue
if args.pixfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
pixfile = io.findfile('pix', night=night, expid=expid, camera=camera,
outdir=args.outdir)
else:
pixfile = args.pixfile
io.write_image(pixfile, img)
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
bias=True
if args.bias : bias=args.bias
if args.nobias : bias=False
dark=True
if args.dark : dark=args.dark
if args.nodark : dark=False
pixflat=True
if args.pixflat : pixflat=args.pixflat
if args.nopixflat : pixflat=False
mask=True
if args.mask : mask=args.mask
if args.nomask : mask=False
if args.cameras is None:
args.cameras = [c+str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None):
if len(args.cameras) > 1:
raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options')
if (args.pixfile is not None):
log.warning('--pixfile is deprecated; please use --outfile instead')
if args.outfile is None:
args.outfile = args.pixfile
else:
log.critical("Set --outfile not --pixfile and certainly not both")
sys.exit(1)
if (args.outfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --outfile option')
if args.outdir is None:
args.outdir = os.getcwd()
log.warning('--outdir not specified; using {}'.format(args.outdir))
ccd_calibration_filename = None
if args.no_ccd_calib_filename :
ccd_calibration_filename = False
elif args.ccd_calib_filename is not None :
ccd_calibration_filename = args.ccd_calib_filename
for camera in args.cameras:
try:
img = io.read_raw(args.infile, camera,
bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub,
nocosmic=args.nocosmic,
cosmics_nsig=args.cosmics_nsig,
cosmics_cfudge=args.cosmics_cfudge,
cosmics_c2fudge=args.cosmics_c2fudge,
ccd_calibration_filename=ccd_calibration_filename,
nocrosstalk=args.nocrosstalk,
nogain=args.nogain,
fill_header=args.fill_header
)
except IOError:
log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile))
continue
if(args.zero_masked) :
img.pix *= (img.mask==0)
if args.outfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
outfile = io.findfile('preproc', night=night, expid=expid, camera=camera,
outdir=args.outdir)
else:
outfile = args.outfile
io.write_image(outfile, img)
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
bias=True
if args.bias : bias=args.bias
if args.nobias : bias=False
dark=True
if args.dark : dark=args.dark
if args.nodark : dark=False
pixflat=True
if args.pixflat : pixflat=args.pixflat
if args.nopixflat : pixflat=False
mask=True
if args.mask : mask=args.mask
if args.nomask : mask=False
if args.cameras is None:
args.cameras = [c+str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None):
if len(args.cameras) > 1:
raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options')
if (args.outfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --outfile option')
if args.outdir is None:
args.outdir = os.getcwd()
log.warning('--outdir not specified; using {}'.format(args.outdir))
ccd_calibration_filename = None
if args.no_ccd_calib_filename :
ccd_calibration_filename = False
elif args.ccd_calib_filename is not None :
ccd_calibration_filename = args.ccd_calib_filename
if args.fibermap and not os.path.exists(fibermap):
raise ValueError('--fibermap {} not found'.format(args.fibermap))
if args.fibermap is None:
datadir, infile = os.path.split(os.path.abspath(args.infile))
fibermapfile = infile.replace('desi-', 'fibermap-').replace('.fits.fz', '.fits')
args.fibermap = os.path.join(datadir, fibermapfile)
if args.nofibermap:
fibermap = None
elif os.path.exists(args.fibermap):
fibermap = io.read_fibermap(args.fibermap)
else:
log.warning('fibermap file not found; creating blank fibermap')
fibermap = io.empty_fibermap(5000)
for camera in args.cameras:
try:
img = io.read_raw(args.infile, camera,
bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub,
nocosmic=args.nocosmic,
cosmics_nsig=args.cosmics_nsig,
cosmics_cfudge=args.cosmics_cfudge,
cosmics_c2fudge=args.cosmics_c2fudge,
ccd_calibration_filename=ccd_calibration_filename,
nocrosstalk=args.nocrosstalk,
nogain=args.nogain,
nodarktrail=args.nodarktrail,
fill_header=args.fill_header,
)
except IOError:
log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile))
continue
if(args.zero_masked) :
img.pix *= (img.mask==0)
if args.outfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
outfile = io.findfile('preproc', night=night, expid=expid, camera=camera,
outdir=args.outdir)
else:
outfile = args.outfile
if fibermap:
petal_loc = int(img.camera[1])
ii = (fibermap['PETAL_LOC'] == petal_loc)
img.fibermap = fibermap[ii]
io.write_image(outfile, img)
log.info("Wrote {}".format(outfile))
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
bias = True
if args.bias: bias = args.bias
if args.nobias: bias = False
dark = True
if args.dark: dark = args.dark
if args.nodark: dark = False
pixflat = True
if args.pixflat: pixflat = args.pixflat
if args.nopixflat: pixflat = False
mask = True
if args.mask: mask = args.mask
if args.nomask: mask = False
if args.cameras is None:
args.cameras = [c + str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (
args.mask is not None) or (args.dark is not None):
if len(args.cameras) > 1:
raise ValueError(
'must use only one camera with --bias, --dark, --pixflat, --mask options'
)
if (args.pixfile is not None):
log.warning('--pixfile is deprecated; please use --outfile instead')
if args.outfile is None:
args.outfile = args.pixfile
else:
log.critical("Set --outfile not --pixfile and certainly not both")
sys.exit(1)
if (args.outfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --outfile option')
if args.outdir is None:
args.outdir = os.getcwd()
log.warning('--outdir not specified; using {}'.format(args.outdir))
ccd_calibration_filename = None
if args.no_ccd_calib_filename:
ccd_calibration_filename = False
elif args.ccd_calib_filename is not None:
ccd_calibration_filename = args.ccd_calib_filename
for camera in args.cameras:
try:
img = io.read_raw(
args.infile,
camera,
bias=bias,
dark=dark,
pixflat=pixflat,
mask=mask,
bkgsub=args.bkgsub,
nocosmic=args.nocosmic,
cosmics_nsig=args.cosmics_nsig,
cosmics_cfudge=args.cosmics_cfudge,
cosmics_c2fudge=args.cosmics_c2fudge,
ccd_calibration_filename=ccd_calibration_filename,
nocrosstalk=args.nocrosstalk,
nogain=args.nogain,
fill_header=args.fill_header)
except IOError:
log.error(
'Error while reading or preprocessing camera {} in {}'.format(
camera, args.infile))
continue
if (args.zero_masked):
img.pix *= (img.mask == 0)
if args.outfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
outfile = io.findfile('preproc',
night=night,
expid=expid,
camera=camera,
outdir=args.outdir)
else:
outfile = args.outfile
io.write_image(outfile, img)
# Get the bias
if args.bias is not None:
biasfile = pyfits.open(args.bias)
bias_img = biasfile[0].data
log.info("read given bias image file")
else:
log.info("No bias image given, will try to find one automatically")
# read raw data and preprocess them
img = io.read_raw(filename,
args.camera,
bias_img=bias_img,
bias=True,
nogain=False,
nocosmic=True,
mask=False,
dark=False,
pixflat=False,
nocrosstalk=True,
ccd_calibration_filename=False)
shape = img.pix.shape
log.info("adding dark %s divided by exposure time %f s" %
(filename, exptime))
images.append(img.pix / exptime)
images = np.array(images)
log.info("compute median image ...")
med_image = np.median(images, axis=0)
def preproc_file(infile,
camera,
outfile=None,
outdir=None,
fibermap=None,
zero_masked=False,
**preproc_opts):
"""
Preprocess a single camera from a single input file
Args:
infile : input raw data file
camera : camera, e.g. 'b0', 'r1', 'z9'
Options:
outfile: output preprocessed image file to write
outdir: output directory; derive filename from infile NIGHT and EXPID
fibermap: fibermap filename to include in output
zero_masked (bool): set masked pixels to 0
preproc_opts: dictionary to pass to preproc
Returns error code (1=error, 0=success) but will not raise exception if
there is an I/O or preprocessing failure (allows other parallel procs to
proceed).
Note: either `outfile` or `outdir` must be provided
"""
try:
img = io.read_raw(infile,
camera,
fibermapfile=fibermap,
**preproc_opts)
except IOError as e:
#- print error and return error code, but don't raise exception so
#- that this won't block other cameras for multiprocessing
log.error(
'Error while reading or preprocessing camera {} in {}'.format(
camera, infile))
log.error(e)
return 1
if zero_masked:
log.info("Setting masked pixels values to zero")
img.pix *= (img.mask == 0)
if outfile is None:
night = img.meta['NIGHT']
if not 'EXPID' in img.meta.keys():
if 'EXPNUM' in img.meta.keys():
img.meta['EXPID'] = img.meta['EXPNUM']
else:
mess = "no EXPID nor EXPNUM in img.meta, cannot create output filename"
log.error(mess)
raise KeyError(mess)
expid = img.meta['EXPID']
outfile = io.findfile('preproc',
night=night,
expid=expid,
camera=camera,
outdir=outdir)
io.write_image(outfile, img)
log.info("Wrote {}".format(outfile))
return 0
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
t0 = time.time()
log = get_logger()
# guess if it is a preprocessed or a raw image
hdulist = fits.open(args.image)
is_input_preprocessed = ("IMAGE" in hdulist) & ("IVAR" in hdulist)
primary_header = hdulist[0].header
hdulist.close()
if is_input_preprocessed:
image = read_image(args.image)
else:
if args.camera is None:
print(
"ERROR: Need to specify camera to open a raw fits image (with all cameras in different fits HDUs)"
)
print(
"Try adding the option '--camera xx', with xx in {brz}{0-9}, like r7, or type 'desi_qproc --help' for more options"
)
sys.exit(12)
image = read_raw(args.image, args.camera, fill_header=[
1,
])
if args.auto:
log.debug("AUTOMATIC MODE")
try:
night = image.meta['NIGHT']
if not 'EXPID' in image.meta:
if 'EXPNUM' in image.meta:
log.warning('using EXPNUM {} for EXPID'.format(
image.meta['EXPNUM']))
image.meta['EXPID'] = image.meta['EXPNUM']
expid = image.meta['EXPID']
except KeyError as e:
log.error(
"Need at least NIGHT and EXPID (or EXPNUM) to run in auto mode. Retry without the --auto option."
)
log.error(str(e))
sys.exit(12)
indir = os.path.dirname(args.image)
if args.fibermap is None:
filename = '{}/fibermap-{:08d}.fits'.format(indir, expid)
if os.path.isfile(filename):
log.debug("auto-mode: found a fibermap, {}, using it!".format(
filename))
args.fibermap = filename
if args.output_preproc is None:
if not is_input_preprocessed:
args.output_preproc = '{}/preproc-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera.lower(), expid)
log.debug("auto-mode: will write preproc in " +
args.output_preproc)
else:
log.debug(
"auto-mode: will not write preproc because input is a preprocessed image"
)
if args.auto_output_dir != '.':
if not os.path.isdir(args.auto_output_dir):
log.debug("auto-mode: creating directory " +
args.auto_output_dir)
os.makedirs(args.auto_output_dir)
if args.output_preproc is not None:
write_image(args.output_preproc, image)
cfinder = None
if args.psf is None:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
args.psf = cfinder.findfile("PSF")
log.info(" Using PSF {}".format(args.psf))
tset = read_xytraceset(args.psf)
# add fibermap
if args.fibermap:
if os.path.isfile(args.fibermap):
fibermap = read_fibermap(args.fibermap)
else:
log.error("no fibermap file {}".format(args.fibermap))
fibermap = None
else:
fibermap = None
if "OBSTYPE" in image.meta:
obstype = image.meta["OBSTYPE"].upper()
image.meta["OBSTYPE"] = obstype # make sure it's upper case
qframe = None
else:
log.warning("No OBSTYPE keyword, trying to guess ...")
qframe = qproc_boxcar_extraction(tset,
image,
width=args.width,
fibermap=fibermap)
obstype = check_qframe_flavor(
qframe, input_flavor=image.meta["FLAVOR"]).upper()
image.meta["OBSTYPE"] = obstype
log.info("OBSTYPE = '{}'".format(obstype))
if args.auto:
# now set the things to do
if obstype == "SKY" or obstype == "TWILIGHT" or obstype == "SCIENCE":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.apply_fiberflat = True
args.skysub = True
args.output_skyframe = '{}/qsky-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.fluxcalib = True
args.outframe = '{}/qcframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
elif obstype == "ARC" or obstype == "TESTARC":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.compute_lsf_sigma = True
elif obstype == "FLAT" or obstype == "TESTFLAT":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.compute_fiberflat = '{}/qfiberflat-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
if args.shift_psf:
# using the trace shift script
if args.auto:
options = option_list({
"psf":
args.psf,
"image":
"dummy",
"outpsf":
"dummy",
"continuum": ((obstype == "FLAT") | (obstype == "TESTFLAT")),
"sky": ((obstype == "SCIENCE") | (obstype == "SKY"))
})
else:
options = option_list({
"psf": args.psf,
"image": "dummy",
"outpsf": "dummy"
})
tmp_args = trace_shifts_script.parse(options=options)
tset = trace_shifts_script.fit_trace_shifts(image=image, args=tmp_args)
qframe = qproc_boxcar_extraction(tset,
image,
width=args.width,
fibermap=fibermap)
if tset.meta is not None:
# add traceshift info in the qframe, this will be saved in the qframe header
if qframe.meta is None:
qframe.meta = dict()
for k in tset.meta.keys():
qframe.meta[k] = tset.meta[k]
if args.output_rawframe is not None:
write_qframe(args.output_rawframe, qframe)
log.info("wrote raw extracted frame in {}".format(
args.output_rawframe))
if args.compute_lsf_sigma:
tset = process_arc(qframe, tset, linelist=None, npoly=2, nbins=2)
if args.output_psf is not None:
for k in qframe.meta:
if k not in tset.meta:
tset.meta[k] = qframe.meta[k]
write_xytraceset(args.output_psf, tset)
if args.compute_fiberflat is not None:
fiberflat = qproc_compute_fiberflat(qframe)
#write_qframe(args.compute_fiberflat,qflat)
write_fiberflat(args.compute_fiberflat, fiberflat, header=qframe.meta)
log.info("wrote fiberflat in {}".format(args.compute_fiberflat))
if args.apply_fiberflat or args.input_fiberflat:
if args.input_fiberflat is None:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
try:
args.input_fiberflat = cfinder.findfile("FIBERFLAT")
except KeyError as e:
log.error("no FIBERFLAT for this spectro config")
sys.exit(12)
log.info("applying fiber flat {}".format(args.input_fiberflat))
flat = read_fiberflat(args.input_fiberflat)
qproc_apply_fiberflat(qframe, flat)
if args.skysub:
log.info("sky subtraction")
if args.output_skyframe is not None:
skyflux = qproc_sky_subtraction(qframe, return_skymodel=True)
sqframe = QFrame(qframe.wave, skyflux, np.ones(skyflux.shape))
write_qframe(args.output_skyframe, sqframe)
log.info("wrote sky model in {}".format(args.output_skyframe))
else:
qproc_sky_subtraction(qframe)
if args.fluxcalib:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
# check for flux calib
if cfinder.haskey("FLUXCALIB"):
fluxcalib_filename = cfinder.findfile("FLUXCALIB")
fluxcalib = read_average_flux_calibration(fluxcalib_filename)
log.info("read average calib in {}".format(fluxcalib_filename))
seeing = qframe.meta["SEEING"]
airmass = qframe.meta["AIRMASS"]
exptime = qframe.meta["EXPTIME"]
exposure_calib = fluxcalib.value(seeing=seeing, airmass=airmass)
for q in range(qframe.nspec):
fiber_calib = np.interp(qframe.wave[q], fluxcalib.wave,
exposure_calib) * exptime
inv_calib = (fiber_calib > 0) / (fiber_calib +
(fiber_calib == 0))
qframe.flux[q] *= inv_calib
qframe.ivar[q] *= fiber_calib**2 * (fiber_calib > 0)
# add keyword in header giving the calibration factor applied at a reference wavelength
band = qframe.meta["CAMERA"].upper()[0]
if band == "B":
refwave = 4500
elif band == "R":
refwave = 6500
else:
refwave = 8500
calvalue = np.interp(refwave, fluxcalib.wave,
exposure_calib) * exptime
qframe.meta["CALWAVE"] = refwave
qframe.meta["CALVALUE"] = calvalue
else:
log.error(
"Cannot calibrate fluxes because no FLUXCALIB keywork in calibration files"
)
fibers = parse_fibers(args.fibers)
if fibers is None:
fibers = qframe.flux.shape[0]
else:
ii = np.arange(qframe.fibers.size)[np.in1d(qframe.fibers, fibers)]
if ii.size == 0:
log.error("no such fibers in frame,")
log.error("fibers are in range [{}:{}]".format(
qframe.fibers[0], qframe.fibers[-1] + 1))
sys.exit(12)
qframe = qframe[ii]
if args.outframe is not None:
write_qframe(args.outframe, qframe)
log.info("wrote {}".format(args.outframe))
t1 = time.time()
log.info("all done in {:3.1f} sec".format(t1 - t0))
if args.plot:
log.info("plotting {} spectra".format(qframe.wave.shape[0]))
import matplotlib.pyplot as plt
fig = plt.figure()
for i in range(qframe.wave.shape[0]):
j = (qframe.ivar[i] > 0)
plt.plot(qframe.wave[i, j], qframe.flux[i, j])
plt.grid()
plt.xlabel("wavelength")
plt.ylabel("flux")
plt.show()
def compute_dark_file(rawfiles,
outfile,
camera,
bias=None,
nocosmic=False,
scale=False,
exptime=None):
"""
Compute classic dark model from input dark images
Args:
rawfiles (list of str): list of input raw data files (desi-*.fits.fz)
outfile (str): output file with dark model to write
camera (str): camera to process, e.g. b0, r1, z9
Options:
bias (str or list): bias file to use, or list of bias files
nocosmic (bool): use medians instead of cosmic identification
scale (bool): apply scale correction for EM0 teststand data
exptime (float): write EXPTIME header keyword; all inputs must match
Note: if bias is None, no bias correction is applied. If it is a single
file, then use that bias for all darks. If it is a list, it must have
len(rawfiles) and gives the per-file bias to use.
Note: this computes a classic dark model without any non-linear terms.
see bin/compute_dark_nonlinear for current DESI dark model.
TODO: separate algorithm from I/O
"""
log = get_logger()
log.info("read images ...")
shape = None
images = []
first_image_header = None
if nocosmic:
masks = None
else:
masks = []
for ifile, filename in enumerate(rawfiles):
log.info(f'Reading {filename} camera {camera}')
# collect exposure times
fitsfile = pyfits.open(filename)
primary_header = fitsfile[0].header
if not "EXPTIME" in primary_header:
primary_header = fitsfile[1].header
if "EXPREQ" in primary_header:
thisexptime = primary_header["EXPREQ"]
log.warning(
"Using EXPREQ and not EXPTIME, because a more accurate quantity on teststand"
)
else:
thisexptime = primary_header["EXPTIME"]
flavor = primary_header['FLAVOR'].upper()
if flavor != 'DARK':
message = f'Input {filename} flavor {flavor} != DARK'
log.error(message)
raise ValueError(message)
if exptime is not None:
if round(exptime) != round(thisexptime):
message = f'Input {filename} exptime {thisexptime} != requested exptime {exptime}'
log.error(message)
raise ValueError(message)
if first_image_header is None:
first_image_header = fitsfile[camera].header
fitsfile.close()
if bias is not None:
if isinstance(bias, str):
thisbias = bias
elif isinstance(bias, (list, tuple, np.array)):
thisbias = bias[ifile]
else:
message = 'bias should be None, str, list, or tuple, not {}'.format(
type(bias))
log.error(message)
raise RuntimeError(message)
else:
thisbias = False
# read raw data and preprocess them
img = io.read_raw(filename,
camera,
bias=thisbias,
nocosmic=nocosmic,
mask=False,
dark=False,
pixflat=False)
# propagate gains to first_image_header
if 'GAINA' in img.meta and 'GAINA' not in first_image_header:
first_image_header['GAINA'] = img.meta['GAINA']
first_image_header['GAINB'] = img.meta['GAINB']
first_image_header['GAINC'] = img.meta['GAINC']
first_image_header['GAIND'] = img.meta['GAIND']
if shape is None:
shape = img.pix.shape
log.info("adding dark %s divided by exposure time %f s" %
(filename, thisexptime))
images.append(img.pix.ravel() / thisexptime)
if masks is not None:
masks.append(img.mask.ravel())
images = np.array(images)
if masks is not None:
masks = np.array(masks)
smask = np.sum(masks, axis=0)
else:
smask = np.zeros(images[0].shape)
log.info("compute median image ...")
medimage = masked_median(images, masks)
if scale:
log.info("compute a scale per image ...")
sm2 = np.sum((smask == 0) * medimage**2)
ok = (medimage > 0.6 * np.median(medimage)) * (smask == 0)
for i, image in enumerate(rawfiles):
s = np.sum((smask == 0) * medimage * image) / sm2
#s=np.median(image[ok]/medimage[ok])
log.info("image %d scale = %f" % (i, s))
images[i] /= s
log.info("recompute median image after scaling ...")
medimage = masked_median(images, masks)
if True:
log.info("compute mask ...")
ares = np.abs(images - medimage)
nsig = 4.
mask = (ares < nsig * 1.4826 * np.median(ares, axis=0))
# average (not median)
log.info("compute average ...")
meanimage = np.sum(images * mask, axis=0) / np.sum(mask, axis=0)
meanimage = meanimage.reshape(shape)
else:
meanimage = medimage.reshape(shape)
log.info("write result in %s ..." % outfile)
hdulist = pyfits.HDUList([pyfits.PrimaryHDU(meanimage.astype('float32'))])
# copy some keywords
for key in [
"TELESCOP",
"INSTRUME",
"SPECGRPH",
"SPECID",
"DETECTOR",
"CAMERA",
"CCDNAME",
"CCDPREP",
"CCDSIZE",
"CCDTEMP",
"CPUTEMP",
"CASETEMP",
"CCDTMING",
"CCDCFG",
"SETTINGS",
"VESSEL",
"FEEVER",
"FEEBOX",
"PRESECA",
"PRRSECA",
"DATASECA",
"TRIMSECA",
"BIASSECA",
"ORSECA",
"CCDSECA",
"DETSECA",
"AMPSECA",
"PRESECB",
"PRRSECB",
"DATASECB",
"TRIMSECB",
"BIASSECB",
"ORSECB",
"CCDSECB",
"DETSECB",
"AMPSECB",
"PRESECC",
"PRRSECC",
"DATASECC",
"TRIMSECC",
"BIASSECC",
"ORSECC",
"CCDSECC",
"DETSECC",
"AMPSECC",
"PRESECD",
"PRRSECD",
"DATASECD",
"TRIMSECD",
"BIASSECD",
"ORSECD",
"CCDSECD",
"DETSECD",
"AMPSECD",
"DAC0",
"DAC1",
"DAC2",
"DAC3",
"DAC4",
"DAC5",
"DAC6",
"DAC7",
"DAC8",
"DAC9",
"DAC10",
"DAC11",
"DAC12",
"DAC13",
"DAC14",
"DAC15",
"DAC16",
"DAC17",
"CLOCK0",
"CLOCK1",
"CLOCK2",
"CLOCK3",
"CLOCK4",
"CLOCK5",
"CLOCK6",
"CLOCK7",
"CLOCK8",
"CLOCK9",
"CLOCK10",
"CLOCK11",
"CLOCK12",
"CLOCK13",
"CLOCK14",
"CLOCK15",
"CLOCK16",
"CLOCK17",
"CLOCK18",
"OFFSET0",
"OFFSET1",
"OFFSET2",
"OFFSET3",
"OFFSET4",
"OFFSET5",
"OFFSET6",
"OFFSET7",
"DELAYS",
"CDSPARMS",
"PGAGAIN",
"OCSVER",
"DOSVER",
"CONSTVER",
"GAINA",
"GAINB",
"GAINC",
"GAIND",
]:
if key in first_image_header:
hdulist[0].header[key] = (first_image_header[key],
first_image_header.comments[key])
if exptime is not None:
hdulist[0].header['EXPTIME'] = exptime
hdulist[0].header["BUNIT"] = "electron/s"
hdulist[0].header["EXTNAME"] = "DARK"
for i, filename in enumerate(rawfiles):
hdulist[0].header["INPUT%03d" % i] = os.path.basename(filename)
hdulist.writeto(outfile, overwrite=True)
log.info(f"Wrote {outfile}")
log.info(f"done")
