def test_pedantic(self):
with self.assertRaises(ValueError):
parse_sec_keyword('blat')
#- should log a warning about large readnoise
rawimage = self.rawimage + np.random.normal(scale=2, size=self.rawimage.shape)
image = preproc(rawimage, self.header, primary_header = self.primary_header)
#- should log an error about huge readnoise
rawimage = self.rawimage + np.random.normal(scale=10, size=self.rawimage.shape)
image = preproc(rawimage, self.header, primary_header = self.primary_header)
#- should log a warning about small readnoise
rdnoise = 0.7 * np.mean(list(self.rdnoise.values()))
rawimage = np.random.normal(scale=rdnoise, size=self.rawimage.shape)
image = preproc(rawimage, self.header, primary_header = self.primary_header)
#- should log a warning about tiny readnoise
rdnoise = 0.01 * np.mean(list(self.rdnoise.values()))
rawimage = np.random.normal(scale=rdnoise, size=self.rawimage.shape)
image = preproc(rawimage, self.header, primary_header = self.primary_header)
#- Missing expected RDNOISE keywords shouldn't be fatal
hdr = self.header.copy()
del hdr['RDNOISEA']
del hdr['RDNOISEB']
del hdr['RDNOISEC']
del hdr['RDNOISED']
image = preproc(self.rawimage, hdr, primary_header = self.primary_header)
#- Missing expected GAIN keywords should log error but not crash
hdr = self.header.copy()
del hdr['GAINA']
del hdr['GAINB']
del hdr['GAINC']
del hdr['GAIND']
image = preproc(self.rawimage, hdr, primary_header = self.primary_header)
def compare_vertical(camera='z3', AMP='D'):
outfile = 'fig_bias_compare_{}-{}.png'.format(camera, AMP)
old_bias, hdu = grab_stack_bias_img(camera, old=True)
new_bias, _ = grab_stack_bias_img(camera, old=False)
jj = preproc.parse_sec_keyword(hdu.header['DATASEC' + AMP])
#kk = preproc.parse_sec_keyword(hdu.header['' + AMP])
ov_col = preproc.parse_sec_keyword(hdu.header['BIASSEC' + AMP])
old_sub_img = old_bias[jj]
new_sub_img = new_bias[jj]
med_new = np.median(new_bias[ov_col])
# Smash and plot
old_smash = np.median(old_sub_img, axis=0)
new_smash = np.median(new_sub_img - med_new, axis=0)
plt.clf()
ax = plt.gca()
ax.plot(old_smash, label='old', color=oclr)
ax.plot(new_smash, label='new', color=nclr)
legend = ax.legend(loc='upper right',
scatterpoints=1,
borderpad=0.2,
fontsize=13)
ax.set_xlabel('Col')
ax.set_ylabel('Median Stacked ADU')
plt.show()
embed()
# Layout and save
print('Writing {:s}'.format(outfile))
plt.tight_layout(pad=0.2, h_pad=0., w_pad=0.1)
# plt.subplots_adjust(hspace=0)
plt.savefig(os.path.join(dpath, 'Figures', outfile),
dpi=500) # , bbox_inches='tight')
plt.close()
def ampregion(image):
"""
Get the pixel boundary regions for amps
Args:
image: desispec.image.Image object
"""
pixboundary = []
for kk in get_amp_ids(image.meta): # A-D or 1-4
#- get the amp region in pix
ampboundary = parse_sec_keyword(image.meta["CCDSEC" + kk])
pixboundary.append(ampboundary)
return pixboundary
def test_preproc_no_orsec(self):
# Strip out ORSEC
old_header = self.header.copy()
old_image = self.rawimage.copy()
for amp in ('A', 'B', 'C', 'D'):
old_header.pop('ORSEC{}'.format(amp))
xy = parse_sec_keyword(self.header['DATASEC'+amp])
#old_image[xy] -= np.int32(self.offset_row[amp]) -- OR_SEC now zero
#
image = preproc(old_image, old_header, primary_header = self.primary_header)
self.assertEqual(image.pix.shape, (2*self.ny, 2*self.nx))
self.assertTrue(np.all(image.ivar <= 1/image.readnoise**2))
for amp in ('A', 'B', 'C', 'D'):
pix = image.pix[self.quad[amp]]
rdnoise = np.median(image.readnoise[self.quad[amp]])
npixover = self.ny * self.noverscan
self.assertAlmostEqual(np.mean(pix), 0.0, delta=1) # Using np.int32 pushed this to 1
self.assertAlmostEqual(np.std(pix), self.rdnoise[amp], delta=0.2)
self.assertAlmostEqual(rdnoise, self.rdnoise[amp], delta=0.2)
def setUp(self):
#- use specter psf for this test
self.psffile = resource_filename('specter', 'test/t/psf-monospot.fits')
#self.psffile=os.environ['DESIMODEL']+'/data/specpsf/psf-b.fits'
self.config = {
"kwargs": {
"refKey": None,
"param": {},
"qso_resid": None
}
}
#- rawimage
hdr = dict()
hdr['CAMERA'] = 'z1'
hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
hdr['PROGRAM'] = 'dark'
hdr['EXPTIME'] = 100
#- Dimensions per amp
ny = self.ny = 500
nx = self.nx = 400
noverscan = nover = 50
hdr['BIASSECA'] = xy2hdr(np.s_[0:ny, nx:nx + nover])
hdr['DATASECA'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSECA'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSECB'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover])
hdr['DATASECB'] = xy2hdr(np.s_[0:ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSECB'] = xy2hdr(np.s_[0:ny, nx:nx + nx])
hdr['BIASSECC'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover])
hdr['DATASECC'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['CCDSECC'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['BIASSECD'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover])
hdr['DATASECD'] = xy2hdr(np.s_[ny:ny + ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSECD'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx])
hdr['NIGHT'] = '20180923'
hdr['EXPID'] = 1
hdr['PROGRAM'] = 'dark'
hdr['FLAVOR'] = 'science'
hdr['EXPTIME'] = 100.0
rawimage = np.zeros((2 * ny, 2 * nx + 2 * noverscan))
offset = {'A': 100.0, 'B': 100.5, 'C': 50.3, 'D': 200.4}
gain = {'A': 1.0, 'B': 1.5, 'C': 0.8, 'D': 1.2}
rdnoise = {'A': 2.0, 'B': 2.2, 'C': 2.4, 'D': 2.6}
obsrdn = {'A': 3.4, 'B': 3.3, 'C': 3.6, 'D': 3.3}
quad = {
'A': np.s_[0:ny, 0:nx],
'B': np.s_[0:ny, nx:nx + nx],
'C': np.s_[ny:ny + ny, 0:nx],
'D': np.s_[ny:ny + ny, nx:nx + nx],
}
for amp in ('A', 'B', 'C', 'D'):
hdr['GAIN' + amp] = gain[amp]
hdr['RDNOISE' + amp] = rdnoise[amp]
hdr['OBSRDN' + amp] = obsrdn[amp]
xy = parse_sec_keyword(hdr['BIASSEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
xy = parse_sec_keyword(hdr['DATASEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
#- set CCD parameters
self.ccdsec1 = hdr["CCDSECA"]
self.ccdsec2 = hdr["CCDSECB"]
self.ccdsec3 = hdr["CCDSECC"]
self.ccdsec4 = hdr["CCDSECD"]
#- raw data are integers, not floats
rawimg = rawimage.astype(np.int32)
self.expid = hdr["EXPID"]
self.camera = hdr["CAMERA"]
#- Confirm that all regions were correctly offset
assert not np.any(rawimage == 0.0)
#- write to the rawfile and read it in QA test
hdr['DOSVER'] = 'SIM'
hdr['FEEVER'] = 'SIM'
hdr['DETECTOR'] = 'SIM'
desispec.io.write_raw(self.rawfile, rawimg, hdr, camera=self.camera)
self.rawimage = fits.open(self.rawfile)
#- read psf, should use specter.PSF.load_psf instead of desispec.PSF(), otherwise need to create a psfboot somewhere.
self.psf = load_psf(self.psffile)
#- make the test pixfile, fibermap file
img_pix = rawimg
img_ivar = np.ones_like(img_pix) / 3.0**2
img_mask = np.zeros(img_pix.shape, dtype=np.uint32)
img_mask[200] = 1
self.image = desispec.image.Image(img_pix,
img_ivar,
img_mask,
camera='z1',
meta=hdr)
desispec.io.write_image(self.pixfile, self.image)
#- Create a fibermap with purposefully overlapping targeting bits
n = 30
self.fibermap = desispec.io.empty_fibermap(n)
self.fibermap['OBJTYPE'][:] = 'TGT'
self.fibermap['DESI_TARGET'][::2] |= desi_mask.ELG
self.fibermap['DESI_TARGET'][::5] |= desi_mask.QSO
self.fibermap['DESI_TARGET'][::7] |= desi_mask.LRG
#- add some arbitrary fluxes
for key in ['FLUX_G', 'FLUX_R', 'FLUX_Z', 'FLUX_W1', 'FLUX_W2']:
self.fibermap[key] = 10**(
(22.5 - np.random.uniform(18, 21, size=n)) / 2.5)
#- Make some standards; these still have OBJTYPE = 'TGT'
ii = [6, 18, 29]
self.fibermap['DESI_TARGET'][ii] = desi_mask.STD_FAINT
#- set some targets to SKY
ii = self.skyfibers = [5, 10, 21]
self.fibermap['OBJTYPE'][ii] = 'SKY'
self.fibermap['DESI_TARGET'][ii] = desi_mask.SKY
for key in ['FLUX_G', 'FLUX_R', 'FLUX_Z', 'FLUX_W1', 'FLUX_W2']:
self.fibermap[key][ii] = np.random.normal(scale=100, size=len(ii))
desispec.io.write_fibermap(self.fibermapfile, self.fibermap)
#- make a test frame file
self.night = hdr['NIGHT']
self.nspec = nspec = 30
wave = np.arange(7600.0, 9800.0, 1.0) #- z channel
nwave = self.nwave = len(wave)
flux = np.random.uniform(size=(nspec, nwave)) + 100.
ivar = np.ones_like(flux)
resolution_data = np.ones((nspec, 13, nwave))
self.frame = desispec.frame.Frame(wave,
flux,
ivar,
resolution_data=resolution_data,
fibermap=self.fibermap)
self.frame.meta = hdr
self.frame.meta['WAVESTEP'] = 0.5
desispec.io.write_frame(self.framefile, self.frame)
#- make a skymodel
sky = np.ones_like(self.frame.flux) * 0.5
skyivar = np.ones_like(sky)
self.mask = np.zeros(sky.shape, dtype=np.uint32)
self.skymodel = desispec.sky.SkyModel(wave, sky, skyivar, self.mask)
self.skyfile = desispec.io.write_sky(self.skyfile, self.skymodel)
#- Make a dummy boundary map for wavelength-flux in pixel space
self.map2pix = {}
self.map2pix["LEFT_MAX_FIBER"] = 14
self.map2pix["RIGHT_MIN_FIBER"] = 17
self.map2pix["BOTTOM_MAX_WAVE_INDEX"] = 900
self.map2pix["TOP_MIN_WAVE_INDEX"] = 1100
def setUp(self):
#- catch specific warnings so that we can find and fix
# warnings.filterwarnings("error", ".*did not parse as fits unit.*")
#- Create temporary calib directory
self.calibdir = os.path.join(os.environ['HOME'], 'preproc_unit_test')
if not os.path.exists(self.calibdir): os.makedirs(self.calibdir)
#- Copy test calibration-data.yaml file
specdir=os.path.join(self.calibdir,"spec/sp0")
if not os.path.isdir(specdir) :
os.makedirs(specdir)
for c in "brz" :
shutil.copy(resource_filename('desispec', 'test/data/ql/{}0.yaml'.format(c)),os.path.join(specdir,"{}0.yaml".format(c)))
#- Set calibration environment variable
os.environ["DESI_SPECTRO_CALIB"] = self.calibdir
self.calibfile = os.path.join(self.calibdir,'test-calib-askjapqwhezcpasehadfaqp.fits')
self.rawfile = os.path.join(self.calibdir,'desi-raw-askjapqwhezcpasehadfaqp.fits')
self.pixfile = os.path.join(self.calibdir,'test-pix-askjapqwhezcpasehadfaqp.fits')
primary_hdr = dict()
primary_hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
primary_hdr['DOSVER'] = 'SIM' # ICS version
hdr = dict()
hdr['CAMERA'] = 'b0'
hdr['DETECTOR'] = 'SIM' # CCD chip identifier
hdr['FEEVER'] = 'SIM' # readout electronic
#- [x,y] 1-indexed for FITS; in reality the amps will be symmetric
#- but the header definitions don't require that to make sure we are
#- getting dimensions correct
#- Dimensions per amp, not full 4-quad CCD
self.ny = ny = 500
self.nx = nx = 400
self.noverscan = nover = 50
self.noverscan_row = nover_row = 50
#- ORSEC = overscan region in raw image (rows)
#- BIASSEC = overscan region in raw image (columns)
#- DATASEC = data region in raw image
#- CCDSEC = where should this go in output
hdr['ORSECA'] = xy2hdr(np.s_[ny:ny+nover_row, 0:nx])
hdr['BIASSECA'] = xy2hdr(np.s_[0:ny, nx:nx+nover])
hdr['DATASECA'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSECA'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['ORSECB'] = xy2hdr(np.s_[ny:ny+nover_row, nx+2*nover:nx+2*nover+nx])
hdr['BIASSECB'] = xy2hdr(np.s_[0:ny, nx+nover:nx+2*nover])
hdr['DATASECB'] = xy2hdr(np.s_[0:ny, nx+2*nover:nx+2*nover+nx])
hdr['CCDSECB'] = xy2hdr(np.s_[0:ny, nx:nx+nx])
hdr['ORSECC'] = xy2hdr(np.s_[ny+nover_row:ny+2*nover_row, 0:nx])
hdr['BIASSECC'] = xy2hdr(np.s_[ny+2*nover_row:ny+ny+2*nover_row, nx:nx+nover])
hdr['DATASECC'] = xy2hdr(np.s_[ny+2*nover_row:ny+ny+2*nover_row, 0:nx])
hdr['CCDSECC'] = xy2hdr(np.s_[ny:ny+ny, 0:nx])
hdr['ORSECD'] = xy2hdr(np.s_[ny+nover_row:ny+2*nover_row, nx+2*nover:nx+2*nover+nx])
hdr['BIASSECD'] = xy2hdr(np.s_[ny+2*nover_row:ny+ny+2*nover_row, nx+nover:nx+2*nover])
hdr['DATASECD'] = xy2hdr(np.s_[ny+2*nover_row:ny+ny+2*nover_row, nx+2*nover:nx+2*nover+nx])
hdr['CCDSECD'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nx])
hdr['NIGHT'] = '20150102'
hdr['EXPID'] = 1
hdr['EXPTIME'] = 10.0
# add to header the minimal set of keywords needed to
# identify the config in the ccd_calibration.yaml file
self.primary_header = primary_hdr
self.header = hdr
self.rawimage = np.zeros((2*self.ny+2*self.noverscan_row, 2*self.nx+2*self.noverscan))
self.offset = {'A':100.0, 'B':100.5, 'C':50.3, 'D':200.4}
self.offset_row = {'A':50.0, 'B':50.5, 'C':20.3, 'D':40.4}
self.gain = {'A':1.0, 'B':1.5, 'C':0.8, 'D':1.2}
self.rdnoise = {'A':2.0, 'B':2.2, 'C':2.4, 'D':2.6}
self.quad = {
'A': np.s_[0:ny, 0:nx], 'B': np.s_[0:ny, nx:nx+nx],
'C': np.s_[ny:ny+ny, 0:nx], 'D': np.s_[ny:ny+ny, nx:nx+nx],
}
for amp in ('A', 'B', 'C', 'D'):
self.header['GAIN'+amp] = self.gain[amp]
self.header['RDNOISE'+amp] = self.rdnoise[amp]
# Overscan row
xy = parse_sec_keyword(hdr['ORSEC'+amp])
shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start]
self.rawimage[xy] += self.offset_row[amp]
self.rawimage[xy] += self.offset[amp]
self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape)/self.gain[amp]
# Overscan col
xy = parse_sec_keyword(hdr['BIASSEC'+amp])
shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start]
self.rawimage[xy] += self.offset[amp]
self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape)/self.gain[amp]
# Extend into the row region
xy_row = parse_sec_keyword(hdr['ORSEC'+amp])
xy = (xy_row[0], xy[1])
shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start]
self.rawimage[xy] += self.offset[amp]
self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape)/self.gain[amp]
# Data
xy = parse_sec_keyword(hdr['DATASEC'+amp])
shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start]
self.rawimage[xy] += self.offset[amp]
#self.rawimage[xy] += self.offset_row[amp]
self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape)/self.gain[amp]
#- raw data are integers, not floats
self.rawimage = self.rawimage.astype(np.int32)
#- Confirm that all regions were correctly offset
assert not np.any(self.rawimage == 0.0)
def bias_stats(camera='z3', use_overscan_row=True):
"""
Measure stats on bias-subtracted frames
Args:
camera:
Returns:
"""
dpath = '/home/xavier/DESI/DESI_SCRATCH/minisv2/data/20200304/'
bias_files = glob.glob(dpath + '000*/desi-*.fz')
bias_files.sort()
# Load bias images
hdul = fits.open(bias_files[0])
hdu = hdul[camera.upper()]
smnum = hdu.header['SPECID']
new_bias_file = os.path.join(dpath, 'bias_{}.fits'.format(camera))
new_bias = fits.open(new_bias_file)[0].data
old_bias_file = os.path.join(
os.getenv('DESI_SPECTRO_CALIB'), 'ccd',
'bias-sm{}-{}-20191021.fits'.format(smnum, camera[0]))
old_bias = fits.open(old_bias_file)[0].data
calib_file = os.path.join(os.getenv('DESI_SPECTRO_CALIB'), 'spec',
'sm{}'.format(smnum),
'sm{}-{}.yaml'.format(smnum, camera[0]))
# Loop and process
stat_tbl = None
for ss, bias_file in enumerate(bias_files):
hdul = fits.open(bias_file)
# Process
tdict = dict(file=[os.path.basename(bias_file)])
lbias = 'new'
for lbias, bimg in zip(['new', 'old'], [new_bias, old_bias]):
img = preproc.preproc(hdu.data,
hdu.header,
hdul[0].header,
pixflat=False,
mask=False,
nocrosstalk=True,
ccd_calibration_filename=calib_file,
use_overscan_row=use_overscan_row,
dark=False,
flag_savgol=False,
bias_img=bimg)
# Stats
amp_ids = preproc.get_amp_ids(hdu.header)
for amp in amp_ids:
kk = preproc.parse_sec_keyword(hdu.header['CCDSEC' + amp])
zero, rms = preproc._overscan(img.pix[kk])
tdict[lbias + amp + '_zero'] = [zero]
tdict[lbias + amp + '_rms'] = [rms]
# Add to Table
if stat_tbl is None:
stat_tbl = Table(tdict)
else:
stat_tbl.add_row(tdict)
# Write
if use_overscan_row:
root = '_'
else:
root = '_norow_'
outbase = 'bias_stats{}{}.fits'.format(root, camera)
outfile = os.path.join(dpath, 'Stats', outbase)
stat_tbl.write(outfile, overwrite=True)
print("Wrote: {}".format(outfile))
def compute_bias_file(rawfiles, outfile, camera, explistfile=None):
"""
Compute a bias file from input ZERO rawfiles
Args:
rawfiles: list of input raw file names
outfile (str): output filename
camera (str): camera, e.g. b0, r1, z9
Options:
explistfile: filename with text list of NIGHT EXPID to use
Notes: explistfile is only used if rawfiles=None; it should have
one NIGHT EXPID entry per line.
"""
log = get_logger()
if explistfile is not None:
if rawfiles is not None:
msg = "specify rawfiles or explistfile, but not both"
log.error(msg)
raise ValueError(msg)
rawfiles = list()
with open(explistfile, 'r') as fx:
for line in fx:
line = line.strip()
if line.startswith('#') or len(line) < 2:
continue
night, expid = map(int, line.split())
filename = io.findfile('raw', night, expid)
if not os.path.exists(filename):
msg = f'Missing {filename}'
log.critical(msg)
raise RuntimeError(msg)
rawfiles.append(filename)
log.info("read images ...")
images = []
shape = None
first_image_header = None
for filename in rawfiles:
log.info("reading %s" % filename)
fitsfile = pyfits.open(filename)
primary_header = fitsfile[0].header
image_header = fitsfile[camera].header
if first_image_header is None:
first_image_header = image_header
flavor = image_header['FLAVOR'].upper()
if flavor != 'ZERO':
message = f'Input {filename} flavor {flavor} != ZERO'
log.error(message)
raise ValueError(message)
# subtract overscan region
cfinder = CalibFinder([image_header, primary_header])
image = fitsfile[camera].data.astype("float64")
if cfinder and cfinder.haskey("AMPLIFIERS"):
amp_ids = list(cfinder.value("AMPLIFIERS"))
else:
amp_ids = ['A', 'B', 'C', 'D']
n0 = image.shape[0] // 2
n1 = image.shape[1] // 2
for a, amp in enumerate(amp_ids):
ii = parse_sec_keyword(image_header['BIASSEC' + amp])
overscan_image = image[ii].copy()
overscan, rdnoise = _overscan(overscan_image)
log.info("amp {} overscan = {}".format(amp, overscan))
if ii[0].start < n0 and ii[1].start < n1:
image[:n0, :n1] -= overscan
elif ii[0].start < n0 and ii[1].start >= n1:
image[:n0, n1:] -= overscan
elif ii[0].start >= n0 and ii[1].start < n1:
image[n0:, :n1] -= overscan
elif ii[0].start >= n0 and ii[1].start >= n1:
image[n0:, n1:] -= overscan
if shape is None:
shape = image.shape
images.append(image.ravel())
fitsfile.close()
images = np.array(images)
print(images.shape)
# compute a mask
log.info("compute median image ...")
medimage = np.median(images, axis=0) #.reshape(shape)
log.info("compute mask ...")
ares = np.abs(images - medimage)
nsig = 4.
mask = (ares < nsig * 1.4826 * np.median(ares, axis=0))
# average (not median)
log.info("compute average ...")
meanimage = np.sum(images * mask, axis=0) / np.sum(mask, axis=0)
meanimage = meanimage.reshape(shape)
log.info("write result in %s ..." % outfile)
hdus = pyfits.HDUList([pyfits.PrimaryHDU(meanimage.astype('float32'))])
# copy some keywords
for key in [
"TELESCOP", "INSTRUME", "SPECGRPH", "SPECID", "DETECTOR", "CAMERA",
"CCDNAME", "CCDPREP", "CCDSIZE", "CCDTEMP", "CPUTEMP", "CASETEMP",
"CCDTMING", "CCDCFG", "SETTINGS", "VESSEL", "FEEVER", "FEEBOX",
"PRESECA", "PRRSECA", "DATASECA", "TRIMSECA", "BIASSECA", "ORSECA",
"CCDSECA", "DETSECA", "AMPSECA", "PRESECB", "PRRSECB", "DATASECB",
"TRIMSECB", "BIASSECB", "ORSECB", "CCDSECB", "DETSECB", "AMPSECB",
"PRESECC", "PRRSECC", "DATASECC", "TRIMSECC", "BIASSECC", "ORSECC",
"CCDSECC", "DETSECC", "AMPSECC", "PRESECD", "PRRSECD", "DATASECD",
"TRIMSECD", "BIASSECD", "ORSECD", "CCDSECD", "DETSECD", "AMPSECD",
"DAC0", "DAC1", "DAC2", "DAC3", "DAC4", "DAC5", "DAC6", "DAC7",
"DAC8", "DAC9", "DAC10", "DAC11", "DAC12", "DAC13", "DAC14",
"DAC15", "DAC16", "DAC17", "CLOCK0", "CLOCK1", "CLOCK2", "CLOCK3",
"CLOCK4", "CLOCK5", "CLOCK6", "CLOCK7", "CLOCK8", "CLOCK9",
"CLOCK10", "CLOCK11", "CLOCK12", "CLOCK13", "CLOCK14", "CLOCK15",
"CLOCK16", "CLOCK17", "CLOCK18", "OFFSET0", "OFFSET1", "OFFSET2",
"OFFSET3", "OFFSET4", "OFFSET5", "OFFSET6", "OFFSET7", "DELAYS",
"CDSPARMS", "PGAGAIN", "OCSVER", "DOSVER", "CONSTVER"
]:
if key in first_image_header:
hdus[0].header[key] = (first_image_header[key],
first_image_header.comments[key])
hdus[0].header["BUNIT"] = "adu"
hdus[0].header["EXTNAME"] = "BIAS"
for filename in rawfiles:
hdus[0].header["COMMENT"] = "Inc. {}".format(
os.path.basename(filename))
hdus.writeto(outfile, overwrite="True")
log.info("done")
def fiducialregion(frame, psf):
"""
Get the fiducial amplifier regions on the CCD pixel to fiber by wavelength space
Args:
frame: desispec.frame.Frame object
psf: desispec.psf.PSF like object
"""
startspec = 0 #- will be None if don't have fibers on the right of the CCD.
endspec = 499 #- will be None if don't have fibers on the right of the CCD
startwave0 = 0 #- lower index for the starting fiber
startwave1 = 0 #- lower index for the last fiber for the amp region
endwave0 = frame.wave.shape[0] #- upper index for the starting fiber
endwave1 = frame.wave.shape[
0] #- upper index for the last fiber for that amp
pixboundary = []
fidboundary = []
#- Adding the min, max boundary individually for the benefit of dumping to yaml.
leftmax = 499 #- for amp 1 and 3
rightmin = 0 #- for amp 2 and 4
bottommax = frame.wave.shape[0] #- for amp 1 and 2
topmin = 0 #- for amp 3 and 4
#- Loop over each amp
for kk in get_amp_ids(frame.meta): # A-D or 1-4
#- get the amp region in pix
ampboundary = parse_sec_keyword(frame.meta["CCDSEC" + kk])
pixboundary.append(ampboundary)
for ispec in range(frame.flux.shape[0]):
if np.all(psf.x(ispec) > ampboundary[1].start):
startspec = ispec
#-cutting off wavelenth boundaries from startspec
yy = psf.y(ispec, frame.wave)
k = np.where(yy > ampboundary[0].start)[0]
startwave0 = k[0]
yy = psf.y(ispec, frame.wave)
k = np.where(yy < ampboundary[0].stop)[0]
endwave0 = k[-1]
break
else:
startspec = None
startwave0 = None
endwave0 = None
if startspec is not None:
for ispec in range(frame.flux.shape[0])[::-1]:
if np.all(psf.x(ispec) < ampboundary[1].stop):
endspec = ispec
#-cutting off wavelenth boundaries from startspec
yy = psf.y(ispec, frame.wave)
k = np.where(yy > ampboundary[0].start)[0]
startwave1 = k[0]
yy = psf.y(ispec, frame.wave)
k = np.where(yy < ampboundary[0].stop)[0]
endwave1 = k[-1]
break
else:
endspec = None
startwave1 = None
endwave1 = None
if startwave0 is not None and startwave1 is not None:
startwave = max(startwave0, startwave1)
else:
startwave = None
if endwave0 is not None and endwave1 is not None:
endwave = min(endwave0, endwave1)
else:
endwave = None
if endspec is not None:
#endspec+=1 #- last entry exclusive in slice, so add 1
#endwave+=1
if endspec < leftmax:
leftmax = endspec
if startspec > rightmin:
rightmin = startspec
if endwave < bottommax:
bottommax = endwave
if startwave > topmin:
topmin = startwave
else:
rightmin = 0 #- Only if no spec in right side of CCD. passing 0 to encertain valid data type. Nontype throws a type error in yaml.dump.
#fiducialb=(slice(startspec,endspec,None),slice(startwave,endwave,None)) #- Note: y,x --> spec, wavelength
#fidboundary.append(fiducialb)
#- return pixboundary,fidboundary
return leftmax, rightmin, bottommax, topmin
def setUp(self):
#- Create temporary calib directory
self.testDir = os.path.join(os.environ['HOME'], 'ql_test_io')
calibDir = os.path.join(self.testDir, 'ql_calib')
if not os.path.exists(calibDir): os.makedirs(calibDir)
#- Generate calib data
for camera in ['b0', 'r0', 'z0']:
#- Fiberflat has to exist but can be a dummpy file
filename = '{}/fiberflat-{}.fits'.format(calibDir, camera)
fx = open(filename, 'w')
fx.write('fiberflat file')
fx.close()
#- PSF has to be real file
psffile = '{}/psf-{}.fits'.format(calibDir, camera)
example_psf = resource_filename(
'desispec', 'test/data/ql/psf-{}.fits'.format(camera))
shutil.copy(example_psf, psffile)
#- Copy test calibration-data.yaml file
specdir = calibDir + "spec/sp0"
if not os.path.isdir(specdir):
os.makedirs(specdir)
for c in "brz":
shutil.copy(
resource_filename('desispec',
'test/data/ql/{}0.yaml'.format(c)),
os.path.join(specdir, "{}0.yaml".format(c)))
#- Set calibration environment variable
os.environ['DESI_SPECTRO_CALIB'] = calibDir
self.rawfile = os.path.join(self.testDir, 'test-raw-abcd.fits')
self.pixfile = os.path.join(self.testDir, 'test-pix-abcd.fits')
self.config = {}
#- rawimage
hdr = dict()
hdr['CAMERA'] = 'b0'
hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
#- Dimensions per amp, not full 4-quad CCD
ny = self.ny = 500
nx = self.nx = 400
noverscan = nover = 50
hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover])
hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover])
hdr['DATASEC2'] = xy2hdr(np.s_[0:ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx])
hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover])
hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover])
hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx])
hdr['NIGHT'] = '20180923'
hdr['EXPID'] = 1
hdr['FLAVOR'] = 'dark'
rawimage = np.zeros((2 * ny, 2 * nx + 2 * noverscan))
offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4}
gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2}
rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6}
quad = {
'1': np.s_[0:ny, 0:nx],
'2': np.s_[0:ny, nx:nx + nx],
'3': np.s_[ny:ny + ny, 0:nx],
'4': np.s_[ny:ny + ny, nx:nx + nx],
}
for amp in ('1', '2', '3', '4'):
hdr['GAIN' + amp] = gain[amp]
hdr['RDNOISE' + amp] = rdnoise[amp]
xy = parse_sec_keyword(hdr['BIASSEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
xy = parse_sec_keyword(hdr['DATASEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
#- raw data are integers, not floats
rawimg = rawimage.astype(np.int32)
self.expid = hdr["EXPID"]
self.camera = hdr["CAMERA"]
#- Confirm that all regions were correctly offset
assert not np.any(rawimage == 0.0)
hdr['DOSVER'] = 'SIM'
hdr['FEEVER'] = 'SIM'
hdr['DETECTOR'] = 'SIM'
desispec.io.write_raw(self.rawfile, rawimg, hdr, camera=self.camera)
self.rawimage = fits.open(self.rawfile)
