def write_bias_subtracted_MEF(self, outfile, gains=None, overwrite=True):
"""
Write a bias-subtracted MEF file with the same format as
the original raw FITS file.
Parameters
----------
outfile: str
Output filename.
gains: dict [None]
Gains to apply to pixel data. If None, then pixel values are
written as ADUs.
overwrite: bool [True]
Flag to overwrite an existing output file.
"""
hdulist = fits.HDUList()
with fits.open(self.imfile) as template:
hdulist.append(template[0])
hdulist[0].header['ORIGFILE'] = hdulist[0].header['FILENAME']
hdulist[0].header['FILENAME'] = outfile
for amp in self:
imarr = self.bias_subtracted_image(amp).getImage().getArray()
if gains is not None:
imarr *= gains[amp]
hdulist.append(
fits.CompImageHDU(data=imarr, header=template[amp].header))
with warnings.catch_warnings():
for warning in (UserWarning, AstropyWarning,
AstropyUserWarning):
warnings.filterwarnings('ignore',
category=warning,
append=True)
fitsWriteto(hdulist, outfile, overwrite=True)
def writeto(self, outfile, pars=None, bitpix=-32, obs_time=None):
ccd_segments = [self.segments[amp] for amp in self.segments]
output = fitsFile(ccd_segments)
if pars is not None:
output[0].header['CCDGAIN'] = pars.system_gain
output[0].header['BIASLVL'] = pars.bias_level
output[0].header['CCDNOISE'] = pars.bias_sigma
output[0].header['RDNOISE'] = pars.read_noise
output[0].header['DARKCURR'] = pars.dark_current
for key, value in self.md.items():
output[0].header[key] = value
if bitpix > 0:
my_round = np.round
else:
#
# Delete any BSCALE and BZERO entries, since we are
# writing image data as floats.
#
try:
del output[amp].header['BSCALE']
del output[amp].header['BZERO']
except KeyError:
pass
my_round = lambda x: x
for hdu in output[1:-2]:
hdu.data = np.array(my_round(hdu.data), dtype=self.dtypes[bitpix])
if obs_time is None:
# Compute the start of the observation from the current time
# minus the exposure time.
obs_time = utcnow(dt=-output[0].header['EXPTIME'])
output[0].header['DATE-OBS'] = obs_time.isot
output[0].header['DATE'] = obs_time.isot
output[0].header.set('MJD-OBS', value=float('%.5f' % obs_time.mjd))
fitsWriteto(output, outfile, clobber=True, checksum=True)
def write(self, outfile, overwrite=True):
"""
Write the results as a FITS binary table.
"""
nrows = sum([len(self[amp]) for amp in self])
output = fits.HDUList()
output.append(fits.PrimaryHDU())
colnames = ['AMPLIFIER', 'XPOS', 'YPOS', 'TRAP_SIZE', 'A0', 'A1']
formats = 'IIIIEE'
units = ['None', 'pixel', 'pixel', 'electrons', 'ADU', 'ADU']
columns = ([np.zeros(nrows, dtype=int)] * 4 +
[np.zeros(nrows, dtype=float)] * 2)
hdu = fitsTableFactory([
fits.Column(name=colname, format=format, unit=unit,
array=column) for colname, format, unit, column in zip(
colnames, formats, units, columns)
])
hdu.name = 'TRAPS'
output.append(hdu)
row = 0
for amp in self:
for xpos, ypos, trap_size, a0, a1 in self[amp]:
output['TRAPS'].data[row]['AMPLIFIER'] = amp
output['TRAPS'].data[row]['XPOS'] = xpos
output['TRAPS'].data[row]['YPOS'] = ypos
output['TRAPS'].data[row]['TRAP_SIZE'] = trap_size
output['TRAPS'].data[row]['A0'] = a0
output['TRAPS'].data[row]['A1'] = a1
row += 1
fitsWriteto(output, outfile, overwrite=overwrite)
def write(self, outfile=None, clobber=True):
"""
Write or update the output file.
"""
if outfile is None:
outfile = self.infile
fitsWriteto(self.output, outfile, clobber=clobber)
def test_raft_image_mosaic(self):
"""
Test of raft-level mosaicking code.
"""
infiles = sorted(
glob.glob(
os.path.join(_root_dir, 'S??', '*_lambda_flat_1000_*.fits')))
infiles = OrderedDict([(filename.split('/')[-2], filename)
for filename in infiles])
test_files = dict()
step = 100
level = step
for slot, infile in list(infiles.items()):
outfile = '%s_test_image_%05i.fits' % (slot, level)
with fits.open(infile) as hdu_list:
for hdu in hdu_list[1:17]:
hdu.data = np.ones(hdu.data.shape,
dtype=np.float32) * level
level += step
fitsWriteto(hdu_list, outfile, overwrite=True)
test_files[slot] = outfile
raft_mosaic = raftTest.RaftMosaic(test_files, bias_subtract=False)
raft_mosaic.plot(title='Test pattern')
plt.savefig(self.outfile)
def write(self, results, outfile, clobber=True):
colnames = ['WAVELENGTH', 'STDEV', 'MEAN']
formats = 'IEE'
my_types = dict((("I", np.int), ("E", np.float)))
columns = [np.zeros(len(results), dtype=my_types[fmt])
for fmt in formats]
units = ['nm', 'rms e-', 'e-']
hdu = fitsTableFactory([fits.Column(name=colnames[i],
format=formats[i],
unit=units[i],
array=columns[i])
for i in range(len(colnames))])
hdu.name = 'PRNU_RESULTS'
for i, wl in enumerate(results.keys()):
hdu.data.field('WAVELENGTH')[i] = wl
hdu.data.field('STDEV')[i] = results[wl][0]
hdu.data.field('MEAN')[i] =results[wl][1]
if os.path.isfile(outfile):
output = fits.open(outfile)
else:
output = fits.HDUList()
output.append(fits.PrimaryHDU())
try:
output[hdu.name] = hdu
except KeyError:
output.append(hdu)
fitsWriteto(output, outfile, clobber=clobber)
def superflat(files,
bias_frame=None,
outfile='superflat.fits',
bitpix=-32,
bias_subtract=True):
"""
The superflat is created by bias-offset correcting the input files
and median-ing them together.
"""
# Get overscan region.
overscan = makeAmplifierGeometry(files[0]).serial_overscan
# Use the first file as a template for the fits output.
output = fits.open(files[0])
for amp in imutils.allAmps(files[0]):
images = afwImage.vectorImageF()
for infile in files:
image = afwImage.ImageF(infile, imutils.dm_hdu(amp))
if bias_subtract:
if bias_frame:
bias_image = afwImage.ImageF(bias_frame,
imutils.dm_hdu(amp))
image = bias_subtracted_image(image, bias_image, overscan)
else:
image -= imutils.bias_image(image,
overscan,
statistic=np.median)
images.push_back(image)
median_image = afwMath.statisticsStack(images, afwMath.MEDIAN)
output[amp].data = median_image.getArray()
if bitpix is not None:
imutils.set_bitpix(output[amp], bitpix)
fitsWriteto(output, outfile, clobber=True)
return outfile
def write(self, times, deferred_charges, outfile, clobber=True):
colnames = ['TIME']
columns = [times]
units = ['s']
for amp in deferred_charges[0]:
colnames.extend(['MEDIAN%02i' % amp, 'STDEV%02i' % amp])
columns.extend([
np.array(
[deferred_charges[i][amp][0] for i in range(len(times))]),
np.array(
[deferred_charges[i][amp][1] for i in range(len(times))])
])
units.extend(['e-/pixel', 'rms e-/pixel'])
formats = ['E'] * len(columns)
HDUList = fits.HDUList()
HDUList.append(fits.PrimaryHDU())
HDUList.append(
fitsTableFactory([
fits.Column(name=colname,
format=format,
unit=unit,
array=column)
for colname, format, unit, column in zip(
colnames, formats, units, columns)
]))
HDUList[-1].name = 'IMAGE_PERSISTENCE_CURVES'
HDUList[0].header['NAMPS'] = len(deferred_charges[0])
fitsWriteto(HDUList, outfile, clobber=clobber)
def write_test_image(outfile,
emin=10,
dark_curr=2e-3,
exptime=10,
gain=5,
ccdtemp=-100,
bias_level=1e2,
bias_sigma=4,
ncols=2,
npix=100):
ccd = sim_tools.CCD(exptime=exptime, gain=gain, ccdtemp=ccdtemp)
ccd.add_bias(bias_level, bias_sigma)
ccd.add_dark_current(level=dark_curr)
#
# Simulation code sets dark pixel values by nsig*seg.sigma(),
# but we want to set using e- per sec, so need to compute
# equivalent nsig.
#
nsig = (emin * exptime /
gain) / ccd.segments[imutils.allAmps[0]].sigma()
#
columns = ccd.generate_dark_cols(ncols)
ccd.add_dark_cols(columns, nsig=nsig)
pixels = ccd.generate_dark_pix(npix)
ccd.add_dark_pix(pixels, nsig=nsig)
fitsWriteto(ccd, outfile)
def write_fits(self, outfile=None, clobber=True):
if outfile is None:
outfile = self.filename
else:
self.filename = outfile
output = fits.HDUList()
output.append(fits.PrimaryHDU(data=self.matrix))
fitsWriteto(output, outfile, clobber=clobber)
def writeFits(ccd_segments, outfile, clobber=True):
output = fitsFile(ccd_segments)
if clobber:
try:
os.remove(outfile)
except OSError:
pass
fitsWriteto(output, outfile, clobber=clobber, checksum=True)
return outfile
def writeFits(ccd_segments, outfile, overwrite=True):
output = fitsFile(ccd_segments)
if overwrite:
try:
os.remove(outfile)
except OSError:
pass
fitsWriteto(output, outfile, overwrite=overwrite, checksum=True)
return outfile
def run(self,
sensor_id,
infiles,
mask_files,
gains,
binsize=1,
bias_frame=None):
outfile = os.path.join(self.config.output_dir,
'%s_ptc.fits' % sensor_id)
all_amps = imutils.allAmps(infiles[0])
ptc_stats = dict([(amp, ([], [])) for amp in all_amps])
exposure = []
file1s = sorted([item for item in infiles if item.find('flat1') != -1])
for flat1 in file1s:
flat2 = find_flat2(flat1)
if self.config.verbose:
self.log.info("processing %s" % flat1)
exposure.append(exptime(flat1))
ccd1 = MaskedCCD(flat1,
mask_files=mask_files,
bias_frame=bias_frame)
ccd2 = MaskedCCD(flat2,
mask_files=mask_files,
bias_frame=bias_frame)
for amp in ccd1:
results = flat_pair_stats(ccd1,
ccd2,
amp,
mask_files=mask_files,
bias_frame=bias_frame)
ptc_stats[amp][0].append(results.flat_mean)
ptc_stats[amp][1].append(results.flat_var)
self._fit_curves(ptc_stats, sensor_id)
output = fits.HDUList()
output.append(fits.PrimaryHDU())
colnames = ['EXPOSURE']
units = ['seconds']
columns = [np.array(exposure, dtype=np.float)]
for amp in all_amps:
colnames.extend(['AMP%02i_MEAN' % amp, 'AMP%02i_VAR' % amp])
units.extend(['ADU', 'ADU**2'])
columns.extend([
np.array(ptc_stats[amp][0], dtype=np.float),
np.array(ptc_stats[amp][1], dtype=np.float)
])
formats = 'E' * len(colnames)
fits_cols = [
fits.Column(name=colnames[i],
format=formats[i],
unit=units[i],
array=columns[i]) for i in range(len(columns))
]
output.append(fitsTableFactory(fits_cols))
output[-1].name = 'PTC_STATS'
output[0].header['NAMPS'] = len(all_amps)
fitsWriteto(output, outfile, clobber=True)
def extract_det_response(self):
outfile = os.path.join(
self.config.output_dir,
'%s_det_response_linearity.fits' % self.sensor_id)
file1s = sorted([
item for item in self.infiles
if item.find('flat1') != -1 or item.find('linearity_flat') != -1
])
if self.config.verbose:
self.log.info("writing to %s" % outfile)
self._create_detresp_fits_output(len(file1s), files1[0])
for row, file1 in enumerate(file1s):
if self.config.verbose:
self.log.info("processing %s" % file1)
try:
file2 = find_flat2(file1)
except IndexError:
# Just use flat1 again since only average is taken and
# FPN subtraction isn't needed.
file2 = file1
flat1 = MaskedCCD(file1,
mask_files=self.mask_files,
bias_frame=self.bias_frame,
linearity_correction=self.linearity_correction)
flat2 = MaskedCCD(file2,
mask_files=self.mask_files,
bias_frame=self.bias_frame,
linearity_correction=self.linearity_correction)
if flat1.md.get('EXPTIME') != flat2.md.get('EXPTIME'):
raise RuntimeError(
"Exposure times do not match for:\n%s\n%s\n" %
(file1, file2))
if (flat1.md.get('MONDIODE') != 0
and flat2.md.get('MONDIODE') != 0):
flux = abs(
flat1.md.get('EXPTIME') * flat1.md.get('MONDIODE') +
flat2.md.get('EXPTIME') * flat2.md.get('MONDIODE')) / 2.
else:
flux = flat1.md.get('EXPTIME')
self.output[-1].data.field('FLUX')[row] = flux
for amp in flat1:
# Convert to e- and write out for each segment.
signal = pair_mean(flat1, flat2, amp) * self.gains[amp]
self.output[-1].data.field('AMP%02i_SIGNAL' %
amp)[row] = signal
self.output[0].header['NAMPS'] = len(flat1)
fitsWriteto(self.output, outfile, overwrite=True)
return outfile
def add_mask_files(mask_files, outfile, overwrite=True):
amp_list = imutils.allAmps(mask_files[0])
masks = dict([(amp, afwImage_Mask(mask_files[0], imutils.dm_hdu(amp)))
for amp in amp_list])
for mask_file in mask_files[1:]:
for amp in masks:
masks[amp] |= afwImage_Mask(mask_file, imutils.dm_hdu(amp))
output = fits.HDUList()
output.append(fits.PrimaryHDU())
output[0].header['MASKTYPE'] = 'SUMMED_MASKS'
fitsWriteto(output, outfile, overwrite=overwrite)
for amp in masks:
md = dafBase.PropertySet()
md.set('EXTNAME', 'SEGMENT%s' % imutils.channelIds[amp])
masks[amp].writeFits(outfile, md, 'a')
return masks
def write_eotest_output(self, BFResults, sensor_id, meanidx=0):
"""Write the correlation curves to a FITS file for plotting,
and the BF results to the eotest results file."""
outfile = os.path.join(self.config.output_dir,
'%s_bf.fits' % sensor_id)
output = fits.HDUList()
output.append(fits.PrimaryHDU())
colnames = []
units = []
columns = []
for amp in BFResults:
colnames.extend(['AMP%02i_XCORR' % amp, 'AMP%02i_XCORR_ERR' % amp,
'AMP%02i_YCORR' % amp, 'AMP%02i_YCORR_ERR' % amp,
'AMP%02i_MEAN' % amp])
units.extend(
['Unitless', 'Unitless', 'Unitless', 'Unitless', 'ADU'])
columns.extend([np.array(BFResults[amp][0], dtype=np.float),
np.array(BFResults[amp][1], dtype=np.float),
np.array(BFResults[amp][2], dtype=np.float),
np.array(BFResults[amp][3], dtype=np.float),
np.array(BFResults[amp][4], dtype=np.float)])
formats = 'E'*len(colnames)
fits_cols = [fits.Column(name=colnames[i], format=formats[i],
unit=units[i], array=columns[i])
for i in range(len(columns))]
output.append(fitsTools.fitsTableFactory(fits_cols))
output[-1].name = 'BF_STATS'
output[0].header['NAMPS'] = len(BFResults)
fitsTools.fitsWriteto(output, outfile, clobber=True)
# Output a file of the coefficients at a given mean, given
# as the index of the exposure in the list.
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
results = EOTestResults(results_file, namps=len(BFResults))
for amp in BFResults:
results.add_seg_result(amp, 'BF_XCORR', BFResults[amp][0][meanidx])
results.add_seg_result(amp, 'BF_XCORR_ERR', BFResults[amp][1][meanidx])
results.add_seg_result(amp, 'BF_YCORR', BFResults[amp][2][meanidx])
results.add_seg_result(amp, 'BF_YCORR_ERR', BFResults[amp][3][meanidx])
results.add_seg_result(amp, 'BF_MEAN', BFResults[amp][4][meanidx])
results.write(clobber=True)
def __call__(self, infile, outfile, overwrite=True):
if self.verbose:
print("processing", infile)
# Recompute the amplifier geometry using NAXIS[12] from
# first image extension of input file.
self.geom.compute_geometry(fitsfile=infile)
self.input = fits.open(infile)
self.output = fits.open(infile, do_not_scale_image_data=True)
prototypes = sensorTest.fits_headers.fits_headers()
# Primary HDU
try:
self.output[0].header.set('MJD',
self.output[0].header['JD'] - 2400000.5)
except KeyError:
from astropy.time import Time
t = Time(self.output[0].header['DATE-OBS'])
self.output[0].header.set('MJD', t.jd - 2400000.5)
self.output[0].header.set('FILENAME', os.path.basename(infile))
self.output[0].header.set('SEQNUM', self._seqnum(infile))
self.output[0].header.set('HEADVER', self._headver())
self.output[0].header.set('LSST_NUM', self.args.sensor_id)
self.output[0].header.set('CCD_MANU', self.args.vendor)
self._update_keywords(0)
# TEST_COND and CCD_COND extensions
self._update_extension('TEST_COND', prototypes['TEST_COND'])
self._update_extension('CCD_COND', prototypes['CCD_COND'])
# Image extensions
self._update_amp_keywords()
# Special handling for monitoring diode current in BNL data.
try:
self.luts[0]['MONDIODE']
except KeyError:
self._set_bnl_mondiode_keyword_value()
fitsWriteto(self.output,
outfile,
overwrite=overwrite,
checksum=True,
output_verify='fix')
def write_fits_tables(self, outfile, clobber=True):
amps = self.qe.keys()
colnames = ['WAVELENGTH']
colnames.extend(['AMP%02i' % i for i in amps])
colnames.append('DEVICE_MEAN')
columns = [self.wlarrs[1]]
columns.extend([self.qe[i] for i in amps])
columns.append(self.ccd_qe)
formats = ["E"] * len(columns)
units = ["nm"]
units.extend(["e-/photon %"] * (len(columns) - 1))
fits_cols = lambda coldata: [
fits.Column(name=colname, format=format, unit=unit, array=column)
for colname, format, unit, column in coldata
]
HDUList = fits.HDUList()
HDUList.append(fits.PrimaryHDU())
HDUList.append(
fitsTableFactory(fits_cols(zip(colnames, formats, units,
columns))))
HDUList[-1].name = 'QE_CURVES'
columns = [self.ccd_qe_band.keys()]
columns.extend([np.array(self.qe_band[amp].values()) for amp in amps])
columns.append(np.array(self.ccd_qe_band.values()))
colnames[0] = 'BAND'
formats[0] = '2A'
units[0] = None
HDUList.append(
fitsTableFactory(fits_cols(zip(colnames, formats, units,
columns))))
HDUList[-1].name = 'QE_BANDS'
HDUList[0].header['NAMPS'] = len(amps)
fitsWriteto(HDUList, outfile, clobber=clobber)
def _write_read_noise_dists(outfile, Ntot, Nsys, gains, bias, sysnoise):
output = fits.HDUList()
output.append(fits.PrimaryHDU())
output[0].header['BIASFILE'] = bias
output[0].header['SYSNFILE'] = str(sysnoise)
for amp in Ntot:
sigtot, sigsys = Ntot[amp], Nsys[amp]
nread_col = fits.Column(name="TOTAL_NOISE",
format="E",
unit="e- rms",
array=sigtot)
nsys_col = fits.Column(name="SYSTEM_NOISE",
format="E",
unit="e- rms",
array=sigsys)
output.append(fitsTableFactory((nread_col, nsys_col)))
output[amp].name = "SEGMENT%s" % imutils.channelIds[amp]
output[0].header["GAIN%s" % imutils.channelIds[amp]] = gains[amp]
output[0].header["SIGTOT%s" % imutils.channelIds[amp]] = \
imutils.median(sigtot)
output[0].header["SIGSYS%s" % imutils.channelIds[amp]] = \
imutils.median(sigsys)
fitsWriteto(output, outfile, clobber=True)
def write_to_fits(self, fits_file):
"""Write this object to a FITS file"""
output = fits.HDUList()
output.append(fits.PrimaryHDU())
col_prof_x = fits.Column(name='prof_x',
format='%iE' % self._nxbins,
unit='ADU',
array=self._prof_x)
col_prof_y = fits.Column(name='prof_y_corr',
format='%iE' % self._nxbins,
unit='ADU',
array=self._prof_y)
col_prof_yerr = fits.Column(name='prof_yerr',
format='%iE' % self._nxbins,
unit='ADU',
array=self._prof_yerr)
fits_cols = [col_prof_x, col_prof_y, col_prof_yerr]
hdu = fitsTableFactory(fits_cols)
hdu.name = 'nonlin'
output.append(hdu)
fitsWriteto(output, fits_file, overwrite=True)
def extract_det_response(self, use_exptime):
max_pd_frac_dev = self.max_pd_frac_dev
outfile = os.path.join(self.config.output_dir,
'%s_det_response.fits' % self.sensor_id)
file1s = sorted(
[item for item in self.infiles if item.find('flat1') != -1])
if self.config.verbose:
self.log.info("writing to %s" % outfile)
self._create_detresp_fits_output(len(file1s))
for row, file1 in enumerate(file1s):
try:
file2 = find_flat2(file1)
except IndexError:
# Just use flat1 again since only average is taken and
# FPN subtraction isn't needed.
file2 = file1
if self.config.verbose:
self.log.info(
"processing\n %s as flat1 and\n %s as flat2" %
(file1, file2))
flat1 = MaskedCCD(file1,
mask_files=self.mask_files,
bias_frame=self.bias_frame)
flat2 = MaskedCCD(file2,
mask_files=self.mask_files,
bias_frame=self.bias_frame)
pd1 = flat1.md.get('MONDIODE')
pd2 = flat2.md.get('MONDIODE')
exptime1 = flat1.md.get('EXPTIME')
exptime2 = flat2.md.get('EXPTIME')
if exptime1 != exptime2:
raise RuntimeError(
"Exposure times do not match for:\n%s\n%s\n" %
(file1, file2))
if (use_exptime or isinstance(pd1, str) or isinstance(pd2, str)
or pd1 == 0 or pd2 == 0):
flux = exptime1
else:
flux = abs(pd1 * exptime1 + pd2 * exptime2) / 2.
if np.abs((pd1 - pd2) / ((pd1 + pd2) / 2.)) > max_pd_frac_dev:
self.log.info(
"Skipping %s and %s since MONDIODE values do not agree to %.1f%%"
% (file1, file2, max_pd_frac_dev * 100.))
continue
if self.config.verbose:
self.log.info(' row = %s' % row)
self.log.info(' pd1, pd2 = %s, %s' % (pd1, pd2))
self.log.info(' exptime1, exptime2 = %s, %s ' %
(exptime1, exptime2))
self.log.info(' flux = %s' % flux)
self.log.info(' flux/exptime = %s' % (flux / exptime1, ))
self.output[-1].data.field('FLUX')[row] = flux
for amp in flat1:
# Convert to e- and write out for each segment.
signal = pair_mean(flat1, flat2, amp) * self.gains[amp]
self.output[-1].data.field('AMP%02i_SIGNAL' %
amp)[row] = signal
self.output[0].header['NAMPS'] = len(flat1)
fitsWriteto(self.output, outfile, clobber=True)
return outfile
amps=tuple(range(1, 17))):
"""
Function to apply distinct levels of parallel cte and/or serial cte
on each amplifier in an input sensor image.
"""
if pcti is None:
pcti = dict([(amp, 0) for amp in amps])
if scti is None:
scti = dict([(amp, 0) for amp in amps])
segments = {}
for amp in amps:
if verbose:
print("apply_cte: working on amp", amp)
image = afwImage.ImageF(infile, imutils.dm_hdu(amp))
geom = sensorTest.makeAmplifierGeometry(infile)
outimage = eotest_utils.ImageTools.applyCTI(image, geom.serial_overscan,
pcti[amp], scti[amp],
verbose)
segments[amp] = outimage
return fitsFile(segments, fits.open(infile))
infiles = sorted(glob.glob('sim_data/000-01_fe55_fe55*.fits'))
nfmin = 0
nfmax = min(len(infiles), 25)
for infile_ in infiles[nfmin:nfmax]:
foo = apply_cte(infile_, pcti=pcti_, scti=scti_, verbose=True)
tokens = os.path.basename(infile_).split('_')
outfile = '_'.join(tuple(tokens[:-1] + ['scti.fits']))
print(outfile)
fitsWriteto(foo, outfile, clobber=True)
def main(sensor_id,
infile,
main_dir,
gain_file=None,
output_dir='./',
no_bias=False):
## Get existing parameter results
param_file = join(main_dir, '{0}_parameter_results.fits'.format(sensor_id))
param_results = OverscanParameterResults.from_fits(param_file)
cti_results = param_results.cti_results
drift_scales = param_results.drift_scales
decay_times = param_results.decay_times
trap_files = [
join(main_dir, '{0}_amp{1}_trap.pkl'.format(sensor_id, i))
for i in range(1, 17)
]
## Get gains
if gain_file is not None:
with open(gain_file, 'rb') as f:
gain_results = pickle.load(f)
gains = gain_results.get_amp_gains(sensor_id)
else:
gains = {i: 1.0 for i in range(1, 17)}
## Output filename
base = splitext(os.path.basename(infile))[0]
outfile = join(output_dir, '{0}_corrected.fits'.format(base))
## Include bias frame for calibration
if no_bias:
bias_frame = None
else:
bias_frame = join(main_dir, '{0}_superbias.fits'.format(sensor_id))
ccd = MaskedCCD(infile, bias_frame=bias_frame)
hdulist = fits.HDUList()
with fits.open(infile) as template:
hdulist.append(template[0])
## Perform correction amp by amp
for amp in range(1, 17):
imarr = ccd.bias_subtracted_image(
amp).getImage().getArray() * gains[amp]
## Electronics Correction
if drift_scales[amp] > 0.:
Linv = electronics_inverse_operator(imarr,
drift_scales[amp],
decay_times[amp],
num_previous_pixels=15)
corrected_imarr = imarr - Linv
else:
corrected_imarr = imarr
## Trap Correction
spltrap = pickle.load(open(trap_files[amp], 'rb'))
Tinv = trap_inverse_operator(corrected_imarr, spltrap)
corrected_imarr = corrected_imarr - (1 - cti_results[amp]) * Tinv
## Reassemble HDUList
hdulist.append(
fits.ImageHDU(data=corrected_imarr / gains[amp],
header=template[amp].header))
with warnings.catch_warnings():
for warning in (UserWarning, AstropyWarning,
AstropyUserWarning):
warnings.filterwarnings('ignore',
category=warning,
append=True)
fitsWriteto(hdulist, outfile, overwrite=True)
def write_results(self, outfile):
"""Export results as a FITs file."""
for amp in range(1, 17):
extname = 'Amp{0:02d}'.format(amp)
nrows1 = len(self.flux[amp])
ncols = len(self.meanrow[amp][0])
meanrow_col = fits.Column('MEANROW',
format='{0}E'.format(ncols),
unit='e-',
array=self.meanrow[amp])
flux_col = fits.Column('FLUX',
format='E',
unit='e-',
array=self.flux[amp])
flux_std_col = fits.Column('FLUX_STD',
format='E',
unit='e-',
array=self.flux_std[amp])
noise_col = fits.Column('NOISE',
format='E',
unit='e-',
array=self.noise[amp])
signal_col = fits.Column('SIGNAL',
format='E',
unit='e-',
array=self.signal[amp])
signal_std_col = fits.Column('SIGNAL_STD',
format='E',
unit='e-',
array=self.signal_std[amp])
tau_col = fits.Column('TAU',
format='E',
unit='None',
array=self.tau[amp])
tau_std_col = fits.Column('TAU_STD',
format='E',
unit='None',
array=self.tau_std[amp])
cti_col = fits.Column('CTI',
format='E',
unit='None',
array=self.cti[amp])
cti_std_col = fits.Column('CTI_STD',
format='E',
unit='None',
array=self.cti_std[amp])
try:
#
# Append new rows if HDU for this segment already exists
#
table_hdu = self.output[extname]
row0 = table_hdu.header['NAXIS2']
nrows = row0 + nrows1
table_hdu = fitsTableFactory(table_hdu.data, nrows=nrows)
table_hdu.data['MEANROW'][row0:] = meanrow_col
table_hdu.data['FLUX'][row0:] = flux_col
table_hdu.data['FLUX_STD'][row0:] = flux_std_col
table_hdu.data['NOISE'][row0:] = noise_col
table_hdu.data['SIGNAL'][row0:] = signal_col
table_hdu.data['SIGNAL_STD'][row0:] = signal_std_col
table_hdu.data['TAU'][row0:] = tau_col
table_hdu.data['TAU_STD'][row0:] = tau_std_col
table_hdu.data['CTI'][row0:] = cti_col
table_hdu.data['CTI_STD'][row0:] = cti_std_col
table_hdu.name = extname
self.output[extname] = table_hdu
except KeyError:
self.output.append(
fitsTableFactory([
meanrow_col, flux_col, flux_std_col, noise_col,
signal_col, signal_std_col, tau_col, tau_std_col,
cti_col, cti_std_col
]))
self.output[-1].name = extname
self.output[0].header['NAMPS'] = 16
fitsWriteto(self.output, outfile, overwrite=True, checksum=True)
def rolloff_mask(infile, outfile,
mask_plane='ROLLOFF_DEFECTS',
tmp_mask_image=None,
outer_edge_width=10,
bloom_stop_width=5,
signal=10,
cleanup=True):
"""
This function creates a file containing masks for each segment to
mask pixels affected the edge roll-off and midline blooming stop
distortions.
infile: Input file to mask.
outfile: The name of the file to contain the masks.
outer_edge_width: This is the width in pixels of the masked region
along the sides closest to the sensor edges of the imaging region
of each segment.
bloom_stop_width: The width in pixels of the masked region of the
imaging region of each segment, on either side of the central
blooming stop implant.
signal: This is the artificial signal written to the mask image so
that the afwDetect code can generate the footprints used define
the masks. It basically just needs to be some positive (non-zero)
number.
"""
#
# Create an empty set of frames to fill with an image of the mask
# from which we will generate the masks. The exposure time and
# system gain need to be set to unity so that the BrightPixels.py
# code can interpret DN directly as e- per second.
#
amp_geom = makeAmplifierGeometry(infile)
gain = 1
exptime = 1
all_amps = imutils.allAmps(infile)
ccd = CCD(exptime=exptime, gain=gain, geometry=amp_geom, amps=all_amps)
#
# Write the output file with a primary HDU so that the DMstack code
# can append only image extensions (and not write to the PHDU).
#
warnings.filterwarnings('ignore', category=fits.verify.VerifyWarning, append=True)
hdulist = fits.HDUList()
hdulist.append(fits.PrimaryHDU())
with fits.open(infile) as fd:
hdulist[0].header.update(fd[0].header)
# Use the mask_plane value ('ROLLOFF_DEFECTS') to distinguish
# this file from other mask files.
hdulist[0].header['MASKTYPE'] = mask_plane
fitsWriteto(hdulist, outfile, overwrite=True)
#
# Amplifiers 1 (AMP10), 8 (AMP17), 9 (AMP07) and 16 (AMP00) are
# along the perimeter, but have different edge rolloff pixels
# depending on the vendor.
#
# These amps have edge roll-off adjacent to the prescan.
amps = (8, 16) if amp_geom.vendor == 'E2V' else (8, 9)
xmin = amp_geom.imaging.getMinX()
xmax = xmin + outer_edge_width
for amp in amps:
if amp not in all_amps:
continue
imarr = ccd.segments[amp].image.getArray()
imarr[:, xmin:xmax] += signal
#
# These amps have edge roll-off adjacent to the serial overscan:
#
amps = (1, 9) if amp_geom.vendor == 'E2V' else (1, 16)
xmax = amp_geom.imaging.getMaxX() + 1
xmin = xmax - outer_edge_width
for amp in amps:
if amp not in all_amps:
continue
imarr = ccd.segments[amp].image.getArray()
imarr[:, xmin:xmax] += signal
#
# Loop over all amps, set signal in perimeter and around blooming
# stop.
#
ymax = amp_geom.imaging.getMaxY() + 1
ymin = ymax - bloom_stop_width
for i, amp in enumerate(ccd.segments):
image = ccd.segments[amp].image
imarr = image.getArray()
#
# Set signal in row direction along perimeter.
#
xmin = amp_geom.imaging.getMinX()
xmax = amp_geom.imaging.getMaxX()
imarr[0:outer_edge_width, xmin:xmax] += signal
if amp_geom.amp_loc == amp_loc['E2V']:
#if True:
#
# Set signal around blooming stop
#
imarr[ymin:ymax, :] += signal
#
# Write the images of the mask regions to the FITS file.
#
if tmp_mask_image is None:
tmp_mask_image = tempfile.mkstemp(suffix='.fits', dir='.')[-1]
fitsWriteto(ccd, tmp_mask_image)
#
# Use BrightPixels code to detect the mask regions and write the mask file.
#
try:
mask = afwImage.Mask(image.getDimensions())
except AttributeError:
mask = afwImage.MaskU(image.getDimensions())
mask.addMaskPlane(mask_plane)
maskedCCD = MaskedCCD(tmp_mask_image)
pixels, columns = {}, {}
for amp in maskedCCD:
bright_pixels = BrightPixels(maskedCCD, amp, exptime, gain,
ethresh=signal/2., mask_plane=mask_plane)
pixels[amp], columns[amp] = bright_pixels.find()
generate_mask(infile, outfile, mask_plane, pixels=pixels, columns=columns)
if cleanup:
os.remove(tmp_mask_image)
def generate_mask(infile,
outfile,
mask_plane,
pixels=None,
columns=None,
temp_mask_image=None):
"""
Generate a mask file for the specified pixels and columns.
The amplifier geometry will be taken from infile.
"""
# Insert artificial signal into specified pixels and columns for
# each segment.
exptime = 1
gain = 1
geometry = makeAmplifierGeometry(infile)
ccd = CCD(exptime=exptime, gain=gain, geometry=geometry)
# Account for prescan in x-coordinate since we want to mask
# columns the full segment.
prescan = geometry.prescan.getWidth()
signal = 10
if pixels is None:
pixels = {}
for amp in pixels:
imarr = ccd.segments[amp].image.getArray()
for ix, iy in pixels[amp]:
imarr[iy][ix + prescan] = signal
if columns is None:
columns = {}
for amp in columns:
imarr = ccd.segments[amp].image.getArray()
for ix in columns[amp]:
imarr[:, ix + prescan] = signal
if temp_mask_image is None:
temp_mask_image = tempfile.mkstemp(suffix='.fits', dir='.')[-1]
fitsWriteto(ccd, temp_mask_image)
# Use the afw code to create a mask file.
afwImage.MaskU.addMaskPlane(mask_plane)
# Create a primary HDU with the input file primary HDU metadata,
# updated with the mask type info.
hdulist = fits.HDUList()
hdulist.append(fits.open(infile)[0])
hdulist[0].header['MASKTYPE'] = mask_plane
fitsWriteto(hdulist, outfile, clobber=True)
# Loop over segments in the temporary file and add all pixels to
# the mask with the inserted signal.
maskedCCD = MaskedCCD(temp_mask_image)
for amp in maskedCCD:
threshold = afwDetect.Threshold(signal / 2. * exptime / gain)
fp_set = afwDetect.FootprintSet(maskedCCD[amp], threshold)
mask = afwImage.MaskU(maskedCCD[amp].getDimensions())
fp_set.setMask(mask, mask_plane)
md = dafBase.PropertySet()
md.set('EXTNAME', 'SEGMENT%s' % imutils.channelIds[amp])
md.set('DETSIZE', geometry[amp]['DETSIZE'])
md.set('DETSEC', geometry[amp]['DETSEC'])
md.set('DATASEC', geometry[amp]['DATASEC'])
mask.writeFits(outfile, md, 'a')
os.remove(temp_mask_image)
def main(sensor_id, infile, output_dir='./', cti=None, do_trapping=False,
do_electronics=False):
processed_file = os.path.join(output_dir,
'{0}_processed.fits'.format(sensor_id))
corrected_file = os.path.join(output_dir,
'{0}_corrected.fits'.format(sensor_id))
## CTI parameters
if cti is None:
do_cti = False
cti = 0.0
else:
do_cti = True
cti_dict = {amp : cti for amp in range(1, 17)}
## Trapping parameters
if do_trapping:
# traps = [LinearTrap(4.0, 0.4, 1, 0.08),
# LogisticTrap(1000.0, 0.4, 1, 60000., 0.0002)]
traps = [LinearTrap(4.0, 0.4, 1, 0.08),
LogisticTrap(40.0, 0.4, 1, 17500., 0.001)]
else:
traps = None
traps_dict = {amp : traps for amp in range(1, 17)}
## Electronics parameters
if do_electronics:
scale = 0.0002
decay_time = 2.5
output_amps = {amp : FloatingOutputAmplifier(1.0, scale, decay_time) for amp in range(1, 17)}
else:
output_amps = {amp : BaseOutputAmplifier(1.0) for amp in range(1, 17)}
## Process infile
imsim = ImageSimulator.from_image_fits(infile, output_amps,
cti=cti_dict, traps=traps_dict)
imarr_results = imsim.image_readout(infile, outfile=processed_file)
ccd = MaskedCCD(processed_file)
hdulist = fits.HDUList()
with fits.open(infile) as template:
hdulist.append(template[0])
hdulist[0].header['ORIGFILE'] = hdulist[0].header['FILENAME']
hdulist[0].header['FILENAME'] = corrected_file
for amp in range(1, 17):
imarr = ccd.bias_subtracted_image(amp).getImage().getArray()
## Electronics Correction
if do_electronics:
E = electronics_operator(imarr, scale, decay_time, num_previous_pixels=15)
corrected_imarr = imarr - E
else:
corrected_imarr = imarr
## Trap Correction
if do_trapping:
T = trap_operator(imarr, *traps)
corrected_imarr = corrected_imarr - (1-cti)*T
else:
pass
## CTI Correction
if do_cti:
Dinv_cti = cti_inverse_operator(cti, imarr.shape[1])
for i in range(imarr.shape[0]):
corrected_imarr[i, :] = Dinv_cti*corrected_imarr[i, :]
else:
pass
hdulist.append(fits.ImageHDU(data=corrected_imarr,
header=template[amp].header))
with warnings.catch_warnings():
for warning in (UserWarning, AstropyWarning,
AstropyUserWarning):
warnings.filterwarnings('ignore', category=warning,
append=True)
fitsWriteto(hdulist, corrected_file, overwrite=True)
def generate_mask(infile,
outfile,
mask_plane,
pixels=None,
columns=None,
temp_mask_image=None):
"""
Generate a mask file for the specified pixels and columns.
The amplifier geometry will be taken from infile.
"""
# Insert artificial signal into specified pixels and columns for
# each segment.
exptime = 1
gain = 1
geometry = makeAmplifierGeometry(infile)
amps = imutils.allAmps(infile)
ccd = CCD(exptime=exptime, gain=gain, geometry=geometry, amps=amps)
# Account for prescan in x-coordinate since we want to mask
# columns the full segment.
prescan = geometry.prescan.getWidth()
signal = 10
if pixels is None:
pixels = {}
for amp in pixels:
imarr = ccd.segments[amp].image.getArray()
for ix, iy in pixels[amp]:
imarr[iy][ix + prescan] = signal
if columns is None:
columns = {}
for amp in columns:
imarr = ccd.segments[amp].image.getArray()
for ix in columns[amp]:
imarr[:, ix + prescan] = signal
if temp_mask_image is None:
temp_mask_image = tempfile.mkstemp(suffix='.fits', dir='.')[-1]
fitsWriteto(ccd, temp_mask_image)
# Use the afw code to create a mask file.
try:
afwImage.Mask.addMaskPlane(mask_plane)
except AttributeError:
afwImage.MaskU.addMaskPlane(mask_plane)
# Loop over segments in the temporary file and add all pixels to
# the mask with the inserted signal.
with fits.open(infile) as hdus:
hdus[0].header['MASKTYPE'] = mask_plane
hdus[0].header['FILENAME'] = outfile
maskedCCD = MaskedCCD(temp_mask_image)
for amp in maskedCCD:
threshold = afwDetect.Threshold(signal / 2. * exptime / gain)
fp_set = afwDetect.FootprintSet(maskedCCD[amp], threshold)
try:
mask = afwImage.Mask(maskedCCD[amp].getDimensions())
except AttributeError:
mask = afwImage.MaskU(maskedCCD[amp].getDimensions())
fp_set.setMask(mask, mask_plane)
hdus[amp].data = mask.array
# add mask plane keywords
for key, value in mask.getMaskPlaneDict().items():
hdus[amp].header['MP_' + key] = value
hdus.writeto(outfile, overwrite=True)
os.remove(temp_mask_image)
def extract_det_response(self, use_exptime):
max_pd_frac_dev = self.max_pd_frac_dev
outfile = os.path.join(self.config.output_dir,
'%s_det_response.fits' % self.sensor_id)
file1s = sorted(
[item for item in self.infiles if item.find('flat1') != -1])
if self.config.verbose:
self.log.info("writing to %s" % outfile)
self._create_detresp_fits_output(len(file1s), file1s[0])
for row, file1 in enumerate(file1s):
try:
file2 = self.find_flat2(file1)
except IndexError:
# Just use flat1 again since only average is taken and
# FPN subtraction isn't needed.
file2 = file1
if self.config.verbose:
self.log.info("processing\n %s as flat1 and\n %s as flat2",
file1, file2)
flat1 = MaskedCCD(file1,
mask_files=self.mask_files,
bias_frame=self.bias_frame,
linearity_correction=self.linearity_correction)
flat2 = MaskedCCD(file2,
mask_files=self.mask_files,
bias_frame=self.bias_frame,
linearity_correction=self.linearity_correction)
seqnum = flat1.md.get('SEQNUM')
exptime1 = flat1.md.get('EXPTIME')
exptime2 = flat2.md.get('EXPTIME')
if exptime1 != exptime2:
raise RuntimeError(
"Exposure times do not match for:\n%s\n%s\n" %
(file1, file2))
if self.mondiode_func is None:
pd1 = flat1.md.get('MONDIODE')
pd2 = flat2.md.get('MONDIODE')
else:
try:
pd1 = self.mondiode_func(file1, exptime1)
pd2 = self.mondiode_func(file2, exptime2)
except KeyError as eobj:
self.log.info(
"KeyError exception computing pd current:\n" +
str(eobj))
continue
if use_exptime:
flux = exptime1
else:
flux = abs(pd1 * exptime1 + pd2 * exptime2) / 2.
if np.abs((pd1 - pd2) / ((pd1 + pd2) / 2.)) > max_pd_frac_dev:
self.log.info(
"Skipping %s and %s since MONDIODE values "
"do not agree to %.1f%%", file1, file2,
max_pd_frac_dev * 100.)
continue
if self.config.verbose:
self.log.info(' row = %s', row)
self.log.info(' pd1, pd2 = %s, %s', pd1, pd2)
self.log.info(' exptime1, exptime2 = %s, %s ', exptime1,
exptime2)
self.log.info(' flux = %s', flux)
self.log.info(' flux/exptime = %s', flux / exptime1)
self.output[-1].data.field('FLUX')[row] = flux
for amp in flat1:
# Convert to e- and write out for each segment.
signal, sig1, sig2 \
= pair_mean(flat1, flat2, amp)*self.gains[amp]
colname = 'AMP%02i_SIGNAL' % amp
self.output[-1].data.field(colname)[row] = signal
self.output[-1].data.field('FLAT1_' + colname)[row] = sig1
self.output[-1].data.field('FLAT2_' + colname)[row] = sig2
self.output[-1].data.field('SEQNUM')[row] = seqnum
self.output[0].header['NAMPS'] = len(flat1)
fitsWriteto(self.output, outfile, overwrite=True)
return outfile
def write_results(self, outfile='fe55_psf_params.fits'):
self.output[0].header['NAMPS'] = len(self.amp_set)
fitsWriteto(self.output, outfile, clobber=True, checksum=True)
def run(self, sensor_id, dark_files, mask_files, gains, bias_frame=None):
imutils.check_temperatures(dark_files,
self.config.temp_set_point_tol,
setpoint=self.config.temp_set_point,
warn_only=True)
median_images = {}
md = imutils.Metadata(dark_files[0], 1)
for amp in imutils.allAmps(dark_files[0]):
median_images[amp] = imutils.fits_median(dark_files,
imutils.dm_hdu(amp))
medfile = os.path.join(self.config.output_dir,
'%s_median_dark_current.fits' % sensor_id)
imutils.writeFits(median_images, medfile, dark_files[0])
ccd = MaskedCCD(medfile, mask_files=mask_files, bias_frame=bias_frame)
dark95s = {}
exptime = md.get('EXPTIME')
if self.config.verbose:
self.log.info("Amp 95 percentile median")
dark_curr_pixels = []
dark_curr_pixels_per_amp = {}
for amp in ccd:
imaging_region = ccd.amp_geom.imaging
overscan = ccd.amp_geom.serial_overscan
image = imutils.unbias_and_trim(ccd[amp].getImage(), overscan,
imaging_region)
mask = imutils.trim(ccd[amp].getMask(), imaging_region)
imarr = image.getArray()
mskarr = mask.getArray()
pixels = imarr.reshape(1, imarr.shape[0] * imarr.shape[1])[0]
masked = mskarr.reshape(1, mskarr.shape[0] * mskarr.shape[1])[0]
unmasked = [
pixels[i] for i in range(len(pixels)) if masked[i] == 0
]
unmasked.sort()
unmasked = np.array(unmasked) * gains[amp] / exptime
dark_curr_pixels_per_amp[amp] = unmasked
dark_curr_pixels.extend(unmasked)
try:
dark95s[amp] = unmasked[int(len(unmasked) * 0.95)]
median = unmasked[len(unmasked) / 2]
except IndexError as eobj:
print str(eobj)
dark95s[amp] = -1.
median = -1.
if self.config.verbose:
self.log.info("%2i %.2e %.2e" %
(amp, dark95s[amp], median))
#
# Compute 95th percentile dark current for CCD as a whole.
#
dark_curr_pixels = sorted(dark_curr_pixels)
darkcurr95 = dark_curr_pixels[int(len(dark_curr_pixels) * 0.95)]
dark95mean = np.mean(dark95s.values())
if self.config.verbose:
#self.log.info("CCD: mean 95 percentile value = %s" % dark95mean)
self.log.info("CCD-wide 95 percentile value = %s" % darkcurr95)
#
# Update header of dark current median image file with dark
# files used and dark95 values, and write dark95 values to the
# eotest results file.
#
results_file = self.config.eotest_results_file
if results_file is None:
results_file = os.path.join(self.config.output_dir,
'%s_eotest_results.fits' % sensor_id)
results = EOTestResults(results_file, namps=len(ccd))
output = fits.open(medfile)
for i, dark in enumerate(dark_files):
output[0].header['DARK%02i' % i] = os.path.basename(dark)
# Write overall dark current 95th percentile
results.output['AMPLIFIER_RESULTS'].header['DARK95'] = darkcurr95
for amp in ccd:
output[0].header['DARK95%s' %
imutils.channelIds[amp]] = dark95s[amp]
results.add_seg_result(amp, 'DARK_CURRENT_95', dark95s[amp])
fitsWriteto(output, medfile, clobber=True, checksum=True)
results.write(clobber=True)
return dark_curr_pixels_per_amp, dark95s
