def action(self, ccd):
"""
Trim an image to bounds given in the widget.
Returns
-------
trimmed : `ccdproc.CCDData`
Trimmed image.
"""
# Don't do anything if not activated
if not self.toggle.value:
pass
whole_axis = slice(None, None)
partial_axis = slice(self._start.value, self._stop.value)
# create a two-element list which will be filled with the appropriate
# slice based on the widget settings.
if self._pre.value == 0:
trimmed = ccdproc.trim_image(ccd[whole_axis, partial_axis])
else:
trimmed = ccdproc.trim_image(ccd[partial_axis, whole_axis])
return trimmed
def action(self, ccd):
"""
Trim an image to bounds given in the widget.
Returns
-------
trimmed : `ccdproc.CCDData`
Trimmed image.
"""
# Don't do anything if not activated
if not self.toggle.value:
pass
whole_axis = slice(None, None)
partial_axis = slice(self._axis_selection.start,
self._axis_selection.stop)
# create a two-element list which will be filled with the appropriate
# slice based on the widget settings.
if self._axis_selection.full_axis == 0:
trimmed = ccdproc.trim_image(ccd[whole_axis, partial_axis])
else:
trimmed = ccdproc.trim_image(ccd[partial_axis, whole_axis])
return trimmed
def process(ccd, gain, oscan, tsec):
"""Basic CCD processing for required for data
"""
#oscan subtract
ccd = ccdproc.subtract_overscan(ccd, overscan=oscan, median=True)
#gain correct
ccd = ccdproc.gain_correct(ccd, gain, gain_unit=u.electron / u.adu)
#trim the image
ccd = ccdproc.trim_image(ccd, fits_section=tsec)
return ccd
def process(ccd, gain, oscan, tsec):
"""Basic CCD processing for required for data
"""
#oscan subtract
ccd = ccdproc.subtract_overscan(ccd, overscan=oscan, median=True)
#gain correct
ccd = ccdproc.gain_correct(ccd, gain, gain_unit=u.electron/u.adu)
#trim the image
ccd = ccdproc.trim_image(ccd, fits_section=tsec)
return ccd
def oscan_trim_file(fname, datahdus=0):
hdulist = fits.open(fname)
nhdus = len(hdulist)
if nhdus > 1:
istart = 1
else:
istart = 0
# loop from first-data to last HDU, unless datahdus is set
hduindexes = list(range(nhdus))[istart:]
if datahdus != 0:
hduindexes = datahdus
for i in hduindexes:
hdulist = fits.open(fname)
data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
data1.header = hdulist[i].header
# What happens if file is already overscan-subtracted?
# We should probably default to using a model
if modeling:
oscan1 = ccdproc.subtract_overscan(
data1,
fits_section=data1.header['BIASSEC'],
add_keyword={
'overscan': True,
'calstat': 'O'
},
model=models.Polynomial1D(1))
else:
oscan1 = ccdproc.subtract_overscan(
data1,
fits_section=data1.header['BIASSEC'],
add_keyword={
'overscan': True,
'calstat': 'O'
},
model=None)
trim1 = ccdproc.trim_image(oscan1,
fits_section=oscan1.header['TRIMSEC'],
add_keyword={
'trimmed': True,
'calstat': 'OT'
})
fits.update(fname, trim1.data, header=trim1.header, ext=i)
hdulist.close()
mylog("Overscan and trim {0}".format(fname))
return
def oscan_and_trim(image_list):
"""
Remove overscan and trim a list of images. The original list is replaced by a list of images
with the changes applied.
Implementation done by ccdproc
Parameters:
----------
image_list :: List of CCDData corresponding to images
"""
for idx, img in enumerate(image_list):
oscan = ccdproc.subtract_overscan(img,overscan=img[:,521:544], add_keyword={'oscan_sub': True, 'calstat': 'O'}, model=models.Polynomial1D(1))
image_list[idx] = ccdproc.trim_image(oscan[:,10:521], add_keyword={'trimmed': True, 'calstat': 'OT'})
def loader_generator(actually_load_data):
# Ideally we would just return a shape instead of an empty data array in
# the case where actually_load_data is false, but we can't use
# `ccdproc.trim_image()` without having a CCDData in hand, so to get the
# right shape we're going to need to create a full CCDData anyway.
for fits_path in glob(join(dirname, '*.fits')):
# `astropy.nddata.ccddata.fits_ccddata_reader` only opens FITS from
# filenames, not from an open HDUList, which means that creating
# multiple CCDDatas from the same FITS file rapidly becomes
# inefficient. So, we emulate its logic.
with fits.open(fits_path) as hdu_list:
for idx, hdu in enumerate(hdu_list):
if idx == 0:
header0 = hdu.header
else:
hdr = hdu.header
hdr.extend(header0, unique=True)
# This ccddata function often generates annoying warnings
with warnings.catch_warnings():
warnings.simplefilter('ignore')
hdr, wcs = ccddata._generate_wcs_and_update_header(
hdr)
# Note: we skip all the unit-handling logic here since the LSST
# sim data I'm using don't have anything useful.
if actually_load_data:
data = hdu.data
else:
data = np.empty(hdu.shape, dtype=np.void)
ccd = ccddata.CCDData(data,
meta=hdr,
unit=unit,
wcs=wcs)
ccd = ccdproc.trim_image(
ccd, fits_section=ccd.header['DATASEC'])
yield (f'{fits_path}:{idx}', ccd)
def trim_bias(refresh=False):
biascollection = ImageFileCollection('HD115709/bias', ext=4)
flag = 0
tbiaspathlist=[]
if refresh == True:
for ccdb, biasn in biascollection.ccds(return_fname=True, ccd_kwargs={'unit': 'adu'}):
print('trimming', biasn)
ccdb.header['imtype'] = ('bias', 'type of image')
if flag == 0:
print('all biases will be trimmed to :', ccdb.meta['trimsec'])
flag = 1
tbias = ccdp.trim_image(ccdb, fits_section=str(ccdb.meta['trimsec']))
tbias.meta['imtype'] = ('trimmed bias', 'type of image')
tbias.meta['taxis1'] = (2048, 'dimension1')
tbias.meta['taxis2'] = (4096, 'dimension2')
tbias.write('Trimmed_Bias/' + biasn[0:8] + '_trim.fits', overwrite=True)
tbiaspathlist.append('Trimmed_Bias/' + biasn[0:8] + '_trim.fits')
return biascollection,tbiaspathlist
def oscan_trim_file(fname):
hdulist = fits.open(fname)
nhdus = len(hdulist)
if nhdus > 1:
istart = 1
else:
istart = 0
# loop from first-data to last HDU.
for i in range(nhdus)[istart:]:
hdulist = fits.open(fname)
data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
data1.header = hdulist[i].header
# What happens if file is already overscan-subtracted?
if modeling:
oscan1 = ccdproc.subtract_overscan(
data1,
fits_section=data1.header['BIASSEC'],
add_keyword={
'overscan': True,
'calstat': 'O'
},
model=models.Polynomial1D(1))
else:
oscan1 = ccdproc.subtract_overscan(
data1,
fits_section=data1.header['BIASSEC'],
add_keyword={
'overscan': True,
'calstat': 'O'
},
model=None)
trim1 = ccdproc.trim_image(oscan1,
fits_section=oscan1.header['TRIMSEC'],
add_keyword={
'trimmed': True,
'calstat': 'OT'
})
fits.update(fname, trim1.data, header=trim1.header, ext=i)
hdulist.close()
return
def _correct_overscan_hdu(hdu_ccd):
"""
Given an HDU loaded as a CCDData object perform overscan correction.
Arguments
---------
hdu_ccd : astropy.nddata.CCDData
HDU to perform correction
Returns
-------
img_trim : astropy.nddata.CCDData
Corrected HDU
"""
img_osub = (ccdproc.subtract_overscan(
hdu_ccd,
fits_section=hdu_ccd.header['BIASSEC'],
model=None,
median=True,
add_keyword={
'overscan': True,
'calstat': 'O'
}))
img_trim = (ccdproc.trim_image(img_osub,
fits_section=img_osub.header['TRIMSEC'],
add_keyword={
'trimmed': True,
'calstat': 'OT'
}))
# Updating header and overwriting processed image
del img_trim.header['BIASSEC']
del img_trim.header['TRIMSEC']
return img_trim
def create_master_flat(list_files, flat_filter=None, fitsfile=None, bias=None, fits_section=None, gain=None,
method='median', key_filter='filter', dfilter={'imagetyp':'FLAT'}, mask=None, key_find='find',
invert_find=False, sjoin=','):
if gain is not None and not isinstance(gain, u.Quantity):
gain = gain * u.electron / u.adu
lflat = []
if dfilter is not None and key_filter is not None and flat_filter is not None:
dfilter = addKeysListDict(dfilter, {key_filter: flat_filter})
list_files = getListFiles(list_files, dfilter, mask, key_find=key_find, invert_find=invert_find)
if len(list_files) == 0:
print ('WARNING: No FLAT files available for filter "%s"' % flat_filter)
return
for filename in list_files:
ccd = CCDData.read(filename, unit= u.adu)
trimmed = True if fits_section is not None else False
ccd = ccdproc.trim_image(ccd, fits_section=fits_section, add_keyword={'trimmed': trimmed})
if gain is not None:
ccd = ccdproc.gain_correct(ccd, gain)
if bias is not None:
if isinstance(bias, str):
bias = fits2CCDData(bias, single=True)
ccd = ccdproc.subtract_bias(ccd, bias)
lflat.append(ccd)
combine = ccdproc.combine(lflat, method=method)
if gain is not None and not 'GAIN' in combine.header:
combine.header.set('GAIN', gain.value, gain.unit)
combine.header['CGAIN'] = True if gain is not None else False
combine.header['IMAGETYP'] = 'FLAT'
combine.header['CMETHOD'] = method
combine.header['CCDVER'] = VERSION
addKeyHdr(combine.header, 'MBIAS', getFilename(bias))
if sjoin is not None:
combine.header['LFLAT'] = sjoin.join([os.path.basename(fits) for fits in list_files])
combine.header['NFLAT'] = len(list_files)
if fitsfile is not None:
combine.header['FILENAME'] = os.path.basename(fitsfile)
combine.write(fitsfile, clobber=True)
return combine
def view_slice(image,
window,
axis=0,
fig_size=(20, 10),
title=None,
traces=None):
trimmed_data = trim_image(image[window[1]:window[3]+1, window[0]:window[2]+1]).data
if axis == 1:
trimmed_data = np.rot90(trimmed_data, k=-1)
elif axis != 0:
raise ValueError("Axis must be 0 or 1, got {}.".format(axis))
projection = np.ma.mean(trimmed_data, axis=0)
figure_ar = fig_size[0] / fig_size[1]
slice_ar = trimmed_data.shape[1] / trimmed_data.shape[0]
f, (ax1, ax2) = plt.subplots(2,
1,
sharex=True,
gridspec_kw={'height_ratios': (1.5, (slice_ar / figure_ar - 1.5))})
ax1.imshow(trimmed_data, origin='lower')
ax2.plot(projection)
ax1.set_xlim(0, trimmed_data.shape[1] - 1)
ax2.set_ylim(0, projection.max() * 1.05)
if traces:
if axis == 1:
mean_pos = (window[2] + window[0]) / 2
else:
mean_pos = (window[3] + window[1]) / 2
ax2.scatter(traces(mean_pos), np.zeros_like(traces(mean_pos)), marker=2, color='g')
f.set_size_inches(fig_size)
if title:
ax1.set_title(title)
plt.tight_layout()
def createmasterbias():
#create biaslist
biaslist = sorted(glob.glob('HD115709/bias/r*.fit'))
print('Number of biases used =', len(biaslist))
#open bias zero for cube dimensions
hdub = fits.open(biaslist[0])
bias0 = hdub[4].data
print(biaslist[0], 'is open, shape:', bias0.shape)
hdub.close()
print(biaslist[0], 'is closed')
#create biascube with shape of science and len(biaslist)
biascube = np.zeros((science0.shape[0], science0.shape[1], len(biaslist)),
dtype=bias0.dtype)
print('biascube created with shape :', biascube.shape)
#crop and stack biases (biasc= counter, and biasn = name)
for biasc, biasn in enumerate(biaslist):
print('Open :', biasn)
hdu = fits.open(biaslist[biasc])
if hdu[4].header['CHIPNAME'] == 'A5382-1-7':
bias = CCDData(hdu[4].data, unit=u.adu)
tbias = ccdproc.trim_image(bias, fits_section=sciencewindow)
biascube[:, :, biasc] = tbias.data
hdu.close()
else:
hdu.close()
print(biasn, 'returned wrong chipname for selected extension')
exit
#take median and write to disk
mbias = np.nanmedian(biascube, 2)
ccdmbias = CCDData(mbias, unit=u.adu)
ccdmbias.write('tmaster_bias.fits', overwrite=True)
print('master bias shape :', ccdmbias.data.shape)
def createmasterflat():
#create flatlist
flatlist = sorted(glob.glob('HD115709/flat_SII/r*.fit'))
print('Number of flats used =', len(flatlist))
#open flat0 for cube len
hduf = fits.open(flatlist[0])
flat0 = hduf[4].data
print(flatlist[0], 'is open, shape:', flat0.shape)
hduf.close()
print(flatlist[0], 'is closed')
#create flatcube with shape of science and len(flatlist)
flatcube = np.zeros((science0.shape[0], science0.shape[1], len(flatlist)),
dtype=flat0.dtype)
#convert trim and populate flatcube after bias correction
for flatc, flatn in enumerate(flatlist):
print('Open :', flatn)
hdu = fits.open(flatlist[flatc])
if hdu[4].header['CHIPNAME'] == 'A5382-1-7':
ccdflat = CCDData(hdu[4].data, unit=u.adu)
ccdtflat = ccdproc.trim_image(ccdflat, fits_section=sciencewindow)
ccdcflat = ccdproc.subtract_bias(ccdtflat, ccdmbias_use)
flatcube[:, :, flatc] = ccdcflat.data
hdu.close()
else:
hdu.close()
print(flatn, 'returned wrong chipname for selected extension')
exit
#take median and write to disk
mflat = np.nanmedian(flatcube, 2)
ccdmflat = CCDData(mflat, unit=u.adu)
ccdmflat.write('tmaster_flat.fits',
overwrite=True) # write the fits to disk
def main(night_path, skip_list_file, mask_file, overwrite=False, plot=False):
"""
See argparse block at bottom of script for description of parameters.
"""
night_path = path.realpath(path.expanduser(night_path))
if not path.exists(night_path):
raise IOError("Path '{}' doesn't exist".format(night_path))
logger.info("Reading data from path: {}".format(night_path))
base_path, night_name = path.split(night_path)
data_path, run_name = path.split(base_path)
output_path = path.realpath(
path.join(data_path, 'processed', run_name, night_name))
os.makedirs(output_path, exist_ok=True)
logger.info("Saving processed files to path: {}".format(output_path))
if plot: # if we're making plots
plot_path = path.realpath(path.join(output_path, 'plots'))
logger.debug("Will make and save plots to: {}".format(plot_path))
os.makedirs(plot_path, exist_ok=True)
else:
plot_path = None
# check for files to skip (e.g., saturated or errored exposures)
if skip_list_file is not None: # a file containing a list of filenames to skip
with open(skip_list_file, 'r') as f:
skip_list = [x.strip() for x in f if x.strip()]
else:
skip_list = None
# look for pixel mask file
if mask_file is not None:
with open(
mask_file, 'r'
) as f: # load YAML file specifying pixel masks for nearby sources
pixel_mask_spec = yaml.load(f.read())
else:
pixel_mask_spec = None
# generate the raw image file collection to process
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
logger.info("Frames to process:")
logger.info("- Bias frames: {}".format(
len(ic.files_filtered(imagetyp='BIAS'))))
logger.info("- Flat frames: {}".format(
len(ic.files_filtered(imagetyp='FLAT'))))
logger.info("- Comparison lamp frames: {}".format(
len(ic.files_filtered(imagetyp='COMP'))))
logger.info("- Object frames: {}".format(
len(ic.files_filtered(imagetyp='OBJECT'))))
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
# ============================
# Create the master bias frame
# ============================
# overscan region of the CCD, using FITS index notation
oscan_fits_section = "[{}:{},:]".format(oscan_idx, oscan_idx + oscan_size)
master_bias_file = path.join(output_path, 'master_bias.fits')
if not os.path.exists(master_bias_file) or overwrite:
# get list of overscan-subtracted bias frames as 2D image arrays
bias_list = []
for hdu, fname in ic.hdus(return_fname=True, imagetyp='BIAS'):
logger.debug('Processing Bias frame: {0}'.format(fname))
ccd = CCDData.read(path.join(ic.location, fname), unit='adu')
ccd = ccdproc.gain_correct(ccd, gain=ccd_gain)
ccd = ccdproc.subtract_overscan(ccd, overscan=ccd[:, oscan_idx:])
ccd = ccdproc.trim_image(ccd,
fits_section="[1:{},:]".format(oscan_idx))
bias_list.append(ccd)
# combine all bias frames into a master bias frame
logger.info("Creating master bias frame")
master_bias = ccdproc.combine(bias_list,
method='average',
clip_extrema=True,
nlow=1,
nhigh=1,
error=True)
master_bias.write(master_bias_file, overwrite=True)
else:
logger.info("Master bias frame file already exists: {}".format(
master_bias_file))
master_bias = CCDData.read(master_bias_file)
if plot:
# TODO: this assumes vertical CCD
assert master_bias.shape[0] > master_bias.shape[1]
aspect_ratio = master_bias.shape[1] / master_bias.shape[0]
fig, ax = plt.subplots(1, 1, figsize=(10, 12 * aspect_ratio))
vmin, vmax = zscaler.get_limits(master_bias.data)
cs = ax.imshow(master_bias.data.T,
origin='bottom',
cmap=cmap,
vmin=max(0, vmin),
vmax=vmax)
ax.set_title('master bias frame [zscale]')
fig.colorbar(cs)
fig.tight_layout()
fig.savefig(path.join(plot_path, 'master_bias.png'))
plt.close(fig)
# ============================
# Create the master flat field
# ============================
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
master_flat_file = path.join(output_path, 'master_flat.fits')
if not os.path.exists(master_flat_file) or overwrite:
# create a list of flat frames
flat_list = []
for hdu, fname in ic.hdus(return_fname=True, imagetyp='FLAT'):
logger.debug('Processing Flat frame: {0}'.format(fname))
ccd = CCDData.read(path.join(ic.location, fname), unit='adu')
ccd = ccdproc.gain_correct(ccd, gain=ccd_gain)
ccd = ccdproc.ccd_process(ccd,
oscan=oscan_fits_section,
trim="[1:{},:]".format(oscan_idx),
master_bias=master_bias)
flat_list.append(ccd)
# combine into a single master flat - use 3*sigma sigma-clipping
logger.info("Creating master flat frame")
master_flat = ccdproc.combine(flat_list,
method='average',
sigma_clip=True,
low_thresh=3,
high_thresh=3)
master_flat.write(master_flat_file, overwrite=True)
# TODO: make plot if requested?
else:
logger.info("Master flat frame file already exists: {}".format(
master_flat_file))
master_flat = CCDData.read(master_flat_file)
if plot:
# TODO: this assumes vertical CCD
assert master_flat.shape[0] > master_flat.shape[1]
aspect_ratio = master_flat.shape[1] / master_flat.shape[0]
fig, ax = plt.subplots(1, 1, figsize=(10, 12 * aspect_ratio))
vmin, vmax = zscaler.get_limits(master_flat.data)
cs = ax.imshow(master_flat.data.T,
origin='bottom',
cmap=cmap,
vmin=max(0, vmin),
vmax=vmax)
ax.set_title('master flat frame [zscale]')
fig.colorbar(cs)
fig.tight_layout()
fig.savefig(path.join(plot_path, 'master_flat.png'))
plt.close(fig)
# =====================
# Process object frames
# =====================
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
logger.info("Beginning object frame processing...")
for hdu, fname in ic.hdus(return_fname=True, imagetyp='OBJECT'):
new_fname = path.join(output_path, 'p_{}'.format(fname))
# -------------------------------------------
# First do the simple processing of the frame
# -------------------------------------------
logger.debug("Processing '{}' [{}]".format(hdu.header['OBJECT'],
fname))
if path.exists(new_fname) and not overwrite:
logger.log(1, "\tAlready processed! {}".format(new_fname))
ext = SourceCCDExtractor(filename=path.join(
ic.location, new_fname),
plot_path=plot_path,
zscaler=zscaler,
cmap=cmap,
**ccd_props)
nccd = ext.ccd
# HACK: F**K this is a bad hack
ext._filename_base = ext._filename_base[2:]
else:
# process the frame!
ext = SourceCCDExtractor(filename=path.join(ic.location, fname),
plot_path=plot_path,
zscaler=zscaler,
cmap=cmap,
unit='adu',
**ccd_props)
_pix_mask = pixel_mask_spec.get(
fname, None) if pixel_mask_spec is not None else None
nccd = ext.process_raw_frame(pixel_mask_spec=_pix_mask,
master_bias=master_bias,
master_flat=master_flat)
nccd.write(new_fname, overwrite=overwrite)
# -------------------------------------------
# Now do the 1D extraction
# -------------------------------------------
fname_1d = path.join(output_path, '1d_{0}'.format(fname))
if path.exists(fname_1d) and not overwrite:
logger.log(1, "\tAlready extracted! {}".format(fname_1d))
continue
else:
logger.debug("\tExtracting to 1D")
# first step is to fit a voigt profile to a middle-ish row to determine LSF
lsf_p = ext.get_lsf_pars() # MAGIC NUMBER
try:
tbl = ext.extract_1d(lsf_p)
except Exception as e:
logger.error('Failed! {}: {}'.format(e.__class__.__name__,
str(e)))
continue
hdu0 = fits.PrimaryHDU(header=nccd.header)
hdu1 = fits.table_to_hdu(tbl)
hdulist = fits.HDUList([hdu0, hdu1])
hdulist.writeto(fname_1d, overwrite=overwrite)
del ext
# ==============================
# Process comparison lamp frames
# ==============================
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
logger.info("Beginning comp. lamp frame processing...")
for hdu, fname in ic.hdus(return_fname=True, imagetyp='COMP'):
new_fname = path.join(output_path, 'p_{}'.format(fname))
logger.debug("\tProcessing '{}'".format(hdu.header['OBJECT']))
if path.exists(new_fname) and not overwrite:
logger.log(1, "\tAlready processed! {}".format(new_fname))
ext = CompCCDExtractor(filename=path.join(ic.location, new_fname),
plot_path=plot_path,
zscaler=zscaler,
cmap=cmap,
**ccd_props)
nccd = ext.ccd
# HACK: F**K this is a bad hack
ext._filename_base = ext._filename_base[2:]
else:
# process the frame!
ext = CompCCDExtractor(filename=path.join(ic.location, fname),
plot_path=plot_path,
unit='adu',
**ccd_props)
_pix_mask = pixel_mask_spec.get(
fname, None) if pixel_mask_spec is not None else None
nccd = ext.process_raw_frame(
pixel_mask_spec=_pix_mask,
master_bias=master_bias,
master_flat=master_flat,
)
nccd.write(new_fname, overwrite=overwrite)
# -------------------------------------------
# Now do the 1D extraction
# -------------------------------------------
fname_1d = path.join(output_path, '1d_{0}'.format(fname))
if path.exists(fname_1d) and not overwrite:
logger.log(1, "\tAlready extracted! {}".format(fname_1d))
continue
else:
logger.debug("\tExtracting to 1D")
try:
tbl = ext.extract_1d()
except Exception as e:
logger.error('Failed! {}: {}'.format(e.__class__.__name__,
str(e)))
continue
hdu0 = fits.PrimaryHDU(header=nccd.header)
hdu1 = fits.table_to_hdu(tbl)
hdulist = fits.HDUList([hdu0, hdu1])
hdulist.writeto(fname_1d, overwrite=overwrite)
def hrs_process(image_name, ampsec=[], oscansec=[], trimsec=[],
masterbias=None, error=False, bad_pixel_mask=None, flip=False,
rdnoise=None, oscan_median=True, oscan_model=None):
"""Processing required for HRS observations. If the images have multiple
amps, then this will process each part of the image and recombine them
into for the final results
Parameters
----------
image_name: str
Name of file to be processed
ampsec: list
List of ampsections. This list should have the same length as the
number of amps in the data set. The sections should be given
in the format of fits_sections (see below).
oscansec: list
List of overscan sections. This list should have the same length as the
number of amps in the data set. The sections should be given
in the format of fits_sections (see below).
trimsec: list
List of overscan sections. This list should have the same length as the
number of amps in the data set. The sections should be given
in the format of fits_sections (see below).
error: boolean
If True, create an uncertainty array for ccd
masterbias: None, `~numpy.ndarray`, or `~ccdproc.CCDData`
A materbias frame to be subtracted from ccd.
bad_pixel_mask: None or `~numpy.ndarray`
A bad pixel mask for the data. The bad pixel mask should be in given
such that bad pixels havea value of 1 and good pixels a value of 0.
flip: boolean
If True, the image will be flipped such that the orders run from the
bottom of the image to the top and the dispersion runs from the left
to the right.
rdnoise: None or `~astropy.Quantity`
Read noise for the observations. The read noise should be in
`~astropy.units.electron`
oscan_median : bool, optional
If true, takes the median of each line. Otherwise, uses the mean
oscan_model : `~astropy.modeling.Model`, optional
Model to fit to the data. If None, returns the values calculated
by the median or the mean.
Returns
-------
ccd: `~ccdproc.CCDData`
Data processed and
Notes
-----
The format of the `fits_section` string follow the rules for slices that
are consistent with the FITS standard (v3) and IRAF usage of keywords like
TRIMSEC and BIASSEC. Its indexes are one-based, instead of the
python-standard zero-based, and the first index is the one that increases
most rapidly as you move through the array in memory order, opposite the
python ordering.
The 'fits_section' argument is provided as a convenience for those who are
processing files that contain TRIMSEC and BIASSEC. The preferred, more
pythonic, way of specifying the overscan is to do it by indexing the data
array directly with the `overscan` argument.
"""
# read in the data
ccd = ccdproc.CCDData.read(image_name, unit=u.adu)
try:
namps = ccd.header['CCDAMPS']
except KeyError:
namps = ccd.header['CCDNAMPS']
# thow errors for the wrong number of amps
if len(ampsec) != namps:
raise ValueError('Number of ampsec does not equal number of amps')
if len(oscansec) != namps:
raise ValueError('Number of oscansec does not equal number of amps')
if len(trimsec) != namps:
raise ValueError('Number of trimsec does not equal number of amps')
if namps == 1:
if ccd.header['OBSTYPE']=='Bias':
gain = None
else:
gain = float(ccd.header['gain'].split()[0]) * u.electron / u.adu
nccd = ccd_process(ccd, oscan=oscansec[0], trim=trimsec[0],
error=error, masterbias=masterbias,
bad_pixel_mask=bad_pixel_mask, gain=gain,
rdnoise=rdnoise, oscan_median=oscan_median,
oscan_model=oscan_model)
else:
ccd_list = []
xsize = 0
ysize = 0
gain_ave = 0
#determine size of the image
ys, xs = ccd.data.shape
x1=0
x2=0
y1=0
y2=ys
for i in range(namps):
amp = ccdproc.utils.slices.slice_from_string(ampsec[i], fits_convention=True)
ay1, ay2, _ = amp[0].indices(ys)
ax1, ax2, _ = amp[1].indices(xs)
trim = ccdproc.utils.slices.slice_from_string(trimsec[i], fits_convention=True)
ty1, ty2, _ = trim[0].indices(ys)
tx1, tx2, _ = trim[1].indices(xs)
dx1 = max(tx1, ax1)
x1 = min(x1 or dx1, dx1)
#assumes bias frames are given in increasing x-order
x2 = ax2 - (ax2-ax1-tx2)
data = np.zeros((y2-y1, x2-x1))
if ccd.mask is not None:
mask = np.zeros(((y2-y1, x2-x1)))
else:
mask = None
if ccd.uncertainty is not None:
raise NotImplementedError(
'Support for uncertainties not implimented yet')
else:
uncertainty = None
for i in range(namps):
gain_ave += float(ccd.header['gain'].split()[i])
gain_ave = gain_ave / namps * u.electron / u.adu
for i in range(namps):
# trim the image to just the amplifier
cc = ccdproc.trim_image(ccd, fits_section=ampsec[i])
# determine its position in the area after correction
amp = ccdproc.utils.slices.slice_from_string(ampsec[i], fits_convention=True)
ay1, ay2, _ = amp[0].indices(ys)
ax1, ax2, _ = amp[1].indices(xs)
trim = ccdproc.utils.slices.slice_from_string(trimsec[i], fits_convention=True)
ty1, ty2, _ = trim[0].indices(ys)
tx1, tx2, _ = trim[1].indices(xs)
dx1 = max(ax1 - x1, 0)
dx2 = ax2 - x1
dy1 = ay1
dy2 = ay2
# gain correct the arrray
gain = float(ccd.header['gain'].split()[i]) * u.electron / u.adu
ncc = ccd_process(cc, oscan=oscansec[i], trim=trimsec[i],
error=False, masterbias=None, gain=gain,
bad_pixel_mask=None, rdnoise=rdnoise,
oscan_median=oscan_median,
oscan_model=oscan_model)
# normalize all to have the same gain
ncc = ncc.divide(gain_ave)
data[dy1:dy2,dx1:dx2] = ncc.data
if ccd.mask is not None:
mask[dy1:dy2,dx1:dx2] = ncc.mask
# create new entryy
nccd = ccdproc.CCDData(data, unit=ncc.unit, mask=mask,
uncertainty=uncertainty)
nccd.header = ccd.header
nccd = ccd_process(nccd, masterbias=masterbias, error=error, gain=gain_ave,
rdnoise=rdnoise, bad_pixel_mask=bad_pixel_mask)
if flip:
nccd.data = nccd.data[::-1, ::-1]
if (nccd.mask is not None):
nccd.mask = nccd.mask[::-1, ::-1]
if (nccd.uncertainty is not None):
nccd.uncertainty = nccd.uncertainty[::-1, ::-1]
return nccd
def process_fits(fitspath,
*,
obstype=None,
object=None,
exposure_times=None,
percentile=None,
percentile_min=None,
percentile_max=None,
window=None,
darks=None,
cosmic_ray=False,
cosmic_ray_kwargs={},
gain=None,
readnoise=None,
normalise=False,
normalise_func=np.ma.average,
combine_type=None,
sigma_clip=False,
low_thresh=3,
high_thresh=3):
"""Combine all FITS images of a given type and exposure time from a given directory.
Parameters
----------
fitspath: str
Path to the FITS images to process. Can be a path to a single file, or a path to a
directory. If the latter the directory will be searched for FITS files and checked
against criteria from obstype, object, exposure_times critera.
obstype: str, optional
Observation type, an 'OBSTYPE' FITS header value e.g. 'DARK', 'OBJ'. If given only files
with matching OBSTYPE will be processed.
object: str, optional
Object name, i.e. 'OBJECT' FITS header value. If given only files with matching OBJECT
will be processed.
exposure_times: float or sequence, optional
Exposure time(s), i.e 'TOTALEXP' FITS header value(s). If given only files with matching
TOTALEXP will be processed.
percentile: float, optional
If given will only images whose percentile value fall between percentile_min and
percentile_max will be processed, e.g. set to 50.0 to select images by median value,
set to 99.5 to select images by their 99.5th percentile value.
percentile_min: float, optional
Minimum percentile value.
percentile_max: float, optional
Maximum percentil value.
window: (int, int, int, int), optional
If given will trim images to the window defined as (x0, y0, x1, y1), where (x0, y0)
and (x1, y1) are the coordinates of the bottom left and top right corners.
darks: str or sequence, optional
Filename(s) of dark frame(s) to subtract from the image(s). If given a dark frame with
matching TOTALEXP will be subtracted from each image during processing.
cosmic_ray: bool, optional
Whether to perform single image cosmic ray removal, using the lacosmic algorithm,
default False. Requires both gain and readnoise to be set.
cosmic_ray_kwargs: dict, optional
Additional keyword arguments to pass to the ccdproc.cosmicray_lacosmic function.
gain: str or astropy.units.Quantity, optional
Either a string indicating the FITS keyword corresponding to the (inverse gain), or
a Quantity containing the gain value to use. If both gain and read noise are given
an uncertainty frame will be created.
readnoise: str or astropy.units.Quantity, optional
Either a string indicating the FITS keyword corresponding to read noise, or a Quantity
containing the read noise value to use. If both read noise and gain are given then an
uncertainty frame will be created.
normalise: bool, optional
If True each image will be normalised. Default False.
normalise_func: callable, optional
Function to use for normalisation. Each image will be divided by normalise_func(image).
Default np.ma.average.
combine_type: str, optional
Type of image combination to use, 'MEAN' or 'MEDIAN'. If None the individual
images will be processed but not combined and the return value will be a list of
CCDData objects. Default None.
sigma_clip: bool, optional
If True will perform sigma clipping on the image stack before combining, default=False.
low_thresh: float, optional
Lower threshold to use for sigma clipping, in standard deviations. Default is 3.0.
high_thresh: float, optional
Upper threshold to use for sigma clipping, in standard deviations. Default is 3.0.
Returns
-------
master: ccdproc.CCDData
Combined image.
"""
if exposure_times:
try:
# Should work for any sequence or iterable type
exposure_times = set(exposure_times)
except TypeError:
# Not a sequence or iterable, try using as a single value.
exposure_times = {
float(exposure_times),
}
if darks:
try:
dark_filenames = set(darks)
except TypeError:
dark_filenames = {
darks,
}
dark_dict = {}
for filename in dark_filenames:
try:
dark_data = CCDData.read(filename)
except ValueError:
# Might be no units in FITS header. Assume ADU.
dark_data = CCDData.read(filename, unit='adu')
dark_dict[dark_data.header['totalexp']] = dark_data
if combine_type and combine_type not in ('MEAN', 'MEDIAN'):
raise ValueError(
"combine_type must be 'MEAN' or 'MEDIAN', got '{}''".format(
combine_type))
fitspath = Path(fitspath)
if fitspath.is_file():
# FITS path points to a single file, turn into a list.
filenames = [
fitspath,
]
elif fitspath.is_dir():
# FITS path is a directory. Find FITS file and collect values of selected FITS headers
ifc = ImageFileCollection(fitspath, keywords='*')
if len(ifc.files) == 0:
raise RuntimeError("No FITS files found in {}".format(fitspath))
# Filter by observation type.
if obstype:
try:
ifc = ifc.filter(obstype=obstype)
except FileNotFoundError:
raise RuntimeError(
"No FITS files with OBSTYPE={}.".format(obstype))
# Filter by object name.
if object:
try:
ifc = ifc.filter(object=object)
except FileNotFoundError:
raise RuntimeError(
"No FITS files with OBJECT={}.".format(object))
filenames = [
Path(ifc.location).joinpath(filename) for filename in ifc.files
]
else:
raise ValueError(
"fitspath '{}' is not an accessible file or directory.".format(
fitspath))
# Load image(s) and process them.
images = []
for filename in filenames:
try:
ccddata = CCDData.read(filename)
except ValueError:
# Might be no units in FITS header. Assume ADU.
ccddata = CCDData.read(filename, unit='adu')
# Filtering by exposure times here because it's hard filter ImageFileCollection
# with an indeterminate number of possible values.
if not exposure_times or ccddata.header['totalexp'] in exposure_times:
if window:
ccddata = ccdproc.trim_image(ccddata[window[1]:window[3] + 1,
window[0]:window[2] + 1])
if percentile:
# Check percentile value is within specified range, otherwise skip to next image.
percentile_value = np.percentile(ccddata.data, percentile)
if percentile_value < percentile_min or percentile_value > percentile_max:
continue
if darks:
try:
ccddata = ccdproc.subtract_dark(
ccddata,
dark_dict[ccddata.header['totalexp']],
exposure_time='totalexp',
exposure_unit=u.second)
except KeyError:
raise RuntimeError(
"No dark with matching totalexp for {}.".format(
filename))
if gain:
if isinstance(gain, str):
egain = ccddata.header[gain]
egain = egain * u.electron / u.adu
elif isinstance(gain, u.Quantity):
try:
egain = gain.to(u.electron / u.adu)
except u.UnitsError:
egain = (1 / gain).to(u.electron / u.adu)
else:
raise ValueError(
f"gain must be a string or Quantity, got {gain}.")
if readnoise:
if isinstance(readnoise, str):
rn = ccddata.header[readnoise]
rn = rn * u.electron
elif isinstance(readnoise, u.Quantity):
try:
rn = readnoise.to(u.electron / u.pixel)
except u.UnitsError:
rn = (readnoise * u.pixel).to(u.electron)
else:
raise ValueError(
f"readnoise must be a string or Quantity, got {readnoise}."
)
if gain and readnoise:
ccddata = ccdproc.create_deviation(ccddata,
gain=egain,
readnoise=rn,
disregard_nan=True)
if gain:
ccddata = ccdproc.gain_correct(ccddata, gain=egain)
if cosmic_ray:
if not gain and readnoise:
raise ValueError(
"Cosmic ray removal required both gain & readnoise.")
ccddata = ccdproc.cosmicray_lacosmic(
ccddata,
gain=1.0, # ccddata already gain corrected
readnoise=rn,
**cosmic_ray_kwargs)
if normalise:
ccddata = ccddata.divide(normalise_func(ccddata.data))
images.append(ccddata)
n_images = len(images)
if n_images == 0:
msg = "No FITS files match exposure time criteria"
raise RuntimeError(msg)
if n_images == 1 and combine_type:
warn(
"Combine type '{}' selected but only 1 matching image, skipping image combination.'"
)
combine_type = None
if combine_type:
combiner = Combiner(images)
# Sigma clip data
if sigma_clip:
if combine_type == 'MEAN':
central_func = np.ma.average
else:
# If not MEAN has to be MEDIAN, checked earlier that it was one or the other.
central_func = np.ma.median
combiner.sigma_clipping(low_thresh=low_thresh,
high_thresh=high_thresh,
func=central_func)
# Stack images.
if combine_type == 'MEAN':
master = combiner.average_combine()
else:
master = combiner.median_combine()
# Populate header of combined image with metadata about the processing.
master.header['fitspath'] = str(fitspath)
if obstype:
master.header['obstype'] = obstype
if exposure_times:
if len(exposure_times) == 1:
master.header['totalexp'] = float(exposure_times.pop())
else:
master.header['totalexp'] = tuple(exposure_times)
master.header['nimages'] = n_images
master.header['combtype'] = combine_type
master.header['sigclip'] = sigma_clip
if sigma_clip:
master.header['lowclip'] = low_thresh
master.header['highclip'] = high_thresh
else:
# No image combination, just processing indivudal image(s)
if n_images == 1:
master = images[0]
else:
master = images
return master
outdir = sys.argv[2]
if not os.path.isdir(outdir): os.mkdir(outdir)
os.chdir(outdir)
#change this to point to your raw data directory
ic1 = ImageFileCollection(indir)
#create the bias frames
blue_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Blue arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
blue_bias_list.append(ccd)
master_bias_blue = ccdproc.combine(blue_bias_list, method='median')
master_bias_blue.write('master_bias_blue.fits', clobber=True)
red_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Red arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
red_bias_list.append(ccd)
master_bias_red = ccdproc.combine(red_bias_list, method='median')
master_bias_red.write('master_bias_red.fits', clobber=True)
def combine_ccd(fitslist=None,
summary_table=None,
table_filecol="file",
trim_fits_section=None,
output=None,
unit='adu',
subtract_frame=None,
combine_method='median',
reject_method=None,
normalize_exposure=False,
normalize_average=False,
exposure_key='EXPTIME',
mem_limit=2e9,
combine_uncertainty_function=None,
extension=0,
type_key=None,
type_val=None,
dtype="float32",
uncertainty_dtype="float32",
output_verify='fix',
overwrite=False,
verbose=True,
**kwargs):
''' Combining images
Slight variant from ccdproc.
# TODO: accept the input like ``sigma_clip_func='median'``, etc.
Parameters
----------
fitslist: path-like, list of path-like, or list of CCDData
The list of path to FITS files or the list of CCDData to be
stacked. It is useful to give list of CCDData if you have
already stacked/loaded FITS file into a list by your own
criteria. If ``None`` (default), you must give ``fitslist`` or
``summary_table``. If it is not ``None``, this function will do
very similar job to that of ``ccdproc.combine``. Although it is
not a good idea, a mixed list of CCDData and paths to the files
is also acceptable.
summary_table: pandas.DataFrame or astropy.table.Table
The table which contains the metadata of files. If there are
many FITS files and you want to use stacking many times, it is
better to make a summary table by ``filemgmt.make_summary`` and
use that instead of opening FITS files' headers every time you
call this function. If you want to use ``summary_table`` instead
of ``fitslist`` and have set ``loadccd=True``, you must not have
``None`` or ``NaN`` value in the
``summary_table[table_filecol]``.
table_filecol: str
The column name of the ``summary_table`` which contains the path
to the FITS files.
trim_fits_section : str or None, optional
The ``fits_section`` of ``ccdproc.trim_image``. Region of
``ccd`` from which the overscan is extracted; see
`~ccdproc.subtract_overscan` for details.
Default is ``None``.
output : path-like or None, optional.
The path if you want to save the resulting ``ccd`` object.
Default is ``None``.
unit : `~astropy.units.Unit` or str, optional.
The units of the data.
Default is ``'adu'``.
subtract_frame : array-like, optional.
The frame you want to subtract from the image after the
combination. It can be, e.g., dark frame, because it is easier
to calculate Poisson error before the dark subtraction and
subtract the dark later.
TODO: This maybe unnecessary.
Default is ``None``.
combine_method : str or None, optinal.
The ``method`` for ``ccdproc.combine``, i.e., {'average',
'median', 'sum'}
Default is ``None``.
reject_method : str
Made for simple use of ``ccdproc.combine``, [None, 'minmax',
'sigclip' == 'sigma_clip', 'extrema' == 'ext']. Automatically
turns on the option, e.g., ``clip_extrema = True`` or
``sigma_clip = True``. Leave it blank for no rejection.
Default is ``None``.
normalize_exposure : bool, optional.
Whether to normalize the values by the exposure time of each
frame before combining.
Default is ``False``.
normalize_average : bool, optional.
Whether to normalize the values by the average value of each
frame before combining.
Default is ``False``.
exposure_key : str, optional
The header keyword for the exposure time.
Default is ``"EXPTIME"``.
combine_uncertainty_function : callable, None, optional
The uncertainty calculation function of ``ccdproc.combine``. If
``None`` use the default uncertainty func when using average,
median or sum combine, otherwise use the function provided.
Default is ``None``.
extension: int or str, optional
The extension to be used.
Default is ``0``.
dtype : str or `numpy.dtype` or None, optional
Allows user to set dtype. See `numpy.array` ``dtype`` parameter
description. If ``None`` it uses ``np.float64``.
Default is ``None``.
type_key, type_val: str, list of str
The header keyword for the ccd type, and the value you want to
match. For an open HDU named ``hdu``, e.g., only the files which
satisfies ``hdu[extension].header[type_key] == type_val`` among
all the ``fitslist`` will be used.
output_verify : str
Output verification option. Must be one of ``"fix"``,
``"silentfix"``, ``"ignore"``, ``"warn"``, or ``"exception"``.
May also be any combination of ``"fix"`` or ``"silentfix"`` with
``"+ignore"``, ``+warn``, or ``+exception" (e.g.
``"fix+warn"``). See the astropy documentation below:
http://docs.astropy.org/en/stable/io/fits/api/verification.html#verify
mem_limit : float, optional
Maximum memory which should be used while combining (in bytes).
Default is ``2.e9``.
**kwarg:
kwargs for the ``ccdproc.combine``. See its documentation. This
includes (RHS are the default values)
```
weights=None,
scale=None,
mem_limit=16000000000.0,
clip_extrema=False,
nlow=1,
nhigh=1,
minmax_clip=False,
minmax_clip_min=None,
minmax_clip_max=None,
sigma_clip=False,
sigma_clip_low_thresh=3,
sigma_clip_high_thresh=3,
sigma_clip_func=<numpy.ma.core._frommethod instance>,
sigma_clip_dev_func=<numpy.ma.core._frommethod instance>,
combine_uncertainty_function=None, **ccdkwargs
```
Returns
-------
master: astropy.nddata.CCDData
Resulting combined ccd.
'''
def _set_reject_method(reject_method):
''' Convenience function for ccdproc.combine reject switches
'''
clip_extrema, minmax_clip, sigma_clip = False, False, False
if reject_method in ['extrema', 'ext']:
clip_extrema = True
elif reject_method in ['minmax']:
minmax_clip = True
elif reject_method in ['sigma_clip', 'sigclip']:
sigma_clip = True
else:
if reject_method not in [None, 'no']:
raise KeyError("reject must be one in [None, 'minmax', " +
"'sigclip'=='sigma_clip', 'extrema'=='ext']")
return clip_extrema, minmax_clip, sigma_clip
def _print_info(combine_method, Nccd, reject_method, **kwargs):
if reject_method is None:
reject_method = 'no'
info_str = ('"{:s}" combine {:d} images by "{:s}" rejection')
print(info_str.format(combine_method, Nccd, reject_method))
print(dict(**kwargs))
return
# def _normalize_exptime(ccdlist, exposure_key):
# _ccdlist = ccdlist.copy()
# exptimes = []
# for i in range(len(_ccdlist)):
# exptime = _ccdlist[i].header[exposure_key]
# exptimes.append(exptime)
# _ccdlist[i] = _ccdlist[i].divide(exptime)
# if verbose:
# if len(np.unique(exptimes)) != 1:
# print('There are more than one exposure times:\n\t', end=' ')
# print(np.unique(exptimes), end=' ')
# print('seconds')
# print(f'Normalized images by exposure time ("{exposure_key}").')
# return _ccdlist
def _add_and_print(s, header, verbose):
header.add_history(s)
if verbose:
print(s)
# Give only one
if ((fitslist is not None) + (summary_table is not None) != 1):
raise ValueError(
"One and only one of [fitslist, summary_table] must be given.")
# If fitslist
if fitslist is not None:
try:
fitslist = list(fitslist)
except TypeError:
raise TypeError("fitslist must be convertable to list. " +
f"It's now {type(fitslist)}.")
# If summary_table
if summary_table is not None:
if ((not isinstance(summary_table, Table))
and (not isinstance(summary_table, pd.DataFrame))):
raise TypeError(
"summary_table must be an astropy Table or Pandas " +
f"DataFrame. It's now {type(summary_table)}.")
# Check for type_key and type_val
if ((type_key is None) ^ (type_val is None)):
raise ValueError(
"type_key and type_val must be both specified or both None.")
if (output is not None) and (Path(output).exists()):
if overwrite:
print(f"{output} already exists:\n\tBut will be overridden.")
else:
print(f"{output} already exists:")
return load_if_exists(output, loader=CCDData.read, if_not=None)
# Do we really need to accept all three of normalize & scale?
# if scale is None:
# scale = np.ones(len(ccdlist))
if (((normalize_average) + (normalize_exposure)) > 1):
raise ValueError("Only up to one of [normalize_average, " +
"normalize_exposure] must be not None.")
# Set history messages
str_history = ('{:d} images with {:s} = {:s} are "{:s}" combined ' +
'using "{:s}" rejection (additional kwargs: {})')
str_nexp = "Each frame normalized by exposure time before combination."
str_navg = "Each frame normalized by average value before combination."
str_subt = "Subtracted a user-provided frame"
str_trim = "Trim by FITS section {}"
if reject_method is None:
reject_method = 'no'
# Select CCDs by
ccdlist = stack_FITS(fitslist=fitslist,
summary_table=summary_table,
table_filecol=table_filecol,
extension=extension,
unit=unit,
type_key=type_key,
type_val=type_val,
loadccd=False)
# trim_fits_section=trim_fits_section,
# loadccd=False: Loading CCD here may cause memory blast...
try:
header = ccdlist[0].header
except AttributeError:
header = fits.getheader(ccdlist[0])
if verbose:
_print_info(combine_method=combine_method,
Nccd=len(ccdlist),
reject_method=reject_method,
dtype=dtype,
**kwargs)
scale = None
# Normalize by exposure
if normalize_exposure:
tmp = make_summary(fitslist=fitslist,
keywords=[exposure_key],
verbose=False,
sort_by=None)
exptimes = tmp[exposure_key].tolist()
scale = 1 / np.array(exptimes)
_add_and_print(str_nexp, header, verbose)
# Normalize by pixel average
if normalize_average:
def invavg(a):
return 1 / np.mean(a)
scale = invavg
_add_and_print(str_navg, header, verbose)
# Set rejection switches
clip_extrema, minmax_clip, sigma_clip = _set_reject_method(reject_method)
if len(ccdlist) == 1:
if isinstance(ccdlist[0], CCDData):
master = ccdlist[0]
else:
master = load_ccd(ccdlist[0], extension=extension, unit=unit)
else:
master = combine(
img_list=ccdlist,
method=combine_method,
clip_extrema=clip_extrema,
minmax_clip=minmax_clip,
sigma_clip=sigma_clip,
mem_limit=mem_limit,
combine_uncertainty_function=combine_uncertainty_function,
unit=unit,
hdu=extension,
scale=scale,
dtype=dtype,
**kwargs)
ncombine = len(ccdlist)
header["COMBVER"] = (ccdproc.__version__,
"ccdproc version used for combine.")
header["NCOMBINE"] = (ncombine, "Number of combined images")
header["COMBMETH"] = (combine_method, "Combining method")
header.add_history(
str_history.format(ncombine, str(type_key), str(type_val),
str(combine_method), str(reject_method), kwargs))
if subtract_frame is not None:
subtract = CCDData(subtract_frame.copy())
master.data = master.subtract(subtract).data
_add_and_print(str_subt, header, verbose)
if trim_fits_section is not None:
master = trim_image(master, fits_section=trim_fits_section)
_add_and_print(str_trim.format(trim_fits_section), header, verbose)
master.header = header
master = CCDData_astype(master,
dtype=dtype,
uncertainty_dtype=uncertainty_dtype)
if output is not None:
if verbose:
print(f"Writing FITS to {output}... ", end='')
master.write(output, output_verify=output_verify, overwrite=overwrite)
if verbose:
print("Saved.")
return master
os.chdir(outdir)
#change this to point to your raw data directory
ic1 = ImageFileCollection(indir)
#create the bias frames
blue_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Blue arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit=u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd,
median=True,
overscan_axis=0,
fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
blue_bias_list.append(ccd)
master_bias_blue = ccdproc.combine(blue_bias_list, method='median')
master_bias_blue.write('master_bias_blue.fits', clobber=True)
red_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Red arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit=u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd,
median=True,
overscan_axis=0,
fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
red_bias_list.append(ccd)
# Table Variables Setting
cards = ['DATE-OBS', 'NAXIS1', 'NAXIS2',
'XPIXSZ','YPIXSZ','XBINNING', 'YBINNING',
'EXPTIME', 'EGAIN',
'OBJECT']
dtypes = ['U24', int, int,
float,float, int, int,
float, float,
'U16']
df = input("현재 작업 파일을 그대로 사용하겠습니까? Y/N, 그 외의 모든 입력은 N으로 처리")
if df == "Y":
for filename in allfitsname:
ccd = trim_image(CCDData.read(filename,unit=u.adu),fits_section=f"[{x1}:{x2}, {y1}:{y2}]")
#16bit를 32비트로 만듦
ccd = yfu.CCDData_astype(ccd,dtype = 'float32')
filename += 's'
ccd.write(newfitspath/filename,overwrite=True)
#fits로 만들어 저장
#%%
# Trimming된 새 Data들로 Path와 directory를 바꿈
os.chdir(newfitspath)
allfitsname = glob.glob('*.fits')
allfitspath = list(newfitspath.glob('*.fits'))
allfitspath.sort()
allfitsname.sort()
)
#parser.add_argument('--gain', dest = 'gain', default= 1.3, help = 'gain in e-/ADU. default is 1.3, which applies to HDI camera')
#parser.add_argument('--rdnoise', dest = 'rdnoise', default= 7.3, help = 'gain in e-/ADU. default is 1.3, which applies to HDI camera')
args = parser.parse_args()
files = sorted(glob.glob(args.filestring + '*.fits'))
nfiles = len(files)
poly_model = models.Polynomial1D(1)
for f in files:
# read in image
# was having trouble getting image into the format that ccdproc wants
print 'working on ', f
# convert data to CCDData format and save header
ccd = CCDData.read(f, unit='adu')
# subtract overscan
o_subtracted = ccdproc.subtract_overscan(ccd,
fits_section=args.irafbiassec,
model=poly_model)
#header['HISTORY'] = 'overscan subtracted '+args.irafbiassec
# trim image
head_updates = {'CCDSEC': args.iraftrimsec, 'BIASSEC': args.irafbiassec}
trimmed = ccdproc.trim_image(o_subtracted,
fits_section=args.iraftrimsec,
add_keyword=head_updates)
trimmed.write('tr' + f, overwrite=True)
def ccdproc(images,
master_bias=None,
master_dark=None,
master_flat=None,
masks=None,
trim=None):
"""
Perform image reduction (bias, dark and flat corretion) on ccd data.
Parameters
----------
images : generator, type of 'ccdproc.CCDData'
Images to be combined.
master_bias : ccdproc.CCDData
Master Bias image.
master_dark : ccdproc.CCDData
Master Dark image.
master_flat : ccdproc.CCDData
Master Flat image.
masks : str, list of str or optional
Area to be masked.
trim : str or optional
Trim section.
Yields
------
'ccdproc.CCDData'
yield the next 'ccdproc.CCDData'.
Examples
--------
>>> from tuglib.io import FitsCollection
>>> from tuglib.reduction import bias_combine, dark_combine, flat_combine
>>> from tuglib.reduction import ccdproc
>>>
>>> path = '/home/user/data/'
>>> masks = ['[:, 1023:1025]', '[:1023, 56:58]']
>>> trim = '[:, 24:2023]'
>>>
>>> images = FitsCollection(location=path, gain=0.57, read_noise=4.11))
>>>
>>> bias_ccds = images.ccds(OBJECT='BIAS', trim=trim, masks=masks)
>>> dark_ccds = images.ccds(OBJECT='DARK', trim=trim, masks=masks)
>>> flat_ccds = images.ccds(OBJECT='FLAT', FILTER='V',
trim=trim, masks=masks)
>>>
>>> sci_ccds = images.ccds(OBJECT='Star', FILTER='V',
trim=trim, masks=masks)
>>>
>>> master_bias = bias_combine(bias_ccds, method='median')
>>> master_dark = dark_combine(dark_ccds, master_bias, method='median')
>>> master_flat = flat_combine(flat_ccds, master_bias, master_dark)
>>>
>>> # Yield a generator which point reduced images.
>>> reduced_ccds = ccdproc(sci_ccds, master_bias, master_dark, master_flat)
"""
if not isinstance(images, types.GeneratorType):
raise TypeError("'images' should be a 'ccdproc.CCDData' object.")
if not isinstance(master_bias, (type(None), CCDData)):
raise TypeError(
"'master_bias' should be 'None' or 'ccdproc.CCDData' object.")
if not isinstance(master_dark, (type(None), CCDData)):
raise TypeError(
"'master_dark' should be 'None' or 'ccdproc.CCDData' object.")
if not isinstance(master_flat, (type(None), CCDData)):
raise TypeError(
"'master_flat' should be 'None' or 'ccdproc.CCDData' object.")
if masks is not None:
if not isinstance(masks, (str, list, type(None))):
raise TypeError(
"'masks' should be 'str', 'list' or 'None' object.")
if trim is not None:
if not isinstance(trim, str):
raise TypeError("'trim' should be a 'str' object.")
mask = None
if master_dark is not None:
dark_exptime = master_dark.meta['EXPTIME'] * u.second
for ccd in images:
if (mask is None) and (masks is not None):
shape = ccd.shape
mask = make_mask(shape, masks)
ccd.mask = mask
ccd = trim_image(ccd, trim)
if master_bias is not None:
ccd = subtract_bias(ccd, master_bias)
if master_dark is not None:
data_exptime = ccd.meta['EXPTIME'] * u.second
ccd = subtract_dark(ccd,
master_dark,
dark_exposure=dark_exptime,
data_exposure=data_exptime,
exposure_time='EXPTIME',
exposure_unit=u.second,
scale=True)
if master_flat is not None:
ccd = flat_correct(ccd, master_flat)
yield ccd
if fil[-1] == '4':
sci_full = CCDData(np.concatenate(sci_final, axis=1),
header=header,
unit=u.electron / u.second)
elif fil[-1] == '8':
sci_full = CCDData(np.concatenate([
sci_final[1],
np.fliplr(sci_final[0]), sci_final[3],
np.fliplr(sci_final[2]), sci_final[5],
np.fliplr(sci_final[4]), sci_final[7],
np.fliplr(sci_final[6])
],
axis=1),
header=header,
unit=u.electron / u.second)
sci_full = ccdproc.trim_image(
sci_full[700:2400, 4100:6100]) #[700:2400,300:7300])
sci_full.header['DATASEC'] = (
'[1:%s,1:%s]' %
(np.shape(sci_full.data)[1], np.shape(sci_full.data)[0]))
reprojected.append(sci_full)
reprojected[i].write(sci_file.replace(sci_path, red_path),
overwrite=True)
align = align_stars(reprojected)
com = [CCDData(x, unit=u.electron / u.second) for x in align]
sci_med = ccdproc.combine(reprojected,
method='median',
sigma_clip=True,
sigma_clip_func=np.ma.median)
sci_med.header = reprojected[0].header
ra = sci_med.header['RA'].split(':')
dec = sci_med.header['DEC'].split(':')
else:
biases = im[i]
#if there is just one, make it two of the same for the combine!
if (len(im) == 1):
biases += ','+im[0]
bias_path += 'master/'
try:
os.mkdir(bias_path)
except:
pass
bias_path += bias_master
logme('Creating master bias frame (%s)...'%(bias_path))
bias = ccdproc.combine(biases, method='median', unit='adu', add_keyword=False)
#trim it, if necessary
if(len(trim_range) > 0):
bias = ccdproc.trim_image(bias, trim_range);
#write master frame to file
hdulist = bias.to_hdu()
hdulist.writeto(bias_path, clobber=True)
#dark
#create master dark frame
im=glob.glob(dark_path+'*.fits')+glob.glob(dark_path+'*.fit')
if(len(im) <= 0):
logme('Error. Dark calibration frame(s) not found (%s).' % dark_path)
log.close()
sys.exit(-1)
darks = None
bias_header = None
for i in range(0,len(im)):
#is (any) dark bias corrected?
from astropy.io import fits
import ccdproc
from os import listdir
import numpy as np
images = listdir('./data_corr/i')
for i in images:
img = ccdproc.CCDData.read('./data_corr/i/'+i, unit='adu')
trimmed = ccdproc.trim_image(img,fits_section='[150:1800, 150:1800]')
trimmed.write('./data_corr_trimmed/i/'+i)
images = listdir('./data_corr/v')
for i in images:
img = ccdproc.CCDData.read('./data_corr/v/'+i, unit='adu')
trimmed = ccdproc.trim_image(img,fits_section='[150:1800, 150:1800]')
trimmed.write('./data_corr_trimmed/v/'+i)
def ccds(self, masks=None, trim=None, **kwargs):
"""
Generator that yields each 'ccdproc.CCDData' objects in the collection.
Parameters
----------
masks : str, list of str or optional
Area to be masked.
trim : str or optional
Trim section.
**kwargs :
Any additional keywords are used to filter the items returned.
Yields
------
'ccdproc.CCDData'
yield the next 'ccdproc.CCDData' in the collection.
Examples
--------
>>> from tuglib.io import FitsCollection
>>>
>>> mask = '[:, 1000:1046]'
>>> trim = '[100:1988, :]'
>>> images = FitsCollection(
location='/home/user/data/fits/', gain=0.57, read_noise=4.11)
>>> biases = images.ccds(OBJECT='BIAS', masks=mask, trim=trim)
"""
if masks is not None:
if not isinstance(masks, (str, list, type(None))):
raise TypeError(
"'masks' should be 'str', 'list' or 'None' object.")
if trim is not None:
if not isinstance(trim, str):
raise TypeError("'trim' should be a 'str' object.")
tmp = np.full(len(self._collection), True, dtype=bool)
if len(kwargs) != 0:
for key, val in kwargs.items():
if key == 'filename':
file_mask = np.array([
fnmatch.fnmatch(filename, kwargs['filename'])
for filename in self._filenames_without_path
],
dtype=bool)
tmp = tmp & file_mask
else:
tmp = tmp & (self._collection[key.upper()] == val)
if np.count_nonzero(tmp) == 0:
yield None
x = self._collection[tmp]['NAXIS1'][0]
y = self._collection[tmp]['NAXIS2'][0]
shape = (y, x)
mask = None
if masks is not None:
mask = make_mask(shape, masks)
for filename in self._collection[tmp]['filename']:
ccd = CCDData.read(filename,
unit=self._unit,
output_verify='silentfix+ignore')
ccd.mask = mask
ccd = trim_image(ccd, trim)
if (self._gain is not None) and (self._read_noise is not None):
data_with_deviation = create_deviation(
ccd,
gain=self._gain,
readnoise=self._read_noise,
disregard_nan=self._disregard_nan)
gain_corrected = gain_correct(data_with_deviation, self._gain)
yield gain_corrected
else:
yield ccd
ind_flat2_2 = (logt['obj']=='flat')*(logt['exp_time']==36)
ind_flat3 = (logt['obj']=='flat')*(logt['exp_time']==144)
flat1_1 = ccdproc.combine(','.join(logt['filename'][ind_flat1_1]), unit='adu', method='median')
flat1_2 = ccdproc.combine(','.join(logt['filename'][ind_flat1_2]), unit='adu', method='median')
flat2_1 = ccdproc.combine(','.join(logt['filename'][ind_flat2_1]), unit='adu', method='median')
flat2_2 = ccdproc.combine(','.join(logt['filename'][ind_flat2_2]), unit='adu', method='median')
flat3 = ccdproc.combine(','.join(logt['filename'][ind_flat3]), unit='adu', method='median')
flat1 = ccdproc.combine([flat1_1.multiply(9.).divide(8.), flat1_2], method='average')
flat2 = ccdproc.combine([flat2_1.multiply(36.).divide(32.), flat2_2], method='average')
flat1_bias = ccdproc.subtract_bias(flat1, bias)
flat2_bias = ccdproc.subtract_bias(flat2, bias)
flat3_bias = ccdproc.subtract_bias(flat3, bias)
flat1_bias_trim = ccdproc.trim_image(flat1_bias[:, :4096])
flat2_bias_trim = ccdproc.trim_image(flat2_bias[:, :4096])
flat3_bias_trim = ccdproc.trim_image(flat3_bias[:, :4096])
""" combine flat """
flat_list = [flat1_bias_trim, flat2_bias_trim, flat3_bias_trim]
# find & combine & group apertures
ap_comb = combine_apertures(flat_list, n_jobs=10)
cheb_coefs, ap_uorder_interp = group_apertures(ap_comb, start_col=2100, order_dist=10)
# combine flat
flat_comb, flat_origin = combine_flat(flat_list, ap_uorder_interp, sat_count=45000, p=95)
flat_comb = ccdproc.CCDData(flat_comb, unit='adu')
def stack_FITS(fitslist=None,
summary_table=None,
extension=0,
unit='adu',
table_filecol="file",
trim_fits_section=None,
loadccd=True,
type_key=None,
type_val=None):
''' Stacks the FITS files specified in fitslist
Parameters
----------
fitslist: None, list of path-like, or list of CCDData
The list of path to FITS files or the list of CCDData to be
stacked. It is useful to give list of CCDData if you have
already stacked/loaded FITS file into a list by your own
criteria. If ``None`` (default), you must give ``fitslist`` or
``summary_table``. If it is not ``None``, this function will do
very similar job to that of ``ccdproc.combine``. Although it is
not a good idea, a mixed list of CCDData and paths to the files
is also acceptable.
summary_table: None, pandas.DataFrame or astropy.table.Table
The table which contains the metadata of files. If there are
many FITS files and you want to use stacking many times, it is
better to make a summary table by ``filemgmt.make_summary`` and
use that instead of opening FITS files' headers every time you
call this function. If you want to use ``summary_table`` instead
of ``fitslist`` and have set ``loadccd=True``, you must not have
``None`` or ``NaN`` value in the
``summary_table[table_filecol]``.
extension: int or str
The extension of FITS to be stacked. For single extension, set
it as 0.
unit: Unit or str, optional
The unit of the CCDs to be loaded.
Used only when ``fitslist`` is not a list of ``CCDData`` and
``loadccd`` is ``True``.
table_filecol: str
The column name of the ``summary_table`` which contains the path
to the FITS files.
trim_fits_section : str or None, optional
The ``fits_section`` of ``ccdproc.trim_image``. Region of
``ccd`` from which the overscan is extracted; see
`~ccdproc.subtract_overscan` for details.
Default is ``None``.
loadccd: bool, optional
Whether to return file paths or loaded CCDData. If ``False``, it
is a function to select FITS files using ``type_key`` and
``type_val`` without using much memory.
This is ignored if ``fitslist`` is given and composed of
``CCDData`` objects.
type_key, type_val: str, list of str
The header keyword for the ccd type, and the value you want to
match.
Return
------
matched: list of Path or list of CCDData
list containing Path to files if ``loadccd`` is ``False``.
Otherwise it is a list containing loaded CCDData after loading
the files. If ``ccdlist`` is given a priori, list of CCDData
will be returned regardless of ``loadccd``.
'''
def _parse_val(value):
val = str(value)
if val.lstrip('+-').isdigit(): # if int
result = int(val)
else:
try:
result = float(val)
except ValueError:
result = str(val)
return result
def _check_mismatch(row):
mismatch = False
for k, v in zip(type_key, type_val):
hdr_val = _parse_val(row[k])
parse_v = _parse_val(v)
if (hdr_val != parse_v):
mismatch = True
break
return mismatch
if ((fitslist is not None) + (summary_table is not None) != 1):
raise ValueError(
"One and only one of fitslist or summary_table must " +
"be not None.")
# If fitslist
if fitslist is not None:
table_mode = False
try:
fitslist = list(fitslist)
except TypeError:
raise TypeError("fitslist must be convertable to list. " +
f"It's now {type(fitslist)}.")
# If summary_table
if summary_table is not None:
table_mode = True
if ((not isinstance(summary_table, Table))
and (not isinstance(summary_table, pd.DataFrame))):
raise TypeError(
"summary_table must be an astropy Table or Pandas " +
f"DataFrame. It's now {type(summary_table)}.")
# Check for type_key and type_val
type_key, type_val, _ = chk_keyval(type_key=type_key,
type_val=type_val,
group_key=None)
# Setting whether to group
grouping = False
if len(type_key) > 0:
grouping = True
print("Analyzing FITS... ", end='')
# Set fitslist and summary_table based on the given input and grouping.
if table_mode:
if isinstance(summary_table, Table):
summary_table = summary_table.to_pandas()
fitslist = summary_table[table_filecol].tolist()
else:
if grouping:
summary_table = make_summary(fitslist,
extension=extension,
verbose=True,
fname_option='relative',
keywords=type_key,
sort_by=None,
pandas=True)
# else: no need to make summary_table.
print("Done", end='')
if loadccd:
print(" and loading FITS... ")
else:
print(".")
matched = []
# Append appropriate CCDs or filepaths to matched
if grouping: # summary_table is used.
for i, row in summary_table.iterrows():
mismatch = _check_mismatch(row)
if mismatch: # skip this row (file)
continue
# if not skipped:
# TODO: Is it better to remove Path here?
if isinstance(fitslist[i], CCDData):
matched.append(fitslist[i])
else: # it must be a path to the file
fpath = Path(fitslist[i])
if loadccd:
ccd_i = load_ccd(fpath, extension=extension, unit=unit)
if trim_fits_section is not None:
ccd_i = trim_image(ccd_i,
fits_section=trim_fits_section)
matched.append(ccd_i)
else:
matched.append(fpath)
else: # summary_table is not used.
for item in fitslist:
if isinstance(item, CCDData):
matched.append(item)
else:
if loadccd:
ccd_i = load_ccd(item, extension=extension, unit=unit)
if trim_fits_section is not None:
ccd_i = trim_image(ccd_i,
fits_section=trim_fits_section)
matched.append(ccd_i)
else: # TODO: Is is better to remove Path here?
matched.append(Path(item))
# Generate warning OR information messages
if len(matched) == 0:
if grouping:
warn('No FITS file had "{:s} = {:s}"'.format(
str(type_key), str(type_val)) +
"Maybe int/float/str confusing?")
else:
warn('No FITS file found')
else:
if grouping:
N = len(matched)
ks = str(type_key)
vs = str(type_val)
if loadccd:
print(f'{N} FITS files with "{ks} = {vs}" are loaded.')
else:
print(f'{N} FITS files with "{ks} = {vs}" are selected.')
else:
if loadccd:
print('{:d} FITS files are loaded.'.format(len(matched)))
return matched
def correctData(filename, master_bias, master_flat, filetype):
"""
Correct a science image using the available
master calibrations. Skip a calibration step if the
master frame does not exist.
No reduced file is written in this new scheme.
Instead, the corrected data is passed directly
to the phot() routine, photometry is done as per
the configuration and the photometry is written out
only.
TODO: Finish docstring
"""
print('Reducing {0:s}...'.format(filename))
with fits.open(filename) as fitsfile:
# correct times for science spectra,
# don't bother for arcs
hdr = fitsfile[0].header
if filetype == 'science':
half_exptime = hdr[EXPTIME_KEYWORD]/2.
utstart = hdr[UTSTART_KEYWORD]
dateobs = hdr[DATEOBS_KEYWORD]
ra = hdr[RA_KEYWORD]
dec = hdr[DEC_KEYWORD]
time_start = Time('{}T{}'.format(dateobs, utstart),
scale='utc',
format='isot',
location=OBSERVATORY)
# correct to mid exposure time
jd_mid = time_start + half_exptime*u.second
ltt_bary, ltt_helio = getLightTravelTimes(ra, dec, jd_mid)
time_bary = jd_mid.tdb + ltt_bary
time_helio = jd_mid.utc + ltt_helio
hdr['BJD-MID'] = time_bary.jd
hdr['HJD-MID'] = time_helio.jd
hdr['JD-MID'] = jd_mid.jd
hdr['UT-MID'] = jd_mid.isot
ccd = CCDData.read(filename, unit=u.adu)
if master_bias:
ccd = subtract_bias(ccd, master_bias)
else:
print('No master bias, skipping correction...')
if master_flat:
ccd = flat_correct(ccd, master_flat)
else:
print('No master flat, skipping correction...')
# after calibrating we get np.float64 data
# if there are no calibrations we maintain dtype = np.uint16
# sep weeps
# fix this by doing the following
if isinstance(ccd.data[0][0], np.uint16):
ccd.data = ccd.data.astype(np.float64)
# trim the data
ccd_trimmed = trim_image(ccd[1000:3001, :])
# write out the trimmed file and the updated header
#ccd_trimmed.write(filename, hdr, clobber=True)
trimmed_filename = '{}_t.fits'.format(filename.split('.')[0])
fits.writeto(trimmed_filename, ccd_trimmed.data, hdr)
# remove the old untrimmed data
os.system('rm {}'.format(filename))
def bdf_process(ccd,
output=None,
mbiaspath=None,
mdarkpath=None,
mflatpath=None,
fits_section=None,
calc_err=False,
unit='adu',
gain=None,
rdnoise=None,
gain_key="GAIN",
rdnoise_key="RDNOISE",
gain_unit=u.electron / u.adu,
rdnoise_unit=u.electron,
dark_exposure=None,
data_exposure=None,
exposure_key="EXPTIME",
exposure_unit=u.s,
dark_scale=False,
min_value=None,
norm_value=None,
verbose=True,
output_verify='fix',
overwrite=True,
dtype="float32"):
''' Do bias, dark and flat process.
Parameters
----------
ccd: array-like
The ccd to be processed.
output: path-like
Saving directory
'''
proc = CCDData(ccd)
hdr_new = proc.header
if mbiaspath is None:
do_bias = False
# mbias = CCDData(np.zeros_like(ccd), unit=unit)
else:
do_bias = True
mbias = CCDData.read(mbiaspath, unit=unit)
hdr_new.add_history(f"Bias subtracted using {mbiaspath}")
if mdarkpath is None:
do_dark = False
mdark = None
else:
do_dark = True
mdark = CCDData.read(mdarkpath, unit=unit)
hdr_new.add_history(f"Dark subtracted using {mdarkpath}")
if dark_scale:
hdr_new.add_history(
f"Dark scaling {dark_scale} using {exposure_key}")
if mflatpath is None:
do_flat = False
# mflat = CCDData(np.ones_like(ccd), unit=unit)
else:
do_flat = True
mflat = CCDData.read(mflatpath)
hdr_new.add_history(f"Flat corrected using {mflatpath}")
if fits_section is not None:
proc = trim_image(proc, fits_section)
mbias = trim_image(mbias, fits_section)
mdark = trim_image(mdark, fits_section)
mflat = trim_image(mflat, fits_section)
hdr_new.add_history(f"Trim by FITS section {fits_section}")
if do_bias:
proc = subtract_bias(proc, mbias)
if do_dark:
proc = subtract_dark(proc,
mdark,
dark_exposure=dark_exposure,
data_exposure=data_exposure,
exposure_time=exposure_key,
exposure_unit=exposure_unit,
scale=dark_scale)
# if calc_err and verbose:
# if mdark.uncertainty is not None:
# print("Dark has uncertainty frame: Propagate in arithmetics.")
# else:
# print("Dark does NOT have uncertainty frame")
if calc_err:
if gain is None:
gain = fu.get_from_header(hdr_new,
gain_key,
unit=gain_unit,
verbose=verbose,
default=1.).value
if rdnoise is None:
rdnoise = fu.get_from_header(hdr_new,
rdnoise_key,
unit=rdnoise_unit,
verbose=verbose,
default=0.).value
err = fu.make_errmap(proc, gain_epadu=gain, subtracted_dark=mdark)
proc.uncertainty = StdDevUncertainty(err)
errstr = (f"Error calculated using gain = {gain:.3f} [e/ADU] and " +
f"rdnoise = {rdnoise:.3f} [e].")
hdr_new.add_history(errstr)
if do_flat:
if calc_err:
if (mflat.uncertainty is not None) and verbose:
print("Flat has uncertainty frame: Propagate in arithmetics.")
hdr_new.add_history(
"Flat had uncertainty and is also propagated.")
proc = flat_correct(proc,
mflat,
min_value=min_value,
norm_value=norm_value)
proc = fu.CCDData_astype(proc, dtype=dtype)
proc.header = hdr_new
if output is not None:
proc.write(output, output_verify=output_verify, overwrite=overwrite)
return proc
def flux_extraction(file_name, path, out_path, images=True):
"""
Parameters
----------
----------
file_name : str
Name of the image/telluric file
from which flux has to be extracted
path : str
Path of the desired image file
out_path : str
Path of the output data and/or image file
images : bool
True if one wants to save visualization of flux data
False if not.
Default is True
----------
returns
----------
flux : data file
.dat file containing the flux at
various pixel values
Path of this file would be similar to
that of image file.
----------
"""
pt = Path(path)
f1 = ccdp.ImageFileCollection(pt)
ccd = CCDData.read(path + file_name) # + '.fits')
# Trimming the Image
trimmed = ccdp.trim_image(ccd, fits_section='[1:256, 100:1000]')
trimmed.meta['TRIM'] = True
trimmed.header = ccd.header
#trimmed.write(file_name + '_trim.fits')
# Reading the data from Trimmed image
data = trimmed.data
# Creating a function to detect the edges of slit
# For lower edge
def xlow(raw_data):
"""
Parameters
----------
----------
raw_data : numpy.ndarray
Array containing flux at some particular wavelength
----------
returns
----------
number : float
A pixel number showing the lower edge of slit
----------
"""
j = 0
for i in range(int(len(raw_data) / 5)):
st = np.std(raw_data[j:j + 5])
xlw = 0
if st < 2:
xlw = j
if xlw != 0:
break
j = j + 5
return xlw
# For upper edge
def xup(raw_data):
"""
Parameters
----------
----------
raw_data : numpy.ndarray
Array containing flux at some particular wavelength
----------
returns
----------
number : float
A pixel number showing the upper edge of slit
----------
"""
j = 255
for i in range(int(len(raw_data) / 5)):
st = np.std(raw_data[j - 5:j])
xup = 0
if st < 2:
xup = j
if xup != 0:
break
j = j - 5
return xup
# Defining line and inverse line
def line(x, m, c):
return m * x + c
def inv_line(x, m, c):
bc = (x - c) / m
return bc
# Detecting the edges of the spectrum
ys = np.array([150, 300, 450, 600, 750])
xs_left = np.array([])
xs_right = np.array([])
xs_mid = np.array([])
for i in range(len(ys)):
dd1 = data[ys[i]]
xll = xlow(dd1)
xs_left = np.hstack((xs_left, xll))
xuu = xup(dd1)
xs_right = np.hstack((xs_right, xuu))
popt_l, pcov_l = cft(line, xs_left, ys)
popt_r, pcov_r = cft(line, xs_right, ys)
# Detecting a line where spectrum could reside
for i in range(len(ys)):
ran_l = inv_line(ys[i], popt_l[0], popt_l[1])
ran_r = inv_line(ys[i], popt_r[0], popt_r[1])
xd1 = data[ys[i]]
xd = xd1[int(ran_l):int(ran_r)]
ma = np.max(xd)
ab = np.where(xd == ma)
xs_mid = np.hstack((xs_mid, ab[0][0] + ran_l))
popt_m, pcov_m = cft(line, xs_mid, ys)
# Finding total flux
def total_flux(lam, xlim=20):
ydata = data[lam]
xmid = inv_line(lam, popt_m[0], popt_m[1])
xlow = xmid - xlim
xup = xmid + xlim
total_flux1 = 0
xdata = np.arange(int(xlow), int(xup + 1), 1)
for i in range(len(xdata)):
total_flux1 = total_flux1 + ydata[xdata[i]]
return total_flux1
# Flux as a function of pixel
flux = np.array([])
y11 = np.arange(0, 900, 1)
for i in range(len(y11)):
f11 = total_flux(y11[i])
flux = np.hstack((flux, f11))
# Saving the image file for flux
if images == True:
fig1 = plt.figure(figsize=(20, 10))
plt.plot(flux)
plt.xlabel('Pixel Number')
plt.ylabel('Total Flux')
plt.title('Total flux for ' + file_name + ' observation')
plt.grid()
plt.savefig(out_path + '/' + file_name + '_flux.png')
plt.close(fig1)
# Saving Data file of the flux
f1 = open(out_path + '/' + file_name + '_flux.dat', 'w')
f1.write('#Pixel\t\tFlux\n')
for i in range(len(y11)):
f1.write(str(y11[i]) + '\t\t' + str(flux[i]) + '\n')
f1.close()
def hrs_process(image_name, ampsec=[], oscansec=[], trimsec=[],
masterbias=None, error=False, bad_pixel_mask=None, flip=False,
rdnoise=None, oscan_median=True, oscan_model=None):
"""Processing required for HRS observations. If the images have multiple
amps, then this will process each part of the image and recombine them
into for the final results
Parameters
----------
image_name: str
Name of file to be processed
ampsec: list
List of ampsections. This list should have the same length as the
number of amps in the data set. The sections should be given
in the format of fits_sections (see below).
oscansec: list
List of overscan sections. This list should have the same length as the
number of amps in the data set. The sections should be given
in the format of fits_sections (see below).
trimsec: list
List of overscan sections. This list should have the same length as the
number of amps in the data set. The sections should be given
in the format of fits_sections (see below).
error: boolean
If True, create an uncertainty array for ccd
masterbias: None, `~numpy.ndarray`, or `~ccdproc.CCDData`
A materbias frame to be subtracted from ccd.
bad_pixel_mask: None or `~numpy.ndarray`
A bad pixel mask for the data. The bad pixel mask should be in given
such that bad pixels havea value of 1 and good pixels a value of 0.
flip: boolean
If True, the image will be flipped such that the orders run from the
bottom of the image to the top and the dispersion runs from the left
to the right.
rdnoise: None or `~astropy.Quantity`
Read noise for the observations. The read noise should be in
`~astropy.units.electron`
oscan_median : bool, optional
If true, takes the median of each line. Otherwise, uses the mean
oscan_model : `~astropy.modeling.Model`, optional
Model to fit to the data. If None, returns the values calculated
by the median or the mean.
Returns
-------
ccd: `~ccdproc.CCDData`
Data processed and
Notes
-----
The format of the `fits_section` string follow the rules for slices that
are consistent with the FITS standard (v3) and IRAF usage of keywords like
TRIMSEC and BIASSEC. Its indexes are one-based, instead of the
python-standard zero-based, and the first index is the one that increases
most rapidly as you move through the array in memory order, opposite the
python ordering.
The 'fits_section' argument is provided as a convenience for those who are
processing files that contain TRIMSEC and BIASSEC. The preferred, more
pythonic, way of specifying the overscan is to do it by indexing the data
array directly with the `overscan` argument.
"""
# read in the data
ccd = ccdproc.CCDData.read(image_name, unit=u.adu)
try:
namps = ccd.header['CCDAMPS']
except KeyError:
namps = ccd.header['CCDNAMPS']
# thow errors for the wrong number of amps
if len(ampsec) != namps:
raise ValueError('Number of ampsec does not equal number of amps')
if len(oscansec) != namps:
raise ValueError('Number of oscansec does not equal number of amps')
if len(trimsec) != namps:
raise ValueError('Number of trimsec does not equal number of amps')
if namps == 1:
gain = float(ccd.header['gain'].split()[0]) * u.electron / u.adu
nccd = ccd_process(ccd, oscan=oscansec[0], trim=trimsec[0],
error=error, masterbias=masterbias,
bad_pixel_mask=bad_pixel_mask, gain=gain,
rdnoise=rdnoise, oscan_median=oscan_median,
oscan_model=oscan_model)
else:
ccd_list = []
xsize = 0
for i in range(namps):
cc = ccdproc.trim_image(ccd, fits_section=ampsec[i])
gain = float(ccd.header['gain'].split()[i]) * u.electron / u.adu
ncc = ccd_process(cc, oscan=oscansec[i], trim=trimsec[i],
error=False, masterbias=None, gain=gain,
bad_pixel_mask=None, rdnoise=rdnoise,
oscan_median=oscan_median,
oscan_model=oscan_model)
xsize = xsize + ncc.shape[1]
ysize = ncc.shape[0]
ccd_list.append(ncc)
# now recombine the processed data
ncc = ccd_list[0]
data = np.zeros((ysize, xsize))
if ncc.mask is not None:
mask = np.zeros((ysize, xsize))
else:
mask = None
if ncc.uncertainty is not None:
raise NotImplementedError(
'Support for uncertainties not implimented yet')
else:
uncertainty = None
x1 = 0
for i in range(namps):
x2 = x1 + ccd_list[i].data.shape[1]
data[:, x1:x2] = ccd_list[i].data
if mask is not None:
mask[:, x1:x2] = ccd_list[i].mask
x1 = x2
nccd = ccdproc.CCDData(data, unit=ncc.unit, mask=mask,
uncertainty=uncertainty)
nccd.header = ccd.header
nccd = ccd_process(nccd, masterbias=masterbias, error=error, gain=None,
rdnoise=rdnoise, bad_pixel_mask=bad_pixel_mask)
if flip:
nccd.data = nccd.data[::-1, ::-1]
if (nccd.mask is not None):
nccd.mask = nccd.mask[::-1, ::-1]
if (nccd.uncertainty is not None):
nccd.uncertainty = nccd.uncertainty[::-1, ::-1]
return nccd
from astropy.io import fits
import ccdproc
from os import listdir
import numpy as np
images = listdir('./data_corr/i')
for i in images:
img = ccdproc.CCDData.read('./data_corr/i/' + i, unit='adu')
trimmed = ccdproc.trim_image(img, fits_section='[150:1800, 150:1800]')
trimmed.write('./data_corr_trimmed/i/' + i)
images = listdir('./data_corr/v')
for i in images:
img = ccdproc.CCDData.read('./data_corr/v/' + i, unit='adu')
trimmed = ccdproc.trim_image(img, fits_section='[150:1800, 150:1800]')
trimmed.write('./data_corr_trimmed/v/' + i)
def ccd_process(ccd, oscan=None, trim=None, error=False, masterbias=None,
bad_pixel_mask=None, gain=None, rdnoise=None,
oscan_median=True, oscan_model=None):
"""Perform basic processing on ccd data.
The following steps can be included:
* overscan correction
* trimming of the image
* create edeviation frame
* gain correction
* add a mask to the data
* subtraction of master bias
The task returns a processed `ccdproc.CCDData` object.
Parameters
----------
ccd: `ccdproc.CCDData`
Frame to be reduced
oscan: None, str, or, `~ccdproc.ccddata.CCDData`
For no overscan correction, set to None. Otherwise proivde a region
of `ccd` from which the overscan is extracted, using the FITS
conventions for index order and index start, or a
slice from `ccd` that contains the overscan.
trim: None or str
For no trim correction, set to None. Otherwise proivde a region
of `ccd` from which the image should be trimmed, using the FITS
conventions for index order and index start.
error: boolean
If True, create an uncertainty array for ccd
masterbias: None, `~numpy.ndarray`, or `~ccdproc.CCDData`
A materbias frame to be subtracted from ccd.
bad_pixel_mask: None or `~numpy.ndarray`
A bad pixel mask for the data. The bad pixel mask should be in given
such that bad pixels havea value of 1 and good pixels a value of 0.
gain: None or `~astropy.Quantity`
Gain value to multiple the image by to convert to electrons
rdnoise: None or `~astropy.Quantity`
Read noise for the observations. The read noise should be in
`~astropy.units.electron`
oscan_median : bool, optional
If true, takes the median of each line. Otherwise, uses the mean
oscan_model : `~astropy.modeling.Model`, optional
Model to fit to the data. If None, returns the values calculated
by the median or the mean.
Returns
-------
ccd: `ccdproc.CCDData`
Reduded ccd
Examples
--------
1. To overscan, trim, and gain correct a data set:
>>> import numpy as np
>>> from astropy import units as u
>>> from hrsprocess import ccd_process
>>> ccd = CCDData(np.ones([100, 100]), unit=u.adu)
>>> nccd = ccd_process(ccd, oscan='[1:10,1:100]', trim='[10:100, 1,100]',
error=False, gain=2.0*u.electron/u.adu)
"""
# make a copy of the object
nccd = ccd.copy()
# apply the overscan correction
if isinstance(oscan, ccdproc.CCDData):
nccd = ccdproc.subtract_overscan(nccd, overscan=oscan,
median=oscan_median,
model=oscan_model)
elif isinstance(oscan, six.string_types):
nccd = ccdproc.subtract_overscan(nccd, fits_section=oscan,
median=oscan_median,
model=oscan_model)
elif oscan is None:
pass
else:
raise TypeError('oscan is not None, a string, or CCDData object')
# apply the trim correction
if isinstance(trim, six.string_types):
nccd = ccdproc.trim_image(nccd, fits_section=trim)
elif trim is None:
pass
else:
raise TypeError('trim is not None or a string')
# create the error frame
if error and gain is not None and rdnoise is not None:
nccd = ccdproc.create_deviation(nccd, gain=gain, rdnoise=rdnoise)
elif error and (gain is None or rdnoise is None):
raise ValueError(
'gain and rdnoise must be specified to create error frame')
# apply the bad pixel mask
if isinstance(bad_pixel_mask, np.ndarray):
nccd.mask = bad_pixel_mask
elif bad_pixel_mask is None:
pass
else:
raise TypeError('bad_pixel_mask is not None or numpy.ndarray')
# apply the gain correction
if isinstance(gain, u.quantity.Quantity):
nccd = ccdproc.gain_correct(nccd, gain)
elif gain is None:
pass
else:
raise TypeError('gain is not None or astropy.Quantity')
# test subtracting the master bias
if isinstance(masterbias, ccdproc.CCDData):
nccd = ccdproc.subtract_bias(nccd, masterbias)
elif isinstance(masterbias, np.ndarray):
nccd.data = nccd.data - masterbias
elif masterbias is None:
pass
else:
raise TypeError(
'masterbias is not None, numpy.ndarray, or a CCDData object')
return nccd
def __call__(self, collection=None, masks=None, trim=None, **kwargs):
"""
Generator that yields each 'ccdproc.CCDData' objects in the collection.
Parameters
----------
collection : 'FitsCollection.collection' or optional
Filtered collection.
masks : str, list of str or optional
Area to be masked.
trim : str or optional
Trim section.
**kwargs :
Any additional keywords are used to filter the items returned.
Yields
------
'ccdproc.CCDData'
yield the next 'ccdproc.CCDData' in the collection.
Examples
--------
>>> from tuglib.io import FitsCollection
>>>
>>> mask = '[:, 1000:1046]'
>>> trim = '[100:1988, :]'
>>>
>>> images = FitsCollection(
location='/home/user/data/fits/', gain=0.57, read_noise=4.11)
>>>
>>> query = images['EXPTIME'] == 100.0
>>> sub_collections = images[query]
>>>
>>> ccds = images(sub_collections, masks=mask, trim=trim)
"""
if masks is not None:
if not isinstance(masks, (str, list, type(None))):
raise TypeError(
"'masks' should be 'str', 'list' or 'None' object.")
if trim is not None:
if not isinstance(trim, str):
raise TypeError("'trim' should be a 'str' object.")
if collection is None:
return self.ccds(masks=masks, trim=trim, **kwargs)
tmp = np.full(len(collection), True, dtype=bool)
if len(kwargs) != 0:
for key, val in kwargs.items():
tmp = tmp & (collection[key] == val)
x = collection[tmp]['NAXIS1'][0]
y = collection[tmp]['NAXIS2'][0]
shape = (y, x)
mask = None
if masks is not None:
mask = make_mask(shape, masks)
if (self._gain is not None) and (self._read_noise is not None):
for filename in collection[tmp]['filename']:
ccd = CCDData.read(filename, unit=self._unit)
ccd.mask = mask
ccd = trim_image(ccd, trim)
data_with_deviation = create_deviation(
ccd,
gain=self._gain,
readnoise=self._read_noise,
disregard_nan=self._disregard_nan)
gain_corrected = gain_correct(data_with_deviation, self._gain)
yield gain_corrected
else:
for filename in collection[tmp]['filename']:
ccd = CCDData.read(filename, unit=self._unit)
ccd.mask = mask
ccd = trim_image(ccd, trim)
yield ccd
def trim_image(fn, trim, logfile):
print('trimming image')
print('trimming image with {}'.format(trim), file=logfile)
ccd = ccdproc.trim_image(ccdproc.CCDData.read(fn)[trim])
return ccd
def analyze_image(imagefile, dark=None, flat=None, box_size=30, medfilt=False,
plot=False, seeing=0, pixelscale=0.1798, verbose=False):
im = reduce_image(imagefile, dark=dark, flat=flat)
hdul = fits.open(imagefile)
# Get info about alignment box positions
alignment_box_table = Table(hdul[4].data)
if plot == True:
plt.figure(figsize=(16,6))
pixels = []
targets = []
for i,box in enumerate(alignment_box_table):
result = None
slitno = int(box['Slit_Number'])
bar_nos = slit_to_bars(slitno)
bar_pos = [hdul[0].header.get(f'B{b:02d}POS') for b in bar_nos]
box_pos = np.mean(bar_pos)
box_pix = physical_to_pixel([[box_pos, slitno]])[0]
boxat = [int(box_pix[0]), int(box_pix[1])]
fits_section = f'[{boxat[0]-box_size:d}:{boxat[0]+box_size:d}, '\
f'{boxat[1]-box_size:d}:{boxat[1]+box_size:d}]'
region = trim_image(im, fits_section=fits_section)
targ_pos = float(box['Target_to_center_of_slit_distance'])/pixelscale
if plot == True:
plt.subplot(1,len(alignment_box_table),i+1, aspect='equal')
plt.title(f"Alignment Box {i+1}\n{fits_section}")
plt.imshow(region.data, origin='lower',
vmin=np.percentile(region.data, 85)*0.95,
vmax=region.data.max()*1.02)
# try:
result = fit_alignment_box(region, box_size=box_size, verbose=False,
seeing=seeing, medfilt=medfilt)
star_pix = np.array([result['Star X']+boxat[0]-box_size,
result['Star Y']+boxat[1]-box_size])
fitted_box_pix = np.array([result['Box X']+boxat[0]-box_size,
result['Box Y']+boxat[1]-box_size])
slitang = 0.22*np.pi/180
targ_pix_im = (result['Box X']-np.sin(slitang)*targ_pos,
result['Box Y']+np.cos(slitang)*targ_pos)
targ_pix = np.array([targ_pix_im[0]+boxat[0]-box_size,
targ_pix_im[1]+boxat[1]-box_size])
pixels.append(list(star_pix))
targets.append(list(targ_pix))
pix_err = targ_pix - star_pix
pos_err = pix_err*pixelscale
if plot == True:
cxy = (result['Star X'], result['Star Y'])
c = plt.Circle(cxy, result['FWHM pix'], linewidth=2, ec='g', fc='none', alpha=0.3)
ax = plt.gca()
ax.add_artist(c)
plt.plot(result['Star X'], result['Star Y'], 'g.')
# plt.plot(result['Box X'], result['Box Y'], 'y+', alpha=0.5, ms=10)
plt.plot(targ_pix_im[0], targ_pix_im[1], 'rx', alpha=0.5)
if verbose:
print(f"Alignment Box {i+1} results:")
print(f" Sky Amplitude: {result['Sky Amplitude']:.0f} ADU")
print(f" Star Amplitude: {result['Star Amplitude']:.0f} ADU")
print(f" Star FWHM: {result['FWHM arcsec']:.2f}")
print(f" Star Position: {star_pix[0]:.1f}, {star_pix[1]:.1f}")
print(f" Target Position: {targ_pix[0]:.1f}, {targ_pix[1]:.1f}")
print(f" Position Error: {pos_err[0]:+.2f}, {pos_err[1]:+.2f} arcsec")
# except:
# print(f'Alignment Box {i+1} failed: {result}')
if plot == True:
plt.xticks([], [])
plt.yticks([], [])
# Calculate Transformation
off_Xpix, off_Ypix, off_R, err_R, A = fit_transforms(pixels, targets)
off_X = off_Xpix * pixelscale
off_Y = off_Ypix * pixelscale
th_XY = 0.10
th_R = 0.030
send_X = off_X if abs(off_X) > th_XY else 0
send_Y = off_Y if abs(off_Y) > th_XY else 0
send_R = off_R if abs(off_R) > th_R else 0
print()
print(f" Calculated Err Send (Threshold)")
print(f"Offset X = {off_X:+.2f} {send_X:+.2f} arcsec ({th_XY:.2f})")
print(f"Offset Y = {off_Y:+.2f} {send_Y:+.2f} arcsec ({th_XY:.2f})")
print(f"Rotation = {off_R:+.3f} {err_R:.3f} {send_R:+.3f} deg ({th_R:.3f})")
if plot == True:
plt.show()
parser.add_argument('--irafbiassec', dest = 'irafbiassec', default= '[4100:4150, 1:4150]', help = 'biassec in iraf notation. default is [4100:4150, 1:4150], which applies to HDI camera')
parser.add_argument('--iraftrimsec', dest = 'iraftrimsec', default= '[1:4095, 1:4109]', help = 'biassec in iraf notation. default is [4100:4150, 1:4150], which applies to HDI camera')
#parser.add_argument('--gain', dest = 'gain', default= 1.3, help = 'gain in e-/ADU. default is 1.3, which applies to HDI camera')
#parser.add_argument('--rdnoise', dest = 'rdnoise', default= 7.3, help = 'gain in e-/ADU. default is 1.3, which applies to HDI camera')
args = parser.parse_args()
files = sorted(glob.glob(args.filestring+'*.fits'))
nfiles=len(files)
poly_model = models.Polynomial1D(1)
for f in files:
# read in image
# was having trouble getting image into the format that ccdproc wants
print 'working on ',f
# convert data to CCDData format and save header
ccd = CCDData.read(f, unit='adu')
# subtract overscan
o_subtracted = ccdproc.subtract_overscan(ccd, fits_section = args.irafbiassec, model=poly_model)
header['HISTORY'] = 'overscan subtracted '+args.irafbiassec
# trim image
trimmed = ccdproc.trim_image(o_subtracted, fits_section = args.iraftrimsec)
head_updates = {'CCDSEC':args.iraftrimsec}
trimmed.write('tr'+f,add_keyword=head_updates)
