def t120_mkflat(flat_dir=t120.t120_flat_dir,
master_name_root='master',
master_offset=t120.t120_ofst_dir + t120.t120_master_name):
# read offset
hdu_offset_list = fits.open(master_offset)
offset = CCDData(hdu_offset_list[0].data, unit=u.adu)
dict_ccd_data = {}
list_ccd_data = []
for fit_file in glob.glob(flat_dir + '*.fit'):
t120.log.info('now opening file: ' + fit_file)
hdu = fits.open(fit_file)
filter_name = hdu[0].header['FILTER']
t120.log.info('filter=' + filter_name)
if not dict_ccd_data.has_key(filter_name):
dict_ccd_data[filter_name] = []
else:
dict_ccd_data[filter_name].append(
subtract_overscan(CCDData(hdu[0].data, unit=u.adu), offset))
t120.log.info('now loop over the filters')
for filter_name in dict_ccd_data:
combiner = Combiner(dict_ccd_data[filter_name])
master_flat = combiner.median_combine()
hdu = master_flat.to_hdu()
master_file = flat_dir + master_name_root + '-' + filter_name + '.fits'
hdu.writeto(master_file, overwrite=True)
t120.log.info('Master flat saved in ' + master_file)
return
def subtract_and_trim_overscan(ccd, config):
'''
Subtracts the overscan from ccd and, then, trim the overscan regions.
'''
for i_region in range(len(config.overscan_regions)):
overscan_region = config.overscan_regions[i_region]
science_region = config.science_regions[i_region]
ccd.data[science_region] = ccdproc.subtract_overscan(ccd[science_region],
overscan=ccd[overscan_region],
overscan_axis=config.overscan_axis,
model=None).data # FIXME: Add model option to the config file - e.g. models.Polynomial1D(1)
# Due a possible bug on WCS, this seems to not work:
# ccd = ccdproc.trim_image(ccd[config_values['science_trim']])
# This is the workaround I found.
if config.overscan_axis == 0:
ccd.data = ccd.data[config.science_trim]
if ccd.uncertainty is not None:
ccd.uncertainty = ccd.uncertainty[config.science_trim]
if ccd.mask is not None:
ccd.mask = ccd.mask[config.science_trim]
else:
ccd.data = ccd.data[:, config.science_trim]
if ccd.uncertainty is not None:
ccd.uncertainty = ccd.uncertainty[:, config.science_trim]
if ccd.mask is not None:
ccd.mask = ccd.mask[:, config.science_trim]
return ccd
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction, dark subtraction, overscan correction, flat correction, and trimming image.')
raw = CCDData.read(sci,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of science image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)#dark_exposure=1*u.second, data_exposure=red.header['EXPTIME']*u.second, scale=True)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
red = ccdproc.flat_correct(red, mflat)
processed_data = ccdproc.ccd_process(red, trim=raw.header['DATASEC'])
log.info('File proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci,processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(red.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (510, 510), filter_size=(9, 9),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits'),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['EXPTIME']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/'),overwrite=True)
processed.append(final)
return processed, masks
def create_flat(flat_list,fil,red_path,mbias=None,log=None):
log.info('Processing files for filter: '+fil)
log.info(str(len(flat_list))+' files found.')
flats = []
flat_scale = []
for flat in flat_list:
log.info('Loading file: '+flat)
raw = CCDData.read(flat,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
mask = make_source_mask(red, nsigma=3, npixels=5)
bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
masked = np.array(red)
masked[mask] = bkg.background[mask]
log.info('Median flat level: '+str(np.median(masked)))
norm = 1/np.median(masked[500:1500,500:1500])
log.info('Flat normalization: '+str(norm))
flat_scale.append(norm)
flats.append(CCDData(masked,meta=red.header,unit=u.electron))
mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True)
log.info('Created master flat for filter: '+fil)
mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True)
log.info('Master flat written to mflat_'+fil+'.fits')
return
def oscan_trim_file(fname, datahdus=0):
hdulist = fits.open(fname)
nhdus = len(hdulist)
if nhdus > 1:
istart = 1
else:
istart = 0
# loop from first-data to last HDU, unless datahdus is set
hduindexes = list(range(nhdus))[istart:]
if datahdus != 0:
hduindexes = datahdus
for i in hduindexes:
hdulist = fits.open(fname)
data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
data1.header = hdulist[i].header
# What happens if file is already overscan-subtracted?
# We should probably default to using a model
if modeling:
oscan1 = ccdproc.subtract_overscan(
data1,
fits_section=data1.header['BIASSEC'],
add_keyword={
'overscan': True,
'calstat': 'O'
},
model=models.Polynomial1D(1))
else:
oscan1 = ccdproc.subtract_overscan(
data1,
fits_section=data1.header['BIASSEC'],
add_keyword={
'overscan': True,
'calstat': 'O'
},
model=None)
trim1 = ccdproc.trim_image(oscan1,
fits_section=oscan1.header['TRIMSEC'],
add_keyword={
'trimmed': True,
'calstat': 'OT'
})
fits.update(fname, trim1.data, header=trim1.header, ext=i)
hdulist.close()
mylog("Overscan and trim {0}".format(fname))
return
def make_master_bias(self, list_of_biasfiles):
bias_list = [0]*len(list_of_biasfiles)
for ii, kk in enumerate(list_of_biasfiles):
fitsfile = fits.open(kk)
bias = ccdproc.CCDData(data=fitsfile[1].data, meta=fitsfile[1].header, unit="adu")
bias_list[ii] = ccdproc.subtract_overscan(bias, fits_section='[5:35, :]')
self.master_bias = ccdproc.combine(bias_list, method="median")
self.master_bias.write(self.output_dir+"/"+self.filename+"_masterbias.fits", overwrite=True)
def ARTNreduce(filename='', to_fits=False):
"""
Take a raw FITS image from an ARTN imager (currently only mont4k is supported), perform basic overscan subtraction
and trimming, and then stitch the images into a single image with the correct sky orientation (E left, N up).
"""
reduced = []
hdus = (1, 2)
fullim = CCDData.read(filename)
xbin = fullim.header['CCDBIN1']
ybin = fullim.header['CCDBIN2']
airm = fullim.header['AIRMASS']
for h in hdus:
im = CCDData.read(filename, hdu=h)
oscansec = im.header['BIASSEC']
trimsec = im.header['TRIMSEC']
poly_model = models.Polynomial1D(1)
im = ccdproc.subtract_overscan(im,
fits_section=oscansec,
overscan_axis=None,
model=poly_model)
im = ccdproc.trim_image(im, fits_section=trimsec)
reduced.append(im)
if xbin != ybin:
raise Exception("ARTNreduce requires equal binning in both axes.")
# hacky, hard-coded way to make combined image for mont4k. result is E to left, N up.
w = reduced[1].wcs.copy()
ysize, xsize = reduced[1].shape
w.array_shape = (ysize, xsize * 2)
blank = np.zeros((ysize, xsize * 2))
blank[:, :xsize] = reduced[1].data
blank[:, xsize:] = np.fliplr(reduced[0].data)
# line added by PND to do the correct flip!
blank[:, :] = np.fliplr(blank)
cr_mask, clean_data = detect_cosmics(blank,
sigclip=5.,
niter=5,
cleantype='medmask',
psffwhm=30. / xbin)
stitched = CCDData(clean_data, wcs=w, unit=reduced[0].unit)
#stitched.header['BINNING'] = xbin
#stitched.header['AIRMASS'] = airm
stitched.header = fullim.header
# added by PND
#This is for quick testing
if to_fits:
_base = os.path.basename(filename)
_fits = f"{_base.split('.')[0]}_stitched.fits"
if os.path.isfile(_fits):
os.remove(_fits)
stitched.write(_fits)
return stitched
def oscan_trim_file(fname):
hdulist = fits.open(fname)
nhdus = len(hdulist)
if nhdus > 1:
istart = 1
else:
istart = 0
# loop from first-data to last HDU.
for i in range(nhdus)[istart:]:
hdulist = fits.open(fname)
data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
data1.header = hdulist[i].header
# What happens if file is already overscan-subtracted?
if modeling:
oscan1 = ccdproc.subtract_overscan(
data1,
fits_section=data1.header['BIASSEC'],
add_keyword={
'overscan': True,
'calstat': 'O'
},
model=models.Polynomial1D(1))
else:
oscan1 = ccdproc.subtract_overscan(
data1,
fits_section=data1.header['BIASSEC'],
add_keyword={
'overscan': True,
'calstat': 'O'
},
model=None)
trim1 = ccdproc.trim_image(oscan1,
fits_section=oscan1.header['TRIMSEC'],
add_keyword={
'trimmed': True,
'calstat': 'OT'
})
fits.update(fname, trim1.data, header=trim1.header, ext=i)
hdulist.close()
return
def overscan_sub(hdul):
exts = len(hdul)
trimmed_hdul = ModHDUList([hdu.copy() for hdu in hdul])
for i in range(exts):
if hdul[i].data is not None:
if 'BIASSEC' in hdul[i].header and 'TRIMSEC' in hdul[i].header:
data = hdul[i].data
ccdData = CCDData(data, unit=u.adu)
poly_model = models.Polynomial1D(3)
oscan_subtracted = subtract_overscan(ccdData, fits_section=hdul[i].header['BIASSEC'], model=poly_model)
trimmed_hdul[i].data = trim_image(oscan_subtracted, fits_section=hdul[i].header['TRIMSEC']).data
return trimmed_hdul
def imreduce(im):
"""
Take a raw FITS image, subtract overscan, and trim image.
"""
oscansec = im.header['BIASSEC']
trimsec = im.header['TRIMSEC']
im = ccdproc.subtract_overscan(im,
fits_section=oscansec,
overscan_axis=None)
im = ccdproc.trim_image(im, fits_section=trimsec)
return im
def make_master_flat(self, list_of_flatfiles):
flat_list = [0]*len(list_of_flatfiles)
for ii, kk in enumerate(list_of_flatfiles):
fitsfile = fits.open(kk)
flat = ccdproc.CCDData(data=fitsfile[1].data, meta=fitsfile[1].header, unit="adu")
flat_scan = ccdproc.subtract_overscan(flat, fits_section='[5:35, :]')
flat_bias = ccdproc.subtract_bias(flat_scan, self.master_bias)
flat_bias.data /= np.median(flat_bias.data[100:-100, 100:-100])
flat_list[ii] = flat_bias
self.master_flat = ccdproc.combine(flat_list, method="average", sigma_clip=True)
self.master_flat.write(self.output_dir+"/"+self.filename+"_masterflat.fits", overwrite=True)
def process(ccd, gain, oscan, tsec):
"""Basic CCD processing for required for data
"""
#oscan subtract
ccd = ccdproc.subtract_overscan(ccd, overscan=oscan, median=True)
#gain correct
ccd = ccdproc.gain_correct(ccd, gain, gain_unit=u.electron/u.adu)
#trim the image
ccd = ccdproc.trim_image(ccd, fits_section=tsec)
return ccd
def process(ccd, gain, oscan, tsec):
"""Basic CCD processing for required for data
"""
#oscan subtract
ccd = ccdproc.subtract_overscan(ccd, overscan=oscan, median=True)
#gain correct
ccd = ccdproc.gain_correct(ccd, gain, gain_unit=u.electron / u.adu)
#trim the image
ccd = ccdproc.trim_image(ccd, fits_section=tsec)
return ccd
def oscan_and_trim(image_list):
"""
Remove overscan and trim a list of images. The original list is replaced by a list of images
with the changes applied.
Implementation done by ccdproc
Parameters:
----------
image_list :: List of CCDData corresponding to images
"""
for idx, img in enumerate(image_list):
oscan = ccdproc.subtract_overscan(img,overscan=img[:,521:544], add_keyword={'oscan_sub': True, 'calstat': 'O'}, model=models.Polynomial1D(1))
image_list[idx] = ccdproc.trim_image(oscan[:,10:521], add_keyword={'trimmed': True, 'calstat': 'OT'})
def _trim_oscan(ccd, biassec, trimsec, model=None):
"""Subtract the overscan region and trim image to desired size.
The CCDPROC function subtract_overscan() expects the TRIMSEC of the image
(the part you want to keep) to span the entirety of one dimension, with the
BIASSEC (overscan section) being at the end of the other dimension.
Both LMI and DeVeny have edge effects on all sides of their respective
chips, and so the TRIMSEC and BIASSEC do not meet the expectations of
subtract_overscan().
Therefore, this function is a wrapper to first remove the undesired ROWS
from top and bottom, then perform the subtract_overscan() fitting and
subtraction, followed by trimming off the now-spent overscan region.
Args:
ccd (:TYPE:`internal link or datatype`)
Description.
biassec (:TYPE:`str`)
Description.
trimsec (:TYPE:`str`)
Description.
model (:TYPE:internal link or datatype`)
Description.
Returns:
ccd (:TYPE:`internal link or datatype`)
Description.
"""
# Convert the FITS bias & trim sections into slice classes for use
yb, xb = slice_from_string(biassec, fits_convention=True)
yt, xt = slice_from_string(trimsec, fits_convention=True)
# First trim off the top & bottom rows
ccd = ccdp.trim_image(ccd[yt.start:yt.stop, :])
# Model & Subtract the overscan
if model is None:
model = models.Chebyshev1D(1) # Chebyshev 1st order function
else:
model = models.Chebyshev1D(1) # Figure out how to incorporate others
ccd = ccdp.subtract_overscan(ccd,
overscan=ccd[:, xb.start:xb.stop],
median=True,
model=model)
# Trim the overscan & return
return ccdp.trim_image(ccd[:, xt.start:xt.stop])
def make_science(self, list_of_sciencefiles):
science_list = [0]*len(list_of_sciencefiles)
science_names = [0]*len(list_of_sciencefiles)
for ii, kk in enumerate(list_of_sciencefiles):
fitsfile = fits.open(kk)
science = ccdproc.CCDData(data=fitsfile[1].data, meta=fitsfile[1].header, unit="adu")
# science_cos = ccdproc.cosmicray_lacosmic(science, verbose = True)
overscan_sub = ccdproc.subtract_overscan(science, fits_section='[5:35, :]')
bias_sub = ccdproc.subtract_bias(overscan_sub, self.master_bias)
flat_corr = ccdproc.flat_correct(bias_sub, self.master_flat)
# science_list[ii] = flat_corr
flat_corr.write(self.output_dir+"/"+self.filename+"_"+str(ii)+".fits", overwrite=True)
science_names[ii] = self.output_dir+"/"+self.filename+"_"+str(ii)+".fits"
# science_list = [0]*len(list_of_sciencefiles)
coverages = [0]*len(list_of_sciencefiles)
for ii, kk in enumerate(science_names):
fitsfile = fits.open(kk)
fitsfile_common, coverage = reproject_interp(fitsfile, fits.open(science_names[0])[0].header, hdu_in=0)
science = ccdproc.CCDData(data=fitsfile_common, meta=fitsfile[0].header, unit="adu")
science_list[ii] = science
coverages[ii] = ccdproc.CCDData(data=coverage, meta=fitsfile[0].header, unit="adu")
self.combined_science = ccdproc.combine(science_list, method="median")
# self.combined_coverage = ccdproc.combine(coverages, method="sum")
header0 = fits.open(list_of_sciencefiles[0])[0].header
for key in header0:
# print(str(key), header0[str(key)])
if "COMMENT" in key:
continue
self.combined_science.header[str(key)] = header0[str(key)]
self.combined_science.write(self.output_dir+"/"+self.filename+".fits", overwrite=True)
def _correct_overscan_hdu(hdu_ccd):
"""
Given an HDU loaded as a CCDData object perform overscan correction.
Arguments
---------
hdu_ccd : astropy.nddata.CCDData
HDU to perform correction
Returns
-------
img_trim : astropy.nddata.CCDData
Corrected HDU
"""
img_osub = (ccdproc.subtract_overscan(
hdu_ccd,
fits_section=hdu_ccd.header['BIASSEC'],
model=None,
median=True,
add_keyword={
'overscan': True,
'calstat': 'O'
}))
img_trim = (ccdproc.trim_image(img_osub,
fits_section=img_osub.header['TRIMSEC'],
add_keyword={
'trimmed': True,
'calstat': 'OT'
}))
# Updating header and overwriting processed image
del img_trim.header['BIASSEC']
del img_trim.header['TRIMSEC']
return img_trim
def t120_mkdark(dark_dir=t120.t120_dark_dir,
master_offset=t120.t120_ofst_dir + t120.t120_master_name,
master_file_name=t120.t120_master_name):
# read offset
hdu_offset_list = fits.open(master_offset)
offset = CCDData(hdu_offset_list[0].data, unit=u.adu)
master_file = dark_dir + master_file_name
listimg = ImageFileCollection(
dark_dir) #,glob_include='*.fit',glob_exclude='*.fits')
dict_ccd_data = {}
list_ccd_data = []
for fit_file in glob.glob(dark_dir + '*.fit'):
t120.log.info('now opening file: ' + fit_file)
hdu = fits.open(fit_file)
exp_time = hdu[0].header['EXPTIME']
strexptime = "%3.1f" % exp_time
t120.log.info('EXPTIME=' + str(exp_time) + ' strexptime=' + strexptime)
if not dict_ccd_data.has_key(strexptime):
dict_ccd_data[strexptime] = []
else:
dict_ccd_data[strexptime].append(
subtract_overscan(CCDData(hdu[0].data, unit=u.adu), offset))
t120.log.info('now loop over the exp_time')
for strexp_time in dict_ccd_data:
t120.log.info('exp_time: ' + strexp_time)
combiner = Combiner(dict_ccd_data[strexp_time])
master_dark = combiner.median_combine()
master_file = dark_dir + master_file_name.replace(
'.fits', '') + '-' + strexp_time + '.fits'
hdu = master_dark.to_hdu()
#hdu[0].header.set('EXPTIME',value=exp_time,comment='Exposure time in sec')
#hdu[0].header.set('EXPOSURE',value=exp_time,comment='Exposure time in sec')
#hdu.writeto(master_file,overwrite=True)
fits_ccddata_writer(master_dark, master_file)
t120.log.info('Master dark saved in ' + master_file)
return
def action(self, ccd):
"""
Subtract overscan from image based on settings.
Parameters
----------
ccd : `ccdproc.CCDData`
Image to be reduced.
"""
if not self.toggle.value:
pass
whole_axis = slice(None, None)
partial_axis = slice(self._axis_selection.start,
self._axis_selection.stop)
# create a two-element list which will be filled with the appropriate
# slice based on the widget settings.
if self._axis_selection.full_axis == 0:
first_axis = whole_axis
second_axis = partial_axis
oscan_axis = 1
else:
first_axis = partial_axis
second_axis = whole_axis
oscan_axis = 0
if self._polyfit.toggle.value:
poly_model = models.Polynomial1D(self.polynomial_order)
else:
poly_model = None
reduced = ccdproc.subtract_overscan(ccd,
overscan=ccd[first_axis,
second_axis],
overscan_axis=oscan_axis,
model=poly_model)
return reduced
def action(self, ccd):
"""
Subtract overscan from image based on settings.
Parameters
----------
ccd : `ccdproc.CCDData`
Image to be reduced.
"""
if not self.toggle.value:
pass
whole_axis = slice(None, None)
partial_axis = slice(self._axis_selection.start,
self._axis_selection.stop)
# create a two-element list which will be filled with the appropriate
# slice based on the widget settings.
if self._axis_selection.full_axis == 0:
first_axis = whole_axis
second_axis = partial_axis
oscan_axis = 1
else:
first_axis = partial_axis
second_axis = whole_axis
oscan_axis = 0
if self._polyfit.toggle.value:
poly_model = models.Polynomial1D(self.polynomial_order)
else:
poly_model = None
reduced = ccdproc.subtract_overscan(ccd,
overscan=ccd[first_axis, second_axis],
overscan_axis=oscan_axis,
model=poly_model)
return reduced
indir = sys.argv[1]
outdir = sys.argv[2]
if not os.path.isdir(outdir): os.mkdir(outdir)
os.chdir(outdir)
#change this to point to your raw data directory
ic1 = ImageFileCollection(indir)
#create the bias frames
blue_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Blue arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
blue_bias_list.append(ccd)
master_bias_blue = ccdproc.combine(blue_bias_list, method='median')
master_bias_blue.write('master_bias_blue.fits', clobber=True)
red_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Red arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
red_bias_list.append(ccd)
master_bias_red = ccdproc.combine(red_bias_list, method='median')
master_bias_red.write('master_bias_red.fits', clobber=True)
parser.add_argument('--irafbiassec', dest = 'irafbiassec', default= '[4100:4150, 1:4150]', help = 'biassec in iraf notation. default is [4100:4150, 1:4150], which applies to HDI camera')
parser.add_argument('--iraftrimsec', dest = 'iraftrimsec', default= '[1:4095, 1:4109]', help = 'biassec in iraf notation. default is [4100:4150, 1:4150], which applies to HDI camera')
#parser.add_argument('--gain', dest = 'gain', default= 1.3, help = 'gain in e-/ADU. default is 1.3, which applies to HDI camera')
#parser.add_argument('--rdnoise', dest = 'rdnoise', default= 7.3, help = 'gain in e-/ADU. default is 1.3, which applies to HDI camera')
args = parser.parse_args()
files = sorted(glob.glob(args.filestring+'*.fits'))
nfiles=len(files)
poly_model = models.Polynomial1D(1)
for f in files:
# read in image
# was having trouble getting image into the format that ccdproc wants
print 'working on ',f
# convert data to CCDData format and save header
ccd = CCDData.read(f, unit='adu')
# subtract overscan
o_subtracted = ccdproc.subtract_overscan(ccd, fits_section = args.irafbiassec, model=poly_model)
header['HISTORY'] = 'overscan subtracted '+args.irafbiassec
# trim image
trimmed = ccdproc.trim_image(o_subtracted, fits_section = args.iraftrimsec)
head_updates = {'CCDSEC':args.iraftrimsec}
trimmed.write('tr'+f,add_keyword=head_updates)
allfiles = cp.ImageFileCollection(path,keywords = fileKeys)
gain = 2.89 * u.electron/u.adu
rdnoise = 6 * u.electron
# correct bias images for overscan and trim the image:
biasImages=[]
for filename in allfiles.files_filtered(NAXIS1=3128,NAXIS2=3080,OBSTYPE = 'BIAS'):
ccd = fits.getdata(allfiles.location + filename)
print(ccd)
ccd = cp.CCDData(ccd, unit = u.adu)
print(ccd)
ccd = cp.subtract_overscan(ccd,overscan_axis = 1, overscan = ccd[:,3099:3125])
print(ccd)
ccd = cp.trim_image(ccd,fits_section = '[27:3095,3:3078]')
print(ccd)
biasImages.append(ccd)
# Create the master bias frame and write to file
master_bias = cp.combine(biasImages,output_file = path + 'mbias_avg.fits', method='average')
#master_bias.write('mbias_avg.fits',clobber=True)
# assemble lists of filter-specific flat files.
flatlist_OH = []
for filename in allfiles.files_filtered(OBSTYPE='SKY FLAT',FILTERS='OH'):
ccd = fits.getdata(allfiles.location + filename)
def correct_overscan(file_path):
"""
Given a list of FITS filenames, does overscan correction over all image
extensions.
Arguments
---------
files_path : str
Path to the file to process.
File transformations
--------------------
Re-write FITS file, with overscan correction and updated header.
Returns
-------
None
"""
hdul = fits.open(file_path)
image_indices = _check_image_extensions(hdul)
# Loop over extensions
for i in image_indices:
# Aborting if the keyword 'BIASSEC' isn't defined
if not ('BIASSEC' in hdul[i].header):
print(
f"Skipping overscan correction on: {file_path:1s}[{i:1.0f}] - BIASSEC keyword not found"
)
continue
img = CCDData(data=hdul[i].data, header=hdul[i].header, unit="adu")
img_osub = (ccdproc.subtract_overscan(
img,
fits_section=img.header['BIASSEC'],
model=None,
median=True,
add_keyword={
'overscan': True,
'calstat': 'O'
}))
img_trim = (ccdproc.trim_image(img_osub,
fits_section=img_osub.header['TRIMSEC'],
add_keyword={
'trimmed': True,
'calstat': 'OT'
}))
# Updating header and overwriting processed image
del img_trim.header['BIASSEC']
del img_trim.header['TRIMSEC']
fits.update(file_path,
img_trim.data.astype(np.float32),
header=img_trim.header,
ext=i)
hdul.close()
)
print(
"So just put the name you want the file and the computer will add 'overscan0.fits' and so on"
)
# Iterate over how many files there are
for n in range(len(filearray)):
im = filearray[n]
imreplace = filelist[n]
outover = imreplace.replace('.fits', '')
# If the overscan is in the x direction
if overscanx != 0:
# Find the edge of the image where the overscan is
x2 = xsize - overscanx
# Subtract the overscan from the image
oscan = ccdproc.subtract_overscan(im,
overscan=im[0:ysize,
x2:xsize],
overscan_axis=1)
# Save the image
astropy.nddata.fits_ccddata_writer(
oscan, outover + 'overscan' + str(n) + '.fits')
# Paste the header into the new file
file2 = fits.open(outover + 'overscan' + str(n) + '.fits')
hdu2 = file2[0].header
hdu2.clear()
hdu2.extend(headerlist[n])
file2.close()
# If the overscan is in the y direction
if overscany != 0:
# Find the edge of the iamge where the overscan is
y2 = ysize - overscany
# Subtract the overscan from the image
def bias_correction(target_dir, ic, config, logger):
import os
import numpy as np
import astropy.units as u
from ccdproc import CCDData, combine
from ccdproc import subtract_bias, subtract_overscan, trim_image
logger.debug('Bias correction.')
fn = os.sep.join((target_dir, 'bias.fits'))
if os.path.exists(fn) and not config.reprocess_all:
logger.info('Read bias = {}.'.format(fn))
bias = CCDData.read(fn)
elif len(ic.files_filtered(imagetyp='bias')) == 0:
logger.warning('No bias files provided.')
logger.warning('{} not found'.format(fn))
try:
last_bias = find_last(target_dir, config, 'bias.fits')
bias = CCDData.read(last_bias)
logger.info('Using {}.'.format(last_bias))
except AssertionError:
logger.warning('No previous bias found. Not subtracting bias.')
bias = 0 * u.electron
else:
logger.info('Create bias = {}.'.format(fn))
files = ic.files_filtered(include_path=True, imagetyp='bias')
bias = combine(files,
method='average',
clip_extrema=True,
nlow=1,
nhigh=1)
bias.meta['FILENAME'] = os.path.basename(fn)
n = str([int(f.split('.')[-2]) for f in files])
bias.meta.add_history('Created from file numbers: {}'.format(n))
bias.write(fn, overwrite=True)
logger.info('Bias subtract and trim data.')
i = ic.summary['subbias'].mask
logger.debug('Bias subtract {} files.'.format(sum(i)))
meanbias = np.mean(bias)
for fn in ic.summary['file'][i]:
ccd = CCDData.read(os.sep.join([ic.location, fn]), unit='electron')
logger.debug(fn)
ccd = subtract_bias(ccd, bias)
ccd.meta['BIASFILE'] = (bias.meta['FILENAME'],
'Name of the bias frame used.')
ccd.meta['MEANBIAS'] = (meanbias, 'Mean bias level, electrons')
if config.overscan_correct:
overscan = CCDData(np.hstack([ccd[:, :50], ccd[:, 2115:]]),
unit=ccd.unit)
ccd = subtract_overscan(ccd, overscan, median=True)
for s in config.trim_sections:
ccd = trim_image(ccd[s])
ccd.write(os.sep.join([ic.location, fn]), overwrite=True)
ic.refresh()
return ic
def ccd_process(ccd,
oscan=None,
trim=None,
error=False,
masterbias=None,
bad_pixel_mask=None,
gain=None,
rdnoise=None,
oscan_median=True,
oscan_model=None):
"""Perform basic processing on ccd data.
The following steps can be included:
* overscan correction
* trimming of the image
* create edeviation frame
* gain correction
* add a mask to the data
* subtraction of master bias
The task returns a processed `ccdproc.CCDData` object.
Parameters
----------
ccd: `ccdproc.CCDData`
Frame to be reduced
oscan: None, str, or, `~ccdproc.ccddata.CCDData`
For no overscan correction, set to None. Otherwise proivde a region
of `ccd` from which the overscan is extracted, using the FITS
conventions for index order and index start, or a
slice from `ccd` that contains the overscan.
trim: None or str
For no trim correction, set to None. Otherwise proivde a region
of `ccd` from which the image should be trimmed, using the FITS
conventions for index order and index start.
error: boolean
If True, create an uncertainty array for ccd
masterbias: None, `~numpy.ndarray`, or `~ccdproc.CCDData`
A materbias frame to be subtracted from ccd.
bad_pixel_mask: None or `~numpy.ndarray`
A bad pixel mask for the data. The bad pixel mask should be in given
such that bad pixels havea value of 1 and good pixels a value of 0.
gain: None or `~astropy.Quantity`
Gain value to multiple the image by to convert to electrons
rdnoise: None or `~astropy.Quantity`
Read noise for the observations. The read noise should be in
`~astropy.units.electron`
oscan_median : bool, optional
If true, takes the median of each line. Otherwise, uses the mean
oscan_model : `~astropy.modeling.Model`, optional
Model to fit to the data. If None, returns the values calculated
by the median or the mean.
Returns
-------
ccd: `ccdproc.CCDData`
Reduded ccd
Examples
--------
1. To overscan, trim, and gain correct a data set:
>>> import numpy as np
>>> from astropy import units as u
>>> from hrsprocess import ccd_process
>>> ccd = CCDData(np.ones([100, 100]), unit=u.adu)
>>> nccd = ccd_process(ccd, oscan='[1:10,1:100]', trim='[10:100, 1,100]',
error=False, gain=2.0*u.electron/u.adu)
"""
# make a copy of the object
nccd = ccd.copy()
# apply the overscan correction
if isinstance(oscan, ccdproc.CCDData):
nccd = ccdproc.subtract_overscan(nccd,
overscan=oscan,
median=oscan_median,
model=oscan_model)
elif isinstance(oscan, six.string_types):
nccd = ccdproc.subtract_overscan(nccd,
fits_section=oscan,
median=oscan_median,
model=oscan_model)
elif oscan is None:
pass
else:
raise TypeError('oscan is not None, a string, or CCDData object')
# apply the trim correction
if isinstance(trim, six.string_types):
nccd = ccdproc.trim_image(nccd, fits_section=trim)
elif trim is None:
pass
else:
raise TypeError('trim is not None or a string')
# create the error frame
if error and gain is not None and rdnoise is not None:
nccd = ccdproc.create_deviation(nccd, gain=gain, rdnoise=rdnoise)
elif error and (gain is None or rdnoise is None):
raise ValueError(
'gain and rdnoise must be specified to create error frame')
# apply the bad pixel mask
if isinstance(bad_pixel_mask, np.ndarray):
nccd.mask = bad_pixel_mask
elif bad_pixel_mask is None:
pass
else:
raise TypeError('bad_pixel_mask is not None or numpy.ndarray')
# apply the gain correction
if isinstance(gain, u.quantity.Quantity):
nccd = ccdproc.gain_correct(nccd, gain)
elif gain is None:
pass
else:
raise TypeError('gain is not None or astropy.Quantity')
# test subtracting the master bias
if isinstance(masterbias, ccdproc.CCDData):
nccd = ccdproc.subtract_bias(nccd, masterbias)
elif isinstance(masterbias, np.ndarray):
nccd.data = nccd.data - masterbias
elif masterbias is None:
pass
else:
raise TypeError(
'masterbias is not None, numpy.ndarray, or a CCDData object')
return nccd
outdir = sys.argv[2]
if not os.path.isdir(outdir): os.mkdir(outdir)
os.chdir(outdir)
#change this to point to your raw data directory
ic1 = ImageFileCollection(indir)
#create the bias frames
blue_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Blue arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit=u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd,
median=True,
overscan_axis=0,
fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
blue_bias_list.append(ccd)
master_bias_blue = ccdproc.combine(blue_bias_list, method='median')
master_bias_blue.write('master_bias_blue.fits', clobber=True)
red_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Red arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit=u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd,
median=True,
overscan_axis=0,
fits_section='[1:966,4105:4190]')
def main(night_path, skip_list_file, mask_file, overwrite=False, plot=False):
"""
See argparse block at bottom of script for description of parameters.
"""
night_path = path.realpath(path.expanduser(night_path))
if not path.exists(night_path):
raise IOError("Path '{}' doesn't exist".format(night_path))
logger.info("Reading data from path: {}".format(night_path))
base_path, night_name = path.split(night_path)
data_path, run_name = path.split(base_path)
output_path = path.realpath(
path.join(data_path, 'processed', run_name, night_name))
os.makedirs(output_path, exist_ok=True)
logger.info("Saving processed files to path: {}".format(output_path))
if plot: # if we're making plots
plot_path = path.realpath(path.join(output_path, 'plots'))
logger.debug("Will make and save plots to: {}".format(plot_path))
os.makedirs(plot_path, exist_ok=True)
else:
plot_path = None
# check for files to skip (e.g., saturated or errored exposures)
if skip_list_file is not None: # a file containing a list of filenames to skip
with open(skip_list_file, 'r') as f:
skip_list = [x.strip() for x in f if x.strip()]
else:
skip_list = None
# look for pixel mask file
if mask_file is not None:
with open(
mask_file, 'r'
) as f: # load YAML file specifying pixel masks for nearby sources
pixel_mask_spec = yaml.load(f.read())
else:
pixel_mask_spec = None
# generate the raw image file collection to process
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
logger.info("Frames to process:")
logger.info("- Bias frames: {}".format(
len(ic.files_filtered(imagetyp='BIAS'))))
logger.info("- Flat frames: {}".format(
len(ic.files_filtered(imagetyp='FLAT'))))
logger.info("- Comparison lamp frames: {}".format(
len(ic.files_filtered(imagetyp='COMP'))))
logger.info("- Object frames: {}".format(
len(ic.files_filtered(imagetyp='OBJECT'))))
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
# ============================
# Create the master bias frame
# ============================
# overscan region of the CCD, using FITS index notation
oscan_fits_section = "[{}:{},:]".format(oscan_idx, oscan_idx + oscan_size)
master_bias_file = path.join(output_path, 'master_bias.fits')
if not os.path.exists(master_bias_file) or overwrite:
# get list of overscan-subtracted bias frames as 2D image arrays
bias_list = []
for hdu, fname in ic.hdus(return_fname=True, imagetyp='BIAS'):
logger.debug('Processing Bias frame: {0}'.format(fname))
ccd = CCDData.read(path.join(ic.location, fname), unit='adu')
ccd = ccdproc.gain_correct(ccd, gain=ccd_gain)
ccd = ccdproc.subtract_overscan(ccd, overscan=ccd[:, oscan_idx:])
ccd = ccdproc.trim_image(ccd,
fits_section="[1:{},:]".format(oscan_idx))
bias_list.append(ccd)
# combine all bias frames into a master bias frame
logger.info("Creating master bias frame")
master_bias = ccdproc.combine(bias_list,
method='average',
clip_extrema=True,
nlow=1,
nhigh=1,
error=True)
master_bias.write(master_bias_file, overwrite=True)
else:
logger.info("Master bias frame file already exists: {}".format(
master_bias_file))
master_bias = CCDData.read(master_bias_file)
if plot:
# TODO: this assumes vertical CCD
assert master_bias.shape[0] > master_bias.shape[1]
aspect_ratio = master_bias.shape[1] / master_bias.shape[0]
fig, ax = plt.subplots(1, 1, figsize=(10, 12 * aspect_ratio))
vmin, vmax = zscaler.get_limits(master_bias.data)
cs = ax.imshow(master_bias.data.T,
origin='bottom',
cmap=cmap,
vmin=max(0, vmin),
vmax=vmax)
ax.set_title('master bias frame [zscale]')
fig.colorbar(cs)
fig.tight_layout()
fig.savefig(path.join(plot_path, 'master_bias.png'))
plt.close(fig)
# ============================
# Create the master flat field
# ============================
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
master_flat_file = path.join(output_path, 'master_flat.fits')
if not os.path.exists(master_flat_file) or overwrite:
# create a list of flat frames
flat_list = []
for hdu, fname in ic.hdus(return_fname=True, imagetyp='FLAT'):
logger.debug('Processing Flat frame: {0}'.format(fname))
ccd = CCDData.read(path.join(ic.location, fname), unit='adu')
ccd = ccdproc.gain_correct(ccd, gain=ccd_gain)
ccd = ccdproc.ccd_process(ccd,
oscan=oscan_fits_section,
trim="[1:{},:]".format(oscan_idx),
master_bias=master_bias)
flat_list.append(ccd)
# combine into a single master flat - use 3*sigma sigma-clipping
logger.info("Creating master flat frame")
master_flat = ccdproc.combine(flat_list,
method='average',
sigma_clip=True,
low_thresh=3,
high_thresh=3)
master_flat.write(master_flat_file, overwrite=True)
# TODO: make plot if requested?
else:
logger.info("Master flat frame file already exists: {}".format(
master_flat_file))
master_flat = CCDData.read(master_flat_file)
if plot:
# TODO: this assumes vertical CCD
assert master_flat.shape[0] > master_flat.shape[1]
aspect_ratio = master_flat.shape[1] / master_flat.shape[0]
fig, ax = plt.subplots(1, 1, figsize=(10, 12 * aspect_ratio))
vmin, vmax = zscaler.get_limits(master_flat.data)
cs = ax.imshow(master_flat.data.T,
origin='bottom',
cmap=cmap,
vmin=max(0, vmin),
vmax=vmax)
ax.set_title('master flat frame [zscale]')
fig.colorbar(cs)
fig.tight_layout()
fig.savefig(path.join(plot_path, 'master_flat.png'))
plt.close(fig)
# =====================
# Process object frames
# =====================
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
logger.info("Beginning object frame processing...")
for hdu, fname in ic.hdus(return_fname=True, imagetyp='OBJECT'):
new_fname = path.join(output_path, 'p_{}'.format(fname))
# -------------------------------------------
# First do the simple processing of the frame
# -------------------------------------------
logger.debug("Processing '{}' [{}]".format(hdu.header['OBJECT'],
fname))
if path.exists(new_fname) and not overwrite:
logger.log(1, "\tAlready processed! {}".format(new_fname))
ext = SourceCCDExtractor(filename=path.join(
ic.location, new_fname),
plot_path=plot_path,
zscaler=zscaler,
cmap=cmap,
**ccd_props)
nccd = ext.ccd
# HACK: F**K this is a bad hack
ext._filename_base = ext._filename_base[2:]
else:
# process the frame!
ext = SourceCCDExtractor(filename=path.join(ic.location, fname),
plot_path=plot_path,
zscaler=zscaler,
cmap=cmap,
unit='adu',
**ccd_props)
_pix_mask = pixel_mask_spec.get(
fname, None) if pixel_mask_spec is not None else None
nccd = ext.process_raw_frame(pixel_mask_spec=_pix_mask,
master_bias=master_bias,
master_flat=master_flat)
nccd.write(new_fname, overwrite=overwrite)
# -------------------------------------------
# Now do the 1D extraction
# -------------------------------------------
fname_1d = path.join(output_path, '1d_{0}'.format(fname))
if path.exists(fname_1d) and not overwrite:
logger.log(1, "\tAlready extracted! {}".format(fname_1d))
continue
else:
logger.debug("\tExtracting to 1D")
# first step is to fit a voigt profile to a middle-ish row to determine LSF
lsf_p = ext.get_lsf_pars() # MAGIC NUMBER
try:
tbl = ext.extract_1d(lsf_p)
except Exception as e:
logger.error('Failed! {}: {}'.format(e.__class__.__name__,
str(e)))
continue
hdu0 = fits.PrimaryHDU(header=nccd.header)
hdu1 = fits.table_to_hdu(tbl)
hdulist = fits.HDUList([hdu0, hdu1])
hdulist.writeto(fname_1d, overwrite=overwrite)
del ext
# ==============================
# Process comparison lamp frames
# ==============================
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
logger.info("Beginning comp. lamp frame processing...")
for hdu, fname in ic.hdus(return_fname=True, imagetyp='COMP'):
new_fname = path.join(output_path, 'p_{}'.format(fname))
logger.debug("\tProcessing '{}'".format(hdu.header['OBJECT']))
if path.exists(new_fname) and not overwrite:
logger.log(1, "\tAlready processed! {}".format(new_fname))
ext = CompCCDExtractor(filename=path.join(ic.location, new_fname),
plot_path=plot_path,
zscaler=zscaler,
cmap=cmap,
**ccd_props)
nccd = ext.ccd
# HACK: F**K this is a bad hack
ext._filename_base = ext._filename_base[2:]
else:
# process the frame!
ext = CompCCDExtractor(filename=path.join(ic.location, fname),
plot_path=plot_path,
unit='adu',
**ccd_props)
_pix_mask = pixel_mask_spec.get(
fname, None) if pixel_mask_spec is not None else None
nccd = ext.process_raw_frame(
pixel_mask_spec=_pix_mask,
master_bias=master_bias,
master_flat=master_flat,
)
nccd.write(new_fname, overwrite=overwrite)
# -------------------------------------------
# Now do the 1D extraction
# -------------------------------------------
fname_1d = path.join(output_path, '1d_{0}'.format(fname))
if path.exists(fname_1d) and not overwrite:
logger.log(1, "\tAlready extracted! {}".format(fname_1d))
continue
else:
logger.debug("\tExtracting to 1D")
try:
tbl = ext.extract_1d()
except Exception as e:
logger.error('Failed! {}: {}'.format(e.__class__.__name__,
str(e)))
continue
hdu0 = fits.PrimaryHDU(header=nccd.header)
hdu1 = fits.table_to_hdu(tbl)
hdulist = fits.HDUList([hdu0, hdu1])
hdulist.writeto(fname_1d, overwrite=overwrite)
def process_raw_frame(self, master_bias, master_flat, pixel_mask_spec=None):
"""
Bias and flat-correct a raw CCD frame. Trim off the overscan
region. Identify cosmic rays using "lacosmic" and inflat
uncertainties where CR's are found. If specified, mask out
nearby sources by setting pixel uncertainty to infinity (or
inverse-variance to 0).
Returns
-------
nccd : `ccdproc.CCDData`
A copy of the original ``CCDData`` object but after the
above procedures have been run.
"""
oscan_fits_section = "[{}:{},:]".format(self.oscan_idx,
self.oscan_idx+self.oscan_size)
# make a copy of the object
nccd = self.ccd.copy()
# apply the overscan correction
poly_model = Polynomial1D(2)
nccd = ccdproc.subtract_overscan(nccd, fits_section=oscan_fits_section,
model=poly_model)
# trim the image (remove overscan region)
nccd = ccdproc.trim_image(nccd, fits_section='[1:{},:]'.format(self.oscan_idx))
# create the error frame
nccd = ccdproc.create_deviation(nccd, gain=self.ccd_gain,
readnoise=self.ccd_readnoise)
# now correct for the ccd gain
nccd = ccdproc.gain_correct(nccd, gain=self.ccd_gain)
# correct for master bias frame
# - this does some crazy shit at the blue end, but we can live with it
nccd = ccdproc.subtract_bias(nccd, master_bias)
# correct for master flat frame
nccd = ccdproc.flat_correct(nccd, master_flat)
# comsic ray cleaning - this updates the uncertainty array as well
nccd = ccdproc.cosmicray_lacosmic(nccd, sigclip=8.)
# replace ccd with processed ccd
self.ccd = nccd
# check for a pixel mask
if pixel_mask_spec is not None:
mask = self.make_nearby_source_mask(pixel_mask_spec)
logger.debug("\t\tSource mask loaded.")
stddev = nccd.uncertainty.array
stddev[mask] = np.inf
nccd.uncertainty = StdDevUncertainty(stddev)
if self.plot_path is not None:
# TODO: this assumes vertical CCD
aspect_ratio = nccd.shape[1]/nccd.shape[0]
fig,axes = plt.subplots(2, 1, figsize=(10,2 * 12*aspect_ratio),
sharex=True, sharey=True)
vmin,vmax = self.zscaler.get_limits(nccd.data)
axes[0].imshow(nccd.data.T, origin='bottom',
cmap=self.cmap, vmin=max(0,vmin), vmax=vmax)
stddev = nccd.uncertainty.array
vmin,vmax = self.zscaler.get_limits(stddev[np.isfinite(stddev)])
axes[1].imshow(stddev.T, origin='bottom',
cmap=self.cmap, vmin=max(0,vmin), vmax=vmax)
axes[0].set_title('Object: {0}, flux'.format(self._obj_name))
axes[1].set_title('root-variance'.format(self._obj_name))
fig.tight_layout()
fig.savefig(path.join(self.plot_path, '{}_frame.png'.format(self._filename_base)))
plt.close(fig)
return nccd
if not flat:
log.critical(
'No master flat present, check data before rerunning.')
logging.shutdown()
sys.exit(-1)
with fits.open(flat) as hdr:
mflat = hdr[1].data
mflat[np.isnan(mflat)] = np.nanmedian(mflat)
mflat = CCDData(mflat, unit=u.electron)
masks = []
processed = []
reprojected = []
for sci in sci_list:
raw = CCDData.read(sci, hdu=1, unit=u.adu)
red = ccdproc.subtract_overscan(raw,
overscan=raw[0:4, :],
overscan_axis=0,
model=models.Chebyshev1D(3))
red = ccdproc.ccd_process(
red,
gain=raw.header['GAIN'] * u.electron / u.adu,
readnoise=raw.header['RDNOISE'] * u.electron)
red = ccdproc.subtract_dark(red,
mdark,
exposure_time='EXPTIME',
exposure_unit=u.second)
red = ccdproc.flat_correct(red, mflat)
processed_data = ccdproc.ccd_process(red,
trim=raw.header['DATASEC'])
sigma_clip = SigmaClip(sigma=3)
bkg_estimator = MedianBackground()
_, median, std = sigma_clipped_stats(processed_data, sigma=3.0)
)
#parser.add_argument('--gain', dest = 'gain', default= 1.3, help = 'gain in e-/ADU. default is 1.3, which applies to HDI camera')
#parser.add_argument('--rdnoise', dest = 'rdnoise', default= 7.3, help = 'gain in e-/ADU. default is 1.3, which applies to HDI camera')
args = parser.parse_args()
files = sorted(glob.glob(args.filestring + '*.fits'))
nfiles = len(files)
poly_model = models.Polynomial1D(1)
for f in files:
# read in image
# was having trouble getting image into the format that ccdproc wants
print 'working on ', f
# convert data to CCDData format and save header
ccd = CCDData.read(f, unit='adu')
# subtract overscan
o_subtracted = ccdproc.subtract_overscan(ccd,
fits_section=args.irafbiassec,
model=poly_model)
#header['HISTORY'] = 'overscan subtracted '+args.irafbiassec
# trim image
head_updates = {'CCDSEC': args.iraftrimsec, 'BIASSEC': args.irafbiassec}
trimmed = ccdproc.trim_image(o_subtracted,
fits_section=args.iraftrimsec,
add_keyword=head_updates)
trimmed.write('tr' + f, overwrite=True)
def ccd_process(ccd, oscan=None, trim=None, error=False, masterbias=None,
bad_pixel_mask=None, gain=None, rdnoise=None,
oscan_median=True, oscan_model=None):
"""Perform basic processing on ccd data.
The following steps can be included:
* overscan correction
* trimming of the image
* create edeviation frame
* gain correction
* add a mask to the data
* subtraction of master bias
The task returns a processed `ccdproc.CCDData` object.
Parameters
----------
ccd: `ccdproc.CCDData`
Frame to be reduced
oscan: None, str, or, `~ccdproc.ccddata.CCDData`
For no overscan correction, set to None. Otherwise proivde a region
of `ccd` from which the overscan is extracted, using the FITS
conventions for index order and index start, or a
slice from `ccd` that contains the overscan.
trim: None or str
For no trim correction, set to None. Otherwise proivde a region
of `ccd` from which the image should be trimmed, using the FITS
conventions for index order and index start.
error: boolean
If True, create an uncertainty array for ccd
masterbias: None, `~numpy.ndarray`, or `~ccdproc.CCDData`
A materbias frame to be subtracted from ccd.
bad_pixel_mask: None or `~numpy.ndarray`
A bad pixel mask for the data. The bad pixel mask should be in given
such that bad pixels havea value of 1 and good pixels a value of 0.
gain: None or `~astropy.Quantity`
Gain value to multiple the image by to convert to electrons
rdnoise: None or `~astropy.Quantity`
Read noise for the observations. The read noise should be in
`~astropy.units.electron`
oscan_median : bool, optional
If true, takes the median of each line. Otherwise, uses the mean
oscan_model : `~astropy.modeling.Model`, optional
Model to fit to the data. If None, returns the values calculated
by the median or the mean.
Returns
-------
ccd: `ccdproc.CCDData`
Reduded ccd
Examples
--------
1. To overscan, trim, and gain correct a data set:
>>> import numpy as np
>>> from astropy import units as u
>>> from hrsprocess import ccd_process
>>> ccd = CCDData(np.ones([100, 100]), unit=u.adu)
>>> nccd = ccd_process(ccd, oscan='[1:10,1:100]', trim='[10:100, 1,100]',
error=False, gain=2.0*u.electron/u.adu)
"""
# make a copy of the object
nccd = ccd.copy()
# apply the overscan correction
if isinstance(oscan, ccdproc.CCDData):
nccd = ccdproc.subtract_overscan(nccd, overscan=oscan,
median=oscan_median,
model=oscan_model)
elif isinstance(oscan, six.string_types):
nccd = ccdproc.subtract_overscan(nccd, fits_section=oscan,
median=oscan_median,
model=oscan_model)
elif oscan is None:
pass
else:
raise TypeError('oscan is not None, a string, or CCDData object')
# apply the trim correction
if isinstance(trim, six.string_types):
nccd = ccdproc.trim_image(nccd, fits_section=trim)
elif trim is None:
pass
else:
raise TypeError('trim is not None or a string')
# create the error frame
if error and gain is not None and rdnoise is not None:
nccd = ccdproc.create_deviation(nccd, gain=gain, rdnoise=rdnoise)
elif error and (gain is None or rdnoise is None):
raise ValueError(
'gain and rdnoise must be specified to create error frame')
# apply the bad pixel mask
if isinstance(bad_pixel_mask, np.ndarray):
nccd.mask = bad_pixel_mask
elif bad_pixel_mask is None:
pass
else:
raise TypeError('bad_pixel_mask is not None or numpy.ndarray')
# apply the gain correction
if isinstance(gain, u.quantity.Quantity):
nccd = ccdproc.gain_correct(nccd, gain)
elif gain is None:
pass
else:
raise TypeError('gain is not None or astropy.Quantity')
# test subtracting the master bias
if isinstance(masterbias, ccdproc.CCDData):
nccd = ccdproc.subtract_bias(nccd, masterbias)
elif isinstance(masterbias, np.ndarray):
nccd.data = nccd.data - masterbias
elif masterbias is None:
pass
else:
raise TypeError(
'masterbias is not None, numpy.ndarray, or a CCDData object')
return nccd
def go_overscan(image_collection,
lbc_chips=True,
objects_only=True,
image_directory='./',
raw_directory='./raw/',
verbose=True,
return_files=True):
"""Remove overscan and trim images."""
##
## Fit, subtract overscan
##
## Notes:
## image_collection -- A ccdproc-style image collection.
## image_directory -- Where to store the output images.
## raw_directory -- Where to find the raw data
## return_files -- Return a list of subtracted files (default: True)
###### Define which chips to extract if default is chosen:
if lbc_chips == True:
lbc_chips = [1, 2, 3, 4]
# By default we're only doing this to the object files.
# Flats and biases are corrected when producing the master calibration images.
if objects_only == True:
over_files = image_collection.files_filtered(imagetyp='object')
else:
over_files = image_collection.files
over_files_out = []
# Loop through the files
for filename in over_files:
# Create the output filename.
output_filename = filename.split('.fits')[0] + '_over.fits'
# Set up the output HDU list
# Capture the 0th header
base_header = fits.getheader(raw_directory + filename)
master_hdu = fits.PrimaryHDU(header=base_header)
# Start output HDU list:
output_hdu = fits.HDUList([master_hdu])
# Loop through the chips
for chip in lbc_chips:
# Create the CCDData version of this chip
ccd = CCDData.read(raw_directory + filename, chip, unit=u.adu)
# Fit, subtract overscan
poly_model = models.Polynomial1D(4)
ccd = ccdproc.subtract_overscan(ccd,
overscan_axis=1,
model=poly_model,
fits_section=ccd.header['BIASSEC'])
# Trim the image
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
# Convert to 32 bit
ccd.data = ccd.data.astype('float32')
# Remove unneeded header keywords. Makes this consistent
# with IRAF treatment. Also take out potentially confusing
# LBC keywords
temp_header = ccd.header
bd_keywords = [
'TRIMSEC', 'BIASSEC', 'DATASEC', 'ROTANGLE', 'PARANGLE'
]
for ky in bd_keywords:
try:
temp_header.pop(ky)
except:
pass
# The LBC includes duplicate astrometric keywords that confuse SCAMP/SWARP.
# These all end in "A" and must be removed in order for the astrometric
# solution to work.
xxx = temp_header['C*A']
for ky in xxx.keys():
try:
temp_header.pop(ky)
except:
pass
# Replace the header with the edited version
ccd.header = temp_header
# Append the current chip into the hdu:
output_hdu.append(ccd.to_hdu()[0])
# Write the data
output_hdu.writeto(output_filename, overwrite=True)
# Keep track of what files we've written.
over_files_out.append(output_filename)
# Report:
if verbose:
print("Created {0}".format(output_filename))
# Return the corrected filenames if requested (True is default).
if return_files == True:
return over_files_out
#for j in bias_set2:
#print(j.shape)
# Assert: each bias frame in the set must have equal dimensions.
# concatenate component lists.
all_bias = bias_set1 + bias_set2
# Create master-bias image by combining bias images and taking the mean. **ccdproc admits a list of fits files. output file must be .fits else error.
for img in all_bias:
ccd = cp.CCDData.read(img, unit=u.adu)
print(ccd)
ccd = cp.subtract_overscan(ccd,
overscan_axis=1,
fits_section='[3100:3129,:]')
ccd = cp.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
biasImages.append(ccd)
mbias_avg = ccd.combine(biasImages,
output_file=path + 'mbias_avg.fits',
method='average')
### Subtract mbias from each flat ###
# Create lists of flats in order to iterate the mbias subtraction over them using the information from reduc_tbl() (observation log)
flatlist_OH = [(path + 'lmi.00%d.fits' % n) for n in range(41, 51)]
flatlist_NH = [(path + 'lmi.00%d.fits' % n) for n in range(51, 62)]
