def combine_flats(self, flat_list, master_dark, method="median"):
"""Combine and reduce a series of flat frames into a normalized flatfield via CCDPROC"""
if method == "median":
print("Combining flats by median")
combined_flat = ccdproc.combine(flat_list, method="median", unit="adu", mem_limit=6e9)
elif method == "mean":
print("Combining flats by mean")
combined_flat = ccdproc.combine(flat_list, method="mean", unit="adu", mem_limit=6e9)
else:
print("Combining flats by median")
combined_flat = ccdproc.combine(flat_list, method="median", unit="adu", mem_limit=6e9)
print("Subtracting master dark from combined flat")
master_flat = ccdproc.subtract_dark(combined_flat, master_dark, data_exposure=combined_flat.header["exposure"]*u.second, dark_exposure=master_dark.header["exposure"]*u.second, scale=True)
print("Reading master flat data")
master_flat_data = np.asarray(master_flat)
print("Creating normalized flatfield")
flatfield_data = master_flat_data / np.mean(master_flat_data)
print("Converting flatfield data to CCDData")
flatfield = ccdproc.CCDData(flatfield_data, unit="adu")
return flatfield
def create_flat(flat_list,fil,red_path,mbias=None,log=None):
log.info('Processing files for filter: '+fil)
log.info(str(len(flat_list))+' files found.')
flats = []
flat_scale = []
for flat in flat_list:
log.info('Loading file: '+flat)
raw = CCDData.read(flat,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
mask = make_source_mask(red, nsigma=3, npixels=5)
bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
masked = np.array(red)
masked[mask] = bkg.background[mask]
log.info('Median flat level: '+str(np.median(masked)))
norm = 1/np.median(masked[500:1500,500:1500])
log.info('Flat normalization: '+str(norm))
flat_scale.append(norm)
flats.append(CCDData(masked,meta=red.header,unit=u.electron))
mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True)
log.info('Created master flat for filter: '+fil)
mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True)
log.info('Master flat written to mflat_'+fil+'.fits')
return
def combineFlats(flatlist, dark=None, bias=None):
"""Combine all flat files into a flat master. Subtract dark or bias if provided."""
ccdflatlist = [
ccdproc.CCDData.read(aflat, unit="adu") for aflat in flatlist
]
if dark is not None and bias is None:
flat_sub = [
ccdproc.subtract_dark(aflat,
dark,
exposure_time='exptime',
exposure_unit=u.second)
for aflat in ccdflatlist
]
elif dark is None and bias is not None:
flat_sub = [
ccdproc.subtract_bias(aflat, bias) for aflat in ccdflatlist
]
else:
flat_sub = ccdflatlist
flatComb = ccdproc.Combiner(flat_sub)
flatComb.sigma_clipping(low_thresh=3, high_thresh=3, func=np.ma.median)
flatComb.scaling = lambda arr: 1. / np.ma.average(arr)
flatmaster = flatComb.average_combine()
return flatmaster
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction, dark subtraction, overscan correction, flat correction, and trimming image.')
raw = CCDData.read(sci,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of science image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)#dark_exposure=1*u.second, data_exposure=red.header['EXPTIME']*u.second, scale=True)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
red = ccdproc.flat_correct(red, mflat)
processed_data = ccdproc.ccd_process(red, trim=raw.header['DATASEC'])
log.info('File proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci,processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(red.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (510, 510), filter_size=(9, 9),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits'),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['EXPTIME']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/'),overwrite=True)
processed.append(final)
return processed, masks
def subs_dark(flat_dir, flat_name, chip, day):
#function to substract de dark from the flat
dark_name = 'Dark_10s_' + chip + '.fits'
if os.path.isfile(main_path + day + '/darks/' + dark_name):
if os.path.isfile(main_path + day + '/flats/DS_' + flat_name):
#print "esta la imagen $$$$$$$"
os.system('rm ' + main_path + day + '/flats/DS_' + flat_name)
print "image deleted %s" % (main_path + day + '/flats/DS_' +
flat_name)
#the flat is trimmed
tflat = ccd.trim_image(ccd.CCDData.read(flat_dir, unit="adu"),
fits_section='[5:595,5:2395]')
#print "$$$$$$$$$$ trim done $$$$$$$$$$$$$$$"
#the darks is substracted
dsflat = ccd.subtract_dark(tflat,
ccd.CCDData.read(main_path + day +
'/darks/' + dark_name,
unit="adu"),
exposure_time='EXPTIME',
exposure_unit=u.second,
scale=False)
#print "$$$$$$$$$$ subs dark done $$$$$$$$$$$$$$$"
ccd.fits_ccddata_writer(dsflat,
main_path + day + '/flats/DS_' + flat_name)
del tflat
del dsflat
print "masterdark %s substracted from flat %s for epoch %s" % (
dark_name, flat_name, day)
return (dark_name)
def preprocess(fits_path):
"""Apply basic CCD reduction tasks.
Ask for the path to all the calibration and science files, and perform Bias, Dark and Flat combinations and proper substractions.
Pipeline que aplica una reduccion basica a imagenes de tipo CCD.
Realiza combinacion de Bias, de Darks, y Flats. Luego realiza las restas inherentes.
"""
import os
import sys
from image_collection import ImageFileCollection
#Collect all FITS files in path in fitslist
fitslist = []
for root, dirs, files in os.walk(fits_path, topdown=True):
fitslist.extend([os.path.join(root, name) for name in files if '.fit' in name])
#Dictionary with the image type (value) for each image file name (key)
imagetypes = {img:fits.getval(img, 'IMAGETYP') for img in fitslist}
#Make different lists for different image types
biaslist = []
sciencelist = []
flatlist = []
darklist = []
unknown = []
for k, v in imagetypes.iteritems():
if 'LIGHT' in v.upper() or v.upper()=='OBJECT':
sciencelist.append(k)
elif 'BIAS' in v.upper() or v.upper()=='ZERO':
biaslist.append(k)
elif 'FLAT' in v.upper():
flatlist.append(k)
elif 'DARK' in v.upper():
darklist.append(k)
else:
unknown.append(k)
#Create the flat master
if darklist:
darkmaster = combineDarks(darklist_flat)
flatmaster = combineFlats(flatlist, dark=darkmaster)
elif biaslist:
biasmaster = combineBias(biaslist)
flatmaster = combineFlats(flatlist, bias=biasmaster)
else:
flatmaster = combineFlats(flatlist)
for ascience in sciencelist:
sci_image = ccdproc.CCDData.read(ascience, unit='adu')
sci_darksub = ccdproc.subtract_dark(sci_image, darkmaster, exposure_time='exptime', exposure_unit=u.second)
sci_flatcorrected = ccdproc.flat_correct(sci_darksub, flatmaster)
outpath = os.path.join('/Users/utb/Desktop/reduced/', 'reduced_' + os.path.basename(ascience))
hdulist = sci_flatcorrected.to_hdu()
hdulist.writeto(outpath, clobber=True)
return
def flat_creation(bdf_files, calibrated_path,calibrated_images, output_path, master_images, args):
set(bdf_files.summary['exptime'][bdf_files.summary['imagetyp'] == 'Flat Field'])
if args.cal_bias:
combined_bias = list(master_images.ccds(combined=True, imagetyp='Bias Frame'))[0]
if args.cal_dark:
combined_dark = CCDData.read(master_images.files_filtered(imagetyp='Dark Frame',
combined=True,
include_path=True)[0])
print('list of the flat files')
for a_flat, f_name in bdf_files.ccds(imagetyp='Flat Field', return_fname=True, ccd_kwargs={'unit': 'adu'}):
print(f_name)
if args.cal_bias:
print('BIAS')
a_flat = ccdp.subtract_bias(a_flat, combined_bias)
print('DONE BIAS')
if args.cal_dark:
print('DARK')
a_flat = ccdp.subtract_dark(a_flat, combined_dark, exposure_time='EXPOSURE', exposure_unit=u.s, scale=True)
print('done dark')
a_flat.write(calibrated_path / f_name, overwrite=True)
calibrated_images.refresh()
flats = calibrated_images.summary['imagetyp'] == 'Flat Field'
flat_filters = set(calibrated_images.summary['filter'][flats])
print('flat filters:', flat_filters)
for filtr in sorted(flat_filters):
calibrated_flats = calibrated_images.files_filtered(imagetyp='Flat Field', filter=filtr,
include_path=True)
print(len(calibrated_flats))
combined_flat = ccdp.combine(calibrated_flats,
method='median', scale=inv_median,
sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
signma_clip_dev_func=mad_std,
mem_limit=350e6)
combined_flat.meta['combined'] = True
flat_file_name = f'combined_flat_{filtr}.fit'
combined_flat.write(output_path / flat_file_name, overwrite=True)
calibrated_images.refresh()
master_images.refresh()
return flat_filters
def _subtractDark(self, ccd):
print('Subtracting dark...')
masterDark = self.getDarkImage()
# TODO: add 'dark_exposure' and 'data_exposure'
ccd = ccdproc.subtract_dark(
ccd,
masterDark,
exposure_time='DIT',
exposure_unit=u.second,
add_keyword={'HIERARCH GIULIA DARK SUBTRACTION': True})
return ccd
def action(self, ccd):
from astropy import units as u
select_dict = {'imagetyp': 'dark'}
for keyword in self.match_on:
if keyword in select_dict:
raise ValueError("Keyword {} already has a value set".format(keyword))
select_dict[keyword] = ccd.header[keyword]
master = self._master_image(select_dict)
return ccdproc.subtract_dark(ccd, master,
exposure_time='exposure',
exposure_unit=u.second)
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation.
"""
self.log.info(f"Running {self.__class__.__name__} action")
self.log.info(f" Found master dark file: {self.master_dark.name}")
kd_master_dark = fits_reader(self.master_dark, datatype=VYSOS20)
self.action.args.kd.pixeldata[0] = ccdproc.subtract_dark(
self.action.args.kd.pixeldata[0],
kd_master_dark.pixeldata[0],
dark_exposure=self.action.args.kd.exptime() * u.second,
data_exposure=kd_master_dark.exptime() * u.second)
return self.action.args
def combineFlats(flatlist, dark=None, bias=None):
"""Combine all flat files into a flat master. Subtract dark or bias if provided."""
ccdflatlist = [ccdproc.CCDData.read(aflat, unit="adu") for aflat in flatlist]
if dark is not None and bias is None:
flat_sub = [ccdproc.subtract_dark(aflat, dark, exposure_time='exptime', exposure_unit=u.second) for aflat in ccdflatlist]
elif dark is None and bias is not None:
flat_sub = [ccdproc.subtract_bias(aflat, bias) for aflat in ccdflatlist]
else:
flat_sub = ccdflatlist
flatComb = ccdproc.Combiner(flat_sub)
flatComb.sigma_clipping(low_thresh=3, high_thresh=3, func=np.ma.median)
flatComb.scaling = lambda arr: 1./np.ma.average(arr)
flatmaster = flatComb.average_combine()
return flatmaster
def combineflat(cal_dir="../Data/20181207/cals", mast_dir=".", filt=[], binning=2):
#Generates master flat file from calibration directory
#Get master bias and dark files
if os.path.isfile(mast_dir + "/master/master_bias.FIT") != True:
print "No master bias file"
return False
if os.path.isfile(mast_dir + "/master/master_dark.FIT") != True:
print "No master dark file"
return False
master_bias = CCDData.read(mast_dir + "/master/master_bias.FIT")
master_dark = CCDData.read(mast_dir + "/master/master_dark.FIT")
#Generate image list
imagelist = gen.getimages(cal_dir, filt=filt)
flat_list = []
for img in imagelist:
ccd = CCDData.read(cal_dir + '/' + img, unit=u.adu)
if ccd.header["IMAGETYP"].strip() == "FLAT":
#Rebin images if needed
if ccd.header["XBINNING"] > binning:
print "ERROR: Binning too low"
return False
elif ccd.header["XBINNING"] < binning:
ccd.data = gen.rebin(ccd.data, oldbin=ccd.header["XBINNING"], newbin=binning)
ccd.header["XBINNING"] = binning
ccd.header["YBINNING"] = binning
#Remove bias and dark effects
ccd = ccdproc.subtract_bias(ccd, master_bias)
ccd = ccdproc.subtract_dark(ccd, master_dark, \
dark_exposure=master_dark.header["EXPTIME"]*u.s, \
data_exposure=ccd.header["EXPTIME"]*u.s, scale=True)
ccd.data = ccd.data/np.median(ccd.data)
flat_list.append(ccd)
if len(flat_list) == 0:
print "ERROR: no flat files"
return False
#Generate master file
master_flat = ccdproc.combine(flat_list, method='median', dtype="float32")
master_flat.write(mast_dir + "/master/master_flat_" + ".FIT", \
overwrite=True)
'''
print "Created master_flat file for " + filter_name + " filter"
'''
return True
def reduceframes(img_dir="../Data/20181207/imgs", mast_dir=".", mast_cal_dir=False):
#Removes effects from bias, dark, and flat master files and makes calibrated images
#Get master bias and dark frames - get flat later once have filter_name
if not mast_cal_dir:
mast_cal_dir = mast_dir
if os.path.isfile(mast_cal_dir + "/master/master_bias.FIT") != True:
print "No master bias file"
return False
if os.path.isfile(mast_cal_dir + "/master/master_dark.FIT") != True:
print "No master dark file"
return False
master_bias = CCDData.read(mast_cal_dir + "/master/master_bias.FIT")
master_dark = CCDData.read(mast_cal_dir + "/master/master_dark.FIT")
# if not os.path.exists(mast_dir + "/frames"):
# makedirs(mast_dir + "/frames")
#Reduce images
raw_image_names = gen.getimages(img_dir, filt=[])
for img in raw_image_names:
print img
ccd = CCDData.read(img_dir + '/' + img, unit=u.adu)
#Mask saturated pixels - not doing any more
'''mask_saturated = (ccd.data > 50000)
.data = np.array(ccd.data, dtype=np.float32)
.data[mask_saturated] = np.nan'''
ccd = ccdproc.subtract_bias(ccd, master_bias)
ccd = ccdproc.subtract_dark(ccd, master_dark, \
dark_exposure=master_dark.header["EXPTIME"]*u.s, \
data_exposure=ccd.header["EXPTIME"]*u.s, scale=True)
mean, background, std = sigma_clipped_stats(ccd.data, sigma=3.0, iters=5)
ccd.data = ccd.data - background
ccd.data = ccd.data/ccd.header["EXPTIME"]
ccd.unit = u.adu/u.s
#Add info about background and raw image name to header
ccd.header['SKY'] = background
ccd.header['RAWFILE'] = img
#Save calibrated frame
ccd.write(mast_dir + '/frames/' + img[:-4] + '-calibrated.FIT' , overwrite=True)
print "Created all calibrated frames in " + mast_dir + '/frames'
return True
def dark(self,im,superdark=None) :
""" Superdark subtraction
"""
# only subtract if we are given a superdark!
if superdark is None : return im
# work with lists so that we can handle multi-channel instruments
if type(im) is not list : ims=[im]
else : ims = im
if type(superdark) is not list : superdarks=[superdark]
else : superdarks = superdark
out=[]
for im,dark in zip(ims,superdarks) :
if self.verbose : print(' subtracting superdark...')
out.append(ccdproc.subtract_dark(im,dark,exposure_time='EXPTIME',exposure_unit=units.s))
if len(out) == 1 : return out[0]
else : return out
def action(self, ccd):
from astropy import units as u
select_dict = {'imagetyp': 'dark'}
for keyword in self.match_on:
if keyword in select_dict:
raise ValueError("Keyword {} already has a value set".format(keyword))
select_dict[keyword] = ccd.header[keyword]
if self._scale.scale:
master = self._master_image(select_dict, closest=self.match_on[0])
if not 'subbias' in master.meta:
raise RuntimeError("Bias has not been subtracted from dark, "
"so cannot scale dark")
else:
master = self._master_image(select_dict)
return ccdproc.subtract_dark(ccd, master,
exposure_time='exposure',
exposure_unit=u.second,
scale=self._scale.scale)
def reduceFrames(self):
self.getMedianForObjects()
log.info('Reducing frames started')
print 'Reducing frames started'
for obj in self.objects.filesList:
data = ccdproc.CCDData.read(obj, unit=u.adu)
dataWithDeviation = ccdproc.create_deviation(data,
gain=1.5*u.electron/u.adu,
readnoise=5*u.electron)
reducedObject = ccdproc.gain_correct(dataWithDeviation,
1.5*u.electron/u.adu)
if self.biases.isExists:
reducedObject = ccdproc.subtract_bias(reducedObject,
self.biases.masterCCD)
if self.darks.isExists:
reducedObject = ccdproc.subtract_dark(reducedObject,
self.darks.masterCCD,
exposure_time=cfg.exptime,
exposure_unit=u.second,
scale=True)
if self.flats.isExists:
reducedObject = ccdproc.flat_correct(reducedObject,
self.flats.masterCCD)
self.directory = '../../Reduction/' + directoryName
if not os.path.exists(self.directory):
os.makedirs(self.directory)
reducedObject.write(self.directory + '/' + obj, clobber=True)
os.system('solve-field ' + self.directory + '/' + obj) # + ' --overwrite')
objName, objExtension = os.path.splitext(self.directory + '/' + obj)
if not os.path.exists(objName + '.new'):
log.warning(objName + ' cannot be solved')
else:
newObjectsList.append(objName + '.new')
log.info('Frame ' + objName + ' reduced')
log.info('Reduced ' + str(len(newObjectsList)) + ' frames')
print 'Reduced ' + str(len(newObjectsList)) + ' frames'
self.clean()
def filtering(in1, in2, out_filter):
image = CCDData.read(in1, unit="adu")
#save_as_png(image.data, 800, 1200, '/tmp/a.png')
dark = CCDData.read(in2, unit="adu")
image_exposure = image.header.get('EXPTIME')
dark_exposure = image.header.get('EXPTIME')
dark_sub = ccdproc.subtract_dark(image, dark, dark_exposure=image_exposure*u.second, data_exposure=dark_exposure*u.second)
hdu = fits.open(in1)
hdu[0].data = dark_sub
#hdu.header['telescop'] = 'CREDO'
hdu.writeto(out_filter, overwrite=True)
hdu.close()
return dark_sub
def action(self, ccd):
from astropy import units as u
select_dict = {'imagetyp': 'dark'}
for keyword in self.match_on:
if keyword in select_dict:
raise ValueError(
"Keyword {} already has a value set".format(keyword))
select_dict[keyword] = ccd.header[keyword]
if self._scale.scale:
master = self._master_image(select_dict, closest=self.match_on[0])
if not 'subbias' in master.meta:
raise RuntimeError("Bias has not been subtracted from dark, "
"so cannot scale dark")
else:
master = self._master_image(select_dict)
return ccdproc.subtract_dark(ccd,
master,
exposure_time='exposure',
exposure_unit=u.second,
scale=self._scale.scale)
def make_master_flat(path, flat_dark):
"""
Make master flat.
Args:
path: path to directory of fits files to be combined
flat_dark: CCDData object of dark for flat
Returns:
master flat as a median combined CCDData object
"""
combined_flat = median_combine(path)
exposure_time = combined_flat.header['EXPTIME']
dark_exposure = flat_dark.header['EXPTIME']
master_flat = ccdproc.subtract_dark(combined_flat,
flat_dark,
dark_exposure=dark_exposure * u.s,
data_exposure=exposure_time * u.s,
exposure_unit=u.s,
scale=True)
return master_flat
def create_super_flat(input_images, oPath, super_name, super_bias_name,
super_dark_name):
#NB Should cull low count input frames.
inputs = []
print('SF: ', len(input_images))
super_bias = ccdproc.CCDData.read(super_bias_name, ignore_missing_end=True)
super_dark = ccdproc.CCDData.read(super_dark_name, ignore_missing_end=True)
#super_dark = super_dark.subtract(super_dark.meta['PEDASTAL']*u.adu)
for img in range(len(input_images)):
img_in = ccdproc.CCDData.read(input_images[img],
unit='adu',
ignore_missing_end=True)
bias_corr = ccdproc.subtract_bias(img_in, super_bias)
print('Hello: ', super_dark.meta['EXPTIME'], img_in.meta['EXPTIME'],
type(bias_corr), type(super_dark), img_in.meta)
corr_flat = ccdproc.subtract_dark(bias_corr, super_dark, scale=True, \
dark_exposure=super_dark.meta['EXPTIME']*u.s, \
data_exposure =img_in.meta['EXPTIME']*u.s)
#corr_flat = ccdproc.
inputs.append(corr_flat)
combiner = Combiner(inputs)
super_img = combiner.median_combine()
super_img.meta = inputs[0].meta
super_img.meta['NCOMBINE'] = len(inputs)
s_name = super_name.split('.')
print('s_name_split: ', s_name[0])
tstring = datetime.datetime.now().isoformat().split('.')[0].split(':')
wstring = str(oPath + '\\' + s_name[0] + '_' + \
tstring[0]+tstring[1]+tstring[2] + \
'.fits')
super_img.write(wstring, overwrite=True)
#Turn the above into a circle region.
return
outpath = os.path.join(preprocessedpath, \
'dark_master_' + str(adarkexp) + 's.fits')
saveCCDDataAndLog(outpath, darklists[adarkexp])
outpath = os.path.join(preprocessedpath, 'flat_master.fits')
saveCCDDataAndLog(outpath, flatmaster)
for ascience in sciencelist:
try:
sci_image = ccdproc.CCDData.read(ascience, unit='adu')
if args.usebias:
sci_biassub = ccdproc.subtract_bias(sci_image, biasmaster)
sci_flatcorrected = ccdproc.flat_correct(sci_biassub, flatmaster)
else:
exp_time = fits.getval(ascience, 'exptime')
darkmaster = chooseClosestDark(darklists, exp_time)
sci_darksub = ccdproc.subtract_dark(sci_image, darkmaster, \
exposure_time='exptime', exposure_unit=u.second)
sci_flatcorrected = ccdproc.flat_correct(sci_darksub, flatmaster)
except:
logger.error("Couldn't reduce image %s." % (ascience))
continue
outpath = os.path.join(preprocessedpath, \
'preprocessed_' + os.path.basename(ascience))
#deadpixmaskfilename = "../stackImages/deadpix.fits"
deadpixmaskfilename = None
hdu_img = fits.PrimaryHDU(sci_flatcorrected.data, \
header=fits.getheader(ascience))
if deadpixmaskfilename is not None:
hdu_deadpix_mask = fits.ImageHDU(fits.getdata(deadpixmaskfilename), \
name='DEAD_PIX_MASK')
def make_flatdark_correction():
""" for each light exposure do:
subtract dark from light
divide by dark-subtracted flat
save fits to file
add entry in database for each calibrated file generated"""
# Eventually should be done as described in:
# https://ccdproc.readthedocs.io/en/latest/reduction_toolbox.html
# Or here:
# https://mwcraig.github.io/ccd-as-book
import ccdproc
from astropy.nddata import CCDData
import astroscrappy
import numpy as np
from astropy import units as u
nb = models.NightBundle.get(telescope_night_bundle_id=1)
nb_dir = nb.directory_path
light_list_q = nb.exposures.select(
lambda d: d.exposure_type == models.EXP_TYPE_CODES["LIGHT"])
light_list = [os.path.join(nb_dir, f.filename) for f in light_list_q]
master_dark_q = nb.combinations.select(
lambda d: d.combination_type == models.COMB_TYPE_CODES["DARKM"]).get()
master_dark_path = os.path.join(nb_dir, "products", master_dark_q.filename)
master_flat_q = nb.combinations.select(
lambda d: d.combination_type == models.COMB_TYPE_CODES["FLATM"]).get()
master_flat_path = os.path.join(nb_dir, "products", master_flat_q.filename)
for light_q, light_fname in zip(light_list_q, light_list):
raw_data = CCDData.read(light_fname, unit="adu")
master_dark = CCDData.read(master_dark_path, unit="adu")
master_flat = CCDData.read(master_flat_path, unit="adu")
# cr_cleaned = ccdproc.cosmicray_lacosmic(
# raw_data,
# satlevel=np.inf,
# sepmed=False,
# cleantype="medmask",
# fsmode="median",
# )
# crmask, cr_cleaned = astroscrappy.detect_cosmics(
# raw_data,
# inmask=None,
# satlevel=np.inf,
# sepmed=False,
# cleantype="medmask",
# fsmode="median",
# )
# cr_cleaned.unit = "adu"
cr_cleaned = raw_data
dark_subtracted = ccdproc.subtract_dark(
cr_cleaned,
master_dark,
exposure_time="EXPTIME",
exposure_unit=u.second,
scale=True,
)
reduced_image = ccdproc.flat_correct(dark_subtracted,
master_flat,
min_value=0.9)
reduced_filename = "calib_{}".format(os.path.basename(light_fname))
reduced_path = os.path.join(nb_dir, "products", reduced_filename)
makedirs = os.path.dirname(reduced_path)
if makedirs:
os.makedirs(makedirs, exist_ok=True)
reduced_image.write(reduced_path, overwrite=True)
reduced_comb = models.ExposureCombination(
night_bundle=nb,
filename=reduced_filename,
combination_type=models.COMB_TYPE_CODES["CALIB_LIGHT"],
exposures=light_q,
uses_combinations=[master_flat_q, master_dark_q],
)
def ccdproc(images,
master_bias=None,
master_dark=None,
master_flat=None,
masks=None,
trim=None):
"""
Perform image reduction (bias, dark and flat corretion) on ccd data.
Parameters
----------
images : generator, type of 'ccdproc.CCDData'
Images to be combined.
master_bias : ccdproc.CCDData
Master Bias image.
master_dark : ccdproc.CCDData
Master Dark image.
master_flat : ccdproc.CCDData
Master Flat image.
masks : str, list of str or optional
Area to be masked.
trim : str or optional
Trim section.
Yields
------
'ccdproc.CCDData'
yield the next 'ccdproc.CCDData'.
Examples
--------
>>> from tuglib.io import FitsCollection
>>> from tuglib.reduction import bias_combine, dark_combine, flat_combine
>>> from tuglib.reduction import ccdproc
>>>
>>> path = '/home/user/data/'
>>> masks = ['[:, 1023:1025]', '[:1023, 56:58]']
>>> trim = '[:, 24:2023]'
>>>
>>> images = FitsCollection(location=path, gain=0.57, read_noise=4.11))
>>>
>>> bias_ccds = images.ccds(OBJECT='BIAS', trim=trim, masks=masks)
>>> dark_ccds = images.ccds(OBJECT='DARK', trim=trim, masks=masks)
>>> flat_ccds = images.ccds(OBJECT='FLAT', FILTER='V',
trim=trim, masks=masks)
>>>
>>> sci_ccds = images.ccds(OBJECT='Star', FILTER='V',
trim=trim, masks=masks)
>>>
>>> master_bias = bias_combine(bias_ccds, method='median')
>>> master_dark = dark_combine(dark_ccds, master_bias, method='median')
>>> master_flat = flat_combine(flat_ccds, master_bias, master_dark)
>>>
>>> # Yield a generator which point reduced images.
>>> reduced_ccds = ccdproc(sci_ccds, master_bias, master_dark, master_flat)
"""
if not isinstance(images, types.GeneratorType):
raise TypeError("'images' should be a 'ccdproc.CCDData' object.")
if not isinstance(master_bias, (type(None), CCDData)):
raise TypeError(
"'master_bias' should be 'None' or 'ccdproc.CCDData' object.")
if not isinstance(master_dark, (type(None), CCDData)):
raise TypeError(
"'master_dark' should be 'None' or 'ccdproc.CCDData' object.")
if not isinstance(master_flat, (type(None), CCDData)):
raise TypeError(
"'master_flat' should be 'None' or 'ccdproc.CCDData' object.")
if masks is not None:
if not isinstance(masks, (str, list, type(None))):
raise TypeError(
"'masks' should be 'str', 'list' or 'None' object.")
if trim is not None:
if not isinstance(trim, str):
raise TypeError("'trim' should be a 'str' object.")
mask = None
if master_dark is not None:
dark_exptime = master_dark.meta['EXPTIME'] * u.second
for ccd in images:
if (mask is None) and (masks is not None):
shape = ccd.shape
mask = make_mask(shape, masks)
ccd.mask = mask
ccd = trim_image(ccd, trim)
if master_bias is not None:
ccd = subtract_bias(ccd, master_bias)
if master_dark is not None:
data_exptime = ccd.meta['EXPTIME'] * u.second
ccd = subtract_dark(ccd,
master_dark,
dark_exposure=dark_exptime,
data_exposure=data_exptime,
exposure_time='EXPTIME',
exposure_unit=u.second,
scale=True)
if master_flat is not None:
ccd = flat_correct(ccd, master_flat)
yield ccd
######## FIND DARK WITH CLOSEST EXPOSURE TIME
#################################################
# find dark with closest exposure time
flat_exptime = my_master_flat.header['EXPTIME']
# find dark with closest exposure time
delta_t = abs(flat_exptime - dark_exptimes_set)
closest_dark = dark_exptimes_set[delta_t == min(delta_t)]
# open the appropriate dark
hdu1 = fits.open('dark-combined-'+str(int(closest_dark[0]))+'.fits')
header = hdu1[0].header
gaincorrected_dark = CCDData(hdu1[0].data, unit=u.electron, meta=header)
hdu1.close()
#master_flat_dark = ccdproc.ccd_process(gaincorrected_master_flat, readnoise=rdnoise, dark_frame=gaincorrected_dark, exposure_key='exposure', exposure_unit=u.second, dark_scale=True, gain_corrected=True)
master_flat_dark = ccdproc.subtract_dark(gaincorrected_master_flat, gaincorrected_dark, exposure_time='exposure', exposure_unit=u.second,scale=False)
print('writing out combined flat for filter ',filt)
# write output
master_flat_dark.write('flat-'+filt+'.fits',overwrite=True)
else:
print('skipping flat combine')
#####################################################
###### PROCESS SCIENCE IMAGES
#####################################################
if process_science:
print('\n Processing science frames!!!')
for filt in all_filters:
sci_files = icz.files_filtered(imagetyp = ccdkeyword['light'], filter = filt)
master_flat = 'flat-'+filt+'.fits'
def preprocess(fits_path):
"""Apply basic CCD reduction tasks.
Ask for the path to all the calibration and science files, and perform Bias, Dark and Flat combinations and proper substractions.
Pipeline que aplica una reduccion basica a imagenes de tipo CCD.
Realiza combinacion de Bias, de Darks, y Flats. Luego realiza las restas inherentes.
"""
import os
import sys
from image_collection import ImageFileCollection
#Collect all FITS files in path in fitslist
#fitslist = []
#for root, dirs, files in os.walk(fits_path, topdown=True):
# fitslist.extend([os.path.join(root, name) for name in files if '.fit' in name])
#
##Dictionary with the image type (value) for each image file name (key)
#imagetypes = {img:fits.getval(img, 'IMAGETYP') for img in fitslist}
# now we make different lists for different image types
#biaslist = []
#sciencelist = []
#flatlist = []
#darklist = []
#unknown = []
#for k, v in imagetypes.iteritems():
# if 'LIGHT' in v.upper() or v.upper()=='OBJECT':
# sciencelist.append(k)
# elif 'BIAS' in v.upper() or v.upper()=='ZERO':
# biaslist.append(k)
# elif 'FLAT' in v.upper():
# flatlist.append(k)
# elif 'DARK' in v.upper():
# darklist.append(k)
# else:
# unknown.append(k)
#Create an image file collection storing the following keys
keys = ['imagetyp', 'object', 'filter', 'exptime']
allfits = ImageFileCollection('.', keywords=keys)
#Collect all dark files and make a dark frame for each different exposure time
dark_matches = np.ma.array(['dark' in typ.lower() for typ in allfits.summary['imagetyp']])
darkexp_set = set(allfits.summary['exptime'][dark_matches])
darklists = {}
for anexp in darkexp_set:
my_darks = allfits.summary['file'][(allfits.summary['exptime'] == anexp) & dark_matches]
darklists[anexp] = combineDarks(my_darks)
#Collect all science files
sciencelist = allfits.files_filtered(imagetyp='light')
#Collect all bias files
#bias_matches = ([('zero' in typ.lower() or 'bias' in typ.lower()) for typ in allfits.summary['imagetyp']]
#biaslist = allfits.summary['file'][bias_matches]
#Create the flat master
flatlist = allfits.files_filtered(imagetyp='flat')
exptime = fits.getval(flatlist[0], 'exptime')
darkmaster = chooseClosestDark(darklists, exptime)
flatmaster = combineFlats(flatlist, dark=darkmaster)
for ascience in sciencelist:
sci_image = ccdproc.CCDData.read(ascience, unit='adu')
exp_time = fits.getval(ascience, 'exptime')
darkmaster = chooseClosestDark(darklists, exp_time)
try:
sci_darksub = ccdproc.subtract_dark(sci_image, darkmaster, exposure_time='exptime', exposure_unit=u.second)
sci_flatcorrected = ccdproc.flat_correct(sci_darksub, flatmaster)
outpath = os.path.join('/Users/utb/Desktop/reduced/', 'reduced_' + os.path.basename(ascience))
except:
sci_flatcorrected = sci_image
outpath = os.path.join('/Users/utb/Desktop/reduced/', 'failed_' + os.path.basename(ascience))
hdulist = sci_flatcorrected.to_hdu()
hdulist.writeto(outpath, clobber=True)
return
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
# Find master dark to subtract from master dark
biaslist = self.loadauxname('bias', multi=False)
darklist = self.loadauxname('dark', multi=False)
if (len(biaslist) == 0):
self.log.error('No bias calibration frames found.')
if (len(darklist) == 0):
self.log.error('No bias calibration frames found.')
self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True)
self.dark = ccdproc.CCDData.read(darklist, unit='adu', relax=True)
# Create empy list for filenames of loaded frames
filelist = []
for fin in self.datain:
self.log.debug("Input filename = %s" % fin.filename)
filelist.append(fin.filename)
# Make a dummy dataout
self.dataout = DataFits(config=self.config)
if len(self.datain) == 0:
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file(s) loaded')
self.log.debug('Creating master flat frame...')
# Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias
if (len(filelist) == 1):
self.flat = ccdproc.CCDData.read(filelist[0],
unit='adu',
relax=True)
self.flat = ccdproc.subtract_bias(self.flat,
self.bias,
add_keyword=False)
self.flat = ccdproc.subtract_dark(self.flat,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second,
add_keyword=False)
else:
#bias and dark correct frames
flatlist = []
for i in filelist:
flat = ccdproc.CCDData.read(i, unit='adu', relax=True)
flatsubbias = ccdproc.subtract_bias(flat,
self.bias,
add_keyword=False)
flatsubbiasdark = ccdproc.subtract_dark(
flatsubbias,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second,
add_keyword=False)
flatlist.append(flatsubbiasdark)
#scale the flat component frames to have the same mean value, 10000.0
scaling_func = lambda arr: 10000.0 / numpy.ma.median(arr)
#combine them
self.flat = ccdproc.combine(flatlist,
method=self.getarg('combinemethod'),
scale=scaling_func,
unit='adu',
add_keyword=False)
# set output header, put image into output
self.dataout.header = self.datain[0].header
self.dataout.imageset(self.flat)
# rename output filename
outputfolder = self.getarg('outputfolder')
if outputfolder != '':
outputfolder = os.path.expandvars(outputfolder)
self.dataout.filename = os.path.join(outputfolder,
os.path.split(filelist[0])[1])
else:
self.dataout.filename = filelist[0]
# Add history
self.dataout.setheadval('HISTORY',
'MasterFlat: %d files used' % len(filelist))
hdulist = flat.to_hdu()
#add bias correction to header
header=hdulist[0].header
header['BIASCORR'] = flat_bias
hdulist.writeto(flat_corrected, clobber=True)
flat_bias = bias_master
else:
logme('Flat frame (%s) is *already* bias corrected (%s).'%(im[i],flat_bias))
#dark correct, if necessary
if(not flat_is_dark_corrected):
#logme('Subtracting master dark frame from flat frame...')
flat = ccdproc.CCDData.read(flat_corrected, unit='adu', relax=True)
##trim it, if necessary
#if(len(trim_range) > 0):
# flat = ccdproc.trim_image(flat, trim_range);
flat = ccdproc.subtract_dark(flat, dark, scale=True, exposure_time=exposure_label, exposure_unit=u.second, add_keyword=False)
hdulist = flat.to_hdu()
#add bias correction to header
header=hdulist[0].header
header['DARKCORR'] = dark_bias
hdulist.writeto(flat_corrected, clobber=True)
flat_dark = dark_master
else:
logme('Flat frame (%s) is *already* dark corrected (%s).'%(im[i],flat_dark) )
if(flats):
flats += ','+flat_corrected
else:
flats = flat_corrected
#calc average exposure time for potential dark correction
if(header.get('EXPTIME') != None):
#print header.get('EXPTIME')
def preprocess(fits_path):
"""Apply basic CCD reduction tasks.
Ask for the path to all the calibration and science files, and perform Bias, Dark and Flat combinations and proper substractions.
Pipeline que aplica una reduccion basica a imagenes de tipo CCD.
Realiza combinacion de Bias, de Darks, y Flats. Luego realiza las restas inherentes.
"""
import os
import sys
from image_collection import ImageFileCollection
#Collect all FITS files in path in fitslist
fitslist = []
for root, dirs, files in os.walk(fits_path, topdown=True):
fitslist.extend(
[os.path.join(root, name) for name in files if '.fit' in name])
#Dictionary with the image type (value) for each image file name (key)
imagetypes = {img: fits.getval(img, 'IMAGETYP') for img in fitslist}
#Make different lists for different image types
biaslist = []
sciencelist = []
flatlist = []
darklist = []
unknown = []
for k, v in imagetypes.iteritems():
if 'LIGHT' in v.upper() or v.upper() == 'OBJECT':
sciencelist.append(k)
elif 'BIAS' in v.upper() or v.upper() == 'ZERO':
biaslist.append(k)
elif 'FLAT' in v.upper():
flatlist.append(k)
elif 'DARK' in v.upper():
darklist.append(k)
else:
unknown.append(k)
#Create the flat master
if darklist:
darkmaster = combineDarks(darklist_flat)
flatmaster = combineFlats(flatlist, dark=darkmaster)
elif biaslist:
biasmaster = combineBias(biaslist)
flatmaster = combineFlats(flatlist, bias=biasmaster)
else:
flatmaster = combineFlats(flatlist)
for ascience in sciencelist:
sci_image = ccdproc.CCDData.read(ascience, unit='adu')
sci_darksub = ccdproc.subtract_dark(sci_image,
darkmaster,
exposure_time='exptime',
exposure_unit=u.second)
sci_flatcorrected = ccdproc.flat_correct(sci_darksub, flatmaster)
outpath = os.path.join('/Users/utb/Desktop/reduced/',
'reduced_' + os.path.basename(ascience))
hdulist = sci_flatcorrected.to_hdu()
hdulist.writeto(outpath, clobber=True)
return
if args.bias:
mbias = CCDData.read(args.bias, unit=u.adu)
else:
mbias = None
if args.flat:
raise Exception('Flat fielding is not currently implemented')
if args.dark:
mdark = CCDData.read(args.dark, unit=u.adu)
else:
mdark = None
for infile in infiles:
ccd = CCDData.read(infile, unit=u.adu)
if mdark:
ccd = ccdproc.subtract_dark(ccd, mdark, exposure_time='EXPTIME', exposure_unit=u.second)
ccd = ccdproc.ccd_process(ccd, oscan='[1121:1181, 1:330]', oscan_median=True,
trim='[17:1116,1:330]', master_bias=mbias,
error=True, gain=1.0 * u.electron/u.adu,
readnoise=5.0 * u.electron)
if args.cray:
ccd = ccdproc.cosmicray_lacosmic(ccd, sigclip=4.5, sigfrac=0.3,
objlim=5.0, gain=1.0, readnoise=6.5,
satlevel=4096.0, pssl=0.0, niter=4,
sepmed=True, cleantype='meanmask', fsmode='median',
psfmodel='gauss', psffwhm=2.5, psfsize=7,
psfk=None, psfbeta=4.765, verbose=False)
ccd.write('p'+os.path.basename(infile), clobber=True)
def process_fits(fitspath,
*,
obstype=None,
object=None,
exposure_times=None,
percentile=None,
percentile_min=None,
percentile_max=None,
window=None,
darks=None,
cosmic_ray=False,
cosmic_ray_kwargs={},
gain=None,
readnoise=None,
normalise=False,
normalise_func=np.ma.average,
combine_type=None,
sigma_clip=False,
low_thresh=3,
high_thresh=3):
"""Combine all FITS images of a given type and exposure time from a given directory.
Parameters
----------
fitspath: str
Path to the FITS images to process. Can be a path to a single file, or a path to a
directory. If the latter the directory will be searched for FITS files and checked
against criteria from obstype, object, exposure_times critera.
obstype: str, optional
Observation type, an 'OBSTYPE' FITS header value e.g. 'DARK', 'OBJ'. If given only files
with matching OBSTYPE will be processed.
object: str, optional
Object name, i.e. 'OBJECT' FITS header value. If given only files with matching OBJECT
will be processed.
exposure_times: float or sequence, optional
Exposure time(s), i.e 'TOTALEXP' FITS header value(s). If given only files with matching
TOTALEXP will be processed.
percentile: float, optional
If given will only images whose percentile value fall between percentile_min and
percentile_max will be processed, e.g. set to 50.0 to select images by median value,
set to 99.5 to select images by their 99.5th percentile value.
percentile_min: float, optional
Minimum percentile value.
percentile_max: float, optional
Maximum percentil value.
window: (int, int, int, int), optional
If given will trim images to the window defined as (x0, y0, x1, y1), where (x0, y0)
and (x1, y1) are the coordinates of the bottom left and top right corners.
darks: str or sequence, optional
Filename(s) of dark frame(s) to subtract from the image(s). If given a dark frame with
matching TOTALEXP will be subtracted from each image during processing.
cosmic_ray: bool, optional
Whether to perform single image cosmic ray removal, using the lacosmic algorithm,
default False. Requires both gain and readnoise to be set.
cosmic_ray_kwargs: dict, optional
Additional keyword arguments to pass to the ccdproc.cosmicray_lacosmic function.
gain: str or astropy.units.Quantity, optional
Either a string indicating the FITS keyword corresponding to the (inverse gain), or
a Quantity containing the gain value to use. If both gain and read noise are given
an uncertainty frame will be created.
readnoise: str or astropy.units.Quantity, optional
Either a string indicating the FITS keyword corresponding to read noise, or a Quantity
containing the read noise value to use. If both read noise and gain are given then an
uncertainty frame will be created.
normalise: bool, optional
If True each image will be normalised. Default False.
normalise_func: callable, optional
Function to use for normalisation. Each image will be divided by normalise_func(image).
Default np.ma.average.
combine_type: str, optional
Type of image combination to use, 'MEAN' or 'MEDIAN'. If None the individual
images will be processed but not combined and the return value will be a list of
CCDData objects. Default None.
sigma_clip: bool, optional
If True will perform sigma clipping on the image stack before combining, default=False.
low_thresh: float, optional
Lower threshold to use for sigma clipping, in standard deviations. Default is 3.0.
high_thresh: float, optional
Upper threshold to use for sigma clipping, in standard deviations. Default is 3.0.
Returns
-------
master: ccdproc.CCDData
Combined image.
"""
if exposure_times:
try:
# Should work for any sequence or iterable type
exposure_times = set(exposure_times)
except TypeError:
# Not a sequence or iterable, try using as a single value.
exposure_times = {
float(exposure_times),
}
if darks:
try:
dark_filenames = set(darks)
except TypeError:
dark_filenames = {
darks,
}
dark_dict = {}
for filename in dark_filenames:
try:
dark_data = CCDData.read(filename)
except ValueError:
# Might be no units in FITS header. Assume ADU.
dark_data = CCDData.read(filename, unit='adu')
dark_dict[dark_data.header['totalexp']] = dark_data
if combine_type and combine_type not in ('MEAN', 'MEDIAN'):
raise ValueError(
"combine_type must be 'MEAN' or 'MEDIAN', got '{}''".format(
combine_type))
fitspath = Path(fitspath)
if fitspath.is_file():
# FITS path points to a single file, turn into a list.
filenames = [
fitspath,
]
elif fitspath.is_dir():
# FITS path is a directory. Find FITS file and collect values of selected FITS headers
ifc = ImageFileCollection(fitspath, keywords='*')
if len(ifc.files) == 0:
raise RuntimeError("No FITS files found in {}".format(fitspath))
# Filter by observation type.
if obstype:
try:
ifc = ifc.filter(obstype=obstype)
except FileNotFoundError:
raise RuntimeError(
"No FITS files with OBSTYPE={}.".format(obstype))
# Filter by object name.
if object:
try:
ifc = ifc.filter(object=object)
except FileNotFoundError:
raise RuntimeError(
"No FITS files with OBJECT={}.".format(object))
filenames = [
Path(ifc.location).joinpath(filename) for filename in ifc.files
]
else:
raise ValueError(
"fitspath '{}' is not an accessible file or directory.".format(
fitspath))
# Load image(s) and process them.
images = []
for filename in filenames:
try:
ccddata = CCDData.read(filename)
except ValueError:
# Might be no units in FITS header. Assume ADU.
ccddata = CCDData.read(filename, unit='adu')
# Filtering by exposure times here because it's hard filter ImageFileCollection
# with an indeterminate number of possible values.
if not exposure_times or ccddata.header['totalexp'] in exposure_times:
if window:
ccddata = ccdproc.trim_image(ccddata[window[1]:window[3] + 1,
window[0]:window[2] + 1])
if percentile:
# Check percentile value is within specified range, otherwise skip to next image.
percentile_value = np.percentile(ccddata.data, percentile)
if percentile_value < percentile_min or percentile_value > percentile_max:
continue
if darks:
try:
ccddata = ccdproc.subtract_dark(
ccddata,
dark_dict[ccddata.header['totalexp']],
exposure_time='totalexp',
exposure_unit=u.second)
except KeyError:
raise RuntimeError(
"No dark with matching totalexp for {}.".format(
filename))
if gain:
if isinstance(gain, str):
egain = ccddata.header[gain]
egain = egain * u.electron / u.adu
elif isinstance(gain, u.Quantity):
try:
egain = gain.to(u.electron / u.adu)
except u.UnitsError:
egain = (1 / gain).to(u.electron / u.adu)
else:
raise ValueError(
f"gain must be a string or Quantity, got {gain}.")
if readnoise:
if isinstance(readnoise, str):
rn = ccddata.header[readnoise]
rn = rn * u.electron
elif isinstance(readnoise, u.Quantity):
try:
rn = readnoise.to(u.electron / u.pixel)
except u.UnitsError:
rn = (readnoise * u.pixel).to(u.electron)
else:
raise ValueError(
f"readnoise must be a string or Quantity, got {readnoise}."
)
if gain and readnoise:
ccddata = ccdproc.create_deviation(ccddata,
gain=egain,
readnoise=rn,
disregard_nan=True)
if gain:
ccddata = ccdproc.gain_correct(ccddata, gain=egain)
if cosmic_ray:
if not gain and readnoise:
raise ValueError(
"Cosmic ray removal required both gain & readnoise.")
ccddata = ccdproc.cosmicray_lacosmic(
ccddata,
gain=1.0, # ccddata already gain corrected
readnoise=rn,
**cosmic_ray_kwargs)
if normalise:
ccddata = ccddata.divide(normalise_func(ccddata.data))
images.append(ccddata)
n_images = len(images)
if n_images == 0:
msg = "No FITS files match exposure time criteria"
raise RuntimeError(msg)
if n_images == 1 and combine_type:
warn(
"Combine type '{}' selected but only 1 matching image, skipping image combination.'"
)
combine_type = None
if combine_type:
combiner = Combiner(images)
# Sigma clip data
if sigma_clip:
if combine_type == 'MEAN':
central_func = np.ma.average
else:
# If not MEAN has to be MEDIAN, checked earlier that it was one or the other.
central_func = np.ma.median
combiner.sigma_clipping(low_thresh=low_thresh,
high_thresh=high_thresh,
func=central_func)
# Stack images.
if combine_type == 'MEAN':
master = combiner.average_combine()
else:
master = combiner.median_combine()
# Populate header of combined image with metadata about the processing.
master.header['fitspath'] = str(fitspath)
if obstype:
master.header['obstype'] = obstype
if exposure_times:
if len(exposure_times) == 1:
master.header['totalexp'] = float(exposure_times.pop())
else:
master.header['totalexp'] = tuple(exposure_times)
master.header['nimages'] = n_images
master.header['combtype'] = combine_type
master.header['sigclip'] = sigma_clip
if sigma_clip:
master.header['lowclip'] = low_thresh
master.header['highclip'] = high_thresh
else:
# No image combination, just processing indivudal image(s)
if n_images == 1:
master = images[0]
else:
master = images
return master
def bdf_process(ccd,
output=None,
mbiaspath=None,
mdarkpath=None,
mflatpath=None,
fits_section=None,
calc_err=False,
unit='adu',
gain=None,
rdnoise=None,
gain_key="GAIN",
rdnoise_key="RDNOISE",
gain_unit=u.electron / u.adu,
rdnoise_unit=u.electron,
dark_exposure=None,
data_exposure=None,
exposure_key="EXPTIME",
exposure_unit=u.s,
dark_scale=False,
min_value=None,
norm_value=None,
verbose=True,
output_verify='fix',
overwrite=True,
dtype="float32"):
''' Do bias, dark and flat process.
Parameters
----------
ccd: array-like
The ccd to be processed.
output: path-like
Saving directory
'''
proc = CCDData(ccd)
hdr_new = proc.header
if mbiaspath is None:
do_bias = False
# mbias = CCDData(np.zeros_like(ccd), unit=unit)
else:
do_bias = True
mbias = CCDData.read(mbiaspath, unit=unit)
hdr_new.add_history(f"Bias subtracted using {mbiaspath}")
if mdarkpath is None:
do_dark = False
mdark = None
else:
do_dark = True
mdark = CCDData.read(mdarkpath, unit=unit)
hdr_new.add_history(f"Dark subtracted using {mdarkpath}")
if dark_scale:
hdr_new.add_history(
f"Dark scaling {dark_scale} using {exposure_key}")
if mflatpath is None:
do_flat = False
# mflat = CCDData(np.ones_like(ccd), unit=unit)
else:
do_flat = True
mflat = CCDData.read(mflatpath)
hdr_new.add_history(f"Flat corrected using {mflatpath}")
if fits_section is not None:
proc = trim_image(proc, fits_section)
mbias = trim_image(mbias, fits_section)
mdark = trim_image(mdark, fits_section)
mflat = trim_image(mflat, fits_section)
hdr_new.add_history(f"Trim by FITS section {fits_section}")
if do_bias:
proc = subtract_bias(proc, mbias)
if do_dark:
proc = subtract_dark(proc,
mdark,
dark_exposure=dark_exposure,
data_exposure=data_exposure,
exposure_time=exposure_key,
exposure_unit=exposure_unit,
scale=dark_scale)
# if calc_err and verbose:
# if mdark.uncertainty is not None:
# print("Dark has uncertainty frame: Propagate in arithmetics.")
# else:
# print("Dark does NOT have uncertainty frame")
if calc_err:
if gain is None:
gain = fu.get_from_header(hdr_new,
gain_key,
unit=gain_unit,
verbose=verbose,
default=1.).value
if rdnoise is None:
rdnoise = fu.get_from_header(hdr_new,
rdnoise_key,
unit=rdnoise_unit,
verbose=verbose,
default=0.).value
err = fu.make_errmap(proc, gain_epadu=gain, subtracted_dark=mdark)
proc.uncertainty = StdDevUncertainty(err)
errstr = (f"Error calculated using gain = {gain:.3f} [e/ADU] and " +
f"rdnoise = {rdnoise:.3f} [e].")
hdr_new.add_history(errstr)
if do_flat:
if calc_err:
if (mflat.uncertainty is not None) and verbose:
print("Flat has uncertainty frame: Propagate in arithmetics.")
hdr_new.add_history(
"Flat had uncertainty and is also propagated.")
proc = flat_correct(proc,
mflat,
min_value=min_value,
norm_value=norm_value)
proc = fu.CCDData_astype(proc, dtype=dtype)
proc.header = hdr_new
if output is not None:
proc.write(output, output_verify=output_verify, overwrite=overwrite)
return proc
warning("Filter mismatch: " + i + " (" +
headers[args.fits_header_filter].strip() +
" filter)")
light_one = CCDData.read(i, unit='adu')
lights.append(light_one)
if biascal == True:
log("Calibrating bias...")
lights[lightcnt] = subtract_bias(lights[lightcnt], bias)
if darkcal == True:
log("Calibrating dark...")
lights[lightcnt] = subtract_dark(
lights[lightcnt],
dark,
dark_exposure=darkexp * units.s,
data_exposure=exptimenow * units.s,
scale=True)
if flatcal == True:
log("Calibrating flat...")
if lightfilter != flatfilter:
warning("Filter mismatch with flat: " + i + " (" +
headers[args.fits_header_filter].strip() +
" filter)")
lights[lightcnt] = flat_correct(lights[lightcnt], flat)
#regularize exposure time
lights[lightcnt] = lights[lightcnt].multiply(exptime /
exptimenow)
lights[lightcnt] = lights[lightcnt].add(args.pedestal *
units.astrophys.adu)
def run(self):
""" Runs the calibrating algorithm. The calibrated data is
returned in self.dataout
"""
### Preparation
# Load bias files if necessary
if not self.biasloaded or self.getarg('reload'):
self.loadbias()
# Else: check data for correct instrument configuration - currently not in use(need improvement)
else:
for keyind in range(len(self.biasfitkeys)):
if self.biaskeyvalues[keyind] != self.datain.getheadval(
self.biasfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.biasfitkeys[keyind])
# Load dark files if necessary
if not self.darkloaded or self.getarg('reload'):
self.loaddark()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.darkfitkeys)):
if self.darkkeyvalues[keyind] != self.datain.getheadval(
self.darkfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.darkfitkeys[keyind])
# Load flat files if necessary
if not self.flatloaded or self.getarg('reload'):
self.loadflat()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.flatfitkeys)):
if self.flatkeyvalues[keyind] != self.datain.getheadval(
self.flatfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.flatfitkeys[keyind])
#convert self.datain to CCD Data object
image = ccdproc.CCDData(self.datain.image, unit='adu')
image.header = self.datain.header
#subtract bias from image
image = ccdproc.subtract_bias(image, self.bias, add_keyword=False)
#subtract dark from image
image = ccdproc.subtract_dark(image,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second,
add_keyword=False)
#apply flat correction to image
image = ccdproc.flat_correct(image, self.flat, add_keyword=False)
# copy calibrated image into self.dataout - make sure self.dataout is a pipedata object
self.dataout = DataFits(config=self.datain.config)
self.dataout.image = image.data
self.dataout.header = image.header
self.dataout.filename = self.datain.filename
### Finish - cleanup
# Update DATATYPE
self.dataout.setheadval('DATATYPE', 'IMAGE')
# Add bias, dark files to History
self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname)
self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname)
self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname)
def flat_combine(images,
master_bias=None,
master_dark=None,
method='median',
output=None,
masks=None,
trim=None):
"""
Flat Combine.
Parameters
----------
images : generator or list of 'ccdproc.CCDData'
Images to be combined.
master_bias : ccdproc.CCDData
Master Bias image.
master_dark : ccdproc.CCDData
Master Dark image.
method : str
Method to combine images:
- average : To combine by calculating the average.
- median : To combine by calculating the median.
- sum : To combine by calculating the sum.
Default is 'median'.
output : None or str
If it is None, function returns just 'ccdproc.CCDData'.
If it is 'str', function returns 'ccdproc.CCDData' and creates file.
masks : str, list of str or optional
Area to be masked.
trim : str or optional
Trim section.
Returns
-------
master_ccd : ccdproc.CCDData
Combined Images.
Examples
--------
>>> from tuglib.io import FitsCollection
>>> from tuglib.reduction import bias_combine, dark_combine, flat_combine
>>>
>>> path = '/home/user/data/'
>>> masks = ['[:, 1023:1025]', '[:1023, 56:58]']
>>> trim = '[:, 24:2023]'
>>>
>>> images = FitsCollection(location=path, gain=0.57, read_noise=4.11))
>>>
>>> bias_ccds = images.ccds(OBJECT='BIAS', trim=trim, masks=masks)
>>> dark_ccds = images.ccds(OBJECT='DARK', trim=trim, masks=masks)
>>> flat_ccds = images.ccds(OBJECT='FLAT', FILTER='V',
trim=trim, masks=masks)
>>>
>>> master_bias = bias_combine(bias_ccds, method='median')
>>> master_dark = dark_combine(dark_ccds, master_bias, method='median')
>>> master_flat = flat_combine(flat_ccds, master_bias, master_dark)
"""
if not isinstance(images, (list, types.GeneratorType)):
raise TypeError("'images' should be a 'ccdproc.CCDData' object.")
if not isinstance(master_bias, (type(None), CCDData)):
raise TypeError(
"'master_bias' should be 'None' or 'ccdproc.CCDData' object.")
if not isinstance(master_dark, (type(None), CCDData)):
raise TypeError(
"'master_dark' should be 'None' or 'ccdproc.CCDData' object.")
if method not in ('average', 'median', 'sum'):
raise ValueError("'method' should be 'average', 'median' or 'sum'.")
if masks is not None:
if not isinstance(masks, (str, list, type(None))):
raise TypeError(
"'masks' should be 'str', 'list' or 'None' object.")
if trim is not None:
if not isinstance(trim, str):
raise TypeError("'trim' should be a 'str' object.")
if not isinstance(output, (type(None), str)):
raise TypeError("'output' should be 'None' or 'str' objects.")
if isinstance(images, types.GeneratorType):
try:
ccds = list(images)
except IndexError:
return None
else:
ccds = images
mask = None
if masks is not None:
shape = ccds[0].shape
mask = make_mask(shape, masks)
for i, ccd in enumerate(ccds):
if mask is not None:
ccd.mask = mask
ccd = trim_image(ccd, trim)
ccds[i] = ccd
if master_dark is not None:
dark_exptime = master_dark.meta['EXPTIME'] * u.second
for i, ccd in enumerate(ccds):
if master_bias is not None:
ccd = subtract_bias(ccd, master_bias)
if master_dark is not None:
data_exptime = ccd.meta['EXPTIME'] * u.second
ccd = subtract_dark(ccd,
master_dark,
dark_exposure=dark_exptime,
data_exposure=data_exptime,
exposure_time='EXPTIME',
exposure_unit=u.second,
scale=True)
ccds[i] = ccd
master_ccd = combine(ccds, method=method, output_verify='silentfix+ignore')
if output is not None:
master_ccd.write(output,
overwrite=True,
output_verify='silentfix+ignore')
return master_ccd
mdark_fdic[exptlist[k]] = CCDData.read(prepath/dark_fdic[exptlist[k]]
,unit = u.adu)
#%%
if os.path.exists(prepath/flat_fname):
mflat = CCDData.read(prepath/flat_fname, unit = u.adu)
print("이전에 만든 bias 사용")
else:
images=[]
for i in range(len(flattab)):
cc = CCDData.read(flattab[i]['FILE'], unit = u.adu)
cc = subtract_bias(cc,mbias)
cc = subtract_dark(cc,mdark_fdic[int(flattab[i]['EXPTIME'])]
,flattab[i]['EXPTIME']*u.second,flattab[i]['EXPTIME']*u.second)
# 각 Image의 평균으로 Normalize
hdr = cc.header
cc = CCDData(cc/np.mean(cc),unit = u.adu)
cc.header = hdr
images.append(cc)
mflat = combine(images,method = 'median')
mflat.header = cc.header
mflat.header.add_history(f"{len(flattab)} image(s) median combined flat frames with b&d sub")
mflat = yfu.CCDData_astype(mflat,dtype = 'float32')
mflat.write(prepath/flat_fname,overwrite=True)
fig, axs = plt.subplots(1,1)
im = yfu.zimshow(axs,mflat)
axs.set_xlabel("Flat", fontsize=12)
def calibrate_images(
x_d,
x_f,
x_s,
it_s='object',
x_b='',
):
"""
Parameters
----------
----------
x_b : str
Path of the bias files.
By default bias is not provided
x_d : str
Path of the dark files
x_f : str
Path of flat files
x_s : str
Path of Science files
it_s : str
Imagetyp of fits science file
Default is set to object
-----------
"""
path_b = Path(x_b)
path_d = Path(x_d)
path_f = Path(x_f)
path_s = Path(x_s)
#-----Bias
if x_b == '' and path_b == Path(''):
print(
'Be aware: You did not provide the Bias files; the process will still continue though.'
)
files_b = None
elif not path_b.is_dir():
raise RuntimeError(
'The path you provided for the Bias files does not exist.')
else:
files_b = ccdp.ImageFileCollection(path_b)
#-----Dark
if x_d == '' or not path_d.is_dir():
raise RuntimeError(
'You must provide Dark files for processing.\n Or the path you provided does not exist.'
)
else:
files_d = ccdp.ImageFileCollection(path_d)
#-----Flat
if x_f == '' or not path_f.is_dir():
raise RuntimeError(
'You must provide Flatfield files for processing.\n Or the path you provided does not exist.'
)
else:
files_f = ccdp.ImageFileCollection(path_f)
#-----Science
if x_s == '' or not path_s.is_dir():
raise RuntimeError(
'You must provide Science images for processing.\n Or the path you provided does not exist.'
)
else:
files_s = ccdp.ImageFileCollection(path_s)
#-----------------------------------
#
#--------Calibrating Images---------
#
#-----------------------------------
if files_b is not None:
#-------------------------------
#------Creating Master-bias-----
#-------------------------------
cali_bias_path = Path(path_b / 'cali_bias')
cali_bias_path.mkdir(exist_ok=True)
files_b_cali = files_b.files_filtered(imagetyp='bias',
include_path=True)
combined_bias = ccdp.combine(files_b_cali,\
method='average',\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_bias.meta['combined'] = True
combined_bias.write(cali_bias_path / 'master_bias.fits')
# Reading master bias
master_bias = CCDData.read(cali_bias_path / 'master_bias.fits')
else:
master_bias = None
#-------------------------------
#-------Calibrating Darks-------
#-------------------------------
cali_dark_path = Path(path_d / 'cali_dark')
cali_dark_path.mkdir(exist_ok=True)
files_d_cali = files_d.files_filtered(imagetyp='DARK', include_path=True)
for ccd, file_name in files_d.ccds(imagetyp='DARK',
return_fname=True,
ccd_kwargs={'unit': 'adu'}):
if master_bias is not None:
# Subtract bias
ccd = ccdp.subtract_bias(ccd, master_bias)
else:
ccd = ccd
# Save the result
ccd.write(cali_dark_path / file_name)
#--------------------------------
#------Creating Master-Dark------
#--------------------------------
red_dark = ccdp.ImageFileCollection(cali_dark_path)
# Calculating exposure times of DARK images
dark_times = set(
red_dark.summary['exptime'][red_dark.summary['imagetyp'] == 'DARK'])
for exposure in sorted(dark_times):
cali_darks = red_dark.files_filtered(imagetyp = 'dark',\
exptime = exposure,\
include_path = True)
combined_dark = ccdp.combine(cali_darks,\
method='average',\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_dark.meta['combined'] = True
com_dark_name = 'combined_dark_{:6.3f}.fits'.format(exposure)
combined_dark.write(cali_dark_path / com_dark_name)
# Reading master dark of various exposure times
red_dark = ccdp.ImageFileCollection(cali_dark_path)
combined_darks = {
ccd.header['exptime']: ccd
for ccd in red_dark.ccds(imagetyp='DARK', combined=True)
}
#--------------------------------
#-------Calibrating Flats--------
#--------------------------------
cali_flat_path = Path(path_f / 'cali_flat')
cali_flat_path.mkdir(exist_ok=True)
files_f_cali = files_f.files_filtered(imagetyp='FLAT', include_path=True)
for ccd, file_name in files_f.ccds(imagetyp='FLAT',
ccd_kwargs={'unit': 'adu'},
return_fname=True):
# Subtract bias
if master_bias is not None:
ccd = ccdp.subtract_bias(ccd, master_bias)
else:
ccd = ccd
closest_dark = utl.find_nearest_dark_exposure(ccd, dark_times)
if closest_dark is None:
closest_dark1 = utl.find_nearest_dark_exposure(ccd,
dark_times,
tolerance=100)
# Subtract scaled Dark
ccd = ccdp.subtract_dark(ccd, combined_darks[closest_dark1],\
exposure_time = 'exptime',\
exposure_unit = u.second,\
scale = True)
ccd.write(cali_flat_path / ('flat-' + file_name))
else:
closest_dark2 = utl.find_nearest_dark_exposure(ccd, dark_times)
# Subtracting Darks
ccd = ccdp.subtract_dark(ccd,
combined_darks[closest_dark2],
exposure_time='exptime',
exposure_unit=u.second)
ccd.write(cali_flat_path / ('flat-' + file_name))
#--------------------------------
#-----Creating Master-Flat-------
#--------------------------------
red_flats = ccdp.ImageFileCollection(cali_flat_path)
cali_flats = red_flats.files_filtered(imagetyp='FLAT', include_path=True)
combined_flat = ccdp.combine(cali_flats,\
method='average',\
scale = utl.inverse_median,\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_flat.meta['combined'] = True
combined_flat.write(cali_flat_path / 'master_flat.fits')
# Reading master flat
red_flats = ccdp.ImageFileCollection(cali_flat_path)
combined_flat = CCDData.read(cali_flat_path / 'master_flat.fits')
#--------------------------------
#---Calibrating Science Images---
#--------------------------------
# Creating a list of spectrum images
files_spec = files_s.summary['file', 'view_pos']
files_spec_list = np.array([])
for i in range(len(files_spec)):
xxx = files_spec['view_pos'][i]
if xxx[0:4] == 'open':
files_spec_list = np.hstack(
(files_spec_list, files_spec['file'][i]))
# Sky subtracting images
cali_science_path1 = Path(path_s / 'Sky_subtracted_science')
cali_science_path1.mkdir(exist_ok=True)
j = 0
for i in range(int(len(files_spec_list) / 2)):
ccd1 = CCDData.read(x_s + files_spec_list[j], unit='adu')
ccd2 = CCDData.read(x_s + files_spec_list[j + 1], unit='adu')
sky_sub1 = ccd1.data - ccd2.data
ss1 = CCDData(sky_sub1, unit='adu')
ss1.header = ccd1.header
ss1.meta['sky_sub'] = True
name1 = 'sky_sub_' + files_spec_list[j]
ss1.write(cali_science_path1 / name1)
sky_sub2 = ccd2.data - ccd1.data
ss2 = CCDData(sky_sub2, unit='adu')
ss2.header = ccd2.header
ss2.meta['sky_sub'] = True
name2 = 'sky_sub_' + files_spec_list[j + 1]
ss2.write(cali_science_path1 / name2)
j = j + 2
files_s1 = ccdp.ImageFileCollection(cali_science_path1)
final_calibrated = Path(path_s / 'Final_calibrated_science')
final_calibrated.mkdir(exist_ok=True)
# Correcting for flat
for ccd, file_name in files_s1.ccds(imagetyp=it_s,
ccd_kwargs={'unit': 'adu'},
return_fname=True):
# Subtract scaled Dark
ccd = ccdp.flat_correct(ccd, combined_flat) #['FLAT'])
ccd.write(final_calibrated / file_name)
# convert data to CCDData format and save header
ccd = CCDData(hdu1[0].data, unit=u.adu)
header = hdu1[0].header
if args.zap:
zccd = ccdproc.cosmicray_lacosmic(ccd, gain = float(args.gain), readnoise = float(args.rdnoise))
outfile = 'z'+outfile
# subtract dark
if args.dark:
if args.zap:
infile=zccd
else:
infile = ccd
dccd = ccdproc.subtract_dark(infile, dark)
outfile = 'd'+outfile
# flatten
if args.flat:
# flatten
if args.dark:
infile = dccd
else:
if args.zap:
infile = zccd
else:
infile = ccd
fccd = ccdproc.flat_correct(infile, gain = float(args.gain), readnoise = float(args.rdnoise),add_keywork='flat_corrected')
outfile = 'f'+outfile
fccd.write(outfile)
mflat = CCDData(mflat, unit=u.electron)
masks = []
processed = []
reprojected = []
for sci in sci_list:
raw = CCDData.read(sci, hdu=1, unit=u.adu)
red = ccdproc.subtract_overscan(raw,
overscan=raw[0:4, :],
overscan_axis=0,
model=models.Chebyshev1D(3))
red = ccdproc.ccd_process(
red,
gain=raw.header['GAIN'] * u.electron / u.adu,
readnoise=raw.header['RDNOISE'] * u.electron)
red = ccdproc.subtract_dark(red,
mdark,
exposure_time='EXPTIME',
exposure_unit=u.second)
red = ccdproc.flat_correct(red, mflat)
processed_data = ccdproc.ccd_process(red,
trim=raw.header['DATASEC'])
sigma_clip = SigmaClip(sigma=3)
bkg_estimator = MedianBackground()
_, median, std = sigma_clipped_stats(processed_data, sigma=3.0)
m = np.ma.masked_greater(processed_data, median + 3 * std)
masks.append(m.mask)
bkg = Background2D(processed_data, (30, 30),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=m.mask,
exclude_percentile=80)
def reduce_night(science_collection, dark_collection, flat_collection, config, config_arguments):
"""
This function reduce science data of one night and save the results to a folder named "reduced". The reduction
is performed as follows:
- Create a list of masterdarks (each masterdark has a different value of the exposure time) ^1
- Create a list of masterflats (each masterflat has a different value of the filter) ^1
- Reduce the science data as follows:
*For each filter:
*For each exposure time with that filter:
-- Look for the masterdark with the nearest exposure time
-- Look for the masterflat with the current filter.
-- Substract masterdark
-- Flat field correct the data
-- Clean cosmic rays (if requested)
-- Save image to ./Calibrated folder
(1) The master(flat/dark)s are created using mean combine.
:param science_collection: Numpy array - A numpy array with the science collection data produced by FitsLookup.
:param dark_collection: Numpy array - A numpy array with the dark collection data produced by FitsLookup.
:param flat_collection: Numpy array - A numpy array with the flat collection data produced by FitsLookup.
:param config_values: Dictionary - Dictionary - A dictionary provided by the function get_config_dict that
contains the config of the fits files ( readed from conf.INI ).
:param config_arguments: Dictionary - A dictionary provided by argparse initialization that contain the current flags.
:return: Integer - 0 if no errors raised 1 if errors raised.
"""
# Supress astropy warnings
warnings.filterwarnings('ignore')
# Renaming some config_arguments for easy acess
work_dir = config_arguments.dir[0]
# Get the filter and exposure collection of science and flat images
science_filter_collection = set(science_collection['filter'])
science_exposures_collection = set(science_collection['exptime'])
dark_exposures_collection = set(dark_collection['exptime'])
flat_filter_collection = set(flat_collection['filter'])
# Inform the user of the filter / exptime found.
science_exp_times_as_string = ", ".join(
[str(x) for x in science_exposures_collection])
dark_exp_times_as_string = ", ".join(
[str(x) for x in dark_exposures_collection])
module_logger.info("We have found {0} filters in the science images: {1}".format(
len(science_filter_collection), ", ".join(science_filter_collection)))
module_logger.info("We have found {0} exposure times science images: {1}".format(
len(science_exposures_collection), science_exp_times_as_string))
module_logger.info("We have found {0} exposure times dark calibrators: {1}".format(
len(dark_exposures_collection), dark_exp_times_as_string))
module_logger.info("We have found {0} filters in the flat calibrators {1}".format(
len(flat_filter_collection), ", ".join(flat_filter_collection)))
# Check if we have the same filters in flats and science, if not, get the
# intersection
if not science_filter_collection.issubset(flat_filter_collection):
module_logger.warning(
"There are more filters in the science images than in the flat calibrators")
science_filter_collection = science_filter_collection.intersection(
flat_filter_collection)
module_logger.warning("Triying to work with common filters.")
module_logger.info("We have found {0} common filters in the science images: {1}".format(
len(science_filter_collection), ", ".join(science_filter_collection)))
if not science_filter_collection:
module_logger.warning(
"There are no common filters between science images and flat calibrators")
module_logger.warning("This night will be skiped.")
return 1
# Warn the user if we found science images of 0 seconds
if 0 in science_exposures_collection:
number_of_null_images = len(filter_collection(
science_collection, [('exptime', 0)]))
module_logger.warning(
"We have found {0} science images with 0 seconds of exposure time.".format(number_of_null_images))
science_exposures_collection.discard(0)
module_logger.warning(
"Discarding images with 0 seconds of exposure time for this night: {0} exposure(s) remain.".format(
len(science_exposures_collection)))
# ------- MASTER DARK CREATION --------
module_logger.info("Starting the creation of the master dark")
module_logger.info("{0} different exposures for masterdarks were found.".format(
len(dark_exposures_collection)))
master_dark_collection = dict()
# Loop over each exposure time.
for dark_exposure_item in dark_exposures_collection:
module_logger.info("Creating masterdark with exposure of {0}s".format(dark_exposure_item))
# Initializate dark list for current collection.
exposure_dark_list = list()
for dark_image_data in filter_collection(dark_collection, [('exptime', dark_exposure_item)]):
# Open the images and append to the dark list
dark_image = dark_image_data['filename']
ccd = CCDData.read(dark_image, unit=config.image_units)
# If we have overscan, subtract and trim.
if config.subtract_overscan:
if config_arguments.verbose_flag_2:
module_logger.info("Subtracting overscan of {0}".format(dark_image))
ccd = subtract_and_trim_overscan(ccd, config)
exposure_dark_list.append(ccd)
# median combine the data
cb = ccdproc.Combiner(exposure_dark_list)
master_dark = cb.median_combine(median_func=np.median)
# Add the masterdark to the master_flat collection
master_dark_collection.update({dark_exposure_item: master_dark})
# Save the masterdark if needed.
if config.save_masterdark:
# Filename to save
aux = '{0}/masterdark_{1}.fits'.format(config_arguments.save_path, dark_exposure_item)
module_logger.info('Saving master dark to {0}'.format(aux))
master_dark.to_hdu().writeto(aux)
# ------- MASTER FLAT CREATION --------
module_logger.info("Starting the creation of the master flats")
module_logger.info("{0} different filters for masterflats were found".format(
len(flat_filter_collection)))
master_flat_collection = dict()
# Go thought the different filters in the collection
for flat_filter in flat_filter_collection:
module_logger.info(
"Creating masterflat with filter {0}".format(flat_filter))
# Initializate the list that will carry the flat images of the actual
# filter
filter_flat_list = list()
for flat_image_data in filter_collection(flat_collection, [('filter', flat_filter)]):
# Open the images and append to the filter's flat list
flat_image = flat_image_data['filename']
ccd = CCDData.read(flat_image, unit=config.image_units)
# Subtract and trim overscan
if config.subtract_overscan:
if config_arguments.verbose_flag_2:
module_logger.info("Subtracting overscan of {0}".format(flat_image))
ccd = subtract_and_trim_overscan(ccd, config)
filter_flat_list.append(ccd)
# median combine the flats after scaling each by its mean
cb = ccdproc.Combiner(filter_flat_list)
cb.scaling = lambda x: 1.0 / np.mean(x)
master_flat = cb.median_combine(median_func=np.median)
# Add the masterflat to the master_flat collection
master_flat_collection.update({flat_filter: master_flat})
# Save the masterflat if needed.
if config.save_masterflat:
aux = '{0}/masterflat_{1}.fits'.format(config_arguments.save_path, flat_filter)
module_logger.info('Saving master flat to {0}'.format(aux))
master_flat.to_hdu().writeto(aux)
# ------- REDUCE SCIENCE DATA --------
module_logger.info("Starting the calibration of the science images")
# Go thought the different files in the collection
for image_filter in science_filter_collection:
module_logger.info("Now calibrating filter: {0}".format(image_filter))
# Iterate thought each different exposure. This is because the dark files
# can have different exposures and the calibration must be performed with
# the masterdark with the nearest exposure time.
for science_exposure in science_exposures_collection:
# Important!! If you have more classifiers in the numpy dtype and you want
# to use them, you must modify the code here. For example, if you want to
# use a 'temp' value as classifier, after modify the dtype following the
# instructions in FitsLookup, you must add a loop here and modify the sub_collection.
# Once you have the 'temp' in the dtype, you must add a loop here as:
#
# >>>for temp_value in set(science_collection['temp']):
# module_logger.info("Now calibrating temp: {0}".format(temp_value))
#
# After this, you MUST indent all the following code (of this function) four spaces to
# the right, of course. Then, you only have to modify the science_subcollection as follows:
#
# >>> science_subcollection = filter_collection(
# science_collection, [('filter', image_filter),
# ('exptime', science_exposure),
# ('temp', temp_value) ])
#
# Follow this steps for every classifier you want to add. Yay!
# --------------------------------------------------------------
# Science subcollection is a really bad name, but is descriptive. Remember that this subcollection
# are the images with the current filter that has the current
# exposure time. E.g. ('r' and 20', 'r' and 30).
science_subcollection = filter_collection(
science_collection, [('filter', image_filter),
('exptime', science_exposure)])
# Continue if we have files to process. This will check if for some filter
# there are not enought images with the actual exposure time.
if science_subcollection.size:
module_logger.info(
"Now calibrating exposure: {0}".format(science_exposure))
# Determine if we have a masterdark with the science exposure file.
#
# - If we have a exposure matching masterdark, use it.
# - If we do not have a exposure matching masterdark, use the nearest.
try:
selected_masterdark = master_dark_collection[science_exposure]
nearest_exposure = 0, science_exposure
except KeyError:
# Get the nearest exoposure in the dark collection.
nearest_exposure = min(enumerate(master_dark_collection.keys()),
key=lambda x: abs(x[1] - science_exposure))
# Notice that nearest_exposure is a tuple of the form
# (index,exposure).
selected_masterdark = master_dark_collection[
nearest_exposure[1]]
# Initialize the progress bar variables
total_len = len(science_subcollection)
meantime = []
# Loop for each image with current (filter,exptime).
for contador, science_image_data_with_current_exposure in enumerate(science_subcollection):
# To supress astropy warnings.
devnull = open(os.devnull, "w")
sys.stdout = devnull
# Notice that until sys stdout is reasigned, no printing
# will be allowed in the following lines.
# Start timing
start = time.time()
# Extract the filename from the image data
science_image = science_image_data_with_current_exposure['filename']
# Read the image
ccd = CCDData.read(science_image, unit=config.image_units, wcs=None)
# Subtract overscan
if config.subtract_overscan:
if config_arguments.verbose_flag_2:
module_logger.info("Subtracting overscan of {0}".format(science_image))
ccd = subtract_and_trim_overscan(ccd, config)
# Master dark substraction
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info("Subtracting dark of image {0} of {1}".format(contador + 1, total_len))
sys.stdout = devnull
else:
module_logger.debug("Subtracting dark of image {0} of {1}".format(contador + 1, total_len))
selected_masterdark._wcs = ccd._wcs # FIXME: currently needed due to bug
ccd = ccdproc.subtract_dark(ccd, selected_masterdark, dark_exposure=nearest_exposure[1] * u.second,
data_exposure=science_exposure * u.second)
# flat-field correct the data
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info("Flat-field correcting image {0} of {1}".format(contador + 1, total_len))
sys.stdout = devnull
else:
module_logger.debug("Flat-field correcting image {0} of {1}".format(contador + 1, total_len))
current_master_flat = master_flat_collection[image_filter]
current_master_flat._wcs = ccd._wcs # FIXME: currently needed due to bug
ccd = ccdproc.flat_correct(ccd, current_master_flat)
# If we need to clean cosmic rays, do it.
if config_arguments.cosmic_flag:
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info(
"Cosmic ray cleaning of image {0} of {1}".format(contador + 1, total_len))
sys.stdout = devnull
else:
module_logger.debug(
"Cosmic ray cleaning of image {0} of {1}".format(contador + 1, total_len))
ccd = ccdproc.cosmicray_lacosmic(ccd, error_image=None, thresh=5, mbox=11, rbox=11, gbox=5)
# Save the calibrated image to a file
output_filename = os.path.join(config_arguments.save_path, os.path.basename(science_image))
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info(
"Saving image {0} of {1} to {2}".format(contador + 1, total_len, output_filename))
sys.stdout = devnull
else:
module_logger.debug(
"Saving image {0} of {1} to {2}".format(contador + 1, total_len, output_filename))
ccd.write(output_filename, clobber=True)
end = time.time()
meantime.append(end - start)
sys.stdout = sys.__stdout__ # Restart stdout printing
# Progressbar in case that we have not activated the no_interaction flag nor the advance
# verbose flag.
if not config_arguments.no_interaction and not config_arguments.verbose_flag_2:
if config_arguments.verbose_flag:
update_progress(float(contador + 1) / total_len,
np.mean(meantime) * (total_len - (contador + 1)))
return 0
def calibrate_image(self, options, img):
if 'biasonly' in options:
master_bias = self.find_master_bias(img)
if master_bias:
print(
f'{Fore.GREEN + Style.BRIGHT}Bias substraction...{Style.RESET_ALL}'
)
img = ccdp.subtract_bias(img, master_bias)
elif 'flatonly' in options:
master_flat = self.find_master_flat(img)
if master_flat:
print(
f'{Fore.GREEN + Style.BRIGHT}Flat correction...{Style.RESET_ALL}'
)
img = ccdp.flat_correct(img, master_flat)
else:
master_dark = self.find_master_dark(img)
if master_dark:
print(
f'{Fore.GREEN + Style.BRIGHT}Dark substraction...{Style.RESET_ALL}'
)
img = ccdp.subtract_dark(img,
master_dark,
exposure_time='exptime',
exposure_unit=u.second)
else:
master_dark = self.find_master_dark_c(img)
if master_dark:
master_bias = self.find_master_bias(img)
if master_bias:
print(
f'{Fore.GREEN + Style.BRIGHT}Bias substraction...{Style.RESET_ALL}'
)
img = ccdp.subtract_bias(img, master_bias)
print(
f'{Fore.GREEN + Style.BRIGHT}Calibrated dark substraction...{Style.RESET_ALL}'
)
else:
print(
f'{Fore.YELLOW + Style.BRIGHT}Substracted calibrated dark without bias...{Style.RESET_ALL}'
)
img = ccdp.subtract_dark(img,
master_dark,
exposure_time='exptime',
exposure_unit=u.second,
scale=True)
else:
master_bias = self.find_master_bias(img)
if master_bias:
print(
f'{Fore.RED + Style.BRIGHT}No dark substraction.{Style.RESET_ALL}'
)
print(
f'{Fore.GREEN + Style.BRIGHT}Bias substraction...{Style.RESET_ALL}'
)
img = ccdp.subtract_bias(img, master_bias)
else:
print(
f'{Fore.RED + Style.BRIGHT}No dark or bias substraction.{Style.RESET_ALL}'
)
if not 'noflat' in options:
master_flat = self.find_master_flat(img)
if master_flat:
print(
f'{Fore.GREEN + Style.BRIGHT}Flat correction...{Style.RESET_ALL}'
)
img = ccdp.flat_correct(img, master_flat)
else:
print(
f'{Fore.RED + Style.BRIGHT}No flat correction.{Style.RESET_ALL}'
)
else:
print(
f'{Fore.YELLOW + Style.BRIGHT}Skipping flat correction.{Style.RESET_ALL}'
)
return img
