def divide_flat(self):
"""Divides all LMI science frames by the appropriate flat field image
This method is LMI-specific, rather than being wrapper for a more
general function. Basic emulation of IRAF's ccdproc/flatcor function.
:return: None
"""
# Load the list of master flats and bias-subtracted data frames
flat_cl = ccdp.ImageFileCollection(
self.path, glob_include=f'flat_bin{self.bin_factor}_*.fits')
sci_cl = ccdp.ImageFileCollection(
self.path, glob_include=f'{self.prefix}.*b.fits')
# Check to be sure there are, indeed, flats...
if flat_cl.files:
# Loop through the filters present
for filt in sorted(list(flat_cl.summary['filters'])):
# Load in the master flat for this filter
if self.debug:
print(f'Dividing by master flat for filter: {filt}')
master_flat, mflat_fn = next(
flat_cl.ccds(ccdsum=self.binning,
filters=filt,
return_fname=True))
# Set up a progress bar, so we can see how the process is going
sci_filt_files = sci_cl.files_filtered(filters=filt)
prog_bar = tqdm(total=len(sci_filt_files),
unit='frame',
unit_scale=False,
colour='#D8BFD8')
# Loop through the science frames to correct
for ccd, sci_fn in sci_cl.ccds(ccdsum=self.binning,
filters=filt,
return_fname=True):
# Divide by master flat
ccdp.flat_correct(ccd, master_flat)
# Update the header
ccd.header['flatcor'] = True
ccd.header['HISTORY'] = PKG_NAME
ccd.header['HISTORY'] = 'Flat-corrected image saved: ' + \
_savetime()
ccd.header['HISTORY'] = f'Divided by flat: {mflat_fn}'
ccd.header['HISTORY'] = f'Previous filename: {sci_fn}'
# Save the result (suffix = 'f'); delete the input file
ccd.write(f'{self.path}/{sci_fn[:-6]}f{sci_fn[-5:]}',
overwrite=True)
os.remove(f'{self.path}/{sci_fn}')
# Update the progress bar
prog_bar.update(1)
# Close the progress bar, end of loop
prog_bar.close()
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction and flat correction.')
with fits.open(sci) as hdr:
header = hdr[0].header
data = hdr[0].data
data[np.isnan(data)] = np.nanmedian(data)
raw = CCDData(data,meta=header,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron)
log.info('Exposure time of science image is '+str(red.header['TRUITIME']*red.header['COADDONE']))
flat = np.array(ccdproc.flat_correct(red, mflat))
flat[np.isnan(flat)] = np.nanmedian(flat)
processed_data = CCDData(flat,unit=u.electron,header=red.header,wcs=red.wcs)
log.info('File proccessed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci.replace('.gz',''),processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(raw.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (128, 128), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits').replace('.gz',''),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['TRUITIME']*red.header['COADDONE']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/').replace('.gz',''),overwrite=True)
processed.append(final)
return processed, masks
def flat_file_by_file(fname, flatname, datahdus=0):
hdulist = fits.open(fname)
hduflat = fits.open(flatname)
nhdus = len(hdulist)
if nhdus > 1:
istart = 1
else:
istart = 0
hduindexes = list(range(nhdus))[istart:]
if datahdus != 0:
hduindexes = datahdus
for i in hduindexes:
data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
data1.header = hdulist[i].header
flat1 = ccdproc.CCDData(hduflat[i].data, unit="adu")
flat1.header = hduflat[i].header
if i == 1:
flatscale = np.mean(flat1)
# flat1 = flat1/flatscale
commentstr = "Flat image is " + flatname + " with scale" + str(
flatscale)
proc1 = ccdproc.flat_correct(data1,
flat1,
add_keyword={
'flat': True,
'calstat': 'OTZF',
'history': commentstr
})
fits.update(fname, proc1.data, header=proc1.header, ext=i)
# fits.update(fname, proc1.data, ext=i)
hdulist.close()
hduflat.close()
mylog("Flat corrected {0} with {1}".format(fname, flatname))
return
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction, dark subtraction, overscan correction, flat correction, and trimming image.')
raw = CCDData.read(sci,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of science image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)#dark_exposure=1*u.second, data_exposure=red.header['EXPTIME']*u.second, scale=True)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
red = ccdproc.flat_correct(red, mflat)
processed_data = ccdproc.ccd_process(red, trim=raw.header['DATASEC'])
log.info('File proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci,processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(red.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (510, 510), filter_size=(9, 9),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits'),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['EXPTIME']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/'),overwrite=True)
processed.append(final)
return processed, masks
def action(self, ccd):
select_dict = {'imagetyp': 'flat'}
for keyword in self.match_on:
if keyword in select_dict:
raise ValueError("Keyword {} already has a value set".format(keyword))
select_dict[keyword] = ccd.header[keyword]
master = self._master_image(select_dict)
return ccdproc.flat_correct(ccd, master)
def action(self, ccd):
select_dict = {'imagetyp': 'flat'}
for keyword in self.match_on:
if keyword in select_dict:
raise ValueError(
"Keyword {} already has a value set".format(keyword))
select_dict[keyword] = ccd.header[keyword]
master = self._master_image(select_dict)
return ccdproc.flat_correct(ccd, master)
def preprocess(fits_path):
"""Apply basic CCD reduction tasks.
Ask for the path to all the calibration and science files, and perform Bias, Dark and Flat combinations and proper substractions.
Pipeline que aplica una reduccion basica a imagenes de tipo CCD.
Realiza combinacion de Bias, de Darks, y Flats. Luego realiza las restas inherentes.
"""
import os
import sys
from image_collection import ImageFileCollection
#Collect all FITS files in path in fitslist
fitslist = []
for root, dirs, files in os.walk(fits_path, topdown=True):
fitslist.extend([os.path.join(root, name) for name in files if '.fit' in name])
#Dictionary with the image type (value) for each image file name (key)
imagetypes = {img:fits.getval(img, 'IMAGETYP') for img in fitslist}
#Make different lists for different image types
biaslist = []
sciencelist = []
flatlist = []
darklist = []
unknown = []
for k, v in imagetypes.iteritems():
if 'LIGHT' in v.upper() or v.upper()=='OBJECT':
sciencelist.append(k)
elif 'BIAS' in v.upper() or v.upper()=='ZERO':
biaslist.append(k)
elif 'FLAT' in v.upper():
flatlist.append(k)
elif 'DARK' in v.upper():
darklist.append(k)
else:
unknown.append(k)
#Create the flat master
if darklist:
darkmaster = combineDarks(darklist_flat)
flatmaster = combineFlats(flatlist, dark=darkmaster)
elif biaslist:
biasmaster = combineBias(biaslist)
flatmaster = combineFlats(flatlist, bias=biasmaster)
else:
flatmaster = combineFlats(flatlist)
for ascience in sciencelist:
sci_image = ccdproc.CCDData.read(ascience, unit='adu')
sci_darksub = ccdproc.subtract_dark(sci_image, darkmaster, exposure_time='exptime', exposure_unit=u.second)
sci_flatcorrected = ccdproc.flat_correct(sci_darksub, flatmaster)
outpath = os.path.join('/Users/utb/Desktop/reduced/', 'reduced_' + os.path.basename(ascience))
hdulist = sci_flatcorrected.to_hdu()
hdulist.writeto(outpath, clobber=True)
return
def _correctForFlat(self, ccd):
masterFlat = self.getFlatImage()
print('Correcting for flat...')
ccd = ccdproc.flat_correct(
ccd,
masterFlat,
min_value=0.1,
add_keyword={'HIERARCH GIULIA FLATFIELD CORRECTION': True})
return ccd
def reduce_image(imagefile, dark=None, flat=None):
im = CCDData.read(imagefile, unit='adu')
if dark is not None:
dark = CCDData.read(dark, unit='adu')
im = im.subtract(dark)
if flat is not None:
# masterflat = CCDData.read(flat, unit='adu')
hdul = fits.open(flat)
masterflat = CCDData(data=hdul[0].data, uncertainty=None, meta=hdul[0].header, unit='adu')
im = flat_correct(im, masterflat)
return im
def reduceFrames(self):
self.getMedianForObjects()
log.info('Reducing frames started')
print 'Reducing frames started'
for obj in self.objects.filesList:
data = ccdproc.CCDData.read(obj, unit=u.adu)
dataWithDeviation = ccdproc.create_deviation(data,
gain=1.5*u.electron/u.adu,
readnoise=5*u.electron)
reducedObject = ccdproc.gain_correct(dataWithDeviation,
1.5*u.electron/u.adu)
if self.biases.isExists:
reducedObject = ccdproc.subtract_bias(reducedObject,
self.biases.masterCCD)
if self.darks.isExists:
reducedObject = ccdproc.subtract_dark(reducedObject,
self.darks.masterCCD,
exposure_time=cfg.exptime,
exposure_unit=u.second,
scale=True)
if self.flats.isExists:
reducedObject = ccdproc.flat_correct(reducedObject,
self.flats.masterCCD)
self.directory = '../../Reduction/' + directoryName
if not os.path.exists(self.directory):
os.makedirs(self.directory)
reducedObject.write(self.directory + '/' + obj, clobber=True)
os.system('solve-field ' + self.directory + '/' + obj) # + ' --overwrite')
objName, objExtension = os.path.splitext(self.directory + '/' + obj)
if not os.path.exists(objName + '.new'):
log.warning(objName + ' cannot be solved')
else:
newObjectsList.append(objName + '.new')
log.info('Frame ' + objName + ' reduced')
log.info('Reduced ' + str(len(newObjectsList)) + ' frames')
print 'Reduced ' + str(len(newObjectsList)) + ' frames'
self.clean()
def correctscience():
for sciencec, sciencen in enumerate(sciencelist):
hdu = fits.open(sciencelist[sciencec])
print('open', sciencen)
if hdu[1].header['CHIPNAME'] == 'A5382-1-7':
ccdscience = CCDData(hdu[1].data, unit=u.adu)
if not os.path.exists('Corrected_Science'):
os.makedirs('Corrected_Science')
cccdscience = ccdproc.flat_correct(
ccdproc.subtract_bias(ccdscience, ccdmbias_use),
ccdmflat_use,
norm_value=np.nanmedian(ccdmflat_use))
path = 'Corrected_Science/' + sciencen[-12:]
cccdscience.write(path, overwrite=True)
hdu.close()
else:
hdu.close()
print(sciencen, 'returned wrong chipname for selected extension')
exit
def flat_correction(textlist_files, master_flat, prefix_str='f'):
"""
To flat field the science images using master flat
Args:
textlist_files : A python list object with paths/names to the individual files.
master_flat : Master flat used to flat field the science image
prefix_str : String to be added to newly created sceince image
Returns:
None
"""
master = CCDData.read(master_flat, unit=u.adu)
for filename in textlist_files:
ccd = CCDData.read(filename, unit=u.adu)
flat_corrected = ccdp.flat_correct(ccd=ccd, flat=master, min_value=0.9)
flat_corrected.meta['flatcorr'] = True
flat_corrected.data = flat_corrected.data.astype('float32')
flat_corrected.write(prefix_str + filename,
hdu_mask=None,
hdu_uncertainty=None)
def make_science(self, list_of_sciencefiles):
science_list = [0]*len(list_of_sciencefiles)
science_names = [0]*len(list_of_sciencefiles)
for ii, kk in enumerate(list_of_sciencefiles):
fitsfile = fits.open(kk)
science = ccdproc.CCDData(data=fitsfile[1].data, meta=fitsfile[1].header, unit="adu")
# science_cos = ccdproc.cosmicray_lacosmic(science, verbose = True)
overscan_sub = ccdproc.subtract_overscan(science, fits_section='[5:35, :]')
bias_sub = ccdproc.subtract_bias(overscan_sub, self.master_bias)
flat_corr = ccdproc.flat_correct(bias_sub, self.master_flat)
# science_list[ii] = flat_corr
flat_corr.write(self.output_dir+"/"+self.filename+"_"+str(ii)+".fits", overwrite=True)
science_names[ii] = self.output_dir+"/"+self.filename+"_"+str(ii)+".fits"
# science_list = [0]*len(list_of_sciencefiles)
coverages = [0]*len(list_of_sciencefiles)
for ii, kk in enumerate(science_names):
fitsfile = fits.open(kk)
fitsfile_common, coverage = reproject_interp(fitsfile, fits.open(science_names[0])[0].header, hdu_in=0)
science = ccdproc.CCDData(data=fitsfile_common, meta=fitsfile[0].header, unit="adu")
science_list[ii] = science
coverages[ii] = ccdproc.CCDData(data=coverage, meta=fitsfile[0].header, unit="adu")
self.combined_science = ccdproc.combine(science_list, method="median")
# self.combined_coverage = ccdproc.combine(coverages, method="sum")
header0 = fits.open(list_of_sciencefiles[0])[0].header
for key in header0:
# print(str(key), header0[str(key)])
if "COMMENT" in key:
continue
self.combined_science.header[str(key)] = header0[str(key)]
self.combined_science.write(self.output_dir+"/"+self.filename+".fits", overwrite=True)
def flat(self,im,superflat=None,display=None) :
""" Flat fielding
"""
# only flatfield if we are given a superflat!
if superflat is None : return im
if type(im) is not list : ims=[im]
else : ims = im
if type(superflat) is not list : superflats=[superflat]
else : superflats = superflat
out=[]
for im,flat in zip(ims,superflats) :
if self.verbose : print(' flat fielding...')
if display is not None :
display.tv(im)
corr = ccdproc.flat_correct(im,flat)
out.append(corr)
if display is not None :
display.tv(corr)
#plot central crossections
display.plotax1.cla()
dim=corr.data.shape
col = int(dim[1]/2)
row = corr.data[:,col]
display.plotax1.plot(row)
min,max=tv.minmax(row,low=5,high=5)
display.plotax1.set_ylim(min,max)
display.plotax1.set_xlabel('row')
display.plotax1.text(0.05,0.95,'Column {:d}'.format(col),transform=display.plotax1.transAxes)
display.plotax2.cla()
row = int(dim[0]/2)
col = corr.data[row,:]
min,max=tv.minmax(col,low=10,high=10)
display.plotax2.plot(col)
display.plotax2.set_xlabel('col')
display.plotax2.text(0.05,0.95,'Row {:d}'.format(row),transform=display.plotax2.transAxes)
display.plotax2.set_ylim(min,max)
input(" See flat-fielded image and original with - key. Hit any key to continue")
if len(out) == 1 : return out[0]
else : return out
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
ccd = ccdproc.subtract_bias(ccd, master_bias_blue)
blue_flat_list.append(ccd)
master_flat_blue = ccdproc.combine(blue_flat_list, method='median')
master_flat_blue.write('master_flat_blue.fits', clobber=True)
#reduce the arc frames
for filename in ic1.files_filtered(obstype='Arc', isiarm='Blue arm'):
hdu = fits.open(ic1.location + filename)
ccd = CCDData(hdu[1].data, header=hdu[0].header+hdu[1].header, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
ccd = ccdproc.subtract_bias(ccd, master_bias_blue)
ccd = ccdproc.flat_correct(ccd, master_flat_blue)
ccd.data = ccd.data.T
ccd.write('arc_'+filename, clobber=True)
red_flat_list = []
for filename in ic1.files_filtered(obstype='Arc', isiarm='Red arm'):
hdu = fits.open(ic1.location + filename)
ccd = CCDData(hdu[1].data, header=hdu[0].header+hdu[1].header, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
ccd = ccdproc.subtract_bias(ccd, master_bias_red)
ccd = ccdproc.flat_correct(ccd, master_flat_red)
ccd.data = ccd.data.T
ccd.write('arc_'+filename, clobber=True)
ppdpath = Path(newfitspath/'PreprocessedImages')
if not ppdpath.exists():
ppdpath.mkdir()
for i in ob_dic:
print(i)
for j in range(len(ob_dic[i])):
c = CCDData(fits.getdata(ob_dic[i][j]['FILE']), unit = u.adu)
hdr = fits.getheader(ob_dic[i][j]['FILE'])
#Exposure Time에 맞는 Dark
mdark = mdark_fdic[hdr['EXPTIME']]
c = subtract_bias(c,mbias)
c = subtract_dark(c,mdark
,hdr['EXPTIME']*u.second,hdr['EXPTIME']*u.second)
c = flat_correct(c,mflat,min_value=0.01)
c.header = hdr
c.header.add_history("B & D subtracted and F corrected")
c = yfu.CCDData_astype(c,dtype = 'float32')
c.write(ppdpath/ob_dic[i][j]['FILE'],overwrite=True)
#%%
fig, axs = plt.subplots(2,len(ob_list),figsize=(12,12))
for i in range(len(ob_list)):
data1 = CCDData.read(f"{ob_list[i]}-0001.fits")
data2 = CCDData.read(ppdpath/f"{ob_list[i]}-0001.fits")
im1 = yfu.zimshow(axs[0][i],data1)
im2 = yfu.zimshow(axs[1][i],data2)
def make_flatdark_correction():
""" for each light exposure do:
subtract dark from light
divide by dark-subtracted flat
save fits to file
add entry in database for each calibrated file generated"""
# Eventually should be done as described in:
# https://ccdproc.readthedocs.io/en/latest/reduction_toolbox.html
# Or here:
# https://mwcraig.github.io/ccd-as-book
import ccdproc
from astropy.nddata import CCDData
import astroscrappy
import numpy as np
from astropy import units as u
nb = models.NightBundle.get(telescope_night_bundle_id=1)
nb_dir = nb.directory_path
light_list_q = nb.exposures.select(
lambda d: d.exposure_type == models.EXP_TYPE_CODES["LIGHT"])
light_list = [os.path.join(nb_dir, f.filename) for f in light_list_q]
master_dark_q = nb.combinations.select(
lambda d: d.combination_type == models.COMB_TYPE_CODES["DARKM"]).get()
master_dark_path = os.path.join(nb_dir, "products", master_dark_q.filename)
master_flat_q = nb.combinations.select(
lambda d: d.combination_type == models.COMB_TYPE_CODES["FLATM"]).get()
master_flat_path = os.path.join(nb_dir, "products", master_flat_q.filename)
for light_q, light_fname in zip(light_list_q, light_list):
raw_data = CCDData.read(light_fname, unit="adu")
master_dark = CCDData.read(master_dark_path, unit="adu")
master_flat = CCDData.read(master_flat_path, unit="adu")
# cr_cleaned = ccdproc.cosmicray_lacosmic(
# raw_data,
# satlevel=np.inf,
# sepmed=False,
# cleantype="medmask",
# fsmode="median",
# )
# crmask, cr_cleaned = astroscrappy.detect_cosmics(
# raw_data,
# inmask=None,
# satlevel=np.inf,
# sepmed=False,
# cleantype="medmask",
# fsmode="median",
# )
# cr_cleaned.unit = "adu"
cr_cleaned = raw_data
dark_subtracted = ccdproc.subtract_dark(
cr_cleaned,
master_dark,
exposure_time="EXPTIME",
exposure_unit=u.second,
scale=True,
)
reduced_image = ccdproc.flat_correct(dark_subtracted,
master_flat,
min_value=0.9)
reduced_filename = "calib_{}".format(os.path.basename(light_fname))
reduced_path = os.path.join(nb_dir, "products", reduced_filename)
makedirs = os.path.dirname(reduced_path)
if makedirs:
os.makedirs(makedirs, exist_ok=True)
reduced_image.write(reduced_path, overwrite=True)
reduced_comb = models.ExposureCombination(
night_bundle=nb,
filename=reduced_filename,
combination_type=models.COMB_TYPE_CODES["CALIB_LIGHT"],
exposures=light_q,
uses_combinations=[master_flat_q, master_dark_q],
)
def process_spectroscopy_science(self, science_group, save_all=False):
"""Process Spectroscopy science images.
This function handles the full image reduction process for science
files. if save_all is set to True, all intermediate steps are saved.
Args:
science_group (object): :class:`~pandas.DataFrame` instance that contains a
list of science images that where observed at the same pointing
and time. It also contains a set of selected keywords from the
image's header.
save_all (bool): If True the pipeline will save all the intermadiate
files such as after overscan correction or bias corrected and
etc.
"""
# TODO (simon): The code here is too crowded.
# TODO cont. Create other functions as necessary
target_name = ''
slit_trim = None
master_bias = None
master_flat = None
master_flat_name = None
obstype = science_group.obstype.unique()
# print(obstype)
if 'OBJECT' in obstype or 'COMP' in obstype:
object_comp_group = science_group[
(science_group.obstype == 'OBJECT') |
(science_group.obstype == 'COMP')]
if 'OBJECT' in obstype:
target_name = science_group.object[science_group.obstype ==
'OBJECT'].unique()[0]
self.log.info('Processing Science Target: '
'{:s}'.format(target_name))
else:
# TODO (simon): This does not make any sense
self.log.info('Processing Comparison Lamp: '
'{:s}'.format(target_name))
if 'FLAT' in obstype and not self.args.ignore_flats:
flat_sub_group = science_group[science_group.obstype == 'FLAT']
master_flat, master_flat_name = \
self.create_master_flats(flat_group=flat_sub_group,
target_name=target_name)
elif self.args.ignore_flats:
self.log.warning(
'Ignoring creation of Master Flat by request.')
master_flat = None
master_flat_name = None
else:
self.log.info('Attempting to find a suitable Master Flat')
object_list = object_comp_group.file.tolist()
# grab a random image from the list
random_image = random.choice(object_list)
# define random image full path
random_image_full = os.path.join(self.args.raw_path,
random_image)
# read the random chosen file
ccd = CCDData.read(random_image_full, unit=u.adu)
if not self.args.ignore_flats:
# define the master flat name
master_flat_name = self.name_master_flats(
header=ccd.header, group=object_comp_group, get=True)
# load the best flat based on the name previously defined
master_flat, master_flat_name = \
get_best_flat(flat_name=master_flat_name,
path=self.args.red_path)
if (master_flat is None) and (master_flat_name is None):
self.log.critical('Failed to obtain master flat')
if master_flat is not None and not self.args.ignore_flats:
self.log.debug('Attempting to find slit trim section')
slit_trim = get_slit_trim_section(master_flat=master_flat)
elif self.args.ignore_flats:
self.log.warning('Slit Trimming will be skipped, '
'--ignore-flats is activated')
else:
self.log.info("Master flat inexistent, can't find slit trim "
"section")
if slit_trim is not None:
master_flat = image_trim(ccd=master_flat,
trim_section=slit_trim,
trim_type='slit')
if self.master_bias is not None:
master_bias = image_trim(ccd=self.master_bias,
trim_section=slit_trim,
trim_type='slit')
else:
try:
master_bias = self.master_bias.copy()
except AttributeError:
master_bias = None
norm_master_flat = None
norm_master_flat_name = None
all_object_image = []
all_comp_image = []
for science_image in object_comp_group.file.tolist():
self.out_prefix = ''
# define image full path
image_full_path = os.path.join(self.args.raw_path,
science_image)
# load image
ccd = read_fits(image_full_path, technique=self.technique)
# apply overscan
ccd = image_overscan(ccd, overscan_region=self.overscan_region)
self.out_prefix += 'o_'
if save_all:
full_path = os.path.join(self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
if slit_trim is not None:
# There is a double trimming of the image, this is to match
# the size of the other data
# TODO (simon): Potential problem here
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
ccd = image_trim(ccd=ccd,
trim_section=slit_trim,
trim_type='slit')
self.out_prefix = 'st' + self.out_prefix
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
else:
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
self.out_prefix = 't' + self.out_prefix
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
if not self.args.ignore_bias:
# TODO (simon): Add check that bias is compatible
ccd = ccdproc.subtract_bias(ccd=ccd,
master=master_bias,
add_keyword=False)
self.out_prefix = 'z' + self.out_prefix
ccd.header['GSP_BIAS'] = (os.path.basename(
self.master_bias_name), 'Master bias image')
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
else:
self.log.warning('Ignoring bias correction by request.')
# Do flat correction
if master_flat is None or master_flat_name is None:
self.log.warning('The file {:s} will not be '
'flatfielded'.format(science_image))
elif self.args.ignore_flats:
self.log.warning('Ignoring flatfielding by request.')
else:
if norm_master_flat is None:
norm_master_flat, norm_master_flat_name = \
normalize_master_flat(
master=master_flat,
name=master_flat_name,
method=self.args.flat_normalize,
order=self.args.norm_order)
ccd = ccdproc.flat_correct(ccd=ccd,
flat=norm_master_flat,
add_keyword=False)
self.out_prefix = 'f' + self.out_prefix
ccd.header['GSP_FLAT'] = (
os.path.basename(norm_master_flat_name),
'Master flat image')
# propagate master flat normalization method
ccd.header['GSP_NORM'] = norm_master_flat.header[
'GSP_NORM']
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
ccd, prefix = call_cosmic_rejection(
ccd=ccd,
image_name=science_image,
out_prefix=self.out_prefix,
red_path=self.args.red_path,
dcr_par=self.args.dcr_par_dir,
keep_files=self.args.keep_cosmic_files,
method=self.args.clean_cosmic,
save=True)
self.out_prefix = prefix
if ccd is not None:
if ccd.header['OBSTYPE'] == 'OBJECT':
self.log.debug(
"Appending OBJECT image for combination")
all_object_image.append(ccd)
elif ccd.header['OBSTYPE'] == 'COMP':
self.log.debug("Appending COMP image for combination")
all_comp_image.append(ccd)
else:
self.log.error("Unknown OBSTYPE = {:s}"
"".format(ccd.header['OBSTYPE']))
else:
self.log.warning("Cosmic ray rejection returned a None.")
if self.args.combine:
self.log.warning("Combination of data is experimental.")
if len(all_object_image) > 1:
print(len(all_object_image))
self.log.info("Combining {:d} OBJECT images"
"".format(len(all_object_image)))
object_group = object_comp_group[object_comp_group.obstype
== "OBJECT"]
print(object_group, len(all_object_image))
combine_data(all_object_image,
dest_path=self.args.red_path,
prefix=self.out_prefix,
save=True)
elif len(all_object_image) == 1:
# write_fits(all_object_image[0])
pass
else:
self.log.error("No OBJECT images to combine")
if len(all_comp_image) > 1:
self.log.info("Combining {:d} COMP images"
"".format(len(all_comp_image)))
# comp_group = object_comp_group[
# object_comp_group.obstype == "COMP"]
combine_data(all_comp_image,
dest_path=self.args.red_path,
prefix=self.out_prefix,
save=True)
else:
self.log.error("No COMP images to combine")
else:
self.log.debug("Combine is disabled (default)")
elif 'FLAT' in obstype:
self.queue.append(science_group)
self.log.warning('Only flats found in this group')
flat_sub_group = science_group[science_group.obstype == 'FLAT']
# TODO (simon): Find out if these variables are useful or not
self.create_master_flats(flat_group=flat_sub_group)
else:
self.log.error('There is no valid datatype in this group')
def run(self):
""" Runs the calibrating algorithm. The calibrated data is
returned in self.dataout
"""
### Preparation
# Load bias files if necessary
if not self.biasloaded or self.getarg('reload'):
self.loadbias()
# Else: check data for correct instrument configuration - currently not in use(need improvement)
else:
for keyind in range(len(self.biasfitkeys)):
if self.biaskeyvalues[keyind] != self.datain.getheadval(
self.biasfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.biasfitkeys[keyind])
# Load dark files if necessary
if not self.darkloaded or self.getarg('reload'):
self.loaddark()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.darkfitkeys)):
if self.darkkeyvalues[keyind] != self.datain.getheadval(
self.darkfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.darkfitkeys[keyind])
# Load flat files if necessary
if not self.flatloaded or self.getarg('reload'):
self.loadflat()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.flatfitkeys)):
if self.flatkeyvalues[keyind] != self.datain.getheadval(
self.flatfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.flatfitkeys[keyind])
#convert self.datain to CCD Data object
image = ccdproc.CCDData(self.datain.image, unit='adu')
image.header = self.datain.header
#subtract bias from image
image = ccdproc.subtract_bias(image, self.bias, add_keyword=False)
#subtract dark from image
image = ccdproc.subtract_dark(image,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second,
add_keyword=False)
#apply flat correction to image
image = ccdproc.flat_correct(image, self.flat, add_keyword=False)
# copy calibrated image into self.dataout - make sure self.dataout is a pipedata object
self.dataout = DataFits(config=self.datain.config)
self.dataout.image = image.data
self.dataout.header = image.header
self.dataout.filename = self.datain.filename
### Finish - cleanup
# Update DATATYPE
self.dataout.setheadval('DATATYPE', 'IMAGE')
# Add bias, dark files to History
self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname)
self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname)
self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname)
def process_imaging_science(self, imaging_group):
"""Does image reduction for science imaging data.
Args:
imaging_group (object): pandas.DataFrame instance that contains a
list of science data that are compatible with a given
instrument configuration and can be reduced together.
"""
# pick a random image in order to get a header
random_image = random.choice(imaging_group.file.tolist())
path_random_image = os.path.join(self.args.raw_path, random_image)
sample_file = CCDData.read(path_random_image, unit=u.adu)
master_flat_name = self.name_master_flats(header=sample_file.header,
group=imaging_group,
get=True)
log.debug('Got {:s} for master flat name'.format(master_flat_name))
master_flat, master_flat_name = get_best_flat(
flat_name=master_flat_name)
if master_flat is not None:
for image_file in imaging_group.file.tolist():
# start with an empty prefix
self.out_prefix = ''
image_full_path = os.path.join(self.args.raw_path, image_file)
ccd = CCDData.read(image_full_path, unit=u.adu)
# Trim image
ccd = image_trim(ccd, trim_section=self.trim_section)
self.out_prefix = 't_'
if not self.args.ignore_bias:
ccd = ccdproc.subtract_bias(ccd,
self.master_bias,
add_keyword=False)
self.out_prefix = 'z' + self.out_prefix
ccd.header.add_history('Bias subtracted image')
# apply flat correction
ccd = ccdproc.flat_correct(ccd, master_flat, add_keyword=False)
self.out_prefix = 'f' + self.out_prefix
ccd.header.add_history('Flat corrected '
'{:s}'.format(
master_flat_name.split('/')[-1]))
if self.args.clean_cosmic:
ccd = lacosmic_cosmicray_rejection(ccd=ccd)
self.out_prefix = 'c' + self.out_prefix
else:
print('Clean Cosmic ' + str(self.args.clean_cosmic))
final_name = os.path.join(self.args.red_path,
self.out_prefix + image_file)
ccd.write(final_name, clobber=True)
log.info('Created science file: {:s}'.format(final_name))
else:
log.error('Can not process data without a master flat')
def process_spectroscopy_science(self, science_group, save_all=False):
"""Process Spectroscopy science images.
This function handles the full image reduction process for science
files. if save_all is used, all intermediate steps are saved.
Args:
science_group (object): pandas.DataFrame instance that contains a
list of science images that where observed at the same pointing
and time. It also contains a set of selected keywords from the
image's header.
save_all (bool): If True the pipeline will save all the intermadiate
files such as after overscan correction or bias corrected and
etc.
"""
# TODO (simon): The code here is too crowded.
# TODO cont. Create other functions as necessary
target_name = ''
slit_trim = None
master_flat = None
master_flat_name = None
obstype = science_group.obstype.unique()
# print(obstype)
if 'OBJECT' in obstype or 'COMP' in obstype:
object_group = science_group[(science_group.obstype == 'OBJECT') |
(science_group.obstype == 'COMP')]
if 'OBJECT' in obstype:
target_name = science_group.object[science_group.obstype ==
'OBJECT'].unique()[0]
log.info('Processing Science Target: {:s}'.format(target_name))
else:
log.info(
'Processing Comparison Lamp: {:s}'.format(target_name))
if 'FLAT' in obstype and not self.args.ignore_flats:
flat_sub_group = science_group[science_group.obstype == 'FLAT']
master_flat, master_flat_name =\
self.create_master_flats(flat_group=flat_sub_group,
target_name=target_name)
elif self.args.ignore_flats:
log.warning('Ignoring creation of Master Flat by request.')
master_flat = None
master_flat_name = None
else:
log.info('Attempting to find a suitable Master Flat')
object_list = object_group.file.tolist()
# grab a random image from the list
random_image = random.choice(object_list)
# define random image full path
random_image_full = os.path.join(self.args.raw_path,
random_image)
# read the random chosen file
ccd = CCDData.read(random_image_full, unit=u.adu)
if not self.args.ignore_flats:
# define the master flat name
master_flat_name = self.name_master_flats(
header=ccd.header, group=object_group, get=True)
# load the best flat based on the name previously defined
master_flat, master_flat_name = \
get_best_flat(flat_name=master_flat_name)
if (master_flat is None) and (master_flat_name is None):
log.critical('Failed to obtain master flat')
if master_flat is not None and not self.args.ignore_flats:
log.debug('Attempting to find slit trim section')
slit_trim = get_slit_trim_section(master_flat=master_flat)
elif self.args.ignore_flats:
log.warning('Slit Trimming will be skipped, --ignore-flats is '
'activated')
else:
log.info('Master flat inexistent, cant find slit trim section')
if slit_trim is not None:
master_flat = image_trim(ccd=master_flat,
trim_section=slit_trim)
if self.master_bias is not None:
master_bias = image_trim(ccd=self.master_bias,
trim_section=slit_trim)
else:
try:
master_bias = self.master_bias.copy()
except AttributeError:
master_bias = None
norm_master_flat = None
for science_image in object_group.file.tolist():
self.out_prefix = ''
# define image full path
image_full_path = os.path.join(self.args.raw_path,
science_image)
# load image
ccd = CCDData.read(image_full_path, unit=u.adu)
# apply overscan
ccd = image_overscan(ccd, overscan_region=self.overscan_region)
self.out_prefix += 'o_'
if save_all:
full_path = os.path.join(self.args.red_path,
self.out_prefix + science_image)
ccd.write(full_path, clobber=True)
if slit_trim is not None:
# There is a double trimming of the image, this is to match
# the size of the other data
# TODO (simon): Potential problem here
ccd = image_trim(ccd=ccd, trim_section=self.trim_section)
ccd = image_trim(ccd=ccd, trim_section=slit_trim)
self.out_prefix = 'st' + self.out_prefix
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
ccd.write(full_path, clobber=True)
else:
ccd = image_trim(ccd=ccd, trim_section=self.trim_section)
self.out_prefix = 't' + self.out_prefix
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
ccd.write(full_path, clobber=True)
if not self.args.ignore_bias:
# TODO (simon): Add check that bias is compatible
ccd = ccdproc.subtract_bias(ccd=ccd,
master=master_bias,
add_keyword=False)
self.out_prefix = 'z' + self.out_prefix
ccd.header.add_history('Bias subtracted image')
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
ccd.write(full_path, clobber=True)
else:
log.warning('Ignoring bias correction by request.')
if master_flat is None or master_flat_name is None:
log.warning('The file {:s} will not be '
'flatfielded'.format(science_image))
elif self.args.ignore_flats:
log.warning('Ignoring flatfielding by request.')
else:
if norm_master_flat is None:
norm_master_flat = normalize_master_flat(
master=master_flat,
name=master_flat_name,
method=self.args.flat_normalize,
order=self.args.norm_order)
ccd = ccdproc.flat_correct(ccd=ccd,
flat=norm_master_flat,
add_keyword=False)
self.out_prefix = 'f' + self.out_prefix
ccd.header.add_history('master flat norm_'
'{:s}'.format(master_flat_name))
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
ccd.write(full_path, clobber=True)
call_cosmic_rejection(ccd=ccd,
image_name=science_image,
out_prefix=self.out_prefix,
red_path=self.args.red_path,
dcr_par=self.args.dcr_par_dir,
keep_files=self.args.keep_cosmic_files,
method=self.args.clean_cosmic)
# print(science_group)
elif 'FLAT' in obstype:
self.queue.append(science_group)
log.warning('Only flats found in this group')
flat_sub_group = science_group[science_group.obstype == 'FLAT']
# TODO (simon): Find out if these variables are useful or not
master_flat, master_flat_name = \
self.create_master_flats(flat_group=flat_sub_group)
else:
log.error('There is no valid datatype in this group')
def calibrate_image(self, options, img):
if 'biasonly' in options:
master_bias = self.find_master_bias(img)
if master_bias:
print(
f'{Fore.GREEN + Style.BRIGHT}Bias substraction...{Style.RESET_ALL}'
)
img = ccdp.subtract_bias(img, master_bias)
elif 'flatonly' in options:
master_flat = self.find_master_flat(img)
if master_flat:
print(
f'{Fore.GREEN + Style.BRIGHT}Flat correction...{Style.RESET_ALL}'
)
img = ccdp.flat_correct(img, master_flat)
else:
master_dark = self.find_master_dark(img)
if master_dark:
print(
f'{Fore.GREEN + Style.BRIGHT}Dark substraction...{Style.RESET_ALL}'
)
img = ccdp.subtract_dark(img,
master_dark,
exposure_time='exptime',
exposure_unit=u.second)
else:
master_dark = self.find_master_dark_c(img)
if master_dark:
master_bias = self.find_master_bias(img)
if master_bias:
print(
f'{Fore.GREEN + Style.BRIGHT}Bias substraction...{Style.RESET_ALL}'
)
img = ccdp.subtract_bias(img, master_bias)
print(
f'{Fore.GREEN + Style.BRIGHT}Calibrated dark substraction...{Style.RESET_ALL}'
)
else:
print(
f'{Fore.YELLOW + Style.BRIGHT}Substracted calibrated dark without bias...{Style.RESET_ALL}'
)
img = ccdp.subtract_dark(img,
master_dark,
exposure_time='exptime',
exposure_unit=u.second,
scale=True)
else:
master_bias = self.find_master_bias(img)
if master_bias:
print(
f'{Fore.RED + Style.BRIGHT}No dark substraction.{Style.RESET_ALL}'
)
print(
f'{Fore.GREEN + Style.BRIGHT}Bias substraction...{Style.RESET_ALL}'
)
img = ccdp.subtract_bias(img, master_bias)
else:
print(
f'{Fore.RED + Style.BRIGHT}No dark or bias substraction.{Style.RESET_ALL}'
)
if not 'noflat' in options:
master_flat = self.find_master_flat(img)
if master_flat:
print(
f'{Fore.GREEN + Style.BRIGHT}Flat correction...{Style.RESET_ALL}'
)
img = ccdp.flat_correct(img, master_flat)
else:
print(
f'{Fore.RED + Style.BRIGHT}No flat correction.{Style.RESET_ALL}'
)
else:
print(
f'{Fore.YELLOW + Style.BRIGHT}Skipping flat correction.{Style.RESET_ALL}'
)
return img
def preprocess(fits_path):
"""Apply basic CCD reduction tasks.
Ask for the path to all the calibration and science files, and perform Bias, Dark and Flat combinations and proper substractions.
Pipeline que aplica una reduccion basica a imagenes de tipo CCD.
Realiza combinacion de Bias, de Darks, y Flats. Luego realiza las restas inherentes.
"""
import os
import sys
from image_collection import ImageFileCollection
#Collect all FITS files in path in fitslist
#fitslist = []
#for root, dirs, files in os.walk(fits_path, topdown=True):
# fitslist.extend([os.path.join(root, name) for name in files if '.fit' in name])
#
##Dictionary with the image type (value) for each image file name (key)
#imagetypes = {img:fits.getval(img, 'IMAGETYP') for img in fitslist}
# now we make different lists for different image types
#biaslist = []
#sciencelist = []
#flatlist = []
#darklist = []
#unknown = []
#for k, v in imagetypes.iteritems():
# if 'LIGHT' in v.upper() or v.upper()=='OBJECT':
# sciencelist.append(k)
# elif 'BIAS' in v.upper() or v.upper()=='ZERO':
# biaslist.append(k)
# elif 'FLAT' in v.upper():
# flatlist.append(k)
# elif 'DARK' in v.upper():
# darklist.append(k)
# else:
# unknown.append(k)
#Create an image file collection storing the following keys
keys = ['imagetyp', 'object', 'filter', 'exptime']
allfits = ImageFileCollection('.', keywords=keys)
#Collect all dark files and make a dark frame for each different exposure time
dark_matches = np.ma.array(['dark' in typ.lower() for typ in allfits.summary['imagetyp']])
darkexp_set = set(allfits.summary['exptime'][dark_matches])
darklists = {}
for anexp in darkexp_set:
my_darks = allfits.summary['file'][(allfits.summary['exptime'] == anexp) & dark_matches]
darklists[anexp] = combineDarks(my_darks)
#Collect all science files
sciencelist = allfits.files_filtered(imagetyp='light')
#Collect all bias files
#bias_matches = ([('zero' in typ.lower() or 'bias' in typ.lower()) for typ in allfits.summary['imagetyp']]
#biaslist = allfits.summary['file'][bias_matches]
#Create the flat master
flatlist = allfits.files_filtered(imagetyp='flat')
exptime = fits.getval(flatlist[0], 'exptime')
darkmaster = chooseClosestDark(darklists, exptime)
flatmaster = combineFlats(flatlist, dark=darkmaster)
for ascience in sciencelist:
sci_image = ccdproc.CCDData.read(ascience, unit='adu')
exp_time = fits.getval(ascience, 'exptime')
darkmaster = chooseClosestDark(darklists, exp_time)
try:
sci_darksub = ccdproc.subtract_dark(sci_image, darkmaster, exposure_time='exptime', exposure_unit=u.second)
sci_flatcorrected = ccdproc.flat_correct(sci_darksub, flatmaster)
outpath = os.path.join('/Users/utb/Desktop/reduced/', 'reduced_' + os.path.basename(ascience))
except:
sci_flatcorrected = sci_image
outpath = os.path.join('/Users/utb/Desktop/reduced/', 'failed_' + os.path.basename(ascience))
hdulist = sci_flatcorrected.to_hdu()
hdulist.writeto(outpath, clobber=True)
return
def process_spectroscopy_science(self, science_group, save_all=False):
"""Process Spectroscopy science images.
This function handles the full image reduction process for science
files. if save_all is set to True, all intermediate steps are saved.
Args:
science_group (object): :class:`~pandas.DataFrame` instance that contains a
list of science images that where observed at the same pointing
and time. It also contains a set of selected keywords from the
image's header.
save_all (bool): If True the pipeline will save all the intermadiate
files such as after overscan correction or bias corrected and
etc.
"""
# TODO (simon): The code here is too crowded.
# TODO cont. Create other functions as necessary
target_name = ''
slit_trim = None
master_bias = None
master_flat = None
master_flat_name = None
obstype = science_group.obstype.unique()
# print(obstype)
if 'OBJECT' in obstype or 'COMP' in obstype:
object_comp_group = science_group[
(science_group.obstype == 'OBJECT') |
(science_group.obstype == 'COMP')]
if 'OBJECT' in obstype:
target_name = science_group.object[science_group.obstype ==
'OBJECT'].unique()[0]
self.log.info('Processing Science Target: '
'{:s}'.format(target_name))
else:
# TODO (simon): This does not make any sense
self.log.info('Processing Comparison Lamp: '
'{:s}'.format(target_name))
if 'FLAT' in obstype and not self.args.ignore_flats:
flat_sub_group = science_group[science_group.obstype == 'FLAT']
master_flat, master_flat_name = \
self.create_master_flats(flat_group=flat_sub_group,
target_name=target_name)
elif self.args.ignore_flats:
self.log.warning('Ignoring creation of Master Flat by request.')
master_flat = None
master_flat_name = None
else:
self.log.info('Attempting to find a suitable Master Flat')
object_list = object_comp_group.file.tolist()
# grab a random image from the list
random_image = random.choice(object_list)
# define random image full path
random_image_full = os.path.join(self.args.raw_path,
random_image)
# read the random chosen file
ccd = CCDData.read(random_image_full, unit=u.adu)
if not self.args.ignore_flats:
# define the master flat name
master_flat_name = self.name_master_flats(
header=ccd.header,
group=object_comp_group,
get=True)
# load the best flat based on the name previously defined
master_flat, master_flat_name = \
get_best_flat(flat_name=master_flat_name,
path=self.args.red_path)
if (master_flat is None) and (master_flat_name is None):
self.log.critical('Failed to obtain master flat')
if master_flat is not None and not self.args.ignore_flats:
self.log.debug('Attempting to find slit trim section')
slit_trim = get_slit_trim_section(master_flat=master_flat)
elif self.args.ignore_flats:
self.log.warning('Slit Trimming will be skipped, '
'--ignore-flats is activated')
else:
self.log.info("Master flat inexistent, can't find slit trim "
"section")
if slit_trim is not None:
master_flat = image_trim(ccd=master_flat,
trim_section=slit_trim,
trim_type='slit')
if self.master_bias is not None:
master_bias = image_trim(ccd=self.master_bias,
trim_section=slit_trim,
trim_type='slit')
else:
try:
master_bias = self.master_bias.copy()
except AttributeError:
master_bias = None
norm_master_flat = None
norm_master_flat_name = None
all_object_image = []
all_comp_image = []
for science_image in object_comp_group.file.tolist():
self.out_prefix = ''
# define image full path
image_full_path = os.path.join(self.args.raw_path,
science_image)
# load image
ccd = read_fits(image_full_path, technique=self.technique)
# apply overscan
ccd = image_overscan(ccd, overscan_region=self.overscan_region)
self.out_prefix += 'o_'
if save_all:
full_path = os.path.join(self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
if slit_trim is not None:
# There is a double trimming of the image, this is to match
# the size of the other data
# TODO (simon): Potential problem here
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
ccd = image_trim(ccd=ccd,
trim_section=slit_trim,
trim_type='slit')
self.out_prefix = 'st' + self.out_prefix
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
else:
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
self.out_prefix = 't' + self.out_prefix
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
if not self.args.ignore_bias:
# TODO (simon): Add check that bias is compatible
ccd = ccdproc.subtract_bias(ccd=ccd,
master=master_bias,
add_keyword=False)
self.out_prefix = 'z' + self.out_prefix
ccd.header['GSP_BIAS'] = (
os.path.basename(self.master_bias_name),
'Master bias image')
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
else:
self.log.warning('Ignoring bias correction by request.')
# Do flat correction
if master_flat is None or master_flat_name is None:
self.log.warning('The file {:s} will not be '
'flatfielded'.format(science_image))
elif self.args.ignore_flats:
self.log.warning('Ignoring flatfielding by request.')
else:
if norm_master_flat is None:
norm_master_flat, norm_master_flat_name = \
normalize_master_flat(
master=master_flat,
name=master_flat_name,
method=self.args.flat_normalize,
order=self.args.norm_order)
ccd = ccdproc.flat_correct(ccd=ccd,
flat=norm_master_flat,
add_keyword=False)
self.out_prefix = 'f' + self.out_prefix
ccd.header['GSP_FLAT'] = (
os.path.basename(norm_master_flat_name),
'Master flat image')
# propagate master flat normalization method
ccd.header['GSP_NORM'] = norm_master_flat.header['GSP_NORM']
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
ccd, prefix = call_cosmic_rejection(
ccd=ccd,
image_name=science_image,
out_prefix=self.out_prefix,
red_path=self.args.red_path,
dcr_par=self.args.dcr_par_dir,
keep_files=self.args.keep_cosmic_files,
method=self.args.clean_cosmic,
save=True)
self.out_prefix = prefix
if ccd is not None:
if ccd.header['OBSTYPE'] == 'OBJECT':
self.log.debug("Appending OBJECT image for combination")
all_object_image.append(ccd)
elif ccd.header['OBSTYPE'] == 'COMP':
self.log.debug("Appending COMP image for combination")
all_comp_image.append(ccd)
else:
self.log.error("Unknown OBSTYPE = {:s}"
"".format(ccd.header['OBSTYPE']))
else:
self.log.warning("Cosmic ray rejection returned a None.")
if self.args.combine:
self.log.warning("Combination of data is experimental.")
if len(all_object_image) > 1:
print(len(all_object_image))
self.log.info("Combining {:d} OBJECT images"
"".format(len(all_object_image)))
object_group = object_comp_group[
object_comp_group.obstype == "OBJECT"]
print(object_group, len(all_object_image))
combine_data(all_object_image,
dest_path=self.args.red_path,
prefix=self.out_prefix,
save=True)
elif len(all_object_image) == 1:
# write_fits(all_object_image[0])
pass
else:
self.log.error("No OBJECT images to combine")
if len(all_comp_image) > 1:
self.log.info("Combining {:d} COMP images"
"".format(len(all_comp_image)))
# comp_group = object_comp_group[
# object_comp_group.obstype == "COMP"]
combine_data(all_comp_image,
dest_path=self.args.red_path,
prefix=self.out_prefix,
save=True)
else:
self.log.error("No COMP images to combine")
else:
self.log.debug("Combine is disabled (default)")
elif 'FLAT' in obstype:
self.queue.append(science_group)
self.log.warning('Only flats found in this group')
flat_sub_group = science_group[science_group.obstype == 'FLAT']
# TODO (simon): Find out if these variables are useful or not
self.create_master_flats(flat_group=flat_sub_group)
else:
self.log.error('There is no valid datatype in this group')
zccd = ccdproc.cosmicray_lacosmic(ccd, gain = float(args.gain), readnoise = float(args.rdnoise))
outfile = 'z'+outfile
# subtract dark
if args.dark:
if args.zap:
infile=zccd
else:
infile = ccd
dccd = ccdproc.subtract_dark(infile, dark)
outfile = 'd'+outfile
# flatten
if args.flat:
# flatten
if args.dark:
infile = dccd
else:
if args.zap:
infile = zccd
else:
infile = ccd
fccd = ccdproc.flat_correct(infile, gain = float(args.gain), readnoise = float(args.rdnoise),add_keywork='flat_corrected')
outfile = 'f'+outfile
fccd.write(outfile)
i += 1
print '\n'
reprojected = []
for sci in sci_list:
raw = CCDData.read(sci, hdu=1, unit=u.adu)
red = ccdproc.subtract_overscan(raw,
overscan=raw[0:4, :],
overscan_axis=0,
model=models.Chebyshev1D(3))
red = ccdproc.ccd_process(
red,
gain=raw.header['GAIN'] * u.electron / u.adu,
readnoise=raw.header['RDNOISE'] * u.electron)
red = ccdproc.subtract_dark(red,
mdark,
exposure_time='EXPTIME',
exposure_unit=u.second)
red = ccdproc.flat_correct(red, mflat)
processed_data = ccdproc.ccd_process(red,
trim=raw.header['DATASEC'])
sigma_clip = SigmaClip(sigma=3)
bkg_estimator = MedianBackground()
_, median, std = sigma_clipped_stats(processed_data, sigma=3.0)
m = np.ma.masked_greater(processed_data, median + 3 * std)
masks.append(m.mask)
bkg = Background2D(processed_data, (30, 30),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=m.mask,
exclude_percentile=80)
processed.append(
processed_data.subtract(CCDData(bkg.background,
lights[lightcnt] = subtract_bias(lights[lightcnt], bias)
if darkcal == True:
log("Calibrating dark...")
lights[lightcnt] = subtract_dark(
lights[lightcnt],
dark,
dark_exposure=darkexp * units.s,
data_exposure=exptimenow * units.s,
scale=True)
if flatcal == True:
log("Calibrating flat...")
if lightfilter != flatfilter:
warning("Filter mismatch with flat: " + i + " (" +
headers[args.fits_header_filter].strip() +
" filter)")
lights[lightcnt] = flat_correct(lights[lightcnt], flat)
#regularize exposure time
lights[lightcnt] = lights[lightcnt].multiply(exptime /
exptimenow)
lights[lightcnt] = lights[lightcnt].add(args.pedestal *
units.astrophys.adu)
if args.offset != None:
try:
light_one = np.roll(light_one, -offset[i][0], axis=1)
light_one = np.roll(light_one, -offset[i][1], axis=0)
light_one = CCDData(light_one, unit='adu')
log("Shifted by " + str(-offset[i][0]) + ", " +
str(-offset[i][1]))
except KeyError:
master_flat_blue.write('master_flat_blue.fits', clobber=True)
#reduce the arc frames
for filename in ic1.files_filtered(obstype='Arc', isiarm='Blue arm'):
hdu = fits.open(ic1.location + filename)
ccd = CCDData(hdu[1].data,
header=hdu[0].header + hdu[1].header,
unit=u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd,
median=True,
overscan_axis=0,
fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
ccd = ccdproc.subtract_bias(ccd, master_bias_blue)
ccd = ccdproc.flat_correct(ccd, master_flat_blue)
ccd.data = ccd.data.T
ccd.write('arc_' + filename, clobber=True)
red_flat_list = []
for filename in ic1.files_filtered(obstype='Arc', isiarm='Red arm'):
hdu = fits.open(ic1.location + filename)
ccd = CCDData(hdu[1].data,
header=hdu[0].header + hdu[1].header,
unit=u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd,
median=True,
overscan_axis=0,
fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
def process_science(sci_list,
fil,
red_path,
mbias=None,
mflat=None,
proc=None,
log=None):
masks = []
processed = []
flat_left = mflat[0]
flat_right = mflat[1]
left_list = []
right_list = []
if proc:
for j, sci in enumerate(sci_list):
log.info('Loading file: ' + sci)
log.info('Applying flat correction and trimming left image.')
left = CCDData.read(sci, hdu=1, unit=u.electron)
left = ccdproc.flat_correct(left, flat_left)
left = ccdproc.ccd_process(left, trim=left.header['DATASEC'])
log.info('Left image proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(left, nsigma=3, npixels=5)
clean, com_mask = create_mask.create_mask(sci, left,
'_mask_left.fits',
static_mask(proc)[0],
mask,
saturation(left.header),
binning(proc, 'left'),
rdnoise(left.header),
cr_clean_sigclip(),
cr_clean_sigcfrac(),
cr_clean_objlim(), log)
left.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(left, (120, 120),
filter_size=(3, 3),
sigma_clip=SigmaClip(sigma=3),
bkg_estimator=MeanBackground(),
mask=mask,
exclude_percentile=80)
log.info('Median background for left iamge: ' +
str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/',
'/red/').replace('.fits',
'_bkg_left.fits'),
bkg.background,
overwrite=True)
left = left.subtract(CCDData(bkg.background, unit=u.electron),
propagate_uncertainties=True,
handle_meta='first_found')
log.info('Exposure time of left image is ' +
str(left.header['EXPTIME']))
left = left.divide(left.header['EXPTIME'],
propagate_uncertainties=True,
handle_meta='first_found')
log.info(
'Background subtracted and image divided by exposure time.')
left.header['DATASEC'] = '[1:' + str(
np.shape(left)[1]) + ',1:' + str(np.shape(left)[0]) + ']'
left_list.append(left)
log.info('Applying flat correction and trimming right image.')
right = CCDData.read(sci, hdu=2, unit=u.electron)
right = ccdproc.flat_correct(right, flat_right)
right = ccdproc.ccd_process(right, trim=right.header['DATASEC'])
log.info('Right image proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(right, nsigma=3, npixels=5)
clean, com_mask = create_mask.create_mask(sci, right,
'_mask_right.fits',
static_mask(proc)[1],
mask,
saturation(right.header),
binning(proc, 'right'),
rdnoise(right.header),
cr_clean_sigclip(),
cr_clean_sigcfrac(),
cr_clean_objlim(), log)
right.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(right, (120, 120),
filter_size=(3, 3),
sigma_clip=SigmaClip(sigma=3),
bkg_estimator=MeanBackground(),
mask=mask,
exclude_percentile=80)
log.info('Median background for right image : ' +
str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/',
'/red/').replace('.fits',
'_bkg_right.fits'),
bkg.background,
overwrite=True)
right = right.subtract(CCDData(bkg.background, unit=u.electron),
propagate_uncertainties=True,
handle_meta='first_found')
log.info('Exposure time of right image is ' +
str(right.header['EXPTIME']))
right = right.divide(right.header['EXPTIME'],
propagate_uncertainties=True,
handle_meta='first_found')
log.info(
'Background subtracted and image divided by exposure time.')
right.header['DATASEC'] = '[1:' + str(
np.shape(right)[1]) + ',1:' + str(np.shape(right)[0]) + ']'
right_list.append(right)
else:
for j, sci in enumerate(sci_list):
log.info('Loading file: ' + sci)
log.info(
'Applying gain correction, overscan correction, flat correction, and trimming image.'
)
with fits.open(sci) as hdr:
header_left = hdr[1].header
header_right = hdr[6].header
data_list = []
for i in range(8):
data = ccdproc.CCDData.read(sci, hdu=i + 1, unit=u.adu)
red = ccdproc.ccd_process(data,
oscan=data[:, 0:50],
oscan_model=models.Chebyshev1D(3),
trim='[1200:2098,210:2056]',
gain=gain()[i] * u.electron / u.adu,
readnoise=4 * u.electron)
data_list.append(np.asarray(red).astype(np.float32))
top_left = np.concatenate(
[data_list[0], np.fliplr(data_list[1])], axis=1)
bot_left = np.flipud(
np.concatenate(
[data_list[3], np.fliplr(data_list[2])], axis=1))
left = CCDData(np.concatenate([top_left, bot_left]),
unit=u.electron,
header=header_left)
left = ccdproc.flat_correct(left, flat_left[209:3903, 1149:2947])
log.info('Left image proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(left, nsigma=3, npixels=5)
# clean, com_mask = create_mask.create_mask(sci,left,static_mask(proc)[0],mask,saturation(left.header),binning(proc,'left'),rdnoise(left.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
# processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(left, (120, 120),
filter_size=(3, 3),
sigma_clip=SigmaClip(sigma=3),
bkg_estimator=MeanBackground(),
mask=mask,
exclude_percentile=80)
log.info('Median background for left image : ' +
str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/',
'/red/').replace('.fits',
'_bkg_left.fits'),
bkg.background,
overwrite=True)
left = left.subtract(CCDData(bkg.background, unit=u.electron),
propagate_uncertainties=True,
handle_meta='first_found')
log.info('Exposure time of left image is ' +
str(left.header['EXPTIME']))
left = left.divide(left.header['EXPTIME'] * u.second,
propagate_uncertainties=True,
handle_meta='first_found')
log.info(
'Background subtracted and image divided by exposure time.')
left.header['DATASEC'] = '[1:1798,1:3694]'
left.header['RADECSYS'] = 'ICRS'
left.header['CUNIT1'] = 'deg'
left.header['CUNIT2'] = 'deg'
left.header['CTYPE1'] = 'RA---TAN'
left.header['CTYPE2'] = 'DEC--TAN'
left.header['CRPIX1'] = 2301
left.header['CRPIX2'] = 1846
coord = util.parse_coord(left.header['RA'], left.header['DEC'])
left.header['CRVAL1'] = coord.ra.deg
left.header['CRVAL2'] = coord.dec.deg
left.header['PC1_1'] = -pixscale() / 3600 * np.sin(
np.pi / 180. * (left.header['POSANG'] + 90))
left.header['PC1_2'] = pixscale() / 3600 * np.cos(
np.pi / 180. * (left.header['POSANG'] + 90))
left.header['PC2_1'] = -pixscale() / 3600 * np.cos(
np.pi / 180. * (left.header['POSANG'] + 90))
left.header['PC2_2'] = pixscale() / 3600 * np.sin(
np.pi / 180. * (left.header['POSANG'] + 90))
left.write(sci.replace('/raw/',
'/red/').replace('.fits', '_left.fits'),
overwrite=True)
left_list.append(left)
top_right = np.concatenate(
[data_list[6], np.fliplr(data_list[7])], axis=1)
bot_right = np.flipud(
np.concatenate(
[data_list[5], np.fliplr(data_list[4])], axis=1))
right = CCDData(np.concatenate([top_right, bot_right]),
unit=u.electron,
header=header_right)
right = ccdproc.flat_correct(right, flat_right[209:3903,
1149:2947])
log.info('Right image proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(right, nsigma=3, npixels=5)
# clean, com_mask = create_mask.create_mask(sci,right,static_mask(proc)[1],mask,saturation(right.header),binning(proc,'right'),rdnoise(right.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
# processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(right, (120, 120),
filter_size=(3, 3),
sigma_clip=SigmaClip(sigma=3),
bkg_estimator=MeanBackground(),
mask=mask,
exclude_percentile=80)
log.info('Median background for right image : ' +
str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/',
'/red/').replace('.fits',
'_bkg_right.fits'),
bkg.background,
overwrite=True)
right = right.subtract(CCDData(bkg.background, unit=u.electron),
propagate_uncertainties=True,
handle_meta='first_found')
log.info('Exposure time of right image is ' +
str(right.header['EXPTIME']))
right = right.divide(right.header['EXPTIME'] * u.second,
propagate_uncertainties=True,
handle_meta='first_found')
log.info(
'Background subtracted and image divided by exposure time.')
right.header['DATASEC'] = '[1:1798,1:3694]'
right.header['RADECSYS'] = 'ICRS'
right.header['CUNIT1'] = 'deg'
right.header['CUNIT2'] = 'deg'
right.header['CTYPE1'] = 'RA---TAN'
right.header['CTYPE2'] = 'DEC--TAN'
right.header['CRPIX1'] = -504
right.header['CRPIX2'] = 1845
coord = util.parse_coord(right.header['RA'], right.header['DEC'])
right.header['CRVAL1'] = coord.ra.deg
right.header['CRVAL2'] = coord.dec.deg
right.header['PC1_1'] = -pixscale() / 3600 * np.sin(
np.pi / 180. * (right.header['POSANG'] + 90))
right.header['PC1_2'] = pixscale() / 3600 * np.cos(
np.pi / 180. * (right.header['POSANG'] + 90))
right.header['PC2_1'] = -pixscale() / 3600 * np.cos(
np.pi / 180. * (right.header['POSANG'] + 90))
right.header['PC2_2'] = pixscale() / 3600 * np.sin(
np.pi / 180. * (right.header['POSANG'] + 90))
right.write(sci.replace('/raw/',
'/red/').replace('.fits', '_right.fits'),
overwrite=True)
right_list.append(right)
return [left_list, right_list], None
def process_imaging_science(self, imaging_group):
"""Does image reduction for science imaging data.
Args:
imaging_group (object): :class:`~pandas.DataFrame` instance that contains a
list of science data that are compatible with a given
instrument configuration and can be reduced together.
"""
# pick a random image in order to get a header
random_image = random.choice(imaging_group.file.tolist())
path_random_image = os.path.join(self.args.raw_path, random_image)
sample_file = CCDData.read(path_random_image, unit=u.adu)
master_flat_name = self.name_master_flats(header=sample_file.header,
group=imaging_group,
get=True)
self.log.debug(
'Got {:s} for master flat name'.format(master_flat_name))
master_flat, master_flat_name = get_best_flat(
flat_name=master_flat_name, path=self.args.red_path)
if master_flat is not None:
for image_file in imaging_group.file.tolist():
# start with an empty prefix
self.out_prefix = ''
image_full_path = os.path.join(self.args.raw_path, image_file)
ccd = read_fits(image_full_path, technique=self.technique)
# Trim image
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
self.out_prefix = 't_'
if not self.args.ignore_bias:
ccd = ccdproc.subtract_bias(ccd,
self.master_bias,
add_keyword=False)
self.out_prefix = 'z' + self.out_prefix
ccd.header['GSP_BIAS'] = (os.path.basename(
self.master_bias_name), 'Master bias image')
# apply flat correction
ccd = ccdproc.flat_correct(ccd, master_flat, add_keyword=False)
self.out_prefix = 'f' + self.out_prefix
ccd.header['GSP_FLAT'] = (os.path.basename(master_flat_name),
'Master Flat image')
if self.args.clean_cosmic:
ccd = astroscrappy_lacosmic(
ccd=ccd,
red_path=self.args.red_path,
save_mask=self.args.keep_cosmic_files)
self.out_prefix = 'c' + self.out_prefix
else:
print('Clean Cosmic ' + str(self.args.clean_cosmic))
final_name = os.path.join(self.args.red_path,
self.out_prefix + image_file)
# ccd.write(final_name, clobber=True)
write_fits(ccd=ccd, full_path=final_name)
self.log.info('Created science file: {:s}'.format(final_name))
else:
self.log.error('Can not process data without a master flat')
def reduce_night(science_collection, dark_collection, flat_collection, config, config_arguments):
"""
This function reduce science data of one night and save the results to a folder named "reduced". The reduction
is performed as follows:
- Create a list of masterdarks (each masterdark has a different value of the exposure time) ^1
- Create a list of masterflats (each masterflat has a different value of the filter) ^1
- Reduce the science data as follows:
*For each filter:
*For each exposure time with that filter:
-- Look for the masterdark with the nearest exposure time
-- Look for the masterflat with the current filter.
-- Substract masterdark
-- Flat field correct the data
-- Clean cosmic rays (if requested)
-- Save image to ./Calibrated folder
(1) The master(flat/dark)s are created using mean combine.
:param science_collection: Numpy array - A numpy array with the science collection data produced by FitsLookup.
:param dark_collection: Numpy array - A numpy array with the dark collection data produced by FitsLookup.
:param flat_collection: Numpy array - A numpy array with the flat collection data produced by FitsLookup.
:param config_values: Dictionary - Dictionary - A dictionary provided by the function get_config_dict that
contains the config of the fits files ( readed from conf.INI ).
:param config_arguments: Dictionary - A dictionary provided by argparse initialization that contain the current flags.
:return: Integer - 0 if no errors raised 1 if errors raised.
"""
# Supress astropy warnings
warnings.filterwarnings('ignore')
# Renaming some config_arguments for easy acess
work_dir = config_arguments.dir[0]
# Get the filter and exposure collection of science and flat images
science_filter_collection = set(science_collection['filter'])
science_exposures_collection = set(science_collection['exptime'])
dark_exposures_collection = set(dark_collection['exptime'])
flat_filter_collection = set(flat_collection['filter'])
# Inform the user of the filter / exptime found.
science_exp_times_as_string = ", ".join(
[str(x) for x in science_exposures_collection])
dark_exp_times_as_string = ", ".join(
[str(x) for x in dark_exposures_collection])
module_logger.info("We have found {0} filters in the science images: {1}".format(
len(science_filter_collection), ", ".join(science_filter_collection)))
module_logger.info("We have found {0} exposure times science images: {1}".format(
len(science_exposures_collection), science_exp_times_as_string))
module_logger.info("We have found {0} exposure times dark calibrators: {1}".format(
len(dark_exposures_collection), dark_exp_times_as_string))
module_logger.info("We have found {0} filters in the flat calibrators {1}".format(
len(flat_filter_collection), ", ".join(flat_filter_collection)))
# Check if we have the same filters in flats and science, if not, get the
# intersection
if not science_filter_collection.issubset(flat_filter_collection):
module_logger.warning(
"There are more filters in the science images than in the flat calibrators")
science_filter_collection = science_filter_collection.intersection(
flat_filter_collection)
module_logger.warning("Triying to work with common filters.")
module_logger.info("We have found {0} common filters in the science images: {1}".format(
len(science_filter_collection), ", ".join(science_filter_collection)))
if not science_filter_collection:
module_logger.warning(
"There are no common filters between science images and flat calibrators")
module_logger.warning("This night will be skiped.")
return 1
# Warn the user if we found science images of 0 seconds
if 0 in science_exposures_collection:
number_of_null_images = len(filter_collection(
science_collection, [('exptime', 0)]))
module_logger.warning(
"We have found {0} science images with 0 seconds of exposure time.".format(number_of_null_images))
science_exposures_collection.discard(0)
module_logger.warning(
"Discarding images with 0 seconds of exposure time for this night: {0} exposure(s) remain.".format(
len(science_exposures_collection)))
# ------- MASTER DARK CREATION --------
module_logger.info("Starting the creation of the master dark")
module_logger.info("{0} different exposures for masterdarks were found.".format(
len(dark_exposures_collection)))
master_dark_collection = dict()
# Loop over each exposure time.
for dark_exposure_item in dark_exposures_collection:
module_logger.info("Creating masterdark with exposure of {0}s".format(dark_exposure_item))
# Initializate dark list for current collection.
exposure_dark_list = list()
for dark_image_data in filter_collection(dark_collection, [('exptime', dark_exposure_item)]):
# Open the images and append to the dark list
dark_image = dark_image_data['filename']
ccd = CCDData.read(dark_image, unit=config.image_units)
# If we have overscan, subtract and trim.
if config.subtract_overscan:
if config_arguments.verbose_flag_2:
module_logger.info("Subtracting overscan of {0}".format(dark_image))
ccd = subtract_and_trim_overscan(ccd, config)
exposure_dark_list.append(ccd)
# median combine the data
cb = ccdproc.Combiner(exposure_dark_list)
master_dark = cb.median_combine(median_func=np.median)
# Add the masterdark to the master_flat collection
master_dark_collection.update({dark_exposure_item: master_dark})
# Save the masterdark if needed.
if config.save_masterdark:
# Filename to save
aux = '{0}/masterdark_{1}.fits'.format(config_arguments.save_path, dark_exposure_item)
module_logger.info('Saving master dark to {0}'.format(aux))
master_dark.to_hdu().writeto(aux)
# ------- MASTER FLAT CREATION --------
module_logger.info("Starting the creation of the master flats")
module_logger.info("{0} different filters for masterflats were found".format(
len(flat_filter_collection)))
master_flat_collection = dict()
# Go thought the different filters in the collection
for flat_filter in flat_filter_collection:
module_logger.info(
"Creating masterflat with filter {0}".format(flat_filter))
# Initializate the list that will carry the flat images of the actual
# filter
filter_flat_list = list()
for flat_image_data in filter_collection(flat_collection, [('filter', flat_filter)]):
# Open the images and append to the filter's flat list
flat_image = flat_image_data['filename']
ccd = CCDData.read(flat_image, unit=config.image_units)
# Subtract and trim overscan
if config.subtract_overscan:
if config_arguments.verbose_flag_2:
module_logger.info("Subtracting overscan of {0}".format(flat_image))
ccd = subtract_and_trim_overscan(ccd, config)
filter_flat_list.append(ccd)
# median combine the flats after scaling each by its mean
cb = ccdproc.Combiner(filter_flat_list)
cb.scaling = lambda x: 1.0 / np.mean(x)
master_flat = cb.median_combine(median_func=np.median)
# Add the masterflat to the master_flat collection
master_flat_collection.update({flat_filter: master_flat})
# Save the masterflat if needed.
if config.save_masterflat:
aux = '{0}/masterflat_{1}.fits'.format(config_arguments.save_path, flat_filter)
module_logger.info('Saving master flat to {0}'.format(aux))
master_flat.to_hdu().writeto(aux)
# ------- REDUCE SCIENCE DATA --------
module_logger.info("Starting the calibration of the science images")
# Go thought the different files in the collection
for image_filter in science_filter_collection:
module_logger.info("Now calibrating filter: {0}".format(image_filter))
# Iterate thought each different exposure. This is because the dark files
# can have different exposures and the calibration must be performed with
# the masterdark with the nearest exposure time.
for science_exposure in science_exposures_collection:
# Important!! If you have more classifiers in the numpy dtype and you want
# to use them, you must modify the code here. For example, if you want to
# use a 'temp' value as classifier, after modify the dtype following the
# instructions in FitsLookup, you must add a loop here and modify the sub_collection.
# Once you have the 'temp' in the dtype, you must add a loop here as:
#
# >>>for temp_value in set(science_collection['temp']):
# module_logger.info("Now calibrating temp: {0}".format(temp_value))
#
# After this, you MUST indent all the following code (of this function) four spaces to
# the right, of course. Then, you only have to modify the science_subcollection as follows:
#
# >>> science_subcollection = filter_collection(
# science_collection, [('filter', image_filter),
# ('exptime', science_exposure),
# ('temp', temp_value) ])
#
# Follow this steps for every classifier you want to add. Yay!
# --------------------------------------------------------------
# Science subcollection is a really bad name, but is descriptive. Remember that this subcollection
# are the images with the current filter that has the current
# exposure time. E.g. ('r' and 20', 'r' and 30).
science_subcollection = filter_collection(
science_collection, [('filter', image_filter),
('exptime', science_exposure)])
# Continue if we have files to process. This will check if for some filter
# there are not enought images with the actual exposure time.
if science_subcollection.size:
module_logger.info(
"Now calibrating exposure: {0}".format(science_exposure))
# Determine if we have a masterdark with the science exposure file.
#
# - If we have a exposure matching masterdark, use it.
# - If we do not have a exposure matching masterdark, use the nearest.
try:
selected_masterdark = master_dark_collection[science_exposure]
nearest_exposure = 0, science_exposure
except KeyError:
# Get the nearest exoposure in the dark collection.
nearest_exposure = min(enumerate(master_dark_collection.keys()),
key=lambda x: abs(x[1] - science_exposure))
# Notice that nearest_exposure is a tuple of the form
# (index,exposure).
selected_masterdark = master_dark_collection[
nearest_exposure[1]]
# Initialize the progress bar variables
total_len = len(science_subcollection)
meantime = []
# Loop for each image with current (filter,exptime).
for contador, science_image_data_with_current_exposure in enumerate(science_subcollection):
# To supress astropy warnings.
devnull = open(os.devnull, "w")
sys.stdout = devnull
# Notice that until sys stdout is reasigned, no printing
# will be allowed in the following lines.
# Start timing
start = time.time()
# Extract the filename from the image data
science_image = science_image_data_with_current_exposure['filename']
# Read the image
ccd = CCDData.read(science_image, unit=config.image_units, wcs=None)
# Subtract overscan
if config.subtract_overscan:
if config_arguments.verbose_flag_2:
module_logger.info("Subtracting overscan of {0}".format(science_image))
ccd = subtract_and_trim_overscan(ccd, config)
# Master dark substraction
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info("Subtracting dark of image {0} of {1}".format(contador + 1, total_len))
sys.stdout = devnull
else:
module_logger.debug("Subtracting dark of image {0} of {1}".format(contador + 1, total_len))
selected_masterdark._wcs = ccd._wcs # FIXME: currently needed due to bug
ccd = ccdproc.subtract_dark(ccd, selected_masterdark, dark_exposure=nearest_exposure[1] * u.second,
data_exposure=science_exposure * u.second)
# flat-field correct the data
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info("Flat-field correcting image {0} of {1}".format(contador + 1, total_len))
sys.stdout = devnull
else:
module_logger.debug("Flat-field correcting image {0} of {1}".format(contador + 1, total_len))
current_master_flat = master_flat_collection[image_filter]
current_master_flat._wcs = ccd._wcs # FIXME: currently needed due to bug
ccd = ccdproc.flat_correct(ccd, current_master_flat)
# If we need to clean cosmic rays, do it.
if config_arguments.cosmic_flag:
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info(
"Cosmic ray cleaning of image {0} of {1}".format(contador + 1, total_len))
sys.stdout = devnull
else:
module_logger.debug(
"Cosmic ray cleaning of image {0} of {1}".format(contador + 1, total_len))
ccd = ccdproc.cosmicray_lacosmic(ccd, error_image=None, thresh=5, mbox=11, rbox=11, gbox=5)
# Save the calibrated image to a file
output_filename = os.path.join(config_arguments.save_path, os.path.basename(science_image))
if config_arguments.verbose_flag_2:
sys.stdout = sys.__stdout__ # Restart stdout printing
module_logger.info(
"Saving image {0} of {1} to {2}".format(contador + 1, total_len, output_filename))
sys.stdout = devnull
else:
module_logger.debug(
"Saving image {0} of {1} to {2}".format(contador + 1, total_len, output_filename))
ccd.write(output_filename, clobber=True)
end = time.time()
meantime.append(end - start)
sys.stdout = sys.__stdout__ # Restart stdout printing
# Progressbar in case that we have not activated the no_interaction flag nor the advance
# verbose flag.
if not config_arguments.no_interaction and not config_arguments.verbose_flag_2:
if config_arguments.verbose_flag:
update_progress(float(contador + 1) / total_len,
np.mean(meantime) * (total_len - (contador + 1)))
return 0
def correctData(filename, master_bias, master_flat, filetype):
"""
Correct a science image using the available
master calibrations. Skip a calibration step if the
master frame does not exist.
No reduced file is written in this new scheme.
Instead, the corrected data is passed directly
to the phot() routine, photometry is done as per
the configuration and the photometry is written out
only.
TODO: Finish docstring
"""
print('Reducing {0:s}...'.format(filename))
with fits.open(filename) as fitsfile:
# correct times for science spectra,
# don't bother for arcs
hdr = fitsfile[0].header
if filetype == 'science':
half_exptime = hdr[EXPTIME_KEYWORD]/2.
utstart = hdr[UTSTART_KEYWORD]
dateobs = hdr[DATEOBS_KEYWORD]
ra = hdr[RA_KEYWORD]
dec = hdr[DEC_KEYWORD]
time_start = Time('{}T{}'.format(dateobs, utstart),
scale='utc',
format='isot',
location=OBSERVATORY)
# correct to mid exposure time
jd_mid = time_start + half_exptime*u.second
ltt_bary, ltt_helio = getLightTravelTimes(ra, dec, jd_mid)
time_bary = jd_mid.tdb + ltt_bary
time_helio = jd_mid.utc + ltt_helio
hdr['BJD-MID'] = time_bary.jd
hdr['HJD-MID'] = time_helio.jd
hdr['JD-MID'] = jd_mid.jd
hdr['UT-MID'] = jd_mid.isot
ccd = CCDData.read(filename, unit=u.adu)
if master_bias:
ccd = subtract_bias(ccd, master_bias)
else:
print('No master bias, skipping correction...')
if master_flat:
ccd = flat_correct(ccd, master_flat)
else:
print('No master flat, skipping correction...')
# after calibrating we get np.float64 data
# if there are no calibrations we maintain dtype = np.uint16
# sep weeps
# fix this by doing the following
if isinstance(ccd.data[0][0], np.uint16):
ccd.data = ccd.data.astype(np.float64)
# trim the data
ccd_trimmed = trim_image(ccd[1000:3001, :])
# write out the trimmed file and the updated header
#ccd_trimmed.write(filename, hdr, clobber=True)
trimmed_filename = '{}_t.fits'.format(filename.split('.')[0])
fits.writeto(trimmed_filename, ccd_trimmed.data, hdr)
# remove the old untrimmed data
os.system('rm {}'.format(filename))
from os import listdir
bias_hdu = ccdproc.CCDData.read('data_bias/master_bias.fits', unit='adu',)
flat_hdu_i = ccdproc.CCDData.read('data_flat/i/master_flat.fits', unit='adu')
flat_hdu_v = ccdproc.CCDData.read('data_flat/v/master_flat.fits', unit='adu')
print "Now cleaning I"
#Band I
subtracted_flat_i = flat_hdu_i
names_i = listdir('./data_obj/i/')
images_i = [ccdproc.CCDData.read('data_obj/i/'+i, unit='adu') for i in listdir('./data_obj/i')]
for i in range(len(names_i)):
subt_obj = ccdproc.subtract_bias(images_i[i], bias_hdu)
corr_obj = ccdproc.flat_correct(subt_obj, subtracted_flat_i, min_value=0.000000001)
corr_obj.write('./data_corr/i/'+names_i[i])
subt_obj.write('./data_biascorr/i/'+names_i[i])
print "Now cleaning V"
#Band V
subtracted_flat_v = flat_hdu_v
names_v = listdir('./data_obj/v/')
images_v = [ccdproc.CCDData.read('data_obj/v/'+i, unit='adu') for i in listdir('./data_obj/v')]
for i in range(len(names_v)):
subt_obj = ccdproc.subtract_bias(images_v[i], bias_hdu)
corr_obj = ccdproc.flat_correct(subt_obj, subtracted_flat_v, min_value=0.000000001)
corr_obj.write('./data_corr/v/'+names_v[i])
outpath = os.path.join(preprocessedpath, 'bias_master.fits')
saveCCDDataAndLog(outpath, biasmaster)
else:
for adarkexp in darkexp_set:
outpath = os.path.join(preprocessedpath, \
'dark_master_' + str(adarkexp) + 's.fits')
saveCCDDataAndLog(outpath, darklists[adarkexp])
outpath = os.path.join(preprocessedpath, 'flat_master.fits')
saveCCDDataAndLog(outpath, flatmaster)
for ascience in sciencelist:
try:
sci_image = ccdproc.CCDData.read(ascience, unit='adu')
if args.usebias:
sci_biassub = ccdproc.subtract_bias(sci_image, biasmaster)
sci_flatcorrected = ccdproc.flat_correct(sci_biassub, flatmaster)
else:
exp_time = fits.getval(ascience, 'exptime')
darkmaster = chooseClosestDark(darklists, exp_time)
sci_darksub = ccdproc.subtract_dark(sci_image, darkmaster, \
exposure_time='exptime', exposure_unit=u.second)
sci_flatcorrected = ccdproc.flat_correct(sci_darksub, flatmaster)
except:
logger.error("Couldn't reduce image %s." % (ascience))
continue
outpath = os.path.join(preprocessedpath, \
'preprocessed_' + os.path.basename(ascience))
#deadpixmaskfilename = "../stackImages/deadpix.fits"
deadpixmaskfilename = None
def preprocess(fits_path):
"""Apply basic CCD reduction tasks.
Ask for the path to all the calibration and science files, and perform Bias, Dark and Flat combinations and proper substractions.
Pipeline que aplica una reduccion basica a imagenes de tipo CCD.
Realiza combinacion de Bias, de Darks, y Flats. Luego realiza las restas inherentes.
"""
import os
import sys
from image_collection import ImageFileCollection
#Collect all FITS files in path in fitslist
fitslist = []
for root, dirs, files in os.walk(fits_path, topdown=True):
fitslist.extend(
[os.path.join(root, name) for name in files if '.fit' in name])
#Dictionary with the image type (value) for each image file name (key)
imagetypes = {img: fits.getval(img, 'IMAGETYP') for img in fitslist}
#Make different lists for different image types
biaslist = []
sciencelist = []
flatlist = []
darklist = []
unknown = []
for k, v in imagetypes.iteritems():
if 'LIGHT' in v.upper() or v.upper() == 'OBJECT':
sciencelist.append(k)
elif 'BIAS' in v.upper() or v.upper() == 'ZERO':
biaslist.append(k)
elif 'FLAT' in v.upper():
flatlist.append(k)
elif 'DARK' in v.upper():
darklist.append(k)
else:
unknown.append(k)
#Create the flat master
if darklist:
darkmaster = combineDarks(darklist_flat)
flatmaster = combineFlats(flatlist, dark=darkmaster)
elif biaslist:
biasmaster = combineBias(biaslist)
flatmaster = combineFlats(flatlist, bias=biasmaster)
else:
flatmaster = combineFlats(flatlist)
for ascience in sciencelist:
sci_image = ccdproc.CCDData.read(ascience, unit='adu')
sci_darksub = ccdproc.subtract_dark(sci_image,
darkmaster,
exposure_time='exptime',
exposure_unit=u.second)
sci_flatcorrected = ccdproc.flat_correct(sci_darksub, flatmaster)
outpath = os.path.join('/Users/utb/Desktop/reduced/',
'reduced_' + os.path.basename(ascience))
hdulist = sci_flatcorrected.to_hdu()
hdulist.writeto(outpath, clobber=True)
return
def clean_the_images(path, filename):
#ast=AstrometryNet()
#ast.api_key= 'iqmqwvazpvolmjmn'
dir = path
gain = 2 * u.electron / u.adu
readnoise = 7.5 * u.electron
ra = input('Enter the RA of the source: ')
dec = input('Enter the DEC of the source: ')
'''
wcs_header=ast.solve_from_image(path+filename)
wcs=WCS(wcs_header)
ran,decn=wcs.all_pix2world(1024,1024,0)
print(ran,decn)
'''
file_name = os.path.join(dir, filename)
image = ccdproc.CCDData.read(file_name, unit='adu')
header = fits.getheader(file_name, 0)
time = header['DATE']
t = Time(time, format='isot', scale='utc')
print(t.jd, t.mjd)
header.insert(15, ('RA', ra))
header.insert(16, ('DEC', dec))
a = sorted(glob(os.path.join(dir, 'bias*.fits')))
biaslist = []
for i in range(0, len(a)):
data = ccdproc.CCDData.read(a[i], unit='adu')
#data = ccdproc.create_deviation(data, gain=gain, readnoise=readnoise)
#data= data-(data.uncertainty.array)
biaslist.append(data)
combiner = ccdproc.Combiner(biaslist)
masterbias = combiner.median_combine()
masterbias.write('masterbias.fit', overwrite=True)
mbias = ccdproc.CCDData.read('masterbias.fit', unit='adu')
#masterbias.meta=image.meta
print('master bias generated')
print(np.mean(masterbias), np.median(masterbias))
c = sorted(glob(os.path.join(dir, 'flat*.fits')))
flatlist = []
for j in range(0, len(c)):
flat = ccdproc.CCDData.read(c[j], unit='adu')
#flat= ccdproc.create_deviation(flat, gain=gain, readnoise=readnoise)
flat = ccdproc.subtract_bias(flat, masterbias)
flatlist.append(flat)
combiner = ccdproc.Combiner(flatlist)
masterflat = combiner.median_combine()
masterflat.write('masterflat.fits', overwrite=True)
mflat = ccdproc.CCDData.read('masterflat.fits', unit='adu')
print('master flat generated')
print(np.mean(masterflat), np.median(masterflat))
#masterflat.meta=image.meta
bias_subtracted = ccdproc.subtract_bias(image, masterbias)
flat_corrected = ccdproc.flat_correct(bias_subtracted, masterflat)
cr_cleaned = ccdproc.cosmicray_lacosmic(flat_corrected,
readnoise=7.5,
sigclip=5)
print('cosmic ray removed')
fits.writeto(dir + 'j_0947_i_1_clean.fits',
cr_cleaned,
header,
overwrite=True)
print('image cleaned')
for image, imname in mastercollection.ccds(imtype='subflat',return_fname=True):
trimage=ccdp.trim_image(image,fits_section=str(sciwin))
trimage.meta['trimwind']=(str(sciwin),'readout window')
trimage.meta['imtype'] = ('mflat', 'windowed master flat')
trimage.write(path+imname,overwrite=True)
tmastercollection=ImageFileCollection(cmpath)
'''check shape'''
#check shape
# print(sci.data.shape)
# for master in tmastercollection.ccds():
# print(master.data.shape)
crop_masters()
for mbias, mbiasn in tmastercollection.ccds(imtype='mbias', combined='sigma_clip average',return_fname=True):
print('using',mbiasn)
for mflat, mflatn in tmastercollection.ccds(imtype='mflat', flatcom='sigma', return_fname=True):
print('using', mflatn)
start = time.time()
for sci, scin in sciencecollection.ccds(return_fname=True, ccd_kwargs={'unit': 'adu'}):
print('correcting',scin)
subsci=ccdp.subtract_bias(sci,mbias,add_keyword={'subsci':'sigmbias'})
corsci=ccdp.flat_correct(subsci,mflat,norm_value=float(mflat.header['normmed']))
corsci.write(cspath+scin,overwrite=True)
print (time.time()-start)
os.remove(f)
#get a list of all FITS files in the input directory
fits_files=glob.glob(input_path+'*.fits')+glob.glob(input_path+'*.fit')
#loop through all qualifying files and perform plate-solving
logme('Calibrating images in %s' %input_path)
for fits_file in fits_files:
#open image
image = ccdproc.CCDData.read(fits_file, unit='adu', relax=True)
#trim it, if necessary
if(len(trim_range) > 0):
image = ccdproc.trim_image(image, trim_range);
#subtract bias from light, dark, and flat frames
image = ccdproc.subtract_bias(image, bias, add_keyword=False)
image = ccdproc.subtract_dark(image, dark, scale=True, exposure_time=exposure_label, exposure_unit=u.second, add_keyword=False)
image = ccdproc.flat_correct(image, flat, add_keyword=False)
#save calibrated image
output_file = "%s"%(fits_file.rsplit('.',1)[0])+output_suffix+".fits"
output_file = output_file.rsplit('/',1)[1]
output_file = output_path+output_file
#scale calibrated image back to int16, some FITS programs don't like float
hdulist = image.to_hdu()
#bzero has to be non-zero, use int32 to handle negative values effectively
#hdulist[0].scale('int32', bzero=1)
hdulist[0].scale('int16', bzero=32768)
hdulist[0].header['BIASCORR'] = bias_master
hdulist[0].header['DARKCORR'] = dark_master
hdulist[0].header['FLATCORR'] = flat_master
if(len(trim_range) > 0):
hdulist[0].header['NAXIS1'] = '%d'%((naxis1_end-naxis1_start))
hdulist[0].header['NAXIS2'] = '%d'%((naxis2_end-naxis2_start))
def calibrate_images(
x_d,
x_f,
x_s,
it_s='object',
x_b='',
):
"""
Parameters
----------
----------
x_b : str
Path of the bias files.
By default bias is not provided
x_d : str
Path of the dark files
x_f : str
Path of flat files
x_s : str
Path of Science files
it_s : str
Imagetyp of fits science file
Default is set to object
-----------
"""
path_b = Path(x_b)
path_d = Path(x_d)
path_f = Path(x_f)
path_s = Path(x_s)
#-----Bias
if x_b == '' and path_b == Path(''):
print(
'Be aware: You did not provide the Bias files; the process will still continue though.'
)
files_b = None
elif not path_b.is_dir():
raise RuntimeError(
'The path you provided for the Bias files does not exist.')
else:
files_b = ccdp.ImageFileCollection(path_b)
#-----Dark
if x_d == '' or not path_d.is_dir():
raise RuntimeError(
'You must provide Dark files for processing.\n Or the path you provided does not exist.'
)
else:
files_d = ccdp.ImageFileCollection(path_d)
#-----Flat
if x_f == '' or not path_f.is_dir():
raise RuntimeError(
'You must provide Flatfield files for processing.\n Or the path you provided does not exist.'
)
else:
files_f = ccdp.ImageFileCollection(path_f)
#-----Science
if x_s == '' or not path_s.is_dir():
raise RuntimeError(
'You must provide Science images for processing.\n Or the path you provided does not exist.'
)
else:
files_s = ccdp.ImageFileCollection(path_s)
#-----------------------------------
#
#--------Calibrating Images---------
#
#-----------------------------------
if files_b is not None:
#-------------------------------
#------Creating Master-bias-----
#-------------------------------
cali_bias_path = Path(path_b / 'cali_bias')
cali_bias_path.mkdir(exist_ok=True)
files_b_cali = files_b.files_filtered(imagetyp='bias',
include_path=True)
combined_bias = ccdp.combine(files_b_cali,\
method='average',\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_bias.meta['combined'] = True
combined_bias.write(cali_bias_path / 'master_bias.fits')
# Reading master bias
master_bias = CCDData.read(cali_bias_path / 'master_bias.fits')
else:
master_bias = None
#-------------------------------
#-------Calibrating Darks-------
#-------------------------------
cali_dark_path = Path(path_d / 'cali_dark')
cali_dark_path.mkdir(exist_ok=True)
files_d_cali = files_d.files_filtered(imagetyp='DARK', include_path=True)
for ccd, file_name in files_d.ccds(imagetyp='DARK',
return_fname=True,
ccd_kwargs={'unit': 'adu'}):
if master_bias is not None:
# Subtract bias
ccd = ccdp.subtract_bias(ccd, master_bias)
else:
ccd = ccd
# Save the result
ccd.write(cali_dark_path / file_name)
#--------------------------------
#------Creating Master-Dark------
#--------------------------------
red_dark = ccdp.ImageFileCollection(cali_dark_path)
# Calculating exposure times of DARK images
dark_times = set(
red_dark.summary['exptime'][red_dark.summary['imagetyp'] == 'DARK'])
for exposure in sorted(dark_times):
cali_darks = red_dark.files_filtered(imagetyp = 'dark',\
exptime = exposure,\
include_path = True)
combined_dark = ccdp.combine(cali_darks,\
method='average',\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_dark.meta['combined'] = True
com_dark_name = 'combined_dark_{:6.3f}.fits'.format(exposure)
combined_dark.write(cali_dark_path / com_dark_name)
# Reading master dark of various exposure times
red_dark = ccdp.ImageFileCollection(cali_dark_path)
combined_darks = {
ccd.header['exptime']: ccd
for ccd in red_dark.ccds(imagetyp='DARK', combined=True)
}
#--------------------------------
#-------Calibrating Flats--------
#--------------------------------
cali_flat_path = Path(path_f / 'cali_flat')
cali_flat_path.mkdir(exist_ok=True)
files_f_cali = files_f.files_filtered(imagetyp='FLAT', include_path=True)
for ccd, file_name in files_f.ccds(imagetyp='FLAT',
ccd_kwargs={'unit': 'adu'},
return_fname=True):
# Subtract bias
if master_bias is not None:
ccd = ccdp.subtract_bias(ccd, master_bias)
else:
ccd = ccd
closest_dark = utl.find_nearest_dark_exposure(ccd, dark_times)
if closest_dark is None:
closest_dark1 = utl.find_nearest_dark_exposure(ccd,
dark_times,
tolerance=100)
# Subtract scaled Dark
ccd = ccdp.subtract_dark(ccd, combined_darks[closest_dark1],\
exposure_time = 'exptime',\
exposure_unit = u.second,\
scale = True)
ccd.write(cali_flat_path / ('flat-' + file_name))
else:
closest_dark2 = utl.find_nearest_dark_exposure(ccd, dark_times)
# Subtracting Darks
ccd = ccdp.subtract_dark(ccd,
combined_darks[closest_dark2],
exposure_time='exptime',
exposure_unit=u.second)
ccd.write(cali_flat_path / ('flat-' + file_name))
#--------------------------------
#-----Creating Master-Flat-------
#--------------------------------
red_flats = ccdp.ImageFileCollection(cali_flat_path)
cali_flats = red_flats.files_filtered(imagetyp='FLAT', include_path=True)
combined_flat = ccdp.combine(cali_flats,\
method='average',\
scale = utl.inverse_median,\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_flat.meta['combined'] = True
combined_flat.write(cali_flat_path / 'master_flat.fits')
# Reading master flat
red_flats = ccdp.ImageFileCollection(cali_flat_path)
combined_flat = CCDData.read(cali_flat_path / 'master_flat.fits')
#--------------------------------
#---Calibrating Science Images---
#--------------------------------
# Creating a list of spectrum images
files_spec = files_s.summary['file', 'view_pos']
files_spec_list = np.array([])
for i in range(len(files_spec)):
xxx = files_spec['view_pos'][i]
if xxx[0:4] == 'open':
files_spec_list = np.hstack(
(files_spec_list, files_spec['file'][i]))
# Sky subtracting images
cali_science_path1 = Path(path_s / 'Sky_subtracted_science')
cali_science_path1.mkdir(exist_ok=True)
j = 0
for i in range(int(len(files_spec_list) / 2)):
ccd1 = CCDData.read(x_s + files_spec_list[j], unit='adu')
ccd2 = CCDData.read(x_s + files_spec_list[j + 1], unit='adu')
sky_sub1 = ccd1.data - ccd2.data
ss1 = CCDData(sky_sub1, unit='adu')
ss1.header = ccd1.header
ss1.meta['sky_sub'] = True
name1 = 'sky_sub_' + files_spec_list[j]
ss1.write(cali_science_path1 / name1)
sky_sub2 = ccd2.data - ccd1.data
ss2 = CCDData(sky_sub2, unit='adu')
ss2.header = ccd2.header
ss2.meta['sky_sub'] = True
name2 = 'sky_sub_' + files_spec_list[j + 1]
ss2.write(cali_science_path1 / name2)
j = j + 2
files_s1 = ccdp.ImageFileCollection(cali_science_path1)
final_calibrated = Path(path_s / 'Final_calibrated_science')
final_calibrated.mkdir(exist_ok=True)
# Correcting for flat
for ccd, file_name in files_s1.ccds(imagetyp=it_s,
ccd_kwargs={'unit': 'adu'},
return_fname=True):
# Subtract scaled Dark
ccd = ccdp.flat_correct(ccd, combined_flat) #['FLAT'])
ccd.write(final_calibrated / file_name)
def bdf_process(ccd,
output=None,
mbiaspath=None,
mdarkpath=None,
mflatpath=None,
fits_section=None,
calc_err=False,
unit='adu',
gain=None,
rdnoise=None,
gain_key="GAIN",
rdnoise_key="RDNOISE",
gain_unit=u.electron / u.adu,
rdnoise_unit=u.electron,
dark_exposure=None,
data_exposure=None,
exposure_key="EXPTIME",
exposure_unit=u.s,
dark_scale=False,
min_value=None,
norm_value=None,
verbose=True,
output_verify='fix',
overwrite=True,
dtype="float32"):
''' Do bias, dark and flat process.
Parameters
----------
ccd: array-like
The ccd to be processed.
output: path-like
Saving directory
'''
proc = CCDData(ccd)
hdr_new = proc.header
if mbiaspath is None:
do_bias = False
# mbias = CCDData(np.zeros_like(ccd), unit=unit)
else:
do_bias = True
mbias = CCDData.read(mbiaspath, unit=unit)
hdr_new.add_history(f"Bias subtracted using {mbiaspath}")
if mdarkpath is None:
do_dark = False
mdark = None
else:
do_dark = True
mdark = CCDData.read(mdarkpath, unit=unit)
hdr_new.add_history(f"Dark subtracted using {mdarkpath}")
if dark_scale:
hdr_new.add_history(
f"Dark scaling {dark_scale} using {exposure_key}")
if mflatpath is None:
do_flat = False
# mflat = CCDData(np.ones_like(ccd), unit=unit)
else:
do_flat = True
mflat = CCDData.read(mflatpath)
hdr_new.add_history(f"Flat corrected using {mflatpath}")
if fits_section is not None:
proc = trim_image(proc, fits_section)
mbias = trim_image(mbias, fits_section)
mdark = trim_image(mdark, fits_section)
mflat = trim_image(mflat, fits_section)
hdr_new.add_history(f"Trim by FITS section {fits_section}")
if do_bias:
proc = subtract_bias(proc, mbias)
if do_dark:
proc = subtract_dark(proc,
mdark,
dark_exposure=dark_exposure,
data_exposure=data_exposure,
exposure_time=exposure_key,
exposure_unit=exposure_unit,
scale=dark_scale)
# if calc_err and verbose:
# if mdark.uncertainty is not None:
# print("Dark has uncertainty frame: Propagate in arithmetics.")
# else:
# print("Dark does NOT have uncertainty frame")
if calc_err:
if gain is None:
gain = fu.get_from_header(hdr_new,
gain_key,
unit=gain_unit,
verbose=verbose,
default=1.).value
if rdnoise is None:
rdnoise = fu.get_from_header(hdr_new,
rdnoise_key,
unit=rdnoise_unit,
verbose=verbose,
default=0.).value
err = fu.make_errmap(proc, gain_epadu=gain, subtracted_dark=mdark)
proc.uncertainty = StdDevUncertainty(err)
errstr = (f"Error calculated using gain = {gain:.3f} [e/ADU] and " +
f"rdnoise = {rdnoise:.3f} [e].")
hdr_new.add_history(errstr)
if do_flat:
if calc_err:
if (mflat.uncertainty is not None) and verbose:
print("Flat has uncertainty frame: Propagate in arithmetics.")
hdr_new.add_history(
"Flat had uncertainty and is also propagated.")
proc = flat_correct(proc,
mflat,
min_value=min_value,
norm_value=norm_value)
proc = fu.CCDData_astype(proc, dtype=dtype)
proc.header = hdr_new
if output is not None:
proc.write(output, output_verify=output_verify, overwrite=overwrite)
return proc
def process_imaging_science(self, imaging_group):
"""Does image reduction for science imaging data.
Args:
imaging_group (object): :class:`~pandas.DataFrame` instance that contains a
list of science data that are compatible with a given
instrument configuration and can be reduced together.
"""
# pick a random image in order to get a header
random_image = random.choice(imaging_group.file.tolist())
path_random_image = os.path.join(self.args.raw_path, random_image)
sample_file = CCDData.read(path_random_image, unit=u.adu)
master_flat_name = self.name_master_flats(header=sample_file.header,
group=imaging_group,
get=True)
self.log.debug('Got {:s} for master flat name'.format(master_flat_name))
master_flat, master_flat_name = get_best_flat(
flat_name=master_flat_name,
path=self.args.red_path)
if master_flat is not None:
for image_file in imaging_group.file.tolist():
# start with an empty prefix
self.out_prefix = ''
image_full_path = os.path.join(self.args.raw_path, image_file)
ccd = read_fits(image_full_path, technique=self.technique)
# Trim image
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
self.out_prefix = 't_'
if not self.args.ignore_bias:
ccd = ccdproc.subtract_bias(ccd,
self.master_bias,
add_keyword=False)
self.out_prefix = 'z' + self.out_prefix
ccd.header['GSP_BIAS'] = (
os.path.basename(self.master_bias_name),
'Master bias image')
# apply flat correction
ccd = ccdproc.flat_correct(ccd,
master_flat,
add_keyword=False)
self.out_prefix = 'f' + self.out_prefix
ccd.header['GSP_FLAT'] = (
os.path.basename(master_flat_name),
'Master Flat image')
if self.args.clean_cosmic:
ccd = astroscrappy_lacosmic(
ccd=ccd,
red_path=self.args.red_path,
save_mask=self.args.keep_cosmic_files)
self.out_prefix = 'c' + self.out_prefix
else:
print('Clean Cosmic ' + str(self.args.clean_cosmic))
final_name = os.path.join(self.args.red_path,
self.out_prefix + image_file)
# ccd.write(final_name, clobber=True)
write_fits(ccd=ccd, full_path=final_name)
self.log.info('Created science file: {:s}'.format(final_name))
else:
self.log.error('Can not process data without a master flat')
