def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
# Find master bias to subtract from master dark
biaslist = self.loadauxname('bias', multi=False)
if (len(biaslist) == 0):
self.log.error('No bias calibration frames found.')
self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True)
# Create empy list for filenames of loaded frames
filelist = []
for fin in self.datain:
self.log.debug("Input filename = %s" % fin.filename)
filelist.append(fin.filename)
# Make a dummy dataout
self.dataout = DataFits(config=self.config)
if len(self.datain) == 0:
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file(s) loaded')
self.log.debug('Creating master flat frame...')
# Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias
if (len(filelist) == 1):
self.dark = ccdproc.CCDData.read(filelist[0],
unit='adu',
relax=True)
self.dark = ccdproc.subtract_bias(self.dark,
self.bias,
add_keyword=False)
else:
darklist = []
for i in filelist:
dark = ccdproc.CCDData.read(i, unit='adu', relax=True)
darksubbias = ccdproc.subtract_bias(dark,
self.bias,
add_keyword=False)
darklist.append(darksubbias)
self.dark = ccdproc.combine(darklist,
method=self.getarg('combinemethod'),
unit='adu',
add_keyword=True)
# set output header, put image into output
self.dataout.header = self.datain[0].header
self.dataout.imageset(self.dark)
# rename output filename
outputfolder = self.getarg('outputfolder')
if outputfolder != '':
outputfolder = os.path.expandvars(outputfolder)
self.dataout.filename = os.path.join(outputfolder,
os.path.split(filelist[0])[1])
else:
self.dataout.filename = filelist[0]
# Add history
self.dataout.setheadval('HISTORY',
'MasterDark: %d files used' % len(filelist))
def bias_file_by_file(fname, biasname, datahdus=0):
hdulist = fits.open(fname)
hdubias = fits.open(biasname)
nhdus = len(hdulist)
if nhdus > 1:
istart = 1
else:
istart = 0
hduindexes = list(range(nhdus))[istart:]
if datahdus != 0:
hduindexes = datahdus
for i in hduindexes:
data1 = ccdproc.CCDData(hdulist[i].data, unit="adu")
data1.header = hdulist[i].header
bias1 = ccdproc.CCDData(hdubias[i].data, unit="adu")
bias1.header = hdubias[i].header
commentstr = "Bias image is " + biasname
# proc1 = ccdproc.subtract_bias(data1,bias1,add_keyword={'bias': True, 'calstat': 'OTZ', 'history':commentstr} )
proc1 = ccdproc.subtract_bias(data1,
bias1,
add_keyword={
'bias': True,
'calstat': 'OTZ'
})
fits.update(fname, proc1.data, header=proc1.header, ext=i)
# fits.update(fname, proc1.data, ext=i)
hdulist.close()
hdubias.close()
mylog("Bias corrected {0} with {1}".format(fname, biasname))
return
def makeMasterFlat(images, master_bias):
"""
Flats are corrected for their bias level (if
master_bias)
TODO: Finish docstring
"""
try:
fitsfile = 'master_flat.fits'
master_flat = CCDData.read(fitsfile, unit=u.adu)
return master_flat
except FileNotFoundError:
# empty list for the flats
flat_list = []
# create the master flat field
print('Reducing flats')
for f in images.files_filtered(imagetyp=FLAT_KEYWORD):
print(f)
with fits.open(f) as fitsfile:
data_exp = fitsfile[0].header[EXPTIME_KEYWORD]
ccd = CCDData.read(f, unit=u.adu)
if master_bias:
ccd = subtract_bias(ccd, master_bias)
else:
print('No master bias, skipping correction...')
flat_list.append(ccd)
try:
master_flat = combine(flat_list, method='median')
master_flat.write('master_flat.fits', clobber=True)
return master_flat
except IndexError:
print('There are no flats, skipping...')
master_flat = None
def combinedark(cal_dir="../Data/20181207/cals", mast_dir=".", filt=[]):
#Generates master dark file from calibration directory - uses max exptime dark images
#Get master bias file
if os.path.isfile(mast_dir + "/master/master_bias.FIT") != True:
print "No master bias file"
return False
master_bias = CCDData.read(mast_dir + "/master/master_bias.FIT")
#Generate image list
imagelist = gen.getimages(cal_dir, filt=filt)
full_dark_list = []
for img in imagelist:
ccd = CCDData.read(cal_dir + '/' + img, unit=u.adu)
if ccd.header["IMAGETYP"].strip() == "Dark Frame":
img_exptime = ccd.header["EXPTIME"]
ccd = ccdproc.subtract_bias(ccd, master_bias)
full_dark_list.append([ccd, img_exptime])
print img
#Select only those with max exptime
exptime = max(full_dark_list, key=lambda x: x[1])[1]
dark_list = [x[0] for x in full_dark_list if x[1] == exptime]
if len(dark_list) == 0:
print "ERROR: no dark files"
return False
#Generate master file
master_dark = ccdproc.combine(dark_list, method='median', dtype="float32")
master_dark.write(mast_dir + "/master/master_dark.FIT", overwrite=True)
'''
print "Created master_dark file with " + str(exptime) + " exptime"
'''
return True
def sub_bias(refresh='2', bias='2'):
tflatcollection = ImageFileCollection('Trimmed_Flat')
if bias == '1':
biaspath = 'Master_Files/mbias_median.fits'
dest = 'Trimmed_Flat/subflatsmed/'
elif bias == '2':
biaspath = 'Master_Files/mbias.fits'
dest = 'Trimmed_Flat/subflatssig/'
if refresh == '1':
subflatpathlist = []
mbias = CCDData.read(biaspath, unit='adu')
for ccdf, flatn in tflatcollection.ccds(imtype='trimmed flat',
return_fname=True):
subflat = ccdp.subtract_bias(ccdf, mbias, add_keyword='subbias')
subflat.meta['imtype'] = ('subflat', 'bias subtracted flat')
subflat.write(dest + flatn[0:8] + '_subbias.fits', overwrite=True)
subflatpathlist.append(dest + flatn[0:8] + '_subbias.fits')
else:
try:
subflatcollection = ImageFileCollection(dest)
subflatpathlist = subflatcollection.files_filtered(
imtype='subflat', include_path=True)
print('found', len(subflatpathlist), 'subflats')
except:
print('can\'t locate subflats, create or check directory')
sys.exit()
return tflatcollection, subflatpathlist
def combineFlats(flatlist, dark=None, bias=None):
"""Combine all flat files into a flat master. Subtract dark or bias if provided."""
ccdflatlist = [
ccdproc.CCDData.read(aflat, unit="adu") for aflat in flatlist
]
if dark is not None and bias is None:
flat_sub = [
ccdproc.subtract_dark(aflat,
dark,
exposure_time='exptime',
exposure_unit=u.second)
for aflat in ccdflatlist
]
elif dark is None and bias is not None:
flat_sub = [
ccdproc.subtract_bias(aflat, bias) for aflat in ccdflatlist
]
else:
flat_sub = ccdflatlist
flatComb = ccdproc.Combiner(flat_sub)
flatComb.sigma_clipping(low_thresh=3, high_thresh=3, func=np.ma.median)
flatComb.scaling = lambda arr: 1. / np.ma.average(arr)
flatmaster = flatComb.average_combine()
return flatmaster
def bias_subtract(self):
"""
:return:
"""
if self.binning is None:
raise InputError('Binning not set.')
if self.debug:
print("Subtracting bias from remaining images...")
# Refresh the ImageFileCollection
self.icl.refresh()
# Load the appropriate bias frame to subtract
if not os.path.isfile(f'{self.path}/{self.zerofn}'):
self._biascombine(binning=self.binning)
try:
combined_bias = CCDData.read(f'{self.path}/{self.zerofn}')
except FileNotFoundError:
# Just skip the bias subtraction
print(f"Skipping bias subtraction for lack of {self.zerofn}")
return
# Set up a progress bar, so we can see how the process is going...
prog_bar = tqdm(total=len(self.icl.files),
unit='frame',
unit_scale=False,
colour='blue')
# Loop through files,
for ccd, file_name in self.icl.ccds(ccdsum=self.binning,
bitpix=16,
return_fname=True):
# Fit the overscan section, subtract it, then trim the image
ccd = _trim_oscan(ccd, self.biassec, self.trimsec)
# Subtract master bias
ccd = ccdp.subtract_bias(ccd, combined_bias)
# Update the header
ccd.header['HISTORY'] = PKG_NAME
ccd.header['HISTORY'] = 'Bias-subtracted image saved: ' + \
_savetime()
ccd.header['HISTORY'] = f'Subtracted bias: {self.zerofn}'
ccd.header['HISTORY'] = f'Original filename: {file_name}'
# Save the result (suffix = 'b'); delete input file
ccd.write(f'{self.path}/{file_name[:-5]}b{file_name[-5:]}',
overwrite=True)
os.remove(f'{self.path}/{file_name}')
# Update the progress bar
prog_bar.update(1)
# Close the progress bar, end of loop
prog_bar.close()
def loaddark(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master dark frame by ccdproc.
Also bias corrects dark files if not already done.
"""
#master dark frame
dark_is_bias_corrected = False
dark_bias = None
namelist = self.loadauxname('dark', multi=True)
if (len(namelist) == 0):
self.log.error('Dark calibration frame(s) not found.')
raise RuntimeError('No dark file loaded')
darks = None
for name in namelist:
#is (any) dark file bias corrected?
header = fits.getheader(name)
if (header.get('BIAS') != None):
dark_is_bias_corrected = True
dark_bias = header.get('BIAS')
elif (header.get('BIASCORR') != None):
dark_is_bias_corrected = True
dark_bias = header.get('BIASCORR')
if (darks):
darks += ',' + name
else:
darks = name
self.log.debug('Creating master dark frame...')
#if there is just one, use it as darkfile or else combine all to make a master dark
if (len(namelist) == 1):
self.dark = ccdproc.CCDData.read(namelist[0],
unit='adu',
relax=True)
else:
self.dark = ccdproc.combine(darks,
method='median',
unit='adu',
add_keyword=False,
**{'verify': 'ignore'})
#bias correct, if necessary
if (not dark_is_bias_corrected):
#Subtracting master bias frame from master dark frame
self.dark = ccdproc.subtract_bias(self.dark,
self.bias,
add_keyword=False)
else:
self.log.debug(
'Master dark frame is *already* bias corrected (%s).' %
dark_bias)
# Finish up
self.darkloaded = True
self.darkname = namelist[0]
self.log.debug('LoadDark: done')
def flat_creation(bdf_files, calibrated_path,calibrated_images, output_path, master_images, args):
set(bdf_files.summary['exptime'][bdf_files.summary['imagetyp'] == 'Flat Field'])
if args.cal_bias:
combined_bias = list(master_images.ccds(combined=True, imagetyp='Bias Frame'))[0]
if args.cal_dark:
combined_dark = CCDData.read(master_images.files_filtered(imagetyp='Dark Frame',
combined=True,
include_path=True)[0])
print('list of the flat files')
for a_flat, f_name in bdf_files.ccds(imagetyp='Flat Field', return_fname=True, ccd_kwargs={'unit': 'adu'}):
print(f_name)
if args.cal_bias:
print('BIAS')
a_flat = ccdp.subtract_bias(a_flat, combined_bias)
print('DONE BIAS')
if args.cal_dark:
print('DARK')
a_flat = ccdp.subtract_dark(a_flat, combined_dark, exposure_time='EXPOSURE', exposure_unit=u.s, scale=True)
print('done dark')
a_flat.write(calibrated_path / f_name, overwrite=True)
calibrated_images.refresh()
flats = calibrated_images.summary['imagetyp'] == 'Flat Field'
flat_filters = set(calibrated_images.summary['filter'][flats])
print('flat filters:', flat_filters)
for filtr in sorted(flat_filters):
calibrated_flats = calibrated_images.files_filtered(imagetyp='Flat Field', filter=filtr,
include_path=True)
print(len(calibrated_flats))
combined_flat = ccdp.combine(calibrated_flats,
method='median', scale=inv_median,
sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
signma_clip_dev_func=mad_std,
mem_limit=350e6)
combined_flat.meta['combined'] = True
flat_file_name = f'combined_flat_{filtr}.fit'
combined_flat.write(output_path / flat_file_name, overwrite=True)
calibrated_images.refresh()
master_images.refresh()
return flat_filters
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation.
"""
self.log.info(f"Running {self.__class__.__name__} action")
self.log.info(f" Found master bias file: {self.master_bias.name}")
kd_master_bias = fits_reader(self.master_bias, datatype=VYSOS20)
self.action.args.kd.pixeldata[0] = ccdproc.subtract_bias(
self.action.args.kd.pixeldata[0], kd_master_bias.pixeldata[0])
return self.action.args
def _combine_calib_images(
images: List[Image], bias: Optional[Image] = None, normalize: bool = False, method: str = "average"
) -> Image:
"""Combine a list of given images.
Args:
images: List of images to combine.
bias: If given, subtract from images before combining them.
normalize: If True, images are normalized to median of 1 before and after combining them.
method: Method for combining images.
"""
import ccdproc
# get CCDData objects
data = [image.to_ccddata() for image in images]
# subtract bias?
if bias is not None:
bias_data = bias.to_ccddata()
data = [ccdproc.subtract_bias(d, bias_data) for d in data]
# normalize?
if normalize:
data = [d.divide(np.median(d.data), handle_meta="first_found") for d in data]
# combine image
combined = ccdproc.combine(
data,
method=method,
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
mem_limit=350e6,
unit="adu",
combine_uncertainty_function=np.ma.std,
)
# normalize?
if normalize:
combined = combined.divide(np.median(combined.data), handle_meta="first_found")
# to Image and copy header
image = Image.from_ccddata(combined)
# add history
for i, src in enumerate(images, 1):
basename = src.header["FNAME"].replace(".fits.fz", "").replace(".fits", "")
image.header["L1AVG%03d" % i] = (basename, "Image used for average")
image.header["RLEVEL"] = (1, "Reduction level")
# finished
return image
def _perform(self):
"""
Returns an Argument() with the parameters that depend on this
operation.
"""
self.log.info(f"Running {self.__class__.__name__} action")
self.log.info(
f" Found master dark file: {self.master_dark_file.name}")
master_dark_ccddata = CCDData.read(self.master_dark_file, unit="adu")
self.log.info(f" Subtracting dark")
self.action.args.ccddata = ccdproc.subtract_bias(
self.action.args.ccddata, master_dark_ccddata)
return self.action.args
def make_master_flat(self, list_of_flatfiles):
flat_list = [0]*len(list_of_flatfiles)
for ii, kk in enumerate(list_of_flatfiles):
fitsfile = fits.open(kk)
flat = ccdproc.CCDData(data=fitsfile[1].data, meta=fitsfile[1].header, unit="adu")
flat_scan = ccdproc.subtract_overscan(flat, fits_section='[5:35, :]')
flat_bias = ccdproc.subtract_bias(flat_scan, self.master_bias)
flat_bias.data /= np.median(flat_bias.data[100:-100, 100:-100])
flat_list[ii] = flat_bias
self.master_flat = ccdproc.combine(flat_list, method="average", sigma_clip=True)
self.master_flat.write(self.output_dir+"/"+self.filename+"_masterflat.fits", overwrite=True)
def combineFlats(flatlist, dark=None, bias=None):
"""Combine all flat files into a flat master. Subtract dark or bias if provided."""
ccdflatlist = [ccdproc.CCDData.read(aflat, unit="adu") for aflat in flatlist]
if dark is not None and bias is None:
flat_sub = [ccdproc.subtract_dark(aflat, dark, exposure_time='exptime', exposure_unit=u.second) for aflat in ccdflatlist]
elif dark is None and bias is not None:
flat_sub = [ccdproc.subtract_bias(aflat, bias) for aflat in ccdflatlist]
else:
flat_sub = ccdflatlist
flatComb = ccdproc.Combiner(flat_sub)
flatComb.sigma_clipping(low_thresh=3, high_thresh=3, func=np.ma.median)
flatComb.scaling = lambda arr: 1./np.ma.average(arr)
flatmaster = flatComb.average_combine()
return flatmaster
def combineflat(cal_dir="../Data/20181207/cals", mast_dir=".", filt=[], binning=2):
#Generates master flat file from calibration directory
#Get master bias and dark files
if os.path.isfile(mast_dir + "/master/master_bias.FIT") != True:
print "No master bias file"
return False
if os.path.isfile(mast_dir + "/master/master_dark.FIT") != True:
print "No master dark file"
return False
master_bias = CCDData.read(mast_dir + "/master/master_bias.FIT")
master_dark = CCDData.read(mast_dir + "/master/master_dark.FIT")
#Generate image list
imagelist = gen.getimages(cal_dir, filt=filt)
flat_list = []
for img in imagelist:
ccd = CCDData.read(cal_dir + '/' + img, unit=u.adu)
if ccd.header["IMAGETYP"].strip() == "FLAT":
#Rebin images if needed
if ccd.header["XBINNING"] > binning:
print "ERROR: Binning too low"
return False
elif ccd.header["XBINNING"] < binning:
ccd.data = gen.rebin(ccd.data, oldbin=ccd.header["XBINNING"], newbin=binning)
ccd.header["XBINNING"] = binning
ccd.header["YBINNING"] = binning
#Remove bias and dark effects
ccd = ccdproc.subtract_bias(ccd, master_bias)
ccd = ccdproc.subtract_dark(ccd, master_dark, \
dark_exposure=master_dark.header["EXPTIME"]*u.s, \
data_exposure=ccd.header["EXPTIME"]*u.s, scale=True)
ccd.data = ccd.data/np.median(ccd.data)
flat_list.append(ccd)
if len(flat_list) == 0:
print "ERROR: no flat files"
return False
#Generate master file
master_flat = ccdproc.combine(flat_list, method='median', dtype="float32")
master_flat.write(mast_dir + "/master/master_flat_" + ".FIT", \
overwrite=True)
'''
print "Created master_flat file for " + filter_name + " filter"
'''
return True
def reduceframes(img_dir="../Data/20181207/imgs", mast_dir=".", mast_cal_dir=False):
#Removes effects from bias, dark, and flat master files and makes calibrated images
#Get master bias and dark frames - get flat later once have filter_name
if not mast_cal_dir:
mast_cal_dir = mast_dir
if os.path.isfile(mast_cal_dir + "/master/master_bias.FIT") != True:
print "No master bias file"
return False
if os.path.isfile(mast_cal_dir + "/master/master_dark.FIT") != True:
print "No master dark file"
return False
master_bias = CCDData.read(mast_cal_dir + "/master/master_bias.FIT")
master_dark = CCDData.read(mast_cal_dir + "/master/master_dark.FIT")
# if not os.path.exists(mast_dir + "/frames"):
# makedirs(mast_dir + "/frames")
#Reduce images
raw_image_names = gen.getimages(img_dir, filt=[])
for img in raw_image_names:
print img
ccd = CCDData.read(img_dir + '/' + img, unit=u.adu)
#Mask saturated pixels - not doing any more
'''mask_saturated = (ccd.data > 50000)
.data = np.array(ccd.data, dtype=np.float32)
.data[mask_saturated] = np.nan'''
ccd = ccdproc.subtract_bias(ccd, master_bias)
ccd = ccdproc.subtract_dark(ccd, master_dark, \
dark_exposure=master_dark.header["EXPTIME"]*u.s, \
data_exposure=ccd.header["EXPTIME"]*u.s, scale=True)
mean, background, std = sigma_clipped_stats(ccd.data, sigma=3.0, iters=5)
ccd.data = ccd.data - background
ccd.data = ccd.data/ccd.header["EXPTIME"]
ccd.unit = u.adu/u.s
#Add info about background and raw image name to header
ccd.header['SKY'] = background
ccd.header['RAWFILE'] = img
#Save calibrated frame
ccd.write(mast_dir + '/frames/' + img[:-4] + '-calibrated.FIT' , overwrite=True)
print "Created all calibrated frames in " + mast_dir + '/frames'
return True
def test_implicit_logging():
ccd_data = ccd_data_func()
# If nothing is supplied for the add_keyword argument then the following
# should happen:
# + A key named func.__name__ is created, with
# + value that is the list of arguments the function was called with.
bias = CCDData(np.zeros_like(ccd_data.data), unit="adu")
result = subtract_bias(ccd_data, bias)
assert "subtract_bias" in result.header
assert result.header['subtract_bias'] == (
'subbias', 'Shortened name for ccdproc command')
assert result.header['subbias'] == "ccd=<CCDData>, master=<CCDData>"
result = create_deviation(ccd_data, readnoise=3 * ccd_data.unit)
assert result.header['create_deviation'] == (
'creatvar', 'Shortened name for ccdproc command')
assert ("readnoise=" + str(3 * ccd_data.unit) in result.header['creatvar'])
def bias(self,im,superbias=None) :
""" Superbias subtraction
"""
# only subtract if we are given a superbias!
if superbias is None : return im
# work with lists so that we can handle multi-channel instruments
if type(im) is not list : ims=[im]
else : ims = im
if type(superbias) is not list : superbiases=[superbias]
else : superbiases = superbias
out=[]
for im,bias in zip(ims,superbiases) :
if self.verbose : print(' subtracting superbias...')
out.append(ccdproc.subtract_bias(im,bias))
if len(out) == 1 : return out[0]
else : return out
def reduceFrames(self):
self.getMedianForObjects()
log.info('Reducing frames started')
print 'Reducing frames started'
for obj in self.objects.filesList:
data = ccdproc.CCDData.read(obj, unit=u.adu)
dataWithDeviation = ccdproc.create_deviation(data,
gain=1.5*u.electron/u.adu,
readnoise=5*u.electron)
reducedObject = ccdproc.gain_correct(dataWithDeviation,
1.5*u.electron/u.adu)
if self.biases.isExists:
reducedObject = ccdproc.subtract_bias(reducedObject,
self.biases.masterCCD)
if self.darks.isExists:
reducedObject = ccdproc.subtract_dark(reducedObject,
self.darks.masterCCD,
exposure_time=cfg.exptime,
exposure_unit=u.second,
scale=True)
if self.flats.isExists:
reducedObject = ccdproc.flat_correct(reducedObject,
self.flats.masterCCD)
self.directory = '../../Reduction/' + directoryName
if not os.path.exists(self.directory):
os.makedirs(self.directory)
reducedObject.write(self.directory + '/' + obj, clobber=True)
os.system('solve-field ' + self.directory + '/' + obj) # + ' --overwrite')
objName, objExtension = os.path.splitext(self.directory + '/' + obj)
if not os.path.exists(objName + '.new'):
log.warning(objName + ' cannot be solved')
else:
newObjectsList.append(objName + '.new')
log.info('Frame ' + objName + ' reduced')
log.info('Reduced ' + str(len(newObjectsList)) + ' frames')
print 'Reduced ' + str(len(newObjectsList)) + ' frames'
self.clean()
def dark_creation(bdf_files, calibrated_path, calibrated_images, output_path,
master_images, args):
if args.cal_bias:
combined_bias = CCDData.read(
master_images.files_filtered(imagetyp='Bias Frame',
combined=True,
include_path=True)[0])
print('list of the dark files')
for ccd, f_name in bdf_files.ccds(imagetyp='Dark Frame',
return_fname=True,
ccd_kwargs={'unit': 'adu'}):
print(f_name)
if args.cal_bias:
ccd = ccdp.subtract_bias(ccd, combined_bias)
ccd.write(calibrated_path / f_name, overwrite=True)
calibrated_images.refresh()
darks = calibrated_images.summary['imagetyp'] == 'Dark Frame'
dark_times = set(calibrated_images.summary['exptime'][darks])
print('the exposure time of the darks:', dark_times)
for exp_time in sorted(dark_times):
calibrated_darks = calibrated_images.files_filtered(
imagetyp='Dark Frame', exptime=exp_time, include_path=True)
combined_dark = ccdp.combine(calibrated_darks,
method='median',
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
signma_clip_dev_func=mad_std,
mem_limit=350e6)
combined_dark.meta['combined'] = True
dark_file_name = 'combined_dark_{:6.3f}.fit'.format(exp_time)
combined_dark.write(output_path / dark_file_name, overwrite=True)
return combined_dark
def measure_xshift_2dcorr(imfps, bias_temp, flat_temp, xmax=12):
drifts = np.zeros(len(imfps))
cfn = sys._getframe().f_code.co_name
for i in trange(len(imfps),
ncols=100,
ascii=False,
desc="**" + cfn + "**",
unit=" file"):
data = ccdproc.CCDData.read(imfps[i], unit="adu")
data = ccdproc.subtract_bias(data, bias_temp)
shift, corr2d = thar_corr2d(data.data,
flat_temp.data,
xtrim=(512, 1536),
ytrim=(512, 1536),
x_shiftmax=xmax,
y_shiftmax=0,
verbose=False)
drifts[i] = np.int(shift[0])
return drifts
def correctscience():
for sciencec, sciencen in enumerate(sciencelist):
hdu = fits.open(sciencelist[sciencec])
print('open', sciencen)
if hdu[1].header['CHIPNAME'] == 'A5382-1-7':
ccdscience = CCDData(hdu[1].data, unit=u.adu)
if not os.path.exists('Corrected_Science'):
os.makedirs('Corrected_Science')
cccdscience = ccdproc.flat_correct(
ccdproc.subtract_bias(ccdscience, ccdmbias_use),
ccdmflat_use,
norm_value=np.nanmedian(ccdmflat_use))
path = 'Corrected_Science/' + sciencen[-12:]
cccdscience.write(path, overwrite=True)
hdu.close()
else:
hdu.close()
print(sciencen, 'returned wrong chipname for selected extension')
exit
def subtract_bias(textlist_files, master_bias='mbias.fits', prefix_str='bs_'):
"""
Subtract bias from the given files list
Args:
textlist_files : A python list object with paths/names to the individual files.
master_bias : Master bias used for subtraction (default 'mbias.fits')
prefix_str : String to be prefixed for subtracted files
Returns:
None
"""
master = CCDData.read(master_bias, unit=u.adu)
for filename in textlist_files:
ccd = CCDData.read(filename, unit=u.adu)
bias_subtracted = ccdp.subtract_bias(ccd=ccd, master=master)
bias_subtracted.meta['biassub'] = True
bias_subtracted.data = bias_subtracted.data.astype('float32')
bias_subtracted.write(prefix_str + filename,
hdu_mask=None,
hdu_uncertainty=None)
def make_science(self, list_of_sciencefiles):
science_list = [0]*len(list_of_sciencefiles)
science_names = [0]*len(list_of_sciencefiles)
for ii, kk in enumerate(list_of_sciencefiles):
fitsfile = fits.open(kk)
science = ccdproc.CCDData(data=fitsfile[1].data, meta=fitsfile[1].header, unit="adu")
# science_cos = ccdproc.cosmicray_lacosmic(science, verbose = True)
overscan_sub = ccdproc.subtract_overscan(science, fits_section='[5:35, :]')
bias_sub = ccdproc.subtract_bias(overscan_sub, self.master_bias)
flat_corr = ccdproc.flat_correct(bias_sub, self.master_flat)
# science_list[ii] = flat_corr
flat_corr.write(self.output_dir+"/"+self.filename+"_"+str(ii)+".fits", overwrite=True)
science_names[ii] = self.output_dir+"/"+self.filename+"_"+str(ii)+".fits"
# science_list = [0]*len(list_of_sciencefiles)
coverages = [0]*len(list_of_sciencefiles)
for ii, kk in enumerate(science_names):
fitsfile = fits.open(kk)
fitsfile_common, coverage = reproject_interp(fitsfile, fits.open(science_names[0])[0].header, hdu_in=0)
science = ccdproc.CCDData(data=fitsfile_common, meta=fitsfile[0].header, unit="adu")
science_list[ii] = science
coverages[ii] = ccdproc.CCDData(data=coverage, meta=fitsfile[0].header, unit="adu")
self.combined_science = ccdproc.combine(science_list, method="median")
# self.combined_coverage = ccdproc.combine(coverages, method="sum")
header0 = fits.open(list_of_sciencefiles[0])[0].header
for key in header0:
# print(str(key), header0[str(key)])
if "COMMENT" in key:
continue
self.combined_science.header[str(key)] = header0[str(key)]
self.combined_science.write(self.output_dir+"/"+self.filename+".fits", overwrite=True)
def create_super_dark(input_images, oPath, super_name, super_bias_name):
inputs = []
print('SD: ', len(input_images), input_images, super_bias_name)
super_bias_img = ccdproc.CCDData.read(super_bias_name,
ignore_missing_end=True)
for img in range(len(input_images)):
corr_dark = ccdproc.subtract_bias(
(ccdproc.CCDData.read(input_images[img], unit='adu')),
super_bias_img)
im = corr_dark
im.data = im.data.astype(np.float32)
im_offset = imageOffset(im, p_median=True)
im_offset = float(im_offset)
im.data -= im_offset
inputs.append(im)
combiner = Combiner(inputs)
super_img = combiner.median_combine()
#mn, std = imageStats(super_img)
#super_img = super_img.add(100*u.adu)
super_img.meta = inputs[0].meta
#super_img.meta['PEDESTAL'] = -100
super_img.meta['NCOMBINE'] = len(inputs)
#super_img.meta['CNTRMEAN'] = mn
#super_img.meta['CNTRSTD'] = std
s_name = super_name.split('.')
print('s_name_split: ', s_name[0])
tstring = datetime.datetime.now().isoformat().split('.')[0].split(':')
wstring = str(oPath + '\\' + s_name[0] + '_' + \
tstring[0]+tstring[1]+tstring[2] + \
'.fits')
super_img.write(wstring, overwrite=True)
hots = hot_pixels(super_img)
print(len(hots), hots)
'''
Need to trim negatives, and find hot pixels to create map.
'''
return
def create_master_flat(list_files, flat_filter=None, fitsfile=None, bias=None, fits_section=None, gain=None,
method='median', key_filter='filter', dfilter={'imagetyp':'FLAT'}, mask=None, key_find='find',
invert_find=False, sjoin=','):
if gain is not None and not isinstance(gain, u.Quantity):
gain = gain * u.electron / u.adu
lflat = []
if dfilter is not None and key_filter is not None and flat_filter is not None:
dfilter = addKeysListDict(dfilter, {key_filter: flat_filter})
list_files = getListFiles(list_files, dfilter, mask, key_find=key_find, invert_find=invert_find)
if len(list_files) == 0:
print ('WARNING: No FLAT files available for filter "%s"' % flat_filter)
return
for filename in list_files:
ccd = CCDData.read(filename, unit= u.adu)
trimmed = True if fits_section is not None else False
ccd = ccdproc.trim_image(ccd, fits_section=fits_section, add_keyword={'trimmed': trimmed})
if gain is not None:
ccd = ccdproc.gain_correct(ccd, gain)
if bias is not None:
if isinstance(bias, str):
bias = fits2CCDData(bias, single=True)
ccd = ccdproc.subtract_bias(ccd, bias)
lflat.append(ccd)
combine = ccdproc.combine(lflat, method=method)
if gain is not None and not 'GAIN' in combine.header:
combine.header.set('GAIN', gain.value, gain.unit)
combine.header['CGAIN'] = True if gain is not None else False
combine.header['IMAGETYP'] = 'FLAT'
combine.header['CMETHOD'] = method
combine.header['CCDVER'] = VERSION
addKeyHdr(combine.header, 'MBIAS', getFilename(bias))
if sjoin is not None:
combine.header['LFLAT'] = sjoin.join([os.path.basename(fits) for fits in list_files])
combine.header['NFLAT'] = len(list_files)
if fitsfile is not None:
combine.header['FILENAME'] = os.path.basename(fitsfile)
combine.write(fitsfile, clobber=True)
return combine
def create_super_flat(input_images, oPath, super_name, super_bias_name,
super_dark_name):
#NB Should cull low count input frames.
inputs = []
print('SF: ', len(input_images))
super_bias = ccdproc.CCDData.read(super_bias_name, ignore_missing_end=True)
super_dark = ccdproc.CCDData.read(super_dark_name, ignore_missing_end=True)
#super_dark = super_dark.subtract(super_dark.meta['PEDASTAL']*u.adu)
for img in range(len(input_images)):
img_in = ccdproc.CCDData.read(input_images[img],
unit='adu',
ignore_missing_end=True)
bias_corr = ccdproc.subtract_bias(img_in, super_bias)
print('Hello: ', super_dark.meta['EXPTIME'], img_in.meta['EXPTIME'],
type(bias_corr), type(super_dark), img_in.meta)
corr_flat = ccdproc.subtract_dark(bias_corr, super_dark, scale=True, \
dark_exposure=super_dark.meta['EXPTIME']*u.s, \
data_exposure =img_in.meta['EXPTIME']*u.s)
#corr_flat = ccdproc.
inputs.append(corr_flat)
combiner = Combiner(inputs)
super_img = combiner.median_combine()
super_img.meta = inputs[0].meta
super_img.meta['NCOMBINE'] = len(inputs)
s_name = super_name.split('.')
print('s_name_split: ', s_name[0])
tstring = datetime.datetime.now().isoformat().split('.')[0].split(':')
wstring = str(oPath + '\\' + s_name[0] + '_' + \
tstring[0]+tstring[1]+tstring[2] + \
'.fits')
super_img.write(wstring, overwrite=True)
#Turn the above into a circle region.
return
def createmasterflat():
#create flatlist
flatlist = sorted(glob.glob('HD115709/flat_SII/r*.fit'))
print('Number of flats used =', len(flatlist))
#open flat0 for cube len
hduf = fits.open(flatlist[0])
flat0 = hduf[4].data
print(flatlist[0], 'is open, shape:', flat0.shape)
hduf.close()
print(flatlist[0], 'is closed')
#create flatcube with shape of science and len(flatlist)
flatcube = np.zeros((science0.shape[0], science0.shape[1], len(flatlist)),
dtype=flat0.dtype)
#convert trim and populate flatcube after bias correction
for flatc, flatn in enumerate(flatlist):
print('Open :', flatn)
hdu = fits.open(flatlist[flatc])
if hdu[4].header['CHIPNAME'] == 'A5382-1-7':
ccdflat = CCDData(hdu[4].data, unit=u.adu)
ccdtflat = ccdproc.trim_image(ccdflat, fits_section=sciencewindow)
ccdcflat = ccdproc.subtract_bias(ccdtflat, ccdmbias_use)
flatcube[:, :, flatc] = ccdcflat.data
hdu.close()
else:
hdu.close()
print(flatn, 'returned wrong chipname for selected extension')
exit
#take median and write to disk
mflat = np.nanmedian(flatcube, 2)
ccdmflat = CCDData(mflat, unit=u.adu)
ccdmflat.write('tmaster_flat.fits',
overwrite=True) # write the fits to disk
def action(self, ccd):
select_dict = {'imagetyp': 'bias'}
master = self._master_image(select_dict)
return ccdproc.subtract_bias(ccd, master)
def process_spectroscopy_science(self, science_group, save_all=False):
"""Process Spectroscopy science images.
This function handles the full image reduction process for science
files. if save_all is set to True, all intermediate steps are saved.
Args:
science_group (object): :class:`~pandas.DataFrame` instance that contains a
list of science images that where observed at the same pointing
and time. It also contains a set of selected keywords from the
image's header.
save_all (bool): If True the pipeline will save all the intermadiate
files such as after overscan correction or bias corrected and
etc.
"""
# TODO (simon): The code here is too crowded.
# TODO cont. Create other functions as necessary
target_name = ''
slit_trim = None
master_bias = None
master_flat = None
master_flat_name = None
obstype = science_group.obstype.unique()
# print(obstype)
if 'OBJECT' in obstype or 'COMP' in obstype:
object_comp_group = science_group[
(science_group.obstype == 'OBJECT') |
(science_group.obstype == 'COMP')]
if 'OBJECT' in obstype:
target_name = science_group.object[science_group.obstype ==
'OBJECT'].unique()[0]
self.log.info('Processing Science Target: '
'{:s}'.format(target_name))
else:
# TODO (simon): This does not make any sense
self.log.info('Processing Comparison Lamp: '
'{:s}'.format(target_name))
if 'FLAT' in obstype and not self.args.ignore_flats:
flat_sub_group = science_group[science_group.obstype == 'FLAT']
master_flat, master_flat_name = \
self.create_master_flats(flat_group=flat_sub_group,
target_name=target_name)
elif self.args.ignore_flats:
self.log.warning('Ignoring creation of Master Flat by request.')
master_flat = None
master_flat_name = None
else:
self.log.info('Attempting to find a suitable Master Flat')
object_list = object_comp_group.file.tolist()
# grab a random image from the list
random_image = random.choice(object_list)
# define random image full path
random_image_full = os.path.join(self.args.raw_path,
random_image)
# read the random chosen file
ccd = CCDData.read(random_image_full, unit=u.adu)
if not self.args.ignore_flats:
# define the master flat name
master_flat_name = self.name_master_flats(
header=ccd.header,
group=object_comp_group,
get=True)
# load the best flat based on the name previously defined
master_flat, master_flat_name = \
get_best_flat(flat_name=master_flat_name,
path=self.args.red_path)
if (master_flat is None) and (master_flat_name is None):
self.log.critical('Failed to obtain master flat')
if master_flat is not None and not self.args.ignore_flats:
self.log.debug('Attempting to find slit trim section')
slit_trim = get_slit_trim_section(master_flat=master_flat)
elif self.args.ignore_flats:
self.log.warning('Slit Trimming will be skipped, '
'--ignore-flats is activated')
else:
self.log.info("Master flat inexistent, can't find slit trim "
"section")
if slit_trim is not None:
master_flat = image_trim(ccd=master_flat,
trim_section=slit_trim,
trim_type='slit')
if self.master_bias is not None:
master_bias = image_trim(ccd=self.master_bias,
trim_section=slit_trim,
trim_type='slit')
else:
try:
master_bias = self.master_bias.copy()
except AttributeError:
master_bias = None
norm_master_flat = None
norm_master_flat_name = None
all_object_image = []
all_comp_image = []
for science_image in object_comp_group.file.tolist():
self.out_prefix = ''
# define image full path
image_full_path = os.path.join(self.args.raw_path,
science_image)
# load image
ccd = read_fits(image_full_path, technique=self.technique)
# apply overscan
ccd = image_overscan(ccd, overscan_region=self.overscan_region)
self.out_prefix += 'o_'
if save_all:
full_path = os.path.join(self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
if slit_trim is not None:
# There is a double trimming of the image, this is to match
# the size of the other data
# TODO (simon): Potential problem here
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
ccd = image_trim(ccd=ccd,
trim_section=slit_trim,
trim_type='slit')
self.out_prefix = 'st' + self.out_prefix
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
else:
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
self.out_prefix = 't' + self.out_prefix
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
if not self.args.ignore_bias:
# TODO (simon): Add check that bias is compatible
ccd = ccdproc.subtract_bias(ccd=ccd,
master=master_bias,
add_keyword=False)
self.out_prefix = 'z' + self.out_prefix
ccd.header['GSP_BIAS'] = (
os.path.basename(self.master_bias_name),
'Master bias image')
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
else:
self.log.warning('Ignoring bias correction by request.')
# Do flat correction
if master_flat is None or master_flat_name is None:
self.log.warning('The file {:s} will not be '
'flatfielded'.format(science_image))
elif self.args.ignore_flats:
self.log.warning('Ignoring flatfielding by request.')
else:
if norm_master_flat is None:
norm_master_flat, norm_master_flat_name = \
normalize_master_flat(
master=master_flat,
name=master_flat_name,
method=self.args.flat_normalize,
order=self.args.norm_order)
ccd = ccdproc.flat_correct(ccd=ccd,
flat=norm_master_flat,
add_keyword=False)
self.out_prefix = 'f' + self.out_prefix
ccd.header['GSP_FLAT'] = (
os.path.basename(norm_master_flat_name),
'Master flat image')
# propagate master flat normalization method
ccd.header['GSP_NORM'] = norm_master_flat.header['GSP_NORM']
if save_all:
full_path = os.path.join(
self.args.red_path,
self.out_prefix + science_image)
# ccd.write(full_path, clobber=True)
write_fits(ccd=ccd, full_path=full_path)
ccd, prefix = call_cosmic_rejection(
ccd=ccd,
image_name=science_image,
out_prefix=self.out_prefix,
red_path=self.args.red_path,
dcr_par=self.args.dcr_par_dir,
keep_files=self.args.keep_cosmic_files,
method=self.args.clean_cosmic,
save=True)
self.out_prefix = prefix
if ccd is not None:
if ccd.header['OBSTYPE'] == 'OBJECT':
self.log.debug("Appending OBJECT image for combination")
all_object_image.append(ccd)
elif ccd.header['OBSTYPE'] == 'COMP':
self.log.debug("Appending COMP image for combination")
all_comp_image.append(ccd)
else:
self.log.error("Unknown OBSTYPE = {:s}"
"".format(ccd.header['OBSTYPE']))
else:
self.log.warning("Cosmic ray rejection returned a None.")
if self.args.combine:
self.log.warning("Combination of data is experimental.")
if len(all_object_image) > 1:
print(len(all_object_image))
self.log.info("Combining {:d} OBJECT images"
"".format(len(all_object_image)))
object_group = object_comp_group[
object_comp_group.obstype == "OBJECT"]
print(object_group, len(all_object_image))
combine_data(all_object_image,
dest_path=self.args.red_path,
prefix=self.out_prefix,
save=True)
elif len(all_object_image) == 1:
# write_fits(all_object_image[0])
pass
else:
self.log.error("No OBJECT images to combine")
if len(all_comp_image) > 1:
self.log.info("Combining {:d} COMP images"
"".format(len(all_comp_image)))
# comp_group = object_comp_group[
# object_comp_group.obstype == "COMP"]
combine_data(all_comp_image,
dest_path=self.args.red_path,
prefix=self.out_prefix,
save=True)
else:
self.log.error("No COMP images to combine")
else:
self.log.debug("Combine is disabled (default)")
elif 'FLAT' in obstype:
self.queue.append(science_group)
self.log.warning('Only flats found in this group')
flat_sub_group = science_group[science_group.obstype == 'FLAT']
# TODO (simon): Find out if these variables are useful or not
self.create_master_flats(flat_group=flat_sub_group)
else:
self.log.error('There is no valid datatype in this group')
darks += ','+im[0]
dark_path += 'master/'
try:
os.mkdir(dark_path)
except:
pass
dark_path += dark_master
logme('Creating master dark frame (%s)...'%(dark_path))
dark = ccdproc.combine(darks, method='median', unit='adu', add_keyword=False, **{'verify': 'ignore'})
#trim it, if necessary
if(len(trim_range) > 0):
dark = ccdproc.trim_image(dark, trim_range);
#bias correct, if necessary
if(not dark_is_bias_corrected):
#logme('Subtracting master bias frame from master dark frame...')
dark = ccdproc.subtract_bias(dark, bias, add_keyword=False)
dark_bias = bias_master
else:
logme('Master dark frame is *already* bias corrected (%s).'%dark_bias)
#write master dark frame
hdulist = dark.to_hdu()
#add bias correction to header
header=hdulist[0].header
header['BIASCORR'] = dark_bias
hdulist.writeto(dark_path, clobber=True)
#flat
#create master flat frame
im=glob.glob(flat_path+'*.fits')+glob.glob(flat_path+'*.fit')
if(len(im) <= 0):
logme('Error. Flat calibration frame(s) not found (%s).' % flat_path)
mdark_fdic[exptlist[k]] = CCDData.read(prepath/dark_fdic[exptlist[k]]
,unit = u.adu)
print(f"이전에 만든 {exptlist[k]}초 Dark 사용")
else:
images=[]
for i in range(len(darkdic[exptlist[k]])):
cc = CCDData.read(darkdic[exptlist[k]][i]['FILE'], unit= u.adu)
images.append(cc)
print(darkdic[exptlist[k]][i]['FILE'])
print(len(images), "장의 Image를 Combine")
mdark_fdic[exptlist[k]] = combine(images,method = 'median')
print(f"{exptlist[k]}초 Master Dark",sep='\n')
# Bias Subtraction
mdark_fdic[exptlist[k]] = subtract_bias(mdark_fdic[exptlist[k]],mbias)
# Header 추가
mdark_fdic[exptlist[k]].header = cc.header
mdark_fdic[exptlist[k]].header.add_history(f"{len(darkdic[exptlist[k]])} image(s) median combined dark frames with bias sub")
mdark_fdic[exptlist[k]] = yfu.CCDData_astype(mdark_fdic[exptlist[k]],dtype = 'float32')
mdark_fdic[exptlist[k]].write(prepath/dark_fdic[exptlist[k]],overwrite=True)
fig, axs = plt.subplots(1,len(exptlist))
for i in range(len(exptlist)):
im = yfu.zimshow(axs[i],mdark_fdic[exptlist[i]])
axs[i].set_xlabel(f"{exptlist[i]} s", fontsize=12)
mdark_fdic={}
for k in range(len(exptlist)):
if os.path.exists(prepath/dark_fdic[exptlist[k]]):
def process_imaging_science(self, imaging_group):
"""Does image reduction for science imaging data.
Args:
imaging_group (object): :class:`~pandas.DataFrame` instance that contains a
list of science data that are compatible with a given
instrument configuration and can be reduced together.
"""
# pick a random image in order to get a header
random_image = random.choice(imaging_group.file.tolist())
path_random_image = os.path.join(self.args.raw_path, random_image)
sample_file = CCDData.read(path_random_image, unit=u.adu)
master_flat_name = self.name_master_flats(header=sample_file.header,
group=imaging_group,
get=True)
self.log.debug('Got {:s} for master flat name'.format(master_flat_name))
master_flat, master_flat_name = get_best_flat(
flat_name=master_flat_name,
path=self.args.red_path)
if master_flat is not None:
for image_file in imaging_group.file.tolist():
# start with an empty prefix
self.out_prefix = ''
image_full_path = os.path.join(self.args.raw_path, image_file)
ccd = read_fits(image_full_path, technique=self.technique)
# Trim image
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
self.out_prefix = 't_'
if not self.args.ignore_bias:
ccd = ccdproc.subtract_bias(ccd,
self.master_bias,
add_keyword=False)
self.out_prefix = 'z' + self.out_prefix
ccd.header['GSP_BIAS'] = (
os.path.basename(self.master_bias_name),
'Master bias image')
# apply flat correction
ccd = ccdproc.flat_correct(ccd,
master_flat,
add_keyword=False)
self.out_prefix = 'f' + self.out_prefix
ccd.header['GSP_FLAT'] = (
os.path.basename(master_flat_name),
'Master Flat image')
if self.args.clean_cosmic:
ccd = astroscrappy_lacosmic(
ccd=ccd,
red_path=self.args.red_path,
save_mask=self.args.keep_cosmic_files)
self.out_prefix = 'c' + self.out_prefix
else:
print('Clean Cosmic ' + str(self.args.clean_cosmic))
final_name = os.path.join(self.args.red_path,
self.out_prefix + image_file)
# ccd.write(final_name, clobber=True)
write_fits(ccd=ccd, full_path=final_name)
self.log.info('Created science file: {:s}'.format(final_name))
else:
self.log.error('Can not process data without a master flat')
def clean_the_images(path, filename):
#ast=AstrometryNet()
#ast.api_key= 'iqmqwvazpvolmjmn'
dir = path
gain = 2 * u.electron / u.adu
readnoise = 7.5 * u.electron
ra = input('Enter the RA of the source: ')
dec = input('Enter the DEC of the source: ')
'''
wcs_header=ast.solve_from_image(path+filename)
wcs=WCS(wcs_header)
ran,decn=wcs.all_pix2world(1024,1024,0)
print(ran,decn)
'''
file_name = os.path.join(dir, filename)
image = ccdproc.CCDData.read(file_name, unit='adu')
header = fits.getheader(file_name, 0)
time = header['DATE']
t = Time(time, format='isot', scale='utc')
print(t.jd, t.mjd)
header.insert(15, ('RA', ra))
header.insert(16, ('DEC', dec))
a = sorted(glob(os.path.join(dir, 'bias*.fits')))
biaslist = []
for i in range(0, len(a)):
data = ccdproc.CCDData.read(a[i], unit='adu')
#data = ccdproc.create_deviation(data, gain=gain, readnoise=readnoise)
#data= data-(data.uncertainty.array)
biaslist.append(data)
combiner = ccdproc.Combiner(biaslist)
masterbias = combiner.median_combine()
masterbias.write('masterbias.fit', overwrite=True)
mbias = ccdproc.CCDData.read('masterbias.fit', unit='adu')
#masterbias.meta=image.meta
print('master bias generated')
print(np.mean(masterbias), np.median(masterbias))
c = sorted(glob(os.path.join(dir, 'flat*.fits')))
flatlist = []
for j in range(0, len(c)):
flat = ccdproc.CCDData.read(c[j], unit='adu')
#flat= ccdproc.create_deviation(flat, gain=gain, readnoise=readnoise)
flat = ccdproc.subtract_bias(flat, masterbias)
flatlist.append(flat)
combiner = ccdproc.Combiner(flatlist)
masterflat = combiner.median_combine()
masterflat.write('masterflat.fits', overwrite=True)
mflat = ccdproc.CCDData.read('masterflat.fits', unit='adu')
print('master flat generated')
print(np.mean(masterflat), np.median(masterflat))
#masterflat.meta=image.meta
bias_subtracted = ccdproc.subtract_bias(image, masterbias)
flat_corrected = ccdproc.flat_correct(bias_subtracted, masterflat)
cr_cleaned = ccdproc.cosmicray_lacosmic(flat_corrected,
readnoise=7.5,
sigclip=5)
print('cosmic ray removed')
fits.writeto(dir + 'j_0947_i_1_clean.fits',
cr_cleaned,
header,
overwrite=True)
print('image cleaned')
if args.usebias:
outpath = os.path.join(preprocessedpath, 'bias_master.fits')
saveCCDDataAndLog(outpath, biasmaster)
else:
for adarkexp in darkexp_set:
outpath = os.path.join(preprocessedpath, \
'dark_master_' + str(adarkexp) + 's.fits')
saveCCDDataAndLog(outpath, darklists[adarkexp])
outpath = os.path.join(preprocessedpath, 'flat_master.fits')
saveCCDDataAndLog(outpath, flatmaster)
for ascience in sciencelist:
try:
sci_image = ccdproc.CCDData.read(ascience, unit='adu')
if args.usebias:
sci_biassub = ccdproc.subtract_bias(sci_image, biasmaster)
sci_flatcorrected = ccdproc.flat_correct(sci_biassub, flatmaster)
else:
exp_time = fits.getval(ascience, 'exptime')
darkmaster = chooseClosestDark(darklists, exp_time)
sci_darksub = ccdproc.subtract_dark(sci_image, darkmaster, \
exposure_time='exptime', exposure_unit=u.second)
sci_flatcorrected = ccdproc.flat_correct(sci_darksub, flatmaster)
except:
logger.error("Couldn't reduce image %s." % (ascience))
continue
outpath = os.path.join(preprocessedpath, \
'preprocessed_' + os.path.basename(ascience))
#deadpixmaskfilename = "../stackImages/deadpix.fits"
deadpixmaskfilename = None
def ccd_process(ccd, oscan=None, trim=None, error=False, masterbias=None,
bad_pixel_mask=None, gain=None, rdnoise=None,
oscan_median=True, oscan_model=None):
"""Perform basic processing on ccd data.
The following steps can be included:
* overscan correction
* trimming of the image
* create edeviation frame
* gain correction
* add a mask to the data
* subtraction of master bias
The task returns a processed `ccdproc.CCDData` object.
Parameters
----------
ccd: `ccdproc.CCDData`
Frame to be reduced
oscan: None, str, or, `~ccdproc.ccddata.CCDData`
For no overscan correction, set to None. Otherwise proivde a region
of `ccd` from which the overscan is extracted, using the FITS
conventions for index order and index start, or a
slice from `ccd` that contains the overscan.
trim: None or str
For no trim correction, set to None. Otherwise proivde a region
of `ccd` from which the image should be trimmed, using the FITS
conventions for index order and index start.
error: boolean
If True, create an uncertainty array for ccd
masterbias: None, `~numpy.ndarray`, or `~ccdproc.CCDData`
A materbias frame to be subtracted from ccd.
bad_pixel_mask: None or `~numpy.ndarray`
A bad pixel mask for the data. The bad pixel mask should be in given
such that bad pixels havea value of 1 and good pixels a value of 0.
gain: None or `~astropy.Quantity`
Gain value to multiple the image by to convert to electrons
rdnoise: None or `~astropy.Quantity`
Read noise for the observations. The read noise should be in
`~astropy.units.electron`
oscan_median : bool, optional
If true, takes the median of each line. Otherwise, uses the mean
oscan_model : `~astropy.modeling.Model`, optional
Model to fit to the data. If None, returns the values calculated
by the median or the mean.
Returns
-------
ccd: `ccdproc.CCDData`
Reduded ccd
Examples
--------
1. To overscan, trim, and gain correct a data set:
>>> import numpy as np
>>> from astropy import units as u
>>> from hrsprocess import ccd_process
>>> ccd = CCDData(np.ones([100, 100]), unit=u.adu)
>>> nccd = ccd_process(ccd, oscan='[1:10,1:100]', trim='[10:100, 1,100]',
error=False, gain=2.0*u.electron/u.adu)
"""
# make a copy of the object
nccd = ccd.copy()
# apply the overscan correction
if isinstance(oscan, ccdproc.CCDData):
nccd = ccdproc.subtract_overscan(nccd, overscan=oscan,
median=oscan_median,
model=oscan_model)
elif isinstance(oscan, six.string_types):
nccd = ccdproc.subtract_overscan(nccd, fits_section=oscan,
median=oscan_median,
model=oscan_model)
elif oscan is None:
pass
else:
raise TypeError('oscan is not None, a string, or CCDData object')
# apply the trim correction
if isinstance(trim, six.string_types):
nccd = ccdproc.trim_image(nccd, fits_section=trim)
elif trim is None:
pass
else:
raise TypeError('trim is not None or a string')
# create the error frame
if error and gain is not None and rdnoise is not None:
nccd = ccdproc.create_deviation(nccd, gain=gain, rdnoise=rdnoise)
elif error and (gain is None or rdnoise is None):
raise ValueError(
'gain and rdnoise must be specified to create error frame')
# apply the bad pixel mask
if isinstance(bad_pixel_mask, np.ndarray):
nccd.mask = bad_pixel_mask
elif bad_pixel_mask is None:
pass
else:
raise TypeError('bad_pixel_mask is not None or numpy.ndarray')
# apply the gain correction
if isinstance(gain, u.quantity.Quantity):
nccd = ccdproc.gain_correct(nccd, gain)
elif gain is None:
pass
else:
raise TypeError('gain is not None or astropy.Quantity')
# test subtracting the master bias
if isinstance(masterbias, ccdproc.CCDData):
nccd = ccdproc.subtract_bias(nccd, masterbias)
elif isinstance(masterbias, np.ndarray):
nccd.data = nccd.data - masterbias
elif masterbias is None:
pass
else:
raise TypeError(
'masterbias is not None, numpy.ndarray, or a CCDData object')
return nccd
import ccdproc
from os import listdir
bias_hdu = ccdproc.CCDData.read('data_bias/master_bias.fits', unit='adu',)
flat_hdu_i = ccdproc.CCDData.read('data_flat/i/master_flat.fits', unit='adu')
flat_hdu_v = ccdproc.CCDData.read('data_flat/v/master_flat.fits', unit='adu')
print "Now cleaning I"
#Band I
subtracted_flat_i = flat_hdu_i
names_i = listdir('./data_obj/i/')
images_i = [ccdproc.CCDData.read('data_obj/i/'+i, unit='adu') for i in listdir('./data_obj/i')]
for i in range(len(names_i)):
subt_obj = ccdproc.subtract_bias(images_i[i], bias_hdu)
corr_obj = ccdproc.flat_correct(subt_obj, subtracted_flat_i, min_value=0.000000001)
corr_obj.write('./data_corr/i/'+names_i[i])
subt_obj.write('./data_biascorr/i/'+names_i[i])
print "Now cleaning V"
#Band V
subtracted_flat_v = flat_hdu_v
names_v = listdir('./data_obj/v/')
images_v = [ccdproc.CCDData.read('data_obj/v/'+i, unit='adu') for i in listdir('./data_obj/v')]
for i in range(len(names_v)):
subt_obj = ccdproc.subtract_bias(images_v[i], bias_hdu)
corr_obj = ccdproc.flat_correct(subt_obj, subtracted_flat_v, min_value=0.000000001)
import numpy as np
import pdb
print "Combining..."
# All images: biases
images = [ccdproc.CCDData.read('data_bias/'+i, unit='adu') for i in listdir('./data_bias')]
c_images = combiner.Combiner(images)
master_bias = c_images.average_combine()
master_bias.write('data_bias/master_bias.fits')
# All images: flats i
images = [ccdproc.CCDData.read('data_flat/i/'+i, unit="adu") for i in listdir('./data_flat/i/')]
sub_images = [ccdproc.subtract_bias(i,master_bias) for i in images]
c_images = combiner.Combiner(sub_images)
c_images.scaling = [1/float(fits.open('data_flat/i/'+i)[0].header["EXPTIME"]) for i in listdir("./data_flat/i")]
master_flat_i = c_images.average_combine()
master_flat_i.data = master_flat_i.data/np.ma.average(master_flat_i.data)
master_flat_i.write('data_flat/i/master_flat.fits')
# All images: flats v
images = [ccdproc.CCDData.read('data_flat/v/'+i, unit="adu") for i in listdir('./data_flat/v/')]
sub_images = [ccdproc.subtract_bias(i,master_bias) for i in images]
c_images = combiner.Combiner(sub_images)
c_images.scaling = [1/float(fits.open('data_flat/v/'+i)[0].header["EXPTIME"]) for i in listdir("./data_flat/v")]
master_flat_v = c_images.average_combine()
master_flat_v.data = master_flat_v.data/np.ma.average(master_flat_v.data)
master_flat_v.write('data_flat/v/master_flat.fits')
def create_master_flats(self, flat_group, target_name=''):
"""Creates master flats
Using a list of compatible flat images it combines them using median and
1-sigma clipping. Also it apply all previous standard calibrations to
each image.
Args:
flat_group (DataFrame): :class:`~pandas.DataFrame` instance.
Contains a list of compatible flat images
target_name (str): Science target name. This is used in some science
case uses only.
Returns:
The master flat :class:`~astropy.nddata.CCDData` instance and the name of under which
the master flat was stored.
"""
flat_file_list = flat_group.file.tolist()
cleaned_flat_list = []
master_flat_list = []
master_flat_name = None
self.log.info('Creating Master Flat')
for flat_file in flat_file_list:
# print(f_file)
image_full_path = os.path.join(self.args.raw_path, flat_file)
ccd = read_fits(image_full_path, technique=self.technique)
self.log.debug('Loading flat image: ' + image_full_path)
if master_flat_name is None:
master_flat_name = self.name_master_flats(
header=ccd.header,
group=flat_group,
target_name=target_name)
if self.technique == 'Spectroscopy':
ccd = image_overscan(ccd, overscan_region=self.overscan_region)
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
ccd = ccdproc.subtract_bias(ccd,
self.master_bias,
add_keyword=False)
ccd.header['GSP_BIAS'] = (
os.path.basename(self.master_bias_name),
'Master bias image')
elif self.technique == 'Imaging':
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
ccd = ccdproc.subtract_bias(ccd,
self.master_bias,
add_keyword=False)
ccd.header['GSP_BIAS'] = (
os.path.basename(self.master_bias_name),
'Master bias image')
else:
self.log.error('Unknown observation technique: ' +
self.technique)
if self._is_file_saturated(ccd=ccd):
self.log.warning('Removing saturated image {:s}. '
'Use --saturation to change saturation '
'level'.format(flat_file))
continue
else:
cleaned_flat_list.append(flat_file)
master_flat_list.append(ccd)
if master_flat_list != []:
master_flat = ccdproc.combine(master_flat_list,
method='median',
sigma_clip=True,
sigma_clip_low_thresh=1.0,
sigma_clip_high_thresh=1.0,
add_keyword=False)
# add name of images used to create master bias
for n in range(len(cleaned_flat_list)):
master_flat.header['GSP_IC{:02d}'.format(n + 1)] = (
cleaned_flat_list[n],
'Image used to create master flat')
write_fits(ccd=master_flat,
full_path=master_flat_name,
combined=True)
self.log.info('Created Master Flat: ' + master_flat_name)
return master_flat, master_flat_name
# print(master_flat_name)
else:
self.log.error('Empty flat list. Check that they do not exceed the '
'saturation limit.')
return None, None
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
red_bias_list.append(ccd)
master_bias_red = ccdproc.combine(red_bias_list, method='median')
master_bias_red.write('master_bias_red.fits', clobber=True)
#create the flat fields
red_flat_list = []
for filename in ic1.files_filtered(obstype='Flat', isiarm='Red arm'):
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
ccd = ccdproc.subtract_bias(ccd, master_bias_red)
red_flat_list.append(ccd)
master_flat_red = ccdproc.combine(red_flat_list, method='median')
master_flat_red.write('master_flat_red.fits', clobber=True)
blue_flat_list = []
for filename in ic1.files_filtered(obstype='Flat', isiarm='Blue arm'):
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
ccd = ccdproc.subtract_bias(ccd, master_bias_blue)
blue_flat_list.append(ccd)
master_flat_blue = ccdproc.combine(blue_flat_list, method='median')
master_flat_blue.write('master_flat_blue.fits', clobber=True)
def action(self, ccd):
select_dict = {'imagetyp': 'bias'}
master = self._master_image(select_dict)
return ccdproc.subtract_bias(ccd, master)
import numpy as np import matplotlib.pyplot as plt from astropy.io import fits from astropy import units as u from astropy.nddata import CCDData import ccdproc bias_file = ".\\testimages\\bias.fits" image_file = ".\\testimages\\image.fits" # bias_list = fits.open(bias_file) # image_list = fits.open(image_file) bias_data = fits.getdata(bias_file) image_data = fits.getdata(image_file) bias = CCDData(bias_data, unit=u.adu) data = CCDData(image_data, unit=u.adu) bias_subtracted = ccdproc.subtract_bias(data, bias) plt.imshow(bias, vmin=100, vmax=120) plt.show() plt.imshow(bias_subtracted, vmin=-5, vmax=5) plt.show()
def correctData(filename, master_bias, master_flat, filetype):
"""
Correct a science image using the available
master calibrations. Skip a calibration step if the
master frame does not exist.
No reduced file is written in this new scheme.
Instead, the corrected data is passed directly
to the phot() routine, photometry is done as per
the configuration and the photometry is written out
only.
TODO: Finish docstring
"""
print('Reducing {0:s}...'.format(filename))
with fits.open(filename) as fitsfile:
# correct times for science spectra,
# don't bother for arcs
hdr = fitsfile[0].header
if filetype == 'science':
half_exptime = hdr[EXPTIME_KEYWORD]/2.
utstart = hdr[UTSTART_KEYWORD]
dateobs = hdr[DATEOBS_KEYWORD]
ra = hdr[RA_KEYWORD]
dec = hdr[DEC_KEYWORD]
time_start = Time('{}T{}'.format(dateobs, utstart),
scale='utc',
format='isot',
location=OBSERVATORY)
# correct to mid exposure time
jd_mid = time_start + half_exptime*u.second
ltt_bary, ltt_helio = getLightTravelTimes(ra, dec, jd_mid)
time_bary = jd_mid.tdb + ltt_bary
time_helio = jd_mid.utc + ltt_helio
hdr['BJD-MID'] = time_bary.jd
hdr['HJD-MID'] = time_helio.jd
hdr['JD-MID'] = jd_mid.jd
hdr['UT-MID'] = jd_mid.isot
ccd = CCDData.read(filename, unit=u.adu)
if master_bias:
ccd = subtract_bias(ccd, master_bias)
else:
print('No master bias, skipping correction...')
if master_flat:
ccd = flat_correct(ccd, master_flat)
else:
print('No master flat, skipping correction...')
# after calibrating we get np.float64 data
# if there are no calibrations we maintain dtype = np.uint16
# sep weeps
# fix this by doing the following
if isinstance(ccd.data[0][0], np.uint16):
ccd.data = ccd.data.astype(np.float64)
# trim the data
ccd_trimmed = trim_image(ccd[1000:3001, :])
# write out the trimmed file and the updated header
#ccd_trimmed.write(filename, hdr, clobber=True)
trimmed_filename = '{}_t.fits'.format(filename.split('.')[0])
fits.writeto(trimmed_filename, ccd_trimmed.data, hdr)
# remove the old untrimmed data
os.system('rm {}'.format(filename))