def combine_stack(self, stack_list, method="median"):
"""Combine a series of aligned object frames into a master stack via CCDPROC"""
if method == "median":
print("Combining stack by median")
stack = ccdproc.combine(stack_list,
method="median",
unit="adu",
mem_limit=2e9)
elif method == "mean":
print("Combining stack by mean")
stack = ccdproc.combine(stack_list,
method="mean",
unit="adu",
mem_limit=6e9)
elif method == "sum":
print("Combining stack by sum")
stack = ccdproc.combine(stack_list,
method="sum",
unit="adu",
mem_limit=6e9)
else:
print("Combining stack by median")
stack = ccdproc.combine(stack_list,
method="median",
unit="adu",
mem_limit=6e9)
return stack
def combine_darks(self, dark_list, method="median"):
"""Combine a series of dark frames into a master dark via CCDPROC"""
if method == "median":
print("Combining darks by median")
master_dark = ccdproc.combine(dark_list,
method="median",
unit="adu",
mem_limit=6e9)
elif method == "mean":
print("Combining darks by mean")
master_dark = ccdproc.combine(dark_list,
method="mean",
unit="adu",
mem_limit=6e9)
else:
print("Combining darks by median")
master_dark = ccdproc.combine(dark_list,
method="median",
unit="adu",
mem_limit=6e9)
return master_dark
def combine_flats(self, flat_list, master_dark, method="median"):
"""Combine and reduce a series of flat frames into a normalized flatfield via CCDPROC"""
if method == "median":
print("Combining flats by median")
combined_flat = ccdproc.combine(flat_list, method="median", unit="adu", mem_limit=6e9)
elif method == "mean":
print("Combining flats by mean")
combined_flat = ccdproc.combine(flat_list, method="mean", unit="adu", mem_limit=6e9)
else:
print("Combining flats by median")
combined_flat = ccdproc.combine(flat_list, method="median", unit="adu", mem_limit=6e9)
print("Subtracting master dark from combined flat")
master_flat = ccdproc.subtract_dark(combined_flat, master_dark, data_exposure=combined_flat.header["exposure"]*u.second, dark_exposure=master_dark.header["exposure"]*u.second, scale=True)
print("Reading master flat data")
master_flat_data = np.asarray(master_flat)
print("Creating normalized flatfield")
flatfield_data = master_flat_data / np.mean(master_flat_data)
print("Converting flatfield data to CCDData")
flatfield = ccdproc.CCDData(flatfield_data, unit="adu")
return flatfield
def combine_flats(refresh='2', method='2'):
if method == '1':
meta = 'med'
source = 'Trimmed_Flat/subflatsmed'
dest = 'Master_Files/mflat_median.fits'
elif method == '2':
meta = 'sig'
source = 'Trimmed_Flat/subflatssig'
dest = 'Master_Files/mflat.fits'
subflatcollection = ImageFileCollection(source)
combtime = 0
if refresh == '1':
print('found', len(subflatcollection.values('file')), 'subflats')
start = time.time()
if method == '1':
mflat = ccdp.combine(subflatcollection.files_filtered(
imtype='subflat', include_path=True),
method='median')
mflat.meta['flatcom'] = 'median'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
elif method == '2':
mflat = ccdp.combine(subflatcollection.files_filtered(
imtype='subflat', include_path=True),
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian,
sigma_clip_dev_func=mad_std)
mflat.meta['flatcom'] = 'sigma'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
mflat.meta['normmed'] = (np.nanmedian(mflat),
'nanmedian of the master flat')
mflat.meta['subflats'] = meta
mflat.write(dest[0:-5] + '_' + meta + '.fits', overwrite=True)
else:
try:
if method == '1':
mflat = CCDData.read('Master_Files/mflat_median_med.fits',
unit='adu')
elif method == '2':
mflat = CCDData.read('Master_Files/mflat_sig.fits', unit='adu')
except:
print('can\'t locate master flat, create or check directory')
sys.exit()
return subflatcollection, mflat, dest, combtime
def combine_BiasDark(file_list, c_method, base_dir_name, master_file_dir_name,
current_dir_name):
try:
combine_result = combine(
file_list, # ccdproc does not accept numpy.ndarray, but only python list.
method=c_method, # default is average so I specified median.
unit='adu')
combine_result.data = np.array(combine_result.data, dtype=np.float32)
combine_result.write('{0}/{1}{2}_master_{3}_float32.fit'\
.format(base_dir_name, master_file_dir_name,
current_dir_name, c_method), overwrite =True, format='fits')
##### fits header update
with fits.open('{0}/{1}{2}_master_{3}_float32.fit'\
.format(base_dir_name, master_file_dir_name, current_dir_name, c_method),
mode='update') as hdul:
hdul[0].header.append(
('COMMENT', ', '.join(file_list), 'combine file list'))
hdul[0].header.append('COMMENT', '{0}'.format(len(file_list)),
'combine file number')
except Exception as err:
print('{5} ::: {4} with {0}/{1}{2}_master_{3}_float32.fit ...'\
.format(base_dir_name, master_file_dir_name,
current_dir_name, c_method, err, datetime.now()))
return 0
def create_master_bias(list_files, fitsfile=None, fits_section=None, gain=None, method='median',
dfilter={'imagetyp':'bias'}, mask=None, key_find='find', invert_find=False, sjoin=','):
if gain is not None and not isinstance(gain, u.Quantity):
gain = gain * u.electron / u.adu
lbias = []
list_files = getListFiles(list_files, dfilter, mask, key_find=key_find, invert_find=invert_find)
for filename in list_files:
ccd = CCDData.read(filename, unit= u.adu)
trimmed = True if fits_section is not None else False
ccd = ccdproc.trim_image(ccd, fits_section=fits_section, add_keyword={'trimmed': trimmed})
if gain is not None:
ccd = ccdproc.gain_correct(ccd, gain)
lbias.append(ccd)
combine = ccdproc.combine(lbias, method=method)
if gain is not None and not 'GAIN' in combine.header:
combine.header.set('GAIN', gain.value, gain.unit)
combine.header['CGAIN'] = True if gain is not None else False
combine.header['IMAGETYP'] = 'BIAS'
combine.header['CMETHOD'] = method
combine.header['CCDVER'] = VERSION
if sjoin is not None:
combine.header['LBIAS'] = sjoin.join([os.path.basename(fits) for fits in list_files])
combine.header['NBIAS'] = len(list_files)
if fitsfile is not None:
combine.header['FILENAME'] = os.path.basename(fitsfile)
combine.write(fitsfile, clobber=True)
return combine
def makeMasterFlat(images, master_bias):
"""
Flats are corrected for their bias level (if
master_bias)
TODO: Finish docstring
"""
try:
fitsfile = 'master_flat.fits'
master_flat = CCDData.read(fitsfile, unit=u.adu)
return master_flat
except FileNotFoundError:
# empty list for the flats
flat_list = []
# create the master flat field
print('Reducing flats')
for f in images.files_filtered(imagetyp=FLAT_KEYWORD):
print(f)
with fits.open(f) as fitsfile:
data_exp = fitsfile[0].header[EXPTIME_KEYWORD]
ccd = CCDData.read(f, unit=u.adu)
if master_bias:
ccd = subtract_bias(ccd, master_bias)
else:
print('No master bias, skipping correction...')
flat_list.append(ccd)
try:
master_flat = combine(flat_list, method='median')
master_flat.write('master_flat.fits', clobber=True)
return master_flat
except IndexError:
print('There are no flats, skipping...')
master_flat = None
def make_flat_master():
""" for each filter do:
stack all flat exposures
save fits to file
add entry in database for each file (stack) generated"""
import ccdproc
nb = models.NightBundle.get(telescope_night_bundle_id=1)
nb_dir = nb.directory_path
flat_list_q = nb.exposures.select(
lambda d: d.exposure_type == models.EXP_TYPE_CODES["FLAT"])
flat_list = [os.path.join(nb_dir, f.filename) for f in flat_list_q]
# Create dark master and save to file
master_flat = ccdproc.combine(flat_list, method="average", unit="adu")
flat_hdu = fits.PrimaryHDU(master_flat)
flat_hdu.header["IMAGETYP"] = "FLATM"
flat_hdu.header["EXPTIME"] = 60.0
file_path = os.path.join(nb_dir, "products", "flat_master.fits")
makedirs = os.path.dirname(file_path)
if makedirs:
os.makedirs(makedirs, exist_ok=True)
flat_hdu.writeto(file_path)
# Add entry to database
flat_comb = models.ExposureCombination(
night_bundle=nb,
filename="flat_master.fits",
combination_type=models.COMB_TYPE_CODES["FLATM"],
exposures=flat_list_q,
)
def loadbias(self):
""" Loads the bias information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master bias frame by ccdproc.
"""
#master bias frame
#Search for bias and load it into data object
namelist = self.loadauxname('bias', multi=True)
if (len(namelist) == 0):
self.log.error('Bias calibration frame not found.')
raise RuntimeError('No bias file loaded')
self.log.debug('Creating master bias frame...')
#if there is just one, use it as biasfile or else combine all to make a master bias
if (len(namelist) == 1):
self.bias = ccdproc.CCDData.read(namelist[0],
unit='adu',
relax=True)
else:
self.bias = ccdproc.combine(namelist,
method='median',
unit='adu',
add_keyword=False)
# Finish up
self.biasloaded = True
self.biasname = namelist[0]
self.log.debug('LoadBias: done')
def create_flat(flat_list,fil,red_path,mbias=None,log=None):
log.info('Processing files for filter: '+fil)
log.info(str(len(flat_list))+' files found.')
flats = []
flat_scale = []
for flat in flat_list:
log.info('Loading file: '+flat)
raw = CCDData.read(flat,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
mask = make_source_mask(red, nsigma=3, npixels=5)
bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
masked = np.array(red)
masked[mask] = bkg.background[mask]
log.info('Median flat level: '+str(np.median(masked)))
norm = 1/np.median(masked[500:1500,500:1500])
log.info('Flat normalization: '+str(norm))
flat_scale.append(norm)
flats.append(CCDData(masked,meta=red.header,unit=u.electron))
mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True)
log.info('Created master flat for filter: '+fil)
mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True)
log.info('Master flat written to mflat_'+fil+'.fits')
return
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation.
"""
self.log.info(f"Running {self.__class__.__name__} action")
filedate = max([d.obstime() for d in self.darks[self.exptime]])
filedate_str = filedate.strftime('%Y%m%dUT')
calibrated_darks = [d.pixeldata[0] for d in self.darks[self.exptime]]
combined_dark = ccdproc.combine(
calibrated_darks,
method='average',
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=stats.mad_std,
)
self.log.info(f" Combined.")
combined_dark.meta['combined'] = True
combined_dark.meta['ncomb'] = len(calibrated_darks)
combined_dark_filename = f'MasterDark_{self.exptime:03d}s_{filedate_str}.fits'
combined_dark_filepath = Path(
self.cfg['Calibrations'].get('DirectoryForMasters'))
combined_dark_file = combined_dark_filepath.joinpath(
combined_dark_filename)
if combined_dark_file.exists() is True:
self.log.debug(f" Deleting existing: {combined_dark_file.name}")
combined_dark_file.unlink()
self.log.info(f" Saving: {combined_dark_file.name}")
combined_dark.write(combined_dark_file)
return self.action.args
def combine_master_flat_file(file_list, c_method, base_dir_name,
master_file_dir_name, current_dir_name, chl):
try:
combine_result = combine(
file_list, # ccdproc does not accept numpy.ndarray, but only python list.
method=c_method, # default is average so I specified median.
sigma_clip=True,
sigma_clip_low_thresh=3,
sigma_clip_high_thresh=3,
unit='adu') # unit is required: it's ADU in our case.
combine_result.data = np.array(combine_result.data, dtype=np.float32)
combine_result.write('{0}{1}{2}_master_{3}_{4}_float32.fit'\
.format(base_dir_name, master_file_dir_name,
current_dir_name, c_method, chl), overwrite =True, format='fits')
##### fits header update
with fits.open('{0}{1}{2}_master_{3}_{4}_float32.fit'\
.format(base_dir_name, master_file_dir_name,
current_dir_name, c_method, chl),
mode='update') as hdul:
hdul[0].header.append(
('COMMENT', ', '.join(file_list), 'combine file list'))
hdul[0].header.append('COMMENT', '{0}'.format(len(file_list)),
'combine file number')
except Exception as err:
print('{6} ::: {5} with {0}{1}{2}_master_{3}_{4}_float32.fit ...'\
.format(base_dir_name, master_file_dir_name,
current_dir_name, c_method, chl, err, datetime.now()))
return 0
def combinedark(cal_dir="../Data/20181207/cals", mast_dir=".", filt=[]):
#Generates master dark file from calibration directory - uses max exptime dark images
#Get master bias file
if os.path.isfile(mast_dir + "/master/master_bias.FIT") != True:
print "No master bias file"
return False
master_bias = CCDData.read(mast_dir + "/master/master_bias.FIT")
#Generate image list
imagelist = gen.getimages(cal_dir, filt=filt)
full_dark_list = []
for img in imagelist:
ccd = CCDData.read(cal_dir + '/' + img, unit=u.adu)
if ccd.header["IMAGETYP"].strip() == "Dark Frame":
img_exptime = ccd.header["EXPTIME"]
ccd = ccdproc.subtract_bias(ccd, master_bias)
full_dark_list.append([ccd, img_exptime])
print img
#Select only those with max exptime
exptime = max(full_dark_list, key=lambda x: x[1])[1]
dark_list = [x[0] for x in full_dark_list if x[1] == exptime]
if len(dark_list) == 0:
print "ERROR: no dark files"
return False
#Generate master file
master_dark = ccdproc.combine(dark_list, method='median', dtype="float32")
master_dark.write(mast_dir + "/master/master_dark.FIT", overwrite=True)
'''
print "Created master_dark file with " + str(exptime) + " exptime"
'''
return True
def create_flat(flat_list,fil,red_path,mbias=None,log=None):
log.info('Processing files for filter: '+fil)
log.info(str(len(flat_list))+' files found.')
flats = []
masks = []
flat_scale = []
for flat in flat_list:
log.info('Loading file: '+flat)
raw = CCDData.read(flat,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron)
log.info('Exposure time of image is '+str(red.header['TRUITIME']*red.header['COADDONE']))
mask = make_source_mask(red, nsigma=3, npixels=5)
bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
masked = np.array(red)
masked[mask] = bkg.background[mask]
log.info('Median flat level: '+str(np.median(masked)))
norm = 1/np.median(masked[224:1824,224:1824])
log.info('Flat normalization: '+str(norm))
flat_scale.append(norm)
flats.append(CCDData(masked,meta=red.header,unit=u.electron))
mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True)
log.info('Created master flat for filter: '+fil)
mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True)
log.info('Master flat written to mflat_'+fil+'.fits')
return
def combine_dark(day, chip, exptime, data_chip):
#function to combine the darks for certain chip at certain exposure time
output_name = 'Dark_' + exptime + '_' + chip + '.fits'
if os.path.isfile(main_path + day + '/darks/' + output_name):
cmd = 'rm ' + main_path + day + '/darks/' + output_name
os.system(cmd)
print "file %s deleted" % (main_path + day + '/darks/' + output_name)
print main_path + day + '/darks/' + output_name
mdark = ccd.combine(data_chip,
output_file=main_path + day + '/darks/' + output_name,
method=u'average',
mem_limit=1600000000000,
sigma_clip=True,
sigma_clip_low_thresh=3,
sigma_clip_high_thresh=3,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=np.ma.std,
dtype=np.float64)
#tmdark=ccd.trim_image(mdark,fits_section='[5:595,5:2395]')
#ccd.fits_ccddata_writer(mdark,main_path+day+'/darks/'+output_name)
#cmd='rm '+main_path+day+'/darks/nt_'+output_name
#os.system(cmd)
del mdark
#del tmdark
#cmd='mv '+output_name+' '+main_path+day+'/darks'
#os.system(cmd)
print "masterdark %s created for chip %s for epoch %s" % (exptime, chip,
day)
return (output_name)
def run(self):
assert isinstance(self.input_list, list)
if len(self.input_list) == 0:
return
if self.output_filename is not None:
assert isinstance(self.output_filename, str)
header = pyfits.getheader(self.input_list[0])
bx, by = header['CCDSUM'].strip().split()
# Parameter obtained from PySOAR, written by Luciano Fraga
try:
master_bias = ccdproc.combine(self.input_list,
method='average',
mem_limit=6.4e7,
minmax_clip=True,
unit='adu')
except ZeroDivisionError:
raise RuntimeError('CCDProc.combine raised an error. '
'Try again with a different number of input '
'files')
if self.output_filename is None:
self.output_filename = "0Zero{}x{}".format(bx, by)
pyfits.writeto(self.output_filename, master_bias.data, header)
def image_integration(self, flat=False, confirm=True):
print(
f'{Style.BRIGHT}Integrating {len(self.collection.files)} files.{Style.RESET_ALL}'
)
if confirm:
hlp.prompt()
scale = None
if flat:
def inv_median(a):
return 1 / np.median(a)
scale = inv_median
print('Integrating...')
stack = ccdp.combine(self.collection.files_filtered(include_path=True),
method='average',
scale=scale,
sigma_clip=True,
sigma_clip_low_thresh=cfg.SIGMA_LOW,
sigma_clip_high_thresh=cfg.SIGMA_HIGH,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=mad_std,
mem_limit=600e7)
return stack
def median_combine(path):
"""
Median combine with from a list within a directory.
Args:
path: path to directory of fits files to be combined
Returns:
a median combined CCDData object
"""
ic1 = ccdproc.ImageFileCollection(location=path)
validate_units(ic1)
framelist = ic1.files_filtered(BUNIT='adu', include_path=True)
median_frame = ccdproc.combine(framelist,
method='median',
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=mad_std,
mem_limit=350e6)
return median_frame
def flat_combine(chip, chip_flats, chip_means):
#function to combine the flat for a certain chip
output_name = 'Flat_' + period + '_' + chip + '.fits'
#nn=np.where(flat_chip==chip)
#chip_flats=flat_list[nn]
#chip_means=flat_mean[nn]
if os.path.isfile(main_path + 'MasterFlats/' + period + '/' + output_name):
os.system('rm ' + main_path + 'MasterFlats/' + period + '/' +
output_name)
print "image deleted %s" % (main_path + 'MasterFlats/' + period + '/' +
output_name)
mean_counts = np.mean(chip_means)
mm = np.where((chip_means > mean_counts) & (chip_means < 15000))
flat_selected = chip_flats[mm]
print "###########", output_name, len(flat_selected), mean_counts
if len(flat_selected) > 0:
data_chip = read_data(chip, flat_selected)
print "combining chip=%s" % (chip)
mflat = ccd.combine(data_chip,
output_file=main_path + 'MasterFlats/' + period +
'/' + output_name,
method=u'median',
mem_limit=1600000000000,
sigma_clip=True,
sigma_clip_low_thresh=3,
sigma_clip_high_thresh=3,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=np.ma.std,
dtype=np.float64)
del mflat
print "masterflat %s created for chip %s for epoch %s" % (output_name,
chip, period)
else:
print "chip = %s without normal masterdark " % (chip)
a = pf.open(main_path + chip_flats[0])
head = a[0].header
dat = a[0].data
fake_data = np.ones(dat.shape)
pf.writeto(main_path + 'MasterFlats/' + period + '/' + output_name,
data=fake_data,
header=head)
print "fake masterflat %s created for chip %s for epoch %s" % (
output_name, chip, period)
return (output_name)
def flat_combine(textlist_files, band, outfile='mflat'):
"""
Combines multiple flats for a given input files list
Args:
textlist_files : A python list object with paths/names to the individual files.
band : Filter name associated with the flat files
outfile : Master flat name (default outfile + band + '.fits')
Returns:
None
"""
new_list = [
CCDData.read(filename, unit=u.adu) for filename in textlist_files
]
combined_flat = ccdp.combine(img_list=new_list,
method='average',
scale=inv_median,
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=mad_std)
combined_flat.meta['combined'] = True
file_name = outfile + band + '.fits'
combined_flat.data = combined_flat.data.astype('float32')
combined_flat.write(file_name, hdu_mask=None, hdu_uncertainty=None)
def _action_for_one_group(self, filter_dict=None):
combined_dict = self.apply_to.copy()
if filter_dict is not None:
combined_dict.update(filter_dict)
file_list = [
os.path.join(self.image_source.location, f)
for f in self.image_source.files_filtered(**combined_dict)
]
combine_keyword_args = {
'minmax_clip': self._clipping_widget.min_max,
'sigma_clip': self._clipping_widget.sigma_clip,
}
if self._combine_method.method == 'Average':
combine_keyword_args['method'] = 'average'
elif self._combine_method.method == 'Median':
combine_keyword_args['method'] = 'median'
if combine_keyword_args['minmax_clip']:
combine_keyword_args['minmax_clip_min'] = \
self._clipping_widget.min_max.min
combine_keyword_args['minmax_clip_max'] = \
self._clipping_widget.min_max.max
if combine_keyword_args['sigma_clip']:
combine_keyword_args['sigma_clip_low_thresh'] = \
self._clipping_widget.sigma_clip.min
combine_keyword_args['sigma_clip_low_thresh'] = \
self._clipping_widget.sigma_clip.min
combine_keyword_args['sigma_clip_func'] = np.ma.median
combine_keyword_args['sigma_clip_dev_func'] = \
median_absolute_deviation
if self._combine_method.scaling_func:
combine_keyword_args['scale'] = self._combine_method.scaling_func
combined = ccdproc.combine(file_list,
mem_limit=DEFAULT_MEMORY_LIMIT,
**combine_keyword_args)
sample_image = ccdproc.CCDData.read(file_list[0])
combined.header = sample_image.header
combined.header['master'] = True
if combined.data.dtype != sample_image.dtype:
combined.data = np.array(combined.data, dtype=sample_image.dtype)
try:
if isinstance(combined.uncertainty.array, np.ma.masked_array):
combined.uncertainty.array = np.array(
combined.uncertainty.array)
except AttributeError:
pass
# Do not keep the mask or uncertainty if the data has neither
if sample_image.mask is None and sample_image.uncertainty is None:
combined.mask = None
combined.uncertainty = None
return combined
def produce_master(t,
method="median",
imagetp='FLAT',
slc=None,
ccdread_unit='adu'):
""" a method to process master frames
Parameters
----------
t: astropy.table.Table
the table of SONG observation
method:
the method adopted when combining frames
imagetp: string
{'BIAS', 'FLAT', 'FLATI2', 'THAR', 'THARI2'}
slc: slice
to denote the fraction of being used
ccdread_unit: string/unit
default is 'adu'
Returns
-------
mst: ccdproc.CCDData
the (combined) master frame
"""
assert method in {'average', 'median'}
# 1. produce ind of master frames
ind_mst = np.where(t['IMAGETYP'] == imagetp)[0]
# check for positive number of master frames
try:
assert len(ind_mst) > 0
except AssertionError:
raise IOError("There is no image of type %s!" % imagetp)
# in default, combine all masters available
if slc is not None:
# combine a fraction of masters available
assert isinstance(slc, slice)
ind_mst = ind_mst[slc]
# check for positive number of master frames
try:
assert len(ind_mst) > 0
except AssertionError:
raise IOError("There is no image of type %s! (slice is bad)" % imagetp)
# 2. read master frames
print("@SONG: trying really hard to produce the final %s frame!" % imagetp)
for _ in t['fps'][ind_mst]:
print("+ %s" % _)
fps = ','.join(t['fps'][ind_mst])
if len(ind_mst) == 1:
mst = ccdproc.CCDData.read(fps, unit=ccdread_unit)
else:
mst = ccdproc.combine(fps, unit=ccdread_unit, method=method)
return mst
def make_master(self, output_master_file):
"""Combine the raw calibration files into a master file and
write it to the specified file name.
"""
# Recommended settings
comb_method = 'average'
do_sig_clip = True
sig_clip_lothresh = 5
sig_clip_hithresh = 5
max_ram_bytes = 5e8
data_units = 'adu'
# Calculate the header keywords to update
kw_dict = self._generate_final_keywords()
cal_type = kw_dict['IMAGETYP'][0]
nfiles = len(self._files.summary)
# Combine files
msg = (
f'About to combine {nfiles} {cal_type} files, method={comb_method}'
f' sigma_clip={do_sig_clip} sig_clip_lothresh={sig_clip_lothresh}'
f' sig_clip_hithresh={sig_clip_hithresh} max_ram_bytes={max_ram_bytes:.3e}.'
)
self._logger.debug(msg)
master = ccdp.combine(self._files.files_filtered(include_path=True),
method=comb_method,
sigma_clip=do_sig_clip,
sigma_clip_low_thresh=sig_clip_lothresh,
sigma_clip_high_thresh=sig_clip_hithresh,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=mad_std,
mem_limit=max_ram_bytes,
unit=data_units,
output_verify='ignore')
# Calculate the header keywords to update, and update them
master.meta['combined'] = True
master.unit = 'adu'
# Remove...
kw_to_remove_list = [
'UT', 'TIME-OBS', 'SWOWNER', 'SWCREATE', 'SBSTDVER'
]
for kw in kw_to_remove_list:
if kw in master.header:
del master.header[kw]
# Update/add
for kw, val in kw_dict.items():
master.header[kw] = val
# Write file
master.write(output_master_file,
overwrite=True,
output_verify='ignore')
self._logger.info(
f'Wrote combined calibration file: {output_master_file}')
return
def make_master_bias(self, list_of_biasfiles):
bias_list = [0]*len(list_of_biasfiles)
for ii, kk in enumerate(list_of_biasfiles):
fitsfile = fits.open(kk)
bias = ccdproc.CCDData(data=fitsfile[1].data, meta=fitsfile[1].header, unit="adu")
bias_list[ii] = ccdproc.subtract_overscan(bias, fits_section='[5:35, :]')
self.master_bias = ccdproc.combine(bias_list, method="median")
self.master_bias.write(self.output_dir+"/"+self.filename+"_masterbias.fits", overwrite=True)
def loaddark(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master dark frame by ccdproc.
Also bias corrects dark files if not already done.
"""
#master dark frame
dark_is_bias_corrected = False
dark_bias = None
namelist = self.loadauxname('dark', multi=True)
if (len(namelist) == 0):
self.log.error('Dark calibration frame(s) not found.')
raise RuntimeError('No dark file loaded')
darks = None
for name in namelist:
#is (any) dark file bias corrected?
header = fits.getheader(name)
if (header.get('BIAS') != None):
dark_is_bias_corrected = True
dark_bias = header.get('BIAS')
elif (header.get('BIASCORR') != None):
dark_is_bias_corrected = True
dark_bias = header.get('BIASCORR')
if (darks):
darks += ',' + name
else:
darks = name
self.log.debug('Creating master dark frame...')
#if there is just one, use it as darkfile or else combine all to make a master dark
if (len(namelist) == 1):
self.dark = ccdproc.CCDData.read(namelist[0],
unit='adu',
relax=True)
else:
self.dark = ccdproc.combine(darks,
method='median',
unit='adu',
add_keyword=False,
**{'verify': 'ignore'})
#bias correct, if necessary
if (not dark_is_bias_corrected):
#Subtracting master bias frame from master dark frame
self.dark = ccdproc.subtract_bias(self.dark,
self.bias,
add_keyword=False)
else:
self.log.debug(
'Master dark frame is *already* bias corrected (%s).' %
dark_bias)
# Finish up
self.darkloaded = True
self.darkname = namelist[0]
self.log.debug('LoadDark: done')
def _action_for_one_group(self, filter_dict=None):
combined_dict = self.apply_to.copy()
if filter_dict is not None:
combined_dict.update(filter_dict)
file_list = [os.path.join(self.image_source.location, f) for f in
self.image_source.files_filtered(**combined_dict)]
combine_keyword_args = {
'minmax_clip': self._clipping_widget.min_max,
'sigma_clip': self._clipping_widget.sigma_clip,
}
if self._combine_method.method == 'Average':
combine_keyword_args['method'] = 'average'
elif self._combine_method.method == 'Median':
combine_keyword_args['method'] = 'median'
if combine_keyword_args['minmax_clip']:
combine_keyword_args['minmax_clip_min'] = \
self._clipping_widget.min_max.min
combine_keyword_args['minmax_clip_max'] = \
self._clipping_widget.min_max.max
if combine_keyword_args['sigma_clip']:
combine_keyword_args['sigma_clip_low_thresh'] = \
self._clipping_widget.sigma_clip.min
combine_keyword_args['sigma_clip_low_thresh'] = \
self._clipping_widget.sigma_clip.min
combine_keyword_args['sigma_clip_func'] = np.ma.median
combine_keyword_args['sigma_clip_dev_func'] = \
median_absolute_deviation
if self._combine_method.scaling_func:
combine_keyword_args['scale'] = self._combine_method.scaling_func
combined = ccdproc.combine(file_list,
mem_limit=DEFAULT_MEMORY_LIMIT,
**combine_keyword_args)
sample_image = ccdproc.CCDData.read(file_list[0])
combined.header = sample_image.header
combined.header['master'] = True
if combined.data.dtype != sample_image.dtype:
combined.data = np.array(combined.data, dtype=sample_image.dtype)
try:
if isinstance(combined.uncertainty.array, np.ma.masked_array):
combined.uncertainty.array = np.array(combined.uncertainty.array)
except AttributeError:
pass
# Do not keep the mask or uncertainty if the data has neither
if sample_image.mask is None and sample_image.uncertainty is None:
combined.mask = None
combined.uncertainty = None
return combined
def bias_combine(refresh='2', method='2'):
tbiascollection = ImageFileCollection('Trimmed_Bias')
combtime = 0
if refresh == '1':
print('found', len(tbiascollection.values('file')), 'trimmed biases')
start = time.time()
if method == '1':
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
method='median')
combbiaspath = 'Master_Files/mbias_median.fits'
combined_bias.meta['combined'] = 'median'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
combined_bias.write(combbiaspath, overwrite=True)
elif method == '2':
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian,
sigma_clip_dev_func=mad_std)
combbiaspath = 'Master_Files/mbias.fits'
combined_bias.meta['combined'] = 'sigma_clip average'
combtime = time.time() - start
print('combination took', combtime, 'seconds')
combined_bias.write(combbiaspath, overwrite=True)
else:
try:
if method == '1':
combined_bias = CCDData.read('Master_Files/mbias_median.fits',
unit='adu')
combbiaspath = 'Master_Files/mbias_median.fits'
elif method == '2':
combined_bias = CCDData.read('Master_Files/mbias.fits',
unit='adu')
combbiaspath = 'Master_Files/mbias.fits'
except:
print('can\'t locate master bias, create or check directory')
sys.exit()
return tbiascollection, combined_bias, combbiaspath, combtime
def flat_creation(bdf_files, calibrated_path,calibrated_images, output_path, master_images, args):
set(bdf_files.summary['exptime'][bdf_files.summary['imagetyp'] == 'Flat Field'])
if args.cal_bias:
combined_bias = list(master_images.ccds(combined=True, imagetyp='Bias Frame'))[0]
if args.cal_dark:
combined_dark = CCDData.read(master_images.files_filtered(imagetyp='Dark Frame',
combined=True,
include_path=True)[0])
print('list of the flat files')
for a_flat, f_name in bdf_files.ccds(imagetyp='Flat Field', return_fname=True, ccd_kwargs={'unit': 'adu'}):
print(f_name)
if args.cal_bias:
print('BIAS')
a_flat = ccdp.subtract_bias(a_flat, combined_bias)
print('DONE BIAS')
if args.cal_dark:
print('DARK')
a_flat = ccdp.subtract_dark(a_flat, combined_dark, exposure_time='EXPOSURE', exposure_unit=u.s, scale=True)
print('done dark')
a_flat.write(calibrated_path / f_name, overwrite=True)
calibrated_images.refresh()
flats = calibrated_images.summary['imagetyp'] == 'Flat Field'
flat_filters = set(calibrated_images.summary['filter'][flats])
print('flat filters:', flat_filters)
for filtr in sorted(flat_filters):
calibrated_flats = calibrated_images.files_filtered(imagetyp='Flat Field', filter=filtr,
include_path=True)
print(len(calibrated_flats))
combined_flat = ccdp.combine(calibrated_flats,
method='median', scale=inv_median,
sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
signma_clip_dev_func=mad_std,
mem_limit=350e6)
combined_flat.meta['combined'] = True
flat_file_name = f'combined_flat_{filtr}.fit'
combined_flat.write(output_path / flat_file_name, overwrite=True)
calibrated_images.refresh()
master_images.refresh()
return flat_filters
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
# Find master bias to subtract from master dark
biaslist = self.loadauxname('bias', multi=False)
if (len(biaslist) == 0):
self.log.error('No bias calibration frames found.')
self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True)
# Create empy list for filenames of loaded frames
filelist = []
for fin in self.datain:
self.log.debug("Input filename = %s" % fin.filename)
filelist.append(fin.filename)
# Make a dummy dataout
self.dataout = DataFits(config=self.config)
if len(self.datain) == 0:
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file(s) loaded')
self.log.debug('Creating master flat frame...')
# Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias
if (len(filelist) == 1):
self.dark = ccdproc.CCDData.read(filelist[0],
unit='adu',
relax=True)
self.dark = ccdproc.subtract_bias(self.dark,
self.bias,
add_keyword=False)
else:
darklist = []
for i in filelist:
dark = ccdproc.CCDData.read(i, unit='adu', relax=True)
darksubbias = ccdproc.subtract_bias(dark,
self.bias,
add_keyword=False)
darklist.append(darksubbias)
self.dark = ccdproc.combine(darklist,
method=self.getarg('combinemethod'),
unit='adu',
add_keyword=True)
# set output header, put image into output
self.dataout.header = self.datain[0].header
self.dataout.imageset(self.dark)
# rename output filename
outputfolder = self.getarg('outputfolder')
if outputfolder != '':
outputfolder = os.path.expandvars(outputfolder)
self.dataout.filename = os.path.join(outputfolder,
os.path.split(filelist[0])[1])
else:
self.dataout.filename = filelist[0]
# Add history
self.dataout.setheadval('HISTORY',
'MasterDark: %d files used' % len(filelist))
def _combine_calib_images(
images: List[Image], bias: Optional[Image] = None, normalize: bool = False, method: str = "average"
) -> Image:
"""Combine a list of given images.
Args:
images: List of images to combine.
bias: If given, subtract from images before combining them.
normalize: If True, images are normalized to median of 1 before and after combining them.
method: Method for combining images.
"""
import ccdproc
# get CCDData objects
data = [image.to_ccddata() for image in images]
# subtract bias?
if bias is not None:
bias_data = bias.to_ccddata()
data = [ccdproc.subtract_bias(d, bias_data) for d in data]
# normalize?
if normalize:
data = [d.divide(np.median(d.data), handle_meta="first_found") for d in data]
# combine image
combined = ccdproc.combine(
data,
method=method,
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
mem_limit=350e6,
unit="adu",
combine_uncertainty_function=np.ma.std,
)
# normalize?
if normalize:
combined = combined.divide(np.median(combined.data), handle_meta="first_found")
# to Image and copy header
image = Image.from_ccddata(combined)
# add history
for i, src in enumerate(images, 1):
basename = src.header["FNAME"].replace(".fits.fz", "").replace(".fits", "")
image.header["L1AVG%03d" % i] = (basename, "Image used for average")
image.header["RLEVEL"] = (1, "Reduction level")
# finished
return image
def bias_combine(refresh=0):
tbiascollection = ImageFileCollection('Trimmed_Bias')
print('found', len(tbiascollection.values('file')), 'trimmed biases')
combined_bias = ccdp.combine(tbiascollection.files_filtered(
imtype='trimmed bias', include_path=True),
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.nanmedian,
sigma_clip_dev_func=mad_std)
combined_bias.meta['combined'] = True
combined_bias.write('Master_Files/mbias.fits', overwrite=True)
return tbiascollection
def align_combine_images(list_files, fitsfile, ref_image_fits=None, precision=100, dout=None, hexp='EXPTIME', force=False,
minmax_clip=False, minmax_clip_min=0.0, sigma_clip=True, date=None, clip_extrema=False, func=np.ma.median, sigclip=5,
cosmic=False, mbox=15, rbox=15, gbox=11, cleantype="medmask", cosmic_method='lacosmic', sky=False, dict_sky={},
dict_combine={}, suffix=None, hfilter='FILTER', key_file='file', hobj='OBJECT', ext=0, method='median',
align=True, **kwargs):
if ref_image_fits is None:
tobj = getTimeTable(list_files, key_time=hexp, key_file=key_file, ext=ext, abspath=True, mask=-1, sort=True, clean=True)
ref_image_fits, list_files = tobj[key_file][-1], tobj[key_file][:-1].data.tolist()
ref_image = CCDData.read(ref_image_fits)
ref_image = ccdproc.cosmicray_lacosmic(ref_image, sigclip=sigclip)
lccd = [ref_image]
lexp = [ref_image.header[hexp]]
all_images = [os.path.basename(ref_image_fits)]
for img in list_files:
image = fits2CCDData(img, single=True)
image_suffix = suffix if suffix is not None else img.split('.')[-1]
if 'REFIMA' in image.header and 'IMAGES' in image.header and not force:
continue
if cosmic:
image = cleanCosmic(image, mbox=mbox, rbox=rbox, gbox=gbox, sigclip=sigclip, cleantype=cleantype, cosmic_method=cosmic_method)
if align:
offset_image = shiftImage(image, ref_image, precision=precision, **kwargs)
img_shifted = '%s_shift.%s' % (img.split('.fit')[0], image_suffix)
all_images.append(os.path.basename(img_shifted))
img_shifted = join_path(img_shifted, dout)
offset_image.write(img_shifted, clobber=True)
lccd.append(offset_image)
else:
all_images.append(os.path.basename(img))
lccd.append(image)
lexp.append(image.header[hexp])
#iraf.unlearn("imshift")
#iraf.imshift.interp_type = "spline3"
#iraf.imshift(img, img, shift[1], shift[0])
# Combine images
lexp = np.array(lexp)
scale_func = None # 1. / lexp
combine = ccdproc.combine(lccd, method=method, scale=scale_func, minmax_clip=minmax_clip, func=func,
minmax_clip_min=minmax_clip_min, sigma_clip=sigma_clip, clip_extrema=clip_extrema, **dict_combine)
if sky:
combine = subtract_sky_ccd(combine, **dict_sky)
combine.header['IMAGES'] = str(' | '.join(all_images))
combine.header['REFIMA'] = os.path.basename(ref_image_fits)
combine.header['IMGSEXP'] = ' | '.join(map(str,lexp[1:].tolist() + [lexp[0]]))
combine.header['CMETHOD'] = method
dir_out = os.path.dirname(ref_image_fits) if dout is None else dout
fitsfile = join_path(fitsfile, dir_out)
combine.header['FILENAME'] = os.path.basename(fitsfile)
combine.header['CCDVER'] = VERSION
combine.write(fitsfile, clobber=True)
def run(self):
list_of_data = []
for f in self.input_list:
hdr = pyfits.getheader(f)
data = pyfits.getdata(f)
data = CCDData(data, unit=u.adu)
list_of_data.append(data)
# Parameter obtained from PySOAR, written by Luciano Fraga
master_bias = combine(list_of_data, method='average', mem_limit=6.4e7,
minmax_clip=True)
master_bias.header = hdr
if self.output_filename is None:
master_bias.write('0ZERO.fits')
else:
master_bias.write(self.output_filename)
def open_files_ccdproc_combine_nochunk(kind):
"""
Open files indirectly as part of ccdproc.combine, ensuring that the
task is not broken into chunks.
"""
global combo
paths = sorted(list(TMPPATH.glob('**/*.' + ALLOWED_EXTENSIONS[kind])))
# We ensure there are no chunks by setting a memory limit large
# enough to hold everything.
with fits.open(paths[0]) as hdulist:
array_size = hdulist[0].data.nbytes
# Why 2x the number of files? To make absolutely sure we don't
# end up chunking the job.
array_size *= 2 * len(paths)
combo = combine(paths)
def open_files_ccdproc_combine_chunk(kind):
"""
Open files indirectly as part of ccdproc.combine, ensuring that the
task is broken into chunks.
"""
global combo
paths = sorted(list(TMPPATH.glob('**/*.' + ALLOWED_EXTENSIONS[kind])))
# We want to force combine to break the task into chunks even
# if the task really would fit in memory; it is in that case that
# we end up with too many open files. We'll open one file, determine
# the size of the data in bytes, and set the memory limit to that.
# That will mean lots of chunks (however many files there are plus one),
# but lots of chunks is fine.
with fits.open(paths[0]) as hdulist:
array_size = hdulist[0].data.nbytes
combo = combine(paths, mem_limit=array_size)
def run(self):
list_of_data = []
for f in self.input_list:
self.debug('Processing file: {:s}'.format(f))
hdr = pyfits.getheader(f)
data = pyfits.getdata(f)
x_center = data.shape[1] // 2
x_bsize = int(0.05 * data.shape[1])
x1, x2 = x_center - x_bsize, x_center + x_bsize
x_where = np.zeros_like(data)
x_where[:, x1:x2] = 1
y_center = data.shape[0] // 2
y_bsize = int(0.05 * data.shape[0])
y1, y2 = y_center - y_bsize, y_center + y_bsize
y_where = np.zeros_like(data)
y_where[y1:y2, :] = 1
where = np.where(x_where * y_where == 1, True, False)
norm_factor = np.median(data[where])
data /= norm_factor
data = CCDData(data, unit=u.adu)
list_of_data.append(data)
# Parameter obtained from PySOAR, written by Luciano Fraga
master_flat = combine(list_of_data, method='median', mem_limit=6.4e7,
sigma_clip=True)
master_flat.header = hdr
if self.output_filename is None:
filter_name = hdr['FILTERS'].strip()
binning = int(hdr['CCDSUM'].strip().split(' ')[0])
self.debug('Binning: {:d}'.format(binning))
filename = '1NSFLAT{0:d}x{0:d}_{1:s}.fits'.format(binning, filter_name)
master_flat.write(filename, overwrite=True)
else:
master_flat.write(self.output_filename, overwrite=True)
def makeMasterBias(images):
"""
Make a master bias using all biases found in
images object
TODO: Finish docstring
"""
try:
master_bias = CCDData.read('master_bias.fits', unit=u.adu)
return master_bias
except FileNotFoundError:
bias_list = []
for f in images.files_filtered(imagetyp=BIAS_KEYWORD):
print(f)
ccd = CCDData.read(f, unit=u.adu)
bias_list.append(ccd)
try:
master_bias = combine(bias_list, method='median')
master_bias.write('master_bias.fits', clobber=True)
return master_bias
except IndexError:
return None
def run(self, input_files, output_file='o.fits'):
from astropy.io import fits as pyfits
from astropy import units as u
from ccdproc import CCDData, combine
list_of_data = []
for f in input_files:
hdr = pyfits.getheader(f)
data = pyfits.getdata(f)
data = CCDData(data, unit=u.adu)
list_of_data.append(data)
final = combine(list_of_data, method='median',
mem_limit=6.4e7,
sigma_clip=True,
sigma_clip_low_thresh=3.0,
sigma_clip_high_thresh=3.0,
scale='mode',
statsec="[200:800,200:800]")
final.write(output_file, clobber=True)
def run(self, input_files, output_file=None):
list_of_data = []
for f in input_files:
hdr = pyfits.getheader(f)
data = pyfits.getdata(f)
data = CCDData(data, unit=u.adu)
list_of_data.append(data)
master_flat = combine(list_of_data, method='median',
mem_limit=6.4e7,
sigma_clip=True,
sigma_clip_low_thresh=3.0,
sigma_clip_high_thresh=3.0,
scale=self.mode)
master_flat.header = hdr
if output_file is None:
master_flat.write('nSFLAT_{:s}.fits'.format(hdr['FILTERS']),
clobber=True)
else:
master_flat.write(output_file, clobber=True)
def create_master_flats(self, flat_group, target_name=''):
"""Creates master flats
Using a list of compatible flat images it combines them using median and
1-sigma clipping. Also it apply all previous standard calibrations to
each image.
Args:
flat_group (DataFrame): :class:`~pandas.DataFrame` instance.
Contains a list of compatible flat images
target_name (str): Science target name. This is used in some science
case uses only.
Returns:
The master flat :class:`~astropy.nddata.CCDData` instance and the name of under which
the master flat was stored.
"""
flat_file_list = flat_group.file.tolist()
cleaned_flat_list = []
master_flat_list = []
master_flat_name = None
self.log.info('Creating Master Flat')
for flat_file in flat_file_list:
# print(f_file)
image_full_path = os.path.join(self.args.raw_path, flat_file)
ccd = read_fits(image_full_path, technique=self.technique)
self.log.debug('Loading flat image: ' + image_full_path)
if master_flat_name is None:
master_flat_name = self.name_master_flats(
header=ccd.header,
group=flat_group,
target_name=target_name)
if self.technique == 'Spectroscopy':
ccd = image_overscan(ccd, overscan_region=self.overscan_region)
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
ccd = ccdproc.subtract_bias(ccd,
self.master_bias,
add_keyword=False)
ccd.header['GSP_BIAS'] = (
os.path.basename(self.master_bias_name),
'Master bias image')
elif self.technique == 'Imaging':
ccd = image_trim(ccd=ccd,
trim_section=self.trim_section,
trim_type='trimsec')
ccd = ccdproc.subtract_bias(ccd,
self.master_bias,
add_keyword=False)
ccd.header['GSP_BIAS'] = (
os.path.basename(self.master_bias_name),
'Master bias image')
else:
self.log.error('Unknown observation technique: ' +
self.technique)
if self._is_file_saturated(ccd=ccd):
self.log.warning('Removing saturated image {:s}. '
'Use --saturation to change saturation '
'level'.format(flat_file))
continue
else:
cleaned_flat_list.append(flat_file)
master_flat_list.append(ccd)
if master_flat_list != []:
master_flat = ccdproc.combine(master_flat_list,
method='median',
sigma_clip=True,
sigma_clip_low_thresh=1.0,
sigma_clip_high_thresh=1.0,
add_keyword=False)
# add name of images used to create master bias
for n in range(len(cleaned_flat_list)):
master_flat.header['GSP_IC{:02d}'.format(n + 1)] = (
cleaned_flat_list[n],
'Image used to create master flat')
write_fits(ccd=master_flat,
full_path=master_flat_name,
combined=True)
self.log.info('Created Master Flat: ' + master_flat_name)
return master_flat, master_flat_name
# print(master_flat_name)
else:
self.log.error('Empty flat list. Check that they do not exceed the '
'saturation limit.')
return None, None
def create_master_bias(self, bias_group):
"""Create Master Bias
Given a :class:`~pandas.DataFrame` object that contains a list of compatible bias.
This function creates the master flat using ccdproc.combine using median
and 3-sigma clipping.
Args:
bias_group (object): :class:`~pandas.DataFrame` instance that contains a list
of bias images compatible with each other.
"""
bias_file_list = bias_group.file.tolist()
default_bias_name = os.path.join(self.args.red_path, 'master_bias.fits')
search_bias_name = re.sub('.fits', '*.fits', default_bias_name)
n_bias = len(glob.glob(search_bias_name))
if n_bias > 0:
self.master_bias_name = re.sub('.fits',
'_{:d}.fits'.format(n_bias + 1),
default_bias_name)
self.log.info('New name for master bias: ' + self.master_bias_name)
else:
self.master_bias_name = default_bias_name
master_bias_list = []
self.log.info('Creating master bias')
for image_file in bias_file_list:
image_full_path = os.path.join(self.args.raw_path, image_file)
ccd = read_fits(image_full_path, technique=self.technique)
self.log.debug('Loading bias image: ' + image_full_path)
if self.technique == 'Spectroscopy':
self.log.debug(
'Overscan Region: {:s}'.format(str(self.overscan_region)))
ccd = image_overscan(ccd=ccd,
overscan_region=self.overscan_region)
ccd = image_trim(ccd,
trim_section=self.trim_section,
trim_type='trimsec')
master_bias_list.append(ccd)
# combine bias for spectroscopy
self.master_bias = ccdproc.combine(master_bias_list,
method='median',
sigma_clip=True,
sigma_clip_low_thresh=3.0,
sigma_clip_high_thresh=3.0,
add_keyword=False)
# add name of images used to create master bias
for n in range(len(bias_file_list)):
self.master_bias.header['GSP_IC{:02d}'.format(n + 1)] = (
bias_file_list[n],
'Image used to create master bias')
write_fits(ccd=self.master_bias,
full_path=self.master_bias_name,
combined=True)
self.log.info('Created master bias: ' + self.master_bias_name)
return self.master_bias, self.master_bias_name
if not os.path.isdir(outdir): os.mkdir(outdir)
os.chdir(outdir)
#change this to point to your raw data directory
ic1 = ImageFileCollection(indir)
#create the bias frames
blue_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Blue arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
blue_bias_list.append(ccd)
master_bias_blue = ccdproc.combine(blue_bias_list, method='median')
master_bias_blue.write('master_bias_blue.fits', clobber=True)
red_bias_list = []
for filename in ic1.files_filtered(obstype='Bias', isiarm='Red arm'):
print ic1.location + filename
ccd = CCDData.read(ic1.location + filename, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
red_bias_list.append(ccd)
master_bias_red = ccdproc.combine(red_bias_list, method='median')
master_bias_red.write('master_bias_red.fits', clobber=True)
#create the flat fields
red_flat_list = []
indices=np.where((array_ftype == f)&(array_filter == element))
if len(indices[0]) > 0:
outfile = open(ftype_filter,'w')
for i in indices[0]:
outfile.write(fnames[i]+'\n')
outfile.close()
#os.remove('tempflats')
for f in flat_filelist:
print 'filelist = ',f
flatimages = []
try:
filelist = open(f,'r')
except IOError:
print('Problem opening file ',f)
print('Hope that is ok...')
for q in filelist: flatimages.append(q.rstrip())
# combine flat images using average combine, scale by median, sigma clip
flat = ccdproc.combine(flatimages,scale=np.median,method='average',sigma_clip=True,unit=u.adu)
#med_flat = ccdproc.combine(flatimages, method='median')
# normalize flat image by dividing by mean
norm_flat = flat / np.mean(flat)
print 'writing fits'
norm_flat.write('n'+f+'.fits', overwrite=True)
'''
import glob
import os
import numpy as np
import ccdproc
import astropy.units as u
import argparse
'''
from pyraf import iraf
iraf.noao()
iraf.imred()
iraf.ccdred()
'''
parser = argparse.ArgumentParser(description ='Groups images by filter and creates flatfield images')
parser.add_argument('--filestring', dest='filestring', default='bias', help='match string for input files (default = bias, which gets bias*.fits)')
args = parser.parse_args()
files = sorted(glob.glob(args.filestring+'*.fits'))
zeros = ccdproc.combine(fimages,method='average',sigma_clip=True,unit=u.adu)
print('writing fits')
zeros.write(args.filestring+'-combined.fits',overwrite=True)
def hrsbias(rawpath, outpath, link=False, mem_limit=1e9, sdb=None, clobber=True):
"""hrsbias processes the HRS red and blue bias frames in a directory
Parameters
----------
rawpath: string
Path to raw data
outpath: string
Path to output result
link: boolean
Add symbolic link to HRS_CALS directory
clobber: boolean
Overwrite existing files
"""
if not os.path.isdir(rawpath): return
image_list = ImageFileCollection(rawpath)
if len(image_list.files)==0: return
#make output directory
if not os.path.isdir(outpath): os.mkdir(outpath)
obsdate=get_obsdate(image_list.summary['file'][0])
#process the red bias frames
matches = (image_list.summary['obstype'] == 'Bias') * (image_list.summary['detnam'] == 'HRDET')
rbias_list = []
for fname in image_list.summary['file'][matches]:
ccd = red_process(rawpath+fname)
rbias_list.append(ccd)
if sdb is not None: dq_ccd_insert(rawpath + fname, sdb)
if rbias_list:
if os.path.isfile("{0}/RBIAS_{1}.fits".format(outpath, obsdate)) and clobber:
os.remove("{0}/RBIAS_{1}.fits".format(outpath, obsdate))
rbias = ccdproc.combine(rbias_list, method='median', output_file="{0}/RBIAS_{1}.fits".format(outpath, obsdate), mem_limit=mem_limit)
del rbias_list
#process the red bias frames
matches = (image_list.summary['obstype'] == 'Bias') * (image_list.summary['detnam'] == 'HBDET')
hbias_list = []
for fname in image_list.summary['file'][matches]:
ccd = blue_process(rawpath+fname)
hbias_list.append(ccd)
if sdb is not None: dq_ccd_insert(rawpath + fname, sdb)
if hbias_list:
if os.path.isfile("{0}/HBIAS_{1}.fits".format(outpath, obsdate)) and clobber:
os.remove("{0}/HBIAS_{1}.fits".format(outpath, obsdate))
hbias = ccdproc.combine(hbias_list, method='median', output_file="{0}/HBIAS_{1}.fits".format(outpath, obsdate), mem_limit=mem_limit)
del hbias_list
#provide the link to the bias frame
if link:
ldir = '/salt/HRS_Cals/CAL_BIAS/{0}/{1}/'.format(obsdate[0:4], obsdate[4:8])
if not os.path.isdir(ldir): os.mkdir(ldir)
ldir = '/salt/HRS_Cals/CAL_BIAS/{0}/{1}/product'.format(obsdate[0:4], obsdate[4:8])
if not os.path.isdir(ldir): os.mkdir(ldir)
infile="{0}/RBIAS_{1}.fits".format(outpath, obsdate)
link='/salt/HRS_Cals/CAL_BIAS/{0}/{1}/product/RBIAS_{2}.fits'.format(obsdate[0:4], obsdate[4:8], obsdate)
if os.path.isfile(link) and clobber: os.remove(link)
os.symlink(infile, link)
infile="{0}/HBIAS_{1}.fits".format(outpath, obsdate)
link='/salt/HRS_Cals/CAL_BIAS/{0}/{1}/product/HBIAS_{2}.fits'.format(obsdate[0:4], obsdate[4:8], obsdate)
if os.path.isfile(link) and clobber: os.remove(link)
os.symlink(infile, link)
def hrsflat(rawpath, outpath, detname, obsmode, master_bias=None, f_limit=1000, first_order=53,
y_start=30, y_limit=3920, smooth_length=20, smooth_fraction=0.4, filter_size=151,
link=False, sdb=None, clobber=True):
"""hrsflat processes the HRS flatfields. It will process for a given detector and a mode
Parameters
----------
rawpath: string
Path to raw data
outpath: string
Path to output result
link: boolean
Add symbolic link to HRS_CALS directory
sdb: sdb_user.mysql
SDB object to upload data quality
clobber: boolean
Overwrite existing files
"""
if not os.path.isdir(rawpath): return
image_list = ImageFileCollection(rawpath)
if len(image_list.files)==0: return
#make output directory
if not os.path.isdir(outpath): os.mkdir(outpath)
#get the observing date
obsdate=get_obsdate(image_list.summary['file'][0])
#setup the instrument prefix
if detname=='HRDET':
prefix='R'
process = red_process
elif detname=='HBDET':
prefix='H'
process = blue_process
else:
raise ValueError('detname must be a valid HRS Detector name')
#process the flat frames
matches = (image_list.summary['obstype'] == 'Flat field') * (image_list.summary['detnam'] == detname) * (image_list.summary['obsmode'] == obsmode)
flat_list = []
for fname in image_list.summary['file'][matches]:
ccd = process(rawpath+fname, masterbias=master_bias)
flat_list.append(ccd)
if sdb is not None: dq_ccd_insert(rawpath + fname, sdb)
if flat_list:
outfile = "{0}/{2}FLAT_{1}_{3}.fits".format(outpath, obsdate, prefix, obsmode.replace(' ', '_'))
if os.path.isfile(outfile) and clobber: os.remove(outfile)
flat = ccdproc.combine(flat_list, method='median', output_file=outfile)
norm = clean_flatimage(flat.data, filter_size=filter_size, flux_limit=0.3,
block_size=100, percentile_low=30, median_size=5)
hdu = fits.PrimaryHDU(norm)
hdu.writeto(prefix+'norm.fits', clobber=True)
norm[norm>0]=1
ys, xs = norm.shape
xc = int(xs/2.0)
if detname=='HRDET':
xc = 1947 #int(xs/2.0)
ndata = norm[:,xc]
detect_kern = ndata[1:100]
#these remove light that has bleed at the edges and may need adjusting
norm[:,:20]=0
norm[:,4040:]=0
elif detname=='HBDET':
ndata = norm[:,xc]
detect_kern = ndata[32:110]
frame = create_orderframe(norm, first_order, xc, detect_kern, smooth_length=smooth_length,
smooth_fraction=smooth_fraction, y_start=y_start, y_limit=y_limit)
order_file = "{0}/{2}ORDER_{1}_{3}.fits".format(outpath, obsdate, prefix, obsmode.replace(' ', '_'))
hdu = fits.PrimaryHDU(frame)
hdu.writeto(prefix+'order.fits', clobber=True)
hdu.writeto(order_file, clobber=True)
if link:
link='/salt/HRS_Cals/CAL_FLAT/{0}/{1}/product/{2}'.format(obsdate[0:4], obsdate[4:8], os.path.basename(outfile))
if os.path.isfile(link) and clobber: os.remove(link)
os.symlink(outfile, link)
olink='/salt/HRS_Cals/CAL_FLAT/{0}/{1}/product/{2}'.format(obsdate[0:4], obsdate[4:8], os.path.basename(order_file))
if os.path.isfile(olink) and clobber: os.remove(olink)
os.symlink(order_file, olink)
def imcombine(filelist, out, options, method="average", reject="none",\
lsigma=3, hsigma=3, mclip=False,\
nlow=None, nhigh=None):
'''Combines images in input list with optional rejection algorithms.
Args:
filelist: The list of files to imcombine
out: The full path to the output file
method: either "average" or "median" combine
options: Options dictionary
bpmask: The full path to the bad pixel mask
reject: none, minmax, sigclip
nlow,nhigh: Parameters for minmax rejection, see iraf docs
mclip: use median as the function to calculate the baseline values for
sigclip rejection?
lsigma, hsigma: low and high sigma rejection thresholds.
Returns:
None
Side effects:
Creates the imcombined file at location `out'
'''
assert method in ['average', 'median']
if os.path.exists(out):
os.remove(out)
if reject == 'none':
info('Combining files using ccdproc.combine task')
info(' reject=none')
for file in filelist:
debug(' Combining: {}'.format(file))
ccdproc.combine(filelist, out, method=method,\
minmax_clip=False,\
iraf_minmax_clip=True,\
sigma_clip=False,\
unit="adu")
info(' Done.')
elif reject == 'minmax':
## The IRAF imcombine minmax rejection behavior is different than the
## ccdproc minmax rejection behavior. We are using the IRAF like
## behavior here. To support this a pull request for the ccdproc
## package has been made:
## https://github.com/astropy/ccdproc/pull/358
##
## Note that the ccdproc behavior still differs slightly from the
## nominal IRAF behavior in that the rejection does not consider whether
## any of the rejected pixels have been rejected for other reasons, so
## if nhigh=1 and that pixel was masked for some other reason, the
## new ccdproc algorithm, will not mask the next highest pixel, it will
## still just mask the highest pixel even if it is already masked.
##
## From IRAF (help imcombine):
## nlow = 1, nhigh = 1 (minmax)
## The number of low and high pixels to be rejected by the
## "minmax" algorithm. These numbers are converted to fractions
## of the total number of input images so that if no rejections
## have taken place the specified number of pixels are rejected
## while if pixels have been rejected by masking, thresholding, or
## non-overlap, then the fraction of the remaining pixels,
## truncated to an integer, is used.
##
## Check that minmax rejection is possible given the number of images
if nlow is None:
nlow = 0
if nhigh is None:
nhigh = 0
if nlow + nhigh >= len(filelist):
warning('nlow + nhigh >= number of input images. Combining without rejection')
nlow = 0
nhigh = 0
if ccdproc.version.major >= 1 and ccdproc.version.minor >= 1\
and ccdproc.version.release:
info('Combining files using ccdproc.combine task')
info(' reject=clip_extrema')
info(' nlow={}'.format(nlow))
info(' nhigh={}'.format(nhigh))
for file in filelist:
info(' {}'.format(file))
ccdproc.combine(filelist, out, method=method,\
minmax_clip=False,\
clip_extrema=True,\
nlow=nlow, nhigh=nhigh,\
sigma_clip=False,\
unit="adu")
info(' Done.')
else:
## If ccdproc does not have new rejection algorithm in:
## https://github.com/astropy/ccdproc/pull/358
## Manually perform rejection using ccdproc.combiner.Combiner object
info('Combining files using local clip_extrema rejection algorithm')
info('and the ccdproc.combiner.Combiner object.')
info(' reject=clip_extrema')
info(' nlow={}'.format(nlow))
info(' nhigh={}'.format(nhigh))
for file in filelist:
info(' {}'.format(file))
ccdlist = []
for file in filelist:
ccdlist.append(ccdproc.CCDData.read(file, unit='adu', hdu=0))
c = ccdproc.combiner.Combiner(ccdlist)
nimages, nx, ny = c.data_arr.mask.shape
argsorted = np.argsort(c.data_arr.data, axis=0)
mg = np.mgrid[0:nx,0:ny]
for i in range(-1*nhigh, nlow):
where = (argsorted[i,:,:].ravel(),
mg[0].ravel(),
mg[1].ravel())
c.data_arr.mask[where] = True
if method == 'average':
result = c.average_combine()
elif method == 'median':
result = c.median_combine()
for key in list(ccdlist[0].header.keys()):
header_entry = ccdlist[0].header[key]
if key != 'COMMENT':
result.header[key] = (header_entry,
ccdlist[0].header.comments[key])
hdul = result.to_hdu()
# print(hdul)
# for hdu in hdul:
# print(type(hdu.data))
hdul[0].writeto(out)
# result.write(out)
info(' Done.')
elif reject == 'sigclip':
info('Combining files using ccdproc.combine task')
info(' reject=sigclip')
info(' mclip={}'.format(mclip))
info(' lsigma={}'.format(lsigma))
info(' hsigma={}'.format(hsigma))
baseline_func = {False: np.mean, True: np.median}
ccdproc.combine(filelist, out, method=method,\
minmax_clip=False,\
clip_extrema=False,\
sigma_clip=True,\
sigma_clip_low_thresh=lsigma,\
sigma_clip_high_thresh=hsigma,\
sigma_clip_func=baseline_func[mclip],\
sigma_clip_dev_func=np.std,\
)
info(' Done.')
else:
raise NotImplementedError('{} rejection unrecognized by MOSFIRE DRP'.format(reject))
