def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction, dark subtraction, overscan correction, flat correction, and trimming image.')
raw = CCDData.read(sci,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of science image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)#dark_exposure=1*u.second, data_exposure=red.header['EXPTIME']*u.second, scale=True)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
red = ccdproc.flat_correct(red, mflat)
processed_data = ccdproc.ccd_process(red, trim=raw.header['DATASEC'])
log.info('File proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci,processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(red.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (510, 510), filter_size=(9, 9),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits'),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['EXPTIME']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/'),overwrite=True)
processed.append(final)
return processed, masks
def ccdproc_images_filter(list_files, image_filter=None, master_flat=None, master_bias=None, fits_section=None, gain=None, readnoise=None,
error=False, sky=True, dout=None, cosmic=False, mbox=15, rbox=15, gbox=11, cleantype="medmask", cosmic_method='lacosmic',
sigclip=5, key_filter='filter', dfilter={'imagetyp':'LIGHT'}, mask=None, key_find='find', invert_find=False, **kwargs):
if error and (gain is None or readnoise is None):
print ('WARNING: You need to provide "gain" and "readnoise" to compute the error!')
return
if gain is not None and not isinstance(gain, u.Quantity):
gain = gain * u.electron / u.adu
if readnoise is not None and not isinstance(readnoise, u.Quantity):
readnoise = readnoise * u.electron
if dfilter is not None and key_filter is not None and image_filter is not None:
dfilter = addKeysListDict(dfilter, {key_filter: image_filter})
list_files = getListFiles(list_files, dfilter, mask, key_find=key_find, invert_find=invert_find)
dccd = {}
for filename in list_files:
ccd = CCDData.read(filename, unit= u.adu)
nccd = ccdproc.ccd_process(ccd, trim=fits_section, gain=gain, master_bias=master_bias, master_flat=master_flat, readnoise=readnoise, error=error)
for key in ccd.header:
if not key in nccd.header:
nccd.header[key] = ccd.header[key]
# Better get rid of the cosmic rays BEFORE subtracting the global sky background
if cosmic:
nccd = cleanCosmic(nccd, mbox=mbox, rbox=rbox, gbox=gbox, sigclip=sigclip, cleantype=cleantype, cosmic_method=cosmic_method)
if sky:
nccd = subtract_sky_ccd(nccd, **kwargs)
addKeyHdr(nccd.header, 'MBIAS', getFilename(master_bias))
addKeyHdr(nccd.header, 'MFLAT', getFilename(master_flat))
filename = 'c%s' % os.path.basename(filename)
dccd[filename] = nccd
filename = join_path(filename, dout)
nccd.header['FILENAME'] = os.path.basename(filename)
nccd.header['CCDVER'] = VERSION
nccd.header = ammendHeader(nccd.header)
nccd.write(filename, clobber=True)
return dccd
def create_flat(flat_list,fil,red_path,mbias=None,log=None):
log.info('Processing files for filter: '+fil)
log.info(str(len(flat_list))+' files found.')
flats = []
flat_scale = []
for flat in flat_list:
log.info('Loading file: '+flat)
raw = CCDData.read(flat,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
mask = make_source_mask(red, nsigma=3, npixels=5)
bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
masked = np.array(red)
masked[mask] = bkg.background[mask]
log.info('Median flat level: '+str(np.median(masked)))
norm = 1/np.median(masked[500:1500,500:1500])
log.info('Flat normalization: '+str(norm))
flat_scale.append(norm)
flats.append(CCDData(masked,meta=red.header,unit=u.electron))
mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True)
log.info('Created master flat for filter: '+fil)
mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True)
log.info('Master flat written to mflat_'+fil+'.fits')
return
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction and flat correction.')
with fits.open(sci) as hdr:
header = hdr[0].header
data = hdr[0].data
data[np.isnan(data)] = np.nanmedian(data)
raw = CCDData(data,meta=header,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron)
log.info('Exposure time of science image is '+str(red.header['TRUITIME']*red.header['COADDONE']))
flat = np.array(ccdproc.flat_correct(red, mflat))
flat[np.isnan(flat)] = np.nanmedian(flat)
processed_data = CCDData(flat,unit=u.electron,header=red.header,wcs=red.wcs)
log.info('File proccessed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci.replace('.gz',''),processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(raw.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (128, 128), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits').replace('.gz',''),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['TRUITIME']*red.header['COADDONE']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/').replace('.gz',''),overwrite=True)
processed.append(final)
return processed, masks
def create_flat(flat_list,fil,red_path,mbias=None,log=None):
log.info('Processing files for filter: '+fil)
log.info(str(len(flat_list))+' files found.')
flats = []
masks = []
flat_scale = []
for flat in flat_list:
log.info('Loading file: '+flat)
raw = CCDData.read(flat,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron)
log.info('Exposure time of image is '+str(red.header['TRUITIME']*red.header['COADDONE']))
mask = make_source_mask(red, nsigma=3, npixels=5)
bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
masked = np.array(red)
masked[mask] = bkg.background[mask]
log.info('Median flat level: '+str(np.median(masked)))
norm = 1/np.median(masked[224:1824,224:1824])
log.info('Flat normalization: '+str(norm))
flat_scale.append(norm)
flats.append(CCDData(masked,meta=red.header,unit=u.electron))
mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True)
log.info('Created master flat for filter: '+fil)
mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True)
log.info('Master flat written to mflat_'+fil+'.fits')
return
def m4kproc( fitsfd ): """m4kproc merges and trims the M4k Images """ dummy_name = "dummy.fits" if os.path.exists( dummy_name ): os.remove( dummy_name ) fitsfd.writeto( dummy_name ) #CRVALs are reversed. This will probably be fixe soon. # and this will need to be removed. for ext in [1,2]: ra=fitsfd[ext].header['CRVAL2'] dec=fitsfd[ext].header['CRVAL1'] fitsfd[ext].header['CRVAL2']=dec fitsfd[ext].header['CRVAL1']=ra amp1 = ccdproc.fits_ccddata_reader( dummy_name, hdu=1 ) amp2 = ccdproc.fits_ccddata_reader( dummy_name, hdu=2 ) procdata1 = ccdproc.ccd_process( amp1, oscan=fitsfd[1].header['biassec'], trim=fitsfd[1].header['TRIMSEC'] ) procdata2 = ccdproc.ccd_process( amp2, oscan=fitsfd[2].header['biassec'], trim=fitsfd[2].header['TRIMSEC'] ) os.remove( dummy_name ) return procdata1, procdata2
def reduce(ccd, master_dark, master_flat, readnoise, gain):
"""
reduce raw data frame for science.
Steps: make dark, make flat, then complete reduction using
ccdproc.ccd_process which dark subtracts and flat correct
Args:
path: path to calibration frames directory to be combined
ccd: CCDData to be reduced
readnoise: Quantity, read noise from CCD
gain: Quantity, gain from CCD
Returns:
reduced frame
"""
# get exposure times for dark and data
dark_exposure = master_dark.header['EXPTIME'] * u.s
data_exposure = ccd.header['EXPTIME'] * u.s
# assign units to gain and readnoise
gain = gain * u.electron / u.adu
readnoise = readnoise * u.electron
# Gain correct dark and flat before processing
master_dark = ccdproc.gain_correct(master_dark, gain)
master_flat = ccdproc.gain_correct(master_flat, gain)
reduced_ccd = ccdproc.ccd_process(ccd,
oscan=None,
trim=None,
error=True,
master_bias=None,
dark_frame=master_dark,
master_flat=master_flat,
bad_pixel_mask=None,
gain=gain,
readnoise=readnoise,
oscan_median=True,
oscan_model=None,
min_value=None,
dark_exposure=dark_exposure,
data_exposure=data_exposure,
exposure_key=None,
exposure_unit=u.s,
dark_scale=True)
return reduced_ccd
def tiled_process(bads: List[CCDData], flat: CCDData,
images: List[CCDData]) -> List[CCDData]:
"""
Do the ccdproc operation on a list of images. includes some extra logic for NOTCAM images to get the
gain and readnoise out of the headers
:param bads:
:param flat:
:param images:
:return:
"""
if bads:
bad = reduce(lambda x, y: x.astype(bool) | y.astype(bool),
(i.data for i in bads)) # combine bad pixel masks
else:
bad = None
reduceds = []
for image in images:
image.mask = bad
# TODO this is really sketchy as it does 4x the work
# Tiling: http://www.not.iac.es/instruments/notcam/guide/observe.html#reductions
# 0-> LL, 1->LR, 2->UR, 3->UL
tile_table = [
None, s_[512:, 0:512], s_[512:, 512:], s_[0:512, 512:], s_[0:512,
0:512]
]
reduced = image.copy()
for idx in range(1, 5):
gain = image.header['GAIN' + str(idx)]
readnoise = image.header['RDNOISE' + str(idx)]
tile = ccdproc.ccd_process(
image,
oscan=None,
error=True,
gain=gain * u.electron / u.count,
# TODO check if this is right or counts->adu required
readnoise=readnoise * u.electron,
dark_frame=None,
master_flat=flat,
bad_pixel_mask=bad)
image.data[tile_table[idx]] = tile.data[tile_table[idx]]
reduced.header = tile.header
reduced.wcs = tile.wcs
reduced.unit = tile.unit
reduceds.append(reduced)
return reduceds
def realtimeRed(storePath, analyPath, masterDark):
neos = ccdproc.ImageFileCollection(location=analyPath)
neoList = []
for neo, fname in neos.hdus(return_fname=True):
meta = neo.header
meta['filename'] = fname
neoList.append(ccdproc.CCDData(data=neo.data, header=meta, unit="adu"))
masterBias_e = ccdproc.gain_correct(masterBias, gain=1 * u.electron / u.adu)
masterDark_e = ccdproc.gain_correct(masterDark, gain=1 * u.electron / u.adu)
masterFlat_e = ccdproc.gain_correct(masterFlat, gain=1 * u.electron / u.adu)
for neo in neoList:
neo_red = ccdproc.ccd_process(neo, master_bias=masterBias_e, dark_frame=masterDark_e, master_flat=masterFlat_e
, gain=1 * u.electron / u.adu, readnoise=readnoise, min_value=1.
, dark_exposure=darkExp * u.second, data_exposure=neo.header['exptime'] * u.second
, exposure_unit=u.second, dark_scale=True)
baseName = os.path.basename(neo.header['filename'])
fits.writeto("{}{}_red.fits".format(storePath, baseName.split('.')[0]), neo_red.data, header=neo_red.header, overwrite=False)
def create_dark(cal_list,cal,mbias,red_path,log):
images = []
processed = []
for dark in cal_list[cal]:
log.info('Creating master dark with exposure time: '+cal.split('_')[1])
log.info('Loading file: '+dark)
try:
raw = CCDData.read(dark, hdu=1, unit='adu')
images.append(dark)
except astropy.io.registry.IORegistryError:
log.error('File not recognized.')
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron, master_bias=mbias)
processed.append(red)
log.info('Creating master dark.')
mdark = ccdproc.combine(processed,method='median')
log.info('Master dark created.')
mdark_hdu = fits.PrimaryHDU(mdark)
mdark_hdu.header['VER'] = (__version__, 'Version of telescope parameter file used.')
for i,im in enumerate(images):
mdark_hdu.header['FILE'+str(i+1)] = im
mdark_hdu.header['EXPTIME'] = (np.float(cal.split('_')[1]), 'Exposure time of master dark (sec).')
mdark_hdu.writeto(red_path+cal+'.fits',overwrite=True)
log.info('Master dark written to '+cal+'.fits')
return
def single_reduction(image, bad, flat):
image.mask = bad
if "GAIN" in image.header: # ALF
gain = image.header['GAIN']
readnoise = image.header['RDNOISE']
else: # NOTCAM
# TODO that's from the quicklook-package, probably would want to do this individually for every sensor area
gain = (image.header['GAIN1'] + image.header['GAIN2'] +
image.header['GAIN3'] + image.header['GAIN4']) / 4
readnoise = (image.header['RDNOISE1'] + image.header['RDNOISE2'] +
image.header['RDNOISE3'] + image.header['RDNOISE4']) / 4
reduced = ccdproc.ccd_process(
image,
oscan=None,
error=True,
gain=gain * u.electron / u.count,
# TODO check if this is right or counts->adu required
readnoise=readnoise * u.electron,
dark_frame=None,
master_flat=flat,
bad_pixel_mask=bad,
add_keyword=None)
return reduced
infiles = args.infile
print infiles
if args.bias:
mbias = CCDData.read(args.bias, unit = u.adu)
else:
mbias = None
if args.flat:
raise Exception('Flat fielding is not currently implemented')
if args.dark:
raise Exception('Dark correction is not currently implemented')
for infile in infiles:
ccd = CCDData.read(infile, unit = u.adu)
ccd = ccdproc.ccd_process(ccd, oscan='[1117:1181, 1:330]', oscan_median=True,
trim='[17:1116,1:330]', master_bias=mbias,
error=True, gain=1.0 * u.electron/u.adu,
readnoise=5.0 * u.electron)
if args.cray:
ccd = ccdproc.cosmicray_lacosmic(ccd, sigclip=4.5, sigfrac=0.3,
objlim=5.0, gain=1.0, readnoise=6.5,
satlevel=65536.0, pssl=0.0, niter=4,
sepmed=True, cleantype='meanmask', fsmode='median',
psfmodel='gauss', psffwhm=2.5, psfsize=7,
psfk=None, psfbeta=4.765, verbose=False)
ccd.write('p'+os.path.basename(infile), clobber=True)
#%%
f_name_bias = 'bias-median.fits'
f_name_dark = 'dark-median.fits'
f_name_flat0 = 'flat0-median.fits'
#%%
# (1) Open master bias, dark, and flat
bias = CCDData.read(dir_name + f_name_bias, unit='adu')
dark = CCDData.read(dir_name + f_name_dark, unit='adu')
flat0 = CCDData.read(dir_name + f_name_flat0,
unit='adu') # Bias NOT subtracted
#%%
# (2) Reduce each object image separately.
# Then save it with prefix 'p_' which means "preprocessed"
for objname in file_list:
obj = CCDData.read(objname, unit='adu')
reduced = ccd_process(
obj, # The input image
master_bias=bias, # Master bias
dark_frame=dark, # dark
master_flat=flat0, # non-calibrated flat
min_value=30000, # flat.min() should be 30000
exposure_key='exptime', # exposure time keyword
exposure_unit=u.s, # exposure time unit
dark_scale=True) # whether to scale dark frame
reduced.data = np.array(reduced.data, dtype=np.uint16)
print('reduced', reduced.data.dtype, reduced.data.min(),
reduced.data.max(), reduced)
reduced.write(objname[:-5] + '_p.fits', overwrite=True)
def create_bias(cal_list):
for i, bias_file in enumerate(cal_list['BIAS4']):
with fits.open(bias_file) as hdr:
header = hdr[0].header
bias_raw = [
CCDData.read(bias_file, hdu=x + 1, unit='adu') for x in range(4)
]
bias_processed = [
ccdproc.ccd_process(x,
oscan=x[:, 0:13],
oscan_model=models.Chebyshev1D(3),
trim='[13:2060,1:2601]',
gain=gain * u.electron / u.adu,
readnoise=rdnoise * u.electron)
for j, x in enumerate(bias_raw)
]
bias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)])
for x in bias_processed:
bias_hdu.append(fits.ImageHDU(x.data, header=x.header))
bias_hdu.writeto(bias_file.replace(cal_path, red_path), overwrite=True)
cal_list['BIAS4'][i] = bias_file.replace(cal_path, red_path)
if len(cal_list['BIAS4']) != 0:
mbias4 = [
ccdproc.combine(cal_list['BIAS4'], hdu=x + 1, unit=u.electron)
for x in range(4)
]
mbias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)])
for x in mbias4:
mbias_hdu.append(fits.ImageHDU(x.data, header=x.header))
mbias_hdu.writeto(red_path + 'mbias4.fits', overwrite=True)
for bias in cal_list['BIAS4']:
os.remove(bias)
else:
mbias4 = None
for i, bias_file in enumerate(cal_list['BIAS8']):
with fits.open(bias_file) as hdr:
header = hdr[0].header
bias_raw = [
CCDData.read(bias_file, hdu=x + 1, unit='adu') for x in range(8)
]
bias_processed = [
ccdproc.ccd_process(x,
oscan=x[:, 0:13],
oscan_model=models.Chebyshev1D(3),
trim='[13:1036,1:2601]',
gain=gain * u.electron / u.adu,
readnoise=rdnoise * u.electron)
for j, x in enumerate(bias_raw)
]
bias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)])
for x in bias_processed:
bias_hdu.append(fits.ImageHDU(x.data, header=x.header))
bias_hdu.writeto(bias_file.replace(cal_path, red_path), overwrite=True)
cal_list['BIAS8'][i] = bias_file.replace(cal_path, red_path)
if len(cal_list['BIAS8']) != 0:
mbias8 = [
ccdproc.combine(cal_list['BIAS8'], hdu=x + 1, unit=u.electron)
for x in range(8)
]
mbias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)])
for x in mbias8:
mbias_hdu.append(fits.ImageHDU(x.data, header=x.header))
mbias_hdu.writeto(red_path + 'mbias8.fits', overwrite=True)
for bias in cal_list['BIAS8']:
os.remove(bias)
else:
mbias8 = None
return mbias4, mbias8
break
scale = []
for i, flat_file in enumerate(cal_list[cal]):
with fits.open(flat_file) as hdr:
header = hdr[0].header
data = hdr[int(int(cal[-1]) / 2)].data
scale.append(1 / np.median(data[1200:2000, 800:1600]))
flat_raw = [
CCDData.read(flat_file, hdu=x + 1, unit='adu')
for x in range(int(cal[-1]))
]
flat_processed = [
ccdproc.ccd_process(x,
oscan=x[:, 0:13],
oscan_model=models.Chebyshev1D(3),
trim=x.header['DATASEC'],
gain=gain * u.electron / u.adu,
readnoise=rdnoise * u.electron,
master_bias=mbias[k],
gain_corrected=True)
for k, x in enumerate(flat_raw)
]
flat_hdu = fits.HDUList([fits.PrimaryHDU(header=header)])
for x in flat_processed:
flat_hdu.append(fits.ImageHDU(x.data, header=x.header))
flat_hdu.writeto(flat_file.replace(cal_path, red_path),
overwrite=True)
cal_list[cal][i] = flat_file.replace(cal_path, red_path)
mflat = [
ccdproc.combine(cal_list[cal],
hdu=x + 1,
unit=u.electron,
def main(night_path, skip_list_file, mask_file, overwrite=False, plot=False):
"""
See argparse block at bottom of script for description of parameters.
"""
night_path = path.realpath(path.expanduser(night_path))
if not path.exists(night_path):
raise IOError("Path '{}' doesn't exist".format(night_path))
logger.info("Reading data from path: {}".format(night_path))
base_path, night_name = path.split(night_path)
data_path, run_name = path.split(base_path)
output_path = path.realpath(
path.join(data_path, 'processed', run_name, night_name))
os.makedirs(output_path, exist_ok=True)
logger.info("Saving processed files to path: {}".format(output_path))
if plot: # if we're making plots
plot_path = path.realpath(path.join(output_path, 'plots'))
logger.debug("Will make and save plots to: {}".format(plot_path))
os.makedirs(plot_path, exist_ok=True)
else:
plot_path = None
# check for files to skip (e.g., saturated or errored exposures)
if skip_list_file is not None: # a file containing a list of filenames to skip
with open(skip_list_file, 'r') as f:
skip_list = [x.strip() for x in f if x.strip()]
else:
skip_list = None
# look for pixel mask file
if mask_file is not None:
with open(
mask_file, 'r'
) as f: # load YAML file specifying pixel masks for nearby sources
pixel_mask_spec = yaml.load(f.read())
else:
pixel_mask_spec = None
# generate the raw image file collection to process
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
logger.info("Frames to process:")
logger.info("- Bias frames: {}".format(
len(ic.files_filtered(imagetyp='BIAS'))))
logger.info("- Flat frames: {}".format(
len(ic.files_filtered(imagetyp='FLAT'))))
logger.info("- Comparison lamp frames: {}".format(
len(ic.files_filtered(imagetyp='COMP'))))
logger.info("- Object frames: {}".format(
len(ic.files_filtered(imagetyp='OBJECT'))))
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
# ============================
# Create the master bias frame
# ============================
# overscan region of the CCD, using FITS index notation
oscan_fits_section = "[{}:{},:]".format(oscan_idx, oscan_idx + oscan_size)
master_bias_file = path.join(output_path, 'master_bias.fits')
if not os.path.exists(master_bias_file) or overwrite:
# get list of overscan-subtracted bias frames as 2D image arrays
bias_list = []
for hdu, fname in ic.hdus(return_fname=True, imagetyp='BIAS'):
logger.debug('Processing Bias frame: {0}'.format(fname))
ccd = CCDData.read(path.join(ic.location, fname), unit='adu')
ccd = ccdproc.gain_correct(ccd, gain=ccd_gain)
ccd = ccdproc.subtract_overscan(ccd, overscan=ccd[:, oscan_idx:])
ccd = ccdproc.trim_image(ccd,
fits_section="[1:{},:]".format(oscan_idx))
bias_list.append(ccd)
# combine all bias frames into a master bias frame
logger.info("Creating master bias frame")
master_bias = ccdproc.combine(bias_list,
method='average',
clip_extrema=True,
nlow=1,
nhigh=1,
error=True)
master_bias.write(master_bias_file, overwrite=True)
else:
logger.info("Master bias frame file already exists: {}".format(
master_bias_file))
master_bias = CCDData.read(master_bias_file)
if plot:
# TODO: this assumes vertical CCD
assert master_bias.shape[0] > master_bias.shape[1]
aspect_ratio = master_bias.shape[1] / master_bias.shape[0]
fig, ax = plt.subplots(1, 1, figsize=(10, 12 * aspect_ratio))
vmin, vmax = zscaler.get_limits(master_bias.data)
cs = ax.imshow(master_bias.data.T,
origin='bottom',
cmap=cmap,
vmin=max(0, vmin),
vmax=vmax)
ax.set_title('master bias frame [zscale]')
fig.colorbar(cs)
fig.tight_layout()
fig.savefig(path.join(plot_path, 'master_bias.png'))
plt.close(fig)
# ============================
# Create the master flat field
# ============================
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
master_flat_file = path.join(output_path, 'master_flat.fits')
if not os.path.exists(master_flat_file) or overwrite:
# create a list of flat frames
flat_list = []
for hdu, fname in ic.hdus(return_fname=True, imagetyp='FLAT'):
logger.debug('Processing Flat frame: {0}'.format(fname))
ccd = CCDData.read(path.join(ic.location, fname), unit='adu')
ccd = ccdproc.gain_correct(ccd, gain=ccd_gain)
ccd = ccdproc.ccd_process(ccd,
oscan=oscan_fits_section,
trim="[1:{},:]".format(oscan_idx),
master_bias=master_bias)
flat_list.append(ccd)
# combine into a single master flat - use 3*sigma sigma-clipping
logger.info("Creating master flat frame")
master_flat = ccdproc.combine(flat_list,
method='average',
sigma_clip=True,
low_thresh=3,
high_thresh=3)
master_flat.write(master_flat_file, overwrite=True)
# TODO: make plot if requested?
else:
logger.info("Master flat frame file already exists: {}".format(
master_flat_file))
master_flat = CCDData.read(master_flat_file)
if plot:
# TODO: this assumes vertical CCD
assert master_flat.shape[0] > master_flat.shape[1]
aspect_ratio = master_flat.shape[1] / master_flat.shape[0]
fig, ax = plt.subplots(1, 1, figsize=(10, 12 * aspect_ratio))
vmin, vmax = zscaler.get_limits(master_flat.data)
cs = ax.imshow(master_flat.data.T,
origin='bottom',
cmap=cmap,
vmin=max(0, vmin),
vmax=vmax)
ax.set_title('master flat frame [zscale]')
fig.colorbar(cs)
fig.tight_layout()
fig.savefig(path.join(plot_path, 'master_flat.png'))
plt.close(fig)
# =====================
# Process object frames
# =====================
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
logger.info("Beginning object frame processing...")
for hdu, fname in ic.hdus(return_fname=True, imagetyp='OBJECT'):
new_fname = path.join(output_path, 'p_{}'.format(fname))
# -------------------------------------------
# First do the simple processing of the frame
# -------------------------------------------
logger.debug("Processing '{}' [{}]".format(hdu.header['OBJECT'],
fname))
if path.exists(new_fname) and not overwrite:
logger.log(1, "\tAlready processed! {}".format(new_fname))
ext = SourceCCDExtractor(filename=path.join(
ic.location, new_fname),
plot_path=plot_path,
zscaler=zscaler,
cmap=cmap,
**ccd_props)
nccd = ext.ccd
# HACK: F**K this is a bad hack
ext._filename_base = ext._filename_base[2:]
else:
# process the frame!
ext = SourceCCDExtractor(filename=path.join(ic.location, fname),
plot_path=plot_path,
zscaler=zscaler,
cmap=cmap,
unit='adu',
**ccd_props)
_pix_mask = pixel_mask_spec.get(
fname, None) if pixel_mask_spec is not None else None
nccd = ext.process_raw_frame(pixel_mask_spec=_pix_mask,
master_bias=master_bias,
master_flat=master_flat)
nccd.write(new_fname, overwrite=overwrite)
# -------------------------------------------
# Now do the 1D extraction
# -------------------------------------------
fname_1d = path.join(output_path, '1d_{0}'.format(fname))
if path.exists(fname_1d) and not overwrite:
logger.log(1, "\tAlready extracted! {}".format(fname_1d))
continue
else:
logger.debug("\tExtracting to 1D")
# first step is to fit a voigt profile to a middle-ish row to determine LSF
lsf_p = ext.get_lsf_pars() # MAGIC NUMBER
try:
tbl = ext.extract_1d(lsf_p)
except Exception as e:
logger.error('Failed! {}: {}'.format(e.__class__.__name__,
str(e)))
continue
hdu0 = fits.PrimaryHDU(header=nccd.header)
hdu1 = fits.table_to_hdu(tbl)
hdulist = fits.HDUList([hdu0, hdu1])
hdulist.writeto(fname_1d, overwrite=overwrite)
del ext
# ==============================
# Process comparison lamp frames
# ==============================
# HACK:
ic = GlobImageFileCollection(night_path, skip_filenames=skip_list)
logger.info("Beginning comp. lamp frame processing...")
for hdu, fname in ic.hdus(return_fname=True, imagetyp='COMP'):
new_fname = path.join(output_path, 'p_{}'.format(fname))
logger.debug("\tProcessing '{}'".format(hdu.header['OBJECT']))
if path.exists(new_fname) and not overwrite:
logger.log(1, "\tAlready processed! {}".format(new_fname))
ext = CompCCDExtractor(filename=path.join(ic.location, new_fname),
plot_path=plot_path,
zscaler=zscaler,
cmap=cmap,
**ccd_props)
nccd = ext.ccd
# HACK: F**K this is a bad hack
ext._filename_base = ext._filename_base[2:]
else:
# process the frame!
ext = CompCCDExtractor(filename=path.join(ic.location, fname),
plot_path=plot_path,
unit='adu',
**ccd_props)
_pix_mask = pixel_mask_spec.get(
fname, None) if pixel_mask_spec is not None else None
nccd = ext.process_raw_frame(
pixel_mask_spec=_pix_mask,
master_bias=master_bias,
master_flat=master_flat,
)
nccd.write(new_fname, overwrite=overwrite)
# -------------------------------------------
# Now do the 1D extraction
# -------------------------------------------
fname_1d = path.join(output_path, '1d_{0}'.format(fname))
if path.exists(fname_1d) and not overwrite:
logger.log(1, "\tAlready extracted! {}".format(fname_1d))
continue
else:
logger.debug("\tExtracting to 1D")
try:
tbl = ext.extract_1d()
except Exception as e:
logger.error('Failed! {}: {}'.format(e.__class__.__name__,
str(e)))
continue
hdu0 = fits.PrimaryHDU(header=nccd.header)
hdu1 = fits.table_to_hdu(tbl)
hdulist = fits.HDUList([hdu0, hdu1])
hdulist.writeto(fname_1d, overwrite=overwrite)
if args.skip_red == 'True' or args.skip_red == 'yes':
if os.path.exists(red_path + 'mdark.fits'):
mdark = CCDData.read(red_path + 'mdark.fits', unit=u.electron)
process_bias = False
else:
log.error('No master dark found, creating master dark.')
if process_dark:
processed = []
for dark in dark_list:
try:
raw = CCDData.read(dark, hdu=1, unit='adu')
except astropy.io.registry.IORegistryError:
log.error('File ' + dark + ' not recognized.')
processed.append(
ccdproc.ccd_process(
raw,
gain=raw.header['GAIN'] * u.electron / u.adu,
readnoise=raw.header['RDNOISE'] * u.electron))
mdark = ccdproc.combine(processed, method='median')
mdark.write(red_path + 'mdark.fits', overwrite=True)
else:
con = input(
'No darks present. Continue without dark subtraction? (True or False) '
)
if con == 'True':
log.error('No darks present, continuing without dark subtraction.')
mdark = None
else:
log.critical('No darks present, check data before rerunning.')
logging.shutdown()
sys.exit(-1)
sys.exit(-1)
process_bias = True
if args.skip_red == 'True' or args.skip_red == 'yes':
if os.path.exists(red_path+'mbias_blue.fits') and os.path.exists(red_path+'mbias_red.fits'):
mbias_blue = [CCDData.read(red_path+'mbias_blue.fits', hdu=x+1, unit=u.electron) for x in range(4)]
mbias_red = [CCDData.read(red_path+'mbias_red.fits', hdu=x+1, unit=u.electron) for x in range(4)]
process_bias = False
else:
log.error('No bias master bias found, creating master bias.')
if process_bias:
for i, bias_file in enumerate(cal_list['BIAS_blue']):
with fits.open(bias_file) as hdr:
header = hdr[0].header
bias_raw = [CCDData.read(bias_file, hdu=x+1, unit='adu') for x in range(4)]
bias_processed = [ccdproc.ccd_process(x, oscan=x[:,0:51], oscan_model=models.Chebyshev1D(3), trim=x.header['DATASEC'], gain=gain_blue[j]*u.electron/u.adu, readnoise=rdnoise_blue[j]*u.electron) for j,x in enumerate(bias_raw)]
bias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)])
for x in bias_processed: bias_hdu.append(fits.ImageHDU(x.data,header=x.header))
bias_hdu.writeto(bias_file.replace(cal_path,red_path),overwrite=True)
cal_list['BIAS_blue'][i] = bias_file.replace(cal_path,red_path)
mbias_blue = [ccdproc.combine(cal_list['BIAS_blue'],hdu=x+1,unit=u.electron) for x in range(4)]
mbias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)])
for x in mbias_blue: mbias_hdu.append(fits.ImageHDU(x.data,header=x.header))
mbias_hdu.writeto(red_path+'mbias_blue.fits',overwrite=True)
for bias in cal_list['BIAS_blue']: os.remove(bias)
for i, bias_file in enumerate(cal_list['BIAS_red']):
with fits.open(bias_file) as hdr:
header = hdr[0].header
bias_raw = [CCDData.read(bias_file, hdu=x+1, unit='adu') for x in range(4)]
bias_processed = [ccdproc.ccd_process(x, oscan=x[:,0:7], oscan_model=models.Chebyshev1D(3), trim=x.header['DATASEC'], gain=gain_red[j]*u.electron/u.adu, readnoise=rdnoise_red[j]*u.electron) for j,x in enumerate(bias_raw)]
bias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)])
def process_science(sci_list,
fil,
red_path,
mbias=None,
mflat=None,
proc=None,
log=None):
masks = []
processed = []
flat_left = mflat[0]
flat_right = mflat[1]
left_list = []
right_list = []
if proc:
for j, sci in enumerate(sci_list):
log.info('Loading file: ' + sci)
log.info('Applying flat correction and trimming left image.')
left = CCDData.read(sci, hdu=1, unit=u.electron)
left = ccdproc.flat_correct(left, flat_left)
left = ccdproc.ccd_process(left, trim=left.header['DATASEC'])
log.info('Left image proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(left, nsigma=3, npixels=5)
clean, com_mask = create_mask.create_mask(sci, left,
'_mask_left.fits',
static_mask(proc)[0],
mask,
saturation(left.header),
binning(proc, 'left'),
rdnoise(left.header),
cr_clean_sigclip(),
cr_clean_sigcfrac(),
cr_clean_objlim(), log)
left.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(left, (120, 120),
filter_size=(3, 3),
sigma_clip=SigmaClip(sigma=3),
bkg_estimator=MeanBackground(),
mask=mask,
exclude_percentile=80)
log.info('Median background for left iamge: ' +
str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/',
'/red/').replace('.fits',
'_bkg_left.fits'),
bkg.background,
overwrite=True)
left = left.subtract(CCDData(bkg.background, unit=u.electron),
propagate_uncertainties=True,
handle_meta='first_found')
log.info('Exposure time of left image is ' +
str(left.header['EXPTIME']))
left = left.divide(left.header['EXPTIME'],
propagate_uncertainties=True,
handle_meta='first_found')
log.info(
'Background subtracted and image divided by exposure time.')
left.header['DATASEC'] = '[1:' + str(
np.shape(left)[1]) + ',1:' + str(np.shape(left)[0]) + ']'
left_list.append(left)
log.info('Applying flat correction and trimming right image.')
right = CCDData.read(sci, hdu=2, unit=u.electron)
right = ccdproc.flat_correct(right, flat_right)
right = ccdproc.ccd_process(right, trim=right.header['DATASEC'])
log.info('Right image proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(right, nsigma=3, npixels=5)
clean, com_mask = create_mask.create_mask(sci, right,
'_mask_right.fits',
static_mask(proc)[1],
mask,
saturation(right.header),
binning(proc, 'right'),
rdnoise(right.header),
cr_clean_sigclip(),
cr_clean_sigcfrac(),
cr_clean_objlim(), log)
right.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(right, (120, 120),
filter_size=(3, 3),
sigma_clip=SigmaClip(sigma=3),
bkg_estimator=MeanBackground(),
mask=mask,
exclude_percentile=80)
log.info('Median background for right image : ' +
str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/',
'/red/').replace('.fits',
'_bkg_right.fits'),
bkg.background,
overwrite=True)
right = right.subtract(CCDData(bkg.background, unit=u.electron),
propagate_uncertainties=True,
handle_meta='first_found')
log.info('Exposure time of right image is ' +
str(right.header['EXPTIME']))
right = right.divide(right.header['EXPTIME'],
propagate_uncertainties=True,
handle_meta='first_found')
log.info(
'Background subtracted and image divided by exposure time.')
right.header['DATASEC'] = '[1:' + str(
np.shape(right)[1]) + ',1:' + str(np.shape(right)[0]) + ']'
right_list.append(right)
else:
for j, sci in enumerate(sci_list):
log.info('Loading file: ' + sci)
log.info(
'Applying gain correction, overscan correction, flat correction, and trimming image.'
)
with fits.open(sci) as hdr:
header_left = hdr[1].header
header_right = hdr[6].header
data_list = []
for i in range(8):
data = ccdproc.CCDData.read(sci, hdu=i + 1, unit=u.adu)
red = ccdproc.ccd_process(data,
oscan=data[:, 0:50],
oscan_model=models.Chebyshev1D(3),
trim='[1200:2098,210:2056]',
gain=gain()[i] * u.electron / u.adu,
readnoise=4 * u.electron)
data_list.append(np.asarray(red).astype(np.float32))
top_left = np.concatenate(
[data_list[0], np.fliplr(data_list[1])], axis=1)
bot_left = np.flipud(
np.concatenate(
[data_list[3], np.fliplr(data_list[2])], axis=1))
left = CCDData(np.concatenate([top_left, bot_left]),
unit=u.electron,
header=header_left)
left = ccdproc.flat_correct(left, flat_left[209:3903, 1149:2947])
log.info('Left image proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(left, nsigma=3, npixels=5)
# clean, com_mask = create_mask.create_mask(sci,left,static_mask(proc)[0],mask,saturation(left.header),binning(proc,'left'),rdnoise(left.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
# processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(left, (120, 120),
filter_size=(3, 3),
sigma_clip=SigmaClip(sigma=3),
bkg_estimator=MeanBackground(),
mask=mask,
exclude_percentile=80)
log.info('Median background for left image : ' +
str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/',
'/red/').replace('.fits',
'_bkg_left.fits'),
bkg.background,
overwrite=True)
left = left.subtract(CCDData(bkg.background, unit=u.electron),
propagate_uncertainties=True,
handle_meta='first_found')
log.info('Exposure time of left image is ' +
str(left.header['EXPTIME']))
left = left.divide(left.header['EXPTIME'] * u.second,
propagate_uncertainties=True,
handle_meta='first_found')
log.info(
'Background subtracted and image divided by exposure time.')
left.header['DATASEC'] = '[1:1798,1:3694]'
left.header['RADECSYS'] = 'ICRS'
left.header['CUNIT1'] = 'deg'
left.header['CUNIT2'] = 'deg'
left.header['CTYPE1'] = 'RA---TAN'
left.header['CTYPE2'] = 'DEC--TAN'
left.header['CRPIX1'] = 2301
left.header['CRPIX2'] = 1846
coord = util.parse_coord(left.header['RA'], left.header['DEC'])
left.header['CRVAL1'] = coord.ra.deg
left.header['CRVAL2'] = coord.dec.deg
left.header['PC1_1'] = -pixscale() / 3600 * np.sin(
np.pi / 180. * (left.header['POSANG'] + 90))
left.header['PC1_2'] = pixscale() / 3600 * np.cos(
np.pi / 180. * (left.header['POSANG'] + 90))
left.header['PC2_1'] = -pixscale() / 3600 * np.cos(
np.pi / 180. * (left.header['POSANG'] + 90))
left.header['PC2_2'] = pixscale() / 3600 * np.sin(
np.pi / 180. * (left.header['POSANG'] + 90))
left.write(sci.replace('/raw/',
'/red/').replace('.fits', '_left.fits'),
overwrite=True)
left_list.append(left)
top_right = np.concatenate(
[data_list[6], np.fliplr(data_list[7])], axis=1)
bot_right = np.flipud(
np.concatenate(
[data_list[5], np.fliplr(data_list[4])], axis=1))
right = CCDData(np.concatenate([top_right, bot_right]),
unit=u.electron,
header=header_right)
right = ccdproc.flat_correct(right, flat_right[209:3903,
1149:2947])
log.info('Right image proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(right, nsigma=3, npixels=5)
# clean, com_mask = create_mask.create_mask(sci,right,static_mask(proc)[1],mask,saturation(right.header),binning(proc,'right'),rdnoise(right.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
# processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(right, (120, 120),
filter_size=(3, 3),
sigma_clip=SigmaClip(sigma=3),
bkg_estimator=MeanBackground(),
mask=mask,
exclude_percentile=80)
log.info('Median background for right image : ' +
str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/',
'/red/').replace('.fits',
'_bkg_right.fits'),
bkg.background,
overwrite=True)
right = right.subtract(CCDData(bkg.background, unit=u.electron),
propagate_uncertainties=True,
handle_meta='first_found')
log.info('Exposure time of right image is ' +
str(right.header['EXPTIME']))
right = right.divide(right.header['EXPTIME'] * u.second,
propagate_uncertainties=True,
handle_meta='first_found')
log.info(
'Background subtracted and image divided by exposure time.')
right.header['DATASEC'] = '[1:1798,1:3694]'
right.header['RADECSYS'] = 'ICRS'
right.header['CUNIT1'] = 'deg'
right.header['CUNIT2'] = 'deg'
right.header['CTYPE1'] = 'RA---TAN'
right.header['CTYPE2'] = 'DEC--TAN'
right.header['CRPIX1'] = -504
right.header['CRPIX2'] = 1845
coord = util.parse_coord(right.header['RA'], right.header['DEC'])
right.header['CRVAL1'] = coord.ra.deg
right.header['CRVAL2'] = coord.dec.deg
right.header['PC1_1'] = -pixscale() / 3600 * np.sin(
np.pi / 180. * (right.header['POSANG'] + 90))
right.header['PC1_2'] = pixscale() / 3600 * np.cos(
np.pi / 180. * (right.header['POSANG'] + 90))
right.header['PC2_1'] = -pixscale() / 3600 * np.cos(
np.pi / 180. * (right.header['POSANG'] + 90))
right.header['PC2_2'] = pixscale() / 3600 * np.sin(
np.pi / 180. * (right.header['POSANG'] + 90))
right.write(sci.replace('/raw/',
'/red/').replace('.fits', '_right.fits'),
overwrite=True)
right_list.append(right)
return [left_list, right_list], None
def ccdred_list(image_path_list, mbias_path, mflat_path):
"""
Given a list of FITS files process it by applying master calibrations and
overscan.
Arguments
---------
image_file : list like with strings
Path to the files to process
mbias_path : str
Path to the master bias
mflat_path : str
Path to the master flats
File transformations
--------------------
Re-write FITS file, with overscan, bias and flat corrections and
updated header.
Returns
-------
None
"""
# Load masters
master_bias = fits.open(mbias_path)
master_flat = fits.open(mflat_path)
image_indices = _check_image_extensions(master_bias)
for image_file in image_path_list:
# Looping over extensions
for i in image_indices:
# Get CCDData object for this extension
ccd = CCDData.read(image_file, unit="adu", hdu=i)
mbias_ccd = CCDData(data=master_bias[i].data,
header=master_bias[i].header,
unit="adu")
mflat_ccd = CCDData(data=master_flat[i].data,
header=master_flat[i].header,
unit="adu")
if 'CCDPROC' in ccd.header:
print(
f"Skipping image: {image_file:1s}[{i:1.0f}] - Already processed."
)
continue
# Overscan section variables
biassec = None
trimsec = None
if "BIASSEC" in ccd.header:
biassec = ccd.header["BIASSEC"]
if "TRIMSEC" in ccd.header:
trimsec = ccd.header["TRIMSEC"]
# Applying corrections
ccd = ccdproc.ccd_process(ccd,
oscan=biassec,
trim=trimsec,
master_bias=mbias_ccd,
master_flat=mbias_ccd)
## Can add more information on the function above for the pixel by
## pixel error calculation (e.g. gain, read noise).
# Updating existing image
ccd.header["CCDPROC"] = True # Added processed flag
fits.update(image_file,
ccd.data.astype(np.float32),
header=ccd.header,
ext=i)
print(f"Processed image: {image_file}")
master_bias.close()
master_flat.close()
#################################################
######## FIND DARK WITH CLOSEST EXPOSURE TIME
#################################################
# find dark with closest exposure time
sci_exptime = header['EXPTIME']
# find dark with closest exposure time
delta_t = abs(sci_exptime - dark_exptimes_set)
closest_dark = dark_exptimes_set[delta_t == min(delta_t)]
# open the appropriate dark
hdu1 = fits.open('dark-combined-'+str(int(closest_dark[0]))+'.fits')
dheader = hdu1[0].header
gaincorrected_dark = CCDData(hdu1[0].data, unit=u.electron, meta=dheader)
hdu1.close()
newccd = ccdproc.ccd_process(ccd,error=True, gain=gain, readnoise=rdnoise, dark_frame=gaincorrected_dark, exposure_key='exposure', exposure_unit=u.second,master_flat = gaincorrected_master_flat, gain_corrected=True)
header['HISTORY'] = '= Processing by ccdproc: dark, flat '
#fits.writeto('fdb'+f,newccd,header, overwrite=True)
fits.writeto('fd'+f,newccd,header, overwrite=True)
hdu1.close()
else:
print('skipping science frames')
'''
# then move on to running scamp and swarp to solve WCS
# and make mosaics
# let's start 01-28-2019 data
'''
dark_list, # ccdproc does not accept numpy.ndarray, but only python list.
method='median', # default is average so I specified median.
unit='adu')
#%%
#save fits file
dark0.write(dir_name + f_name_dark0, overwrite=True)
print('dark0', dark0.data.min(), dark0.data.max(), dark0)
# This dark isn't bias subtracted yet, so let's subtract bias:
#%%
# (1) Open master bias
bias = CCDData.read(dir_name + f_name_bias, unit='adu')
print('bias', bias.data.min(), bias.data.max(), bias.data)
# `unit='adu'` does not necessarily be set if you have header keyword for it.
#%%
# (2) Subtract bias:
dark = ccd_process(dark0, master_bias=bias)
print('dark', dark.data.min(), dark.data.max(), dark.data)
# This automatically does "dark-bias"
# You can do it by the function "subtract_bias", or just normal pythonic arithmetic.
#%%
# (3) Sigma clip the dark
dark_clip = sigma_clip(dark)
print('dark_clip', dark_clip.data.min(), dark_clip.data.max(), dark_clip.data)
# by default, astropy.stats.sigma_clip does "3-sigma, 10-iterations" clipping.
# You can tune the parameters by optional keywords (e.g., sigma, iters, ...).
# dark_clip is "numpy masked array". It contains data, mask, and filled_value.
# filled_value is the value which is used to fill the data which are masked (rejected).
# I will change filled_value to be the median value.
#%%
# (4) For rejected pixels, insert the median value
dark_clip.fill_value = np.median(dark_clip.data)
def imreduce(target,
bias=None,
dark=None,
flat=None,
expkey='EXPTIME',
expunit=u.s,
normkey='NORMALZD',
biaskey='BIASSUB',
maskfile=None,
fixpix=False):
# must read bias and dark in here since CCDData not picklable
bias = CCDData.read(bias, unit='adu') if bias else None
dark = CCDData.read(dark, unit='adu') if dark else None
if dark:
# if dark is already normalized, don't normalize again
if normkey in dark.header and dark.header[normkey]:
print(dark.header[normkey])
dark_exposure = None
dark_scale = False
else:
dark_exposure = dark.header[expkey] * expunit
dark_scale = True
# if bias frame, bias subtract darks first
if bias:
# if dark frame is already bias subtracted, skip
if biaskey in dark.header and dark.header[biaskey]:
pass
else:
dark = subtract_bias(dark, bias)
else:
dark_exposure = None
dark_scale = False
if flat:
if isinstance(flat, str):
# flat passed in as string, must read to CCDData
ffile = flat
flat = CCDData.read(ffile, unit='adu')
flat.header['OFNAME'] = ffile
# ccdproc has issues with nans when flatfielding
# -> replace nans with 1
mask = np.isnan(flat.data)
flat.data[mask] = 1.
if isinstance(target, str):
# target passed in as string, must read to CCDData
tfile = target
target = CCDData.read(tfile, unit='adu')
target.header['OFNAME'] = tfile
data_exposure = target.header[expkey] * expunit
# read in bad pixel mask if specified
if maskfile:
target.header.add_history('%s - masked points from %s' %
(os.path.basename(__file__), maskfile))
maskfile = fits.getdata(maskfile)
# do the ccd reduction!
ccd = ccd_process(
target,
master_bias=bias,
dark_frame=dark,
master_flat=flat,
#exposure_key=expkey,exposure_unit=expunit,
data_exposure=data_exposure,
dark_exposure=dark_exposure,
dark_scale=dark_scale,
bad_pixel_mask=maskfile)
# fix bad pixels through grid interpolation
if fixpix:
if maskfile is None:
warn(RuntimeWarning('No maskfile specified: --fixpix ignored'))
else:
from photutils.utils import interpolate_masked_data
from photutils.utils import ShepardIDWInterpolator as idw
from scipy.interpolate import griddata
mask = np.asarray(maskfile, dtype=bool)
goodpoints = np.where(~mask)
goodvals = ccd.data[goodpoints]
badpoints = np.where(mask)
# interpolate and replace values
res = griddata(goodpoints, goodvals, badpoints)
ccd.data[badpoints] = res
'''
mask = np.asarray(maskfile,dtype=bool)
goodcoord = np.where(~mask)
print(goodcoord,len(goodcoord))
goodval = ccd.data[goodcoord]
print(len(goodval));exit()
iz = idw(goodcoord,goodval)
fixed = iz(np.where(mask))
ccd.data[mask] = fixed
#fixed,_,_ = interpolate_masked_data(ccd.data,np.asarray(maskfile,dtype=bool))
#ccd.data = fixed
'''
ccd.header.add_history('%s - interpolated over bad pixels' %
os.path.basename(__file__))
return ccd.to_hdu(hdu_mask=None, hdu_uncertainty=None)
else: #default to use science files for master flat creation
flat_type = 'sky'
log.info('Using science files to create master flat')
flat_list = sky_list[fil]
flat_scale = lambda arr: 1.0 / np.ma.median(
arr[224:1824, 224:1824]) #scaling function for normalization
if len(flat_list) > 31:
random.seed(4)
flat_list = random.sample(flat_list, k=31)
image_list = []
for image in flat_list:
sci_raw = CCDData.read(image, unit=u.adu)
image_list.append(
ccdproc.ccd_process(
sci_raw,
gain=sci_raw.header['SYSGAIN'] * u.electron / u.adu,
readnoise=readnoise[sci_raw.header['NUMREADS']] *
u.electron))
mask = [
make_source_mask(x, nsigma=1, npixels=5)
for x in image_list
] #####make into list
weights = np.ones([len(image_list), 2048, 2048])
for i, n in enumerate(image_list):
weights[i] *= np.median(
n[224:1824, 224:1824]) #flats weighted by median in flat
master_flat = ccdproc.combine(
image_list,
method='median',
weights=weights,
scale=flat_scale,
if args.dark_sigmaclip == 1:
dark_sigmaclip = True
else:
dark_sigmaclip = False
darkscales = np.array(darkscales)
#print(darkscales)
dark = combine(darklist_real,
method=args.dark_method,
unit='adu',
sigma_clip=dark_sigmaclip,
scale=darkscales)
if bias != None:
dark = ccd_process(dark, master_bias=bias)
dark.write(mdark)
hdulist = fits.open(mdark, mode='update')
hdulist[0].header.set(args.fits_header_exptime, args.dark_exptime)
hdulist[0].header.set(args.fits_header_ccdtemp, temperature)
darkexp = args.dark_exptime
hdulist.flush()
hdulist.close()
elif args.masterdark != None:
darkcal = True
mdark = args.masterdark
if os.path.isfile(mdark) == False:
right.write(
red_path +
os.path.basename(sci).replace('.fits', '_right.fits'),
overwrite=True)
right_list.append(right)
else:
with fits.open(sci) as hdr:
header_left = hdr[1].header
header_right = hdr[6].header
data_list = []
for i in range(8):
data = ccdproc.CCDData.read(sci, hdu=i + 1, unit=u.adu)
red = ccdproc.ccd_process(
data,
oscan=data[:, 0:50],
oscan_model=models.Chebyshev1D(3),
trim='[1200:2098,210:2056]',
gain=gain[i] * u.electron / u.adu,
readnoise=4 * u.electron)
data_list.append(np.asarray(red).astype(np.float32))
top_left = np.concatenate(
[data_list[0], np.fliplr(data_list[1])], axis=1)
bot_left = np.flipud(
np.concatenate(
[data_list[3], np.fliplr(data_list[2])], axis=1))
left = CCDData(np.concatenate([top_left, bot_left]),
unit=u.electron,
header=header_left,
wcs=wcs.WCS(header_left))
left = ccdproc.flat_correct(left, flat_left[209:3903,
1199:2997])
for x in range(12)
]
process_bias = False
else:
log.error('No bias master bias found, creatging master bias.')
if process_bias:
for i, bias_file in enumerate(cal_list['BIAS']):
with fits.open(bias_file) as hdr:
header = hdr[0].header
bias_raw = [
CCDData.read(bias_file, hdu=x + 1, unit='adu') for x in range(12)
]
bias_processed = [
ccdproc.ccd_process(x,
oscan=x.header['BIASSEC'],
oscan_model=models.Chebyshev1D(3),
trim=x.header['DATASEC'],
gain=x.header['GAIN'] * u.electron / u.adu,
readnoise=x.header['RDNOISE'] * u.electron)
for x in bias_raw
]
for x in bias_processed:
x.header['DATASEC'] = ('[1:256,1:2112]', 'updated DATASEC')
bias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)])
for x in bias_processed:
bias_hdu.append(fits.ImageHDU(x.data, header=x.header))
bias_hdu.writeto(bias_file.replace(cal_path, red_path).replace(
'.fits.bz2', '.fits'),
overwrite=True)
cal_list['BIAS'][i] = bias_file.replace(cal_path, red_path).replace(
'.fits.bz2', '.fits')
mbias = [
def t120_makecosmetic(work_dir=t120.t120_data_path,
orig_dir_root=t120.t120_orig_dir,
reduc_dir_root=t120.t120_redu_dir,
offset_dir_root=t120.t120_ofst_dir,
flat_dir_root=t120.t120_flat_dir,
dark_dir_root=t120.t120_dark_dir,
common_dir_root=t120.t120_common_root,
scampahead_file=t120.t120_scamp_ahead,
master_offset_name=t120.t120_master_name):
orig_dir = work_dir + orig_dir_root
reduc_dir = work_dir + reduc_dir_root
offset_dir = work_dir + offset_dir_root
flat_dir = work_dir + flat_dir_root
dark_dir = work_dir + dark_dir_root
master_offset_file = offset_dir + t120.t120_master_name
# now check existence of useful files and directories
for subdir in [
work_dir, work_dir, orig_dir, offset_dir, flat_dir,
t120.t120_common_dir
]:
if not os.path.isdir(subdir):
msg = '*** FATAL ERROR: directory ' + subdir + ' does not exist'
t120.log.error(msg)
raise IOError(msg)
try:
master_offset = fits_ccddata_reader(master_offset_file)
except:
msg = '*** FATAL ERROR while reading ' + master_offset_file
t120.log.error(msg)
raise IOError(msg)
# read scamp ahead file
#scamp_header = read_scamp_ahead(scampahead_file)
# loop over imagess
listremove = []
listimg = ImageFileCollection(
orig_dir + '/') #,glob_include='*-c.fits',glob_exclude='*.fit')
for ccd, fit_file in listimg.ccds(
ccd_kwargs={'unit':
'adu'}, return_fname=True, save_location=reduc_dir
): #,save_with_name='-c',save_location=reduc_dir+'/'):
t120.log.info('now treating file: ' + orig_dir + fit_file)
filter_name = ccd.header['FILTER']
flat_name = flat_dir + '/master-' + filter_name + '.fits'
master_flat = fits_ccddata_reader(flat_name, unit=u.adu)
hdu = fits.open(orig_dir + fit_file)
exp_time = hdu[0].header['EXPTIME']
strexptime = "%3.1f" % exp_time
t120.log.info('exp_time=' + strexptime)
dark_name = dark_dir + t120.t120_master_name.replace(
'.fits', '') + '-' + strexptime + '.fits'
master_dark = fits_ccddata_reader(dark_name, unit=u.adu)
t120.log.info('Flat: ' + flat_name)
t120.log.info('Dark: ' + dark_name)
master_dark.header['EXPOSURE'] = ccd.header[
'EXPOSURE'] # for dark subtraction
master_offset.header['EXPOSURE'] = ccd.header[
'EXPOSURE'] # for dark subtraction
ccd_corr = ccd_process(ccd,
exposure_key='EXPOSURE',
exposure_unit=u.second,
dark_frame=master_dark,
master_flat=master_flat)
#ccd_corr.header = put_scamp_header(ccd_corr.header,scamp_header) # update header
skycoo = SkyCoord(ccd_corr.header['OBJCTRA'] + ' ' +
ccd_corr.header['OBJCTDEC'],
unit=(u.hourangle, u.deg))
ccd_corr.header['CRVAL1'] = (
skycoo.ra.to('deg').value,
'Reference Right ascencion in decimal deg')
ccd_corr.header['CRVAL2'] = (skycoo.dec.to('deg').value,
'Reference Declination in decimal deg')
out_fit_file = reduc_dir + '/' + os.path.splitext(
fit_file)[0] + '-c.fits'
if os.path.exists(out_fit_file):
os.system('rm ' + out_fit_file)
t120.log.info('File ' + out_fit_file + ' has been removed')
# make primary HDU
hducorrlist = ccd_corr.to_hdu()
ccd_tosave = CCDData(hducorrlist[0].data, unit=u.adu)
ccd_tosave.header = hducorrlist[0].header
fits_ccddata_writer(ccd_tosave, out_fit_file)
t120.log.info('corrected image saved in ' + out_fit_file)
copy_fit_file = reduc_dir + fit_file
t120.log.info('copy_fit_file ' + copy_fit_file)
listremove.append(copy_fit_file)
# remove copy of original fits files
for file2remove in listremove:
t120.log.info('now removing file ' + file2remove)
if os.path.exists(file2remove):
os.system('rm ' + file2remove)
t120.log.info('File ' + file2remove + ' has been removed')
else:
t120.log.info('File ' + file2remove + ' does not exist')
return
