def load_data(args):
bdf_files = ccdp.ImageFileCollection(args.data_path)
raw_darks = bdf_files.files_filtered(imagetyp='Dark Frame',
include_path=True)
if args.calib_path:
calibrated_path = Path(args.calib_path)
else:
calibrated_path = Path(args.data_path, 'calibrated')
calibrated_path.mkdir(exist_ok=True)
calibrated_images = ccdp.ImageFileCollection(calibrated_path)
if args.output_path:
output_path = Path(args.output_path)
else:
output_path = Path(args.data_path, 'masters')
output_path.mkdir(exist_ok=True)
master_images = ccdp.ImageFileCollection(output_path)
calibrated_images.refresh()
return bdf_files, raw_darks, calibrated_path, calibrated_images, output_path, master_images
def load_data(args):
master_images = ccdp.ImageFileCollection(args.data_path)
raw_gaias = ccdp.ImageFileCollection(args.raw_images)
raw_gaias_im = raw_gaias.files_filtered(imagetyp='Light Frame',
include_path=True)
raw_gaias_im = sorted(raw_gaias_im)
if args.red_path:
reduced_path = Path(args.red_path)
else:
reduced_path = Path(args.raw_images, 'reduced')
reduced_path.mkdir(exist_ok=True)
reduced_images = ccdp.ImageFileCollection(reduced_path)
if args.al_path:
aligned_path = Path(args.al_path)
else:
aligned_path = Path(args.raw_images, 'aligned')
aligned_path.mkdir(exist_ok=True)
aligned_images = ccdp.ImageFileCollection(aligned_path)
if args.st_path:
stacked_path = Path(args.st_path)
else:
stacked_path = Path(args.raw_images, 'stacked')
stacked_path.mkdir(exist_ok=True)
stacked_images = ccdp.ImageFileCollection(stacked_path)
return master_images, raw_gaias, raw_gaias_im, reduced_path, reduced_images, aligned_path, aligned_images, stacked_path, stacked_images
def divide_flat(self):
"""Divides all LMI science frames by the appropriate flat field image
This method is LMI-specific, rather than being wrapper for a more
general function. Basic emulation of IRAF's ccdproc/flatcor function.
:return: None
"""
# Load the list of master flats and bias-subtracted data frames
flat_cl = ccdp.ImageFileCollection(
self.path, glob_include=f'flat_bin{self.bin_factor}_*.fits')
sci_cl = ccdp.ImageFileCollection(
self.path, glob_include=f'{self.prefix}.*b.fits')
# Check to be sure there are, indeed, flats...
if flat_cl.files:
# Loop through the filters present
for filt in sorted(list(flat_cl.summary['filters'])):
# Load in the master flat for this filter
if self.debug:
print(f'Dividing by master flat for filter: {filt}')
master_flat, mflat_fn = next(
flat_cl.ccds(ccdsum=self.binning,
filters=filt,
return_fname=True))
# Set up a progress bar, so we can see how the process is going
sci_filt_files = sci_cl.files_filtered(filters=filt)
prog_bar = tqdm(total=len(sci_filt_files),
unit='frame',
unit_scale=False,
colour='#D8BFD8')
# Loop through the science frames to correct
for ccd, sci_fn in sci_cl.ccds(ccdsum=self.binning,
filters=filt,
return_fname=True):
# Divide by master flat
ccdp.flat_correct(ccd, master_flat)
# Update the header
ccd.header['flatcor'] = True
ccd.header['HISTORY'] = PKG_NAME
ccd.header['HISTORY'] = 'Flat-corrected image saved: ' + \
_savetime()
ccd.header['HISTORY'] = f'Divided by flat: {mflat_fn}'
ccd.header['HISTORY'] = f'Previous filename: {sci_fn}'
# Save the result (suffix = 'f'); delete the input file
ccd.write(f'{self.path}/{sci_fn[:-6]}f{sci_fn[-5:]}',
overwrite=True)
os.remove(f'{self.path}/{sci_fn}')
# Update the progress bar
prog_bar.update(1)
# Close the progress bar, end of loop
prog_bar.close()
def load_data(args):
bdf_files = ccdp.ImageFileCollection(args.data_path)
raw_biases = bdf_files.files_filtered(imagetyp='Bias Frame',
include_path=True)
print(raw_biases)
if args.output_path:
output_path = Path(args.output_path)
else:
output_path = Path(args.data_path, 'masters')
output_path.mkdir(exist_ok=True)
master_images = ccdp.ImageFileCollection(output_path)
return raw_biases, output_path
def create_nirspec_collection(self):
"""Create a collection of Keck NIRSPEC echelle spectra"""
keywords = [
"imagetyp",
"filname",
"slitname",
"itime",
"frameno",
"object",
"ut",
"airmass",
"dispers",
"slitwidt",
"slitlen",
"ra",
"dec",
]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
ims = (ccdproc.ImageFileCollection(
self.root_dir,
keywords=keywords,
glob_include="NS*.fits",
).filter(dispers="high").filter(regex_match=True, slitlen="12|24"))
return ims
def _create_file_collection(self, data_dir, exclude_pattern, file_list):
"""Create a FITS ImageFileCollection for a directory, optionally
including only the files named in fname_list.
:param data_dir: Path to the directory containing the files.
:param exclude_pattern: Pattern used to exclude certain files.
:param file_list: None, or list of file names to read instead
of reading all files in the data_dir (excluding files that
match the exclude_pattern).
"""
if file_list is not None:
msg = (f'Looking for FITS files in {data_dir},'
f' including only files in the list: {file_list}')
else:
msg = (
f'Looking for FITS files in {data_dir},'
f' excluding files matching the pattern "{exclude_pattern}"')
self._logger.info(msg)
file_collection = ccdp.ImageFileCollection(
data_dir,
keywords=self._summary_kw,
glob_exclude=exclude_pattern,
filenames=file_list)
self._logger.info(
f'Found {len(file_collection.summary)} FITS files matching the constraints.'
)
return file_collection
def median_combine(path):
"""
Median combine with from a list within a directory.
Args:
path: path to directory of fits files to be combined
Returns:
a median combined CCDData object
"""
ic1 = ccdproc.ImageFileCollection(location=path)
validate_units(ic1)
framelist = ic1.files_filtered(BUNIT='adu', include_path=True)
median_frame = ccdproc.combine(framelist,
method='median',
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=mad_std,
mem_limit=350e6)
return median_frame
def __init__(self,
path,
biassec=None,
trimsec=None,
prefix=None,
multilamp=False):
"""__init__: Initialize DeVeny class.
Args:
path (:TYPE:`str`)
Path to the directory containing the images to be reduced.
biassec (:TYPE:`str`)
The IRAF-style overscan region to be subtracted from each frame.
If unspecified, use the values suggested in the LMI User Manual.
trimsec (:TYPE:`str`)
The IRAF-style image region to be retained in each frame.
If unspecified, use the values suggested in the LMI User Manual.
"""
super().__init__(path)
self.bin_factor = 1
self.binning = f'{self.bin_factor} {self.bin_factor}'
self.multilamp = multilamp
# Set the BIASSEC and TRIMSEC appropriately
self.biassec = '[2101:2144,5:512]' if biassec is None else biassec
self.trimsec = '[54: 2096,5:512]' if trimsec is None else trimsec
# File prefix -- DeVeny files prefix with the UT date
if prefix is None:
# Look at all the 20*.fits files in this directory, and choose
# Note: This will need to be updated for the year 2100
fitsfiles = glob.glob(self.path + '/' + '20*.????.fits')
if fitsfiles:
slashind = fitsfiles[0].rfind('/')
self.prefix = fitsfiles[0][slashind + 1:slashind + 9]
else:
self.prefix = prefix
if self.debug:
print(f'Directory prefix: {self.prefix}')
# Define standard filenames
self.zerofn = 'bias.fits'
# Define the gratings
self.gratings = {
"DV1": "150/5000",
"DV2": "300/4000",
"DV3": "300/6750",
"DV4": "400/8500",
"DV5": "500/5500",
"DV6": "600/4900",
"DV7": "600/6750",
"DV8": "831/8000",
"DV9": "1200/5000",
"DV10": "2160/5000",
"DVxx": "UNKNOWN"
}
self.icl = ccdp.ImageFileCollection(
self.path, glob_include=f'{self.prefix}.*.fits')
def load_images(data_dir, grating):
"""load_images Load in the images associated with DATA_DIR and grating
[extended_summary]
Parameters
----------
data_dir : `str`
The directory containing the data to analyze
grating : `str`
The grating ID to use
Returns
-------
`ccdproc.image_collection.ImageFileCollection`
IFC of the files meeting the input criteria
"""
# Dictionary
gratid = {'DV1': '150/5000', 'DV2': '300/4000', 'DV5': '500/5500'}
# Load the images of interest in to an ImageFileCollection()
icl = ccdp.ImageFileCollection(data_dir)
# Clean up the telescope altitude
for ccd, fn in icl.ccds(return_fname=True):
ccd.header['telalt'] = np.round(ccd.header['telalt'])
ccd.write(os.path.join(data_dir, fn), overwrite=True)
# Return an ImageFileCollection filtered by the grating desired
return icl.filter(grating=gratid[grating])
def __init__(self, path, biassec=None, trimsec=None, bin_factor=2):
"""__init__: Initialize LMI class.
Args:
path (:TYPE:`str`)
Path to the directory containing the images to be reduced.
biassec (:TYPE:`str`)
The IRAF-style overscan region to be subtracted from each frame.
If unspecified, use the values suggested in the LMI User Manual.
trimsec (:TYPE:`str`)
The IRAF-style image region to be retained in each frame.
If unspecified, use the values suggested in the LMI User Manual.
bin_factor (:TYPE:`int`)
The binning factor used to create the image(s) to be processed.
[Default: 2]
"""
super().__init__(path)
self.bin_factor = int(bin_factor)
self.binning = f'{self.bin_factor} {self.bin_factor}'
# # Set the BIASSEC and TRIMSEC appropriately FOR 2x2 BINNING
# if self.bin_factor == 2:
# self.biassec = '[3100:3124, 3:3079]' if biassec is None else biassec
# self.trimsec = '[30:3094, 3:3079]' if trimsec is None else trimsec
# else:
self.biassec = biassec
self.trimsec = trimsec
# Define file prefix & standard filenames
self.prefix = 'lmi'
self.zerofn = f'bias_bin{self.bin_factor}.fits'
# Load initial ImageFileCollection
self.icl = ccdp.ImageFileCollection(
self.path, glob_include=f'{self.prefix}.*.fits')
def add_units(path, units='adu'):
"""
Check for units in fits header. If none then add Keyword BUNIT with default = adu
Args:
path: path to fits file
units: optional, default set to adu useful options
adu, photon, and electron
Returns:
None
"""
ic1 = ccdproc.ImageFileCollection(location=path)
# check for units in header
if 'bunit' not in ic1.summary.colnames:
warnings.warn('Overwriting Original Header!!!')
for hdu in ic1.hdus(overwrite=True):
hdu.header['bunit'] = 'adu'
def realtimeRed(storePath, analyPath, masterDark):
neos = ccdproc.ImageFileCollection(location=analyPath)
neoList = []
for neo, fname in neos.hdus(return_fname=True):
meta = neo.header
meta['filename'] = fname
neoList.append(ccdproc.CCDData(data=neo.data, header=meta, unit="adu"))
masterBias_e = ccdproc.gain_correct(masterBias, gain=1 * u.electron / u.adu)
masterDark_e = ccdproc.gain_correct(masterDark, gain=1 * u.electron / u.adu)
masterFlat_e = ccdproc.gain_correct(masterFlat, gain=1 * u.electron / u.adu)
for neo in neoList:
neo_red = ccdproc.ccd_process(neo, master_bias=masterBias_e, dark_frame=masterDark_e, master_flat=masterFlat_e
, gain=1 * u.electron / u.adu, readnoise=readnoise, min_value=1.
, dark_exposure=darkExp * u.second, data_exposure=neo.header['exptime'] * u.second
, exposure_unit=u.second, dark_scale=True)
baseName = os.path.basename(neo.header['filename'])
fits.writeto("{}{}_red.fits".format(storePath, baseName.split('.')[0]), neo_red.data, header=neo_red.header, overwrite=False)
def stack(self, tmp_fit_dir_p, filename_p, exposure, image_bitpix):
logger.info(
'Stacking dark frames for exposure %0.1fs, gain %d, bin %d',
exposure, self.gain_v.value, self.bin_v.value)
if image_bitpix == 16:
numpy_type = numpy.uint16
elif image_bitpix == 8:
numpy_type = numpy.uint8
dark_images = ccdproc.ImageFileCollection(tmp_fit_dir_p)
cal_darks = dark_images.files_filtered(imagetyp='Dark Frame',
exptime=exposure,
include_path=True)
start = time.time()
combined_dark = ccdproc.combine(
cal_darks,
method='average',
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=numpy.ma.median,
signma_clip_dev_func=mad_std,
dtype=numpy_type,
mem_limit=350000000,
)
elapsed_s = time.time() - start
logger.info('Exposure sigma clip stacked in %0.4f s', elapsed_s)
combined_dark.meta['combined'] = True
combined_dark.write(filename_p)
def calibrate_collection(self, options, collection):
write_path = Path(f'{os.getcwd()}/calibrated')
write_path.mkdir(exist_ok=True)
count = 1
for img, fname in collection.ccds(return_fname=True):
print(
f'\n{Style.BRIGHT}[{count}/{len(self.collection.files)}] Calibrating: {fname + Style.RESET_ALL}'
)
ccd_temp = img.header['ccd-temp']
gain = img.header['gain']
offset = img.header['offset']
exptime = img.header['exptime']
filter = img.header['filter']
print(f'CCD_TEMP: {ccd_temp}')
print(f'GAIN: {gain}')
print(f'OFFSET: {offset}')
print(f'EXPTIME: {exptime}')
print(f'FILTER: {filter}')
self.calibrate_image(options, img).write(write_path / fname,
overwrite=True)
count += 1
return ccdp.ImageFileCollection(write_path)
from astropy import units as u
from astropy.io import fits
import matplotlib.pyplot as plt
import ccdproc as cp
# The path listed below is for the directory on ShadowHQ not local.
path = '/Users/JPeg/cometdata/'
fileKeys = ['FILENAME','NAXIS1','NAXIS2','OBJECT','OBSTYPE', 'FILTERS' ]
allfiles = cp.ImageFileCollection(path,keywords = fileKeys)
gain = 2.89 * u.electron/u.adu
rdnoise = 6 * u.electron
# correct bias images for overscan and trim the image:
biasImages=[]
for filename in allfiles.files_filtered(NAXIS1=3128,NAXIS2=3080,OBSTYPE = 'BIAS'):
ccd = fits.getdata(allfiles.location + filename)
print(ccd)
ccd = cp.CCDData(ccd, unit = u.adu)
print(ccd)
ccd = cp.subtract_overscan(ccd,overscan_axis = 1, overscan = ccd[:,3099:3125])
print(ccd)
ccd = cp.trim_image(ccd,fits_section = '[27:3095,3:3078]')
print(ccd)
biasImages.append(ccd)
def calibrate_images(
x_d,
x_f,
x_s,
it_s='object',
x_b='',
):
"""
Parameters
----------
----------
x_b : str
Path of the bias files.
By default bias is not provided
x_d : str
Path of the dark files
x_f : str
Path of flat files
x_s : str
Path of Science files
it_s : str
Imagetyp of fits science file
Default is set to object
-----------
"""
path_b = Path(x_b)
path_d = Path(x_d)
path_f = Path(x_f)
path_s = Path(x_s)
#-----Bias
if x_b == '' and path_b == Path(''):
print(
'Be aware: You did not provide the Bias files; the process will still continue though.'
)
files_b = None
elif not path_b.is_dir():
raise RuntimeError(
'The path you provided for the Bias files does not exist.')
else:
files_b = ccdp.ImageFileCollection(path_b)
#-----Dark
if x_d == '' or not path_d.is_dir():
raise RuntimeError(
'You must provide Dark files for processing.\n Or the path you provided does not exist.'
)
else:
files_d = ccdp.ImageFileCollection(path_d)
#-----Flat
if x_f == '' or not path_f.is_dir():
raise RuntimeError(
'You must provide Flatfield files for processing.\n Or the path you provided does not exist.'
)
else:
files_f = ccdp.ImageFileCollection(path_f)
#-----Science
if x_s == '' or not path_s.is_dir():
raise RuntimeError(
'You must provide Science images for processing.\n Or the path you provided does not exist.'
)
else:
files_s = ccdp.ImageFileCollection(path_s)
#-----------------------------------
#
#--------Calibrating Images---------
#
#-----------------------------------
if files_b is not None:
#-------------------------------
#------Creating Master-bias-----
#-------------------------------
cali_bias_path = Path(path_b / 'cali_bias')
cali_bias_path.mkdir(exist_ok=True)
files_b_cali = files_b.files_filtered(imagetyp='bias',
include_path=True)
combined_bias = ccdp.combine(files_b_cali,\
method='average',\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_bias.meta['combined'] = True
combined_bias.write(cali_bias_path / 'master_bias.fits')
# Reading master bias
master_bias = CCDData.read(cali_bias_path / 'master_bias.fits')
else:
master_bias = None
#-------------------------------
#-------Calibrating Darks-------
#-------------------------------
cali_dark_path = Path(path_d / 'cali_dark')
cali_dark_path.mkdir(exist_ok=True)
files_d_cali = files_d.files_filtered(imagetyp='DARK', include_path=True)
for ccd, file_name in files_d.ccds(imagetyp='DARK',
return_fname=True,
ccd_kwargs={'unit': 'adu'}):
if master_bias is not None:
# Subtract bias
ccd = ccdp.subtract_bias(ccd, master_bias)
else:
ccd = ccd
# Save the result
ccd.write(cali_dark_path / file_name)
#--------------------------------
#------Creating Master-Dark------
#--------------------------------
red_dark = ccdp.ImageFileCollection(cali_dark_path)
# Calculating exposure times of DARK images
dark_times = set(
red_dark.summary['exptime'][red_dark.summary['imagetyp'] == 'DARK'])
for exposure in sorted(dark_times):
cali_darks = red_dark.files_filtered(imagetyp = 'dark',\
exptime = exposure,\
include_path = True)
combined_dark = ccdp.combine(cali_darks,\
method='average',\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_dark.meta['combined'] = True
com_dark_name = 'combined_dark_{:6.3f}.fits'.format(exposure)
combined_dark.write(cali_dark_path / com_dark_name)
# Reading master dark of various exposure times
red_dark = ccdp.ImageFileCollection(cali_dark_path)
combined_darks = {
ccd.header['exptime']: ccd
for ccd in red_dark.ccds(imagetyp='DARK', combined=True)
}
#--------------------------------
#-------Calibrating Flats--------
#--------------------------------
cali_flat_path = Path(path_f / 'cali_flat')
cali_flat_path.mkdir(exist_ok=True)
files_f_cali = files_f.files_filtered(imagetyp='FLAT', include_path=True)
for ccd, file_name in files_f.ccds(imagetyp='FLAT',
ccd_kwargs={'unit': 'adu'},
return_fname=True):
# Subtract bias
if master_bias is not None:
ccd = ccdp.subtract_bias(ccd, master_bias)
else:
ccd = ccd
closest_dark = utl.find_nearest_dark_exposure(ccd, dark_times)
if closest_dark is None:
closest_dark1 = utl.find_nearest_dark_exposure(ccd,
dark_times,
tolerance=100)
# Subtract scaled Dark
ccd = ccdp.subtract_dark(ccd, combined_darks[closest_dark1],\
exposure_time = 'exptime',\
exposure_unit = u.second,\
scale = True)
ccd.write(cali_flat_path / ('flat-' + file_name))
else:
closest_dark2 = utl.find_nearest_dark_exposure(ccd, dark_times)
# Subtracting Darks
ccd = ccdp.subtract_dark(ccd,
combined_darks[closest_dark2],
exposure_time='exptime',
exposure_unit=u.second)
ccd.write(cali_flat_path / ('flat-' + file_name))
#--------------------------------
#-----Creating Master-Flat-------
#--------------------------------
red_flats = ccdp.ImageFileCollection(cali_flat_path)
cali_flats = red_flats.files_filtered(imagetyp='FLAT', include_path=True)
combined_flat = ccdp.combine(cali_flats,\
method='average',\
scale = utl.inverse_median,\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_flat.meta['combined'] = True
combined_flat.write(cali_flat_path / 'master_flat.fits')
# Reading master flat
red_flats = ccdp.ImageFileCollection(cali_flat_path)
combined_flat = CCDData.read(cali_flat_path / 'master_flat.fits')
#--------------------------------
#---Calibrating Science Images---
#--------------------------------
# Creating a list of spectrum images
files_spec = files_s.summary['file', 'view_pos']
files_spec_list = np.array([])
for i in range(len(files_spec)):
xxx = files_spec['view_pos'][i]
if xxx[0:4] == 'open':
files_spec_list = np.hstack(
(files_spec_list, files_spec['file'][i]))
# Sky subtracting images
cali_science_path1 = Path(path_s / 'Sky_subtracted_science')
cali_science_path1.mkdir(exist_ok=True)
j = 0
for i in range(int(len(files_spec_list) / 2)):
ccd1 = CCDData.read(x_s + files_spec_list[j], unit='adu')
ccd2 = CCDData.read(x_s + files_spec_list[j + 1], unit='adu')
sky_sub1 = ccd1.data - ccd2.data
ss1 = CCDData(sky_sub1, unit='adu')
ss1.header = ccd1.header
ss1.meta['sky_sub'] = True
name1 = 'sky_sub_' + files_spec_list[j]
ss1.write(cali_science_path1 / name1)
sky_sub2 = ccd2.data - ccd1.data
ss2 = CCDData(sky_sub2, unit='adu')
ss2.header = ccd2.header
ss2.meta['sky_sub'] = True
name2 = 'sky_sub_' + files_spec_list[j + 1]
ss2.write(cali_science_path1 / name2)
j = j + 2
files_s1 = ccdp.ImageFileCollection(cali_science_path1)
final_calibrated = Path(path_s / 'Final_calibrated_science')
final_calibrated.mkdir(exist_ok=True)
# Correcting for flat
for ccd, file_name in files_s1.ccds(imagetyp=it_s,
ccd_kwargs={'unit': 'adu'},
return_fname=True):
# Subtract scaled Dark
ccd = ccdp.flat_correct(ccd, combined_flat) #['FLAT'])
ccd.write(final_calibrated / file_name)
def flux_extraction(file_name, path, out_path, images=True):
"""
Parameters
----------
----------
file_name : str
Name of the image/telluric file
from which flux has to be extracted
path : str
Path of the desired image file
out_path : str
Path of the output data and/or image file
images : bool
True if one wants to save visualization of flux data
False if not.
Default is True
----------
returns
----------
flux : data file
.dat file containing the flux at
various pixel values
Path of this file would be similar to
that of image file.
----------
"""
pt = Path(path)
f1 = ccdp.ImageFileCollection(pt)
ccd = CCDData.read(path + file_name) # + '.fits')
# Trimming the Image
trimmed = ccdp.trim_image(ccd, fits_section='[1:256, 100:1000]')
trimmed.meta['TRIM'] = True
trimmed.header = ccd.header
#trimmed.write(file_name + '_trim.fits')
# Reading the data from Trimmed image
data = trimmed.data
# Creating a function to detect the edges of slit
# For lower edge
def xlow(raw_data):
"""
Parameters
----------
----------
raw_data : numpy.ndarray
Array containing flux at some particular wavelength
----------
returns
----------
number : float
A pixel number showing the lower edge of slit
----------
"""
j = 0
for i in range(int(len(raw_data) / 5)):
st = np.std(raw_data[j:j + 5])
xlw = 0
if st < 2:
xlw = j
if xlw != 0:
break
j = j + 5
return xlw
# For upper edge
def xup(raw_data):
"""
Parameters
----------
----------
raw_data : numpy.ndarray
Array containing flux at some particular wavelength
----------
returns
----------
number : float
A pixel number showing the upper edge of slit
----------
"""
j = 255
for i in range(int(len(raw_data) / 5)):
st = np.std(raw_data[j - 5:j])
xup = 0
if st < 2:
xup = j
if xup != 0:
break
j = j - 5
return xup
# Defining line and inverse line
def line(x, m, c):
return m * x + c
def inv_line(x, m, c):
bc = (x - c) / m
return bc
# Detecting the edges of the spectrum
ys = np.array([150, 300, 450, 600, 750])
xs_left = np.array([])
xs_right = np.array([])
xs_mid = np.array([])
for i in range(len(ys)):
dd1 = data[ys[i]]
xll = xlow(dd1)
xs_left = np.hstack((xs_left, xll))
xuu = xup(dd1)
xs_right = np.hstack((xs_right, xuu))
popt_l, pcov_l = cft(line, xs_left, ys)
popt_r, pcov_r = cft(line, xs_right, ys)
# Detecting a line where spectrum could reside
for i in range(len(ys)):
ran_l = inv_line(ys[i], popt_l[0], popt_l[1])
ran_r = inv_line(ys[i], popt_r[0], popt_r[1])
xd1 = data[ys[i]]
xd = xd1[int(ran_l):int(ran_r)]
ma = np.max(xd)
ab = np.where(xd == ma)
xs_mid = np.hstack((xs_mid, ab[0][0] + ran_l))
popt_m, pcov_m = cft(line, xs_mid, ys)
# Finding total flux
def total_flux(lam, xlim=20):
ydata = data[lam]
xmid = inv_line(lam, popt_m[0], popt_m[1])
xlow = xmid - xlim
xup = xmid + xlim
total_flux1 = 0
xdata = np.arange(int(xlow), int(xup + 1), 1)
for i in range(len(xdata)):
total_flux1 = total_flux1 + ydata[xdata[i]]
return total_flux1
# Flux as a function of pixel
flux = np.array([])
y11 = np.arange(0, 900, 1)
for i in range(len(y11)):
f11 = total_flux(y11[i])
flux = np.hstack((flux, f11))
# Saving the image file for flux
if images == True:
fig1 = plt.figure(figsize=(20, 10))
plt.plot(flux)
plt.xlabel('Pixel Number')
plt.ylabel('Total Flux')
plt.title('Total flux for ' + file_name + ' observation')
plt.grid()
plt.savefig(out_path + '/' + file_name + '_flux.png')
plt.close(fig1)
# Saving Data file of the flux
f1 = open(out_path + '/' + file_name + '_flux.dat', 'w')
f1.write('#Pixel\t\tFlux\n')
for i in range(len(y11)):
f1.write(str(y11[i]) + '\t\t' + str(flux[i]) + '\n')
f1.close()
def reduce_night(year=datetime.date.today().year,
month=datetime.date.today().month,
day=datetime.date.today().day,
telescope="C28",
config_file="config.ini"):
config = get_config(config_file)
log = init_log(time.strftime("%Y%m%d_%H%M%S", time.gmtime()), config_file)
t = datetime.date(year, month, day)
t_str = datetime.date.strftime(t, format="%Y%m%d")
im_path = config.get("GENERAL", "PATH") + config.get(
telescope, "PATH") + t_str + config.get(telescope,
"DIR_SUFFIX") + os.sep
if not os.path.isdir(im_path):
log.warning(f"Folder {im_path} doesn't exist!")
return
log.info(
f"""Creating {telescope} master calibration frames for {t_str}...""")
calframes.create_masters(year,
month,
day,
telescope,
log=log,
config_file=config_file)
imlist = ccdproc.ImageFileCollection(im_path, keywords='*')
files = imlist.files_filtered(imagetyp="LIGHT")
if len(files.tolist()) == 0:
log.warning(f"No science frames in {t_str}.")
return
imlist = ccdproc.ImageFileCollection(im_path,
keywords='*',
filenames=files.tolist())
save_uncertainty = config.getboolean("GENERAL", "SAVE_UNCERTAINTY")
is_overwrite = config.getboolean("GENERAL", "OVERWRITE")
instrument = imlist.values("instrume", True)[0]
ccd_shape = utils.get_ccd_shape(imlist, telescope)
reduced_path = im_path + config.get("GENERAL", "REDUCED_DIR") + os.sep
# create reduced folder
if not os.path.exists(reduced_path):
os.makedirs(reduced_path)
log.debug(f"""ccd_shape length {len(ccd_shape["x_naxis"])}""")
for i in range(len(ccd_shape["x_naxis"])):
ccd_set = utils.get_set_from_dict(ccd_shape, i)
ccd_str = utils.get_ccd_str(ccd_shape, idx=i)
filters = np.unique(imlist.summary["filter"])
log.debug(f"{filters}")
for filt in filters:
bias_file, dark_file, flat_file = calframes.get_calframes(
year,
month,
day,
filt,
ccd_str,
telescope=telescope,
instrument=instrument,
log=log,
config_file=config_file)
if (not bias_file) or (not dark_file) or (not flat_file):
log.warning(f"No calibration frames found, skipping.")
continue
bias = ccdproc.CCDData.read(bias_file)
dark = ccdproc.CCDData.read(dark_file)
flat = ccdproc.CCDData.read(flat_file)
kwargs = dict()
kwargs[get_key_name("image_type",
telescope)] = get_key_val("light", telescope)
kwargs[get_key_name("filter", telescope)] = filt
kwargs[get_key_name("x_naxis", telescope)] = ccd_set["x_naxis"]
kwargs[get_key_name("y_naxis", telescope)] = ccd_set["y_naxis"]
kwargs[get_key_name("x_subframe",
telescope)] = ccd_set["x_subframe"]
kwargs[get_key_name("y_subframe",
telescope)] = ccd_set["y_subframe"]
kwargs[get_key_name("x_bin", telescope)] = ccd_set["x_bin"]
kwargs[get_key_name("y_bin", telescope)] = ccd_set["y_bin"]
for im, filename in imlist.hdus(return_fname=True, **kwargs):
log.debug(f"{filename}")
obj = im.header[get_key_name("object", telescope)]
jd = im.header[get_key_name("jd", telescope)] * u.day
# fix JD for RBI flood delay (C28):
if telescope == "C28":
if im.header[get_key_name("readout",
telescope)] == get_key_val(
"rbi", telescope):
jd = jd + get_key_val("rbi_delay", telescope)
im.header[get_key_name("jd",
telescope)] = jd.to_value()
im.header[get_key_name("rbi_delay",
telescope)] = "TRUE"
im.header.comments[get_key_name(
"rbi_delay", telescope
)] = f"""Corrected JD by {get_key_val("rbi_delay", telescope)} of RBI flood delay."""
log.warning(
f"""Corrected RBI flood delay of {get_key_val("rbi_delay", telescope)}."""
)
jd = str(jd.to_value()).replace(".", "_")
filename = f"{obj}_{jd}_{filt}_{telescope}"
file_exists = os.path.isfile(reduced_path + filename + ".fits")
if (not file_exists) | (file_exists & is_overwrite):
im.data = im.data.astype("float32")
im = ccdproc.subtract_bias(
ccdproc.CCDData(data=im.data,
unit=u.adu,
header=im.header),
bias,
add_keyword=ccdproc.Keyword("DEBIAS",
value=bias_file.split(
os.sep)[-1]))
im = ccdproc.subtract_dark(
im,
dark,
exposure_time=get_key_name("exptime", telescope),
exposure_unit=u.s,
scale=True,
add_keyword=ccdproc.Keyword("DEDARK",
value=dark_file.split(
os.sep)[-1]))
im = ccdproc.flat_correct(im,
flat,
add_keyword=ccdproc.Keyword(
"DEFLAT",
value=flat_file.split(
os.sep)[-1]))
if not save_uncertainty:
im.uncertainty = None
im.mask = None
im.data = im.data.astype('float32')
im.write(reduced_path + filename + ".fits", overwrite=True)
close_log(log)
return reduced_path
def __init__(self, data=None):
self.main_path = Path(cfg.CALIBRATION_PATH)
self.masters = ccdp.ImageFileCollection(self.main_path)
if data:
self.collection = ccdp.ImageFileCollection(filenames=data)
def flat_combine(self):
"""
:return:
"""
if self.debug:
print("Combining flats...")
# Load the list of bias-subtracted data frames
bsub_cl = ccdp.ImageFileCollection(
self.path, glob_include=f'{self.prefix}.*b.fits')
# Find just the flats
# NOTE: When CCDPROC filters an ImgFileCol, the resulting filenames
# have the path attached. This allows for generality, but will
# need to be accounted for.
flats_cl = bsub_cl.filter(imagetyp="dome flat")
# Check that we have any
if flats_cl.files:
# In case more than one grating was used (unlikely except eng)
for grname in list(set(list(flats_cl.summary['grat_id']))):
# Filter the ImgFileColl to include only this grating
gr_cl = flats_cl.filter(grat_id=grname)
# In case more than one grating tilt angle was used (possible)
for gra in list(set(list(gr_cl.summary['grangle']))):
# Filter the ImgFileColl to include only this tilt
gra_cl = gr_cl.filter(grangle=gra)
# In case more than one order-blocking filter was used (???)
for filt in list(set(list(gra_cl.summary['filtrear']))):
# Filter the ImgFileColl to include only this filter
filt_cl = gra_cl.filter(filtrear=filt)
# For engineering, possibly use different lamps for comp
if self.multilamp:
lamps = list(set(list(filt_cl.summary['comment'])))
else:
lamps = ['domelamp']
if self.debug:
print(f'Flat lamps used: {lamps}')
for this_lamp in lamps:
if self.multilamp:
lamp_cl = filt_cl.filter(comment=this_lamp)
lname = '_TRING' if this_lamp[0:3] == 'Top' \
else '_FLOOD'
else:
lamp_cl = filt_cl
lname = ''
# Actually do the flat combining
cflat = ccdp.combine(lamp_cl.files,
method='median',
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=mad_std,
mem_limit=4e9)
# Add FITS keyword NCOMBINE and HISTORY
cflat.header.set(
'ncombine', len(lamp_cl.files),
'# of input images in combination')
cflat.header['HISTORY'] = PKG_NAME
cflat.header['HISTORY'] = 'Combined flat ' + \
'created: ' + _savetime()
cflat.header['HISTORY'] = \
f'Median combined {len(lamp_cl.files)} files:'
for fn in lamp_cl.files:
# Note: These filenames have the path attached,
# via the .filter() method of ImgFileCol.
# Include just the pathless filename.
cflat.header['HISTORY'] = fn[fn.rfind('/') +
1:]
# Build filename, save, remove input files
flat_fn = f'flat_{grname}_{gra}_{filt}{lname}.fits'
if self.debug:
print(f'Saving combined flat as {flat_fn}')
cflat.write(f'{self.path}/{flat_fn}',
overwrite=True)
for fn in lamp_cl.files:
# Note: These filenames have the path already
# attached, via the .filter() method of
# ImgFileCol.
os.remove(f'{fn}')
else:
print("No flats to be combined.")
def imcombine(*infiles,
inlist=None,
outfn=None,
del_input=False,
combine=None,
printstat=True,
overwrite=True,
returnccd=False):
"""Combine a collection of images
This function (crudely) emulates the IRAF imcombine function. Pass in a
list of images to be combined, and the result is written to disk with an
optionally specified output filename.
:param infiles: `list`: List of filenames to combine
:param inlist: `str`: Filename of text file listing images to be combined
:param outfn: `str`: Filename to write combined image. Default: append
'_comb' to first filename in the input list.
:param del_input: `bool`: Delete the input files after combination.
Default: `false`
:param combine: `str`: Combine method. 'median' (default), or 'mean'
:param printstat: `bool`: Print image statistics to screen
:param overwrite: `bool`: Overwrite the output file. Default: True
:return: None (Unless returnccd=True, then it returns the combined CCDData)
"""
# Unpack the single-item tuple *infiles
if len(infiles) > 0:
files, = infiles
else:
files = []
# Check for inputs
if len(files) > 0 and inlist is not None:
print("Only one of files or inlist may be specified, not both.")
raise Exception()
# Read in the text list inlist, if specified
if inlist is not None:
with open(inlist, 'r', encoding='utf-8') as f:
files = []
for line in f:
files.append(line.rstrip())
# Check for proper file list
if len(files) < 3:
print("Combination requires at least three input images.")
raise Exception()
# Check that specified input files exist
for f in files:
if not os.path.isfile(f):
print(f"File {f} does not exist.")
raise Exception()
# Determine combine method (default = 'median')
if combine != 'median' or combine != 'mean':
combine = 'median'
# Create an ImgFileColl using the input files
file_cl = ccdp.ImageFileCollection(filenames=files)
if printstat:
# Print out the statistics, for clarity
for img, fn in file_cl.ccds(return_fname=True):
mini, maxi, mean, stdv = mmms(img)
print(f'{fn}:: Min: {mini:.2f} Max: {maxi:.2f} ' +
f'Mean: {mean:.2f} Stddev: {stdv:.2f}')
comb_img = ccdp.combine(file_cl.files,
method=combine,
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=mad_std,
mem_limit=4e9)
# Add FITS keyword NCOMBINE and add HISTORY
comb_img.header.set('ncombine', len(file_cl.files),
'# of input images in combination')
comb_img.header['HISTORY'] = PKG_NAME
comb_img.header['HISTORY'] = 'Combined image created: ' + _savetime()
comb_img.header['HISTORY'] = f'{combine.title()} combined ' + \
f'{len(file_cl.files)} files:'
for fn in file_cl.files:
comb_img.header['HISTORY'] = fn
# If returnccd is True, return now before thinking about saving.
if returnccd:
return comb_img
# Build filename (if not specified in call), save, remove input files
if outfn is None:
outfn = f'{files[0][:-5]}_comb{files[0][-5:]}'
print(f'Saving combined image as {outfn}')
comb_img.write(f'{outfn}', overwrite=overwrite)
if del_input:
for f in files:
os.remove(f'{f}')
return None
def get_log(folder, extra_keys=[], write=True):
"""
Given a folder, it generate a log file with the listing of the keys given
on the keys parameter. Default are the keys from OPD.
Parameters
----------
folder : String
Path to the night run folder with the FITS files.
extra_keys : List
Listing of keys to look at in order, in addition to the default ones.
write : bool
If True writes an output csv (default is True).
Retuns
------
log_tab: DataFrame
Table containing information given on keys
File transformations
--------------------
Write to disk a file named w/ the folder name + "_night.log" containing the
log inside the folder when write is True.
"""
out = Path(folder)
# Check if there are fles
if len(list(out.glob("*.fits"))) == 0:
print(
"ERROR: No files found, can't create log dataframe for orientation. Returning None"
)
return None
out = out / f"{str(out)}_night.log"
keys = ["DATE-OBS", "OBJECT", "FILTER", "EXPTIME", "AIRMASS", "COMMENT"]
keys.extend(extra_keys)
ifc = ccdproc.ImageFileCollection(folder, keywords=keys)
df = ifc.summary.to_pandas(index="file")
# Cleaning OPD comment and exptime. Then standardizing OBJECT and FILTER.
def standardize(value):
"""
Put values on standard way for folder creation by removing certain
characteres, and having it in upper case. (str -> str)
"""
translator = str.maketrans({" ": "", "\\": "-", "/": "-"})
return value.translate(translator).upper()
cleaners = {
"COMMENT": lambda value: value.split("'")[1].strip(),
"EXPTIME": lambda value: int(value.split(",")[0]),
"OBJECT": standardize,
"FILTER": standardize
}
for key in cleaners:
df[key] = df[key].apply(cleaners[key])
if write:
df.to_csv(out)
return [df, str(out)]
"""
Created on Thu May 7 16:56:18 2020
@author: ambar
"""
import sp
from astropy.io import ascii
import time
import ccdproc as ccdp
import numpy as np
a='fasfas'
print(type(a))
science=ccdp.ImageFileCollection('Corrected_Science')
#ascii.write(sc,'sc_summary.csv',format='csv',overwrite=True)
for i,j in science.hdus(return_fname=True):
print(i)
#found=np.zeros(len(hdulist),dtype=type(a))
'''
start_time=time.time()
for imsc, imsd in enumerate(science.data()):
f=sp.find(imsd,hmin=1000,fwhm=12)
found[imsc]=len(f[0])
ascii.write(found,'found.csv',format='csv',overwrite=True)
'''
def flat_combine(self):
"""Finds and combines flat frames with the indicated binning
:return: None
"""
# Load the list of bias-subtracted data frames -- check binning
bsub_cl = ccdp.ImageFileCollection(
self.path, glob_include=f'{self.prefix}.*b.fits')
if not bsub_cl.files:
print("Nothing to be done for flat_combine()!")
return
if self.debug:
print("Normalizing flat field frames...")
# Filter here to get # of images for progress bar
flat_cl = bsub_cl.filter(ccdsum=self.binning,
imagetyp='[a-z]+ flat',
regex_match=True)
# Set up a progress bar, so we can see how the process is going...
prog_bar = tqdm(total=len(flat_cl.files),
unit='frame',
unit_scale=False,
colour='yellow')
# Normalize flat field images by the mean value
for ccd, flat_fn in flat_cl.ccds(return_fname=True):
# Perform the division
ccd = ccd.divide(np.mean(ccd), handle_meta='first_found')
# Update the header
ccd.header['HISTORY'] = 'Normalized flat saved: ' + _savetime()
ccd.header['HISTORY'] = f'Previous filename: {flat_fn}'
# Save the result (suffix = 'n'); delete the input file
ccd.write(f'{self.path}/{flat_fn[:-6]}n{flat_fn[-5:]}',
overwrite=True)
os.remove(f'{self.path}/{flat_fn}')
# Update the progress bar
prog_bar.update(1)
# Close the progress bar, end of loop
prog_bar.close()
# Load the list of normalized flat field images
norm_cl = ccdp.ImageFileCollection(
self.path, glob_include=f'{self.prefix}.*n.fits')
if norm_cl.files:
# Create a unique list of the filter collection found in this set
filters = list(norm_cl.summary['filters'])
unique_filters = sorted(list(set(filters)))
# Combine flat field frames for each filt in unique_filters
for filt in unique_filters:
flats = norm_cl.files_filtered(filters=filt, include_path=True)
print(f"Combining flats for filter {filt}...")
cflat = ccdp.combine(flats,
method='median',
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=mad_std,
mem_limit=4e9)
# Add FITS keyword NCOMBINE and HISTORY
cflat.header.set('ncombine', len(flats),
'# of input images in combination')
cflat.header['HISTORY'] = PKG_NAME
cflat.header['HISTORY'] = 'Combined flat created: ' + \
_savetime()
cflat.header['HISTORY'] = 'Median combined ' + \
f'{len(flats)} files:'
for fn in flats:
# Remove the path portion of the filename for the HISTORY
cflat.header['HISTORY'] = fn[fn.rfind('/') + 1:]
# Build filename, save, remove input files
flat_fn = f'flat_bin{self.bin_factor}_{filt}.fits'
if self.debug:
print(f'Saving combined flat as {flat_fn}')
cflat.write(f'{self.path}/{flat_fn}', overwrite=True)
for fn in flats:
# Path name is already included
os.remove(f'{fn}')
else:
print("No flats to be combined.")
if len(LWTdata_flatPaths) < 5:
os.system("cp /LWTdata/LWT_{0}/flat/*V_Astrodon* /LWTanaly/{0}/flat/.".format(date))
else:
for path in LWTdata_flatPaths[:5]:
os.system("cp {} /LWTanaly/{}/flat/.".format(path, date))
gain = 1.4 * u.electron / u.adu
gainValue = 1.4
readnoise = 8.2 * u.electron
readnoiseValue = 8.2
biasPath = "/LWTanaly/{}/bias/".format(date)
darkPath_test = "/LWTanaly/{}/dark_test/".format(date)
flatPath = "/LWTanaly/{}/flat/".format(date)
# Create master bias frame.
biases = ccdproc.ImageFileCollection(location=biasPath)
biasList = []
for bias, fname in biases.hdus(return_fname=True):
meta = bias.header
meta['filename'] = fname
biasList.append(ccdproc.CCDData(data=bias.data, meta=meta, unit='adu'))
masterBias = ccdproc.combine(biasList, output_file="/LWTanaly/{}/bias/Master_Bias.fits".format(date), method='median'
, clip_extrema=True, nlow=0, nhigh=1)
# Create master dark frame for each NEO.
darkPaths = [i for i in glob("/LWTanaly/{}/dark_*".format(date)) if 'test' not in i]
masterDark = []
for path in darkPaths:
darks = ccdproc.ImageFileCollection(location=path+'/')
darkList = []
for dark, fname in darks.hdus(return_fname=True):
from pathlib import Path
from subprocess import run
from datetime import datetime
import ccdproc as ccdp
from astropy.stats import mad_std
import numpy as np
from matplotlib import pyplot as plt
from calibrate import calibrate_collection
import helpers as hlp
import config as cfg
calibration_path = Path(cfg.CALIBRATION_PATH)
calibration_masters = ccdp.ImageFileCollection(calibration_path)
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(
description='Master flat generation script. Uses average combination and sigma clipping pixel rejection. Generates bias-substracted and raw masters.')
parser.add_argument(
'files', help='input files (FITS only)', type=str, nargs='+')
parser.add_argument('-c', '--calibrated', action='store_true',
help='generate bias-substracted master dark')
parser.add_argument(
'-s', '--sigmalow', help='sigma low threshold for pixel rejection', default=5)
parser.add_argument(
'-S', '--sigmahigh', help='sigma high threshold for pixel rejection', default=5)
'-R', '--radius', help='search radius around center (degrees, default=15 unless no RA or DEC: 360)', type=str, dest='radius', default=15)
parser.add_argument(
'-s', '--scale', help='estimated field scale (arcminutes, online solver only)', type=int, dest='scaleEst')
parser.add_argument(
'-e', '--error', help='estimated field scale error (percent, online solver only)', type=int, dest='scaleErr')
parser.add_argument('-D', '--downsample',
help='downsampling amount', type=int)
parser.add_argument('-b', '--blind',
action='store_true', help='blind solve')
parser.add_argument('-y', '--noconfirm',
action='store_true', help='skip confirmation')
parser.add_argument('-o', '--online', action='store_true',
help='use online solver (requires internet connection)')
args = parser.parse_args()
images = ccdp.ImageFileCollection(filenames=args.files)
print(
f'{Style.BRIGHT}Platesolving {len(images.files)} files.{Style.RESET_ALL}')
if (not args.ra and not args.dec) and not args.blind:
try:
args.ra = ccdp.CCDData.read(args.files[0]).header['ra']
args.dec = ccdp.CCDData.read(args.files[0]).header['dec']
print(
f'{Style.BRIGHT + Fore.GREEN}Found WCS in file, using as target.{Style.RESET_ALL}')
except:
print(
f'{Style.BRIGHT + Fore.RED}No WCS found.{Style.RESET_ALL}')
args.blind = True
if not args.blind:
def reduce_sami(path):
log.info('SAMI Data-Reduction Pipeline')
log.info('Version {}'.format(version.__str__))
sami_merger = merge.SamiMerger()
reduced_path = os.path.join(path, 'RED')
if os.path.exists(reduced_path):
log.warning('Skipping existing directory: {}'.format(reduced_path))
else:
log.info(
'Creating directory for reduced data: {}'.format(reduced_path))
os.makedirs(os.path.join(path, "RED"), exist_ok=True)
list_of_files = glob.glob(os.path.join(path, '*.fits'))
log.info('Reading raw files')
table = []
for _file in list_of_files:
try:
hdu = pyfits.open(_file)
except OSError:
log.warning("Could not read file: {}".format(_file))
continue
if numpy.std(hdu[1].data.ravel()) == 0:
log.warning("Bad data found on file: {}".format(_file))
continue
row = {
'filename': _file,
'obstype': hdu[0].header['obstype'],
'filter_id': hdu[0].header['filters'],
'filter_name': hdu[0].header['filter1'],
'binning':
[int(b) for b in hdu[1].header['ccdsum'].strip().split(' ')]
}
table.append(row)
list_of_binning = []
for row in table:
if row['binning'] not in list_of_binning:
list_of_binning.append(row['binning'])
log.info('Found new binning mode: {}'.format(
row['binning'][0], row['binning'][1]))
for binning in list_of_binning:
log.info('Organizing data.')
sub_table = [row for row in table if row['binning'] == binning]
zero_table = [row for row in sub_table if row['obstype'] == 'ZERO']
dflat_table = [row for row in sub_table if row['obstype'] == 'DFLAT']
sflat_table = [row for row in sub_table if row['obstype'] == 'SFLAT']
obj_table = [row for row in sub_table if row['obstype'] == 'OBJECT']
log.info('Processing ZERO files')
zero_files = [r['filename'] for r in zero_table]
zero_files.sort()
zero_list_name = os.path.join(
path, "RED", "0Zero{}x{}".format(binning[0], binning[1]))
with open(zero_list_name, 'w') as zero_list_buffer:
for zero_file in zero_files:
sami_merger.zero_file = None
sami_merger.flat_file = None
path, fname = os.path.split(zero_file)
prefix = sami_merger.get_prefix()
output_zero_file = os.path.join(path, 'RED', prefix + fname)
if os.path.exists(output_zero_file):
log.warning(
'Skipping existing file: {}'.format(output_zero_file))
continue
log.info('Processing ZERO file: {}'.format(zero_file))
data = sami_merger.get_joined_data(zero_file)
header = pyfits.getheader(zero_file)
data, header, prefix = sami_merger.join_and_process(
data, header)
pyfits.writeto(output_zero_file, data, header)
zero_list_buffer.write('{:s}\n'.format(zero_file))
log.info('Combining ZERO files.')
master_zero_fname = zero_list_name + '.fits'
if os.path.exists(master_zero_fname):
log.warning('Skipping existing MASTER ZERO: {:s}'.format(
master_zero_fname))
else:
ic = ccdproc.ImageFileCollection(location=os.path.join(
path, 'RED'),
glob_include='*.fits',
keywords=KEYWORDS)
zero_combine_files = [
os.path.join(path, 'RED', f)
for f in ic.files_filtered(obstype='ZERO')
]
log.info("Writing master zero to: {}".format(master_zero_fname))
zero_combine = combine.ZeroCombine(input_list=zero_combine_files,
output_file=master_zero_fname)
zero_combine.run()
log.info('Done.')
log.info('Processing FLAT files (SFLAT + DFLAT)')
all_flats = sflat_table + dflat_table
filters_used = []
for row in all_flats:
if row['filter_id'] not in filters_used:
filters_used.append(row['filter_id'])
log.info('Found new filter: {}'.format(row['filter_name']))
for _filter in filters_used:
log.info('Processing FLATs for filter: {}'.format(_filter))
sub_table_by_filter = [
row for row in all_flats if row['filter_id'] == _filter
]
flat_list_name = os.path.join(
path, 'RED', "1FLAT_{}x{}_{}".format(binning[0], binning[1],
_filter))
flat_files = [row['filename'] for row in sub_table_by_filter]
flat_files.sort()
flat_combine_files = []
with open(flat_list_name, 'w') as flat_list_buffer:
for flat_file in flat_files:
sami_merger.zero_file = master_zero_fname
sami_merger.flat_file = None
prefix = sami_merger.get_prefix()
path, fname = os.path.split(flat_file)
output_flat_file = os.path.join(path, 'RED',
prefix + fname)
if os.path.exists(os.path.join(path, output_flat_file)):
log.warning('Skipping existing FLAT file: {}'.format(
output_flat_file))
continue
log.info('Processing FLAT file: {}'.format(flat_file))
d = sami_merger.get_joined_data(flat_file)
h = pyfits.getheader(flat_file)
d, h, p = sami_merger.join_and_process(d, h)
pyfits.writeto(output_flat_file, d, h)
flat_list_buffer.write('{:s}\n'.format(output_flat_file))
flat_combine_files.append(output_flat_file)
master_flat_fname = flat_list_name + '.fits'
if os.path.exists(master_flat_fname):
log.warning('Skipping existing MASTER FLAT: {:s}'.format(
master_flat_fname))
else:
log.info('Writing master FLAT to file: {}'.format(
master_flat_fname))
flat_combine = combine.FlatCombine(
input_list=flat_combine_files,
output_file=master_flat_fname)
flat_combine.run()
for _filter in filters_used:
master_flat_fname = os.path.join(
path, 'RED',
"1FLAT_{}x{}_{}.fits".format(binning[0], binning[1], _filter))
sami_merger.zero_file = master_zero_fname
sami_merger.flat_file = master_flat_fname
sami_merger.cosmic_rays = True
sub_table_by_filter = [
row for row in obj_table if row['filter_id'] == _filter
]
log.info('Processing OBJECT files with filter: {}'.format(_filter))
obj_list_name = os.path.join(
path, 'RED', "2OBJECT_{}x{}_{}".format(binning[0], binning[1],
_filter))
obj_files = [row['filename'] for row in sub_table_by_filter]
obj_files.sort()
with open(obj_list_name, 'w') as obj_list_buffer:
for obj_file in obj_files:
path, fname = os.path.split(obj_file)
prefix = sami_merger.get_prefix()
output_obj_file = os.path.join(path, 'RED', prefix + fname)
if os.path.exists(output_obj_file):
log.warning('Skipping existing OBJECT file: {}'.format(
output_obj_file))
continue
log.info('Processing OBJECT file: {}'.format(obj_file))
d = sami_merger.get_joined_data(obj_file)
h = sami_merger.get_header(obj_file)
h = sami_merger.add_wcs(d, h)
d, h, p = sami_merger.join_and_process(d, h)
pyfits.writeto(output_obj_file, d, h)
obj_list_buffer.write('\n'.format(obj_file))
log.info('All done.')
def __init__(self, data=None):
if data:
self.collection = ccdp.ImageFileCollection(filenames=data)
def _biascombine(self, binning=None, output="bias.fits"):
"""Finds and combines bias frames with the indicated binning
:param binning:
:param output:
:return:
"""
if binning is None:
raise InputError('Binning not set.')
if self.debug:
print("Trimming and combining bias frames with binning "
f"{binning.replace(' ','x')} into {output}...")
# First, refresh the ImageFileCollection
self.icl.refresh()
# Set up a progress bar, so we can see how the process is going...
bias_files = self.icl.files_filtered(imagetyp='bias')
prog_bar = tqdm(total=len(bias_files),
unit='frame',
unit_scale=False,
colour='#808080')
# Loop through files,
for ccd, file_name in self.icl.ccds(ccdsum=binning,
imagetyp='bias',
bitpix=16,
return_fname=True):
# Fit the overscan section, subtract it, then trim the image
ccd = _trim_oscan(ccd, self.biassec, self.trimsec)
# Update the header
ccd.header['HISTORY'] = PKG_NAME
ccd.header['HISTORY'] = 'Trimmed bias saved: ' + _savetime()
ccd.header['HISTORY'] = f'Original filename: {file_name}'
# Save the result (suffix = 't'); delete the input file
ccd.write(f'{self.path}/{file_name[:-5]}t{file_name[-5:]}',
overwrite=True)
os.remove(f'{self.path}/{file_name}')
# Update the progress bar
prog_bar.update(1)
# Close the progress bar, end of loop
prog_bar.close()
# Collect the trimmed biases
t_bias_cl = ccdp.ImageFileCollection(
self.path, glob_include=f'{self.prefix}.*t.fits')
# If we have a fresh list of trimmed biases to work with...
if t_bias_cl.files:
if self.debug:
print("Doing median combine now...")
comb_bias = ccdp.combine(
[f'{self.path}/{fn}' for fn in t_bias_cl.files],
method='median',
sigma_clip=True,
sigma_clip_low_thresh=5,
sigma_clip_high_thresh=5,
sigma_clip_func=np.ma.median,
sigma_clip_dev_func=mad_std,
mem_limit=4e9)
# Add FITS keyword NCOMBINE and HISTORY
comb_bias.header.set('ncombine', len(t_bias_cl.files),
'# of input images in combination')
comb_bias.header['HISTORY'] = 'Combined bias created: ' + \
_savetime()
comb_bias.header['HISTORY'] = 'Median combined ' + \
f'{len(t_bias_cl.files)} files:'
for f in t_bias_cl.files:
comb_bias.header['HISTORY'] = f
# Save the result; delete the input files
comb_bias.write(f'{self.path}/{output}', overwrite=True)
for f in t_bias_cl.files:
os.remove(f'{self.path}/{f}')
