def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation.
"""
self.log.info(f"Running {self.__class__.__name__} action")
header_keywords = ['RN', 'READNOISE']
for kw in header_keywords:
rn = self.action.args.kd.get(kw, None)
if rn is not None: break
if rn is not None:
self.log.debug(f' Got read noise from header: {rn}')
if rn is None:
rn = self.cfg['Telescope'].getfloat('RN', None)
self.log.debug(f' Got read noise from config: {rn}')
self.action.args.kd.headers.append(fits.Header({'READNOISE': rn}))
for i, pd in enumerate(self.action.args.kd.pixeldata):
if pd.unit == u.electron:
self.action.args.kd.pixeldata[i] = ccdproc.create_deviation(
pd, readnoise=rn * u.electron)
elif pd.unit == u.adu:
self.action.args.kd.pixeldata[i] = ccdproc.create_deviation(
pd,
gain=float(self.action.args.kd.get('GAIN')),
readnoise=rn * u.electron)
else:
self.log.error('Could not estimate uncertainty')
return self.action.args
def test_log_bad_type_fails():
ccd_data = ccd_data_func()
add_key = 15 # anything not string and not dict-like will work here
# Do we fail with non-string, non-Keyword, non-dict-like value?
with pytest.raises(AttributeError):
create_deviation(ccd_data,
readnoise=3 * ccd_data.unit,
add_keyword=add_key)
def test_log_keyword():
ccd_data = ccd_data_func()
key = 'filter'
key_val = 'V'
kwd = Keyword(key, value=key_val)
new = create_deviation(ccd_data,
readnoise=3 * ccd_data.unit,
add_keyword=kwd)
# Was the Keyword added with the correct value?
assert kwd.name in new.meta
assert kwd.name not in ccd_data.meta
assert new.meta[kwd.name] == key_val
def test_log_string(key):
ccd_data = ccd_data_func()
add_key = key
new = create_deviation(ccd_data,
readnoise=3 * ccd_data.unit,
add_keyword=add_key)
# Keys should be added to new but not to ccd_data and should have
# no value.
assert add_key in new.meta
assert add_key not in ccd_data.meta
# Long keyword names should be accessible with just the keyword name
# without HIERARCH -- is it?
assert new.meta[add_key] is None
def test_log_dict():
ccd_data = ccd_data_func()
keys_to_add = {
'process': 'Added deviation',
'n_images_input': 1,
'current_temp': 42.9
}
new = create_deviation(ccd_data,
readnoise=3 * ccd_data.unit,
add_keyword=keys_to_add)
for k, v in keys_to_add.items():
# Were all dictionary items added?
assert k in new.meta
assert k not in ccd_data.meta
assert new.meta[k] == v
def test_implicit_logging():
ccd_data = ccd_data_func()
# If nothing is supplied for the add_keyword argument then the following
# should happen:
# + A key named func.__name__ is created, with
# + value that is the list of arguments the function was called with.
bias = CCDData(np.zeros_like(ccd_data.data), unit="adu")
result = subtract_bias(ccd_data, bias)
assert "subtract_bias" in result.header
assert result.header['subtract_bias'] == (
'subbias', 'Shortened name for ccdproc command')
assert result.header['subbias'] == "ccd=<CCDData>, master=<CCDData>"
result = create_deviation(ccd_data, readnoise=3 * ccd_data.unit)
assert result.header['create_deviation'] == (
'creatvar', 'Shortened name for ccdproc command')
assert ("readnoise=" + str(3 * ccd_data.unit) in result.header['creatvar'])
def _perform(self):
"""
Returns an Argument() with the parameters that depend on this
operation.
"""
self.log.info(f"Running {self.__class__.__name__} action")
read_noise = self.action.args.meta.get('read_noise', None)
if read_noise is not None:
self.log.debug(f' Using read_noise = {read_noise}')
if read_noise is None:
read_noise = self.cfg['Telescope'].getfloat('read_noise', None)
self.log.debug(f' Got read_noise from config: {read_noise}')
self.action.args.ccddata = ccdproc.create_deviation(
self.action.args.ccddata, readnoise=read_noise * u.electron)
return self.action.args
def reduceFrames(self):
self.getMedianForObjects()
log.info('Reducing frames started')
print 'Reducing frames started'
for obj in self.objects.filesList:
data = ccdproc.CCDData.read(obj, unit=u.adu)
dataWithDeviation = ccdproc.create_deviation(data,
gain=1.5*u.electron/u.adu,
readnoise=5*u.electron)
reducedObject = ccdproc.gain_correct(dataWithDeviation,
1.5*u.electron/u.adu)
if self.biases.isExists:
reducedObject = ccdproc.subtract_bias(reducedObject,
self.biases.masterCCD)
if self.darks.isExists:
reducedObject = ccdproc.subtract_dark(reducedObject,
self.darks.masterCCD,
exposure_time=cfg.exptime,
exposure_unit=u.second,
scale=True)
if self.flats.isExists:
reducedObject = ccdproc.flat_correct(reducedObject,
self.flats.masterCCD)
self.directory = '../../Reduction/' + directoryName
if not os.path.exists(self.directory):
os.makedirs(self.directory)
reducedObject.write(self.directory + '/' + obj, clobber=True)
os.system('solve-field ' + self.directory + '/' + obj) # + ' --overwrite')
objName, objExtension = os.path.splitext(self.directory + '/' + obj)
if not os.path.exists(objName + '.new'):
log.warning(objName + ' cannot be solved')
else:
newObjectsList.append(objName + '.new')
log.info('Frame ' + objName + ' reduced')
log.info('Reduced ' + str(len(newObjectsList)) + ' frames')
print 'Reduced ' + str(len(newObjectsList)) + ' frames'
self.clean()
def convert_to_ccddata(images, gain=None, read_noise=None):
"""
Convert 'fits' file to 'ccdproc.CCCData' object.
Parameters
----------
images : str or list of str.
Images to converted.
gain : float
Gain (u.electron / u.adu)
read_noise : float
Read Noise (u.electron).
Yields
------
'ccdproc.CCDData'
yield the next 'ccdproc.CCDData'.
Examples
--------
>>> from tuglib.io import convert_to_ccddata
>>> from glob import glob
>>>
>>> images = glob('/home/user/data/image*.fits')
>>>
>>> ccds = convert_to_ccddata(images, gain=0.37, read_noise=4.11)
"""
if not isinstance(images, (str, list)):
raise TypeError("'images' should be 'str' or 'list' object.")
if not isinstance(gain, (type(None), float)):
raise TypeError("'gain' should be 'None' or 'float' object.")
if not isinstance(read_noise, (type(None), float)):
raise TypeError("'read_noise' should be 'None' or 'float' object.")
if isinstance(images, str):
images = [images]
unit = u.adu
if gain is not None:
gain = gain * u.electron / u.adu
if read_noise is not None:
read_noise = read_noise * u.electron
for image in images:
ccd = CCDData.read(image, unit=unit, output_verify='silentfix+ignore')
if (gain is not None) and (read_noise is not None):
data_with_deviation = create_deviation(ccd,
gain=gain,
readnoise=read_noise)
gain_corrected = gain_correct(data_with_deviation, gain)
yield gain_corrected
else:
yield ccd
def test_log_set_to_None_does_not_change_header():
ccd_data = ccd_data_func()
new = create_deviation(ccd_data,
readnoise=3 * ccd_data.unit,
add_keyword=None)
assert new.meta.keys() == ccd_data.header.keys()
def process_raw_frame(self, master_bias, master_flat, pixel_mask_spec=None):
"""
Bias and flat-correct a raw CCD frame. Trim off the overscan
region. Identify cosmic rays using "lacosmic" and inflat
uncertainties where CR's are found. If specified, mask out
nearby sources by setting pixel uncertainty to infinity (or
inverse-variance to 0).
Returns
-------
nccd : `ccdproc.CCDData`
A copy of the original ``CCDData`` object but after the
above procedures have been run.
"""
oscan_fits_section = "[{}:{},:]".format(self.oscan_idx,
self.oscan_idx+self.oscan_size)
# make a copy of the object
nccd = self.ccd.copy()
# apply the overscan correction
poly_model = Polynomial1D(2)
nccd = ccdproc.subtract_overscan(nccd, fits_section=oscan_fits_section,
model=poly_model)
# trim the image (remove overscan region)
nccd = ccdproc.trim_image(nccd, fits_section='[1:{},:]'.format(self.oscan_idx))
# create the error frame
nccd = ccdproc.create_deviation(nccd, gain=self.ccd_gain,
readnoise=self.ccd_readnoise)
# now correct for the ccd gain
nccd = ccdproc.gain_correct(nccd, gain=self.ccd_gain)
# correct for master bias frame
# - this does some crazy shit at the blue end, but we can live with it
nccd = ccdproc.subtract_bias(nccd, master_bias)
# correct for master flat frame
nccd = ccdproc.flat_correct(nccd, master_flat)
# comsic ray cleaning - this updates the uncertainty array as well
nccd = ccdproc.cosmicray_lacosmic(nccd, sigclip=8.)
# replace ccd with processed ccd
self.ccd = nccd
# check for a pixel mask
if pixel_mask_spec is not None:
mask = self.make_nearby_source_mask(pixel_mask_spec)
logger.debug("\t\tSource mask loaded.")
stddev = nccd.uncertainty.array
stddev[mask] = np.inf
nccd.uncertainty = StdDevUncertainty(stddev)
if self.plot_path is not None:
# TODO: this assumes vertical CCD
aspect_ratio = nccd.shape[1]/nccd.shape[0]
fig,axes = plt.subplots(2, 1, figsize=(10,2 * 12*aspect_ratio),
sharex=True, sharey=True)
vmin,vmax = self.zscaler.get_limits(nccd.data)
axes[0].imshow(nccd.data.T, origin='bottom',
cmap=self.cmap, vmin=max(0,vmin), vmax=vmax)
stddev = nccd.uncertainty.array
vmin,vmax = self.zscaler.get_limits(stddev[np.isfinite(stddev)])
axes[1].imshow(stddev.T, origin='bottom',
cmap=self.cmap, vmin=max(0,vmin), vmax=vmax)
axes[0].set_title('Object: {0}, flux'.format(self._obj_name))
axes[1].set_title('root-variance'.format(self._obj_name))
fig.tight_layout()
fig.savefig(path.join(self.plot_path, '{}_frame.png'.format(self._filename_base)))
plt.close(fig)
return nccd
def calibrate_images(x_d, x_f, x_s, it_s = 'object', x_b = '', ):
"""
Parameters
----------
----------
x_b : str
Path of the bias files.
By default bias is not provided
x_d : str
Path of the dark files
x_f : str
Path of flat files
x_s : str
Path of Science files
it_s : str
Imagetyp of fits science file
Default is set to object
-----------
"""
path_b = Path(x_b)
path_d = Path(x_d)
path_f = Path(x_f)
path_s = Path(x_s)
#-----Bias
if x_b == '' and path_b == Path(''):
print('Be aware: You did not provide the Bias files; the process will still continue though.')
files_b = None
elif not path_b.is_dir():
raise RuntimeError('The path you provided for the Bias files does not exist.')
else:
files_b = ccdp.ImageFileCollection(path_b)
#-----Dark
if x_d == '' or not path_d.is_dir():
raise RuntimeError('You must provide Dark files for processing.\n Or the path you provided does not exist.')
else:
files_d = ccdp.ImageFileCollection(path_d)
#-----Flat
if x_f == '' or not path_f.is_dir():
raise RuntimeError('You must provide Flatfield files for processing.\n Or the path you provided does not exist.')
else:
files_f = ccdp.ImageFileCollection(path_f)
#-----Science
if x_s == '' or not path_s.is_dir():
raise RuntimeError('You must provide Science images for processing.\n Or the path you provided does not exist.')
else:
files_s = ccdp.ImageFileCollection(path_s)
#-----------------------------------
#
#--------Calibrating Images---------
#
#-----------------------------------
if files_b is not None:
#-------------------------------
#------Creating Master-bias-----
#-------------------------------
cali_bias_path = Path(path_b / 'cali_bias')
cali_bias_path.mkdir(exist_ok = True)
files_b_cali = files_b.files_filtered(imagetyp = 'bias', include_path = True)
combined_bias = ccdp.combine(files_b_cali,\
method='average',\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_bias.meta['combined'] = True
combined_bias.write(cali_bias_path / 'master_bias.fits')
# Reading master bias
master_bias = CCDData.read(cali_bias_path / 'master_bias.fits')
else:
master_bias = None
#-------------------------------
#-------Calibrating Darks-------
#-------------------------------
cali_dark_path = Path(path_d / 'cali_dark')
cali_dark_path.mkdir(exist_ok = True)
files_d_cali = files_d.files_filtered(imagetyp = 'DARK', include_path = True)
for ccd, file_name in files_d.ccds(imagetyp = 'DARK', return_fname = True, ccd_kwargs = {'unit':'adu'}):
if master_bias is not None:
# Subtract bias
ccd = ccdp.subtract_bias(ccd, master_bias)
else:
ccd = ccd
# Save the result
ccd.write(cali_dark_path / file_name)
#--------------------------------
#------Creating Master-Dark------
#--------------------------------
red_dark = ccdp.ImageFileCollection(cali_dark_path)
# Calculating exposure times of DARK images
dark_times = set(red_dark.summary['exptime'][red_dark.summary['imagetyp'] == 'DARK'])
for exposure in sorted(dark_times):
cali_darks = red_dark.files_filtered(imagetyp = 'dark',\
exptime = exposure,\
include_path = True)
combined_dark = ccdp.combine(cali_darks,\
method='average',\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_dark.meta['combined'] = True
com_dark_name = 'combined_dark_{:6.3f}.fits'.format(exposure)
combined_dark.write(cali_dark_path / com_dark_name)
# Reading master dark of various exposure times
red_dark = ccdp.ImageFileCollection(cali_dark_path)
combined_darks = {ccd.header['exptime']: ccd for ccd in red_dark.ccds(imagetyp = 'DARK', combined = True)}
#--------------------------------
#-------Calibrating Flats--------
#--------------------------------
cali_flat_path = Path(path_f / 'cali_flat')
cali_flat_path.mkdir(exist_ok = True)
files_f_cali = files_f.files_filtered(imagetyp = 'FLAT', include_path = True)
for ccd, file_name in files_f.ccds(imagetyp = 'FLAT', ccd_kwargs = {'unit' : 'adu'}, return_fname = True):
# Subtract bias
if master_bias is not None:
ccd = ccdp.subtract_bias(ccd, master_bias)
else:
ccd = ccd
closest_dark = utl.find_nearest_dark_exposure(ccd, dark_times)
if closest_dark is None:
closest_dark1 = utl.find_nearest_dark_exposure(ccd, dark_times, tolerance = 100)
# Subtract scaled Dark
ccd = ccdp.subtract_dark(ccd, combined_darks[closest_dark1],\
exposure_time = 'exptime',\
exposure_unit = u.second,\
scale = True)
ccd.write(cali_flat_path / ('flat-' + file_name))
else:
closest_dark2 = utl.find_nearest_dark_exposure(ccd, dark_times)
# Subtracting Darks
ccd = ccdp.subtract_dark(ccd, combined_darks[closest_dark2], exposure_time = 'exptime', exposure_unit = u.second)
ccd.write(cali_flat_path / ('flat-' + file_name))
#--------------------------------
#-----Creating Master-Flat-------
#--------------------------------
red_flats = ccdp.ImageFileCollection(cali_flat_path)
cali_flats = red_flats.files_filtered(imagetyp = 'FLAT', include_path = True)
combined_flat = ccdp.combine(cali_flats,\
method='average',\
scale = utl.inverse_median,\
sigma_clip=True,\
sigma_clip_low_thresh=5,\
sigma_clip_high_thresh=5,\
sigma_clip_func=np.ma.median,\
sigma_clip_dev_func=mad_std,\
mem_limit=350e6)
combined_flat.meta['combined'] = True
combined_flat.write(cali_flat_path / 'master_flat.fits')
# Reading master flat
red_flats = ccdp.ImageFileCollection(cali_flat_path)
combined_flat = CCDData.read(cali_flat_path / 'master_flat.fits')
#--------------------------------
#---Calibrating Science Images---
#--------------------------------
cali_science_path = Path(path_s / 'cali_science')
cali_science_path.mkdir(exist_ok = True)
# Correcting for flat
for ccd, file_name in files_s.ccds(imagetyp = it_s, ccd_kwargs = {'unit' : 'adu'}, return_fname = True):
# Subtract scaled Dark
#ccd = ccdp.subtract_dark(ccd, combined_darks[closest_dark1], exposure_time = 'exptime', exposure_unit = u.second, scale = True)
ccd = ccdp.flat_correct(ccd, combined_flat)#['FLAT'])
ccd.write(cali_science_path / file_name)
files_s1 = ccdp.ImageFileCollection(cali_science_path)
files_s_cali = files_s1.files_filtered(imagetyp = it_s, include_path = True)
# Creating a list of spectrum images
files_spec = files_s1.summary['file', 'view_pos']
files_spec_list = np.array([])
for i in range(len(files_spec)):
xxx = files_spec['view_pos'][i]
if xxx[0:4] == 'open':
files_spec_list = np.hstack((files_spec_list, files_spec['file'][i]))
# Sky subtracting images
final_calibrated = Path(path_s / 'Final_calibrated_science')
final_calibrated.mkdir(exist_ok = True)
# Variance in sky subtracting images
final_calibrated_err = Path(path_s / 'Error_final_calibrated_science')
final_calibrated_err.mkdir(exist_ok = True)
j = 0
for i in range(int(len(files_spec_list)/2)):
# For Reduced Image
ccd1 = CCDData.read(x_s + 'cali_science/' + files_spec_list[j], unit='adu')
ccd2 = CCDData.read(x_s + 'cali_science/' + files_spec_list[j+1], unit = 'adu')
sky_sub1 = ccd1.data - ccd2.data
ss1 = CCDData(sky_sub1, unit='adu')
ss1.header = ccd1.header
ss1.meta['sky_sub'] = True
name1 = 'sky_sub_' + files_spec_list[j]
ss1.write(final_calibrated / name1)
sky_sub2 = ccd2.data - ccd1.data
ss2 = CCDData(sky_sub2, unit='adu')
ss2.header = ccd2.header
ss2.meta['sky_sub'] = True
name2 = 'sky_sub_' + files_spec_list[j+1]
ss2.write(final_calibrated / name2)
# For Errors in Reduced Image
ccd1e = ccdp.create_deviation(ccd1, gain=9.2*u.electron/u.adu, readnoise=40*u.electron)
ccd1er = ccd1e.uncertainty.array
ccd1err = np.nan_to_num(ccd1er)
ccd2e = ccdp.create_deviation(ccd2, gain=9.2*u.electron/u.adu, readnoise=40*u.electron)
ccd2er = ccd2e.uncertainty.array
ccd2err = np.nan_to_num(ccd2er)
data_err = np.sqrt(ccd1err**2 + ccd2err**2 - (2*ccd1err*ccd2err))
data_err1 = CCDData(data_err, unit='adu')
data_err1.meta['Error'] = True
data_err1.header = ccd1.header
name3 = 'sky_sub_err_' + files_spec_list[j]
data_err1.write(final_calibrated_err / name3)
name4 = 'sky_sub_err_' + files_spec_list[j+1]
data_err1.write(final_calibrated_err / name4)
j = j+2
def ccd_process(ccd, oscan=None, trim=None, error=False, masterbias=None,
bad_pixel_mask=None, gain=None, rdnoise=None,
oscan_median=True, oscan_model=None):
"""Perform basic processing on ccd data.
The following steps can be included:
* overscan correction
* trimming of the image
* create edeviation frame
* gain correction
* add a mask to the data
* subtraction of master bias
The task returns a processed `ccdproc.CCDData` object.
Parameters
----------
ccd: `ccdproc.CCDData`
Frame to be reduced
oscan: None, str, or, `~ccdproc.ccddata.CCDData`
For no overscan correction, set to None. Otherwise proivde a region
of `ccd` from which the overscan is extracted, using the FITS
conventions for index order and index start, or a
slice from `ccd` that contains the overscan.
trim: None or str
For no trim correction, set to None. Otherwise proivde a region
of `ccd` from which the image should be trimmed, using the FITS
conventions for index order and index start.
error: boolean
If True, create an uncertainty array for ccd
masterbias: None, `~numpy.ndarray`, or `~ccdproc.CCDData`
A materbias frame to be subtracted from ccd.
bad_pixel_mask: None or `~numpy.ndarray`
A bad pixel mask for the data. The bad pixel mask should be in given
such that bad pixels havea value of 1 and good pixels a value of 0.
gain: None or `~astropy.Quantity`
Gain value to multiple the image by to convert to electrons
rdnoise: None or `~astropy.Quantity`
Read noise for the observations. The read noise should be in
`~astropy.units.electron`
oscan_median : bool, optional
If true, takes the median of each line. Otherwise, uses the mean
oscan_model : `~astropy.modeling.Model`, optional
Model to fit to the data. If None, returns the values calculated
by the median or the mean.
Returns
-------
ccd: `ccdproc.CCDData`
Reduded ccd
Examples
--------
1. To overscan, trim, and gain correct a data set:
>>> import numpy as np
>>> from astropy import units as u
>>> from hrsprocess import ccd_process
>>> ccd = CCDData(np.ones([100, 100]), unit=u.adu)
>>> nccd = ccd_process(ccd, oscan='[1:10,1:100]', trim='[10:100, 1,100]',
error=False, gain=2.0*u.electron/u.adu)
"""
# make a copy of the object
nccd = ccd.copy()
# apply the overscan correction
if isinstance(oscan, ccdproc.CCDData):
nccd = ccdproc.subtract_overscan(nccd, overscan=oscan,
median=oscan_median,
model=oscan_model)
elif isinstance(oscan, six.string_types):
nccd = ccdproc.subtract_overscan(nccd, fits_section=oscan,
median=oscan_median,
model=oscan_model)
elif oscan is None:
pass
else:
raise TypeError('oscan is not None, a string, or CCDData object')
# apply the trim correction
if isinstance(trim, six.string_types):
nccd = ccdproc.trim_image(nccd, fits_section=trim)
elif trim is None:
pass
else:
raise TypeError('trim is not None or a string')
# create the error frame
if error and gain is not None and rdnoise is not None:
nccd = ccdproc.create_deviation(nccd, gain=gain, rdnoise=rdnoise)
elif error and (gain is None or rdnoise is None):
raise ValueError(
'gain and rdnoise must be specified to create error frame')
# apply the bad pixel mask
if isinstance(bad_pixel_mask, np.ndarray):
nccd.mask = bad_pixel_mask
elif bad_pixel_mask is None:
pass
else:
raise TypeError('bad_pixel_mask is not None or numpy.ndarray')
# apply the gain correction
if isinstance(gain, u.quantity.Quantity):
nccd = ccdproc.gain_correct(nccd, gain)
elif gain is None:
pass
else:
raise TypeError('gain is not None or astropy.Quantity')
# test subtracting the master bias
if isinstance(masterbias, ccdproc.CCDData):
nccd = ccdproc.subtract_bias(nccd, masterbias)
elif isinstance(masterbias, np.ndarray):
nccd.data = nccd.data - masterbias
elif masterbias is None:
pass
else:
raise TypeError(
'masterbias is not None, numpy.ndarray, or a CCDData object')
return nccd
def process_fits(fitspath,
*,
obstype=None,
object=None,
exposure_times=None,
percentile=None,
percentile_min=None,
percentile_max=None,
window=None,
darks=None,
cosmic_ray=False,
cosmic_ray_kwargs={},
gain=None,
readnoise=None,
normalise=False,
normalise_func=np.ma.average,
combine_type=None,
sigma_clip=False,
low_thresh=3,
high_thresh=3):
"""Combine all FITS images of a given type and exposure time from a given directory.
Parameters
----------
fitspath: str
Path to the FITS images to process. Can be a path to a single file, or a path to a
directory. If the latter the directory will be searched for FITS files and checked
against criteria from obstype, object, exposure_times critera.
obstype: str, optional
Observation type, an 'OBSTYPE' FITS header value e.g. 'DARK', 'OBJ'. If given only files
with matching OBSTYPE will be processed.
object: str, optional
Object name, i.e. 'OBJECT' FITS header value. If given only files with matching OBJECT
will be processed.
exposure_times: float or sequence, optional
Exposure time(s), i.e 'TOTALEXP' FITS header value(s). If given only files with matching
TOTALEXP will be processed.
percentile: float, optional
If given will only images whose percentile value fall between percentile_min and
percentile_max will be processed, e.g. set to 50.0 to select images by median value,
set to 99.5 to select images by their 99.5th percentile value.
percentile_min: float, optional
Minimum percentile value.
percentile_max: float, optional
Maximum percentil value.
window: (int, int, int, int), optional
If given will trim images to the window defined as (x0, y0, x1, y1), where (x0, y0)
and (x1, y1) are the coordinates of the bottom left and top right corners.
darks: str or sequence, optional
Filename(s) of dark frame(s) to subtract from the image(s). If given a dark frame with
matching TOTALEXP will be subtracted from each image during processing.
cosmic_ray: bool, optional
Whether to perform single image cosmic ray removal, using the lacosmic algorithm,
default False. Requires both gain and readnoise to be set.
cosmic_ray_kwargs: dict, optional
Additional keyword arguments to pass to the ccdproc.cosmicray_lacosmic function.
gain: str or astropy.units.Quantity, optional
Either a string indicating the FITS keyword corresponding to the (inverse gain), or
a Quantity containing the gain value to use. If both gain and read noise are given
an uncertainty frame will be created.
readnoise: str or astropy.units.Quantity, optional
Either a string indicating the FITS keyword corresponding to read noise, or a Quantity
containing the read noise value to use. If both read noise and gain are given then an
uncertainty frame will be created.
normalise: bool, optional
If True each image will be normalised. Default False.
normalise_func: callable, optional
Function to use for normalisation. Each image will be divided by normalise_func(image).
Default np.ma.average.
combine_type: str, optional
Type of image combination to use, 'MEAN' or 'MEDIAN'. If None the individual
images will be processed but not combined and the return value will be a list of
CCDData objects. Default None.
sigma_clip: bool, optional
If True will perform sigma clipping on the image stack before combining, default=False.
low_thresh: float, optional
Lower threshold to use for sigma clipping, in standard deviations. Default is 3.0.
high_thresh: float, optional
Upper threshold to use for sigma clipping, in standard deviations. Default is 3.0.
Returns
-------
master: ccdproc.CCDData
Combined image.
"""
if exposure_times:
try:
# Should work for any sequence or iterable type
exposure_times = set(exposure_times)
except TypeError:
# Not a sequence or iterable, try using as a single value.
exposure_times = {
float(exposure_times),
}
if darks:
try:
dark_filenames = set(darks)
except TypeError:
dark_filenames = {
darks,
}
dark_dict = {}
for filename in dark_filenames:
try:
dark_data = CCDData.read(filename)
except ValueError:
# Might be no units in FITS header. Assume ADU.
dark_data = CCDData.read(filename, unit='adu')
dark_dict[dark_data.header['totalexp']] = dark_data
if combine_type and combine_type not in ('MEAN', 'MEDIAN'):
raise ValueError(
"combine_type must be 'MEAN' or 'MEDIAN', got '{}''".format(
combine_type))
fitspath = Path(fitspath)
if fitspath.is_file():
# FITS path points to a single file, turn into a list.
filenames = [
fitspath,
]
elif fitspath.is_dir():
# FITS path is a directory. Find FITS file and collect values of selected FITS headers
ifc = ImageFileCollection(fitspath, keywords='*')
if len(ifc.files) == 0:
raise RuntimeError("No FITS files found in {}".format(fitspath))
# Filter by observation type.
if obstype:
try:
ifc = ifc.filter(obstype=obstype)
except FileNotFoundError:
raise RuntimeError(
"No FITS files with OBSTYPE={}.".format(obstype))
# Filter by object name.
if object:
try:
ifc = ifc.filter(object=object)
except FileNotFoundError:
raise RuntimeError(
"No FITS files with OBJECT={}.".format(object))
filenames = [
Path(ifc.location).joinpath(filename) for filename in ifc.files
]
else:
raise ValueError(
"fitspath '{}' is not an accessible file or directory.".format(
fitspath))
# Load image(s) and process them.
images = []
for filename in filenames:
try:
ccddata = CCDData.read(filename)
except ValueError:
# Might be no units in FITS header. Assume ADU.
ccddata = CCDData.read(filename, unit='adu')
# Filtering by exposure times here because it's hard filter ImageFileCollection
# with an indeterminate number of possible values.
if not exposure_times or ccddata.header['totalexp'] in exposure_times:
if window:
ccddata = ccdproc.trim_image(ccddata[window[1]:window[3] + 1,
window[0]:window[2] + 1])
if percentile:
# Check percentile value is within specified range, otherwise skip to next image.
percentile_value = np.percentile(ccddata.data, percentile)
if percentile_value < percentile_min or percentile_value > percentile_max:
continue
if darks:
try:
ccddata = ccdproc.subtract_dark(
ccddata,
dark_dict[ccddata.header['totalexp']],
exposure_time='totalexp',
exposure_unit=u.second)
except KeyError:
raise RuntimeError(
"No dark with matching totalexp for {}.".format(
filename))
if gain:
if isinstance(gain, str):
egain = ccddata.header[gain]
egain = egain * u.electron / u.adu
elif isinstance(gain, u.Quantity):
try:
egain = gain.to(u.electron / u.adu)
except u.UnitsError:
egain = (1 / gain).to(u.electron / u.adu)
else:
raise ValueError(
f"gain must be a string or Quantity, got {gain}.")
if readnoise:
if isinstance(readnoise, str):
rn = ccddata.header[readnoise]
rn = rn * u.electron
elif isinstance(readnoise, u.Quantity):
try:
rn = readnoise.to(u.electron / u.pixel)
except u.UnitsError:
rn = (readnoise * u.pixel).to(u.electron)
else:
raise ValueError(
f"readnoise must be a string or Quantity, got {readnoise}."
)
if gain and readnoise:
ccddata = ccdproc.create_deviation(ccddata,
gain=egain,
readnoise=rn,
disregard_nan=True)
if gain:
ccddata = ccdproc.gain_correct(ccddata, gain=egain)
if cosmic_ray:
if not gain and readnoise:
raise ValueError(
"Cosmic ray removal required both gain & readnoise.")
ccddata = ccdproc.cosmicray_lacosmic(
ccddata,
gain=1.0, # ccddata already gain corrected
readnoise=rn,
**cosmic_ray_kwargs)
if normalise:
ccddata = ccddata.divide(normalise_func(ccddata.data))
images.append(ccddata)
n_images = len(images)
if n_images == 0:
msg = "No FITS files match exposure time criteria"
raise RuntimeError(msg)
if n_images == 1 and combine_type:
warn(
"Combine type '{}' selected but only 1 matching image, skipping image combination.'"
)
combine_type = None
if combine_type:
combiner = Combiner(images)
# Sigma clip data
if sigma_clip:
if combine_type == 'MEAN':
central_func = np.ma.average
else:
# If not MEAN has to be MEDIAN, checked earlier that it was one or the other.
central_func = np.ma.median
combiner.sigma_clipping(low_thresh=low_thresh,
high_thresh=high_thresh,
func=central_func)
# Stack images.
if combine_type == 'MEAN':
master = combiner.average_combine()
else:
master = combiner.median_combine()
# Populate header of combined image with metadata about the processing.
master.header['fitspath'] = str(fitspath)
if obstype:
master.header['obstype'] = obstype
if exposure_times:
if len(exposure_times) == 1:
master.header['totalexp'] = float(exposure_times.pop())
else:
master.header['totalexp'] = tuple(exposure_times)
master.header['nimages'] = n_images
master.header['combtype'] = combine_type
master.header['sigclip'] = sigma_clip
if sigma_clip:
master.header['lowclip'] = low_thresh
master.header['highclip'] = high_thresh
else:
# No image combination, just processing indivudal image(s)
if n_images == 1:
master = images[0]
else:
master = images
return master
def ccds(self, masks=None, trim=None, **kwargs):
"""
Generator that yields each 'ccdproc.CCDData' objects in the collection.
Parameters
----------
masks : str, list of str or optional
Area to be masked.
trim : str or optional
Trim section.
**kwargs :
Any additional keywords are used to filter the items returned.
Yields
------
'ccdproc.CCDData'
yield the next 'ccdproc.CCDData' in the collection.
Examples
--------
>>> from tuglib.io import FitsCollection
>>>
>>> mask = '[:, 1000:1046]'
>>> trim = '[100:1988, :]'
>>> images = FitsCollection(
location='/home/user/data/fits/', gain=0.57, read_noise=4.11)
>>> biases = images.ccds(OBJECT='BIAS', masks=mask, trim=trim)
"""
if masks is not None:
if not isinstance(masks, (str, list, type(None))):
raise TypeError(
"'masks' should be 'str', 'list' or 'None' object.")
if trim is not None:
if not isinstance(trim, str):
raise TypeError("'trim' should be a 'str' object.")
tmp = np.full(len(self._collection), True, dtype=bool)
if len(kwargs) != 0:
for key, val in kwargs.items():
if key == 'filename':
file_mask = np.array([
fnmatch.fnmatch(filename, kwargs['filename'])
for filename in self._filenames_without_path
],
dtype=bool)
tmp = tmp & file_mask
else:
tmp = tmp & (self._collection[key.upper()] == val)
if np.count_nonzero(tmp) == 0:
yield None
x = self._collection[tmp]['NAXIS1'][0]
y = self._collection[tmp]['NAXIS2'][0]
shape = (y, x)
mask = None
if masks is not None:
mask = make_mask(shape, masks)
for filename in self._collection[tmp]['filename']:
ccd = CCDData.read(filename,
unit=self._unit,
output_verify='silentfix+ignore')
ccd.mask = mask
ccd = trim_image(ccd, trim)
if (self._gain is not None) and (self._read_noise is not None):
data_with_deviation = create_deviation(
ccd,
gain=self._gain,
readnoise=self._read_noise,
disregard_nan=self._disregard_nan)
gain_corrected = gain_correct(data_with_deviation, self._gain)
yield gain_corrected
else:
yield ccd
def __call__(self, collection=None, masks=None, trim=None, **kwargs):
"""
Generator that yields each 'ccdproc.CCDData' objects in the collection.
Parameters
----------
collection : 'FitsCollection.collection' or optional
Filtered collection.
masks : str, list of str or optional
Area to be masked.
trim : str or optional
Trim section.
**kwargs :
Any additional keywords are used to filter the items returned.
Yields
------
'ccdproc.CCDData'
yield the next 'ccdproc.CCDData' in the collection.
Examples
--------
>>> from tuglib.io import FitsCollection
>>>
>>> mask = '[:, 1000:1046]'
>>> trim = '[100:1988, :]'
>>>
>>> images = FitsCollection(
location='/home/user/data/fits/', gain=0.57, read_noise=4.11)
>>>
>>> query = images['EXPTIME'] == 100.0
>>> sub_collections = images[query]
>>>
>>> ccds = images(sub_collections, masks=mask, trim=trim)
"""
if masks is not None:
if not isinstance(masks, (str, list, type(None))):
raise TypeError(
"'masks' should be 'str', 'list' or 'None' object.")
if trim is not None:
if not isinstance(trim, str):
raise TypeError("'trim' should be a 'str' object.")
if collection is None:
return self.ccds(masks=masks, trim=trim, **kwargs)
tmp = np.full(len(collection), True, dtype=bool)
if len(kwargs) != 0:
for key, val in kwargs.items():
tmp = tmp & (collection[key] == val)
x = collection[tmp]['NAXIS1'][0]
y = collection[tmp]['NAXIS2'][0]
shape = (y, x)
mask = None
if masks is not None:
mask = make_mask(shape, masks)
if (self._gain is not None) and (self._read_noise is not None):
for filename in collection[tmp]['filename']:
ccd = CCDData.read(filename, unit=self._unit)
ccd.mask = mask
ccd = trim_image(ccd, trim)
data_with_deviation = create_deviation(
ccd,
gain=self._gain,
readnoise=self._read_noise,
disregard_nan=self._disregard_nan)
gain_corrected = gain_correct(data_with_deviation, self._gain)
yield gain_corrected
else:
for filename in collection[tmp]['filename']:
ccd = CCDData.read(filename, unit=self._unit)
ccd.mask = mask
ccd = trim_image(ccd, trim)
yield ccd
def ccd_process(ccd,
oscan=None,
trim=None,
error=False,
masterbias=None,
bad_pixel_mask=None,
gain=None,
rdnoise=None,
oscan_median=True,
oscan_model=None):
"""Perform basic processing on ccd data.
The following steps can be included:
* overscan correction
* trimming of the image
* create edeviation frame
* gain correction
* add a mask to the data
* subtraction of master bias
The task returns a processed `ccdproc.CCDData` object.
Parameters
----------
ccd: `ccdproc.CCDData`
Frame to be reduced
oscan: None, str, or, `~ccdproc.ccddata.CCDData`
For no overscan correction, set to None. Otherwise proivde a region
of `ccd` from which the overscan is extracted, using the FITS
conventions for index order and index start, or a
slice from `ccd` that contains the overscan.
trim: None or str
For no trim correction, set to None. Otherwise proivde a region
of `ccd` from which the image should be trimmed, using the FITS
conventions for index order and index start.
error: boolean
If True, create an uncertainty array for ccd
masterbias: None, `~numpy.ndarray`, or `~ccdproc.CCDData`
A materbias frame to be subtracted from ccd.
bad_pixel_mask: None or `~numpy.ndarray`
A bad pixel mask for the data. The bad pixel mask should be in given
such that bad pixels havea value of 1 and good pixels a value of 0.
gain: None or `~astropy.Quantity`
Gain value to multiple the image by to convert to electrons
rdnoise: None or `~astropy.Quantity`
Read noise for the observations. The read noise should be in
`~astropy.units.electron`
oscan_median : bool, optional
If true, takes the median of each line. Otherwise, uses the mean
oscan_model : `~astropy.modeling.Model`, optional
Model to fit to the data. If None, returns the values calculated
by the median or the mean.
Returns
-------
ccd: `ccdproc.CCDData`
Reduded ccd
Examples
--------
1. To overscan, trim, and gain correct a data set:
>>> import numpy as np
>>> from astropy import units as u
>>> from hrsprocess import ccd_process
>>> ccd = CCDData(np.ones([100, 100]), unit=u.adu)
>>> nccd = ccd_process(ccd, oscan='[1:10,1:100]', trim='[10:100, 1,100]',
error=False, gain=2.0*u.electron/u.adu)
"""
# make a copy of the object
nccd = ccd.copy()
# apply the overscan correction
if isinstance(oscan, ccdproc.CCDData):
nccd = ccdproc.subtract_overscan(nccd,
overscan=oscan,
median=oscan_median,
model=oscan_model)
elif isinstance(oscan, six.string_types):
nccd = ccdproc.subtract_overscan(nccd,
fits_section=oscan,
median=oscan_median,
model=oscan_model)
elif oscan is None:
pass
else:
raise TypeError('oscan is not None, a string, or CCDData object')
# apply the trim correction
if isinstance(trim, six.string_types):
nccd = ccdproc.trim_image(nccd, fits_section=trim)
elif trim is None:
pass
else:
raise TypeError('trim is not None or a string')
# create the error frame
if error and gain is not None and rdnoise is not None:
nccd = ccdproc.create_deviation(nccd, gain=gain, rdnoise=rdnoise)
elif error and (gain is None or rdnoise is None):
raise ValueError(
'gain and rdnoise must be specified to create error frame')
# apply the bad pixel mask
if isinstance(bad_pixel_mask, np.ndarray):
nccd.mask = bad_pixel_mask
elif bad_pixel_mask is None:
pass
else:
raise TypeError('bad_pixel_mask is not None or numpy.ndarray')
# apply the gain correction
if isinstance(gain, u.quantity.Quantity):
nccd = ccdproc.gain_correct(nccd, gain)
elif gain is None:
pass
else:
raise TypeError('gain is not None or astropy.Quantity')
# test subtracting the master bias
if isinstance(masterbias, ccdproc.CCDData):
nccd = ccdproc.subtract_bias(nccd, masterbias)
elif isinstance(masterbias, np.ndarray):
nccd.data = nccd.data - masterbias
elif masterbias is None:
pass
else:
raise TypeError(
'masterbias is not None, numpy.ndarray, or a CCDData object')
return nccd
