class StepBiasDarkFlat(StepLoadAux, StepParent):
""" Pipeline Step Object to calibrate Bias/Dark/Flat files
"""
stepver = '0.1' # pipe step version
def __init__(self):
""" Constructor: Initialize data objects and variables
"""
# call superclass constructor (calls setup)
super(StepBiasDarkFlat, self).__init__()
# bias values
self.biasloaded = False # indicates if bias has been loaded
self.bias = None # CCD data object containing arrays with bias values
self.biasdata = DataFits() # Pipedata object containing the bias file
# bias file info and header keywords to fit
self.biasname = '' # name of selected bias file
self.biasfitkeys = [] # FITS keywords that are present in bias
self.biaskeyvalues = [] # values of FITS keywords (from data file)
# dark values
self.darkloaded = False # indicates if dark has been loaded
self.dark = None # CCD data object containing arrays with dark values
self.darkdata = DataFits() # Pipedata object containing the dark file
# dark file info and header keywords to fit
self.darkname = '' # name of selected dark file
self.darkfitkeys = [] # FITS keywords that have to fit for dark
self.darkkeyvalues = [] # values of FITS keywords (from data file)
# flat values
self.flatloaded = False # indicates if flat has been loaded
self.flat = None # CCD data object containing arrays with flat values
self.flatdata = DataFits() # Pipedata object containing the flat file
# flat file info and header keywords to fit
self.flatname = '' # name of selected flat file
self.flatfitkeys = [] # FITS keywords that have to fit for flat
self.flatkeyvalues = [] # values of flat keywords (from data file)
# set configuration
self.log.debug('Init: done')
# This function is directly lifted from CCDProc https://github.com/astropy/ccdproc/blob/master/ccdproc/core.py
# Instead of directly calling CCDProc, we have included the function here to increase educational value
# and to decrease reliance on external libraries.
def subtract_bias(self, image, bias):
"""
Subtract master bias from image.
Parameters
----------
image : `~astropy.nddata.CCDData`
Image from which bias will be subtracted.
bias : `~astropy.nddata.CCDData`
Master image to be subtracted from ``ccd``.
{log}
Returns
-------
result : `~astropy.nddata.CCDData`
CCDData object with bias subtracted.
"""
self.log.debug('Subtracting bias...')
result = image.copy()
try:
result.data = image.data - bias.data
# we believe that we should keep this error detection in theory, the bias
# and image both come from seo, so their units should be the same
except ValueError as e:
if 'operand units' in str(e):
raise u.UnitsError(
"Unit '{}' of the uncalibrated image does not "
"match unit '{}' of the calibration "
"image".format(image.unit, bias.unit))
else:
raise e
self.log.debug('Subtracted bias.')
return result
# this code is also lifted from ccdproc https://github.com/astropy/ccdproc/blob/master/ccdproc/core.py
# some of the code is removed from the original ccdproc because it is not relevant to how SEO currently
# processes data. If you are looking at this code in the future, there is more code available to draw from
def subtract_dark(self,
image,
dark,
scale=False,
exposure_time=None,
exposure_unit=None):
"""
Subtract dark current from an image.
Parameters
----------
image : `~astropy.nddata.CCDData`
Image from which dark will be subtracted.
dark : `~astropy.nddata.CCDData`
Dark image.
exposure_time : str or `~ccdproc.Keyword` or None, optional
Name of key in image metadata that contains exposure time.
Default is ``None``.
exposure_unit : `~astropy.units.Unit` or None, optional
Unit of the exposure time if the value in the meta data does not
include a unit.
Default is ``None``.
scale: bool, optional
If True, scale the dark frame by the exposure times.
Default is ``False``.
{log}
Returns
-------
result : `~astropy.nddata.CCDData`
Dark-subtracted image.
"""
self.log.debug('Subtracting dark...')
result = image.copy()
try:
# if dark current is linear, then this first step scales the provided
# dark to match the exposure time
if scale:
dark_scaled = dark.copy()
data_exposure = image.header[exposure_time]
dark_exposure = dark.header[exposure_time]
# data_exposure and dark_exposure are both quantities,
# so we can just have subtract do the scaling
dark_scaled = dark_scaled.multiply(data_exposure /
dark_exposure)
result.data = image.data - dark_scaled.data
else:
result.data = image.data - dark.data
except (u.UnitsError, u.UnitConversionError, ValueError) as e:
# Make the error message a little more explicit than what is returned
# by default.
raise u.UnitsError("Unit '{}' of the uncalibrated image does not "
"match unit '{}' of the calibration "
"image".format(image.unit, dark.unit))
self.log.debug('Subtracted dark.')
return result
# This code is also from ccdproc. A notable removal is the option to manually choose
# maximum and minimum flat values.
def flat_correct(self, image, flat):
"""Correct the image for flat fielding.
The flat field image is normalized by its mean or a user-supplied value
before flat correcting.
Parameters
----------
ccd : `~astropy.nddata.CCDData`
Data to be transformed.
flat : `~astropy.nddata.CCDData`
Flatfield to apply to the data.
{log}
Returns
-------
ccd : `~astropy.nddata.CCDData`
CCDData object with flat corrected.
"""
self.log.debug('Correcting flat...')
# Use the min_value to replace any values in the flat
flat_corrected = image.copy()
use_flat = flat
flat_mean_val = use_flat.data.mean()
# Normalize the flat.
flat_mean = flat_mean_val * use_flat.unit
flat_normed = use_flat.data / flat_mean
# divide through the flat
flat_corrected.data = image.data / flat_normed
self.log.debug('Corrected flat.')
return flat_corrected
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'biasdarkflat'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'bdf'
# Set Logger for this pipe step
self.log = logging.getLogger('stoneedge.pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append([
'reload', False,
'Set to True to look for new bias files for every input'
])
# Get parameters for StepLoadAux, replace auxfile with biasfile
self.loadauxsetup('bias')
# Get parameters for StepLoadAux, replace auxfile with darkfile
self.loadauxsetup('dark')
# Get parameters for StepLoadAux, replace auxfile with flatfile
self.loadauxsetup('flat')
# confirm end of setup
self.log.debug('Setup: done')
'''# Looking for similar exptime
def closestExp(self):
input_exptime = self.datain.getheadval('EXPTIME')
dark_exptime = self.loadauxname('dark', multi = True).getheadval('EXPTIME')
nearexp = {abs(dark_ave_exptime - exp): exp for exp in dark_exptime}
return nearexp[min(nearexp.keys())]
'''
def run(self):
""" Runs the calibrating algorithm. The calibrated data is
returned in self.dataout
"""
### Preparation
# Load bias files if necessary
if not self.biasloaded or self.getarg('reload'):
self.loadbias()
# Else: check data for correct instrument configuration - currently not in use(need improvement)
else:
for keyind in range(len(self.biasfitkeys)):
if self.biaskeyvalues[keyind] != self.datain.getheadval(
self.biasfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.biasfitkeys[keyind])
# Load dark files if necessary
if not self.darkloaded or self.getarg('reload'):
self.loaddark()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.darkfitkeys)):
if self.darkkeyvalues[keyind] != self.datain.getheadval(
self.darkfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.darkfitkeys[keyind])
# Load flat files if necessary
if not self.flatloaded or self.getarg('reload'):
self.loadflat()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.flatfitkeys)):
if self.flatkeyvalues[keyind] != self.datain.getheadval(
self.flatfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.flatfitkeys[keyind])
# in the config file, set the 'intermediate' variable to either true or false to enable
# saving of intermediate steps
saveIntermediateSteps = self.config['biasdarkflat']['intermediate']
self.dataout = DataFits(config=self.datain.config)
#convert self.datain to CCD Data object
image = CCDData(self.datain.image, unit='adu')
image.header = self.datain.header
# subtract bias from image
image = self.subtract_bias(image, self.bias)
if (saveIntermediateSteps == "true"):
self.dataout.imageset(image.data, imagename="BIAS")
# self.dataout.setheadval('DATATYPE','IMAGE', dataname="BIAS")
self.dataout.setheadval('HISTORY',
'BIAS: %s' % self.biasname,
dataname="BIAS")
# subtract dark from image
image = self.subtract_dark(image,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second)
if (saveIntermediateSteps == "true"):
self.dataout.imageset(image.data, imagename="DARK")
# self.dataout.setheadval('DATATYPE','IMAGE', dataname="DARK")
self.dataout.setheadval('HISTORY',
'BIAS: %s' % self.biasname,
dataname="DARK")
self.dataout.setheadval('HISTORY',
'DARK: %s' % self.darkname,
dataname="DARK")
# apply flat correction to image
image = self.flat_correct(image, self.flat)
# if separating bias,dark,flat steps , save the flat-corrected portion into its own hdu
if (saveIntermediateSteps == "true"):
self.dataout.imageset(image.data, imagename="FLAT")
# self.dataout.setheadval('DATATYPE','IMAGE', dataname="FLAT")
self.dataout.setheadval('HISTORY',
'BIAS: %s' % self.biasname,
dataname="FLAT")
self.dataout.setheadval('HISTORY',
'DARK: %s' % self.darkname,
dataname="FLAT")
self.dataout.setheadval('HISTORY',
'FLAT: %s' % self.flatname,
dataname="FLAT")
else:
# copy calibrated image into self.dataout
self.dataout.image = image.data
self.dataout.header = self.datain.header
### Finish - cleanup
# Update DATATYPE
self.dataout.setheadval('DATATYPE', 'IMAGE')
# Add bias, dark files to History
self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname)
self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname)
self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname)
self.dataout.filename = self.datain.filename
def loadbias(self):
""" Loads the bias information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master bias frame by ccdproc.
"""
#master bias frame
#Search for bias and load it into data object
namelist = self.loadauxname('bias', multi=False)
self.log.info('File loaded: %s' % namelist)
if (len(namelist) == 0):
self.log.error('Bias calibration frame not found.')
raise RuntimeError('No bias file loaded')
self.log.debug('Creating master bias frame...')
#if there is just one, use it as biasfile or else combine all to make a master bias
self.bias = CCDData.read(namelist, unit='adu', relax=True)
# Finish up
self.biasloaded = True
self.biasname = namelist
self.log.debug('LoadBias: done')
def loaddark(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master dark frame by ccdproc.
Also bias corrects dark files if not already done.
"""
#master dark frame
dark_is_bias_corrected = False
dark_bias = None
namelist = self.loadauxname('dark', multi=False)
if (len(namelist) == 0):
self.log.error('Dark calibration frame(s) not found.')
raise RuntimeError('No dark file loaded')
# This has been commented out as it is now in StepMasterDark
# darks = None
# for name in namelist:
# #is (any) dark file bias corrected?
# header = fits.getheader(name)
# if(header.get('BIAS') != None):
# dark_is_bias_corrected = True
# dark_bias = header.get('BIAS')
# elif(header.get('BIASCORR') != None):
# dark_is_bias_corrected = True
# dark_bias = header.get('BIASCORR')
# if(darks):
# darks += ','+name
# else:
# darks = name
self.log.debug('Creating master dark frame...')
#if there is just one, use it as darkfile or else combine all to make a master dark
self.dark = CCDData.read(namelist, unit='adu', relax=True)
#bias correct, if necessary
# if(not dark_is_bias_corrected):
# #Subtracting master bias frame from master dark frame
# self.dark = ccdproc.subtract_bias(self.dark, self.bias, add_keyword=False)
# else:
# self.log.debug('Master dark frame is *already* bias corrected (%s).' % dark_bias)
# Finish up
self.darkloaded = True
self.darkname = namelist
self.log.debug('LoadDark: done')
def loadflat(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master flat frame by ccdproc.
Also biascorrects and dark corrects flat files if not already done.
"""
#create master flat frame
flat_is_bias_corrected = False
flat_bias = None
flat_is_dark_corrected = False
flat_dark = None
flat_ave_exptime = 0
namelist = self.loadauxname('flat', multi=False)
if (len(namelist) == 0):
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file loaded')
count = 0
datalist = []
flat_corrected = None
# This has been commented out as it is now in StepMasterFlat
#check a few things in these flat component frames
# for name in namelist:
# header = fits.getheader(name)
#is this flat bias corrected?
# if(header.get('BIAS') != None):
# flat_is_bias_corrected = True
# flat_bias = header.get('BIAS')
# elif(header.get('BIASCORR') != None):
# flat_is_bias_corrected = True
# flat_bias = header.get('BIASCORR')
# #is this flat dark corrected?
# if(header.get('DARK') != None):
# flat_is_dark_corrected = True
# flat_dark = header.get('DARK')
# elif(header.get('DARKCORR') != None):
# flat_is_dark_corrected = True
# flat_dark = header.get('DARKCORR')
# flat_corrected = "%s"%(name.rsplit('.',1)[0])+".corrected"
# shutil.copy(name, flat_corrected)
# self.log.debug('Copying %s to %s' % (name, flat_corrected))
# self.flat = ccdproc.CCDData.read(flat_corrected, unit='adu', relax=True)
# #bias correct, if necessary
# if(not flat_is_bias_corrected):
# self.log.debug('Subtracting master bias frame from flat frame...')
# self.flat = ccdproc.subtract_bias(self.flat, self.bias, add_keyword=False)
# else:
# self.log.debug('Flat frame (%s) is *already* bias corrected.'%flat_bias)
# #dark correct, if necessary
# if(not flat_is_dark_corrected):
# self.log.debug('Subtracting master dark frame from flat frame...')
# self.flat = ccdproc.subtract_dark(self.flat, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second, add_keyword=False)
# else:
# self.log.debug('Flat frame (%s) is *already* dark corrected.'%flat_dark)
# #create CCD Data object list with corrected flat files
# datalist.append(self.flat)
# #calc average exposure time for potential dark correction
# if(header.get('EXPTIME') != None):
# try:
# exptime = float(header.get('EXPTIME'))
# flat_ave_exptime += exptime
# except ValueError:
# self.log.error('Exposure time (EXPTIME) is not a float (%s).'%(header.get('EXPTIME')))
# count += 1
# #calc average exposure time
# if(count > 0):
# flat_ave_exptime = flat_ave_exptime/count
# self.flat.header['EXPTIME'] = flat_ave_exptime
# self.log.info("Average exposure time for flats is %f"%flat_ave_exptime)
self.log.debug('Creating master flat frame...')
#if there is just one, use it as flatfile or else combine all to make a master flat
self.flat = CCDData.read(namelist, unit='adu', relax=True)
# Finish up
self.flatloaded = True
self.flatname = namelist
self.log.debug('LoadFlat: done')
def reset(self):
""" Resets the step to the same condition as it was when it was
created. Internal variables are reset, any stored data is
erased.
"""
self.biasloaded = False
self.bias = None
self.darkloaded = False
self.dark = None
self.flatloaded = False
self.flat = None
self.log.debug('Reset: done')
class StepMasterDark(StepLoadAux, StepMIParent):
""" Stone Edge Pipeline Step Master Dark Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '0.1' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'masterdark'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'mdark'
# Set Logger for this pipe step
self.log = logging.getLogger('stoneedge.pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters !!!! WHAT PARAMETERS ARE NEEDED ????? !!!!!
self.paramlist.append([
'combinemethod', 'median',
'Specifies how the files should be combined - options are median, average, sum'
])
self.paramlist.append([
'outputfolder', '',
'Output directory location - default is the folder of the input files'
])
# Get parameters for StepLoadAux, replace auxfile with biasfile
self.loadauxsetup('bias')
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
# Find master bias to subtract from master dark
biaslist = self.loadauxname('bias', multi=False)
if (len(biaslist) == 0):
self.log.error('No bias calibration frames found.')
self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True)
# Create empy list for filenames of loaded frames
filelist = []
for fin in self.datain:
self.log.debug("Input filename = %s" % fin.filename)
filelist.append(fin.filename)
# Make a dummy dataout
self.dataout = DataFits(config=self.config)
if len(self.datain) == 0:
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file(s) loaded')
self.log.debug('Creating master flat frame...')
# Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias
if (len(filelist) == 1):
self.dark = ccdproc.CCDData.read(filelist[0],
unit='adu',
relax=True)
self.dark = ccdproc.subtract_bias(self.dark,
self.bias,
add_keyword=False)
else:
darklist = []
for i in filelist:
dark = ccdproc.CCDData.read(i, unit='adu', relax=True)
darksubbias = ccdproc.subtract_bias(dark,
self.bias,
add_keyword=False)
darklist.append(darksubbias)
self.dark = ccdproc.combine(darklist,
method=self.getarg('combinemethod'),
unit='adu',
add_keyword=True)
# set output header, put image into output
self.dataout.header = self.datain[0].header
self.dataout.imageset(self.dark)
# rename output filename
outputfolder = self.getarg('outputfolder')
if outputfolder != '':
outputfolder = os.path.expandvars(outputfolder)
self.dataout.filename = os.path.join(outputfolder,
os.path.split(filelist[0])[1])
else:
self.dataout.filename = filelist[0]
# Add history
self.dataout.setheadval('HISTORY',
'MasterDark: %d files used' % len(filelist))
class StepRGB(StepMIParent):
""" Stone Edge Pipeline Step RGB Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '0.1' # pipe step version
def __init__(self):
""" Constructor: Initialize data objects and variables
"""
# call superclass constructor (calls setup)
super(StepRGB, self).__init__()
# list of data
self.datalist = [] # used in run() for every new input data file
# set configuration
self.log.debug('Init: done')
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'makergb'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'rgb'
# Set Logger for this pipe step
self.log = logging.getLogger('stoneedge.pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append([
'minpercent', 0.05,
'Specifies the percentile for the minimum scaling'
])
self.paramlist.append([
'maxpercent', 0.999,
'Specifies the percentile for the maximum scaling'
])
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
''' Select 3 input dataset to use, store in datause '''
#Store number of inputs
num_inputs = len(self.datain)
# Create variable to hold input files
# Copy input to output header and filename
datause = []
self.log.debug('Number of input files = %d' % num_inputs)
# Ensure datause has 3 elements irrespective of number of input files
if num_inputs == 0: # Raise exception for no input
raise ValueError('No input')
elif num_inputs == 1:
datause = [self.datain[0], self.datain[0], self.datain[0]]
elif num_inputs == 2:
datause = [self.datain[0], self.datain[1], self.datain[1]]
else: # If inputs exceed 2 in number
# Here we know there are at least 3 files
ilist = [] # Make empty lists for each filter
rlist = []
glist = []
other = []
for element in self.datain: # Loop through the input files and add to the lists
fname = element.filename.lower()
if 'i-band' in fname or 'iband' in fname or 'iprime' in fname:
ilist.append(element)
elif 'r-band' in fname or 'rband' in fname or 'rprime' in fname:
rlist.append(element)
elif 'g-band' in fname or 'gband' in fname or 'gprime' in fname:
glist.append(element)
else:
other.append(element)
continue
self.log.debug(
'len(ilist) = %d, len(rlist) = %d, len(glist) = %d' %
(len(ilist), len(rlist), len(glist)))
# If there is at least one i-, r-, and g-band filter found in self.datain (best case)
if len(ilist) >= 1 and len(rlist) >= 1 and len(glist) >= 1:
# The first image from each filter list will be reduced in the correct order.
datause = [ilist[0], rlist[0], glist[0]]
elif len(ilist) == 0 and len(rlist) >= 1 and len(glist) >= 1:
# Cases where there is no ilist
if len(rlist) > len(glist):
datause = [rlist[0], rlist[1], glist[0]]
else:
datause = [rlist[0], glist[0], glist[1]]
elif len(glist) == 0 and len(rlist) >= 1 and len(ilist) >= 1:
# Cases where there is no glist
if len(rlist) > len(ilist):
datause = [rlist[0], rlist[1], ilist[0]]
else:
datause = [rlist[0], ilist[0], ilist[1]]
elif len(ilist) == 0 and len(rlist) >= 1 and len(glist) >= 1:
# Cases where there is no rlist
if len(ilist) > len(glist):
datause = [ilist[0], ilist[1], glist[0]]
else:
datause = [ilist[0], glist[0], glist[1]]
elif len(rlist) == 0 and len(glist) == 0:
# Case where there is only ilist
datause = [ilist[0], ilist[1], ilist[2]]
elif len(rlist) == 0 and len(ilist) == 0:
# Case where there is only glist
datause = [glist[0], glist[1], glist[2]]
elif len(ilist) == 0 and len(glist) == 0:
# Case where there is only rlist
datause = [rlist[0], rlist[1], rlist[2]]
self.log.debug(
'Files used: R = %s G = %s B = %s' %
(datause[0].filename, datause[1].filename, datause[2].filename))
self.dataout = DataFits(config=self.config)
self.dataout.header = datause[0].header
self.dataout.filename = datause[0].filename
img = datause[0].image
img1 = datause[1].image
img2 = datause[2].image
''' Finding Min/Max scaling values '''
# Create a Data Cube with floats
datacube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype=float)
# Enter the image data into the cube so an absolute max can be found
datacube[:, :, 0] = img
datacube[:, :, 1] = img1
datacube[:, :, 2] = img2
# Find how many data points are in the data cube
datalength = img.shape[0] * img.shape[1] * 3
# Create a 1-dimensional array with all the data, then sort it
datacube.shape = (datalength, )
datacube.sort()
# Now use arrays for each filter to find separate min values
rarray = img.copy()
garray = img1.copy()
barray = img2.copy()
# Shape and sort the arrays
arrlength = img.shape[0] * img.shape[1]
rarray.shape = (arrlength, )
rarray.sort()
garray.shape = (arrlength, )
garray.sort()
barray.shape = (arrlength, )
barray.sort()
# Find the min/max percentile values in the data for scaling
# Values are determined by parameters in the pipe configuration file
minpercent = int(arrlength * self.getarg('minpercent'))
maxpercent = int(datalength * self.getarg('maxpercent'))
# Find the final data values to use for scaling from the image data
rminsv = rarray[minpercent] #sv stands for "scalevalue"
gminsv = garray[minpercent]
bminsv = barray[minpercent]
maxsv = datacube[maxpercent]
self.log.info(' Scale min r/g/b: %f/%f/%f' % (rminsv, gminsv, bminsv))
self.log.info(' Scale max: %f' % maxsv)
# The same min/max values will be used to scale all filters
''' Finished Finding scaling values '''
''' Combining Function '''
# Make new cube with the proper data type for color images (uint8)
# Use square root (sqrt) scaling for each filter
# log or asinh scaling is also available
#astropy.vidualizations.SqrtStretch()
imgcube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype='uint8')
minsv = [rminsv, gminsv, bminsv]
for i in range(3):
# Make normalization function
norm = simple_norm(datause[i].image,
'sqrt',
min_cut=minsv[i],
max_cut=maxsv)
# Apply it
imgcube[:, :, i] = norm(datause[i].image) * 255.
self.dataout.image = imgcube
# Create variable containing all the scaled image data
imgcolor = Image.fromarray(self.dataout.image, mode='RGB')
# Save colored image as a .tif file (without the labels)
imgcolortif = imgcube.copy()
imgcolortif.astype('uint16')
### tiff.imsave('%s.tif' % self.dataout.filenamebase, imgcolortif)
''' End of combining function '''
''' Add a Label to the Image '''
draw = ImageDraw.Draw(imgcolor)
# Use a variable to make the positions and size of text relative
imgwidth = img.shape[1]
imgheight = img.shape[0]
# Open Sans-Serif Font with a size relative to the picture size
try:
# This should work on Linux
font = ImageFont.truetype(
'/usr/share/fonts/liberation/LiberationSans-Regular.ttf',
imgheight // 41)
except:
try:
# This should work on Mac
font = ImageFont.truetype('/Library/Fonts/Arial Unicode.ttf',
imgheight // 41)
except:
try:
# This should work on Windows
font = ImageFont.truetype('C:\\Windows\\Fonts\\arial.ttf',
imgheight // 41)
except:
# This should work in Colab
font = ImageFont.truetype(
'/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf',
imgheight // 41)
# If this still doesn't work - then add more code to make it run on YOUR system
# Use the beginning of the FITS filename as the object name
filename = os.path.split(self.dataout.filename)[-1]
try:
objectname = filename.split('_')[0]
objectname = objectname[0].upper() + objectname[1:]
except Exception:
objectname = 'Unknown.'
objectname = 'Object: %s' % objectname
# Read labels at their respective position (kept relative to image size)
# Left corner: object, observer, observatory
# Right corner: Filters used for red, green, and blue colors
draw.text((imgwidth / 100, imgheight / 1.114),
objectname, (255, 255, 255),
font=font)
# Read FITS keywords for the observer, observatory, and filters
if 'OBSERVER' in self.dataout.header:
observer = 'Observer: %s' % self.dataout.getheadval('OBSERVER')
draw.text((imgwidth / 100, imgheight / 1.073),
observer, (255, 255, 255),
font=font)
if 'OBSERVAT' in self.dataout.header:
observatory = 'Observatory: %s' % self.dataout.getheadval(
'OBSERVAT')
draw.text((imgwidth / 100, imgheight / 1.035),
observatory, (255, 255, 255),
font=font)
if 'FILTER' in datause[0].header:
red = 'R: %s' % datause[0].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.114),
red, (255, 255, 255),
font=font)
if 'FILTER' in datause[1].header:
green = 'G: %s' % datause[1].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.073),
green, (255, 255, 255),
font=font)
if 'FILTER' in datause[2].header:
blue = 'B: %s' % datause[2].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.035),
blue, (255, 255, 255),
font=font)
# Make image name
imgname = self.dataout.filenamebegin
if imgname[-1] in '_-,.': imgname = imgname[:-1]
imgname += '.jpg'
# Save the completed image
imgcolor.save(imgname)
self.log.info('Saving file %sjpg' % self.dataout.filenamebegin)
''' End of Label Code '''
# Set complete flag
self.dataout.setheadval('COMPLETE', 1,
'Data Reduction Pipe: Complete Data Flag')
def reset(self):
""" Resets the step to the same condition as it was when it was
created. Internal variables are reset, any stored data is
erased.
"""
self.log.debug('Reset: done')
def test(self):
""" Test Pipe Step Parent Object:
Runs a set of basic tests on the object
"""
# log message
self.log.info('Testing pipe step rgb')
# log message
self.log.info('Testing pipe step rgb - Done')
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in jpeg_dataout.
"""
''' Select 3 input dataset to use, store in datause '''
#Store number of inputs
num_inputs = len(self.datain)
# Create variable to hold input files
# Copy input to output header and filename
datause = [None, None, None]
self.log.debug('Number of input files = %d' % num_inputs)
if num_inputs == 0: # Raise exception for no input
raise ValueError('No input')
elif num_inputs == 1:
datause = [self.datain[0], self.datain[0], self.datain[0]]
elif num_inputs == 2:
datause = [self.datain[0], self.datain[0], self.datain[1]]
else:
filterorder_list = self.getarg('filterorder').split('|')
filterprefs_list = self.getarg('filterprefs').split('|')
datain_filter_list = [
element.getheadval('filter') for element in self.datain
]
used_filter_flags = [False] * len(self.datain)
if len(filterprefs_list) != 3:
self.log.error(
'Invalid number of preferred filters provided (should be 3): '
+ self.getarg('filterprefs'))
else:
# Locate data matching the filters specified in filterprefs
for i, preferred_filter in enumerate(filterprefs_list):
for j, element in enumerate(self.datain):
if element.getheadval('filter') == preferred_filter:
datause[i] = element
used_filter_flags[j] = True
break
filterorder_walker = 0
for i, channel in enumerate(datause):
if channel == None:
for ordered_filter in filterorder_list[
filterorder_walker:]:
filterorder_walker = filterorder_walker + 1
if ordered_filter in datain_filter_list:
datain_index = datain_filter_list.index(
ordered_filter)
if not used_filter_flags[datain_index]:
datause[i] = self.datain[datain_index]
used_filter_flags[datain_index] = True
break
elif channel.getheadval('filter') in filterorder_list:
filterorder_walker = filterorder_list.index(
channel.getheadval('filter'))
for i, channel in enumerate(datause):
if channel == None:
for j, datain_filter in enumerate(datain_filter_list):
if not used_filter_flags[j]:
datause[i] = self.datain[j]
used_filter_flags[j] = True
break
self.log.debug(
'Files used: R = %s G = %s B = %s' %
(datause[0].filename, datause[1].filename, datause[2].filename))
jpeg_dataout = DataFits(config=self.config)
jpeg_dataout.header = datause[0].header
jpeg_dataout.filename = datause[0].filename
img = datause[0].image
img1 = datause[1].image
img2 = datause[2].image
''' Finding Min/Max scaling values '''
# Create a Data Cube with floats
datacube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype=float)
# Enter the image data into the cube so an absolute max can be found
datacube[:, :, 0] = img
datacube[:, :, 1] = img1
datacube[:, :, 2] = img2
# Find how many data points are in the data cube
datalength = img.shape[0] * img.shape[1] * 3
# Create a 1-dimensional array with all the data, then sort it
datacube.shape = (datalength, )
datacube.sort()
# Now use arrays for each filter to find separate min values
rarray = img.copy()
garray = img1.copy()
barray = img2.copy()
# Shape and sort the arrays
arrlength = img.shape[0] * img.shape[1]
rarray.shape = (arrlength, )
rarray.sort()
garray.shape = (arrlength, )
garray.sort()
barray.shape = (arrlength, )
barray.sort()
# Find the min/max percentile values in the data for scaling
# Values are determined by parameters in the pipe configuration file
minpercent = int(arrlength * self.getarg('minpercent'))
maxpercent = int(datalength * self.getarg('maxpercent'))
# Find the final data values to use for scaling from the image data
rminsv = rarray[minpercent] #sv stands for "scalevalue"
gminsv = garray[minpercent]
bminsv = barray[minpercent]
maxsv = datacube[maxpercent]
self.log.info(' Scale min r/g/b: %f/%f/%f' % (rminsv, gminsv, bminsv))
self.log.info(' Scale max: %f' % maxsv)
# The same min/max values will be used to scale all filters
''' Finished Finding scaling values '''
''' Combining Function '''
# Make new cube with the proper data type for color images (uint8)
# Use square root (sqrt) scaling for each filter
# log or asinh scaling is also available
#astropy.vidualizations.SqrtStretch()
imgcube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype='uint8')
minsv = [rminsv, gminsv, bminsv]
for i in range(3):
# Make normalization function
norm = simple_norm(datause[i].image,
'sqrt',
min_cut=minsv[i],
max_cut=maxsv)
# Apply it
imgcube[:, :, i] = norm(datause[i].image) * 255.
jpeg_dataout.image = imgcube
# Create variable containing all the scaled image data
imgcolor = Image.fromarray(jpeg_dataout.image, mode='RGB')
# Save colored image as a .tif file (without the labels)
imgcolortif = imgcube.copy()
imgcolortif.astype('uint16')
### tiff.imsave('%s.tif' % jpeg_dataout.filenamebase, imgcolortif)
''' End of combining function '''
''' Add a Label to the Image '''
draw = ImageDraw.Draw(imgcolor)
# Use a variable to make the positions and size of text relative
imgwidth = img.shape[1]
imgheight = img.shape[0]
# Open Sans-Serif Font with a size relative to the picture size
try:
# This should work on Linux
font = ImageFont.truetype(
'/usr/share/fonts/liberation/LiberationSans-Regular.ttf',
imgheight // 41)
except:
try:
# This should work on Mac
font = ImageFont.truetype('/Library/Fonts/Arial Unicode.ttf',
imgheight // 41)
except:
try:
# This should work on Windows
font = ImageFont.truetype('C:\\Windows\\Fonts\\arial.ttf',
imgheight // 41)
except:
# This should work in Colab
font = ImageFont.truetype(
'/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf',
imgheight // 41)
# If this still doesn't work - then add more code to make it run on YOUR system
# Use the beginning of the FITS filename as the object name
filename = os.path.split(jpeg_dataout.filename)[-1]
try:
objectname = filename.split('_')[0]
objectname = objectname[0].upper() + objectname[1:]
except Exception:
objectname = 'Unknown.'
objectname = 'Object: %s' % objectname
# Read labels at their respective position (kept relative to image size)
# Left corner: object, observer, observatory
# Right corner: Filters used for red, green, and blue colors
draw.text((imgwidth / 100, imgheight / 1.114),
objectname, (255, 255, 255),
font=font)
# Read FITS keywords for the observer, observatory, and filters
if 'OBSERVER' in jpeg_dataout.header:
observer = 'Observer: %s' % jpeg_dataout.getheadval('OBSERVER')
draw.text((imgwidth / 100, imgheight / 1.073),
observer, (255, 255, 255),
font=font)
if 'OBSERVAT' in jpeg_dataout.header:
observatory = 'Observatory: %s' % jpeg_dataout.getheadval(
'OBSERVAT')
draw.text((imgwidth / 100, imgheight / 1.035),
observatory, (255, 255, 255),
font=font)
if 'FILTER' in datause[0].header:
red = 'R: %s' % datause[0].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.114),
red, (255, 255, 255),
font=font)
if 'FILTER' in datause[1].header:
green = 'G: %s' % datause[1].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.073),
green, (255, 255, 255),
font=font)
if 'FILTER' in datause[2].header:
blue = 'B: %s' % datause[2].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.035),
blue, (255, 255, 255),
font=font)
# Make image name
imgname = jpeg_dataout.filenamebegin
if imgname[-1] in '_-,.': imgname = imgname[:-1]
imgname += '.jpg'
# Save the completed image
imgcolor.save(imgname)
self.log.info('Saving file %sjpg' % jpeg_dataout.filenamebegin)
# Optional folder output setup
baseimgname = os.path.basename(imgname)
folderpaths_list = self.getarg('folderpaths').split(':')
for path in folderpaths_list:
path = time.strftime(path, time.localtime())
if not os.path.exists(path):
if self.getarg('createfolders'):
os.makedirs(path)
self.log.info('Creating directory %s' % path)
else:
self.log.info('Invalid folder path %s' % path)
try:
imgcolor.save(os.path.join(path, baseimgname))
except:
self.log.exception('Could not save image to directory %s' %
path)
''' End of Label Code '''
# Set complete flag
jpeg_dataout.setheadval('COMPLETE', 1,
'Data Reduction Pipe: Complete Data Flag')
### Make output data
self.dataout = self.datain.copy()
self.dataout.append(jpeg_dataout)
class StepBiasDarkFlat(StepLoadAux, StepParent):
""" Pipeline Step Object to calibrate Bias/Dark/Flat files
"""
stepver = '0.1' # pipe step version
def __init__(self):
""" Constructor: Initialize data objects and variables
"""
# call superclass constructor (calls setup)
super(StepBiasDarkFlat, self).__init__()
# bias values
self.biasloaded = False # indicates if bias has been loaded
self.bias = None # CCD data object containing arrays with bias values
self.biasdata = DataFits() # Pipedata object containing the bias file
# bias file info and header keywords to fit
self.biasname = '' # name of selected bias file
self.biasfitkeys = [] # FITS keywords that are present in bias
self.biaskeyvalues = [] # values of FITS keywords (from data file)
# dark values
self.darkloaded = False # indicates if dark has been loaded
self.dark = None # CCD data object containing arrays with dark values
self.darkdata = DataFits() # Pipedata object containing the dark file
# dark file info and header keywords to fit
self.darkname = '' # name of selected dark file
self.darkfitkeys = [] # FITS keywords that have to fit for dark
self.darkkeyvalues = [] # values of FITS keywords (from data file)
# flat values
self.flatloaded = False # indicates if flat has been loaded
self.flat = None # CCD data object containing arrays with flat values
self.flatdata = DataFits() # Pipedata object containing the flat file
# flat file info and header keywords to fit
self.flatname = '' # name of selected flat file
self.flatfitkeys = [] # FITS keywords that have to fit for flat
self.flatkeyvalues = [] # values of flat keywords (from data file)
# set configuration
self.log.debug('Init: done')
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'biasdarkflat'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'bdf'
# Set Logger for this pipe step
self.log = logging.getLogger('stoneedge.pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append([
'reload', False,
'Set to True to look for new bias files for every input'
])
# Get parameters for StepLoadAux, replace auxfile with biasfile
self.loadauxsetup('bias')
# Get parameters for StepLoadAux, replace auxfile with darkfile
self.loadauxsetup('dark')
# Get parameters for StepLoadAux, replace auxfile with flatfile
self.loadauxsetup('flat')
# confirm end of setup
self.log.debug('Setup: done')
'''# Looking for similar exptime
def closestExp(self):
input_exptime = self.datain.getheadval('EXPTIME')
dark_exptime = self.loadauxname('dark', multi = True).getheadval('EXPTIME')
nearexp = {abs(dark_ave_exptime - exp): exp for exp in dark_exptime}
return nearexp[min(nearexp.keys())]
'''
def run(self):
""" Runs the calibrating algorithm. The calibrated data is
returned in self.dataout
"""
### Preparation
# Load bias files if necessary
if not self.biasloaded or self.getarg('reload'):
self.loadbias()
# Else: check data for correct instrument configuration - currently not in use(need improvement)
else:
for keyind in range(len(self.biasfitkeys)):
if self.biaskeyvalues[keyind] != self.datain.getheadval(
self.biasfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.biasfitkeys[keyind])
# Load dark files if necessary
if not self.darkloaded or self.getarg('reload'):
self.loaddark()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.darkfitkeys)):
if self.darkkeyvalues[keyind] != self.datain.getheadval(
self.darkfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.darkfitkeys[keyind])
# Load flat files if necessary
if not self.flatloaded or self.getarg('reload'):
self.loadflat()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.flatfitkeys)):
if self.flatkeyvalues[keyind] != self.datain.getheadval(
self.flatfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.flatfitkeys[keyind])
#convert self.datain to CCD Data object
image = ccdproc.CCDData(self.datain.image, unit='adu')
image.header = self.datain.header
#subtract bias from image
image = ccdproc.subtract_bias(image, self.bias, add_keyword=False)
#subtract dark from image
image = ccdproc.subtract_dark(image,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second,
add_keyword=False)
#apply flat correction to image
image = ccdproc.flat_correct(image, self.flat, add_keyword=False)
# copy calibrated image into self.dataout - make sure self.dataout is a pipedata object
self.dataout = DataFits(config=self.datain.config)
self.dataout.image = image.data
self.dataout.header = image.header
self.dataout.filename = self.datain.filename
### Finish - cleanup
# Update DATATYPE
self.dataout.setheadval('DATATYPE', 'IMAGE')
# Add bias, dark files to History
self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname)
self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname)
self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname)
def loadbias(self):
""" Loads the bias information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master bias frame by ccdproc.
"""
#master bias frame
#Search for bias and load it into data object
namelist = self.loadauxname('bias', multi=False)
self.log.info('File loaded: %s' % namelist)
if (len(namelist) == 0):
self.log.error('Bias calibration frame not found.')
raise RuntimeError('No bias file loaded')
self.log.debug('Creating master bias frame...')
#if there is just one, use it as biasfile or else combine all to make a master bias
self.bias = ccdproc.CCDData.read(namelist, unit='adu', relax=True)
# Finish up
self.biasloaded = True
self.biasname = namelist
self.log.debug('LoadBias: done')
def loaddark(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master dark frame by ccdproc.
Also bias corrects dark files if not already done.
"""
#master dark frame
dark_is_bias_corrected = False
dark_bias = None
namelist = self.loadauxname('dark', multi=False)
if (len(namelist) == 0):
self.log.error('Dark calibration frame(s) not found.')
raise RuntimeError('No dark file loaded')
# This has been commented out as it is now in StepMasterDark
# darks = None
# for name in namelist:
# #is (any) dark file bias corrected?
# header = fits.getheader(name)
# if(header.get('BIAS') != None):
# dark_is_bias_corrected = True
# dark_bias = header.get('BIAS')
# elif(header.get('BIASCORR') != None):
# dark_is_bias_corrected = True
# dark_bias = header.get('BIASCORR')
# if(darks):
# darks += ','+name
# else:
# darks = name
self.log.debug('Creating master dark frame...')
#if there is just one, use it as darkfile or else combine all to make a master dark
self.dark = ccdproc.CCDData.read(namelist, unit='adu', relax=True)
#bias correct, if necessary
# if(not dark_is_bias_corrected):
# #Subtracting master bias frame from master dark frame
# self.dark = ccdproc.subtract_bias(self.dark, self.bias, add_keyword=False)
# else:
# self.log.debug('Master dark frame is *already* bias corrected (%s).' % dark_bias)
# Finish up
self.darkloaded = True
self.darkname = namelist
self.log.debug('LoadDark: done')
def loadflat(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master flat frame by ccdproc.
Also biascorrects and dark corrects flat files if not already done.
"""
#create master flat frame
flat_is_bias_corrected = False
flat_bias = None
flat_is_dark_corrected = False
flat_dark = None
flat_ave_exptime = 0
namelist = self.loadauxname('flat', multi=False)
if (len(namelist) == 0):
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file loaded')
count = 0
datalist = []
flat_corrected = None
# This has been commented out as it is now in StepMasterFlat
#check a few things in these flat component frames
# for name in namelist:
# header = fits.getheader(name)
#is this flat bias corrected?
# if(header.get('BIAS') != None):
# flat_is_bias_corrected = True
# flat_bias = header.get('BIAS')
# elif(header.get('BIASCORR') != None):
# flat_is_bias_corrected = True
# flat_bias = header.get('BIASCORR')
# #is this flat dark corrected?
# if(header.get('DARK') != None):
# flat_is_dark_corrected = True
# flat_dark = header.get('DARK')
# elif(header.get('DARKCORR') != None):
# flat_is_dark_corrected = True
# flat_dark = header.get('DARKCORR')
# flat_corrected = "%s"%(name.rsplit('.',1)[0])+".corrected"
# shutil.copy(name, flat_corrected)
# self.log.debug('Copying %s to %s' % (name, flat_corrected))
# self.flat = ccdproc.CCDData.read(flat_corrected, unit='adu', relax=True)
# #bias correct, if necessary
# if(not flat_is_bias_corrected):
# self.log.debug('Subtracting master bias frame from flat frame...')
# self.flat = ccdproc.subtract_bias(self.flat, self.bias, add_keyword=False)
# else:
# self.log.debug('Flat frame (%s) is *already* bias corrected.'%flat_bias)
# #dark correct, if necessary
# if(not flat_is_dark_corrected):
# self.log.debug('Subtracting master dark frame from flat frame...')
# self.flat = ccdproc.subtract_dark(self.flat, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second, add_keyword=False)
# else:
# self.log.debug('Flat frame (%s) is *already* dark corrected.'%flat_dark)
# #create CCD Data object list with corrected flat files
# datalist.append(self.flat)
# #calc average exposure time for potential dark correction
# if(header.get('EXPTIME') != None):
# try:
# exptime = float(header.get('EXPTIME'))
# flat_ave_exptime += exptime
# except ValueError:
# self.log.error('Exposure time (EXPTIME) is not a float (%s).'%(header.get('EXPTIME')))
# count += 1
# #calc average exposure time
# if(count > 0):
# flat_ave_exptime = flat_ave_exptime/count
# self.flat.header['EXPTIME'] = flat_ave_exptime
# self.log.info("Average exposure time for flats is %f"%flat_ave_exptime)
self.log.debug('Creating master flat frame...')
#if there is just one, use it as flatfile or else combine all to make a master flat
self.flat = ccdproc.CCDData.read(namelist, unit='adu', relax=True)
# Finish up
self.flatloaded = True
self.flatname = namelist
self.log.debug('LoadFlat: done')
def reset(self):
""" Resets the step to the same condition as it was when it was
created. Internal variables are reset, any stored data is
erased.
"""
self.biasloaded = False
self.bias = None
self.darkloaded = False
self.dark = None
self.flatloaded = False
self.flat = None
self.log.debug('Reset: done')
class StepAstrometry(StepParent):
""" HAWC Pipeline Step Parent Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '0.2' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'astrometry'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'WCS'
# Set Logger for this pipe step
self.log = logging.getLogger('pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append([
'astrocmd', 'cp %s %s',
'Command to call astrometry, should contain 2' +
'string placeholders for intput and output ' + 'filepathname'
])
self.paramlist.append(
['verbose', False, 'log full astrometry output at DEBUG level'])
self.paramlist.append([
'delete_temp', False,
'Flag to delete temporary files generated by astrometry'
])
self.paramlist.append(
['downsample', [2], 'List of downsample factors to try'])
self.paramlist.append([
'paramoptions', ['--guess-scale'],
'Parameter groups to run if the command fails'
])
self.paramlist.append(
['timeout', 300, 'Timeout for running astrometry (seconds)'])
self.paramlist.append(
['ra', '', 'Option to manually set image center RA'])
self.paramlist.append(
['dec', '', 'Option to manually set image center DEC'])
self.paramlist.append([
'searchradius', 5,
'Only search in indexes within "searchradius" (degrees) of the field center given by --ra and --dec (degrees)'
])
# confirm end of setup
self.log.debug('Setup: done')
def run(self):
""" Runs the data reduction algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
### Preparation
# construct a temp file name that astrometry will output
fp = tempfile.NamedTemporaryFile(suffix=".fits", dir=os.getcwd())
# split off path name, because a path that is too long causes remap to
# crash sometimes
outname = os.path.split(fp.name)[1]
fp.close()
# Add input file path to ouput file and make new name
outpath = os.path.split(self.datain.filename)[0]
outnewname = os.path.join(outpath, outname.replace('.fits', '.new'))
outwcsname = os.path.join(outpath, outname.replace('.fits', '.wcs'))
# Make sure input data exists as file
if not os.path.exists(self.datain.filename):
self.datain.save()
# Make command string
rawcommand = self.getarg('astrocmd') % (self.datain.filename, outname)
# get estimated RA and DEC center values from the config file or input FITS header
raopt = self.getarg('ra')
if raopt != '':
ra = Angle(raopt, unit=u.hour).degree
else:
try:
ra = Angle(self.datain.getheadval('RA'), unit=u.hour).degree
except:
ra = ''
decopt = self.getarg('dec')
if decopt != '':
dec = Angle(decopt, unit=u.deg).degree
else:
try:
dec = Angle(self.datain.getheadval('DEC'), unit=u.deg).degree
except:
dec = ''
if (ra != '') and (dec != ''):
# update command parameters to use these values
rawcommand = rawcommand + ' --ra %f --dec %f --radius %f' % (
ra, dec, self.getarg('searchradius'))
else:
self.log.debug(
'FITS header missing RA/DEC -> searching entire sky')
### Run Astrometry:
# This loop tries the downsample and param options until the fit is successful
# need either --scale-low 0.5 --scale-high 2.0 --sort-column FLUX
# or --guess-scale
downsamples = self.getarg('downsample')
paramoptions = self.getarg('paramoptions')
for option in range(len(downsamples) * len(paramoptions)):
#for downsample in self.getarg('downsample'):
downsample = downsamples[option % len(downsamples)]
paramoption = paramoptions[option // len(downsamples)]
# Add options to command
command = rawcommand + ' --downsample %d' % downsample + ' ' + paramoption
optionstring = "Downsample=%s Paramopts=%s" % (downsample,
paramoption[:10])
# Run the process - see note at the top of the file if using cron
process = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
self.log.debug('running command = %s' % command)
# Wait for the process to be finished or timeout to be reached
timeout = time.time() + self.getarg('timeout')
while time.time() < timeout and process.poll() == None:
time.sleep(1)
poll = process.poll()
if poll == None:
process.kill()
time.sleep(1)
poll = process.poll()
self.log.debug('command returns %d' % poll)
if poll == 0 and os.path.exists(outnewname):
self.log.debug('output file valid -> astrometry successful')
break
else:
self.log.debug('output file missing -> astrometry failed')
# Print the output from astrometry (cut if necessary)
if self.getarg('verbose'):
output = process.stdout.read().decode()
if len(output) > 1000:
outlines = output.split('\n')
output = outlines[:10] + ['...', '...'] + outlines[-7:]
output = '\n'.join(output)
self.log.debug(output)
### Post processing
# Read output file
self.dataout = DataFits(config=self.config)
self.log.debug('Opening astrometry.net output file %s' % outnewname)
try:
self.dataout.load(outnewname)
self.dataout.filename = self.datain.filename
except Exception as error:
self.log.error("Unable to open astrometry. output file = %s" %
outname)
raise error
self.log.debug('Successful parameter options = %s' % optionstring)
# Add history message
histmsg = 'Astrometry.Net: At downsample = %d, search took %d seconds' % (
downsample, time.time() - timeout + 300)
self.dataout.setheadval('HISTORY', histmsg)
# Add RA from astrometry
w = wcs.WCS(self.dataout.header)
n1 = float(self.dataout.header['NAXIS1'] / 2)
n2 = float(self.dataout.header['NAXIS2'] / 2)
ra, dec = w.all_pix2world(n1, n2, 1)
self.dataout.header['CRPIX1'] = n1
self.dataout.header['CRPIX2'] = n2
self.dataout.header['CRVAL1'] = float(ra)
self.dataout.header['CRVAL2'] = float(dec)
self.dataout.header['RA'] = Angle(ra, u.deg).to_string(unit=u.hour,
sep=':')
self.dataout.header['Dec'] = Angle(dec, u.deg).to_string(sep=':')
self.dataout.setheadval('HISTORY',
'Astrometry: Paramopts = ' + optionstring)
# Delete temporary files
if self.getarg('delete_temp'):
os.remove(outnewname)
os.remove(outwcsname)
self.log.debug('Run: Done')
class StepCoadd(StepMIParent):
""" Stone Edge Pipeline Step Master Bias Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '1.2' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name='coadd'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'coadd'
# Set Logger for this pipe step
self.log = logging.getLogger('pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append(['kernel','square',
'Specifies the kernel used to determine spreading of input pixels onto output pixels \
- options are square, point, gaussian, smoothing, tophat'])
self.paramlist.append(['pixfrac', 1.,
'The fraction of an output pixel(s) that an input pixel\'s flux is confined to'])
self.paramlist.append(['resolution', 1.,
'Pixel scale divisor for output image (higher gives more resolution, lower gives less)'])
self.paramlist.append(['pad', 0,
'Extra padding outside maximum extent of inputs'])
self.paramlist.append(['fillval', np.nan,
'Value for filling in the area(s) in the output where there is no input data'])
self.paramlist.append(['drizzleweights','exptime',
'How each input image should be weighted when added to the output \
- options are exptime, expsq and uniform'])
self.paramlist.append(['outangle',0.,
'Output angle of drizzled image (currently not functional)'])
def run(self):
""" Runs the mosaicing algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
#calculate platescale of first input image
try:
det = np.linalg.det(wcs.WCS(self.datain[0].header).wcs.cd)
pscale = np.sqrt(np.abs(det))*3600.
except:
try:
det = np.linalg.det(wcs.WCS(self.datain[0].header).wcs.pc)
pscale = np.sqrt(np.abs(det))*3600.
except:
pscale = self.datain[0].header['PIXSCAL']
#filtering out images which are too far away from the others
#passing images added to a list of (image, WCS) tuples
'''
image_centers = []
for f in self.datain:
image_centers.append((f.header['CRVAL1'], f.header['CRVAL2']))
filtered_datain = []
dist_list = [[[0]*(len(image_centers)-1)]*len(image_centers)]
for i in range(len(image_centers)):
for j in range(len(image_centers)-1):
dist_list[i][j+1] = np.sqrt((image_)**2+()**2)
'''
#calculations necessary for updating wcs information
px = []
py = []
#in order to avoid NaN interactions, creating weight map
weights=[]
for f in self.datain:
weights.append((np.where(np.isnan(f.image) == True, 0, 1)))
for f in self.datain:
px.extend(wcs.WCS(f.header).calc_footprint()[:,0])
py.extend(wcs.WCS(f.header).calc_footprint()[:,1])
x0 = (max(px)+min(px))/2.
y0 = (max(py)+min(py))/2.
sx = (max(px)-min(px))*np.cos(y0/180*np.pi) # arcsec
sy = (max(py)-min(py)) # arcsec
size = (sx*3600+self.getarg('pad')*2, sy*3600+self.getarg('pad')*2)
xpix = size[0]//pscale
ypix = size[1]//pscale
cdelt = [pscale/3600.]*2
#create self.dataout and give it a copy of an input's header
self.dataout = DataFits(config = self.config)
self.dataout.header = self.datain[0].header.copy()
#update header wcs information
self.log.info('Creating new WCS header')
self.dataout.header['CRPIX1'] = xpix/2
self.dataout.header['CRPIX2'] = ypix/2
self.dataout.header['CRVAL1'] = x0
self.dataout.header['CRVAL2'] = y0
self.dataout.header['CD1_1'] = -cdelt[0]
self.dataout.header['CD1_2'] = self.dataout.header['CD2_1'] = 0.
self.dataout.header['CD2_2'] = cdelt[1]
self.dataout.header['NAXIS1'] = int(xpix)
self.dataout.header['NAXIS2'] = int(ypix)
self.dataout.header['CTYPE1'] = 'RA---TAN-SIP'
self.dataout.header['CTYPE2'] = 'DEC--TAN-SIP'
self.dataout.header['RADESYS'] = 'ICRS'
self.dataout.header['EQUINOX'] = 2000
self.dataout.header['LATPOLE'] = self.datain[0].header['CRVAL2']
self.dataout.header['LONPOLE'] = 180
self.dataout.header['PIXASEC'] = pscale
theta_rad = np.deg2rad(self.getarg('outangle'))
rot_matrix = np.array([[np.cos(theta_rad), -np.sin(theta_rad)],
[np.sin(theta_rad), np.cos(theta_rad)]])
rot_cd = np.dot(rot_matrix, np.array([[self.dataout.header['CD1_1'], 0.],[0., self.dataout.header['CD2_2']]]))
for i in [0,1]:
for j in [0,1]:
self.dataout.header['CD{0:d}_{1:d}'.format(i+1, j+1)] = rot_cd[i,j]
#check drizzle arguments
if self.getarg('kernel') == 'smoothing':
kernel = 'lanczos3'
elif self.getarg('kernel') in ['square', 'point', 'gaussian', 'tophat']:
kernel = self.getarg('kernel')
else:
self.log.error('Kernel name not recognized, using default')
kernel = 'square'
if self.getarg('drizzleweights') == 'uniform':
driz_wt = ''
elif self.getarg('drizzleweights') in ['exptime', 'expsq']:
driz_wt = self.getarg('drizzleweights')
else:
self.log.error('Drizzle weighting not recognized, using default')
driz_wt = ''
#create drizzle object and add input images
fullwcs = wcs.WCS(self.dataout.header)
self.log.info('Starting drizzle')
driz = drz.Drizzle(outwcs = fullwcs, pixfrac=self.getarg('pixfrac'), \
kernel=kernel, fillval='10000', wt_scl=driz_wt)
for i,f in enumerate(self.datain):
self.log.info('Adding %s to drizzle stack' % f.filename)
driz.add_image(f.imgdata[0], wcs.WCS(f.header), inwht=weights[i])
try:
fillval=float(self.getarg('fillval'))
except:
fillval=np.nan
self.log.error('Fillvalue not recognized or missing, using default')
#creates output fits file from drizzle output
self.dataout.imageset(np.where(driz.outsci == 10000, fillval, driz.outsci))
self.dataout.imageset(driz.outwht,'OutWeight', self.dataout.header)
self.dataout.filename = self.datain[0].filename
#add history
self.dataout.setheadval('HISTORY','Coadd: %d files combined with %s kernel, pixfrac %f at %f times resolution' \
% (len(self.datain), kernel, self.getarg('pixfrac'), self.getarg('resolution')))