class StepMaskImage(StepParent):
""" HAWC Pipeline Step Parent Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '0.2' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'maskimage'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'MSK'
# Set Logger for this pipe step
self.log = logging.getLogger('pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
# confirm end of setup
self.log.debug('Setup: done')
def mask(self):
'''
Masks the input image file
'''
#Mask the image
image_data = self.datain.image.astype('int32').byteswap(
inplace=True).newbyteorder()
mask = vp.unit(data=image_data, header=self.datain.header)
mask.extract_bkg()
mask.subtract_bkg()
mask.set_primary('bkg_sub')
mask.extract_sources()
mask.build_sources_table()
mask.filter_sources(edgefrac=0.4)
mask.mask_sources()
masked_image = fits.PrimaryHDU(mask.primary, header=self.datain.header)
mask_fp = self.datain.filename.replace('.fits', '_MSK.fits')
masked_image.writeto(mask_fp)
return mask_fp
def run(self):
self.dataout = DataFits(config=self.config)
self.dataout.load(self.mask())
def __init__(self):
""" Constructor: Initialize data objects and variables
"""
# call superclass constructor (calls setup)
super(StepBiasDarkFlat, self).__init__()
# bias values
self.biasloaded = False # indicates if bias has been loaded
self.bias = None # CCD data object containing arrays with bias values
self.biasdata = DataFits() # Pipedata object containing the bias file
# bias file info and header keywords to fit
self.biasname = '' # name of selected bias file
self.biasfitkeys = [] # FITS keywords that are present in bias
self.biaskeyvalues = [] # values of FITS keywords (from data file)
# dark values
self.darkloaded = False # indicates if dark has been loaded
self.dark = None # CCD data object containing arrays with dark values
self.darkdata = DataFits() # Pipedata object containing the dark file
# dark file info and header keywords to fit
self.darkname = '' # name of selected dark file
self.darkfitkeys = [] # FITS keywords that have to fit for dark
self.darkkeyvalues = [] # values of FITS keywords (from data file)
# flat values
self.flatloaded = False # indicates if flat has been loaded
self.flat = None # CCD data object containing arrays with flat values
self.flatdata = DataFits() # Pipedata object containing the flat file
# flat file info and header keywords to fit
self.flatname = '' # name of selected flat file
self.flatfitkeys = [] # FITS keywords that have to fit for flat
self.flatkeyvalues = [] # values of flat keywords (from data file)
# set configuration
self.log.debug('Init: done')
class StepSEP(StepParent):
""" HAWC Pipeline Step Parent Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '0.2' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name='sep'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'SEP'
# Set Logger for this pipe step
self.log = logging.getLogger('pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
# confirm end of setup
self.log.debug('Setup: done')
def source_extract(self):
bkg = sep.Background(self.datain.image.astype('int32'))
primary = self.datain.image.astype('int32') - bkg
objects = sep.extract(primary, 1.5, err=bkg.globalrms)
df = pd.DataFrame()
df['x'] = objects['x']; df['y'] = objects['y']; df['a'] = objects['a']; df['b'] = objects['b']; df['theta'] = objects['theta']; df['npix'] = objects['npix']; df['FLUX'] = objects['cflux']
table_image = Table(df.values)
print(repr(df))
table_fp = self.datain.filename.replace('.fits', '_TABLE.fits')
table_image.write(table_fp, format='fits')
return table_fp
def run(self):
self.dataout = DataFits(config=self.config)
self.dataout.load(self.source_extract())
self.dataout.header['RA'] = self.datain.header['RA']
self.dataout.header['Dec'] = self.datain.header['Dec']
self.dataout.save()
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
# Find master bias to subtract from master dark
biaslist = self.loadauxname('bias', multi=False)
if (len(biaslist) == 0):
self.log.error('No bias calibration frames found.')
self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True)
# Create empy list for filenames of loaded frames
filelist = []
for fin in self.datain:
self.log.debug("Input filename = %s" % fin.filename)
filelist.append(fin.filename)
# Make a dummy dataout
self.dataout = DataFits(config=self.config)
if len(self.datain) == 0:
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file(s) loaded')
self.log.debug('Creating master flat frame...')
# Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias
if (len(filelist) == 1):
self.dark = ccdproc.CCDData.read(filelist[0],
unit='adu',
relax=True)
self.dark = ccdproc.subtract_bias(self.dark,
self.bias,
add_keyword=False)
else:
darklist = []
for i in filelist:
dark = ccdproc.CCDData.read(i, unit='adu', relax=True)
darksubbias = ccdproc.subtract_bias(dark,
self.bias,
add_keyword=False)
darklist.append(darksubbias)
self.dark = ccdproc.combine(darklist,
method=self.getarg('combinemethod'),
unit='adu',
add_keyword=True)
# set output header, put image into output
self.dataout.header = self.datain[0].header
self.dataout.imageset(self.dark)
# rename output filename
outputfolder = self.getarg('outputfolder')
if outputfolder != '':
outputfolder = os.path.expandvars(outputfolder)
self.dataout.filename = os.path.join(outputfolder,
os.path.split(filelist[0])[1])
else:
self.dataout.filename = filelist[0]
# Add history
self.dataout.setheadval('HISTORY',
'MasterDark: %d files used' % len(filelist))
def run(self):
""" Runs the data reduction algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
self.dataout = DataFits(config=self.config)
self.dataout = self.datain.copy()
self.astrometrymaster()
# Update RA/Dec from astrometry
self.dataout.header.update(self.wcs_out)
try:
w = wcs.WCS(self.dataout.header)
n1 = float( self.dataout.header['NAXIS1']/2 )
n2 = float( self.dataout.header['NAXIS2']/2 )
ra, dec = w.all_pix2world(n1, n2, 1)
# No update because update may affect accuracy of WCS solution
# self.dataout.header['CRPIX1']=n1
# self.dataout.header['CRPIX2']=n2
# self.dataout.header['CRVAL1']=float(ra)
# self.dataout.header['CRVAL2']=float(dec)
self.dataout.header['RA'] = Angle(ra, u.deg).to_string(unit=u.hour, sep=':')
self.dataout.header['Dec'] = Angle(dec, u.deg).to_string(sep=':')
except:
self.log.error('Run: Could not update RA/Dec from Astrometry')
else:
self.log.debug('Run: Updated RA/Dec from Astrometry')
self.log.debug('Run: Done')
def __init__(self):
""" Constructor: Initialize data objects and variables
"""
# call superclass constructor (calls setup)
super(StepFlat, self).__init__()
# list of data and flats
self.datalist = [] # used in run() for every new input data file
# flat values
self.flatloaded = 0 # indicates if flat has been loaded
self.flats = [] # list containing arrays with flat values
self.flatdata = DataFits() # Pipedata object containing the flat file
# flat file info and header keywords to fit
self.flatfile = '' # name of selected flat file
self.fitkeys = [] # FITS keywords that have to fit
self.keyvalues = [
] # values of the keywords (from the first data file)
# set configuration
self.log.debug('Init: done')
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
filelist = []
for fin in self.datain:
self.log.debug("Input filename = %s" % fin.filename)
filelist.append(fin.filename)
# Make a dummy dataout
self.dataout = DataFits(config=self.config)
if len(self.datain) == 0:
self.log.error('Bias calibration frame not found.')
raise RuntimeError('No bias file(s) loaded')
# self.log.debug('Creating master bias frame...')
# if there is just one, use it as biasfile or else combine all to make a master bias
if (len(filelist) == 1):
self.bias = ccdproc.CCDData.read(filelist[0],
unit='adu',
relax=True)
else:
self.bias = ccdproc.combine(filelist,
method=self.getarg('combinemethod'),
unit='adu',
add_keyword=True)
# set output header, put image into output
self.dataout.header = self.datain[0].header
self.dataout.imageset(self.bias)
# rename output filename
outputfolder = self.getarg('outputfolder')
if outputfolder != '':
outputfolder = os.path.expandvars(outputfolder)
self.dataout.filename = os.path.join(outputfolder,
os.path.split(filelist[0])[1])
else:
self.dataout.filename = filelist[0]
# Add history
self.dataout.setheadval('HISTORY',
'MasterBias: %d files used' % len(filelist))
def test(self):
""" Test Pipe Step Flat Object: Runs basic tests
"""
# initial log message
self.log.info('Testing pipe step flat')
# get testin and a configuration
if self.config != None and len(
self.config) > 2: # i.e. if real config is loaded
testin = DataFits(config=self.config)
else:
testin = DataFits(config=self.testconf)
# load sample data
datain = DataFits(config=testin.config)
#infile = 'mode_chop/120207_000_00HA012.chop.dmd.fits'
infile = 'mode_chop/120306_000_00HA006.chop.dmd.fits'
#infile = 'mode_chop/120402_000_00HA035.chop.dmd.fits'
#infile = 'sharp/sharc2-048485.dmdsqr.fits'
testfile = os.path.join(datain.config['testing']['testpath'], infile)
#testfile = '/Users/berthoud/testfit.fits'
datain.load(testfile)
if False:
# change data (make complex number array with
# Re=0,1,2,3,4,5,6 . . . in time Im=0)
dataval = numpy.ones(datain.image.shape)
dataval[..., 1] = 0.0
inclist = numpy.arange(dataval.shape[0])
incshape = [1 + i - i for i in dataval.shape[0:-1]]
incshape[0] = dataval.shape[0]
inclist.shape = incshape
dataval[..., 0] = dataval[..., 0] * inclist
#datain.image=dataval
# run first flat
dataout = self(datain)
#print dataout.image[100,...] # print 100th image
#print dataout.image[range(0,dataval.shape[0],1000),0,0] # 1 val per img
dataout.save()
# final log message
self.log.info('Testing pipe step flat - Done')
def run(self):
""" Runs the calibrating algorithm. The calibrated data is
returned in self.dataout
"""
### Preparation
# Load bias files if necessary
if not self.biasloaded or self.getarg('reload'):
self.loadbias()
# Else: check data for correct instrument configuration - currently not in use(need improvement)
else:
for keyind in range(len(self.biasfitkeys)):
if self.biaskeyvalues[keyind] != self.datain.getheadval(
self.biasfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.biasfitkeys[keyind])
# Load dark files if necessary
if not self.darkloaded or self.getarg('reload'):
self.loaddark()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.darkfitkeys)):
if self.darkkeyvalues[keyind] != self.datain.getheadval(
self.darkfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.darkfitkeys[keyind])
# Load flat files if necessary
if not self.flatloaded or self.getarg('reload'):
self.loadflat()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.flatfitkeys)):
if self.flatkeyvalues[keyind] != self.datain.getheadval(
self.flatfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.flatfitkeys[keyind])
# in the config file, set the 'intermediate' variable to either true or false to enable
# saving of intermediate steps
saveIntermediateSteps = self.config['biasdarkflat']['intermediate']
self.dataout = DataFits(config=self.datain.config)
#convert self.datain to CCD Data object
image = CCDData(self.datain.image, unit='adu')
image.header = self.datain.header
# subtract bias from image
image = self.subtract_bias(image, self.bias)
if (saveIntermediateSteps == "true"):
self.dataout.imageset(image.data, imagename="BIAS")
# self.dataout.setheadval('DATATYPE','IMAGE', dataname="BIAS")
self.dataout.setheadval('HISTORY',
'BIAS: %s' % self.biasname,
dataname="BIAS")
# subtract dark from image
image = self.subtract_dark(image,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second)
if (saveIntermediateSteps == "true"):
self.dataout.imageset(image.data, imagename="DARK")
# self.dataout.setheadval('DATATYPE','IMAGE', dataname="DARK")
self.dataout.setheadval('HISTORY',
'BIAS: %s' % self.biasname,
dataname="DARK")
self.dataout.setheadval('HISTORY',
'DARK: %s' % self.darkname,
dataname="DARK")
# apply flat correction to image
image = self.flat_correct(image, self.flat)
# if separating bias,dark,flat steps , save the flat-corrected portion into its own hdu
if (saveIntermediateSteps == "true"):
self.dataout.imageset(image.data, imagename="FLAT")
# self.dataout.setheadval('DATATYPE','IMAGE', dataname="FLAT")
self.dataout.setheadval('HISTORY',
'BIAS: %s' % self.biasname,
dataname="FLAT")
self.dataout.setheadval('HISTORY',
'DARK: %s' % self.darkname,
dataname="FLAT")
self.dataout.setheadval('HISTORY',
'FLAT: %s' % self.flatname,
dataname="FLAT")
else:
# copy calibrated image into self.dataout
self.dataout.image = image.data
self.dataout.header = self.datain.header
### Finish - cleanup
# Update DATATYPE
self.dataout.setheadval('DATATYPE', 'IMAGE')
# Add bias, dark files to History
self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname)
self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname)
self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname)
self.dataout.filename = self.datain.filename
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
''' Select 3 input dataset to use, store in datause '''
#Store number of inputs
num_inputs = len(self.datain)
# Create variable to hold input files
# Copy input to output header and filename
datause = []
self.log.debug('Number of input files = %d' % num_inputs)
# Ensure datause has 3 elements irrespective of number of input files
if num_inputs == 0: # Raise exception for no input
raise ValueError('No input')
elif num_inputs == 1:
datause = [self.datain[0], self.datain[0], self.datain[0]]
elif num_inputs == 2:
datause = [self.datain[0], self.datain[1], self.datain[1]]
else: # If inputs exceed 2 in number
# Here we know there are at least 3 files
ilist = [] # Make empty lists for each filter
rlist = []
glist = []
other = []
for element in self.datain: # Loop through the input files and add to the lists
fname = element.filename.lower()
if 'i-band' in fname or 'iband' in fname or 'iprime' in fname:
ilist.append(element)
elif 'r-band' in fname or 'rband' in fname or 'rprime' in fname:
rlist.append(element)
elif 'g-band' in fname or 'gband' in fname or 'gprime' in fname:
glist.append(element)
else:
other.append(element)
continue
self.log.debug(
'len(ilist) = %d, len(rlist) = %d, len(glist) = %d' %
(len(ilist), len(rlist), len(glist)))
# If there is at least one i-, r-, and g-band filter found in self.datain (best case)
if len(ilist) >= 1 and len(rlist) >= 1 and len(glist) >= 1:
# The first image from each filter list will be reduced in the correct order.
datause = [ilist[0], rlist[0], glist[0]]
elif len(ilist) == 0 and len(rlist) >= 1 and len(glist) >= 1:
# Cases where there is no ilist
if len(rlist) > len(glist):
datause = [rlist[0], rlist[1], glist[0]]
else:
datause = [rlist[0], glist[0], glist[1]]
elif len(glist) == 0 and len(rlist) >= 1 and len(ilist) >= 1:
# Cases where there is no glist
if len(rlist) > len(ilist):
datause = [rlist[0], rlist[1], ilist[0]]
else:
datause = [rlist[0], ilist[0], ilist[1]]
elif len(ilist) == 0 and len(rlist) >= 1 and len(glist) >= 1:
# Cases where there is no rlist
if len(ilist) > len(glist):
datause = [ilist[0], ilist[1], glist[0]]
else:
datause = [ilist[0], glist[0], glist[1]]
elif len(rlist) == 0 and len(glist) == 0:
# Case where there is only ilist
datause = [ilist[0], ilist[1], ilist[2]]
elif len(rlist) == 0 and len(ilist) == 0:
# Case where there is only glist
datause = [glist[0], glist[1], glist[2]]
elif len(ilist) == 0 and len(glist) == 0:
# Case where there is only rlist
datause = [rlist[0], rlist[1], rlist[2]]
self.log.debug(
'Files used: R = %s G = %s B = %s' %
(datause[0].filename, datause[1].filename, datause[2].filename))
self.dataout = DataFits(config=self.config)
self.dataout.header = datause[0].header
self.dataout.filename = datause[0].filename
img = datause[0].image
img1 = datause[1].image
img2 = datause[2].image
''' Finding Min/Max scaling values '''
# Create a Data Cube with floats
datacube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype=float)
# Enter the image data into the cube so an absolute max can be found
datacube[:, :, 0] = img
datacube[:, :, 1] = img1
datacube[:, :, 2] = img2
# Find how many data points are in the data cube
datalength = img.shape[0] * img.shape[1] * 3
# Create a 1-dimensional array with all the data, then sort it
datacube.shape = (datalength, )
datacube.sort()
# Now use arrays for each filter to find separate min values
rarray = img.copy()
garray = img1.copy()
barray = img2.copy()
# Shape and sort the arrays
arrlength = img.shape[0] * img.shape[1]
rarray.shape = (arrlength, )
rarray.sort()
garray.shape = (arrlength, )
garray.sort()
barray.shape = (arrlength, )
barray.sort()
# Find the min/max percentile values in the data for scaling
# Values are determined by parameters in the pipe configuration file
minpercent = int(arrlength * self.getarg('minpercent'))
maxpercent = int(datalength * self.getarg('maxpercent'))
# Find the final data values to use for scaling from the image data
rminsv = rarray[minpercent] #sv stands for "scalevalue"
gminsv = garray[minpercent]
bminsv = barray[minpercent]
maxsv = datacube[maxpercent]
self.log.info(' Scale min r/g/b: %f/%f/%f' % (rminsv, gminsv, bminsv))
self.log.info(' Scale max: %f' % maxsv)
# The same min/max values will be used to scale all filters
''' Finished Finding scaling values '''
''' Combining Function '''
# Make new cube with the proper data type for color images (uint8)
# Use square root (sqrt) scaling for each filter
# log or asinh scaling is also available
#astropy.vidualizations.SqrtStretch()
imgcube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype='uint8')
minsv = [rminsv, gminsv, bminsv]
for i in range(3):
# Make normalization function
norm = simple_norm(datause[i].image,
'sqrt',
min_cut=minsv[i],
max_cut=maxsv)
# Apply it
imgcube[:, :, i] = norm(datause[i].image) * 255.
self.dataout.image = imgcube
# Create variable containing all the scaled image data
imgcolor = Image.fromarray(self.dataout.image, mode='RGB')
# Save colored image as a .tif file (without the labels)
imgcolortif = imgcube.copy()
imgcolortif.astype('uint16')
### tiff.imsave('%s.tif' % self.dataout.filenamebase, imgcolortif)
''' End of combining function '''
''' Add a Label to the Image '''
draw = ImageDraw.Draw(imgcolor)
# Use a variable to make the positions and size of text relative
imgwidth = img.shape[1]
imgheight = img.shape[0]
# Open Sans-Serif Font with a size relative to the picture size
try:
# This should work on Linux
font = ImageFont.truetype(
'/usr/share/fonts/liberation/LiberationSans-Regular.ttf',
imgheight // 41)
except:
try:
# This should work on Mac
font = ImageFont.truetype('/Library/Fonts/Arial Unicode.ttf',
imgheight // 41)
except:
try:
# This should work on Windows
font = ImageFont.truetype('C:\\Windows\\Fonts\\arial.ttf',
imgheight // 41)
except:
# This should work in Colab
font = ImageFont.truetype(
'/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf',
imgheight // 41)
# If this still doesn't work - then add more code to make it run on YOUR system
# Use the beginning of the FITS filename as the object name
filename = os.path.split(self.dataout.filename)[-1]
try:
objectname = filename.split('_')[0]
objectname = objectname[0].upper() + objectname[1:]
except Exception:
objectname = 'Unknown.'
objectname = 'Object: %s' % objectname
# Read labels at their respective position (kept relative to image size)
# Left corner: object, observer, observatory
# Right corner: Filters used for red, green, and blue colors
draw.text((imgwidth / 100, imgheight / 1.114),
objectname, (255, 255, 255),
font=font)
# Read FITS keywords for the observer, observatory, and filters
if 'OBSERVER' in self.dataout.header:
observer = 'Observer: %s' % self.dataout.getheadval('OBSERVER')
draw.text((imgwidth / 100, imgheight / 1.073),
observer, (255, 255, 255),
font=font)
if 'OBSERVAT' in self.dataout.header:
observatory = 'Observatory: %s' % self.dataout.getheadval(
'OBSERVAT')
draw.text((imgwidth / 100, imgheight / 1.035),
observatory, (255, 255, 255),
font=font)
if 'FILTER' in datause[0].header:
red = 'R: %s' % datause[0].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.114),
red, (255, 255, 255),
font=font)
if 'FILTER' in datause[1].header:
green = 'G: %s' % datause[1].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.073),
green, (255, 255, 255),
font=font)
if 'FILTER' in datause[2].header:
blue = 'B: %s' % datause[2].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.035),
blue, (255, 255, 255),
font=font)
# Make image name
imgname = self.dataout.filenamebegin
if imgname[-1] in '_-,.': imgname = imgname[:-1]
imgname += '.jpg'
# Save the completed image
imgcolor.save(imgname)
self.log.info('Saving file %sjpg' % self.dataout.filenamebegin)
''' End of Label Code '''
# Set complete flag
self.dataout.setheadval('COMPLETE', 1,
'Data Reduction Pipe: Complete Data Flag')
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in jpeg_dataout.
"""
''' Select 3 input dataset to use, store in datause '''
#Store number of inputs
num_inputs = len(self.datain)
# Create variable to hold input files
# Copy input to output header and filename
datause = [None, None, None]
self.log.debug('Number of input files = %d' % num_inputs)
if num_inputs == 0: # Raise exception for no input
raise ValueError('No input')
elif num_inputs == 1:
datause = [self.datain[0], self.datain[0], self.datain[0]]
elif num_inputs == 2:
datause = [self.datain[0], self.datain[0], self.datain[1]]
else:
filterorder_list = self.getarg('filterorder').split('|')
filterprefs_list = self.getarg('filterprefs').split('|')
datain_filter_list = [
element.getheadval('filter') for element in self.datain
]
used_filter_flags = [False] * len(self.datain)
if len(filterprefs_list) != 3:
self.log.error(
'Invalid number of preferred filters provided (should be 3): '
+ self.getarg('filterprefs'))
else:
# Locate data matching the filters specified in filterprefs
for i, preferred_filter in enumerate(filterprefs_list):
for j, element in enumerate(self.datain):
if element.getheadval('filter') == preferred_filter:
datause[i] = element
used_filter_flags[j] = True
break
filterorder_walker = 0
for i, channel in enumerate(datause):
if channel == None:
for ordered_filter in filterorder_list[
filterorder_walker:]:
filterorder_walker = filterorder_walker + 1
if ordered_filter in datain_filter_list:
datain_index = datain_filter_list.index(
ordered_filter)
if not used_filter_flags[datain_index]:
datause[i] = self.datain[datain_index]
used_filter_flags[datain_index] = True
break
elif channel.getheadval('filter') in filterorder_list:
filterorder_walker = filterorder_list.index(
channel.getheadval('filter'))
for i, channel in enumerate(datause):
if channel == None:
for j, datain_filter in enumerate(datain_filter_list):
if not used_filter_flags[j]:
datause[i] = self.datain[j]
used_filter_flags[j] = True
break
self.log.debug(
'Files used: R = %s G = %s B = %s' %
(datause[0].filename, datause[1].filename, datause[2].filename))
jpeg_dataout = DataFits(config=self.config)
jpeg_dataout.header = datause[0].header
jpeg_dataout.filename = datause[0].filename
img = datause[0].image
img1 = datause[1].image
img2 = datause[2].image
''' Finding Min/Max scaling values '''
# Create a Data Cube with floats
datacube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype=float)
# Enter the image data into the cube so an absolute max can be found
datacube[:, :, 0] = img
datacube[:, :, 1] = img1
datacube[:, :, 2] = img2
# Find how many data points are in the data cube
datalength = img.shape[0] * img.shape[1] * 3
# Create a 1-dimensional array with all the data, then sort it
datacube.shape = (datalength, )
datacube.sort()
# Now use arrays for each filter to find separate min values
rarray = img.copy()
garray = img1.copy()
barray = img2.copy()
# Shape and sort the arrays
arrlength = img.shape[0] * img.shape[1]
rarray.shape = (arrlength, )
rarray.sort()
garray.shape = (arrlength, )
garray.sort()
barray.shape = (arrlength, )
barray.sort()
# Find the min/max percentile values in the data for scaling
# Values are determined by parameters in the pipe configuration file
minpercent = int(arrlength * self.getarg('minpercent'))
maxpercent = int(datalength * self.getarg('maxpercent'))
# Find the final data values to use for scaling from the image data
rminsv = rarray[minpercent] #sv stands for "scalevalue"
gminsv = garray[minpercent]
bminsv = barray[minpercent]
maxsv = datacube[maxpercent]
self.log.info(' Scale min r/g/b: %f/%f/%f' % (rminsv, gminsv, bminsv))
self.log.info(' Scale max: %f' % maxsv)
# The same min/max values will be used to scale all filters
''' Finished Finding scaling values '''
''' Combining Function '''
# Make new cube with the proper data type for color images (uint8)
# Use square root (sqrt) scaling for each filter
# log or asinh scaling is also available
#astropy.vidualizations.SqrtStretch()
imgcube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype='uint8')
minsv = [rminsv, gminsv, bminsv]
for i in range(3):
# Make normalization function
norm = simple_norm(datause[i].image,
'sqrt',
min_cut=minsv[i],
max_cut=maxsv)
# Apply it
imgcube[:, :, i] = norm(datause[i].image) * 255.
jpeg_dataout.image = imgcube
# Create variable containing all the scaled image data
imgcolor = Image.fromarray(jpeg_dataout.image, mode='RGB')
# Save colored image as a .tif file (without the labels)
imgcolortif = imgcube.copy()
imgcolortif.astype('uint16')
### tiff.imsave('%s.tif' % jpeg_dataout.filenamebase, imgcolortif)
''' End of combining function '''
''' Add a Label to the Image '''
draw = ImageDraw.Draw(imgcolor)
# Use a variable to make the positions and size of text relative
imgwidth = img.shape[1]
imgheight = img.shape[0]
# Open Sans-Serif Font with a size relative to the picture size
try:
# This should work on Linux
font = ImageFont.truetype(
'/usr/share/fonts/liberation/LiberationSans-Regular.ttf',
imgheight // 41)
except:
try:
# This should work on Mac
font = ImageFont.truetype('/Library/Fonts/Arial Unicode.ttf',
imgheight // 41)
except:
try:
# This should work on Windows
font = ImageFont.truetype('C:\\Windows\\Fonts\\arial.ttf',
imgheight // 41)
except:
# This should work in Colab
font = ImageFont.truetype(
'/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf',
imgheight // 41)
# If this still doesn't work - then add more code to make it run on YOUR system
# Use the beginning of the FITS filename as the object name
filename = os.path.split(jpeg_dataout.filename)[-1]
try:
objectname = filename.split('_')[0]
objectname = objectname[0].upper() + objectname[1:]
except Exception:
objectname = 'Unknown.'
objectname = 'Object: %s' % objectname
# Read labels at their respective position (kept relative to image size)
# Left corner: object, observer, observatory
# Right corner: Filters used for red, green, and blue colors
draw.text((imgwidth / 100, imgheight / 1.114),
objectname, (255, 255, 255),
font=font)
# Read FITS keywords for the observer, observatory, and filters
if 'OBSERVER' in jpeg_dataout.header:
observer = 'Observer: %s' % jpeg_dataout.getheadval('OBSERVER')
draw.text((imgwidth / 100, imgheight / 1.073),
observer, (255, 255, 255),
font=font)
if 'OBSERVAT' in jpeg_dataout.header:
observatory = 'Observatory: %s' % jpeg_dataout.getheadval(
'OBSERVAT')
draw.text((imgwidth / 100, imgheight / 1.035),
observatory, (255, 255, 255),
font=font)
if 'FILTER' in datause[0].header:
red = 'R: %s' % datause[0].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.114),
red, (255, 255, 255),
font=font)
if 'FILTER' in datause[1].header:
green = 'G: %s' % datause[1].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.073),
green, (255, 255, 255),
font=font)
if 'FILTER' in datause[2].header:
blue = 'B: %s' % datause[2].getheadval('FILTER')
draw.text((imgwidth / 1.15, imgheight / 1.035),
blue, (255, 255, 255),
font=font)
# Make image name
imgname = jpeg_dataout.filenamebegin
if imgname[-1] in '_-,.': imgname = imgname[:-1]
imgname += '.jpg'
# Save the completed image
imgcolor.save(imgname)
self.log.info('Saving file %sjpg' % jpeg_dataout.filenamebegin)
# Optional folder output setup
baseimgname = os.path.basename(imgname)
folderpaths_list = self.getarg('folderpaths').split(':')
for path in folderpaths_list:
path = time.strftime(path, time.localtime())
if not os.path.exists(path):
if self.getarg('createfolders'):
os.makedirs(path)
self.log.info('Creating directory %s' % path)
else:
self.log.info('Invalid folder path %s' % path)
try:
imgcolor.save(os.path.join(path, baseimgname))
except:
self.log.exception('Could not save image to directory %s' %
path)
''' End of Label Code '''
# Set complete flag
jpeg_dataout.setheadval('COMPLETE', 1,
'Data Reduction Pipe: Complete Data Flag')
### Make output data
self.dataout = self.datain.copy()
self.dataout.append(jpeg_dataout)
class StepBiasDarkFlat(StepLoadAux, StepParent):
""" Pipeline Step Object to calibrate Bias/Dark/Flat files
"""
stepver = '0.1' # pipe step version
def __init__(self):
""" Constructor: Initialize data objects and variables
"""
# call superclass constructor (calls setup)
super(StepBiasDarkFlat, self).__init__()
# bias values
self.biasloaded = False # indicates if bias has been loaded
self.bias = None # CCD data object containing arrays with bias values
self.biasdata = DataFits() # Pipedata object containing the bias file
# bias file info and header keywords to fit
self.biasname = '' # name of selected bias file
self.biasfitkeys = [] # FITS keywords that are present in bias
self.biaskeyvalues = [] # values of FITS keywords (from data file)
# dark values
self.darkloaded = False # indicates if dark has been loaded
self.dark = None # CCD data object containing arrays with dark values
self.darkdata = DataFits() # Pipedata object containing the dark file
# dark file info and header keywords to fit
self.darkname = '' # name of selected dark file
self.darkfitkeys = [] # FITS keywords that have to fit for dark
self.darkkeyvalues = [] # values of FITS keywords (from data file)
# flat values
self.flatloaded = False # indicates if flat has been loaded
self.flat = None # CCD data object containing arrays with flat values
self.flatdata = DataFits() # Pipedata object containing the flat file
# flat file info and header keywords to fit
self.flatname = '' # name of selected flat file
self.flatfitkeys = [] # FITS keywords that have to fit for flat
self.flatkeyvalues = [] # values of flat keywords (from data file)
# set configuration
self.log.debug('Init: done')
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'biasdarkflat'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'bdf'
# Set Logger for this pipe step
self.log = logging.getLogger('stoneedge.pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append([
'reload', False,
'Set to True to look for new bias files for every input'
])
# Get parameters for StepLoadAux, replace auxfile with biasfile
self.loadauxsetup('bias')
# Get parameters for StepLoadAux, replace auxfile with darkfile
self.loadauxsetup('dark')
# Get parameters for StepLoadAux, replace auxfile with flatfile
self.loadauxsetup('flat')
# confirm end of setup
self.log.debug('Setup: done')
'''# Looking for similar exptime
def closestExp(self):
input_exptime = self.datain.getheadval('EXPTIME')
dark_exptime = self.loadauxname('dark', multi = True).getheadval('EXPTIME')
nearexp = {abs(dark_ave_exptime - exp): exp for exp in dark_exptime}
return nearexp[min(nearexp.keys())]
'''
def run(self):
""" Runs the calibrating algorithm. The calibrated data is
returned in self.dataout
"""
### Preparation
# Load bias files if necessary
if not self.biasloaded or self.getarg('reload'):
self.loadbias()
# Else: check data for correct instrument configuration - currently not in use(need improvement)
else:
for keyind in range(len(self.biasfitkeys)):
if self.biaskeyvalues[keyind] != self.datain.getheadval(
self.biasfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.biasfitkeys[keyind])
# Load dark files if necessary
if not self.darkloaded or self.getarg('reload'):
self.loaddark()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.darkfitkeys)):
if self.darkkeyvalues[keyind] != self.datain.getheadval(
self.darkfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.darkfitkeys[keyind])
# Load flat files if necessary
if not self.flatloaded or self.getarg('reload'):
self.loadflat()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.flatfitkeys)):
if self.flatkeyvalues[keyind] != self.datain.getheadval(
self.flatfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.flatfitkeys[keyind])
#convert self.datain to CCD Data object
image = ccdproc.CCDData(self.datain.image, unit='adu')
image.header = self.datain.header
#subtract bias from image
image = ccdproc.subtract_bias(image, self.bias, add_keyword=False)
#subtract dark from image
image = ccdproc.subtract_dark(image,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second,
add_keyword=False)
#apply flat correction to image
image = ccdproc.flat_correct(image, self.flat, add_keyword=False)
# copy calibrated image into self.dataout - make sure self.dataout is a pipedata object
self.dataout = DataFits(config=self.datain.config)
self.dataout.image = image.data
self.dataout.header = image.header
self.dataout.filename = self.datain.filename
### Finish - cleanup
# Update DATATYPE
self.dataout.setheadval('DATATYPE', 'IMAGE')
# Add bias, dark files to History
self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname)
self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname)
self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname)
def loadbias(self):
""" Loads the bias information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master bias frame by ccdproc.
"""
#master bias frame
#Search for bias and load it into data object
namelist = self.loadauxname('bias', multi=False)
self.log.info('File loaded: %s' % namelist)
if (len(namelist) == 0):
self.log.error('Bias calibration frame not found.')
raise RuntimeError('No bias file loaded')
self.log.debug('Creating master bias frame...')
#if there is just one, use it as biasfile or else combine all to make a master bias
self.bias = ccdproc.CCDData.read(namelist, unit='adu', relax=True)
# Finish up
self.biasloaded = True
self.biasname = namelist
self.log.debug('LoadBias: done')
def loaddark(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master dark frame by ccdproc.
Also bias corrects dark files if not already done.
"""
#master dark frame
dark_is_bias_corrected = False
dark_bias = None
namelist = self.loadauxname('dark', multi=False)
if (len(namelist) == 0):
self.log.error('Dark calibration frame(s) not found.')
raise RuntimeError('No dark file loaded')
# This has been commented out as it is now in StepMasterDark
# darks = None
# for name in namelist:
# #is (any) dark file bias corrected?
# header = fits.getheader(name)
# if(header.get('BIAS') != None):
# dark_is_bias_corrected = True
# dark_bias = header.get('BIAS')
# elif(header.get('BIASCORR') != None):
# dark_is_bias_corrected = True
# dark_bias = header.get('BIASCORR')
# if(darks):
# darks += ','+name
# else:
# darks = name
self.log.debug('Creating master dark frame...')
#if there is just one, use it as darkfile or else combine all to make a master dark
self.dark = ccdproc.CCDData.read(namelist, unit='adu', relax=True)
#bias correct, if necessary
# if(not dark_is_bias_corrected):
# #Subtracting master bias frame from master dark frame
# self.dark = ccdproc.subtract_bias(self.dark, self.bias, add_keyword=False)
# else:
# self.log.debug('Master dark frame is *already* bias corrected (%s).' % dark_bias)
# Finish up
self.darkloaded = True
self.darkname = namelist
self.log.debug('LoadDark: done')
def loadflat(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master flat frame by ccdproc.
Also biascorrects and dark corrects flat files if not already done.
"""
#create master flat frame
flat_is_bias_corrected = False
flat_bias = None
flat_is_dark_corrected = False
flat_dark = None
flat_ave_exptime = 0
namelist = self.loadauxname('flat', multi=False)
if (len(namelist) == 0):
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file loaded')
count = 0
datalist = []
flat_corrected = None
# This has been commented out as it is now in StepMasterFlat
#check a few things in these flat component frames
# for name in namelist:
# header = fits.getheader(name)
#is this flat bias corrected?
# if(header.get('BIAS') != None):
# flat_is_bias_corrected = True
# flat_bias = header.get('BIAS')
# elif(header.get('BIASCORR') != None):
# flat_is_bias_corrected = True
# flat_bias = header.get('BIASCORR')
# #is this flat dark corrected?
# if(header.get('DARK') != None):
# flat_is_dark_corrected = True
# flat_dark = header.get('DARK')
# elif(header.get('DARKCORR') != None):
# flat_is_dark_corrected = True
# flat_dark = header.get('DARKCORR')
# flat_corrected = "%s"%(name.rsplit('.',1)[0])+".corrected"
# shutil.copy(name, flat_corrected)
# self.log.debug('Copying %s to %s' % (name, flat_corrected))
# self.flat = ccdproc.CCDData.read(flat_corrected, unit='adu', relax=True)
# #bias correct, if necessary
# if(not flat_is_bias_corrected):
# self.log.debug('Subtracting master bias frame from flat frame...')
# self.flat = ccdproc.subtract_bias(self.flat, self.bias, add_keyword=False)
# else:
# self.log.debug('Flat frame (%s) is *already* bias corrected.'%flat_bias)
# #dark correct, if necessary
# if(not flat_is_dark_corrected):
# self.log.debug('Subtracting master dark frame from flat frame...')
# self.flat = ccdproc.subtract_dark(self.flat, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second, add_keyword=False)
# else:
# self.log.debug('Flat frame (%s) is *already* dark corrected.'%flat_dark)
# #create CCD Data object list with corrected flat files
# datalist.append(self.flat)
# #calc average exposure time for potential dark correction
# if(header.get('EXPTIME') != None):
# try:
# exptime = float(header.get('EXPTIME'))
# flat_ave_exptime += exptime
# except ValueError:
# self.log.error('Exposure time (EXPTIME) is not a float (%s).'%(header.get('EXPTIME')))
# count += 1
# #calc average exposure time
# if(count > 0):
# flat_ave_exptime = flat_ave_exptime/count
# self.flat.header['EXPTIME'] = flat_ave_exptime
# self.log.info("Average exposure time for flats is %f"%flat_ave_exptime)
self.log.debug('Creating master flat frame...')
#if there is just one, use it as flatfile or else combine all to make a master flat
self.flat = ccdproc.CCDData.read(namelist, unit='adu', relax=True)
# Finish up
self.flatloaded = True
self.flatname = namelist
self.log.debug('LoadFlat: done')
def reset(self):
""" Resets the step to the same condition as it was when it was
created. Internal variables are reset, any stored data is
erased.
"""
self.biasloaded = False
self.bias = None
self.darkloaded = False
self.dark = None
self.flatloaded = False
self.flat = None
self.log.debug('Reset: done')
def run(self):
""" Runs the calibrating algorithm. The calibrated data is
returned in self.dataout
"""
### Preparation
# Load bias files if necessary
if not self.biasloaded or self.getarg('reload'):
self.loadbias()
# Else: check data for correct instrument configuration - currently not in use(need improvement)
else:
for keyind in range(len(self.biasfitkeys)):
if self.biaskeyvalues[keyind] != self.datain.getheadval(
self.biasfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.biasfitkeys[keyind])
# Load dark files if necessary
if not self.darkloaded or self.getarg('reload'):
self.loaddark()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.darkfitkeys)):
if self.darkkeyvalues[keyind] != self.datain.getheadval(
self.darkfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.darkfitkeys[keyind])
# Load flat files if necessary
if not self.flatloaded or self.getarg('reload'):
self.loadflat()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.flatfitkeys)):
if self.flatkeyvalues[keyind] != self.datain.getheadval(
self.flatfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.flatfitkeys[keyind])
#convert self.datain to CCD Data object
image = ccdproc.CCDData(self.datain.image, unit='adu')
image.header = self.datain.header
#subtract bias from image
image = ccdproc.subtract_bias(image, self.bias, add_keyword=False)
#subtract dark from image
image = ccdproc.subtract_dark(image,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second,
add_keyword=False)
#apply flat correction to image
image = ccdproc.flat_correct(image, self.flat, add_keyword=False)
# copy calibrated image into self.dataout - make sure self.dataout is a pipedata object
self.dataout = DataFits(config=self.datain.config)
self.dataout.image = image.data
self.dataout.header = image.header
self.dataout.filename = self.datain.filename
### Finish - cleanup
# Update DATATYPE
self.dataout.setheadval('DATATYPE', 'IMAGE')
# Add bias, dark files to History
self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname)
self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname)
self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname)
def run(self):
""" Runs the data reduction algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
### Preparation
# construct a temp file name that astrometry will output
fp = tempfile.NamedTemporaryFile(suffix=".fits", dir=os.getcwd())
# split off path name, because a path that is too long causes remap to
# crash sometimes
outname = os.path.split(fp.name)[1]
fp.close()
# Add input file path to ouput file and make new name
outpath = os.path.split(self.datain.filename)[0]
outnewname = os.path.join(outpath, outname.replace('.fits', '.new'))
outwcsname = os.path.join(outpath, outname.replace('.fits', '.wcs'))
# Make sure input data exists as file
if not os.path.exists(self.datain.filename):
self.datain.save()
# Make command string
rawcommand = self.getarg('astrocmd') % (self.datain.filename, outname)
# get estimated RA and DEC center values from the config file or input FITS header
raopt = self.getarg('ra')
if raopt != '':
ra = Angle(raopt, unit=u.hour).degree
else:
try:
ra = Angle(self.datain.getheadval('RA'), unit=u.hour).degree
except:
ra = ''
decopt = self.getarg('dec')
if decopt != '':
dec = Angle(decopt, unit=u.deg).degree
else:
try:
dec = Angle(self.datain.getheadval('DEC'), unit=u.deg).degree
except:
dec = ''
if (ra != '') and (dec != ''):
# update command parameters to use these values
rawcommand = rawcommand + ' --ra %f --dec %f --radius %f' % (
ra, dec, self.getarg('searchradius'))
else:
self.log.debug(
'FITS header missing RA/DEC -> searching entire sky')
### Run Astrometry:
# This loop tries the downsample and param options until the fit is successful
# need either --scale-low 0.5 --scale-high 2.0 --sort-column FLUX
# or --guess-scale
downsamples = self.getarg('downsample')
paramoptions = self.getarg('paramoptions')
for option in range(len(downsamples) * len(paramoptions)):
#for downsample in self.getarg('downsample'):
downsample = downsamples[option % len(downsamples)]
paramoption = paramoptions[option // len(downsamples)]
# Add options to command
command = rawcommand + ' --downsample %d' % downsample + ' ' + paramoption
optionstring = "Downsample=%s Paramopts=%s" % (downsample,
paramoption[:10])
# Run the process - see note at the top of the file if using cron
process = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
self.log.debug('running command = %s' % command)
# Wait for the process to be finished or timeout to be reached
timeout = time.time() + self.getarg('timeout')
while time.time() < timeout and process.poll() == None:
time.sleep(1)
poll = process.poll()
if poll == None:
process.kill()
time.sleep(1)
poll = process.poll()
self.log.debug('command returns %d' % poll)
if poll == 0 and os.path.exists(outnewname):
self.log.debug('output file valid -> astrometry successful')
break
else:
self.log.debug('output file missing -> astrometry failed')
# Print the output from astrometry (cut if necessary)
if self.getarg('verbose'):
output = process.stdout.read().decode()
if len(output) > 1000:
outlines = output.split('\n')
output = outlines[:10] + ['...', '...'] + outlines[-7:]
output = '\n'.join(output)
self.log.debug(output)
### Post processing
# Read output file
self.dataout = DataFits(config=self.config)
self.log.debug('Opening astrometry.net output file %s' % outnewname)
try:
self.dataout.load(outnewname)
self.dataout.filename = self.datain.filename
except Exception as error:
self.log.error("Unable to open astrometry. output file = %s" %
outname)
raise error
self.log.debug('Successful parameter options = %s' % optionstring)
# Add history message
histmsg = 'Astrometry.Net: At downsample = %d, search took %d seconds' % (
downsample, time.time() - timeout + 300)
self.dataout.setheadval('HISTORY', histmsg)
# Add RA from astrometry
w = wcs.WCS(self.dataout.header)
n1 = float(self.dataout.header['NAXIS1'] / 2)
n2 = float(self.dataout.header['NAXIS2'] / 2)
ra, dec = w.all_pix2world(n1, n2, 1)
self.dataout.header['CRPIX1'] = n1
self.dataout.header['CRPIX2'] = n2
self.dataout.header['CRVAL1'] = float(ra)
self.dataout.header['CRVAL2'] = float(dec)
self.dataout.header['RA'] = Angle(ra, u.deg).to_string(unit=u.hour,
sep=':')
self.dataout.header['Dec'] = Angle(dec, u.deg).to_string(sep=':')
self.dataout.setheadval('HISTORY',
'Astrometry: Paramopts = ' + optionstring)
# Delete temporary files
if self.getarg('delete_temp'):
os.remove(outnewname)
os.remove(outwcsname)
self.log.debug('Run: Done')
def run(self):
self.dataout = DataFits(config=self.config)
self.dataout.load(self.mask())
class StepMasterDark(StepLoadAux, StepMIParent):
""" Stone Edge Pipeline Step Master Dark Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '0.1' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'masterdark'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'mdark'
# Set Logger for this pipe step
self.log = logging.getLogger('stoneedge.pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters !!!! WHAT PARAMETERS ARE NEEDED ????? !!!!!
self.paramlist.append([
'combinemethod', 'median',
'Specifies how the files should be combined - options are median, average, sum'
])
self.paramlist.append([
'outputfolder', '',
'Output directory location - default is the folder of the input files'
])
# Get parameters for StepLoadAux, replace auxfile with biasfile
self.loadauxsetup('bias')
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
# Find master bias to subtract from master dark
biaslist = self.loadauxname('bias', multi=False)
if (len(biaslist) == 0):
self.log.error('No bias calibration frames found.')
self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True)
# Create empy list for filenames of loaded frames
filelist = []
for fin in self.datain:
self.log.debug("Input filename = %s" % fin.filename)
filelist.append(fin.filename)
# Make a dummy dataout
self.dataout = DataFits(config=self.config)
if len(self.datain) == 0:
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file(s) loaded')
self.log.debug('Creating master flat frame...')
# Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias
if (len(filelist) == 1):
self.dark = ccdproc.CCDData.read(filelist[0],
unit='adu',
relax=True)
self.dark = ccdproc.subtract_bias(self.dark,
self.bias,
add_keyword=False)
else:
darklist = []
for i in filelist:
dark = ccdproc.CCDData.read(i, unit='adu', relax=True)
darksubbias = ccdproc.subtract_bias(dark,
self.bias,
add_keyword=False)
darklist.append(darksubbias)
self.dark = ccdproc.combine(darklist,
method=self.getarg('combinemethod'),
unit='adu',
add_keyword=True)
# set output header, put image into output
self.dataout.header = self.datain[0].header
self.dataout.imageset(self.dark)
# rename output filename
outputfolder = self.getarg('outputfolder')
if outputfolder != '':
outputfolder = os.path.expandvars(outputfolder)
self.dataout.filename = os.path.join(outputfolder,
os.path.split(filelist[0])[1])
else:
self.dataout.filename = filelist[0]
# Add history
self.dataout.setheadval('HISTORY',
'MasterDark: %d files used' % len(filelist))
def run(self):
self.dataout = DataFits(config=self.config)
self.dataout.load(self.source_extract())
self.dataout.header['RA'] = self.datain.header['RA']
self.dataout.header['Dec'] = self.datain.header['Dec']
self.dataout.save()
from darepype.drp import DataFits from astropy.io import fits config = '/Users/josh/pipeline/pipeline/Developments/stepwebastrometry/pipeconf_stonedge_auto.txt' fp = '/Users/josh/Desktop/pipeline_test/data/M5_r-band_60s_bin2_200711_053415_itzamna_seo_0_RAW_TABLE.fits' fts = DataFits(config=config) fts.load(fp) # print(repr(fits.HDUList(file=fp))) # fts.header['RA'] = 0 # fts.header['Dec'] = 0 print(repr(fts.header)) print(repr(fts.image)) # print(repr(fts.table))
# print(repr(dfits.header))
### OPTIONAL BUT RECOMMENDED: Check if all necessary files exist
error_flag = False
# Check if configuration file exists
if not os.path.exists(baseconfig):
print(
'ERROR: The config file you specified, %s,\n does NOT exist on your computer, fix "config" above'
% baseconfig)
error_flag = True
# Check if input files exist
for name in infilenames:
if not os.path.exists(name):
print(
'ERROR: The input file you specifed, %s,\n does NOT exist on your computer, fix "inputnames" above'
% name)
error_flag = True
if not error_flag:
print("All Good")
os.chdir('/Users/josh/pipeline/pipeline/Developments/stepwebastrometry')
step = StepWebAstrometry()
indata = []
for f in infilenames:
fits = DataFits(config=baseconfig)
fits.load(f)
indata.append(fits)
outdata = step(indata[0])
print('Done')
def loaddark(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
"""
### identify dark file to load, search if requested
darkfile = self.getarg('darkfile')
if darkfile == 'search' :
# get list of keywords to fit
fitkeys = self.getarg('fitkeys')
# check format (make first element uppercase)
try:
_ = fitkeys[0].upper()
except AttributeError:
# AttributeError if it's not a string
self.log.error('LoadDark: fitkeys config parameter is ' +
'incorrect format')
raise TypeError('fitkeys config parameter is incorrect format')
# get keywords from data
datakeys=[]
for fitkey in fitkeys:
datakeys.append(self.datain.getheadval(fitkey))
# get dark files from darkdir folder
darkfolder = self.getarg('darkfolder')
filelist=[name for name in os.listdir(darkfolder)
if name[0] != '.' and name.find('.fit') > -1 ]
if len(filelist) < 1:
self.log.error('LoadDark: no dark files found in folder ' +
darkfolder)
raise ValueError('no dark files found in folder ' +
darkfolder)
# match dark files, return best dark file
bestind = 0 # index of file with best match in filelist
bestfitn = 0 # number of keywords that match in best match
fileind = 0 # index for going through the list
while fileind < len(filelist) and bestfitn < len(fitkeys):
# load keys of dark file
filehead = pyfits.getheader(darkfolder+'/'+filelist[fileind])
filekeys=[]
for fitkey in fitkeys:
try:
filekeys.append(filehead[fitkey])
except KeyError:
self.log.warning('LoadDark: missing key [%s] in dark <%s>'
% (fitkey, filelist[fileind] ) )
filekeys.append('')
# determine number of fitting keywords
keyfitn=0
while ( keyfitn < len(fitkeys) and
datakeys[keyfitn] == filekeys[keyfitn] ):
keyfitn = keyfitn + 1
# compare with previous best find
if keyfitn > bestfitn:
bestind = fileind
bestfitn = keyfitn
fileind=fileind+1
darkfile = darkfolder+'/'+filelist[ bestind ]
if bestfitn < len(fitkeys):
self.log.warn('Could not find perfect dark file match')
self.log.warn('Best dark file found is <%s>'
% filelist[bestind] )
else:
self.log.info('Best dark file found is <%s>'
% filelist[bestind] )
self.fitkeys = fitkeys
self.keyvalues = datakeys
### load dark data into a DataFits object
self.darkfile = darkfile
darkdata = DataFits(config = self.config)
darkdata.load(self.darkfile)
### find dark image data arrays and store them
# get sizes of input data
datalist = self.getarg('datalist')
if len(datalist) > 0:
# There are items in datalist -> loop over items
self.darks = []
# Check if necessary number of images in darkdata
if len(darkdata.imgdata) < len(datalist):
msg = 'Number of images in dark file < '
msg += 'number of entries in datalist'
self.log.error('LoadDark: %s' % msg)
raise ValueError(msg)
# Loop through datalist items
for dataind in range(len(datalist)):
dataitem = datalist[dataind]
# Search for dataitem in self.datain images
if dataitem.upper() in self.datain.imgnames:
dataimg = self.datain.imageget(dataitem)
self.log.debug('LoadDark: Found image <%s> to subtract dark'
% dataitem)
# Search dataitem in self.table columns
else:
try:
dataimg = self.datain.table[dataitem]
self.log.debug('LoadDark: Found column <%s> to subtract dark'
% dataitem)
except:
msg = 'No data found for <%s>' % dataitem
self.log.error('LoadDark: %s' % msg)
raise ValueError(msg)
# Get dimensions - append dark to list
datasiz = dataimg.shape
if self.getarg('l0method').upper() != 'NO':
datasiz = datasiz[1:]
darksiz = darkdata.imgdata[dataind].shape
self.darks.append(darkdata.imgdata[dataind])
# Check dimension with dark data
print(datasiz,darksiz)
if len(datasiz) >= len(darksiz):
# Data has >= dimensions than dark -> compare
begind = len(datasiz)-len(darksiz)
if datasiz[begind:] != darksiz:
msg = 'Dark "%s" does not fit data - A' % dataitem
self.log.error('LoadDark: %s' % msg)
raise ValueError(msg)
else:
# More dimensions in dark data -> report error
msg = 'Dark "%s" does not fit data - B' % dataitem
self.log.error('LoadDark: %s' % dataitem)
raise ValueError(msg)
else:
# Empty datalist -> Subtract dark from first image in data with first dark
datasiz = self.datain.image.shape
if self.getarg('l0method').upper() != 'NO':
datasiz = datasiz[1:]
darksiz = darkdata.image.shape
if len(datasiz) >= len(darksiz):
# Data has >= dimensions than dark -> compare
begind = len(datasiz)-len(darksiz)
if datasiz[begind:] != darksiz:
self.log.error('LoadDark: Dark does not fit data - A')
raise ValueError('Dark does not fit data - A')
else:
# More dimensions in dark data -> report error
self.log.error('LoadDark: Dark does not fit data - B')
raise ValueError('Dark does not fit data - B')
self.log.debug('LoadDark: Subtracting Dark from first data image with first dark')
self.darks=[darkdata.image]
### make good pixel map for each detector and add to
#darktemp = numpy.abs(data[0,...])+numpy.abs(data[1,...])
#self.goodpixmap = numpy.ones(data.shape[1:])
#self.goodpixmap [ numpy.where(darktemp == 0.0)] = 0.0
# Finish up
self.darkloaded = 1
self.log.debug('LoadDark: done')
class StepBiasDarkFlat(StepLoadAux, StepParent):
""" Pipeline Step Object to calibrate Bias/Dark/Flat files
"""
stepver = '0.1' # pipe step version
def __init__(self):
""" Constructor: Initialize data objects and variables
"""
# call superclass constructor (calls setup)
super(StepBiasDarkFlat, self).__init__()
# bias values
self.biasloaded = False # indicates if bias has been loaded
self.bias = None # CCD data object containing arrays with bias values
self.biasdata = DataFits() # Pipedata object containing the bias file
# bias file info and header keywords to fit
self.biasname = '' # name of selected bias file
self.biasfitkeys = [] # FITS keywords that are present in bias
self.biaskeyvalues = [] # values of FITS keywords (from data file)
# dark values
self.darkloaded = False # indicates if dark has been loaded
self.dark = None # CCD data object containing arrays with dark values
self.darkdata = DataFits() # Pipedata object containing the dark file
# dark file info and header keywords to fit
self.darkname = '' # name of selected dark file
self.darkfitkeys = [] # FITS keywords that have to fit for dark
self.darkkeyvalues = [] # values of FITS keywords (from data file)
# flat values
self.flatloaded = False # indicates if flat has been loaded
self.flat = None # CCD data object containing arrays with flat values
self.flatdata = DataFits() # Pipedata object containing the flat file
# flat file info and header keywords to fit
self.flatname = '' # name of selected flat file
self.flatfitkeys = [] # FITS keywords that have to fit for flat
self.flatkeyvalues = [] # values of flat keywords (from data file)
# set configuration
self.log.debug('Init: done')
# This function is directly lifted from CCDProc https://github.com/astropy/ccdproc/blob/master/ccdproc/core.py
# Instead of directly calling CCDProc, we have included the function here to increase educational value
# and to decrease reliance on external libraries.
def subtract_bias(self, image, bias):
"""
Subtract master bias from image.
Parameters
----------
image : `~astropy.nddata.CCDData`
Image from which bias will be subtracted.
bias : `~astropy.nddata.CCDData`
Master image to be subtracted from ``ccd``.
{log}
Returns
-------
result : `~astropy.nddata.CCDData`
CCDData object with bias subtracted.
"""
self.log.debug('Subtracting bias...')
result = image.copy()
try:
result.data = image.data - bias.data
# we believe that we should keep this error detection in theory, the bias
# and image both come from seo, so their units should be the same
except ValueError as e:
if 'operand units' in str(e):
raise u.UnitsError(
"Unit '{}' of the uncalibrated image does not "
"match unit '{}' of the calibration "
"image".format(image.unit, bias.unit))
else:
raise e
self.log.debug('Subtracted bias.')
return result
# this code is also lifted from ccdproc https://github.com/astropy/ccdproc/blob/master/ccdproc/core.py
# some of the code is removed from the original ccdproc because it is not relevant to how SEO currently
# processes data. If you are looking at this code in the future, there is more code available to draw from
def subtract_dark(self,
image,
dark,
scale=False,
exposure_time=None,
exposure_unit=None):
"""
Subtract dark current from an image.
Parameters
----------
image : `~astropy.nddata.CCDData`
Image from which dark will be subtracted.
dark : `~astropy.nddata.CCDData`
Dark image.
exposure_time : str or `~ccdproc.Keyword` or None, optional
Name of key in image metadata that contains exposure time.
Default is ``None``.
exposure_unit : `~astropy.units.Unit` or None, optional
Unit of the exposure time if the value in the meta data does not
include a unit.
Default is ``None``.
scale: bool, optional
If True, scale the dark frame by the exposure times.
Default is ``False``.
{log}
Returns
-------
result : `~astropy.nddata.CCDData`
Dark-subtracted image.
"""
self.log.debug('Subtracting dark...')
result = image.copy()
try:
# if dark current is linear, then this first step scales the provided
# dark to match the exposure time
if scale:
dark_scaled = dark.copy()
data_exposure = image.header[exposure_time]
dark_exposure = dark.header[exposure_time]
# data_exposure and dark_exposure are both quantities,
# so we can just have subtract do the scaling
dark_scaled = dark_scaled.multiply(data_exposure /
dark_exposure)
result.data = image.data - dark_scaled.data
else:
result.data = image.data - dark.data
except (u.UnitsError, u.UnitConversionError, ValueError) as e:
# Make the error message a little more explicit than what is returned
# by default.
raise u.UnitsError("Unit '{}' of the uncalibrated image does not "
"match unit '{}' of the calibration "
"image".format(image.unit, dark.unit))
self.log.debug('Subtracted dark.')
return result
# This code is also from ccdproc. A notable removal is the option to manually choose
# maximum and minimum flat values.
def flat_correct(self, image, flat):
"""Correct the image for flat fielding.
The flat field image is normalized by its mean or a user-supplied value
before flat correcting.
Parameters
----------
ccd : `~astropy.nddata.CCDData`
Data to be transformed.
flat : `~astropy.nddata.CCDData`
Flatfield to apply to the data.
{log}
Returns
-------
ccd : `~astropy.nddata.CCDData`
CCDData object with flat corrected.
"""
self.log.debug('Correcting flat...')
# Use the min_value to replace any values in the flat
flat_corrected = image.copy()
use_flat = flat
flat_mean_val = use_flat.data.mean()
# Normalize the flat.
flat_mean = flat_mean_val * use_flat.unit
flat_normed = use_flat.data / flat_mean
# divide through the flat
flat_corrected.data = image.data / flat_normed
self.log.debug('Corrected flat.')
return flat_corrected
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'biasdarkflat'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'bdf'
# Set Logger for this pipe step
self.log = logging.getLogger('stoneedge.pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append([
'reload', False,
'Set to True to look for new bias files for every input'
])
# Get parameters for StepLoadAux, replace auxfile with biasfile
self.loadauxsetup('bias')
# Get parameters for StepLoadAux, replace auxfile with darkfile
self.loadauxsetup('dark')
# Get parameters for StepLoadAux, replace auxfile with flatfile
self.loadauxsetup('flat')
# confirm end of setup
self.log.debug('Setup: done')
'''# Looking for similar exptime
def closestExp(self):
input_exptime = self.datain.getheadval('EXPTIME')
dark_exptime = self.loadauxname('dark', multi = True).getheadval('EXPTIME')
nearexp = {abs(dark_ave_exptime - exp): exp for exp in dark_exptime}
return nearexp[min(nearexp.keys())]
'''
def run(self):
""" Runs the calibrating algorithm. The calibrated data is
returned in self.dataout
"""
### Preparation
# Load bias files if necessary
if not self.biasloaded or self.getarg('reload'):
self.loadbias()
# Else: check data for correct instrument configuration - currently not in use(need improvement)
else:
for keyind in range(len(self.biasfitkeys)):
if self.biaskeyvalues[keyind] != self.datain.getheadval(
self.biasfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.biasfitkeys[keyind])
# Load dark files if necessary
if not self.darkloaded or self.getarg('reload'):
self.loaddark()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.darkfitkeys)):
if self.darkkeyvalues[keyind] != self.datain.getheadval(
self.darkfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.darkfitkeys[keyind])
# Load flat files if necessary
if not self.flatloaded or self.getarg('reload'):
self.loadflat()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.flatfitkeys)):
if self.flatkeyvalues[keyind] != self.datain.getheadval(
self.flatfitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.flatfitkeys[keyind])
# in the config file, set the 'intermediate' variable to either true or false to enable
# saving of intermediate steps
saveIntermediateSteps = self.config['biasdarkflat']['intermediate']
self.dataout = DataFits(config=self.datain.config)
#convert self.datain to CCD Data object
image = CCDData(self.datain.image, unit='adu')
image.header = self.datain.header
# subtract bias from image
image = self.subtract_bias(image, self.bias)
if (saveIntermediateSteps == "true"):
self.dataout.imageset(image.data, imagename="BIAS")
# self.dataout.setheadval('DATATYPE','IMAGE', dataname="BIAS")
self.dataout.setheadval('HISTORY',
'BIAS: %s' % self.biasname,
dataname="BIAS")
# subtract dark from image
image = self.subtract_dark(image,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second)
if (saveIntermediateSteps == "true"):
self.dataout.imageset(image.data, imagename="DARK")
# self.dataout.setheadval('DATATYPE','IMAGE', dataname="DARK")
self.dataout.setheadval('HISTORY',
'BIAS: %s' % self.biasname,
dataname="DARK")
self.dataout.setheadval('HISTORY',
'DARK: %s' % self.darkname,
dataname="DARK")
# apply flat correction to image
image = self.flat_correct(image, self.flat)
# if separating bias,dark,flat steps , save the flat-corrected portion into its own hdu
if (saveIntermediateSteps == "true"):
self.dataout.imageset(image.data, imagename="FLAT")
# self.dataout.setheadval('DATATYPE','IMAGE', dataname="FLAT")
self.dataout.setheadval('HISTORY',
'BIAS: %s' % self.biasname,
dataname="FLAT")
self.dataout.setheadval('HISTORY',
'DARK: %s' % self.darkname,
dataname="FLAT")
self.dataout.setheadval('HISTORY',
'FLAT: %s' % self.flatname,
dataname="FLAT")
else:
# copy calibrated image into self.dataout
self.dataout.image = image.data
self.dataout.header = self.datain.header
### Finish - cleanup
# Update DATATYPE
self.dataout.setheadval('DATATYPE', 'IMAGE')
# Add bias, dark files to History
self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname)
self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname)
self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname)
self.dataout.filename = self.datain.filename
def loadbias(self):
""" Loads the bias information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master bias frame by ccdproc.
"""
#master bias frame
#Search for bias and load it into data object
namelist = self.loadauxname('bias', multi=False)
self.log.info('File loaded: %s' % namelist)
if (len(namelist) == 0):
self.log.error('Bias calibration frame not found.')
raise RuntimeError('No bias file loaded')
self.log.debug('Creating master bias frame...')
#if there is just one, use it as biasfile or else combine all to make a master bias
self.bias = CCDData.read(namelist, unit='adu', relax=True)
# Finish up
self.biasloaded = True
self.biasname = namelist
self.log.debug('LoadBias: done')
def loaddark(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master dark frame by ccdproc.
Also bias corrects dark files if not already done.
"""
#master dark frame
dark_is_bias_corrected = False
dark_bias = None
namelist = self.loadauxname('dark', multi=False)
if (len(namelist) == 0):
self.log.error('Dark calibration frame(s) not found.')
raise RuntimeError('No dark file loaded')
# This has been commented out as it is now in StepMasterDark
# darks = None
# for name in namelist:
# #is (any) dark file bias corrected?
# header = fits.getheader(name)
# if(header.get('BIAS') != None):
# dark_is_bias_corrected = True
# dark_bias = header.get('BIAS')
# elif(header.get('BIASCORR') != None):
# dark_is_bias_corrected = True
# dark_bias = header.get('BIASCORR')
# if(darks):
# darks += ','+name
# else:
# darks = name
self.log.debug('Creating master dark frame...')
#if there is just one, use it as darkfile or else combine all to make a master dark
self.dark = CCDData.read(namelist, unit='adu', relax=True)
#bias correct, if necessary
# if(not dark_is_bias_corrected):
# #Subtracting master bias frame from master dark frame
# self.dark = ccdproc.subtract_bias(self.dark, self.bias, add_keyword=False)
# else:
# self.log.debug('Master dark frame is *already* bias corrected (%s).' % dark_bias)
# Finish up
self.darkloaded = True
self.darkname = namelist
self.log.debug('LoadDark: done')
def loadflat(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
If multiple matching files are found, they are combined into a single
master flat frame by ccdproc.
Also biascorrects and dark corrects flat files if not already done.
"""
#create master flat frame
flat_is_bias_corrected = False
flat_bias = None
flat_is_dark_corrected = False
flat_dark = None
flat_ave_exptime = 0
namelist = self.loadauxname('flat', multi=False)
if (len(namelist) == 0):
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file loaded')
count = 0
datalist = []
flat_corrected = None
# This has been commented out as it is now in StepMasterFlat
#check a few things in these flat component frames
# for name in namelist:
# header = fits.getheader(name)
#is this flat bias corrected?
# if(header.get('BIAS') != None):
# flat_is_bias_corrected = True
# flat_bias = header.get('BIAS')
# elif(header.get('BIASCORR') != None):
# flat_is_bias_corrected = True
# flat_bias = header.get('BIASCORR')
# #is this flat dark corrected?
# if(header.get('DARK') != None):
# flat_is_dark_corrected = True
# flat_dark = header.get('DARK')
# elif(header.get('DARKCORR') != None):
# flat_is_dark_corrected = True
# flat_dark = header.get('DARKCORR')
# flat_corrected = "%s"%(name.rsplit('.',1)[0])+".corrected"
# shutil.copy(name, flat_corrected)
# self.log.debug('Copying %s to %s' % (name, flat_corrected))
# self.flat = ccdproc.CCDData.read(flat_corrected, unit='adu', relax=True)
# #bias correct, if necessary
# if(not flat_is_bias_corrected):
# self.log.debug('Subtracting master bias frame from flat frame...')
# self.flat = ccdproc.subtract_bias(self.flat, self.bias, add_keyword=False)
# else:
# self.log.debug('Flat frame (%s) is *already* bias corrected.'%flat_bias)
# #dark correct, if necessary
# if(not flat_is_dark_corrected):
# self.log.debug('Subtracting master dark frame from flat frame...')
# self.flat = ccdproc.subtract_dark(self.flat, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second, add_keyword=False)
# else:
# self.log.debug('Flat frame (%s) is *already* dark corrected.'%flat_dark)
# #create CCD Data object list with corrected flat files
# datalist.append(self.flat)
# #calc average exposure time for potential dark correction
# if(header.get('EXPTIME') != None):
# try:
# exptime = float(header.get('EXPTIME'))
# flat_ave_exptime += exptime
# except ValueError:
# self.log.error('Exposure time (EXPTIME) is not a float (%s).'%(header.get('EXPTIME')))
# count += 1
# #calc average exposure time
# if(count > 0):
# flat_ave_exptime = flat_ave_exptime/count
# self.flat.header['EXPTIME'] = flat_ave_exptime
# self.log.info("Average exposure time for flats is %f"%flat_ave_exptime)
self.log.debug('Creating master flat frame...')
#if there is just one, use it as flatfile or else combine all to make a master flat
self.flat = CCDData.read(namelist, unit='adu', relax=True)
# Finish up
self.flatloaded = True
self.flatname = namelist
self.log.debug('LoadFlat: done')
def reset(self):
""" Resets the step to the same condition as it was when it was
created. Internal variables are reset, any stored data is
erased.
"""
self.biasloaded = False
self.bias = None
self.darkloaded = False
self.dark = None
self.flatloaded = False
self.flat = None
self.log.debug('Reset: done')
def run(self):
""" Runs the combining algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
# Find master dark to subtract from master dark
biaslist = self.loadauxname('bias', multi=False)
darklist = self.loadauxname('dark', multi=False)
if (len(biaslist) == 0):
self.log.error('No bias calibration frames found.')
if (len(darklist) == 0):
self.log.error('No bias calibration frames found.')
self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True)
self.dark = ccdproc.CCDData.read(darklist, unit='adu', relax=True)
# Create empy list for filenames of loaded frames
filelist = []
for fin in self.datain:
self.log.debug("Input filename = %s" % fin.filename)
filelist.append(fin.filename)
# Make a dummy dataout
self.dataout = DataFits(config=self.config)
if len(self.datain) == 0:
self.log.error('Flat calibration frame not found.')
raise RuntimeError('No flat file(s) loaded')
self.log.debug('Creating master flat frame...')
# Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias
if (len(filelist) == 1):
self.flat = ccdproc.CCDData.read(filelist[0],
unit='adu',
relax=True)
self.flat = ccdproc.subtract_bias(self.flat,
self.bias,
add_keyword=False)
self.flat = ccdproc.subtract_dark(self.flat,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second,
add_keyword=False)
else:
#bias and dark correct frames
flatlist = []
for i in filelist:
flat = ccdproc.CCDData.read(i, unit='adu', relax=True)
flatsubbias = ccdproc.subtract_bias(flat,
self.bias,
add_keyword=False)
flatsubbiasdark = ccdproc.subtract_dark(
flatsubbias,
self.dark,
scale=True,
exposure_time='EXPTIME',
exposure_unit=u.second,
add_keyword=False)
flatlist.append(flatsubbiasdark)
#scale the flat component frames to have the same mean value, 10000.0
scaling_func = lambda arr: 10000.0 / numpy.ma.median(arr)
#combine them
self.flat = ccdproc.combine(flatlist,
method=self.getarg('combinemethod'),
scale=scaling_func,
unit='adu',
add_keyword=False)
# set output header, put image into output
self.dataout.header = self.datain[0].header
self.dataout.imageset(self.flat)
# rename output filename
outputfolder = self.getarg('outputfolder')
if outputfolder != '':
outputfolder = os.path.expandvars(outputfolder)
self.dataout.filename = os.path.join(outputfolder,
os.path.split(filelist[0])[1])
else:
self.dataout.filename = filelist[0]
# Add history
self.dataout.setheadval('HISTORY',
'MasterFlat: %d files used' % len(filelist))
class StepCoadd(StepMIParent):
""" Stone Edge Pipeline Step Master Bias Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '1.2' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name='coadd'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'coadd'
# Set Logger for this pipe step
self.log = logging.getLogger('pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append(['kernel','square',
'Specifies the kernel used to determine spreading of input pixels onto output pixels \
- options are square, point, gaussian, smoothing, tophat'])
self.paramlist.append(['pixfrac', 1.,
'The fraction of an output pixel(s) that an input pixel\'s flux is confined to'])
self.paramlist.append(['resolution', 1.,
'Pixel scale divisor for output image (higher gives more resolution, lower gives less)'])
self.paramlist.append(['pad', 0,
'Extra padding outside maximum extent of inputs'])
self.paramlist.append(['fillval', np.nan,
'Value for filling in the area(s) in the output where there is no input data'])
self.paramlist.append(['drizzleweights','exptime',
'How each input image should be weighted when added to the output \
- options are exptime, expsq and uniform'])
self.paramlist.append(['outangle',0.,
'Output angle of drizzled image (currently not functional)'])
def run(self):
""" Runs the mosaicing algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
#calculate platescale of first input image
try:
det = np.linalg.det(wcs.WCS(self.datain[0].header).wcs.cd)
pscale = np.sqrt(np.abs(det))*3600.
except:
try:
det = np.linalg.det(wcs.WCS(self.datain[0].header).wcs.pc)
pscale = np.sqrt(np.abs(det))*3600.
except:
pscale = self.datain[0].header['PIXSCAL']
#filtering out images which are too far away from the others
#passing images added to a list of (image, WCS) tuples
'''
image_centers = []
for f in self.datain:
image_centers.append((f.header['CRVAL1'], f.header['CRVAL2']))
filtered_datain = []
dist_list = [[[0]*(len(image_centers)-1)]*len(image_centers)]
for i in range(len(image_centers)):
for j in range(len(image_centers)-1):
dist_list[i][j+1] = np.sqrt((image_)**2+()**2)
'''
#calculations necessary for updating wcs information
px = []
py = []
#in order to avoid NaN interactions, creating weight map
weights=[]
for f in self.datain:
weights.append((np.where(np.isnan(f.image) == True, 0, 1)))
for f in self.datain:
px.extend(wcs.WCS(f.header).calc_footprint()[:,0])
py.extend(wcs.WCS(f.header).calc_footprint()[:,1])
x0 = (max(px)+min(px))/2.
y0 = (max(py)+min(py))/2.
sx = (max(px)-min(px))*np.cos(y0/180*np.pi) # arcsec
sy = (max(py)-min(py)) # arcsec
size = (sx*3600+self.getarg('pad')*2, sy*3600+self.getarg('pad')*2)
xpix = size[0]//pscale
ypix = size[1]//pscale
cdelt = [pscale/3600.]*2
#create self.dataout and give it a copy of an input's header
self.dataout = DataFits(config = self.config)
self.dataout.header = self.datain[0].header.copy()
#update header wcs information
self.log.info('Creating new WCS header')
self.dataout.header['CRPIX1'] = xpix/2
self.dataout.header['CRPIX2'] = ypix/2
self.dataout.header['CRVAL1'] = x0
self.dataout.header['CRVAL2'] = y0
self.dataout.header['CD1_1'] = -cdelt[0]
self.dataout.header['CD1_2'] = self.dataout.header['CD2_1'] = 0.
self.dataout.header['CD2_2'] = cdelt[1]
self.dataout.header['NAXIS1'] = int(xpix)
self.dataout.header['NAXIS2'] = int(ypix)
self.dataout.header['CTYPE1'] = 'RA---TAN-SIP'
self.dataout.header['CTYPE2'] = 'DEC--TAN-SIP'
self.dataout.header['RADESYS'] = 'ICRS'
self.dataout.header['EQUINOX'] = 2000
self.dataout.header['LATPOLE'] = self.datain[0].header['CRVAL2']
self.dataout.header['LONPOLE'] = 180
self.dataout.header['PIXASEC'] = pscale
theta_rad = np.deg2rad(self.getarg('outangle'))
rot_matrix = np.array([[np.cos(theta_rad), -np.sin(theta_rad)],
[np.sin(theta_rad), np.cos(theta_rad)]])
rot_cd = np.dot(rot_matrix, np.array([[self.dataout.header['CD1_1'], 0.],[0., self.dataout.header['CD2_2']]]))
for i in [0,1]:
for j in [0,1]:
self.dataout.header['CD{0:d}_{1:d}'.format(i+1, j+1)] = rot_cd[i,j]
#check drizzle arguments
if self.getarg('kernel') == 'smoothing':
kernel = 'lanczos3'
elif self.getarg('kernel') in ['square', 'point', 'gaussian', 'tophat']:
kernel = self.getarg('kernel')
else:
self.log.error('Kernel name not recognized, using default')
kernel = 'square'
if self.getarg('drizzleweights') == 'uniform':
driz_wt = ''
elif self.getarg('drizzleweights') in ['exptime', 'expsq']:
driz_wt = self.getarg('drizzleweights')
else:
self.log.error('Drizzle weighting not recognized, using default')
driz_wt = ''
#create drizzle object and add input images
fullwcs = wcs.WCS(self.dataout.header)
self.log.info('Starting drizzle')
driz = drz.Drizzle(outwcs = fullwcs, pixfrac=self.getarg('pixfrac'), \
kernel=kernel, fillval='10000', wt_scl=driz_wt)
for i,f in enumerate(self.datain):
self.log.info('Adding %s to drizzle stack' % f.filename)
driz.add_image(f.imgdata[0], wcs.WCS(f.header), inwht=weights[i])
try:
fillval=float(self.getarg('fillval'))
except:
fillval=np.nan
self.log.error('Fillvalue not recognized or missing, using default')
#creates output fits file from drizzle output
self.dataout.imageset(np.where(driz.outsci == 10000, fillval, driz.outsci))
self.dataout.imageset(driz.outwht,'OutWeight', self.dataout.header)
self.dataout.filename = self.datain[0].filename
#add history
self.dataout.setheadval('HISTORY','Coadd: %d files combined with %s kernel, pixfrac %f at %f times resolution' \
% (len(self.datain), kernel, self.getarg('pixfrac'), self.getarg('resolution')))
def run(self):
""" Runs the mosaicing algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
#calculate platescale of first input image
try:
det = np.linalg.det(wcs.WCS(self.datain[0].header).wcs.cd)
pscale = np.sqrt(np.abs(det))*3600.
except:
try:
det = np.linalg.det(wcs.WCS(self.datain[0].header).wcs.pc)
pscale = np.sqrt(np.abs(det))*3600.
except:
pscale = self.datain[0].header['PIXSCAL']
#filtering out images which are too far away from the others
#passing images added to a list of (image, WCS) tuples
'''
image_centers = []
for f in self.datain:
image_centers.append((f.header['CRVAL1'], f.header['CRVAL2']))
filtered_datain = []
dist_list = [[[0]*(len(image_centers)-1)]*len(image_centers)]
for i in range(len(image_centers)):
for j in range(len(image_centers)-1):
dist_list[i][j+1] = np.sqrt((image_)**2+()**2)
'''
#calculations necessary for updating wcs information
px = []
py = []
#in order to avoid NaN interactions, creating weight map
weights=[]
for f in self.datain:
weights.append((np.where(np.isnan(f.image) == True, 0, 1)))
for f in self.datain:
px.extend(wcs.WCS(f.header).calc_footprint()[:,0])
py.extend(wcs.WCS(f.header).calc_footprint()[:,1])
x0 = (max(px)+min(px))/2.
y0 = (max(py)+min(py))/2.
sx = (max(px)-min(px))*np.cos(y0/180*np.pi) # arcsec
sy = (max(py)-min(py)) # arcsec
size = (sx*3600+self.getarg('pad')*2, sy*3600+self.getarg('pad')*2)
xpix = size[0]//pscale
ypix = size[1]//pscale
cdelt = [pscale/3600.]*2
#create self.dataout and give it a copy of an input's header
self.dataout = DataFits(config = self.config)
self.dataout.header = self.datain[0].header.copy()
#update header wcs information
self.log.info('Creating new WCS header')
self.dataout.header['CRPIX1'] = xpix/2
self.dataout.header['CRPIX2'] = ypix/2
self.dataout.header['CRVAL1'] = x0
self.dataout.header['CRVAL2'] = y0
self.dataout.header['CD1_1'] = -cdelt[0]
self.dataout.header['CD1_2'] = self.dataout.header['CD2_1'] = 0.
self.dataout.header['CD2_2'] = cdelt[1]
self.dataout.header['NAXIS1'] = int(xpix)
self.dataout.header['NAXIS2'] = int(ypix)
self.dataout.header['CTYPE1'] = 'RA---TAN-SIP'
self.dataout.header['CTYPE2'] = 'DEC--TAN-SIP'
self.dataout.header['RADESYS'] = 'ICRS'
self.dataout.header['EQUINOX'] = 2000
self.dataout.header['LATPOLE'] = self.datain[0].header['CRVAL2']
self.dataout.header['LONPOLE'] = 180
self.dataout.header['PIXASEC'] = pscale
theta_rad = np.deg2rad(self.getarg('outangle'))
rot_matrix = np.array([[np.cos(theta_rad), -np.sin(theta_rad)],
[np.sin(theta_rad), np.cos(theta_rad)]])
rot_cd = np.dot(rot_matrix, np.array([[self.dataout.header['CD1_1'], 0.],[0., self.dataout.header['CD2_2']]]))
for i in [0,1]:
for j in [0,1]:
self.dataout.header['CD{0:d}_{1:d}'.format(i+1, j+1)] = rot_cd[i,j]
#check drizzle arguments
if self.getarg('kernel') == 'smoothing':
kernel = 'lanczos3'
elif self.getarg('kernel') in ['square', 'point', 'gaussian', 'tophat']:
kernel = self.getarg('kernel')
else:
self.log.error('Kernel name not recognized, using default')
kernel = 'square'
if self.getarg('drizzleweights') == 'uniform':
driz_wt = ''
elif self.getarg('drizzleweights') in ['exptime', 'expsq']:
driz_wt = self.getarg('drizzleweights')
else:
self.log.error('Drizzle weighting not recognized, using default')
driz_wt = ''
#create drizzle object and add input images
fullwcs = wcs.WCS(self.dataout.header)
self.log.info('Starting drizzle')
driz = drz.Drizzle(outwcs = fullwcs, pixfrac=self.getarg('pixfrac'), \
kernel=kernel, fillval='10000', wt_scl=driz_wt)
for i,f in enumerate(self.datain):
self.log.info('Adding %s to drizzle stack' % f.filename)
driz.add_image(f.imgdata[0], wcs.WCS(f.header), inwht=weights[i])
try:
fillval=float(self.getarg('fillval'))
except:
fillval=np.nan
self.log.error('Fillvalue not recognized or missing, using default')
#creates output fits file from drizzle output
self.dataout.imageset(np.where(driz.outsci == 10000, fillval, driz.outsci))
self.dataout.imageset(driz.outwht,'OutWeight', self.dataout.header)
self.dataout.filename = self.datain[0].filename
#add history
self.dataout.setheadval('HISTORY','Coadd: %d files combined with %s kernel, pixfrac %f at %f times resolution' \
% (len(self.datain), kernel, self.getarg('pixfrac'), self.getarg('resolution')))
class StepFlat(StepLoadAux, StepParent):
""" HAWC Pipeline Flatfielding Step Object
"""
stepver = '0.1' # pipe step version
def __init__(self):
""" Constructor: Initialize data objects and variables
"""
# call superclass constructor (calls setup)
super(StepFlat, self).__init__()
# list of data and flats
self.datalist = [] # used in run() for every new input data file
# flat values
self.flatloaded = 0 # indicates if flat has been loaded
self.flats = [] # list containing arrays with flat values
self.flatdata = DataFits() # Pipedata object containing the flat file
# flat file info and header keywords to fit
self.flatfile = '' # name of selected flat file
self.fitkeys = [] # FITS keywords that have to fit
self.keyvalues = [
] # values of the keywords (from the first data file)
# set configuration
self.log.debug('Init: done')
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'flat'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'fla'
# Set Logger for this pipe step
self.log = logging.getLogger('hawc.pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append([
'reload', 'False',
'Set to True to look for new flat files for every input'
])
self.paramlist.append([
'l0method', 'NO',
'Method to normalize data: NOne, REal, IMag and ABSolute ' +
'(default = NO)'
])
self.paramlist.append([
'datalist', [], 'List of data sets to flatten in intput file ' +
'(default = [] i.e. only flatten image cube in first HDU)'
])
self.paramlist.append([
'addfromfile', [],
'List of data sets from the flat file to add to the output data' +
'(default = [] i.e. no data to add)'
])
# Get parameters for StepLoadAux, replace auxfile with flatfile
self.loadauxsetup('flatfile')
def run(self):
""" Runs the flatfielding algorithm. The flatfielded data is
returned in self.dataout
"""
### Preparation
# Load flat files if necessary
if not self.flatloaded or self.getarg('reload'):
self.loadflat()
# Else: check data for correct instrument configuration
else:
for keyind in range(len(self.fitkeys)):
if self.keyvalues[keyind] != self.datain.getheadval(
self.fitkeys[keyind]):
self.log.warn(
'New data has different FITS key value for keyword %s'
% self.fitkeys[keyind])
### Copy datain to dataout
self.dataout = self.datain.copy()
### Apply Flatfield
# Only one data set -> flatfield it
datalist = self.getarg('datalist')
if len(datalist) == 0:
# Get Image
image = self.datain.image.copy()
# Flatfield it
self.dataout.image = self.flatfield(image, self.flats[0])
# Loop through data sets
else:
for dataind in range(len(datalist)):
dataitem = datalist[dataind]
# Search for dataitem in self.datain images -> flatfield it
if dataitem.upper() in self.datain.imgnames:
image = self.datain.imageget(dataitem)
image = self.flatfield(image, self.flats[dataind])
self.dataout.imageset(image, dataitem)
continue # go to next dataitem (skip end of loop)
# Search for dataitem in self.table columns
try:
image = self.datain.table[dataitem]
except:
msg = 'No data found for %s in file %s' % (
dataitem, self.datain.filename)
self.log.error('Run: %s' % msg)
raise ValueError(msg)
# Flatfield
image = self.flatfield(image, self.flats[dataind])
# Store image in dataout
self.dataout.imageset(image, dataitem)
# Remove table column from dataout
self.dataout.tabledelcol(dataitem)
### Add additional image frames from flat file
for dataitem in self.getarg('addfromfile'):
ind = self.flatdata.imageindex(dataitem.upper())
if ind > -1:
self.dataout.imageset(
self.flatdata.imageget(dataitem),
imagename=dataitem,
imageheader=self.flatdata.getheader(dataitem))
continue
ind = self.flatdata.tableindex(dataitem.upper())
if ind > -1:
self.dataout.tableset(
self.flatdata.tableget(dataitem),
tablename=dataitem,
tableheader=self.flatdata.getheader(dataitem))
continue
msg = 'No data to add found for <%s>' % dataitem
self.log.error('Run: %s' % msg)
raise ValueError(msg)
# Remove the instrumental configuration HDU
if 'CONFIGURATION' in self.dataout.imgnames:
self.dataout.imagedel('CONFIGURATION')
### Finish - cleanup
# Update DATATYPE
self.dataout.setheadval('DATATYPE', 'IMAGE')
# Add flat file to History
self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatdata.filename)
# Update PROCSTAT to level 2
self.dataout.setheadval('PROCSTAT', 'LEVEL_2')
def loadflat(self):
""" Loads the flat information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
"""
### Search for flat and load it into data object
self.flatdata = self.loadauxfile()
### find flatfields data arrays and store them
# get sizes of input data
datalist = self.getarg('datalist')
if len(datalist) == 0:
# Empty datalist -> Flat first image in data with first flat
self.checksize(self.datain.image.shape, self.flatdata.image.shape)
self.log.debug(
'LoadFlat: Flatfielding first data image with first flat')
self.flats = [self.flatdata.image]
else:
# There are items in datalist -> loop over items
self.flats = []
# Check if necessary number of images in flatdata
if len(self.flatdata.imgdata) < len(datalist):
msg = 'Number of images in flatfield file < '
msg += 'number of entries in datalist'
self.log.error('LoadFlat: %s' % msg)
raise ValueError(msg)
# Loop through datalist items
for dataind in range(len(datalist)):
dataitem = datalist[dataind]
# Search for dataitem in self.datain images
if dataitem.upper() in self.datain.imgnames:
dataimg = self.datain.imageget(dataitem)
self.log.debug('LoadFlat: Found image <%s> to flat' %
dataitem)
# Search dataitem in self.table columns
else:
try:
dataimg = self.datain.table[dataitem]
self.log.debug('LoadFlat: Found column <%s> to flat' %
dataitem)
except:
msg = 'No data found for <%s>' % dataitem
self.log.error('LoadFlat: %s' % msg)
raise ValueError(msg)
# Get dimensions - append flat to list
self.checksize(dataimg.shape,
self.flatdata.imgdata[dataind].shape)
self.flats.append(self.flatdata.imgdata[dataind])
### Ensure that data listed in addfromfile is present in flatfile
for dataitem in self.getarg('addfromfile'):
if dataitem.upper() in self.flatdata.imgnames:
pass
elif dataitem.upper() in self.flatdata.tabnames:
pass
else:
msg = 'No data to add found for <%s>' % dataitem
self.log.error('LoadFlat: %s' % msg)
raise ValueError(msg)
# Finish up
self.flatloaded = 1
self.log.debug('LoadFlat: done')
def flatfield(self, imgin, flat):
""" Flatfields an array using flat.
If r0method != NO then the real image is computed
This method checks that imagein and flat are compatible
"""
# Check flatfield dimension
self.checksize(imgin.shape, flat.shape)
# Do r0method correction if necessary
l0method = self.getarg('l0method')
if l0method != 'NO':
# Return L0 for chosen method
if l0method == 'ABS':
data = numpy.zeros(imgin.shape[1:], dtype=complex)
data.real = imgin[0, ...].copy()
data.imag = imgin[1, ...].copy()
imgin = abs(data)
elif l0method == 'IM':
imgin = imgin[1, ...].copy()
else:
imgin = imgin[0, ...].copy()
# Apply flatfield
imgout = imgin * flat
return imgout
def checksize(self, datashape, flatshape):
""" Checks that the shape of the flat is comptatible to be used
with image data of datashape. Raises exceptions otherwise.
"""
if self.getarg('l0method').upper() != 'NO':
datashape = datashape[1:]
if len(datashape) >= len(flatshape):
# Data has >= dimensions than flat -> compare
begind = len(datashape) - len(flatshape)
if datashape[begind:] != flatshape:
msg = 'Flat does not fit data in file %s' % self.datain.filename
self.log.error('FlatField: %s' % msg)
raise ValueError(msg)
else:
# More dimensions in flat data -> report error
msg = 'Flat does not fit data in file %s' % self.datain.filename
self.log.error('LoadFlat: %s' % msg)
raise ValueError(msg)
def reset(self):
""" Resets the step to the same condition it was when it was
created. Stored flatfield data is erased and the configuration
information is cleared.
"""
# initialize input and output
self.flatloaded = 0
self.flatvalue = numpy.zeros([1, 1])
self.flatphase = numpy.zeros([1, 1])
self.flatheader = fits.PrimaryHDU(numpy.array(1))
self.flatfile = ''
def test(self):
""" Test Pipe Step Flat Object: Runs basic tests
"""
# initial log message
self.log.info('Testing pipe step flat')
# get testin and a configuration
if self.config != None and len(
self.config) > 2: # i.e. if real config is loaded
testin = DataFits(config=self.config)
else:
testin = DataFits(config=self.testconf)
# load sample data
datain = DataFits(config=testin.config)
#infile = 'mode_chop/120207_000_00HA012.chop.dmd.fits'
infile = 'mode_chop/120306_000_00HA006.chop.dmd.fits'
#infile = 'mode_chop/120402_000_00HA035.chop.dmd.fits'
#infile = 'sharp/sharc2-048485.dmdsqr.fits'
testfile = os.path.join(datain.config['testing']['testpath'], infile)
#testfile = '/Users/berthoud/testfit.fits'
datain.load(testfile)
if False:
# change data (make complex number array with
# Re=0,1,2,3,4,5,6 . . . in time Im=0)
dataval = numpy.ones(datain.image.shape)
dataval[..., 1] = 0.0
inclist = numpy.arange(dataval.shape[0])
incshape = [1 + i - i for i in dataval.shape[0:-1]]
incshape[0] = dataval.shape[0]
inclist.shape = incshape
dataval[..., 0] = dataval[..., 0] * inclist
#datain.image=dataval
# run first flat
dataout = self(datain)
#print dataout.image[100,...] # print 100th image
#print dataout.image[range(0,dataval.shape[0],1000),0,0] # 1 val per img
dataout.save()
# final log message
self.log.info('Testing pipe step flat - Done')
class StepAstrometry(StepParent):
""" HAWC Pipeline Step Parent Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '0.2' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'astrometry'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'WCS'
# Set Logger for this pipe step
self.log = logging.getLogger('pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append([
'astrocmd', 'cp %s %s',
'Command to call astrometry, should contain 2' +
'string placeholders for intput and output ' + 'filepathname'
])
self.paramlist.append(
['verbose', False, 'log full astrometry output at DEBUG level'])
self.paramlist.append([
'delete_temp', False,
'Flag to delete temporary files generated by astrometry'
])
self.paramlist.append(
['downsample', [2], 'List of downsample factors to try'])
self.paramlist.append([
'paramoptions', ['--guess-scale'],
'Parameter groups to run if the command fails'
])
self.paramlist.append(
['timeout', 300, 'Timeout for running astrometry (seconds)'])
self.paramlist.append(
['ra', '', 'Option to manually set image center RA'])
self.paramlist.append(
['dec', '', 'Option to manually set image center DEC'])
self.paramlist.append([
'searchradius', 5,
'Only search in indexes within "searchradius" (degrees) of the field center given by --ra and --dec (degrees)'
])
# confirm end of setup
self.log.debug('Setup: done')
def run(self):
""" Runs the data reduction algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
### Preparation
# construct a temp file name that astrometry will output
fp = tempfile.NamedTemporaryFile(suffix=".fits", dir=os.getcwd())
# split off path name, because a path that is too long causes remap to
# crash sometimes
outname = os.path.split(fp.name)[1]
fp.close()
# Add input file path to ouput file and make new name
outpath = os.path.split(self.datain.filename)[0]
outnewname = os.path.join(outpath, outname.replace('.fits', '.new'))
outwcsname = os.path.join(outpath, outname.replace('.fits', '.wcs'))
# Make sure input data exists as file
if not os.path.exists(self.datain.filename):
self.datain.save()
# Make command string
rawcommand = self.getarg('astrocmd') % (self.datain.filename, outname)
# get estimated RA and DEC center values from the config file or input FITS header
raopt = self.getarg('ra')
if raopt != '':
ra = Angle(raopt, unit=u.hour).degree
else:
try:
ra = Angle(self.datain.getheadval('RA'), unit=u.hour).degree
except:
ra = ''
decopt = self.getarg('dec')
if decopt != '':
dec = Angle(decopt, unit=u.deg).degree
else:
try:
dec = Angle(self.datain.getheadval('DEC'), unit=u.deg).degree
except:
dec = ''
if (ra != '') and (dec != ''):
# update command parameters to use these values
rawcommand = rawcommand + ' --ra %f --dec %f --radius %f' % (
ra, dec, self.getarg('searchradius'))
else:
self.log.debug(
'FITS header missing RA/DEC -> searching entire sky')
### Run Astrometry:
# This loop tries the downsample and param options until the fit is successful
# need either --scale-low 0.5 --scale-high 2.0 --sort-column FLUX
# or --guess-scale
downsamples = self.getarg('downsample')
paramoptions = self.getarg('paramoptions')
for option in range(len(downsamples) * len(paramoptions)):
#for downsample in self.getarg('downsample'):
downsample = downsamples[option % len(downsamples)]
paramoption = paramoptions[option // len(downsamples)]
# Add options to command
command = rawcommand + ' --downsample %d' % downsample + ' ' + paramoption
optionstring = "Downsample=%s Paramopts=%s" % (downsample,
paramoption[:10])
# Run the process - see note at the top of the file if using cron
process = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
self.log.debug('running command = %s' % command)
# Wait for the process to be finished or timeout to be reached
timeout = time.time() + self.getarg('timeout')
while time.time() < timeout and process.poll() == None:
time.sleep(1)
poll = process.poll()
if poll == None:
process.kill()
time.sleep(1)
poll = process.poll()
self.log.debug('command returns %d' % poll)
if poll == 0 and os.path.exists(outnewname):
self.log.debug('output file valid -> astrometry successful')
break
else:
self.log.debug('output file missing -> astrometry failed')
# Print the output from astrometry (cut if necessary)
if self.getarg('verbose'):
output = process.stdout.read().decode()
if len(output) > 1000:
outlines = output.split('\n')
output = outlines[:10] + ['...', '...'] + outlines[-7:]
output = '\n'.join(output)
self.log.debug(output)
### Post processing
# Read output file
self.dataout = DataFits(config=self.config)
self.log.debug('Opening astrometry.net output file %s' % outnewname)
try:
self.dataout.load(outnewname)
self.dataout.filename = self.datain.filename
except Exception as error:
self.log.error("Unable to open astrometry. output file = %s" %
outname)
raise error
self.log.debug('Successful parameter options = %s' % optionstring)
# Add history message
histmsg = 'Astrometry.Net: At downsample = %d, search took %d seconds' % (
downsample, time.time() - timeout + 300)
self.dataout.setheadval('HISTORY', histmsg)
# Add RA from astrometry
w = wcs.WCS(self.dataout.header)
n1 = float(self.dataout.header['NAXIS1'] / 2)
n2 = float(self.dataout.header['NAXIS2'] / 2)
ra, dec = w.all_pix2world(n1, n2, 1)
self.dataout.header['CRPIX1'] = n1
self.dataout.header['CRPIX2'] = n2
self.dataout.header['CRVAL1'] = float(ra)
self.dataout.header['CRVAL2'] = float(dec)
self.dataout.header['RA'] = Angle(ra, u.deg).to_string(unit=u.hour,
sep=':')
self.dataout.header['Dec'] = Angle(dec, u.deg).to_string(sep=':')
self.dataout.setheadval('HISTORY',
'Astrometry: Paramopts = ' + optionstring)
# Delete temporary files
if self.getarg('delete_temp'):
os.remove(outnewname)
os.remove(outwcsname)
self.log.debug('Run: Done')
