class StepMaskImage(StepParent):
""" HAWC Pipeline Step Parent Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '0.2' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'maskimage'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'MSK'
# Set Logger for this pipe step
self.log = logging.getLogger('pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
# confirm end of setup
self.log.debug('Setup: done')
def mask(self):
'''
Masks the input image file
'''
#Mask the image
image_data = self.datain.image.astype('int32').byteswap(
inplace=True).newbyteorder()
mask = vp.unit(data=image_data, header=self.datain.header)
mask.extract_bkg()
mask.subtract_bkg()
mask.set_primary('bkg_sub')
mask.extract_sources()
mask.build_sources_table()
mask.filter_sources(edgefrac=0.4)
mask.mask_sources()
masked_image = fits.PrimaryHDU(mask.primary, header=self.datain.header)
mask_fp = self.datain.filename.replace('.fits', '_MSK.fits')
masked_image.writeto(mask_fp)
return mask_fp
def run(self):
self.dataout = DataFits(config=self.config)
self.dataout.load(self.mask())
class StepSEP(StepParent):
""" HAWC Pipeline Step Parent Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '0.2' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name='sep'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'SEP'
# Set Logger for this pipe step
self.log = logging.getLogger('pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
# confirm end of setup
self.log.debug('Setup: done')
def source_extract(self):
bkg = sep.Background(self.datain.image.astype('int32'))
primary = self.datain.image.astype('int32') - bkg
objects = sep.extract(primary, 1.5, err=bkg.globalrms)
df = pd.DataFrame()
df['x'] = objects['x']; df['y'] = objects['y']; df['a'] = objects['a']; df['b'] = objects['b']; df['theta'] = objects['theta']; df['npix'] = objects['npix']; df['FLUX'] = objects['cflux']
table_image = Table(df.values)
print(repr(df))
table_fp = self.datain.filename.replace('.fits', '_TABLE.fits')
table_image.write(table_fp, format='fits')
return table_fp
def run(self):
self.dataout = DataFits(config=self.config)
self.dataout.load(self.source_extract())
self.dataout.header['RA'] = self.datain.header['RA']
self.dataout.header['Dec'] = self.datain.header['Dec']
self.dataout.save()
def test(self):
""" Test Pipe Step Flat Object: Runs basic tests
"""
# initial log message
self.log.info('Testing pipe step flat')
# get testin and a configuration
if self.config != None and len(
self.config) > 2: # i.e. if real config is loaded
testin = DataFits(config=self.config)
else:
testin = DataFits(config=self.testconf)
# load sample data
datain = DataFits(config=testin.config)
#infile = 'mode_chop/120207_000_00HA012.chop.dmd.fits'
infile = 'mode_chop/120306_000_00HA006.chop.dmd.fits'
#infile = 'mode_chop/120402_000_00HA035.chop.dmd.fits'
#infile = 'sharp/sharc2-048485.dmdsqr.fits'
testfile = os.path.join(datain.config['testing']['testpath'], infile)
#testfile = '/Users/berthoud/testfit.fits'
datain.load(testfile)
if False:
# change data (make complex number array with
# Re=0,1,2,3,4,5,6 . . . in time Im=0)
dataval = numpy.ones(datain.image.shape)
dataval[..., 1] = 0.0
inclist = numpy.arange(dataval.shape[0])
incshape = [1 + i - i for i in dataval.shape[0:-1]]
incshape[0] = dataval.shape[0]
inclist.shape = incshape
dataval[..., 0] = dataval[..., 0] * inclist
#datain.image=dataval
# run first flat
dataout = self(datain)
#print dataout.image[100,...] # print 100th image
#print dataout.image[range(0,dataval.shape[0],1000),0,0] # 1 val per img
dataout.save()
# final log message
self.log.info('Testing pipe step flat - Done')
class StepAstrometry(StepParent):
""" HAWC Pipeline Step Parent Object
The object is callable. It requires a valid configuration input
(file or object) when it runs.
"""
stepver = '0.2' # pipe step version
def setup(self):
""" ### Names and Parameters need to be Set Here ###
Sets the internal names for the function and for saved files.
Defines the input parameters for the current pipe step.
Setup() is called at the end of __init__
The parameters are stored in a list containing the following
information:
- name: The name for the parameter. This name is used when
calling the pipe step from command line or python shell.
It is also used to identify the parameter in the pipeline
configuration file.
- default: A default value for the parameter. If nothing, set
'' for strings, 0 for integers and 0.0 for floats
- help: A short description of the parameter.
"""
### Set Names
# Name of the pipeline reduction step
self.name = 'astrometry'
# Shortcut for pipeline reduction step and identifier for
# saved file names.
self.procname = 'WCS'
# Set Logger for this pipe step
self.log = logging.getLogger('pipe.step.%s' % self.name)
### Set Parameter list
# Clear Parameter list
self.paramlist = []
# Append parameters
self.paramlist.append([
'astrocmd', 'cp %s %s',
'Command to call astrometry, should contain 2' +
'string placeholders for intput and output ' + 'filepathname'
])
self.paramlist.append(
['verbose', False, 'log full astrometry output at DEBUG level'])
self.paramlist.append([
'delete_temp', False,
'Flag to delete temporary files generated by astrometry'
])
self.paramlist.append(
['downsample', [2], 'List of downsample factors to try'])
self.paramlist.append([
'paramoptions', ['--guess-scale'],
'Parameter groups to run if the command fails'
])
self.paramlist.append(
['timeout', 300, 'Timeout for running astrometry (seconds)'])
self.paramlist.append(
['ra', '', 'Option to manually set image center RA'])
self.paramlist.append(
['dec', '', 'Option to manually set image center DEC'])
self.paramlist.append([
'searchradius', 5,
'Only search in indexes within "searchradius" (degrees) of the field center given by --ra and --dec (degrees)'
])
# confirm end of setup
self.log.debug('Setup: done')
def run(self):
""" Runs the data reduction algorithm. The self.datain is run
through the code, the result is in self.dataout.
"""
### Preparation
# construct a temp file name that astrometry will output
fp = tempfile.NamedTemporaryFile(suffix=".fits", dir=os.getcwd())
# split off path name, because a path that is too long causes remap to
# crash sometimes
outname = os.path.split(fp.name)[1]
fp.close()
# Add input file path to ouput file and make new name
outpath = os.path.split(self.datain.filename)[0]
outnewname = os.path.join(outpath, outname.replace('.fits', '.new'))
outwcsname = os.path.join(outpath, outname.replace('.fits', '.wcs'))
# Make sure input data exists as file
if not os.path.exists(self.datain.filename):
self.datain.save()
# Make command string
rawcommand = self.getarg('astrocmd') % (self.datain.filename, outname)
# get estimated RA and DEC center values from the config file or input FITS header
raopt = self.getarg('ra')
if raopt != '':
ra = Angle(raopt, unit=u.hour).degree
else:
try:
ra = Angle(self.datain.getheadval('RA'), unit=u.hour).degree
except:
ra = ''
decopt = self.getarg('dec')
if decopt != '':
dec = Angle(decopt, unit=u.deg).degree
else:
try:
dec = Angle(self.datain.getheadval('DEC'), unit=u.deg).degree
except:
dec = ''
if (ra != '') and (dec != ''):
# update command parameters to use these values
rawcommand = rawcommand + ' --ra %f --dec %f --radius %f' % (
ra, dec, self.getarg('searchradius'))
else:
self.log.debug(
'FITS header missing RA/DEC -> searching entire sky')
### Run Astrometry:
# This loop tries the downsample and param options until the fit is successful
# need either --scale-low 0.5 --scale-high 2.0 --sort-column FLUX
# or --guess-scale
downsamples = self.getarg('downsample')
paramoptions = self.getarg('paramoptions')
for option in range(len(downsamples) * len(paramoptions)):
#for downsample in self.getarg('downsample'):
downsample = downsamples[option % len(downsamples)]
paramoption = paramoptions[option // len(downsamples)]
# Add options to command
command = rawcommand + ' --downsample %d' % downsample + ' ' + paramoption
optionstring = "Downsample=%s Paramopts=%s" % (downsample,
paramoption[:10])
# Run the process - see note at the top of the file if using cron
process = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
self.log.debug('running command = %s' % command)
# Wait for the process to be finished or timeout to be reached
timeout = time.time() + self.getarg('timeout')
while time.time() < timeout and process.poll() == None:
time.sleep(1)
poll = process.poll()
if poll == None:
process.kill()
time.sleep(1)
poll = process.poll()
self.log.debug('command returns %d' % poll)
if poll == 0 and os.path.exists(outnewname):
self.log.debug('output file valid -> astrometry successful')
break
else:
self.log.debug('output file missing -> astrometry failed')
# Print the output from astrometry (cut if necessary)
if self.getarg('verbose'):
output = process.stdout.read().decode()
if len(output) > 1000:
outlines = output.split('\n')
output = outlines[:10] + ['...', '...'] + outlines[-7:]
output = '\n'.join(output)
self.log.debug(output)
### Post processing
# Read output file
self.dataout = DataFits(config=self.config)
self.log.debug('Opening astrometry.net output file %s' % outnewname)
try:
self.dataout.load(outnewname)
self.dataout.filename = self.datain.filename
except Exception as error:
self.log.error("Unable to open astrometry. output file = %s" %
outname)
raise error
self.log.debug('Successful parameter options = %s' % optionstring)
# Add history message
histmsg = 'Astrometry.Net: At downsample = %d, search took %d seconds' % (
downsample, time.time() - timeout + 300)
self.dataout.setheadval('HISTORY', histmsg)
# Add RA from astrometry
w = wcs.WCS(self.dataout.header)
n1 = float(self.dataout.header['NAXIS1'] / 2)
n2 = float(self.dataout.header['NAXIS2'] / 2)
ra, dec = w.all_pix2world(n1, n2, 1)
self.dataout.header['CRPIX1'] = n1
self.dataout.header['CRPIX2'] = n2
self.dataout.header['CRVAL1'] = float(ra)
self.dataout.header['CRVAL2'] = float(dec)
self.dataout.header['RA'] = Angle(ra, u.deg).to_string(unit=u.hour,
sep=':')
self.dataout.header['Dec'] = Angle(dec, u.deg).to_string(sep=':')
self.dataout.setheadval('HISTORY',
'Astrometry: Paramopts = ' + optionstring)
# Delete temporary files
if self.getarg('delete_temp'):
os.remove(outnewname)
os.remove(outwcsname)
self.log.debug('Run: Done')
from darepype.drp import DataFits from astropy.io import fits config = '/Users/josh/pipeline/pipeline/Developments/stepwebastrometry/pipeconf_stonedge_auto.txt' fp = '/Users/josh/Desktop/pipeline_test/data/M5_r-band_60s_bin2_200711_053415_itzamna_seo_0_RAW_TABLE.fits' fts = DataFits(config=config) fts.load(fp) # print(repr(fits.HDUList(file=fp))) # fts.header['RA'] = 0 # fts.header['Dec'] = 0 print(repr(fts.header)) print(repr(fts.image)) # print(repr(fts.table))
# print(repr(dfits.header))
### OPTIONAL BUT RECOMMENDED: Check if all necessary files exist
error_flag = False
# Check if configuration file exists
if not os.path.exists(baseconfig):
print(
'ERROR: The config file you specified, %s,\n does NOT exist on your computer, fix "config" above'
% baseconfig)
error_flag = True
# Check if input files exist
for name in infilenames:
if not os.path.exists(name):
print(
'ERROR: The input file you specifed, %s,\n does NOT exist on your computer, fix "inputnames" above'
% name)
error_flag = True
if not error_flag:
print("All Good")
os.chdir('/Users/josh/pipeline/pipeline/Developments/stepwebastrometry')
step = StepWebAstrometry()
indata = []
for f in infilenames:
fits = DataFits(config=baseconfig)
fits.load(f)
indata.append(fits)
outdata = step(indata[0])
print('Done')
def loaddark(self):
""" Loads the dark information for the instrument settings
described in the header of self.datain.
If an appropriate file can not be found or the file is invalid
various warnings and errors are returned.
"""
### identify dark file to load, search if requested
darkfile = self.getarg('darkfile')
if darkfile == 'search' :
# get list of keywords to fit
fitkeys = self.getarg('fitkeys')
# check format (make first element uppercase)
try:
_ = fitkeys[0].upper()
except AttributeError:
# AttributeError if it's not a string
self.log.error('LoadDark: fitkeys config parameter is ' +
'incorrect format')
raise TypeError('fitkeys config parameter is incorrect format')
# get keywords from data
datakeys=[]
for fitkey in fitkeys:
datakeys.append(self.datain.getheadval(fitkey))
# get dark files from darkdir folder
darkfolder = self.getarg('darkfolder')
filelist=[name for name in os.listdir(darkfolder)
if name[0] != '.' and name.find('.fit') > -1 ]
if len(filelist) < 1:
self.log.error('LoadDark: no dark files found in folder ' +
darkfolder)
raise ValueError('no dark files found in folder ' +
darkfolder)
# match dark files, return best dark file
bestind = 0 # index of file with best match in filelist
bestfitn = 0 # number of keywords that match in best match
fileind = 0 # index for going through the list
while fileind < len(filelist) and bestfitn < len(fitkeys):
# load keys of dark file
filehead = pyfits.getheader(darkfolder+'/'+filelist[fileind])
filekeys=[]
for fitkey in fitkeys:
try:
filekeys.append(filehead[fitkey])
except KeyError:
self.log.warning('LoadDark: missing key [%s] in dark <%s>'
% (fitkey, filelist[fileind] ) )
filekeys.append('')
# determine number of fitting keywords
keyfitn=0
while ( keyfitn < len(fitkeys) and
datakeys[keyfitn] == filekeys[keyfitn] ):
keyfitn = keyfitn + 1
# compare with previous best find
if keyfitn > bestfitn:
bestind = fileind
bestfitn = keyfitn
fileind=fileind+1
darkfile = darkfolder+'/'+filelist[ bestind ]
if bestfitn < len(fitkeys):
self.log.warn('Could not find perfect dark file match')
self.log.warn('Best dark file found is <%s>'
% filelist[bestind] )
else:
self.log.info('Best dark file found is <%s>'
% filelist[bestind] )
self.fitkeys = fitkeys
self.keyvalues = datakeys
### load dark data into a DataFits object
self.darkfile = darkfile
darkdata = DataFits(config = self.config)
darkdata.load(self.darkfile)
### find dark image data arrays and store them
# get sizes of input data
datalist = self.getarg('datalist')
if len(datalist) > 0:
# There are items in datalist -> loop over items
self.darks = []
# Check if necessary number of images in darkdata
if len(darkdata.imgdata) < len(datalist):
msg = 'Number of images in dark file < '
msg += 'number of entries in datalist'
self.log.error('LoadDark: %s' % msg)
raise ValueError(msg)
# Loop through datalist items
for dataind in range(len(datalist)):
dataitem = datalist[dataind]
# Search for dataitem in self.datain images
if dataitem.upper() in self.datain.imgnames:
dataimg = self.datain.imageget(dataitem)
self.log.debug('LoadDark: Found image <%s> to subtract dark'
% dataitem)
# Search dataitem in self.table columns
else:
try:
dataimg = self.datain.table[dataitem]
self.log.debug('LoadDark: Found column <%s> to subtract dark'
% dataitem)
except:
msg = 'No data found for <%s>' % dataitem
self.log.error('LoadDark: %s' % msg)
raise ValueError(msg)
# Get dimensions - append dark to list
datasiz = dataimg.shape
if self.getarg('l0method').upper() != 'NO':
datasiz = datasiz[1:]
darksiz = darkdata.imgdata[dataind].shape
self.darks.append(darkdata.imgdata[dataind])
# Check dimension with dark data
print(datasiz,darksiz)
if len(datasiz) >= len(darksiz):
# Data has >= dimensions than dark -> compare
begind = len(datasiz)-len(darksiz)
if datasiz[begind:] != darksiz:
msg = 'Dark "%s" does not fit data - A' % dataitem
self.log.error('LoadDark: %s' % msg)
raise ValueError(msg)
else:
# More dimensions in dark data -> report error
msg = 'Dark "%s" does not fit data - B' % dataitem
self.log.error('LoadDark: %s' % dataitem)
raise ValueError(msg)
else:
# Empty datalist -> Subtract dark from first image in data with first dark
datasiz = self.datain.image.shape
if self.getarg('l0method').upper() != 'NO':
datasiz = datasiz[1:]
darksiz = darkdata.image.shape
if len(datasiz) >= len(darksiz):
# Data has >= dimensions than dark -> compare
begind = len(datasiz)-len(darksiz)
if datasiz[begind:] != darksiz:
self.log.error('LoadDark: Dark does not fit data - A')
raise ValueError('Dark does not fit data - A')
else:
# More dimensions in dark data -> report error
self.log.error('LoadDark: Dark does not fit data - B')
raise ValueError('Dark does not fit data - B')
self.log.debug('LoadDark: Subtracting Dark from first data image with first dark')
self.darks=[darkdata.image]
### make good pixel map for each detector and add to
#darktemp = numpy.abs(data[0,...])+numpy.abs(data[1,...])
#self.goodpixmap = numpy.ones(data.shape[1:])
#self.goodpixmap [ numpy.where(darktemp == 0.0)] = 0.0
# Finish up
self.darkloaded = 1
self.log.debug('LoadDark: done')
