class StepMaskImage(StepParent): """ HAWC Pipeline Step Parent Object The object is callable. It requires a valid configuration input (file or object) when it runs. """ stepver = '0.2' # pipe step version def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. Setup() is called at the end of __init__ The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name = 'maskimage' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'MSK' # Set Logger for this pipe step self.log = logging.getLogger('pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters # confirm end of setup self.log.debug('Setup: done') def mask(self): ''' Masks the input image file ''' #Mask the image image_data = self.datain.image.astype('int32').byteswap( inplace=True).newbyteorder() mask = vp.unit(data=image_data, header=self.datain.header) mask.extract_bkg() mask.subtract_bkg() mask.set_primary('bkg_sub') mask.extract_sources() mask.build_sources_table() mask.filter_sources(edgefrac=0.4) mask.mask_sources() masked_image = fits.PrimaryHDU(mask.primary, header=self.datain.header) mask_fp = self.datain.filename.replace('.fits', '_MSK.fits') masked_image.writeto(mask_fp) return mask_fp def run(self): self.dataout = DataFits(config=self.config) self.dataout.load(self.mask())
class StepSEP(StepParent): """ HAWC Pipeline Step Parent Object The object is callable. It requires a valid configuration input (file or object) when it runs. """ stepver = '0.2' # pipe step version def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. Setup() is called at the end of __init__ The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name='sep' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'SEP' # Set Logger for this pipe step self.log = logging.getLogger('pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters # confirm end of setup self.log.debug('Setup: done') def source_extract(self): bkg = sep.Background(self.datain.image.astype('int32')) primary = self.datain.image.astype('int32') - bkg objects = sep.extract(primary, 1.5, err=bkg.globalrms) df = pd.DataFrame() df['x'] = objects['x']; df['y'] = objects['y']; df['a'] = objects['a']; df['b'] = objects['b']; df['theta'] = objects['theta']; df['npix'] = objects['npix']; df['FLUX'] = objects['cflux'] table_image = Table(df.values) print(repr(df)) table_fp = self.datain.filename.replace('.fits', '_TABLE.fits') table_image.write(table_fp, format='fits') return table_fp def run(self): self.dataout = DataFits(config=self.config) self.dataout.load(self.source_extract()) self.dataout.header['RA'] = self.datain.header['RA'] self.dataout.header['Dec'] = self.datain.header['Dec'] self.dataout.save()
def test(self): """ Test Pipe Step Flat Object: Runs basic tests """ # initial log message self.log.info('Testing pipe step flat') # get testin and a configuration if self.config != None and len( self.config) > 2: # i.e. if real config is loaded testin = DataFits(config=self.config) else: testin = DataFits(config=self.testconf) # load sample data datain = DataFits(config=testin.config) #infile = 'mode_chop/120207_000_00HA012.chop.dmd.fits' infile = 'mode_chop/120306_000_00HA006.chop.dmd.fits' #infile = 'mode_chop/120402_000_00HA035.chop.dmd.fits' #infile = 'sharp/sharc2-048485.dmdsqr.fits' testfile = os.path.join(datain.config['testing']['testpath'], infile) #testfile = '/Users/berthoud/testfit.fits' datain.load(testfile) if False: # change data (make complex number array with # Re=0,1,2,3,4,5,6 . . . in time Im=0) dataval = numpy.ones(datain.image.shape) dataval[..., 1] = 0.0 inclist = numpy.arange(dataval.shape[0]) incshape = [1 + i - i for i in dataval.shape[0:-1]] incshape[0] = dataval.shape[0] inclist.shape = incshape dataval[..., 0] = dataval[..., 0] * inclist #datain.image=dataval # run first flat dataout = self(datain) #print dataout.image[100,...] # print 100th image #print dataout.image[range(0,dataval.shape[0],1000),0,0] # 1 val per img dataout.save() # final log message self.log.info('Testing pipe step flat - Done')
class StepAstrometry(StepParent): """ HAWC Pipeline Step Parent Object The object is callable. It requires a valid configuration input (file or object) when it runs. """ stepver = '0.2' # pipe step version def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. Setup() is called at the end of __init__ The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name = 'astrometry' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'WCS' # Set Logger for this pipe step self.log = logging.getLogger('pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters self.paramlist.append([ 'astrocmd', 'cp %s %s', 'Command to call astrometry, should contain 2' + 'string placeholders for intput and output ' + 'filepathname' ]) self.paramlist.append( ['verbose', False, 'log full astrometry output at DEBUG level']) self.paramlist.append([ 'delete_temp', False, 'Flag to delete temporary files generated by astrometry' ]) self.paramlist.append( ['downsample', [2], 'List of downsample factors to try']) self.paramlist.append([ 'paramoptions', ['--guess-scale'], 'Parameter groups to run if the command fails' ]) self.paramlist.append( ['timeout', 300, 'Timeout for running astrometry (seconds)']) self.paramlist.append( ['ra', '', 'Option to manually set image center RA']) self.paramlist.append( ['dec', '', 'Option to manually set image center DEC']) self.paramlist.append([ 'searchradius', 5, 'Only search in indexes within "searchradius" (degrees) of the field center given by --ra and --dec (degrees)' ]) # confirm end of setup self.log.debug('Setup: done') def run(self): """ Runs the data reduction algorithm. The self.datain is run through the code, the result is in self.dataout. """ ### Preparation # construct a temp file name that astrometry will output fp = tempfile.NamedTemporaryFile(suffix=".fits", dir=os.getcwd()) # split off path name, because a path that is too long causes remap to # crash sometimes outname = os.path.split(fp.name)[1] fp.close() # Add input file path to ouput file and make new name outpath = os.path.split(self.datain.filename)[0] outnewname = os.path.join(outpath, outname.replace('.fits', '.new')) outwcsname = os.path.join(outpath, outname.replace('.fits', '.wcs')) # Make sure input data exists as file if not os.path.exists(self.datain.filename): self.datain.save() # Make command string rawcommand = self.getarg('astrocmd') % (self.datain.filename, outname) # get estimated RA and DEC center values from the config file or input FITS header raopt = self.getarg('ra') if raopt != '': ra = Angle(raopt, unit=u.hour).degree else: try: ra = Angle(self.datain.getheadval('RA'), unit=u.hour).degree except: ra = '' decopt = self.getarg('dec') if decopt != '': dec = Angle(decopt, unit=u.deg).degree else: try: dec = Angle(self.datain.getheadval('DEC'), unit=u.deg).degree except: dec = '' if (ra != '') and (dec != ''): # update command parameters to use these values rawcommand = rawcommand + ' --ra %f --dec %f --radius %f' % ( ra, dec, self.getarg('searchradius')) else: self.log.debug( 'FITS header missing RA/DEC -> searching entire sky') ### Run Astrometry: # This loop tries the downsample and param options until the fit is successful # need either --scale-low 0.5 --scale-high 2.0 --sort-column FLUX # or --guess-scale downsamples = self.getarg('downsample') paramoptions = self.getarg('paramoptions') for option in range(len(downsamples) * len(paramoptions)): #for downsample in self.getarg('downsample'): downsample = downsamples[option % len(downsamples)] paramoption = paramoptions[option // len(downsamples)] # Add options to command command = rawcommand + ' --downsample %d' % downsample + ' ' + paramoption optionstring = "Downsample=%s Paramopts=%s" % (downsample, paramoption[:10]) # Run the process - see note at the top of the file if using cron process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) self.log.debug('running command = %s' % command) # Wait for the process to be finished or timeout to be reached timeout = time.time() + self.getarg('timeout') while time.time() < timeout and process.poll() == None: time.sleep(1) poll = process.poll() if poll == None: process.kill() time.sleep(1) poll = process.poll() self.log.debug('command returns %d' % poll) if poll == 0 and os.path.exists(outnewname): self.log.debug('output file valid -> astrometry successful') break else: self.log.debug('output file missing -> astrometry failed') # Print the output from astrometry (cut if necessary) if self.getarg('verbose'): output = process.stdout.read().decode() if len(output) > 1000: outlines = output.split('\n') output = outlines[:10] + ['...', '...'] + outlines[-7:] output = '\n'.join(output) self.log.debug(output) ### Post processing # Read output file self.dataout = DataFits(config=self.config) self.log.debug('Opening astrometry.net output file %s' % outnewname) try: self.dataout.load(outnewname) self.dataout.filename = self.datain.filename except Exception as error: self.log.error("Unable to open astrometry. output file = %s" % outname) raise error self.log.debug('Successful parameter options = %s' % optionstring) # Add history message histmsg = 'Astrometry.Net: At downsample = %d, search took %d seconds' % ( downsample, time.time() - timeout + 300) self.dataout.setheadval('HISTORY', histmsg) # Add RA from astrometry w = wcs.WCS(self.dataout.header) n1 = float(self.dataout.header['NAXIS1'] / 2) n2 = float(self.dataout.header['NAXIS2'] / 2) ra, dec = w.all_pix2world(n1, n2, 1) self.dataout.header['CRPIX1'] = n1 self.dataout.header['CRPIX2'] = n2 self.dataout.header['CRVAL1'] = float(ra) self.dataout.header['CRVAL2'] = float(dec) self.dataout.header['RA'] = Angle(ra, u.deg).to_string(unit=u.hour, sep=':') self.dataout.header['Dec'] = Angle(dec, u.deg).to_string(sep=':') self.dataout.setheadval('HISTORY', 'Astrometry: Paramopts = ' + optionstring) # Delete temporary files if self.getarg('delete_temp'): os.remove(outnewname) os.remove(outwcsname) self.log.debug('Run: Done')
from darepype.drp import DataFits from astropy.io import fits config = '/Users/josh/pipeline/pipeline/Developments/stepwebastrometry/pipeconf_stonedge_auto.txt' fp = '/Users/josh/Desktop/pipeline_test/data/M5_r-band_60s_bin2_200711_053415_itzamna_seo_0_RAW_TABLE.fits' fts = DataFits(config=config) fts.load(fp) # print(repr(fits.HDUList(file=fp))) # fts.header['RA'] = 0 # fts.header['Dec'] = 0 print(repr(fts.header)) print(repr(fts.image)) # print(repr(fts.table))
# print(repr(dfits.header)) ### OPTIONAL BUT RECOMMENDED: Check if all necessary files exist error_flag = False # Check if configuration file exists if not os.path.exists(baseconfig): print( 'ERROR: The config file you specified, %s,\n does NOT exist on your computer, fix "config" above' % baseconfig) error_flag = True # Check if input files exist for name in infilenames: if not os.path.exists(name): print( 'ERROR: The input file you specifed, %s,\n does NOT exist on your computer, fix "inputnames" above' % name) error_flag = True if not error_flag: print("All Good") os.chdir('/Users/josh/pipeline/pipeline/Developments/stepwebastrometry') step = StepWebAstrometry() indata = [] for f in infilenames: fits = DataFits(config=baseconfig) fits.load(f) indata.append(fits) outdata = step(indata[0]) print('Done')
def loaddark(self): """ Loads the dark information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. """ ### identify dark file to load, search if requested darkfile = self.getarg('darkfile') if darkfile == 'search' : # get list of keywords to fit fitkeys = self.getarg('fitkeys') # check format (make first element uppercase) try: _ = fitkeys[0].upper() except AttributeError: # AttributeError if it's not a string self.log.error('LoadDark: fitkeys config parameter is ' + 'incorrect format') raise TypeError('fitkeys config parameter is incorrect format') # get keywords from data datakeys=[] for fitkey in fitkeys: datakeys.append(self.datain.getheadval(fitkey)) # get dark files from darkdir folder darkfolder = self.getarg('darkfolder') filelist=[name for name in os.listdir(darkfolder) if name[0] != '.' and name.find('.fit') > -1 ] if len(filelist) < 1: self.log.error('LoadDark: no dark files found in folder ' + darkfolder) raise ValueError('no dark files found in folder ' + darkfolder) # match dark files, return best dark file bestind = 0 # index of file with best match in filelist bestfitn = 0 # number of keywords that match in best match fileind = 0 # index for going through the list while fileind < len(filelist) and bestfitn < len(fitkeys): # load keys of dark file filehead = pyfits.getheader(darkfolder+'/'+filelist[fileind]) filekeys=[] for fitkey in fitkeys: try: filekeys.append(filehead[fitkey]) except KeyError: self.log.warning('LoadDark: missing key [%s] in dark <%s>' % (fitkey, filelist[fileind] ) ) filekeys.append('') # determine number of fitting keywords keyfitn=0 while ( keyfitn < len(fitkeys) and datakeys[keyfitn] == filekeys[keyfitn] ): keyfitn = keyfitn + 1 # compare with previous best find if keyfitn > bestfitn: bestind = fileind bestfitn = keyfitn fileind=fileind+1 darkfile = darkfolder+'/'+filelist[ bestind ] if bestfitn < len(fitkeys): self.log.warn('Could not find perfect dark file match') self.log.warn('Best dark file found is <%s>' % filelist[bestind] ) else: self.log.info('Best dark file found is <%s>' % filelist[bestind] ) self.fitkeys = fitkeys self.keyvalues = datakeys ### load dark data into a DataFits object self.darkfile = darkfile darkdata = DataFits(config = self.config) darkdata.load(self.darkfile) ### find dark image data arrays and store them # get sizes of input data datalist = self.getarg('datalist') if len(datalist) > 0: # There are items in datalist -> loop over items self.darks = [] # Check if necessary number of images in darkdata if len(darkdata.imgdata) < len(datalist): msg = 'Number of images in dark file < ' msg += 'number of entries in datalist' self.log.error('LoadDark: %s' % msg) raise ValueError(msg) # Loop through datalist items for dataind in range(len(datalist)): dataitem = datalist[dataind] # Search for dataitem in self.datain images if dataitem.upper() in self.datain.imgnames: dataimg = self.datain.imageget(dataitem) self.log.debug('LoadDark: Found image <%s> to subtract dark' % dataitem) # Search dataitem in self.table columns else: try: dataimg = self.datain.table[dataitem] self.log.debug('LoadDark: Found column <%s> to subtract dark' % dataitem) except: msg = 'No data found for <%s>' % dataitem self.log.error('LoadDark: %s' % msg) raise ValueError(msg) # Get dimensions - append dark to list datasiz = dataimg.shape if self.getarg('l0method').upper() != 'NO': datasiz = datasiz[1:] darksiz = darkdata.imgdata[dataind].shape self.darks.append(darkdata.imgdata[dataind]) # Check dimension with dark data print(datasiz,darksiz) if len(datasiz) >= len(darksiz): # Data has >= dimensions than dark -> compare begind = len(datasiz)-len(darksiz) if datasiz[begind:] != darksiz: msg = 'Dark "%s" does not fit data - A' % dataitem self.log.error('LoadDark: %s' % msg) raise ValueError(msg) else: # More dimensions in dark data -> report error msg = 'Dark "%s" does not fit data - B' % dataitem self.log.error('LoadDark: %s' % dataitem) raise ValueError(msg) else: # Empty datalist -> Subtract dark from first image in data with first dark datasiz = self.datain.image.shape if self.getarg('l0method').upper() != 'NO': datasiz = datasiz[1:] darksiz = darkdata.image.shape if len(datasiz) >= len(darksiz): # Data has >= dimensions than dark -> compare begind = len(datasiz)-len(darksiz) if datasiz[begind:] != darksiz: self.log.error('LoadDark: Dark does not fit data - A') raise ValueError('Dark does not fit data - A') else: # More dimensions in dark data -> report error self.log.error('LoadDark: Dark does not fit data - B') raise ValueError('Dark does not fit data - B') self.log.debug('LoadDark: Subtracting Dark from first data image with first dark') self.darks=[darkdata.image] ### make good pixel map for each detector and add to #darktemp = numpy.abs(data[0,...])+numpy.abs(data[1,...]) #self.goodpixmap = numpy.ones(data.shape[1:]) #self.goodpixmap [ numpy.where(darktemp == 0.0)] = 0.0 # Finish up self.darkloaded = 1 self.log.debug('LoadDark: done')