class StepMaskImage(StepParent): """ HAWC Pipeline Step Parent Object The object is callable. It requires a valid configuration input (file or object) when it runs. """ stepver = '0.2' # pipe step version def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. Setup() is called at the end of __init__ The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name = 'maskimage' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'MSK' # Set Logger for this pipe step self.log = logging.getLogger('pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters # confirm end of setup self.log.debug('Setup: done') def mask(self): ''' Masks the input image file ''' #Mask the image image_data = self.datain.image.astype('int32').byteswap( inplace=True).newbyteorder() mask = vp.unit(data=image_data, header=self.datain.header) mask.extract_bkg() mask.subtract_bkg() mask.set_primary('bkg_sub') mask.extract_sources() mask.build_sources_table() mask.filter_sources(edgefrac=0.4) mask.mask_sources() masked_image = fits.PrimaryHDU(mask.primary, header=self.datain.header) mask_fp = self.datain.filename.replace('.fits', '_MSK.fits') masked_image.writeto(mask_fp) return mask_fp def run(self): self.dataout = DataFits(config=self.config) self.dataout.load(self.mask())
def __init__(self): """ Constructor: Initialize data objects and variables """ # call superclass constructor (calls setup) super(StepBiasDarkFlat, self).__init__() # bias values self.biasloaded = False # indicates if bias has been loaded self.bias = None # CCD data object containing arrays with bias values self.biasdata = DataFits() # Pipedata object containing the bias file # bias file info and header keywords to fit self.biasname = '' # name of selected bias file self.biasfitkeys = [] # FITS keywords that are present in bias self.biaskeyvalues = [] # values of FITS keywords (from data file) # dark values self.darkloaded = False # indicates if dark has been loaded self.dark = None # CCD data object containing arrays with dark values self.darkdata = DataFits() # Pipedata object containing the dark file # dark file info and header keywords to fit self.darkname = '' # name of selected dark file self.darkfitkeys = [] # FITS keywords that have to fit for dark self.darkkeyvalues = [] # values of FITS keywords (from data file) # flat values self.flatloaded = False # indicates if flat has been loaded self.flat = None # CCD data object containing arrays with flat values self.flatdata = DataFits() # Pipedata object containing the flat file # flat file info and header keywords to fit self.flatname = '' # name of selected flat file self.flatfitkeys = [] # FITS keywords that have to fit for flat self.flatkeyvalues = [] # values of flat keywords (from data file) # set configuration self.log.debug('Init: done')
class StepSEP(StepParent): """ HAWC Pipeline Step Parent Object The object is callable. It requires a valid configuration input (file or object) when it runs. """ stepver = '0.2' # pipe step version def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. Setup() is called at the end of __init__ The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name='sep' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'SEP' # Set Logger for this pipe step self.log = logging.getLogger('pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters # confirm end of setup self.log.debug('Setup: done') def source_extract(self): bkg = sep.Background(self.datain.image.astype('int32')) primary = self.datain.image.astype('int32') - bkg objects = sep.extract(primary, 1.5, err=bkg.globalrms) df = pd.DataFrame() df['x'] = objects['x']; df['y'] = objects['y']; df['a'] = objects['a']; df['b'] = objects['b']; df['theta'] = objects['theta']; df['npix'] = objects['npix']; df['FLUX'] = objects['cflux'] table_image = Table(df.values) print(repr(df)) table_fp = self.datain.filename.replace('.fits', '_TABLE.fits') table_image.write(table_fp, format='fits') return table_fp def run(self): self.dataout = DataFits(config=self.config) self.dataout.load(self.source_extract()) self.dataout.header['RA'] = self.datain.header['RA'] self.dataout.header['Dec'] = self.datain.header['Dec'] self.dataout.save()
def run(self): """ Runs the combining algorithm. The self.datain is run through the code, the result is in self.dataout. """ # Find master bias to subtract from master dark biaslist = self.loadauxname('bias', multi=False) if (len(biaslist) == 0): self.log.error('No bias calibration frames found.') self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True) # Create empy list for filenames of loaded frames filelist = [] for fin in self.datain: self.log.debug("Input filename = %s" % fin.filename) filelist.append(fin.filename) # Make a dummy dataout self.dataout = DataFits(config=self.config) if len(self.datain) == 0: self.log.error('Flat calibration frame not found.') raise RuntimeError('No flat file(s) loaded') self.log.debug('Creating master flat frame...') # Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias if (len(filelist) == 1): self.dark = ccdproc.CCDData.read(filelist[0], unit='adu', relax=True) self.dark = ccdproc.subtract_bias(self.dark, self.bias, add_keyword=False) else: darklist = [] for i in filelist: dark = ccdproc.CCDData.read(i, unit='adu', relax=True) darksubbias = ccdproc.subtract_bias(dark, self.bias, add_keyword=False) darklist.append(darksubbias) self.dark = ccdproc.combine(darklist, method=self.getarg('combinemethod'), unit='adu', add_keyword=True) # set output header, put image into output self.dataout.header = self.datain[0].header self.dataout.imageset(self.dark) # rename output filename outputfolder = self.getarg('outputfolder') if outputfolder != '': outputfolder = os.path.expandvars(outputfolder) self.dataout.filename = os.path.join(outputfolder, os.path.split(filelist[0])[1]) else: self.dataout.filename = filelist[0] # Add history self.dataout.setheadval('HISTORY', 'MasterDark: %d files used' % len(filelist))
def run(self): """ Runs the data reduction algorithm. The self.datain is run through the code, the result is in self.dataout. """ self.dataout = DataFits(config=self.config) self.dataout = self.datain.copy() self.astrometrymaster() # Update RA/Dec from astrometry self.dataout.header.update(self.wcs_out) try: w = wcs.WCS(self.dataout.header) n1 = float( self.dataout.header['NAXIS1']/2 ) n2 = float( self.dataout.header['NAXIS2']/2 ) ra, dec = w.all_pix2world(n1, n2, 1) # No update because update may affect accuracy of WCS solution # self.dataout.header['CRPIX1']=n1 # self.dataout.header['CRPIX2']=n2 # self.dataout.header['CRVAL1']=float(ra) # self.dataout.header['CRVAL2']=float(dec) self.dataout.header['RA'] = Angle(ra, u.deg).to_string(unit=u.hour, sep=':') self.dataout.header['Dec'] = Angle(dec, u.deg).to_string(sep=':') except: self.log.error('Run: Could not update RA/Dec from Astrometry') else: self.log.debug('Run: Updated RA/Dec from Astrometry') self.log.debug('Run: Done')
def __init__(self): """ Constructor: Initialize data objects and variables """ # call superclass constructor (calls setup) super(StepFlat, self).__init__() # list of data and flats self.datalist = [] # used in run() for every new input data file # flat values self.flatloaded = 0 # indicates if flat has been loaded self.flats = [] # list containing arrays with flat values self.flatdata = DataFits() # Pipedata object containing the flat file # flat file info and header keywords to fit self.flatfile = '' # name of selected flat file self.fitkeys = [] # FITS keywords that have to fit self.keyvalues = [ ] # values of the keywords (from the first data file) # set configuration self.log.debug('Init: done')
def run(self): """ Runs the combining algorithm. The self.datain is run through the code, the result is in self.dataout. """ filelist = [] for fin in self.datain: self.log.debug("Input filename = %s" % fin.filename) filelist.append(fin.filename) # Make a dummy dataout self.dataout = DataFits(config=self.config) if len(self.datain) == 0: self.log.error('Bias calibration frame not found.') raise RuntimeError('No bias file(s) loaded') # self.log.debug('Creating master bias frame...') # if there is just one, use it as biasfile or else combine all to make a master bias if (len(filelist) == 1): self.bias = ccdproc.CCDData.read(filelist[0], unit='adu', relax=True) else: self.bias = ccdproc.combine(filelist, method=self.getarg('combinemethod'), unit='adu', add_keyword=True) # set output header, put image into output self.dataout.header = self.datain[0].header self.dataout.imageset(self.bias) # rename output filename outputfolder = self.getarg('outputfolder') if outputfolder != '': outputfolder = os.path.expandvars(outputfolder) self.dataout.filename = os.path.join(outputfolder, os.path.split(filelist[0])[1]) else: self.dataout.filename = filelist[0] # Add history self.dataout.setheadval('HISTORY', 'MasterBias: %d files used' % len(filelist))
def test(self): """ Test Pipe Step Flat Object: Runs basic tests """ # initial log message self.log.info('Testing pipe step flat') # get testin and a configuration if self.config != None and len( self.config) > 2: # i.e. if real config is loaded testin = DataFits(config=self.config) else: testin = DataFits(config=self.testconf) # load sample data datain = DataFits(config=testin.config) #infile = 'mode_chop/120207_000_00HA012.chop.dmd.fits' infile = 'mode_chop/120306_000_00HA006.chop.dmd.fits' #infile = 'mode_chop/120402_000_00HA035.chop.dmd.fits' #infile = 'sharp/sharc2-048485.dmdsqr.fits' testfile = os.path.join(datain.config['testing']['testpath'], infile) #testfile = '/Users/berthoud/testfit.fits' datain.load(testfile) if False: # change data (make complex number array with # Re=0,1,2,3,4,5,6 . . . in time Im=0) dataval = numpy.ones(datain.image.shape) dataval[..., 1] = 0.0 inclist = numpy.arange(dataval.shape[0]) incshape = [1 + i - i for i in dataval.shape[0:-1]] incshape[0] = dataval.shape[0] inclist.shape = incshape dataval[..., 0] = dataval[..., 0] * inclist #datain.image=dataval # run first flat dataout = self(datain) #print dataout.image[100,...] # print 100th image #print dataout.image[range(0,dataval.shape[0],1000),0,0] # 1 val per img dataout.save() # final log message self.log.info('Testing pipe step flat - Done')
def run(self): """ Runs the calibrating algorithm. The calibrated data is returned in self.dataout """ ### Preparation # Load bias files if necessary if not self.biasloaded or self.getarg('reload'): self.loadbias() # Else: check data for correct instrument configuration - currently not in use(need improvement) else: for keyind in range(len(self.biasfitkeys)): if self.biaskeyvalues[keyind] != self.datain.getheadval( self.biasfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.biasfitkeys[keyind]) # Load dark files if necessary if not self.darkloaded or self.getarg('reload'): self.loaddark() # Else: check data for correct instrument configuration else: for keyind in range(len(self.darkfitkeys)): if self.darkkeyvalues[keyind] != self.datain.getheadval( self.darkfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.darkfitkeys[keyind]) # Load flat files if necessary if not self.flatloaded or self.getarg('reload'): self.loadflat() # Else: check data for correct instrument configuration else: for keyind in range(len(self.flatfitkeys)): if self.flatkeyvalues[keyind] != self.datain.getheadval( self.flatfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.flatfitkeys[keyind]) # in the config file, set the 'intermediate' variable to either true or false to enable # saving of intermediate steps saveIntermediateSteps = self.config['biasdarkflat']['intermediate'] self.dataout = DataFits(config=self.datain.config) #convert self.datain to CCD Data object image = CCDData(self.datain.image, unit='adu') image.header = self.datain.header # subtract bias from image image = self.subtract_bias(image, self.bias) if (saveIntermediateSteps == "true"): self.dataout.imageset(image.data, imagename="BIAS") # self.dataout.setheadval('DATATYPE','IMAGE', dataname="BIAS") self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname, dataname="BIAS") # subtract dark from image image = self.subtract_dark(image, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second) if (saveIntermediateSteps == "true"): self.dataout.imageset(image.data, imagename="DARK") # self.dataout.setheadval('DATATYPE','IMAGE', dataname="DARK") self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname, dataname="DARK") self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname, dataname="DARK") # apply flat correction to image image = self.flat_correct(image, self.flat) # if separating bias,dark,flat steps , save the flat-corrected portion into its own hdu if (saveIntermediateSteps == "true"): self.dataout.imageset(image.data, imagename="FLAT") # self.dataout.setheadval('DATATYPE','IMAGE', dataname="FLAT") self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname, dataname="FLAT") self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname, dataname="FLAT") self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname, dataname="FLAT") else: # copy calibrated image into self.dataout self.dataout.image = image.data self.dataout.header = self.datain.header ### Finish - cleanup # Update DATATYPE self.dataout.setheadval('DATATYPE', 'IMAGE') # Add bias, dark files to History self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname) self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname) self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname) self.dataout.filename = self.datain.filename
def run(self): """ Runs the combining algorithm. The self.datain is run through the code, the result is in self.dataout. """ ''' Select 3 input dataset to use, store in datause ''' #Store number of inputs num_inputs = len(self.datain) # Create variable to hold input files # Copy input to output header and filename datause = [] self.log.debug('Number of input files = %d' % num_inputs) # Ensure datause has 3 elements irrespective of number of input files if num_inputs == 0: # Raise exception for no input raise ValueError('No input') elif num_inputs == 1: datause = [self.datain[0], self.datain[0], self.datain[0]] elif num_inputs == 2: datause = [self.datain[0], self.datain[1], self.datain[1]] else: # If inputs exceed 2 in number # Here we know there are at least 3 files ilist = [] # Make empty lists for each filter rlist = [] glist = [] other = [] for element in self.datain: # Loop through the input files and add to the lists fname = element.filename.lower() if 'i-band' in fname or 'iband' in fname or 'iprime' in fname: ilist.append(element) elif 'r-band' in fname or 'rband' in fname or 'rprime' in fname: rlist.append(element) elif 'g-band' in fname or 'gband' in fname or 'gprime' in fname: glist.append(element) else: other.append(element) continue self.log.debug( 'len(ilist) = %d, len(rlist) = %d, len(glist) = %d' % (len(ilist), len(rlist), len(glist))) # If there is at least one i-, r-, and g-band filter found in self.datain (best case) if len(ilist) >= 1 and len(rlist) >= 1 and len(glist) >= 1: # The first image from each filter list will be reduced in the correct order. datause = [ilist[0], rlist[0], glist[0]] elif len(ilist) == 0 and len(rlist) >= 1 and len(glist) >= 1: # Cases where there is no ilist if len(rlist) > len(glist): datause = [rlist[0], rlist[1], glist[0]] else: datause = [rlist[0], glist[0], glist[1]] elif len(glist) == 0 and len(rlist) >= 1 and len(ilist) >= 1: # Cases where there is no glist if len(rlist) > len(ilist): datause = [rlist[0], rlist[1], ilist[0]] else: datause = [rlist[0], ilist[0], ilist[1]] elif len(ilist) == 0 and len(rlist) >= 1 and len(glist) >= 1: # Cases where there is no rlist if len(ilist) > len(glist): datause = [ilist[0], ilist[1], glist[0]] else: datause = [ilist[0], glist[0], glist[1]] elif len(rlist) == 0 and len(glist) == 0: # Case where there is only ilist datause = [ilist[0], ilist[1], ilist[2]] elif len(rlist) == 0 and len(ilist) == 0: # Case where there is only glist datause = [glist[0], glist[1], glist[2]] elif len(ilist) == 0 and len(glist) == 0: # Case where there is only rlist datause = [rlist[0], rlist[1], rlist[2]] self.log.debug( 'Files used: R = %s G = %s B = %s' % (datause[0].filename, datause[1].filename, datause[2].filename)) self.dataout = DataFits(config=self.config) self.dataout.header = datause[0].header self.dataout.filename = datause[0].filename img = datause[0].image img1 = datause[1].image img2 = datause[2].image ''' Finding Min/Max scaling values ''' # Create a Data Cube with floats datacube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype=float) # Enter the image data into the cube so an absolute max can be found datacube[:, :, 0] = img datacube[:, :, 1] = img1 datacube[:, :, 2] = img2 # Find how many data points are in the data cube datalength = img.shape[0] * img.shape[1] * 3 # Create a 1-dimensional array with all the data, then sort it datacube.shape = (datalength, ) datacube.sort() # Now use arrays for each filter to find separate min values rarray = img.copy() garray = img1.copy() barray = img2.copy() # Shape and sort the arrays arrlength = img.shape[0] * img.shape[1] rarray.shape = (arrlength, ) rarray.sort() garray.shape = (arrlength, ) garray.sort() barray.shape = (arrlength, ) barray.sort() # Find the min/max percentile values in the data for scaling # Values are determined by parameters in the pipe configuration file minpercent = int(arrlength * self.getarg('minpercent')) maxpercent = int(datalength * self.getarg('maxpercent')) # Find the final data values to use for scaling from the image data rminsv = rarray[minpercent] #sv stands for "scalevalue" gminsv = garray[minpercent] bminsv = barray[minpercent] maxsv = datacube[maxpercent] self.log.info(' Scale min r/g/b: %f/%f/%f' % (rminsv, gminsv, bminsv)) self.log.info(' Scale max: %f' % maxsv) # The same min/max values will be used to scale all filters ''' Finished Finding scaling values ''' ''' Combining Function ''' # Make new cube with the proper data type for color images (uint8) # Use square root (sqrt) scaling for each filter # log or asinh scaling is also available #astropy.vidualizations.SqrtStretch() imgcube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype='uint8') minsv = [rminsv, gminsv, bminsv] for i in range(3): # Make normalization function norm = simple_norm(datause[i].image, 'sqrt', min_cut=minsv[i], max_cut=maxsv) # Apply it imgcube[:, :, i] = norm(datause[i].image) * 255. self.dataout.image = imgcube # Create variable containing all the scaled image data imgcolor = Image.fromarray(self.dataout.image, mode='RGB') # Save colored image as a .tif file (without the labels) imgcolortif = imgcube.copy() imgcolortif.astype('uint16') ### tiff.imsave('%s.tif' % self.dataout.filenamebase, imgcolortif) ''' End of combining function ''' ''' Add a Label to the Image ''' draw = ImageDraw.Draw(imgcolor) # Use a variable to make the positions and size of text relative imgwidth = img.shape[1] imgheight = img.shape[0] # Open Sans-Serif Font with a size relative to the picture size try: # This should work on Linux font = ImageFont.truetype( '/usr/share/fonts/liberation/LiberationSans-Regular.ttf', imgheight // 41) except: try: # This should work on Mac font = ImageFont.truetype('/Library/Fonts/Arial Unicode.ttf', imgheight // 41) except: try: # This should work on Windows font = ImageFont.truetype('C:\\Windows\\Fonts\\arial.ttf', imgheight // 41) except: # This should work in Colab font = ImageFont.truetype( '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf', imgheight // 41) # If this still doesn't work - then add more code to make it run on YOUR system # Use the beginning of the FITS filename as the object name filename = os.path.split(self.dataout.filename)[-1] try: objectname = filename.split('_')[0] objectname = objectname[0].upper() + objectname[1:] except Exception: objectname = 'Unknown.' objectname = 'Object: %s' % objectname # Read labels at their respective position (kept relative to image size) # Left corner: object, observer, observatory # Right corner: Filters used for red, green, and blue colors draw.text((imgwidth / 100, imgheight / 1.114), objectname, (255, 255, 255), font=font) # Read FITS keywords for the observer, observatory, and filters if 'OBSERVER' in self.dataout.header: observer = 'Observer: %s' % self.dataout.getheadval('OBSERVER') draw.text((imgwidth / 100, imgheight / 1.073), observer, (255, 255, 255), font=font) if 'OBSERVAT' in self.dataout.header: observatory = 'Observatory: %s' % self.dataout.getheadval( 'OBSERVAT') draw.text((imgwidth / 100, imgheight / 1.035), observatory, (255, 255, 255), font=font) if 'FILTER' in datause[0].header: red = 'R: %s' % datause[0].getheadval('FILTER') draw.text((imgwidth / 1.15, imgheight / 1.114), red, (255, 255, 255), font=font) if 'FILTER' in datause[1].header: green = 'G: %s' % datause[1].getheadval('FILTER') draw.text((imgwidth / 1.15, imgheight / 1.073), green, (255, 255, 255), font=font) if 'FILTER' in datause[2].header: blue = 'B: %s' % datause[2].getheadval('FILTER') draw.text((imgwidth / 1.15, imgheight / 1.035), blue, (255, 255, 255), font=font) # Make image name imgname = self.dataout.filenamebegin if imgname[-1] in '_-,.': imgname = imgname[:-1] imgname += '.jpg' # Save the completed image imgcolor.save(imgname) self.log.info('Saving file %sjpg' % self.dataout.filenamebegin) ''' End of Label Code ''' # Set complete flag self.dataout.setheadval('COMPLETE', 1, 'Data Reduction Pipe: Complete Data Flag')
def run(self): """ Runs the combining algorithm. The self.datain is run through the code, the result is in jpeg_dataout. """ ''' Select 3 input dataset to use, store in datause ''' #Store number of inputs num_inputs = len(self.datain) # Create variable to hold input files # Copy input to output header and filename datause = [None, None, None] self.log.debug('Number of input files = %d' % num_inputs) if num_inputs == 0: # Raise exception for no input raise ValueError('No input') elif num_inputs == 1: datause = [self.datain[0], self.datain[0], self.datain[0]] elif num_inputs == 2: datause = [self.datain[0], self.datain[0], self.datain[1]] else: filterorder_list = self.getarg('filterorder').split('|') filterprefs_list = self.getarg('filterprefs').split('|') datain_filter_list = [ element.getheadval('filter') for element in self.datain ] used_filter_flags = [False] * len(self.datain) if len(filterprefs_list) != 3: self.log.error( 'Invalid number of preferred filters provided (should be 3): ' + self.getarg('filterprefs')) else: # Locate data matching the filters specified in filterprefs for i, preferred_filter in enumerate(filterprefs_list): for j, element in enumerate(self.datain): if element.getheadval('filter') == preferred_filter: datause[i] = element used_filter_flags[j] = True break filterorder_walker = 0 for i, channel in enumerate(datause): if channel == None: for ordered_filter in filterorder_list[ filterorder_walker:]: filterorder_walker = filterorder_walker + 1 if ordered_filter in datain_filter_list: datain_index = datain_filter_list.index( ordered_filter) if not used_filter_flags[datain_index]: datause[i] = self.datain[datain_index] used_filter_flags[datain_index] = True break elif channel.getheadval('filter') in filterorder_list: filterorder_walker = filterorder_list.index( channel.getheadval('filter')) for i, channel in enumerate(datause): if channel == None: for j, datain_filter in enumerate(datain_filter_list): if not used_filter_flags[j]: datause[i] = self.datain[j] used_filter_flags[j] = True break self.log.debug( 'Files used: R = %s G = %s B = %s' % (datause[0].filename, datause[1].filename, datause[2].filename)) jpeg_dataout = DataFits(config=self.config) jpeg_dataout.header = datause[0].header jpeg_dataout.filename = datause[0].filename img = datause[0].image img1 = datause[1].image img2 = datause[2].image ''' Finding Min/Max scaling values ''' # Create a Data Cube with floats datacube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype=float) # Enter the image data into the cube so an absolute max can be found datacube[:, :, 0] = img datacube[:, :, 1] = img1 datacube[:, :, 2] = img2 # Find how many data points are in the data cube datalength = img.shape[0] * img.shape[1] * 3 # Create a 1-dimensional array with all the data, then sort it datacube.shape = (datalength, ) datacube.sort() # Now use arrays for each filter to find separate min values rarray = img.copy() garray = img1.copy() barray = img2.copy() # Shape and sort the arrays arrlength = img.shape[0] * img.shape[1] rarray.shape = (arrlength, ) rarray.sort() garray.shape = (arrlength, ) garray.sort() barray.shape = (arrlength, ) barray.sort() # Find the min/max percentile values in the data for scaling # Values are determined by parameters in the pipe configuration file minpercent = int(arrlength * self.getarg('minpercent')) maxpercent = int(datalength * self.getarg('maxpercent')) # Find the final data values to use for scaling from the image data rminsv = rarray[minpercent] #sv stands for "scalevalue" gminsv = garray[minpercent] bminsv = barray[minpercent] maxsv = datacube[maxpercent] self.log.info(' Scale min r/g/b: %f/%f/%f' % (rminsv, gminsv, bminsv)) self.log.info(' Scale max: %f' % maxsv) # The same min/max values will be used to scale all filters ''' Finished Finding scaling values ''' ''' Combining Function ''' # Make new cube with the proper data type for color images (uint8) # Use square root (sqrt) scaling for each filter # log or asinh scaling is also available #astropy.vidualizations.SqrtStretch() imgcube = numpy.zeros((img.shape[0], img.shape[1], 3), dtype='uint8') minsv = [rminsv, gminsv, bminsv] for i in range(3): # Make normalization function norm = simple_norm(datause[i].image, 'sqrt', min_cut=minsv[i], max_cut=maxsv) # Apply it imgcube[:, :, i] = norm(datause[i].image) * 255. jpeg_dataout.image = imgcube # Create variable containing all the scaled image data imgcolor = Image.fromarray(jpeg_dataout.image, mode='RGB') # Save colored image as a .tif file (without the labels) imgcolortif = imgcube.copy() imgcolortif.astype('uint16') ### tiff.imsave('%s.tif' % jpeg_dataout.filenamebase, imgcolortif) ''' End of combining function ''' ''' Add a Label to the Image ''' draw = ImageDraw.Draw(imgcolor) # Use a variable to make the positions and size of text relative imgwidth = img.shape[1] imgheight = img.shape[0] # Open Sans-Serif Font with a size relative to the picture size try: # This should work on Linux font = ImageFont.truetype( '/usr/share/fonts/liberation/LiberationSans-Regular.ttf', imgheight // 41) except: try: # This should work on Mac font = ImageFont.truetype('/Library/Fonts/Arial Unicode.ttf', imgheight // 41) except: try: # This should work on Windows font = ImageFont.truetype('C:\\Windows\\Fonts\\arial.ttf', imgheight // 41) except: # This should work in Colab font = ImageFont.truetype( '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf', imgheight // 41) # If this still doesn't work - then add more code to make it run on YOUR system # Use the beginning of the FITS filename as the object name filename = os.path.split(jpeg_dataout.filename)[-1] try: objectname = filename.split('_')[0] objectname = objectname[0].upper() + objectname[1:] except Exception: objectname = 'Unknown.' objectname = 'Object: %s' % objectname # Read labels at their respective position (kept relative to image size) # Left corner: object, observer, observatory # Right corner: Filters used for red, green, and blue colors draw.text((imgwidth / 100, imgheight / 1.114), objectname, (255, 255, 255), font=font) # Read FITS keywords for the observer, observatory, and filters if 'OBSERVER' in jpeg_dataout.header: observer = 'Observer: %s' % jpeg_dataout.getheadval('OBSERVER') draw.text((imgwidth / 100, imgheight / 1.073), observer, (255, 255, 255), font=font) if 'OBSERVAT' in jpeg_dataout.header: observatory = 'Observatory: %s' % jpeg_dataout.getheadval( 'OBSERVAT') draw.text((imgwidth / 100, imgheight / 1.035), observatory, (255, 255, 255), font=font) if 'FILTER' in datause[0].header: red = 'R: %s' % datause[0].getheadval('FILTER') draw.text((imgwidth / 1.15, imgheight / 1.114), red, (255, 255, 255), font=font) if 'FILTER' in datause[1].header: green = 'G: %s' % datause[1].getheadval('FILTER') draw.text((imgwidth / 1.15, imgheight / 1.073), green, (255, 255, 255), font=font) if 'FILTER' in datause[2].header: blue = 'B: %s' % datause[2].getheadval('FILTER') draw.text((imgwidth / 1.15, imgheight / 1.035), blue, (255, 255, 255), font=font) # Make image name imgname = jpeg_dataout.filenamebegin if imgname[-1] in '_-,.': imgname = imgname[:-1] imgname += '.jpg' # Save the completed image imgcolor.save(imgname) self.log.info('Saving file %sjpg' % jpeg_dataout.filenamebegin) # Optional folder output setup baseimgname = os.path.basename(imgname) folderpaths_list = self.getarg('folderpaths').split(':') for path in folderpaths_list: path = time.strftime(path, time.localtime()) if not os.path.exists(path): if self.getarg('createfolders'): os.makedirs(path) self.log.info('Creating directory %s' % path) else: self.log.info('Invalid folder path %s' % path) try: imgcolor.save(os.path.join(path, baseimgname)) except: self.log.exception('Could not save image to directory %s' % path) ''' End of Label Code ''' # Set complete flag jpeg_dataout.setheadval('COMPLETE', 1, 'Data Reduction Pipe: Complete Data Flag') ### Make output data self.dataout = self.datain.copy() self.dataout.append(jpeg_dataout)
class StepBiasDarkFlat(StepLoadAux, StepParent): """ Pipeline Step Object to calibrate Bias/Dark/Flat files """ stepver = '0.1' # pipe step version def __init__(self): """ Constructor: Initialize data objects and variables """ # call superclass constructor (calls setup) super(StepBiasDarkFlat, self).__init__() # bias values self.biasloaded = False # indicates if bias has been loaded self.bias = None # CCD data object containing arrays with bias values self.biasdata = DataFits() # Pipedata object containing the bias file # bias file info and header keywords to fit self.biasname = '' # name of selected bias file self.biasfitkeys = [] # FITS keywords that are present in bias self.biaskeyvalues = [] # values of FITS keywords (from data file) # dark values self.darkloaded = False # indicates if dark has been loaded self.dark = None # CCD data object containing arrays with dark values self.darkdata = DataFits() # Pipedata object containing the dark file # dark file info and header keywords to fit self.darkname = '' # name of selected dark file self.darkfitkeys = [] # FITS keywords that have to fit for dark self.darkkeyvalues = [] # values of FITS keywords (from data file) # flat values self.flatloaded = False # indicates if flat has been loaded self.flat = None # CCD data object containing arrays with flat values self.flatdata = DataFits() # Pipedata object containing the flat file # flat file info and header keywords to fit self.flatname = '' # name of selected flat file self.flatfitkeys = [] # FITS keywords that have to fit for flat self.flatkeyvalues = [] # values of flat keywords (from data file) # set configuration self.log.debug('Init: done') def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name = 'biasdarkflat' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'bdf' # Set Logger for this pipe step self.log = logging.getLogger('stoneedge.pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters self.paramlist.append([ 'reload', False, 'Set to True to look for new bias files for every input' ]) # Get parameters for StepLoadAux, replace auxfile with biasfile self.loadauxsetup('bias') # Get parameters for StepLoadAux, replace auxfile with darkfile self.loadauxsetup('dark') # Get parameters for StepLoadAux, replace auxfile with flatfile self.loadauxsetup('flat') # confirm end of setup self.log.debug('Setup: done') '''# Looking for similar exptime def closestExp(self): input_exptime = self.datain.getheadval('EXPTIME') dark_exptime = self.loadauxname('dark', multi = True).getheadval('EXPTIME') nearexp = {abs(dark_ave_exptime - exp): exp for exp in dark_exptime} return nearexp[min(nearexp.keys())] ''' def run(self): """ Runs the calibrating algorithm. The calibrated data is returned in self.dataout """ ### Preparation # Load bias files if necessary if not self.biasloaded or self.getarg('reload'): self.loadbias() # Else: check data for correct instrument configuration - currently not in use(need improvement) else: for keyind in range(len(self.biasfitkeys)): if self.biaskeyvalues[keyind] != self.datain.getheadval( self.biasfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.biasfitkeys[keyind]) # Load dark files if necessary if not self.darkloaded or self.getarg('reload'): self.loaddark() # Else: check data for correct instrument configuration else: for keyind in range(len(self.darkfitkeys)): if self.darkkeyvalues[keyind] != self.datain.getheadval( self.darkfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.darkfitkeys[keyind]) # Load flat files if necessary if not self.flatloaded or self.getarg('reload'): self.loadflat() # Else: check data for correct instrument configuration else: for keyind in range(len(self.flatfitkeys)): if self.flatkeyvalues[keyind] != self.datain.getheadval( self.flatfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.flatfitkeys[keyind]) #convert self.datain to CCD Data object image = ccdproc.CCDData(self.datain.image, unit='adu') image.header = self.datain.header #subtract bias from image image = ccdproc.subtract_bias(image, self.bias, add_keyword=False) #subtract dark from image image = ccdproc.subtract_dark(image, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second, add_keyword=False) #apply flat correction to image image = ccdproc.flat_correct(image, self.flat, add_keyword=False) # copy calibrated image into self.dataout - make sure self.dataout is a pipedata object self.dataout = DataFits(config=self.datain.config) self.dataout.image = image.data self.dataout.header = image.header self.dataout.filename = self.datain.filename ### Finish - cleanup # Update DATATYPE self.dataout.setheadval('DATATYPE', 'IMAGE') # Add bias, dark files to History self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname) self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname) self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname) def loadbias(self): """ Loads the bias information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. If multiple matching files are found, they are combined into a single master bias frame by ccdproc. """ #master bias frame #Search for bias and load it into data object namelist = self.loadauxname('bias', multi=False) self.log.info('File loaded: %s' % namelist) if (len(namelist) == 0): self.log.error('Bias calibration frame not found.') raise RuntimeError('No bias file loaded') self.log.debug('Creating master bias frame...') #if there is just one, use it as biasfile or else combine all to make a master bias self.bias = ccdproc.CCDData.read(namelist, unit='adu', relax=True) # Finish up self.biasloaded = True self.biasname = namelist self.log.debug('LoadBias: done') def loaddark(self): """ Loads the dark information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. If multiple matching files are found, they are combined into a single master dark frame by ccdproc. Also bias corrects dark files if not already done. """ #master dark frame dark_is_bias_corrected = False dark_bias = None namelist = self.loadauxname('dark', multi=False) if (len(namelist) == 0): self.log.error('Dark calibration frame(s) not found.') raise RuntimeError('No dark file loaded') # This has been commented out as it is now in StepMasterDark # darks = None # for name in namelist: # #is (any) dark file bias corrected? # header = fits.getheader(name) # if(header.get('BIAS') != None): # dark_is_bias_corrected = True # dark_bias = header.get('BIAS') # elif(header.get('BIASCORR') != None): # dark_is_bias_corrected = True # dark_bias = header.get('BIASCORR') # if(darks): # darks += ','+name # else: # darks = name self.log.debug('Creating master dark frame...') #if there is just one, use it as darkfile or else combine all to make a master dark self.dark = ccdproc.CCDData.read(namelist, unit='adu', relax=True) #bias correct, if necessary # if(not dark_is_bias_corrected): # #Subtracting master bias frame from master dark frame # self.dark = ccdproc.subtract_bias(self.dark, self.bias, add_keyword=False) # else: # self.log.debug('Master dark frame is *already* bias corrected (%s).' % dark_bias) # Finish up self.darkloaded = True self.darkname = namelist self.log.debug('LoadDark: done') def loadflat(self): """ Loads the dark information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. If multiple matching files are found, they are combined into a single master flat frame by ccdproc. Also biascorrects and dark corrects flat files if not already done. """ #create master flat frame flat_is_bias_corrected = False flat_bias = None flat_is_dark_corrected = False flat_dark = None flat_ave_exptime = 0 namelist = self.loadauxname('flat', multi=False) if (len(namelist) == 0): self.log.error('Flat calibration frame not found.') raise RuntimeError('No flat file loaded') count = 0 datalist = [] flat_corrected = None # This has been commented out as it is now in StepMasterFlat #check a few things in these flat component frames # for name in namelist: # header = fits.getheader(name) #is this flat bias corrected? # if(header.get('BIAS') != None): # flat_is_bias_corrected = True # flat_bias = header.get('BIAS') # elif(header.get('BIASCORR') != None): # flat_is_bias_corrected = True # flat_bias = header.get('BIASCORR') # #is this flat dark corrected? # if(header.get('DARK') != None): # flat_is_dark_corrected = True # flat_dark = header.get('DARK') # elif(header.get('DARKCORR') != None): # flat_is_dark_corrected = True # flat_dark = header.get('DARKCORR') # flat_corrected = "%s"%(name.rsplit('.',1)[0])+".corrected" # shutil.copy(name, flat_corrected) # self.log.debug('Copying %s to %s' % (name, flat_corrected)) # self.flat = ccdproc.CCDData.read(flat_corrected, unit='adu', relax=True) # #bias correct, if necessary # if(not flat_is_bias_corrected): # self.log.debug('Subtracting master bias frame from flat frame...') # self.flat = ccdproc.subtract_bias(self.flat, self.bias, add_keyword=False) # else: # self.log.debug('Flat frame (%s) is *already* bias corrected.'%flat_bias) # #dark correct, if necessary # if(not flat_is_dark_corrected): # self.log.debug('Subtracting master dark frame from flat frame...') # self.flat = ccdproc.subtract_dark(self.flat, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second, add_keyword=False) # else: # self.log.debug('Flat frame (%s) is *already* dark corrected.'%flat_dark) # #create CCD Data object list with corrected flat files # datalist.append(self.flat) # #calc average exposure time for potential dark correction # if(header.get('EXPTIME') != None): # try: # exptime = float(header.get('EXPTIME')) # flat_ave_exptime += exptime # except ValueError: # self.log.error('Exposure time (EXPTIME) is not a float (%s).'%(header.get('EXPTIME'))) # count += 1 # #calc average exposure time # if(count > 0): # flat_ave_exptime = flat_ave_exptime/count # self.flat.header['EXPTIME'] = flat_ave_exptime # self.log.info("Average exposure time for flats is %f"%flat_ave_exptime) self.log.debug('Creating master flat frame...') #if there is just one, use it as flatfile or else combine all to make a master flat self.flat = ccdproc.CCDData.read(namelist, unit='adu', relax=True) # Finish up self.flatloaded = True self.flatname = namelist self.log.debug('LoadFlat: done') def reset(self): """ Resets the step to the same condition as it was when it was created. Internal variables are reset, any stored data is erased. """ self.biasloaded = False self.bias = None self.darkloaded = False self.dark = None self.flatloaded = False self.flat = None self.log.debug('Reset: done')
def run(self): """ Runs the calibrating algorithm. The calibrated data is returned in self.dataout """ ### Preparation # Load bias files if necessary if not self.biasloaded or self.getarg('reload'): self.loadbias() # Else: check data for correct instrument configuration - currently not in use(need improvement) else: for keyind in range(len(self.biasfitkeys)): if self.biaskeyvalues[keyind] != self.datain.getheadval( self.biasfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.biasfitkeys[keyind]) # Load dark files if necessary if not self.darkloaded or self.getarg('reload'): self.loaddark() # Else: check data for correct instrument configuration else: for keyind in range(len(self.darkfitkeys)): if self.darkkeyvalues[keyind] != self.datain.getheadval( self.darkfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.darkfitkeys[keyind]) # Load flat files if necessary if not self.flatloaded or self.getarg('reload'): self.loadflat() # Else: check data for correct instrument configuration else: for keyind in range(len(self.flatfitkeys)): if self.flatkeyvalues[keyind] != self.datain.getheadval( self.flatfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.flatfitkeys[keyind]) #convert self.datain to CCD Data object image = ccdproc.CCDData(self.datain.image, unit='adu') image.header = self.datain.header #subtract bias from image image = ccdproc.subtract_bias(image, self.bias, add_keyword=False) #subtract dark from image image = ccdproc.subtract_dark(image, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second, add_keyword=False) #apply flat correction to image image = ccdproc.flat_correct(image, self.flat, add_keyword=False) # copy calibrated image into self.dataout - make sure self.dataout is a pipedata object self.dataout = DataFits(config=self.datain.config) self.dataout.image = image.data self.dataout.header = image.header self.dataout.filename = self.datain.filename ### Finish - cleanup # Update DATATYPE self.dataout.setheadval('DATATYPE', 'IMAGE') # Add bias, dark files to History self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname) self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname) self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname)
def run(self): """ Runs the data reduction algorithm. The self.datain is run through the code, the result is in self.dataout. """ ### Preparation # construct a temp file name that astrometry will output fp = tempfile.NamedTemporaryFile(suffix=".fits", dir=os.getcwd()) # split off path name, because a path that is too long causes remap to # crash sometimes outname = os.path.split(fp.name)[1] fp.close() # Add input file path to ouput file and make new name outpath = os.path.split(self.datain.filename)[0] outnewname = os.path.join(outpath, outname.replace('.fits', '.new')) outwcsname = os.path.join(outpath, outname.replace('.fits', '.wcs')) # Make sure input data exists as file if not os.path.exists(self.datain.filename): self.datain.save() # Make command string rawcommand = self.getarg('astrocmd') % (self.datain.filename, outname) # get estimated RA and DEC center values from the config file or input FITS header raopt = self.getarg('ra') if raopt != '': ra = Angle(raopt, unit=u.hour).degree else: try: ra = Angle(self.datain.getheadval('RA'), unit=u.hour).degree except: ra = '' decopt = self.getarg('dec') if decopt != '': dec = Angle(decopt, unit=u.deg).degree else: try: dec = Angle(self.datain.getheadval('DEC'), unit=u.deg).degree except: dec = '' if (ra != '') and (dec != ''): # update command parameters to use these values rawcommand = rawcommand + ' --ra %f --dec %f --radius %f' % ( ra, dec, self.getarg('searchradius')) else: self.log.debug( 'FITS header missing RA/DEC -> searching entire sky') ### Run Astrometry: # This loop tries the downsample and param options until the fit is successful # need either --scale-low 0.5 --scale-high 2.0 --sort-column FLUX # or --guess-scale downsamples = self.getarg('downsample') paramoptions = self.getarg('paramoptions') for option in range(len(downsamples) * len(paramoptions)): #for downsample in self.getarg('downsample'): downsample = downsamples[option % len(downsamples)] paramoption = paramoptions[option // len(downsamples)] # Add options to command command = rawcommand + ' --downsample %d' % downsample + ' ' + paramoption optionstring = "Downsample=%s Paramopts=%s" % (downsample, paramoption[:10]) # Run the process - see note at the top of the file if using cron process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) self.log.debug('running command = %s' % command) # Wait for the process to be finished or timeout to be reached timeout = time.time() + self.getarg('timeout') while time.time() < timeout and process.poll() == None: time.sleep(1) poll = process.poll() if poll == None: process.kill() time.sleep(1) poll = process.poll() self.log.debug('command returns %d' % poll) if poll == 0 and os.path.exists(outnewname): self.log.debug('output file valid -> astrometry successful') break else: self.log.debug('output file missing -> astrometry failed') # Print the output from astrometry (cut if necessary) if self.getarg('verbose'): output = process.stdout.read().decode() if len(output) > 1000: outlines = output.split('\n') output = outlines[:10] + ['...', '...'] + outlines[-7:] output = '\n'.join(output) self.log.debug(output) ### Post processing # Read output file self.dataout = DataFits(config=self.config) self.log.debug('Opening astrometry.net output file %s' % outnewname) try: self.dataout.load(outnewname) self.dataout.filename = self.datain.filename except Exception as error: self.log.error("Unable to open astrometry. output file = %s" % outname) raise error self.log.debug('Successful parameter options = %s' % optionstring) # Add history message histmsg = 'Astrometry.Net: At downsample = %d, search took %d seconds' % ( downsample, time.time() - timeout + 300) self.dataout.setheadval('HISTORY', histmsg) # Add RA from astrometry w = wcs.WCS(self.dataout.header) n1 = float(self.dataout.header['NAXIS1'] / 2) n2 = float(self.dataout.header['NAXIS2'] / 2) ra, dec = w.all_pix2world(n1, n2, 1) self.dataout.header['CRPIX1'] = n1 self.dataout.header['CRPIX2'] = n2 self.dataout.header['CRVAL1'] = float(ra) self.dataout.header['CRVAL2'] = float(dec) self.dataout.header['RA'] = Angle(ra, u.deg).to_string(unit=u.hour, sep=':') self.dataout.header['Dec'] = Angle(dec, u.deg).to_string(sep=':') self.dataout.setheadval('HISTORY', 'Astrometry: Paramopts = ' + optionstring) # Delete temporary files if self.getarg('delete_temp'): os.remove(outnewname) os.remove(outwcsname) self.log.debug('Run: Done')
def run(self): self.dataout = DataFits(config=self.config) self.dataout.load(self.mask())
class StepMasterDark(StepLoadAux, StepMIParent): """ Stone Edge Pipeline Step Master Dark Object The object is callable. It requires a valid configuration input (file or object) when it runs. """ stepver = '0.1' # pipe step version def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. Setup() is called at the end of __init__ The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name = 'masterdark' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'mdark' # Set Logger for this pipe step self.log = logging.getLogger('stoneedge.pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters !!!! WHAT PARAMETERS ARE NEEDED ????? !!!!! self.paramlist.append([ 'combinemethod', 'median', 'Specifies how the files should be combined - options are median, average, sum' ]) self.paramlist.append([ 'outputfolder', '', 'Output directory location - default is the folder of the input files' ]) # Get parameters for StepLoadAux, replace auxfile with biasfile self.loadauxsetup('bias') def run(self): """ Runs the combining algorithm. The self.datain is run through the code, the result is in self.dataout. """ # Find master bias to subtract from master dark biaslist = self.loadauxname('bias', multi=False) if (len(biaslist) == 0): self.log.error('No bias calibration frames found.') self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True) # Create empy list for filenames of loaded frames filelist = [] for fin in self.datain: self.log.debug("Input filename = %s" % fin.filename) filelist.append(fin.filename) # Make a dummy dataout self.dataout = DataFits(config=self.config) if len(self.datain) == 0: self.log.error('Flat calibration frame not found.') raise RuntimeError('No flat file(s) loaded') self.log.debug('Creating master flat frame...') # Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias if (len(filelist) == 1): self.dark = ccdproc.CCDData.read(filelist[0], unit='adu', relax=True) self.dark = ccdproc.subtract_bias(self.dark, self.bias, add_keyword=False) else: darklist = [] for i in filelist: dark = ccdproc.CCDData.read(i, unit='adu', relax=True) darksubbias = ccdproc.subtract_bias(dark, self.bias, add_keyword=False) darklist.append(darksubbias) self.dark = ccdproc.combine(darklist, method=self.getarg('combinemethod'), unit='adu', add_keyword=True) # set output header, put image into output self.dataout.header = self.datain[0].header self.dataout.imageset(self.dark) # rename output filename outputfolder = self.getarg('outputfolder') if outputfolder != '': outputfolder = os.path.expandvars(outputfolder) self.dataout.filename = os.path.join(outputfolder, os.path.split(filelist[0])[1]) else: self.dataout.filename = filelist[0] # Add history self.dataout.setheadval('HISTORY', 'MasterDark: %d files used' % len(filelist))
def run(self): self.dataout = DataFits(config=self.config) self.dataout.load(self.source_extract()) self.dataout.header['RA'] = self.datain.header['RA'] self.dataout.header['Dec'] = self.datain.header['Dec'] self.dataout.save()
from darepype.drp import DataFits from astropy.io import fits config = '/Users/josh/pipeline/pipeline/Developments/stepwebastrometry/pipeconf_stonedge_auto.txt' fp = '/Users/josh/Desktop/pipeline_test/data/M5_r-band_60s_bin2_200711_053415_itzamna_seo_0_RAW_TABLE.fits' fts = DataFits(config=config) fts.load(fp) # print(repr(fits.HDUList(file=fp))) # fts.header['RA'] = 0 # fts.header['Dec'] = 0 print(repr(fts.header)) print(repr(fts.image)) # print(repr(fts.table))
# print(repr(dfits.header)) ### OPTIONAL BUT RECOMMENDED: Check if all necessary files exist error_flag = False # Check if configuration file exists if not os.path.exists(baseconfig): print( 'ERROR: The config file you specified, %s,\n does NOT exist on your computer, fix "config" above' % baseconfig) error_flag = True # Check if input files exist for name in infilenames: if not os.path.exists(name): print( 'ERROR: The input file you specifed, %s,\n does NOT exist on your computer, fix "inputnames" above' % name) error_flag = True if not error_flag: print("All Good") os.chdir('/Users/josh/pipeline/pipeline/Developments/stepwebastrometry') step = StepWebAstrometry() indata = [] for f in infilenames: fits = DataFits(config=baseconfig) fits.load(f) indata.append(fits) outdata = step(indata[0]) print('Done')
def loaddark(self): """ Loads the dark information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. """ ### identify dark file to load, search if requested darkfile = self.getarg('darkfile') if darkfile == 'search' : # get list of keywords to fit fitkeys = self.getarg('fitkeys') # check format (make first element uppercase) try: _ = fitkeys[0].upper() except AttributeError: # AttributeError if it's not a string self.log.error('LoadDark: fitkeys config parameter is ' + 'incorrect format') raise TypeError('fitkeys config parameter is incorrect format') # get keywords from data datakeys=[] for fitkey in fitkeys: datakeys.append(self.datain.getheadval(fitkey)) # get dark files from darkdir folder darkfolder = self.getarg('darkfolder') filelist=[name for name in os.listdir(darkfolder) if name[0] != '.' and name.find('.fit') > -1 ] if len(filelist) < 1: self.log.error('LoadDark: no dark files found in folder ' + darkfolder) raise ValueError('no dark files found in folder ' + darkfolder) # match dark files, return best dark file bestind = 0 # index of file with best match in filelist bestfitn = 0 # number of keywords that match in best match fileind = 0 # index for going through the list while fileind < len(filelist) and bestfitn < len(fitkeys): # load keys of dark file filehead = pyfits.getheader(darkfolder+'/'+filelist[fileind]) filekeys=[] for fitkey in fitkeys: try: filekeys.append(filehead[fitkey]) except KeyError: self.log.warning('LoadDark: missing key [%s] in dark <%s>' % (fitkey, filelist[fileind] ) ) filekeys.append('') # determine number of fitting keywords keyfitn=0 while ( keyfitn < len(fitkeys) and datakeys[keyfitn] == filekeys[keyfitn] ): keyfitn = keyfitn + 1 # compare with previous best find if keyfitn > bestfitn: bestind = fileind bestfitn = keyfitn fileind=fileind+1 darkfile = darkfolder+'/'+filelist[ bestind ] if bestfitn < len(fitkeys): self.log.warn('Could not find perfect dark file match') self.log.warn('Best dark file found is <%s>' % filelist[bestind] ) else: self.log.info('Best dark file found is <%s>' % filelist[bestind] ) self.fitkeys = fitkeys self.keyvalues = datakeys ### load dark data into a DataFits object self.darkfile = darkfile darkdata = DataFits(config = self.config) darkdata.load(self.darkfile) ### find dark image data arrays and store them # get sizes of input data datalist = self.getarg('datalist') if len(datalist) > 0: # There are items in datalist -> loop over items self.darks = [] # Check if necessary number of images in darkdata if len(darkdata.imgdata) < len(datalist): msg = 'Number of images in dark file < ' msg += 'number of entries in datalist' self.log.error('LoadDark: %s' % msg) raise ValueError(msg) # Loop through datalist items for dataind in range(len(datalist)): dataitem = datalist[dataind] # Search for dataitem in self.datain images if dataitem.upper() in self.datain.imgnames: dataimg = self.datain.imageget(dataitem) self.log.debug('LoadDark: Found image <%s> to subtract dark' % dataitem) # Search dataitem in self.table columns else: try: dataimg = self.datain.table[dataitem] self.log.debug('LoadDark: Found column <%s> to subtract dark' % dataitem) except: msg = 'No data found for <%s>' % dataitem self.log.error('LoadDark: %s' % msg) raise ValueError(msg) # Get dimensions - append dark to list datasiz = dataimg.shape if self.getarg('l0method').upper() != 'NO': datasiz = datasiz[1:] darksiz = darkdata.imgdata[dataind].shape self.darks.append(darkdata.imgdata[dataind]) # Check dimension with dark data print(datasiz,darksiz) if len(datasiz) >= len(darksiz): # Data has >= dimensions than dark -> compare begind = len(datasiz)-len(darksiz) if datasiz[begind:] != darksiz: msg = 'Dark "%s" does not fit data - A' % dataitem self.log.error('LoadDark: %s' % msg) raise ValueError(msg) else: # More dimensions in dark data -> report error msg = 'Dark "%s" does not fit data - B' % dataitem self.log.error('LoadDark: %s' % dataitem) raise ValueError(msg) else: # Empty datalist -> Subtract dark from first image in data with first dark datasiz = self.datain.image.shape if self.getarg('l0method').upper() != 'NO': datasiz = datasiz[1:] darksiz = darkdata.image.shape if len(datasiz) >= len(darksiz): # Data has >= dimensions than dark -> compare begind = len(datasiz)-len(darksiz) if datasiz[begind:] != darksiz: self.log.error('LoadDark: Dark does not fit data - A') raise ValueError('Dark does not fit data - A') else: # More dimensions in dark data -> report error self.log.error('LoadDark: Dark does not fit data - B') raise ValueError('Dark does not fit data - B') self.log.debug('LoadDark: Subtracting Dark from first data image with first dark') self.darks=[darkdata.image] ### make good pixel map for each detector and add to #darktemp = numpy.abs(data[0,...])+numpy.abs(data[1,...]) #self.goodpixmap = numpy.ones(data.shape[1:]) #self.goodpixmap [ numpy.where(darktemp == 0.0)] = 0.0 # Finish up self.darkloaded = 1 self.log.debug('LoadDark: done')
class StepBiasDarkFlat(StepLoadAux, StepParent): """ Pipeline Step Object to calibrate Bias/Dark/Flat files """ stepver = '0.1' # pipe step version def __init__(self): """ Constructor: Initialize data objects and variables """ # call superclass constructor (calls setup) super(StepBiasDarkFlat, self).__init__() # bias values self.biasloaded = False # indicates if bias has been loaded self.bias = None # CCD data object containing arrays with bias values self.biasdata = DataFits() # Pipedata object containing the bias file # bias file info and header keywords to fit self.biasname = '' # name of selected bias file self.biasfitkeys = [] # FITS keywords that are present in bias self.biaskeyvalues = [] # values of FITS keywords (from data file) # dark values self.darkloaded = False # indicates if dark has been loaded self.dark = None # CCD data object containing arrays with dark values self.darkdata = DataFits() # Pipedata object containing the dark file # dark file info and header keywords to fit self.darkname = '' # name of selected dark file self.darkfitkeys = [] # FITS keywords that have to fit for dark self.darkkeyvalues = [] # values of FITS keywords (from data file) # flat values self.flatloaded = False # indicates if flat has been loaded self.flat = None # CCD data object containing arrays with flat values self.flatdata = DataFits() # Pipedata object containing the flat file # flat file info and header keywords to fit self.flatname = '' # name of selected flat file self.flatfitkeys = [] # FITS keywords that have to fit for flat self.flatkeyvalues = [] # values of flat keywords (from data file) # set configuration self.log.debug('Init: done') # This function is directly lifted from CCDProc https://github.com/astropy/ccdproc/blob/master/ccdproc/core.py # Instead of directly calling CCDProc, we have included the function here to increase educational value # and to decrease reliance on external libraries. def subtract_bias(self, image, bias): """ Subtract master bias from image. Parameters ---------- image : `~astropy.nddata.CCDData` Image from which bias will be subtracted. bias : `~astropy.nddata.CCDData` Master image to be subtracted from ``ccd``. {log} Returns ------- result : `~astropy.nddata.CCDData` CCDData object with bias subtracted. """ self.log.debug('Subtracting bias...') result = image.copy() try: result.data = image.data - bias.data # we believe that we should keep this error detection in theory, the bias # and image both come from seo, so their units should be the same except ValueError as e: if 'operand units' in str(e): raise u.UnitsError( "Unit '{}' of the uncalibrated image does not " "match unit '{}' of the calibration " "image".format(image.unit, bias.unit)) else: raise e self.log.debug('Subtracted bias.') return result # this code is also lifted from ccdproc https://github.com/astropy/ccdproc/blob/master/ccdproc/core.py # some of the code is removed from the original ccdproc because it is not relevant to how SEO currently # processes data. If you are looking at this code in the future, there is more code available to draw from def subtract_dark(self, image, dark, scale=False, exposure_time=None, exposure_unit=None): """ Subtract dark current from an image. Parameters ---------- image : `~astropy.nddata.CCDData` Image from which dark will be subtracted. dark : `~astropy.nddata.CCDData` Dark image. exposure_time : str or `~ccdproc.Keyword` or None, optional Name of key in image metadata that contains exposure time. Default is ``None``. exposure_unit : `~astropy.units.Unit` or None, optional Unit of the exposure time if the value in the meta data does not include a unit. Default is ``None``. scale: bool, optional If True, scale the dark frame by the exposure times. Default is ``False``. {log} Returns ------- result : `~astropy.nddata.CCDData` Dark-subtracted image. """ self.log.debug('Subtracting dark...') result = image.copy() try: # if dark current is linear, then this first step scales the provided # dark to match the exposure time if scale: dark_scaled = dark.copy() data_exposure = image.header[exposure_time] dark_exposure = dark.header[exposure_time] # data_exposure and dark_exposure are both quantities, # so we can just have subtract do the scaling dark_scaled = dark_scaled.multiply(data_exposure / dark_exposure) result.data = image.data - dark_scaled.data else: result.data = image.data - dark.data except (u.UnitsError, u.UnitConversionError, ValueError) as e: # Make the error message a little more explicit than what is returned # by default. raise u.UnitsError("Unit '{}' of the uncalibrated image does not " "match unit '{}' of the calibration " "image".format(image.unit, dark.unit)) self.log.debug('Subtracted dark.') return result # This code is also from ccdproc. A notable removal is the option to manually choose # maximum and minimum flat values. def flat_correct(self, image, flat): """Correct the image for flat fielding. The flat field image is normalized by its mean or a user-supplied value before flat correcting. Parameters ---------- ccd : `~astropy.nddata.CCDData` Data to be transformed. flat : `~astropy.nddata.CCDData` Flatfield to apply to the data. {log} Returns ------- ccd : `~astropy.nddata.CCDData` CCDData object with flat corrected. """ self.log.debug('Correcting flat...') # Use the min_value to replace any values in the flat flat_corrected = image.copy() use_flat = flat flat_mean_val = use_flat.data.mean() # Normalize the flat. flat_mean = flat_mean_val * use_flat.unit flat_normed = use_flat.data / flat_mean # divide through the flat flat_corrected.data = image.data / flat_normed self.log.debug('Corrected flat.') return flat_corrected def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name = 'biasdarkflat' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'bdf' # Set Logger for this pipe step self.log = logging.getLogger('stoneedge.pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters self.paramlist.append([ 'reload', False, 'Set to True to look for new bias files for every input' ]) # Get parameters for StepLoadAux, replace auxfile with biasfile self.loadauxsetup('bias') # Get parameters for StepLoadAux, replace auxfile with darkfile self.loadauxsetup('dark') # Get parameters for StepLoadAux, replace auxfile with flatfile self.loadauxsetup('flat') # confirm end of setup self.log.debug('Setup: done') '''# Looking for similar exptime def closestExp(self): input_exptime = self.datain.getheadval('EXPTIME') dark_exptime = self.loadauxname('dark', multi = True).getheadval('EXPTIME') nearexp = {abs(dark_ave_exptime - exp): exp for exp in dark_exptime} return nearexp[min(nearexp.keys())] ''' def run(self): """ Runs the calibrating algorithm. The calibrated data is returned in self.dataout """ ### Preparation # Load bias files if necessary if not self.biasloaded or self.getarg('reload'): self.loadbias() # Else: check data for correct instrument configuration - currently not in use(need improvement) else: for keyind in range(len(self.biasfitkeys)): if self.biaskeyvalues[keyind] != self.datain.getheadval( self.biasfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.biasfitkeys[keyind]) # Load dark files if necessary if not self.darkloaded or self.getarg('reload'): self.loaddark() # Else: check data for correct instrument configuration else: for keyind in range(len(self.darkfitkeys)): if self.darkkeyvalues[keyind] != self.datain.getheadval( self.darkfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.darkfitkeys[keyind]) # Load flat files if necessary if not self.flatloaded or self.getarg('reload'): self.loadflat() # Else: check data for correct instrument configuration else: for keyind in range(len(self.flatfitkeys)): if self.flatkeyvalues[keyind] != self.datain.getheadval( self.flatfitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.flatfitkeys[keyind]) # in the config file, set the 'intermediate' variable to either true or false to enable # saving of intermediate steps saveIntermediateSteps = self.config['biasdarkflat']['intermediate'] self.dataout = DataFits(config=self.datain.config) #convert self.datain to CCD Data object image = CCDData(self.datain.image, unit='adu') image.header = self.datain.header # subtract bias from image image = self.subtract_bias(image, self.bias) if (saveIntermediateSteps == "true"): self.dataout.imageset(image.data, imagename="BIAS") # self.dataout.setheadval('DATATYPE','IMAGE', dataname="BIAS") self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname, dataname="BIAS") # subtract dark from image image = self.subtract_dark(image, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second) if (saveIntermediateSteps == "true"): self.dataout.imageset(image.data, imagename="DARK") # self.dataout.setheadval('DATATYPE','IMAGE', dataname="DARK") self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname, dataname="DARK") self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname, dataname="DARK") # apply flat correction to image image = self.flat_correct(image, self.flat) # if separating bias,dark,flat steps , save the flat-corrected portion into its own hdu if (saveIntermediateSteps == "true"): self.dataout.imageset(image.data, imagename="FLAT") # self.dataout.setheadval('DATATYPE','IMAGE', dataname="FLAT") self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname, dataname="FLAT") self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname, dataname="FLAT") self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname, dataname="FLAT") else: # copy calibrated image into self.dataout self.dataout.image = image.data self.dataout.header = self.datain.header ### Finish - cleanup # Update DATATYPE self.dataout.setheadval('DATATYPE', 'IMAGE') # Add bias, dark files to History self.dataout.setheadval('HISTORY', 'BIAS: %s' % self.biasname) self.dataout.setheadval('HISTORY', 'DARK: %s' % self.darkname) self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatname) self.dataout.filename = self.datain.filename def loadbias(self): """ Loads the bias information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. If multiple matching files are found, they are combined into a single master bias frame by ccdproc. """ #master bias frame #Search for bias and load it into data object namelist = self.loadauxname('bias', multi=False) self.log.info('File loaded: %s' % namelist) if (len(namelist) == 0): self.log.error('Bias calibration frame not found.') raise RuntimeError('No bias file loaded') self.log.debug('Creating master bias frame...') #if there is just one, use it as biasfile or else combine all to make a master bias self.bias = CCDData.read(namelist, unit='adu', relax=True) # Finish up self.biasloaded = True self.biasname = namelist self.log.debug('LoadBias: done') def loaddark(self): """ Loads the dark information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. If multiple matching files are found, they are combined into a single master dark frame by ccdproc. Also bias corrects dark files if not already done. """ #master dark frame dark_is_bias_corrected = False dark_bias = None namelist = self.loadauxname('dark', multi=False) if (len(namelist) == 0): self.log.error('Dark calibration frame(s) not found.') raise RuntimeError('No dark file loaded') # This has been commented out as it is now in StepMasterDark # darks = None # for name in namelist: # #is (any) dark file bias corrected? # header = fits.getheader(name) # if(header.get('BIAS') != None): # dark_is_bias_corrected = True # dark_bias = header.get('BIAS') # elif(header.get('BIASCORR') != None): # dark_is_bias_corrected = True # dark_bias = header.get('BIASCORR') # if(darks): # darks += ','+name # else: # darks = name self.log.debug('Creating master dark frame...') #if there is just one, use it as darkfile or else combine all to make a master dark self.dark = CCDData.read(namelist, unit='adu', relax=True) #bias correct, if necessary # if(not dark_is_bias_corrected): # #Subtracting master bias frame from master dark frame # self.dark = ccdproc.subtract_bias(self.dark, self.bias, add_keyword=False) # else: # self.log.debug('Master dark frame is *already* bias corrected (%s).' % dark_bias) # Finish up self.darkloaded = True self.darkname = namelist self.log.debug('LoadDark: done') def loadflat(self): """ Loads the dark information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. If multiple matching files are found, they are combined into a single master flat frame by ccdproc. Also biascorrects and dark corrects flat files if not already done. """ #create master flat frame flat_is_bias_corrected = False flat_bias = None flat_is_dark_corrected = False flat_dark = None flat_ave_exptime = 0 namelist = self.loadauxname('flat', multi=False) if (len(namelist) == 0): self.log.error('Flat calibration frame not found.') raise RuntimeError('No flat file loaded') count = 0 datalist = [] flat_corrected = None # This has been commented out as it is now in StepMasterFlat #check a few things in these flat component frames # for name in namelist: # header = fits.getheader(name) #is this flat bias corrected? # if(header.get('BIAS') != None): # flat_is_bias_corrected = True # flat_bias = header.get('BIAS') # elif(header.get('BIASCORR') != None): # flat_is_bias_corrected = True # flat_bias = header.get('BIASCORR') # #is this flat dark corrected? # if(header.get('DARK') != None): # flat_is_dark_corrected = True # flat_dark = header.get('DARK') # elif(header.get('DARKCORR') != None): # flat_is_dark_corrected = True # flat_dark = header.get('DARKCORR') # flat_corrected = "%s"%(name.rsplit('.',1)[0])+".corrected" # shutil.copy(name, flat_corrected) # self.log.debug('Copying %s to %s' % (name, flat_corrected)) # self.flat = ccdproc.CCDData.read(flat_corrected, unit='adu', relax=True) # #bias correct, if necessary # if(not flat_is_bias_corrected): # self.log.debug('Subtracting master bias frame from flat frame...') # self.flat = ccdproc.subtract_bias(self.flat, self.bias, add_keyword=False) # else: # self.log.debug('Flat frame (%s) is *already* bias corrected.'%flat_bias) # #dark correct, if necessary # if(not flat_is_dark_corrected): # self.log.debug('Subtracting master dark frame from flat frame...') # self.flat = ccdproc.subtract_dark(self.flat, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second, add_keyword=False) # else: # self.log.debug('Flat frame (%s) is *already* dark corrected.'%flat_dark) # #create CCD Data object list with corrected flat files # datalist.append(self.flat) # #calc average exposure time for potential dark correction # if(header.get('EXPTIME') != None): # try: # exptime = float(header.get('EXPTIME')) # flat_ave_exptime += exptime # except ValueError: # self.log.error('Exposure time (EXPTIME) is not a float (%s).'%(header.get('EXPTIME'))) # count += 1 # #calc average exposure time # if(count > 0): # flat_ave_exptime = flat_ave_exptime/count # self.flat.header['EXPTIME'] = flat_ave_exptime # self.log.info("Average exposure time for flats is %f"%flat_ave_exptime) self.log.debug('Creating master flat frame...') #if there is just one, use it as flatfile or else combine all to make a master flat self.flat = CCDData.read(namelist, unit='adu', relax=True) # Finish up self.flatloaded = True self.flatname = namelist self.log.debug('LoadFlat: done') def reset(self): """ Resets the step to the same condition as it was when it was created. Internal variables are reset, any stored data is erased. """ self.biasloaded = False self.bias = None self.darkloaded = False self.dark = None self.flatloaded = False self.flat = None self.log.debug('Reset: done')
def run(self): """ Runs the combining algorithm. The self.datain is run through the code, the result is in self.dataout. """ # Find master dark to subtract from master dark biaslist = self.loadauxname('bias', multi=False) darklist = self.loadauxname('dark', multi=False) if (len(biaslist) == 0): self.log.error('No bias calibration frames found.') if (len(darklist) == 0): self.log.error('No bias calibration frames found.') self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True) self.dark = ccdproc.CCDData.read(darklist, unit='adu', relax=True) # Create empy list for filenames of loaded frames filelist = [] for fin in self.datain: self.log.debug("Input filename = %s" % fin.filename) filelist.append(fin.filename) # Make a dummy dataout self.dataout = DataFits(config=self.config) if len(self.datain) == 0: self.log.error('Flat calibration frame not found.') raise RuntimeError('No flat file(s) loaded') self.log.debug('Creating master flat frame...') # Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias if (len(filelist) == 1): self.flat = ccdproc.CCDData.read(filelist[0], unit='adu', relax=True) self.flat = ccdproc.subtract_bias(self.flat, self.bias, add_keyword=False) self.flat = ccdproc.subtract_dark(self.flat, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second, add_keyword=False) else: #bias and dark correct frames flatlist = [] for i in filelist: flat = ccdproc.CCDData.read(i, unit='adu', relax=True) flatsubbias = ccdproc.subtract_bias(flat, self.bias, add_keyword=False) flatsubbiasdark = ccdproc.subtract_dark( flatsubbias, self.dark, scale=True, exposure_time='EXPTIME', exposure_unit=u.second, add_keyword=False) flatlist.append(flatsubbiasdark) #scale the flat component frames to have the same mean value, 10000.0 scaling_func = lambda arr: 10000.0 / numpy.ma.median(arr) #combine them self.flat = ccdproc.combine(flatlist, method=self.getarg('combinemethod'), scale=scaling_func, unit='adu', add_keyword=False) # set output header, put image into output self.dataout.header = self.datain[0].header self.dataout.imageset(self.flat) # rename output filename outputfolder = self.getarg('outputfolder') if outputfolder != '': outputfolder = os.path.expandvars(outputfolder) self.dataout.filename = os.path.join(outputfolder, os.path.split(filelist[0])[1]) else: self.dataout.filename = filelist[0] # Add history self.dataout.setheadval('HISTORY', 'MasterFlat: %d files used' % len(filelist))
class StepCoadd(StepMIParent): """ Stone Edge Pipeline Step Master Bias Object The object is callable. It requires a valid configuration input (file or object) when it runs. """ stepver = '1.2' # pipe step version def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. Setup() is called at the end of __init__ The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name='coadd' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'coadd' # Set Logger for this pipe step self.log = logging.getLogger('pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters self.paramlist.append(['kernel','square', 'Specifies the kernel used to determine spreading of input pixels onto output pixels \ - options are square, point, gaussian, smoothing, tophat']) self.paramlist.append(['pixfrac', 1., 'The fraction of an output pixel(s) that an input pixel\'s flux is confined to']) self.paramlist.append(['resolution', 1., 'Pixel scale divisor for output image (higher gives more resolution, lower gives less)']) self.paramlist.append(['pad', 0, 'Extra padding outside maximum extent of inputs']) self.paramlist.append(['fillval', np.nan, 'Value for filling in the area(s) in the output where there is no input data']) self.paramlist.append(['drizzleweights','exptime', 'How each input image should be weighted when added to the output \ - options are exptime, expsq and uniform']) self.paramlist.append(['outangle',0., 'Output angle of drizzled image (currently not functional)']) def run(self): """ Runs the mosaicing algorithm. The self.datain is run through the code, the result is in self.dataout. """ #calculate platescale of first input image try: det = np.linalg.det(wcs.WCS(self.datain[0].header).wcs.cd) pscale = np.sqrt(np.abs(det))*3600. except: try: det = np.linalg.det(wcs.WCS(self.datain[0].header).wcs.pc) pscale = np.sqrt(np.abs(det))*3600. except: pscale = self.datain[0].header['PIXSCAL'] #filtering out images which are too far away from the others #passing images added to a list of (image, WCS) tuples ''' image_centers = [] for f in self.datain: image_centers.append((f.header['CRVAL1'], f.header['CRVAL2'])) filtered_datain = [] dist_list = [[[0]*(len(image_centers)-1)]*len(image_centers)] for i in range(len(image_centers)): for j in range(len(image_centers)-1): dist_list[i][j+1] = np.sqrt((image_)**2+()**2) ''' #calculations necessary for updating wcs information px = [] py = [] #in order to avoid NaN interactions, creating weight map weights=[] for f in self.datain: weights.append((np.where(np.isnan(f.image) == True, 0, 1))) for f in self.datain: px.extend(wcs.WCS(f.header).calc_footprint()[:,0]) py.extend(wcs.WCS(f.header).calc_footprint()[:,1]) x0 = (max(px)+min(px))/2. y0 = (max(py)+min(py))/2. sx = (max(px)-min(px))*np.cos(y0/180*np.pi) # arcsec sy = (max(py)-min(py)) # arcsec size = (sx*3600+self.getarg('pad')*2, sy*3600+self.getarg('pad')*2) xpix = size[0]//pscale ypix = size[1]//pscale cdelt = [pscale/3600.]*2 #create self.dataout and give it a copy of an input's header self.dataout = DataFits(config = self.config) self.dataout.header = self.datain[0].header.copy() #update header wcs information self.log.info('Creating new WCS header') self.dataout.header['CRPIX1'] = xpix/2 self.dataout.header['CRPIX2'] = ypix/2 self.dataout.header['CRVAL1'] = x0 self.dataout.header['CRVAL2'] = y0 self.dataout.header['CD1_1'] = -cdelt[0] self.dataout.header['CD1_2'] = self.dataout.header['CD2_1'] = 0. self.dataout.header['CD2_2'] = cdelt[1] self.dataout.header['NAXIS1'] = int(xpix) self.dataout.header['NAXIS2'] = int(ypix) self.dataout.header['CTYPE1'] = 'RA---TAN-SIP' self.dataout.header['CTYPE2'] = 'DEC--TAN-SIP' self.dataout.header['RADESYS'] = 'ICRS' self.dataout.header['EQUINOX'] = 2000 self.dataout.header['LATPOLE'] = self.datain[0].header['CRVAL2'] self.dataout.header['LONPOLE'] = 180 self.dataout.header['PIXASEC'] = pscale theta_rad = np.deg2rad(self.getarg('outangle')) rot_matrix = np.array([[np.cos(theta_rad), -np.sin(theta_rad)], [np.sin(theta_rad), np.cos(theta_rad)]]) rot_cd = np.dot(rot_matrix, np.array([[self.dataout.header['CD1_1'], 0.],[0., self.dataout.header['CD2_2']]])) for i in [0,1]: for j in [0,1]: self.dataout.header['CD{0:d}_{1:d}'.format(i+1, j+1)] = rot_cd[i,j] #check drizzle arguments if self.getarg('kernel') == 'smoothing': kernel = 'lanczos3' elif self.getarg('kernel') in ['square', 'point', 'gaussian', 'tophat']: kernel = self.getarg('kernel') else: self.log.error('Kernel name not recognized, using default') kernel = 'square' if self.getarg('drizzleweights') == 'uniform': driz_wt = '' elif self.getarg('drizzleweights') in ['exptime', 'expsq']: driz_wt = self.getarg('drizzleweights') else: self.log.error('Drizzle weighting not recognized, using default') driz_wt = '' #create drizzle object and add input images fullwcs = wcs.WCS(self.dataout.header) self.log.info('Starting drizzle') driz = drz.Drizzle(outwcs = fullwcs, pixfrac=self.getarg('pixfrac'), \ kernel=kernel, fillval='10000', wt_scl=driz_wt) for i,f in enumerate(self.datain): self.log.info('Adding %s to drizzle stack' % f.filename) driz.add_image(f.imgdata[0], wcs.WCS(f.header), inwht=weights[i]) try: fillval=float(self.getarg('fillval')) except: fillval=np.nan self.log.error('Fillvalue not recognized or missing, using default') #creates output fits file from drizzle output self.dataout.imageset(np.where(driz.outsci == 10000, fillval, driz.outsci)) self.dataout.imageset(driz.outwht,'OutWeight', self.dataout.header) self.dataout.filename = self.datain[0].filename #add history self.dataout.setheadval('HISTORY','Coadd: %d files combined with %s kernel, pixfrac %f at %f times resolution' \ % (len(self.datain), kernel, self.getarg('pixfrac'), self.getarg('resolution')))
def run(self): """ Runs the mosaicing algorithm. The self.datain is run through the code, the result is in self.dataout. """ #calculate platescale of first input image try: det = np.linalg.det(wcs.WCS(self.datain[0].header).wcs.cd) pscale = np.sqrt(np.abs(det))*3600. except: try: det = np.linalg.det(wcs.WCS(self.datain[0].header).wcs.pc) pscale = np.sqrt(np.abs(det))*3600. except: pscale = self.datain[0].header['PIXSCAL'] #filtering out images which are too far away from the others #passing images added to a list of (image, WCS) tuples ''' image_centers = [] for f in self.datain: image_centers.append((f.header['CRVAL1'], f.header['CRVAL2'])) filtered_datain = [] dist_list = [[[0]*(len(image_centers)-1)]*len(image_centers)] for i in range(len(image_centers)): for j in range(len(image_centers)-1): dist_list[i][j+1] = np.sqrt((image_)**2+()**2) ''' #calculations necessary for updating wcs information px = [] py = [] #in order to avoid NaN interactions, creating weight map weights=[] for f in self.datain: weights.append((np.where(np.isnan(f.image) == True, 0, 1))) for f in self.datain: px.extend(wcs.WCS(f.header).calc_footprint()[:,0]) py.extend(wcs.WCS(f.header).calc_footprint()[:,1]) x0 = (max(px)+min(px))/2. y0 = (max(py)+min(py))/2. sx = (max(px)-min(px))*np.cos(y0/180*np.pi) # arcsec sy = (max(py)-min(py)) # arcsec size = (sx*3600+self.getarg('pad')*2, sy*3600+self.getarg('pad')*2) xpix = size[0]//pscale ypix = size[1]//pscale cdelt = [pscale/3600.]*2 #create self.dataout and give it a copy of an input's header self.dataout = DataFits(config = self.config) self.dataout.header = self.datain[0].header.copy() #update header wcs information self.log.info('Creating new WCS header') self.dataout.header['CRPIX1'] = xpix/2 self.dataout.header['CRPIX2'] = ypix/2 self.dataout.header['CRVAL1'] = x0 self.dataout.header['CRVAL2'] = y0 self.dataout.header['CD1_1'] = -cdelt[0] self.dataout.header['CD1_2'] = self.dataout.header['CD2_1'] = 0. self.dataout.header['CD2_2'] = cdelt[1] self.dataout.header['NAXIS1'] = int(xpix) self.dataout.header['NAXIS2'] = int(ypix) self.dataout.header['CTYPE1'] = 'RA---TAN-SIP' self.dataout.header['CTYPE2'] = 'DEC--TAN-SIP' self.dataout.header['RADESYS'] = 'ICRS' self.dataout.header['EQUINOX'] = 2000 self.dataout.header['LATPOLE'] = self.datain[0].header['CRVAL2'] self.dataout.header['LONPOLE'] = 180 self.dataout.header['PIXASEC'] = pscale theta_rad = np.deg2rad(self.getarg('outangle')) rot_matrix = np.array([[np.cos(theta_rad), -np.sin(theta_rad)], [np.sin(theta_rad), np.cos(theta_rad)]]) rot_cd = np.dot(rot_matrix, np.array([[self.dataout.header['CD1_1'], 0.],[0., self.dataout.header['CD2_2']]])) for i in [0,1]: for j in [0,1]: self.dataout.header['CD{0:d}_{1:d}'.format(i+1, j+1)] = rot_cd[i,j] #check drizzle arguments if self.getarg('kernel') == 'smoothing': kernel = 'lanczos3' elif self.getarg('kernel') in ['square', 'point', 'gaussian', 'tophat']: kernel = self.getarg('kernel') else: self.log.error('Kernel name not recognized, using default') kernel = 'square' if self.getarg('drizzleweights') == 'uniform': driz_wt = '' elif self.getarg('drizzleweights') in ['exptime', 'expsq']: driz_wt = self.getarg('drizzleweights') else: self.log.error('Drizzle weighting not recognized, using default') driz_wt = '' #create drizzle object and add input images fullwcs = wcs.WCS(self.dataout.header) self.log.info('Starting drizzle') driz = drz.Drizzle(outwcs = fullwcs, pixfrac=self.getarg('pixfrac'), \ kernel=kernel, fillval='10000', wt_scl=driz_wt) for i,f in enumerate(self.datain): self.log.info('Adding %s to drizzle stack' % f.filename) driz.add_image(f.imgdata[0], wcs.WCS(f.header), inwht=weights[i]) try: fillval=float(self.getarg('fillval')) except: fillval=np.nan self.log.error('Fillvalue not recognized or missing, using default') #creates output fits file from drizzle output self.dataout.imageset(np.where(driz.outsci == 10000, fillval, driz.outsci)) self.dataout.imageset(driz.outwht,'OutWeight', self.dataout.header) self.dataout.filename = self.datain[0].filename #add history self.dataout.setheadval('HISTORY','Coadd: %d files combined with %s kernel, pixfrac %f at %f times resolution' \ % (len(self.datain), kernel, self.getarg('pixfrac'), self.getarg('resolution')))
class StepFlat(StepLoadAux, StepParent): """ HAWC Pipeline Flatfielding Step Object """ stepver = '0.1' # pipe step version def __init__(self): """ Constructor: Initialize data objects and variables """ # call superclass constructor (calls setup) super(StepFlat, self).__init__() # list of data and flats self.datalist = [] # used in run() for every new input data file # flat values self.flatloaded = 0 # indicates if flat has been loaded self.flats = [] # list containing arrays with flat values self.flatdata = DataFits() # Pipedata object containing the flat file # flat file info and header keywords to fit self.flatfile = '' # name of selected flat file self.fitkeys = [] # FITS keywords that have to fit self.keyvalues = [ ] # values of the keywords (from the first data file) # set configuration self.log.debug('Init: done') def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name = 'flat' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'fla' # Set Logger for this pipe step self.log = logging.getLogger('hawc.pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters self.paramlist.append([ 'reload', 'False', 'Set to True to look for new flat files for every input' ]) self.paramlist.append([ 'l0method', 'NO', 'Method to normalize data: NOne, REal, IMag and ABSolute ' + '(default = NO)' ]) self.paramlist.append([ 'datalist', [], 'List of data sets to flatten in intput file ' + '(default = [] i.e. only flatten image cube in first HDU)' ]) self.paramlist.append([ 'addfromfile', [], 'List of data sets from the flat file to add to the output data' + '(default = [] i.e. no data to add)' ]) # Get parameters for StepLoadAux, replace auxfile with flatfile self.loadauxsetup('flatfile') def run(self): """ Runs the flatfielding algorithm. The flatfielded data is returned in self.dataout """ ### Preparation # Load flat files if necessary if not self.flatloaded or self.getarg('reload'): self.loadflat() # Else: check data for correct instrument configuration else: for keyind in range(len(self.fitkeys)): if self.keyvalues[keyind] != self.datain.getheadval( self.fitkeys[keyind]): self.log.warn( 'New data has different FITS key value for keyword %s' % self.fitkeys[keyind]) ### Copy datain to dataout self.dataout = self.datain.copy() ### Apply Flatfield # Only one data set -> flatfield it datalist = self.getarg('datalist') if len(datalist) == 0: # Get Image image = self.datain.image.copy() # Flatfield it self.dataout.image = self.flatfield(image, self.flats[0]) # Loop through data sets else: for dataind in range(len(datalist)): dataitem = datalist[dataind] # Search for dataitem in self.datain images -> flatfield it if dataitem.upper() in self.datain.imgnames: image = self.datain.imageget(dataitem) image = self.flatfield(image, self.flats[dataind]) self.dataout.imageset(image, dataitem) continue # go to next dataitem (skip end of loop) # Search for dataitem in self.table columns try: image = self.datain.table[dataitem] except: msg = 'No data found for %s in file %s' % ( dataitem, self.datain.filename) self.log.error('Run: %s' % msg) raise ValueError(msg) # Flatfield image = self.flatfield(image, self.flats[dataind]) # Store image in dataout self.dataout.imageset(image, dataitem) # Remove table column from dataout self.dataout.tabledelcol(dataitem) ### Add additional image frames from flat file for dataitem in self.getarg('addfromfile'): ind = self.flatdata.imageindex(dataitem.upper()) if ind > -1: self.dataout.imageset( self.flatdata.imageget(dataitem), imagename=dataitem, imageheader=self.flatdata.getheader(dataitem)) continue ind = self.flatdata.tableindex(dataitem.upper()) if ind > -1: self.dataout.tableset( self.flatdata.tableget(dataitem), tablename=dataitem, tableheader=self.flatdata.getheader(dataitem)) continue msg = 'No data to add found for <%s>' % dataitem self.log.error('Run: %s' % msg) raise ValueError(msg) # Remove the instrumental configuration HDU if 'CONFIGURATION' in self.dataout.imgnames: self.dataout.imagedel('CONFIGURATION') ### Finish - cleanup # Update DATATYPE self.dataout.setheadval('DATATYPE', 'IMAGE') # Add flat file to History self.dataout.setheadval('HISTORY', 'FLAT: %s' % self.flatdata.filename) # Update PROCSTAT to level 2 self.dataout.setheadval('PROCSTAT', 'LEVEL_2') def loadflat(self): """ Loads the flat information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. """ ### Search for flat and load it into data object self.flatdata = self.loadauxfile() ### find flatfields data arrays and store them # get sizes of input data datalist = self.getarg('datalist') if len(datalist) == 0: # Empty datalist -> Flat first image in data with first flat self.checksize(self.datain.image.shape, self.flatdata.image.shape) self.log.debug( 'LoadFlat: Flatfielding first data image with first flat') self.flats = [self.flatdata.image] else: # There are items in datalist -> loop over items self.flats = [] # Check if necessary number of images in flatdata if len(self.flatdata.imgdata) < len(datalist): msg = 'Number of images in flatfield file < ' msg += 'number of entries in datalist' self.log.error('LoadFlat: %s' % msg) raise ValueError(msg) # Loop through datalist items for dataind in range(len(datalist)): dataitem = datalist[dataind] # Search for dataitem in self.datain images if dataitem.upper() in self.datain.imgnames: dataimg = self.datain.imageget(dataitem) self.log.debug('LoadFlat: Found image <%s> to flat' % dataitem) # Search dataitem in self.table columns else: try: dataimg = self.datain.table[dataitem] self.log.debug('LoadFlat: Found column <%s> to flat' % dataitem) except: msg = 'No data found for <%s>' % dataitem self.log.error('LoadFlat: %s' % msg) raise ValueError(msg) # Get dimensions - append flat to list self.checksize(dataimg.shape, self.flatdata.imgdata[dataind].shape) self.flats.append(self.flatdata.imgdata[dataind]) ### Ensure that data listed in addfromfile is present in flatfile for dataitem in self.getarg('addfromfile'): if dataitem.upper() in self.flatdata.imgnames: pass elif dataitem.upper() in self.flatdata.tabnames: pass else: msg = 'No data to add found for <%s>' % dataitem self.log.error('LoadFlat: %s' % msg) raise ValueError(msg) # Finish up self.flatloaded = 1 self.log.debug('LoadFlat: done') def flatfield(self, imgin, flat): """ Flatfields an array using flat. If r0method != NO then the real image is computed This method checks that imagein and flat are compatible """ # Check flatfield dimension self.checksize(imgin.shape, flat.shape) # Do r0method correction if necessary l0method = self.getarg('l0method') if l0method != 'NO': # Return L0 for chosen method if l0method == 'ABS': data = numpy.zeros(imgin.shape[1:], dtype=complex) data.real = imgin[0, ...].copy() data.imag = imgin[1, ...].copy() imgin = abs(data) elif l0method == 'IM': imgin = imgin[1, ...].copy() else: imgin = imgin[0, ...].copy() # Apply flatfield imgout = imgin * flat return imgout def checksize(self, datashape, flatshape): """ Checks that the shape of the flat is comptatible to be used with image data of datashape. Raises exceptions otherwise. """ if self.getarg('l0method').upper() != 'NO': datashape = datashape[1:] if len(datashape) >= len(flatshape): # Data has >= dimensions than flat -> compare begind = len(datashape) - len(flatshape) if datashape[begind:] != flatshape: msg = 'Flat does not fit data in file %s' % self.datain.filename self.log.error('FlatField: %s' % msg) raise ValueError(msg) else: # More dimensions in flat data -> report error msg = 'Flat does not fit data in file %s' % self.datain.filename self.log.error('LoadFlat: %s' % msg) raise ValueError(msg) def reset(self): """ Resets the step to the same condition it was when it was created. Stored flatfield data is erased and the configuration information is cleared. """ # initialize input and output self.flatloaded = 0 self.flatvalue = numpy.zeros([1, 1]) self.flatphase = numpy.zeros([1, 1]) self.flatheader = fits.PrimaryHDU(numpy.array(1)) self.flatfile = '' def test(self): """ Test Pipe Step Flat Object: Runs basic tests """ # initial log message self.log.info('Testing pipe step flat') # get testin and a configuration if self.config != None and len( self.config) > 2: # i.e. if real config is loaded testin = DataFits(config=self.config) else: testin = DataFits(config=self.testconf) # load sample data datain = DataFits(config=testin.config) #infile = 'mode_chop/120207_000_00HA012.chop.dmd.fits' infile = 'mode_chop/120306_000_00HA006.chop.dmd.fits' #infile = 'mode_chop/120402_000_00HA035.chop.dmd.fits' #infile = 'sharp/sharc2-048485.dmdsqr.fits' testfile = os.path.join(datain.config['testing']['testpath'], infile) #testfile = '/Users/berthoud/testfit.fits' datain.load(testfile) if False: # change data (make complex number array with # Re=0,1,2,3,4,5,6 . . . in time Im=0) dataval = numpy.ones(datain.image.shape) dataval[..., 1] = 0.0 inclist = numpy.arange(dataval.shape[0]) incshape = [1 + i - i for i in dataval.shape[0:-1]] incshape[0] = dataval.shape[0] inclist.shape = incshape dataval[..., 0] = dataval[..., 0] * inclist #datain.image=dataval # run first flat dataout = self(datain) #print dataout.image[100,...] # print 100th image #print dataout.image[range(0,dataval.shape[0],1000),0,0] # 1 val per img dataout.save() # final log message self.log.info('Testing pipe step flat - Done')
class StepAstrometry(StepParent): """ HAWC Pipeline Step Parent Object The object is callable. It requires a valid configuration input (file or object) when it runs. """ stepver = '0.2' # pipe step version def setup(self): """ ### Names and Parameters need to be Set Here ### Sets the internal names for the function and for saved files. Defines the input parameters for the current pipe step. Setup() is called at the end of __init__ The parameters are stored in a list containing the following information: - name: The name for the parameter. This name is used when calling the pipe step from command line or python shell. It is also used to identify the parameter in the pipeline configuration file. - default: A default value for the parameter. If nothing, set '' for strings, 0 for integers and 0.0 for floats - help: A short description of the parameter. """ ### Set Names # Name of the pipeline reduction step self.name = 'astrometry' # Shortcut for pipeline reduction step and identifier for # saved file names. self.procname = 'WCS' # Set Logger for this pipe step self.log = logging.getLogger('pipe.step.%s' % self.name) ### Set Parameter list # Clear Parameter list self.paramlist = [] # Append parameters self.paramlist.append([ 'astrocmd', 'cp %s %s', 'Command to call astrometry, should contain 2' + 'string placeholders for intput and output ' + 'filepathname' ]) self.paramlist.append( ['verbose', False, 'log full astrometry output at DEBUG level']) self.paramlist.append([ 'delete_temp', False, 'Flag to delete temporary files generated by astrometry' ]) self.paramlist.append( ['downsample', [2], 'List of downsample factors to try']) self.paramlist.append([ 'paramoptions', ['--guess-scale'], 'Parameter groups to run if the command fails' ]) self.paramlist.append( ['timeout', 300, 'Timeout for running astrometry (seconds)']) self.paramlist.append( ['ra', '', 'Option to manually set image center RA']) self.paramlist.append( ['dec', '', 'Option to manually set image center DEC']) self.paramlist.append([ 'searchradius', 5, 'Only search in indexes within "searchradius" (degrees) of the field center given by --ra and --dec (degrees)' ]) # confirm end of setup self.log.debug('Setup: done') def run(self): """ Runs the data reduction algorithm. The self.datain is run through the code, the result is in self.dataout. """ ### Preparation # construct a temp file name that astrometry will output fp = tempfile.NamedTemporaryFile(suffix=".fits", dir=os.getcwd()) # split off path name, because a path that is too long causes remap to # crash sometimes outname = os.path.split(fp.name)[1] fp.close() # Add input file path to ouput file and make new name outpath = os.path.split(self.datain.filename)[0] outnewname = os.path.join(outpath, outname.replace('.fits', '.new')) outwcsname = os.path.join(outpath, outname.replace('.fits', '.wcs')) # Make sure input data exists as file if not os.path.exists(self.datain.filename): self.datain.save() # Make command string rawcommand = self.getarg('astrocmd') % (self.datain.filename, outname) # get estimated RA and DEC center values from the config file or input FITS header raopt = self.getarg('ra') if raopt != '': ra = Angle(raopt, unit=u.hour).degree else: try: ra = Angle(self.datain.getheadval('RA'), unit=u.hour).degree except: ra = '' decopt = self.getarg('dec') if decopt != '': dec = Angle(decopt, unit=u.deg).degree else: try: dec = Angle(self.datain.getheadval('DEC'), unit=u.deg).degree except: dec = '' if (ra != '') and (dec != ''): # update command parameters to use these values rawcommand = rawcommand + ' --ra %f --dec %f --radius %f' % ( ra, dec, self.getarg('searchradius')) else: self.log.debug( 'FITS header missing RA/DEC -> searching entire sky') ### Run Astrometry: # This loop tries the downsample and param options until the fit is successful # need either --scale-low 0.5 --scale-high 2.0 --sort-column FLUX # or --guess-scale downsamples = self.getarg('downsample') paramoptions = self.getarg('paramoptions') for option in range(len(downsamples) * len(paramoptions)): #for downsample in self.getarg('downsample'): downsample = downsamples[option % len(downsamples)] paramoption = paramoptions[option // len(downsamples)] # Add options to command command = rawcommand + ' --downsample %d' % downsample + ' ' + paramoption optionstring = "Downsample=%s Paramopts=%s" % (downsample, paramoption[:10]) # Run the process - see note at the top of the file if using cron process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) self.log.debug('running command = %s' % command) # Wait for the process to be finished or timeout to be reached timeout = time.time() + self.getarg('timeout') while time.time() < timeout and process.poll() == None: time.sleep(1) poll = process.poll() if poll == None: process.kill() time.sleep(1) poll = process.poll() self.log.debug('command returns %d' % poll) if poll == 0 and os.path.exists(outnewname): self.log.debug('output file valid -> astrometry successful') break else: self.log.debug('output file missing -> astrometry failed') # Print the output from astrometry (cut if necessary) if self.getarg('verbose'): output = process.stdout.read().decode() if len(output) > 1000: outlines = output.split('\n') output = outlines[:10] + ['...', '...'] + outlines[-7:] output = '\n'.join(output) self.log.debug(output) ### Post processing # Read output file self.dataout = DataFits(config=self.config) self.log.debug('Opening astrometry.net output file %s' % outnewname) try: self.dataout.load(outnewname) self.dataout.filename = self.datain.filename except Exception as error: self.log.error("Unable to open astrometry. output file = %s" % outname) raise error self.log.debug('Successful parameter options = %s' % optionstring) # Add history message histmsg = 'Astrometry.Net: At downsample = %d, search took %d seconds' % ( downsample, time.time() - timeout + 300) self.dataout.setheadval('HISTORY', histmsg) # Add RA from astrometry w = wcs.WCS(self.dataout.header) n1 = float(self.dataout.header['NAXIS1'] / 2) n2 = float(self.dataout.header['NAXIS2'] / 2) ra, dec = w.all_pix2world(n1, n2, 1) self.dataout.header['CRPIX1'] = n1 self.dataout.header['CRPIX2'] = n2 self.dataout.header['CRVAL1'] = float(ra) self.dataout.header['CRVAL2'] = float(dec) self.dataout.header['RA'] = Angle(ra, u.deg).to_string(unit=u.hour, sep=':') self.dataout.header['Dec'] = Angle(dec, u.deg).to_string(sep=':') self.dataout.setheadval('HISTORY', 'Astrometry: Paramopts = ' + optionstring) # Delete temporary files if self.getarg('delete_temp'): os.remove(outnewname) os.remove(outwcsname) self.log.debug('Run: Done')