def reduce(ccd, master_dark, master_flat, readnoise, gain): """ reduce raw data frame for science. Steps: make dark, make flat, then complete reduction using ccdproc.ccd_process which dark subtracts and flat correct Args: path: path to calibration frames directory to be combined ccd: CCDData to be reduced readnoise: Quantity, read noise from CCD gain: Quantity, gain from CCD Returns: reduced frame """ # get exposure times for dark and data dark_exposure = master_dark.header['EXPTIME'] * u.s data_exposure = ccd.header['EXPTIME'] * u.s # assign units to gain and readnoise gain = gain * u.electron / u.adu readnoise = readnoise * u.electron # Gain correct dark and flat before processing master_dark = ccdproc.gain_correct(master_dark, gain) master_flat = ccdproc.gain_correct(master_flat, gain) reduced_ccd = ccdproc.ccd_process(ccd, oscan=None, trim=None, error=True, master_bias=None, dark_frame=master_dark, master_flat=master_flat, bad_pixel_mask=None, gain=gain, readnoise=readnoise, oscan_median=True, oscan_model=None, min_value=None, dark_exposure=dark_exposure, data_exposure=data_exposure, exposure_key=None, exposure_unit=u.s, dark_scale=True) return reduced_ccd
def create_master_bias(list_files, fitsfile=None, fits_section=None, gain=None, method='median', dfilter={'imagetyp':'bias'}, mask=None, key_find='find', invert_find=False, sjoin=','): if gain is not None and not isinstance(gain, u.Quantity): gain = gain * u.electron / u.adu lbias = [] list_files = getListFiles(list_files, dfilter, mask, key_find=key_find, invert_find=invert_find) for filename in list_files: ccd = CCDData.read(filename, unit= u.adu) trimmed = True if fits_section is not None else False ccd = ccdproc.trim_image(ccd, fits_section=fits_section, add_keyword={'trimmed': trimmed}) if gain is not None: ccd = ccdproc.gain_correct(ccd, gain) lbias.append(ccd) combine = ccdproc.combine(lbias, method=method) if gain is not None and not 'GAIN' in combine.header: combine.header.set('GAIN', gain.value, gain.unit) combine.header['CGAIN'] = True if gain is not None else False combine.header['IMAGETYP'] = 'BIAS' combine.header['CMETHOD'] = method combine.header['CCDVER'] = VERSION if sjoin is not None: combine.header['LBIAS'] = sjoin.join([os.path.basename(fits) for fits in list_files]) combine.header['NBIAS'] = len(list_files) if fitsfile is not None: combine.header['FILENAME'] = os.path.basename(fitsfile) combine.write(fitsfile, clobber=True) return combine
def cosmic_ray_corr(textlist_files, prefix_str='c'): """ Gain correction and Cosmic ray correction using LA Cosmic method. Args: textlist_files : A python list object with paths/names to the individual files. prefix_str : String appended to the name of newly created file Returns: None """ for filename in textlist_files: file_corr = CCDData.read(filename, unit=u.adu) file_corr = ccdp.gain_correct(file_corr, gain=GAIN) new_ccd = ccdp.cosmicray_lacosmic(file_corr, readnoise=READ_NOISE, sigclip=7, satlevel=SATURATION, niter=4, gain_apply=False, verbose=True) new_ccd.meta['crcorr'] = True new_ccd.data = new_ccd.data.astype('float32') new_ccd.write(prefix_str + filename, hdu_mask=None, hdu_uncertainty=None)
def realtimeRed(storePath, analyPath, masterDark): neos = ccdproc.ImageFileCollection(location=analyPath) neoList = [] for neo, fname in neos.hdus(return_fname=True): meta = neo.header meta['filename'] = fname neoList.append(ccdproc.CCDData(data=neo.data, header=meta, unit="adu")) masterBias_e = ccdproc.gain_correct(masterBias, gain=1 * u.electron / u.adu) masterDark_e = ccdproc.gain_correct(masterDark, gain=1 * u.electron / u.adu) masterFlat_e = ccdproc.gain_correct(masterFlat, gain=1 * u.electron / u.adu) for neo in neoList: neo_red = ccdproc.ccd_process(neo, master_bias=masterBias_e, dark_frame=masterDark_e, master_flat=masterFlat_e , gain=1 * u.electron / u.adu, readnoise=readnoise, min_value=1. , dark_exposure=darkExp * u.second, data_exposure=neo.header['exptime'] * u.second , exposure_unit=u.second, dark_scale=True) baseName = os.path.basename(neo.header['filename']) fits.writeto("{}{}_red.fits".format(storePath, baseName.split('.')[0]), neo_red.data, header=neo_red.header, overwrite=False)
def process(ccd, gain, oscan, tsec): """Basic CCD processing for required for data """ #oscan subtract ccd = ccdproc.subtract_overscan(ccd, overscan=oscan, median=True) #gain correct ccd = ccdproc.gain_correct(ccd, gain, gain_unit=u.electron / u.adu) #trim the image ccd = ccdproc.trim_image(ccd, fits_section=tsec) return ccd
def process(ccd, gain, oscan, tsec): """Basic CCD processing for required for data """ #oscan subtract ccd = ccdproc.subtract_overscan(ccd, overscan=oscan, median=True) #gain correct ccd = ccdproc.gain_correct(ccd, gain, gain_unit=u.electron/u.adu) #trim the image ccd = ccdproc.trim_image(ccd, fits_section=tsec) return ccd
def _perform(self): """ Returns an Argument() with the parameters that depend on this operation. """ self.log.info(f"Running {self.__class__.__name__} action") gain = self.action.args.meta.get('GAIN', None) if gain is not None: self.log.debug(f' Using gain = {gain}') if gain is None: gain = self.cfg['Telescope'].getfloat('gain', None) self.log.debug(f' Got gain from config: {gain}') self.log.debug(' Gain correcting data') self.action.args.ccddata = ccdproc.gain_correct( self.action.args.ccddata, gain, gain_unit=u.electron / u.adu) return self.action.args
def reduceFrames(self): self.getMedianForObjects() log.info('Reducing frames started') print 'Reducing frames started' for obj in self.objects.filesList: data = ccdproc.CCDData.read(obj, unit=u.adu) dataWithDeviation = ccdproc.create_deviation(data, gain=1.5*u.electron/u.adu, readnoise=5*u.electron) reducedObject = ccdproc.gain_correct(dataWithDeviation, 1.5*u.electron/u.adu) if self.biases.isExists: reducedObject = ccdproc.subtract_bias(reducedObject, self.biases.masterCCD) if self.darks.isExists: reducedObject = ccdproc.subtract_dark(reducedObject, self.darks.masterCCD, exposure_time=cfg.exptime, exposure_unit=u.second, scale=True) if self.flats.isExists: reducedObject = ccdproc.flat_correct(reducedObject, self.flats.masterCCD) self.directory = '../../Reduction/' + directoryName if not os.path.exists(self.directory): os.makedirs(self.directory) reducedObject.write(self.directory + '/' + obj, clobber=True) os.system('solve-field ' + self.directory + '/' + obj) # + ' --overwrite') objName, objExtension = os.path.splitext(self.directory + '/' + obj) if not os.path.exists(objName + '.new'): log.warning(objName + ' cannot be solved') else: newObjectsList.append(objName + '.new') log.info('Frame ' + objName + ' reduced') log.info('Reduced ' + str(len(newObjectsList)) + ' frames') print 'Reduced ' + str(len(newObjectsList)) + ' frames' self.clean()
def _perform(self): """ Returns an Argument() with the parameters that depends on this operation. """ self.log.info(f"Running {self.__class__.__name__} action") gain = self.action.args.kd.get('GAIN', None) if gain is not None: self.log.debug(f' Got gain from header: {gain}') if gain is None: gain = self.cfg['Telescope'].getfloat('gain', None) self.log.debug(f' Got gain from config: {gain}') self.action.args.kd.headers.append(fits.Header({'GAIN': gain})) for i, pd in enumerate(self.action.args.kd.pixeldata): self.log.debug(' Gain correcting pixeldata') self.action.args.kd.pixeldata[i] = ccdproc.gain_correct( pd, gain, gain_unit=u.electron / u.adu) return self.action.args
def create_master_flat(list_files, flat_filter=None, fitsfile=None, bias=None, fits_section=None, gain=None, method='median', key_filter='filter', dfilter={'imagetyp':'FLAT'}, mask=None, key_find='find', invert_find=False, sjoin=','): if gain is not None and not isinstance(gain, u.Quantity): gain = gain * u.electron / u.adu lflat = [] if dfilter is not None and key_filter is not None and flat_filter is not None: dfilter = addKeysListDict(dfilter, {key_filter: flat_filter}) list_files = getListFiles(list_files, dfilter, mask, key_find=key_find, invert_find=invert_find) if len(list_files) == 0: print ('WARNING: No FLAT files available for filter "%s"' % flat_filter) return for filename in list_files: ccd = CCDData.read(filename, unit= u.adu) trimmed = True if fits_section is not None else False ccd = ccdproc.trim_image(ccd, fits_section=fits_section, add_keyword={'trimmed': trimmed}) if gain is not None: ccd = ccdproc.gain_correct(ccd, gain) if bias is not None: if isinstance(bias, str): bias = fits2CCDData(bias, single=True) ccd = ccdproc.subtract_bias(ccd, bias) lflat.append(ccd) combine = ccdproc.combine(lflat, method=method) if gain is not None and not 'GAIN' in combine.header: combine.header.set('GAIN', gain.value, gain.unit) combine.header['CGAIN'] = True if gain is not None else False combine.header['IMAGETYP'] = 'FLAT' combine.header['CMETHOD'] = method combine.header['CCDVER'] = VERSION addKeyHdr(combine.header, 'MBIAS', getFilename(bias)) if sjoin is not None: combine.header['LFLAT'] = sjoin.join([os.path.basename(fits) for fits in list_files]) combine.header['NFLAT'] = len(list_files) if fitsfile is not None: combine.header['FILENAME'] = os.path.basename(fitsfile) combine.write(fitsfile, clobber=True) return combine
def ccds(self, masks=None, trim=None, **kwargs): """ Generator that yields each 'ccdproc.CCDData' objects in the collection. Parameters ---------- masks : str, list of str or optional Area to be masked. trim : str or optional Trim section. **kwargs : Any additional keywords are used to filter the items returned. Yields ------ 'ccdproc.CCDData' yield the next 'ccdproc.CCDData' in the collection. Examples -------- >>> from tuglib.io import FitsCollection >>> >>> mask = '[:, 1000:1046]' >>> trim = '[100:1988, :]' >>> images = FitsCollection( location='/home/user/data/fits/', gain=0.57, read_noise=4.11) >>> biases = images.ccds(OBJECT='BIAS', masks=mask, trim=trim) """ if masks is not None: if not isinstance(masks, (str, list, type(None))): raise TypeError( "'masks' should be 'str', 'list' or 'None' object.") if trim is not None: if not isinstance(trim, str): raise TypeError("'trim' should be a 'str' object.") tmp = np.full(len(self._collection), True, dtype=bool) if len(kwargs) != 0: for key, val in kwargs.items(): if key == 'filename': file_mask = np.array([ fnmatch.fnmatch(filename, kwargs['filename']) for filename in self._filenames_without_path ], dtype=bool) tmp = tmp & file_mask else: tmp = tmp & (self._collection[key.upper()] == val) if np.count_nonzero(tmp) == 0: yield None x = self._collection[tmp]['NAXIS1'][0] y = self._collection[tmp]['NAXIS2'][0] shape = (y, x) mask = None if masks is not None: mask = make_mask(shape, masks) for filename in self._collection[tmp]['filename']: ccd = CCDData.read(filename, unit=self._unit, output_verify='silentfix+ignore') ccd.mask = mask ccd = trim_image(ccd, trim) if (self._gain is not None) and (self._read_noise is not None): data_with_deviation = create_deviation( ccd, gain=self._gain, readnoise=self._read_noise, disregard_nan=self._disregard_nan) gain_corrected = gain_correct(data_with_deviation, self._gain) yield gain_corrected else: yield ccd
def __call__(self, collection=None, masks=None, trim=None, **kwargs): """ Generator that yields each 'ccdproc.CCDData' objects in the collection. Parameters ---------- collection : 'FitsCollection.collection' or optional Filtered collection. masks : str, list of str or optional Area to be masked. trim : str or optional Trim section. **kwargs : Any additional keywords are used to filter the items returned. Yields ------ 'ccdproc.CCDData' yield the next 'ccdproc.CCDData' in the collection. Examples -------- >>> from tuglib.io import FitsCollection >>> >>> mask = '[:, 1000:1046]' >>> trim = '[100:1988, :]' >>> >>> images = FitsCollection( location='/home/user/data/fits/', gain=0.57, read_noise=4.11) >>> >>> query = images['EXPTIME'] == 100.0 >>> sub_collections = images[query] >>> >>> ccds = images(sub_collections, masks=mask, trim=trim) """ if masks is not None: if not isinstance(masks, (str, list, type(None))): raise TypeError( "'masks' should be 'str', 'list' or 'None' object.") if trim is not None: if not isinstance(trim, str): raise TypeError("'trim' should be a 'str' object.") if collection is None: return self.ccds(masks=masks, trim=trim, **kwargs) tmp = np.full(len(collection), True, dtype=bool) if len(kwargs) != 0: for key, val in kwargs.items(): tmp = tmp & (collection[key] == val) x = collection[tmp]['NAXIS1'][0] y = collection[tmp]['NAXIS2'][0] shape = (y, x) mask = None if masks is not None: mask = make_mask(shape, masks) if (self._gain is not None) and (self._read_noise is not None): for filename in collection[tmp]['filename']: ccd = CCDData.read(filename, unit=self._unit) ccd.mask = mask ccd = trim_image(ccd, trim) data_with_deviation = create_deviation( ccd, gain=self._gain, readnoise=self._read_noise, disregard_nan=self._disregard_nan) gain_corrected = gain_correct(data_with_deviation, self._gain) yield gain_corrected else: for filename in collection[tmp]['filename']: ccd = CCDData.read(filename, unit=self._unit) ccd.mask = mask ccd = trim_image(ccd, trim) yield ccd
def convert_to_ccddata(images, gain=None, read_noise=None): """ Convert 'fits' file to 'ccdproc.CCCData' object. Parameters ---------- images : str or list of str. Images to converted. gain : float Gain (u.electron / u.adu) read_noise : float Read Noise (u.electron). Yields ------ 'ccdproc.CCDData' yield the next 'ccdproc.CCDData'. Examples -------- >>> from tuglib.io import convert_to_ccddata >>> from glob import glob >>> >>> images = glob('/home/user/data/image*.fits') >>> >>> ccds = convert_to_ccddata(images, gain=0.37, read_noise=4.11) """ if not isinstance(images, (str, list)): raise TypeError("'images' should be 'str' or 'list' object.") if not isinstance(gain, (type(None), float)): raise TypeError("'gain' should be 'None' or 'float' object.") if not isinstance(read_noise, (type(None), float)): raise TypeError("'read_noise' should be 'None' or 'float' object.") if isinstance(images, str): images = [images] unit = u.adu if gain is not None: gain = gain * u.electron / u.adu if read_noise is not None: read_noise = read_noise * u.electron for image in images: ccd = CCDData.read(image, unit=unit, output_verify='silentfix+ignore') if (gain is not None) and (read_noise is not None): data_with_deviation = create_deviation(ccd, gain=gain, readnoise=read_noise) gain_corrected = gain_correct(data_with_deviation, gain) yield gain_corrected else: yield ccd
def copy_files(source_dir, target_dir, config, logger): import os import re import shutil from glob import glob import astropy.units as u from astropy.io import fits from ccdproc import CCDData, gain_correct pat = os.sep.join((source_dir.strip('/'), '*')) in_list = [ f for f in glob(pat) if len(re.findall(config.file_template, f)) == 1 ] logger.info('Found {} raw files in {}.'.format(len(in_list), source_dir)) assert len(in_list) > 0, 'No files matched: {}'.format( config.file_template) files = [os.path.basename(fn) + '.fits' for fn in in_list] copied = 0 for ifn, ofn in zip(in_list, files): ofn = os.sep.join((target_dir, ofn)) if os.path.exists(ofn) and not config.reprocess_data: continue shutil.copy(ifn, ofn) os.chmod(ofn, 0o644) logger.debug('{} -> {}'.format(ifn, ofn)) copied += 1 # header fix ccd = CCDData.read(ofn, unit=u.adu) if ccd.meta['OBJECT'] == '2016R2 Pan-STARRS': ccd.meta['OBJECT'] = 'C/2016 R2 (PanSTARRS)' elif ccd.meta['OBJECT'] == '2016M1 Pan-STARRS': ccd.meta['OBJECT'] = 'C/2016 M1 (PanSTARRS)' elif ccd.meta['OBJECT'] == '2017T1 Heinze': ccd.meta['OBJECT'] = 'C/2017 T1 (Heinze)' elif ccd.meta['OBJECT'] == '2019Y1 ATLAS': ccd.meta['OBJECT'] = 'C/2019 Y1 (ATLAS)' elif ccd.meta['OBJECT'] == '21P Gia-Zin': ccd.meta['OBJECT'] = '21P/Giacobini-Zinner' elif ccd.meta['OBJECT'] == '29P Schwas-Wach': ccd.meta['OBJECT'] = '29P/Schwassmann-Wachmann 1' elif ccd.meta['OBJECT'] == '29P Schwas-Wach (Low': ccd.meta['OBJECT'] = '29P/Schwassmann-Wachmann 1' elif ccd.meta['OBJECT'] == '38P Ste-Ote': ccd.meta['OBJECT'] = '38P/Stephan-Oterma' elif ccd.meta['OBJECT'] == '46P Wirtanen': ccd.meta['OBJECT'] = '46P/Wirtanen' elif ccd.meta['OBJECT'] == '64P Swift-Gehrels': ccd.meta['OBJECT'] = '64P/Swift-Gehrels' elif ccd.meta['OBJECT'] == '123P Wes-Har': ccd.meta['OBJECT'] = '123P/West-Hartley' elif ccd.meta['OBJECT'] == '123P West-Hartley': ccd.meta['OBJECT'] = '123P/West-Hartley' elif ccd.meta['OBJECT'] == '2018Y1 Iwamoto': ccd.meta['OBJECT'] = 'C/2018 Y1 (Iwamoto)' elif ccd.meta['OBJECT'] == '2019Q4 Borisov (Lowe': ccd.meta['OBJECT'] = 'C/2019 Q4 (Borisov)' elif ccd.meta['OBJECT'] == '2019Q4 Borisov': ccd.meta['OBJECT'] = 'C/2019 Q4 (Borisov)' elif ccd.meta['OBJECT'] == '2I Borisov': ccd.meta['OBJECT'] = 'C/2019 Q4 (Borisov)' elif ccd.meta['OBJECT'] == '155P Shoemaker 3': ccd.meta['OBJECT'] = '155P/Shoemaker 3' ccd.meta['FILTER'] = (ccd.meta['FILTER1'] + ccd.meta['FILTER2']).replace('Open', '').strip() ccd = gain_correct(ccd, ccd.meta['gain'], gain_unit=u.electron / u.adu) ccd.write(ofn, overwrite=True) logger.info('{} files copied and gain corrected.'.format(copied)) return files
def ccd_process(ccd, oscan=None, trim=None, error=False, masterbias=None, bad_pixel_mask=None, gain=None, rdnoise=None, oscan_median=True, oscan_model=None): """Perform basic processing on ccd data. The following steps can be included: * overscan correction * trimming of the image * create edeviation frame * gain correction * add a mask to the data * subtraction of master bias The task returns a processed `ccdproc.CCDData` object. Parameters ---------- ccd: `ccdproc.CCDData` Frame to be reduced oscan: None, str, or, `~ccdproc.ccddata.CCDData` For no overscan correction, set to None. Otherwise proivde a region of `ccd` from which the overscan is extracted, using the FITS conventions for index order and index start, or a slice from `ccd` that contains the overscan. trim: None or str For no trim correction, set to None. Otherwise proivde a region of `ccd` from which the image should be trimmed, using the FITS conventions for index order and index start. error: boolean If True, create an uncertainty array for ccd masterbias: None, `~numpy.ndarray`, or `~ccdproc.CCDData` A materbias frame to be subtracted from ccd. bad_pixel_mask: None or `~numpy.ndarray` A bad pixel mask for the data. The bad pixel mask should be in given such that bad pixels havea value of 1 and good pixels a value of 0. gain: None or `~astropy.Quantity` Gain value to multiple the image by to convert to electrons rdnoise: None or `~astropy.Quantity` Read noise for the observations. The read noise should be in `~astropy.units.electron` oscan_median : bool, optional If true, takes the median of each line. Otherwise, uses the mean oscan_model : `~astropy.modeling.Model`, optional Model to fit to the data. If None, returns the values calculated by the median or the mean. Returns ------- ccd: `ccdproc.CCDData` Reduded ccd Examples -------- 1. To overscan, trim, and gain correct a data set: >>> import numpy as np >>> from astropy import units as u >>> from hrsprocess import ccd_process >>> ccd = CCDData(np.ones([100, 100]), unit=u.adu) >>> nccd = ccd_process(ccd, oscan='[1:10,1:100]', trim='[10:100, 1,100]', error=False, gain=2.0*u.electron/u.adu) """ # make a copy of the object nccd = ccd.copy() # apply the overscan correction if isinstance(oscan, ccdproc.CCDData): nccd = ccdproc.subtract_overscan(nccd, overscan=oscan, median=oscan_median, model=oscan_model) elif isinstance(oscan, six.string_types): nccd = ccdproc.subtract_overscan(nccd, fits_section=oscan, median=oscan_median, model=oscan_model) elif oscan is None: pass else: raise TypeError('oscan is not None, a string, or CCDData object') # apply the trim correction if isinstance(trim, six.string_types): nccd = ccdproc.trim_image(nccd, fits_section=trim) elif trim is None: pass else: raise TypeError('trim is not None or a string') # create the error frame if error and gain is not None and rdnoise is not None: nccd = ccdproc.create_deviation(nccd, gain=gain, rdnoise=rdnoise) elif error and (gain is None or rdnoise is None): raise ValueError( 'gain and rdnoise must be specified to create error frame') # apply the bad pixel mask if isinstance(bad_pixel_mask, np.ndarray): nccd.mask = bad_pixel_mask elif bad_pixel_mask is None: pass else: raise TypeError('bad_pixel_mask is not None or numpy.ndarray') # apply the gain correction if isinstance(gain, u.quantity.Quantity): nccd = ccdproc.gain_correct(nccd, gain) elif gain is None: pass else: raise TypeError('gain is not None or astropy.Quantity') # test subtracting the master bias if isinstance(masterbias, ccdproc.CCDData): nccd = ccdproc.subtract_bias(nccd, masterbias) elif isinstance(masterbias, np.ndarray): nccd.data = nccd.data - masterbias elif masterbias is None: pass else: raise TypeError( 'masterbias is not None, numpy.ndarray, or a CCDData object') return nccd
def process_raw_frame(self, master_bias, master_flat, pixel_mask_spec=None): """ Bias and flat-correct a raw CCD frame. Trim off the overscan region. Identify cosmic rays using "lacosmic" and inflat uncertainties where CR's are found. If specified, mask out nearby sources by setting pixel uncertainty to infinity (or inverse-variance to 0). Returns ------- nccd : `ccdproc.CCDData` A copy of the original ``CCDData`` object but after the above procedures have been run. """ oscan_fits_section = "[{}:{},:]".format(self.oscan_idx, self.oscan_idx+self.oscan_size) # make a copy of the object nccd = self.ccd.copy() # apply the overscan correction poly_model = Polynomial1D(2) nccd = ccdproc.subtract_overscan(nccd, fits_section=oscan_fits_section, model=poly_model) # trim the image (remove overscan region) nccd = ccdproc.trim_image(nccd, fits_section='[1:{},:]'.format(self.oscan_idx)) # create the error frame nccd = ccdproc.create_deviation(nccd, gain=self.ccd_gain, readnoise=self.ccd_readnoise) # now correct for the ccd gain nccd = ccdproc.gain_correct(nccd, gain=self.ccd_gain) # correct for master bias frame # - this does some crazy shit at the blue end, but we can live with it nccd = ccdproc.subtract_bias(nccd, master_bias) # correct for master flat frame nccd = ccdproc.flat_correct(nccd, master_flat) # comsic ray cleaning - this updates the uncertainty array as well nccd = ccdproc.cosmicray_lacosmic(nccd, sigclip=8.) # replace ccd with processed ccd self.ccd = nccd # check for a pixel mask if pixel_mask_spec is not None: mask = self.make_nearby_source_mask(pixel_mask_spec) logger.debug("\t\tSource mask loaded.") stddev = nccd.uncertainty.array stddev[mask] = np.inf nccd.uncertainty = StdDevUncertainty(stddev) if self.plot_path is not None: # TODO: this assumes vertical CCD aspect_ratio = nccd.shape[1]/nccd.shape[0] fig,axes = plt.subplots(2, 1, figsize=(10,2 * 12*aspect_ratio), sharex=True, sharey=True) vmin,vmax = self.zscaler.get_limits(nccd.data) axes[0].imshow(nccd.data.T, origin='bottom', cmap=self.cmap, vmin=max(0,vmin), vmax=vmax) stddev = nccd.uncertainty.array vmin,vmax = self.zscaler.get_limits(stddev[np.isfinite(stddev)]) axes[1].imshow(stddev.T, origin='bottom', cmap=self.cmap, vmin=max(0,vmin), vmax=vmax) axes[0].set_title('Object: {0}, flux'.format(self._obj_name)) axes[1].set_title('root-variance'.format(self._obj_name)) fig.tight_layout() fig.savefig(path.join(self.plot_path, '{}_frame.png'.format(self._filename_base))) plt.close(fig) return nccd
print ('\n') # make an image collection of all the files in the data directory # z is prefix for galaxies that already have cosmic ray rejection ic = ImageFileCollection(os.getcwd(),keywords='*',glob_include='*.fit*') # combine bias frames if args.zerocombine: # select all files with imagetyp=='bias' bias_files = ic.files_filtered(imagetyp = ccdkeyword['bias']) # feed list into ccdproc.combine, output bias master_bias = ccdproc.combine(bias_files,method='average',sigma_clip=True,unit=u.adu) gaincorrected_master_bias = ccdproc.gain_correct(master_bias,float(gain)) print('writing fits file for master bias') gaincorrected_master_bias.write('bias-combined.fits',overwrite=True) else: if args.bias: print('not combining zeros') print('\t reading in bias-combined.fits instead') hdu1 = fits.open('bias-combined.fits') header = hdu1[0].header gaincorrected_master_bias = CCDData(hdu1[0].data, unit=u.electron, meta=header) hdu1.close() else: print('not using a bias frame') ################################################### # COMBINE DARKS ###################################################
def main(night_path, skip_list_file, mask_file, overwrite=False, plot=False): """ See argparse block at bottom of script for description of parameters. """ night_path = path.realpath(path.expanduser(night_path)) if not path.exists(night_path): raise IOError("Path '{}' doesn't exist".format(night_path)) logger.info("Reading data from path: {}".format(night_path)) base_path, night_name = path.split(night_path) data_path, run_name = path.split(base_path) output_path = path.realpath( path.join(data_path, 'processed', run_name, night_name)) os.makedirs(output_path, exist_ok=True) logger.info("Saving processed files to path: {}".format(output_path)) if plot: # if we're making plots plot_path = path.realpath(path.join(output_path, 'plots')) logger.debug("Will make and save plots to: {}".format(plot_path)) os.makedirs(plot_path, exist_ok=True) else: plot_path = None # check for files to skip (e.g., saturated or errored exposures) if skip_list_file is not None: # a file containing a list of filenames to skip with open(skip_list_file, 'r') as f: skip_list = [x.strip() for x in f if x.strip()] else: skip_list = None # look for pixel mask file if mask_file is not None: with open( mask_file, 'r' ) as f: # load YAML file specifying pixel masks for nearby sources pixel_mask_spec = yaml.load(f.read()) else: pixel_mask_spec = None # generate the raw image file collection to process ic = GlobImageFileCollection(night_path, skip_filenames=skip_list) logger.info("Frames to process:") logger.info("- Bias frames: {}".format( len(ic.files_filtered(imagetyp='BIAS')))) logger.info("- Flat frames: {}".format( len(ic.files_filtered(imagetyp='FLAT')))) logger.info("- Comparison lamp frames: {}".format( len(ic.files_filtered(imagetyp='COMP')))) logger.info("- Object frames: {}".format( len(ic.files_filtered(imagetyp='OBJECT')))) # HACK: ic = GlobImageFileCollection(night_path, skip_filenames=skip_list) # ============================ # Create the master bias frame # ============================ # overscan region of the CCD, using FITS index notation oscan_fits_section = "[{}:{},:]".format(oscan_idx, oscan_idx + oscan_size) master_bias_file = path.join(output_path, 'master_bias.fits') if not os.path.exists(master_bias_file) or overwrite: # get list of overscan-subtracted bias frames as 2D image arrays bias_list = [] for hdu, fname in ic.hdus(return_fname=True, imagetyp='BIAS'): logger.debug('Processing Bias frame: {0}'.format(fname)) ccd = CCDData.read(path.join(ic.location, fname), unit='adu') ccd = ccdproc.gain_correct(ccd, gain=ccd_gain) ccd = ccdproc.subtract_overscan(ccd, overscan=ccd[:, oscan_idx:]) ccd = ccdproc.trim_image(ccd, fits_section="[1:{},:]".format(oscan_idx)) bias_list.append(ccd) # combine all bias frames into a master bias frame logger.info("Creating master bias frame") master_bias = ccdproc.combine(bias_list, method='average', clip_extrema=True, nlow=1, nhigh=1, error=True) master_bias.write(master_bias_file, overwrite=True) else: logger.info("Master bias frame file already exists: {}".format( master_bias_file)) master_bias = CCDData.read(master_bias_file) if plot: # TODO: this assumes vertical CCD assert master_bias.shape[0] > master_bias.shape[1] aspect_ratio = master_bias.shape[1] / master_bias.shape[0] fig, ax = plt.subplots(1, 1, figsize=(10, 12 * aspect_ratio)) vmin, vmax = zscaler.get_limits(master_bias.data) cs = ax.imshow(master_bias.data.T, origin='bottom', cmap=cmap, vmin=max(0, vmin), vmax=vmax) ax.set_title('master bias frame [zscale]') fig.colorbar(cs) fig.tight_layout() fig.savefig(path.join(plot_path, 'master_bias.png')) plt.close(fig) # ============================ # Create the master flat field # ============================ # HACK: ic = GlobImageFileCollection(night_path, skip_filenames=skip_list) master_flat_file = path.join(output_path, 'master_flat.fits') if not os.path.exists(master_flat_file) or overwrite: # create a list of flat frames flat_list = [] for hdu, fname in ic.hdus(return_fname=True, imagetyp='FLAT'): logger.debug('Processing Flat frame: {0}'.format(fname)) ccd = CCDData.read(path.join(ic.location, fname), unit='adu') ccd = ccdproc.gain_correct(ccd, gain=ccd_gain) ccd = ccdproc.ccd_process(ccd, oscan=oscan_fits_section, trim="[1:{},:]".format(oscan_idx), master_bias=master_bias) flat_list.append(ccd) # combine into a single master flat - use 3*sigma sigma-clipping logger.info("Creating master flat frame") master_flat = ccdproc.combine(flat_list, method='average', sigma_clip=True, low_thresh=3, high_thresh=3) master_flat.write(master_flat_file, overwrite=True) # TODO: make plot if requested? else: logger.info("Master flat frame file already exists: {}".format( master_flat_file)) master_flat = CCDData.read(master_flat_file) if plot: # TODO: this assumes vertical CCD assert master_flat.shape[0] > master_flat.shape[1] aspect_ratio = master_flat.shape[1] / master_flat.shape[0] fig, ax = plt.subplots(1, 1, figsize=(10, 12 * aspect_ratio)) vmin, vmax = zscaler.get_limits(master_flat.data) cs = ax.imshow(master_flat.data.T, origin='bottom', cmap=cmap, vmin=max(0, vmin), vmax=vmax) ax.set_title('master flat frame [zscale]') fig.colorbar(cs) fig.tight_layout() fig.savefig(path.join(plot_path, 'master_flat.png')) plt.close(fig) # ===================== # Process object frames # ===================== # HACK: ic = GlobImageFileCollection(night_path, skip_filenames=skip_list) logger.info("Beginning object frame processing...") for hdu, fname in ic.hdus(return_fname=True, imagetyp='OBJECT'): new_fname = path.join(output_path, 'p_{}'.format(fname)) # ------------------------------------------- # First do the simple processing of the frame # ------------------------------------------- logger.debug("Processing '{}' [{}]".format(hdu.header['OBJECT'], fname)) if path.exists(new_fname) and not overwrite: logger.log(1, "\tAlready processed! {}".format(new_fname)) ext = SourceCCDExtractor(filename=path.join( ic.location, new_fname), plot_path=plot_path, zscaler=zscaler, cmap=cmap, **ccd_props) nccd = ext.ccd # HACK: F**K this is a bad hack ext._filename_base = ext._filename_base[2:] else: # process the frame! ext = SourceCCDExtractor(filename=path.join(ic.location, fname), plot_path=plot_path, zscaler=zscaler, cmap=cmap, unit='adu', **ccd_props) _pix_mask = pixel_mask_spec.get( fname, None) if pixel_mask_spec is not None else None nccd = ext.process_raw_frame(pixel_mask_spec=_pix_mask, master_bias=master_bias, master_flat=master_flat) nccd.write(new_fname, overwrite=overwrite) # ------------------------------------------- # Now do the 1D extraction # ------------------------------------------- fname_1d = path.join(output_path, '1d_{0}'.format(fname)) if path.exists(fname_1d) and not overwrite: logger.log(1, "\tAlready extracted! {}".format(fname_1d)) continue else: logger.debug("\tExtracting to 1D") # first step is to fit a voigt profile to a middle-ish row to determine LSF lsf_p = ext.get_lsf_pars() # MAGIC NUMBER try: tbl = ext.extract_1d(lsf_p) except Exception as e: logger.error('Failed! {}: {}'.format(e.__class__.__name__, str(e))) continue hdu0 = fits.PrimaryHDU(header=nccd.header) hdu1 = fits.table_to_hdu(tbl) hdulist = fits.HDUList([hdu0, hdu1]) hdulist.writeto(fname_1d, overwrite=overwrite) del ext # ============================== # Process comparison lamp frames # ============================== # HACK: ic = GlobImageFileCollection(night_path, skip_filenames=skip_list) logger.info("Beginning comp. lamp frame processing...") for hdu, fname in ic.hdus(return_fname=True, imagetyp='COMP'): new_fname = path.join(output_path, 'p_{}'.format(fname)) logger.debug("\tProcessing '{}'".format(hdu.header['OBJECT'])) if path.exists(new_fname) and not overwrite: logger.log(1, "\tAlready processed! {}".format(new_fname)) ext = CompCCDExtractor(filename=path.join(ic.location, new_fname), plot_path=plot_path, zscaler=zscaler, cmap=cmap, **ccd_props) nccd = ext.ccd # HACK: F**K this is a bad hack ext._filename_base = ext._filename_base[2:] else: # process the frame! ext = CompCCDExtractor(filename=path.join(ic.location, fname), plot_path=plot_path, unit='adu', **ccd_props) _pix_mask = pixel_mask_spec.get( fname, None) if pixel_mask_spec is not None else None nccd = ext.process_raw_frame( pixel_mask_spec=_pix_mask, master_bias=master_bias, master_flat=master_flat, ) nccd.write(new_fname, overwrite=overwrite) # ------------------------------------------- # Now do the 1D extraction # ------------------------------------------- fname_1d = path.join(output_path, '1d_{0}'.format(fname)) if path.exists(fname_1d) and not overwrite: logger.log(1, "\tAlready extracted! {}".format(fname_1d)) continue else: logger.debug("\tExtracting to 1D") try: tbl = ext.extract_1d() except Exception as e: logger.error('Failed! {}: {}'.format(e.__class__.__name__, str(e))) continue hdu0 = fits.PrimaryHDU(header=nccd.header) hdu1 = fits.table_to_hdu(tbl) hdulist = fits.HDUList([hdu0, hdu1]) hdulist.writeto(fname_1d, overwrite=overwrite)
def _adu2Electron(self, ccd): print('Converting from ADU to e-') return ccdproc.gain_correct(ccd, ccd.header['GAIN'], u.electron / u.adu)
def process_fits(fitspath, *, obstype=None, object=None, exposure_times=None, percentile=None, percentile_min=None, percentile_max=None, window=None, darks=None, cosmic_ray=False, cosmic_ray_kwargs={}, gain=None, readnoise=None, normalise=False, normalise_func=np.ma.average, combine_type=None, sigma_clip=False, low_thresh=3, high_thresh=3): """Combine all FITS images of a given type and exposure time from a given directory. Parameters ---------- fitspath: str Path to the FITS images to process. Can be a path to a single file, or a path to a directory. If the latter the directory will be searched for FITS files and checked against criteria from obstype, object, exposure_times critera. obstype: str, optional Observation type, an 'OBSTYPE' FITS header value e.g. 'DARK', 'OBJ'. If given only files with matching OBSTYPE will be processed. object: str, optional Object name, i.e. 'OBJECT' FITS header value. If given only files with matching OBJECT will be processed. exposure_times: float or sequence, optional Exposure time(s), i.e 'TOTALEXP' FITS header value(s). If given only files with matching TOTALEXP will be processed. percentile: float, optional If given will only images whose percentile value fall between percentile_min and percentile_max will be processed, e.g. set to 50.0 to select images by median value, set to 99.5 to select images by their 99.5th percentile value. percentile_min: float, optional Minimum percentile value. percentile_max: float, optional Maximum percentil value. window: (int, int, int, int), optional If given will trim images to the window defined as (x0, y0, x1, y1), where (x0, y0) and (x1, y1) are the coordinates of the bottom left and top right corners. darks: str or sequence, optional Filename(s) of dark frame(s) to subtract from the image(s). If given a dark frame with matching TOTALEXP will be subtracted from each image during processing. cosmic_ray: bool, optional Whether to perform single image cosmic ray removal, using the lacosmic algorithm, default False. Requires both gain and readnoise to be set. cosmic_ray_kwargs: dict, optional Additional keyword arguments to pass to the ccdproc.cosmicray_lacosmic function. gain: str or astropy.units.Quantity, optional Either a string indicating the FITS keyword corresponding to the (inverse gain), or a Quantity containing the gain value to use. If both gain and read noise are given an uncertainty frame will be created. readnoise: str or astropy.units.Quantity, optional Either a string indicating the FITS keyword corresponding to read noise, or a Quantity containing the read noise value to use. If both read noise and gain are given then an uncertainty frame will be created. normalise: bool, optional If True each image will be normalised. Default False. normalise_func: callable, optional Function to use for normalisation. Each image will be divided by normalise_func(image). Default np.ma.average. combine_type: str, optional Type of image combination to use, 'MEAN' or 'MEDIAN'. If None the individual images will be processed but not combined and the return value will be a list of CCDData objects. Default None. sigma_clip: bool, optional If True will perform sigma clipping on the image stack before combining, default=False. low_thresh: float, optional Lower threshold to use for sigma clipping, in standard deviations. Default is 3.0. high_thresh: float, optional Upper threshold to use for sigma clipping, in standard deviations. Default is 3.0. Returns ------- master: ccdproc.CCDData Combined image. """ if exposure_times: try: # Should work for any sequence or iterable type exposure_times = set(exposure_times) except TypeError: # Not a sequence or iterable, try using as a single value. exposure_times = { float(exposure_times), } if darks: try: dark_filenames = set(darks) except TypeError: dark_filenames = { darks, } dark_dict = {} for filename in dark_filenames: try: dark_data = CCDData.read(filename) except ValueError: # Might be no units in FITS header. Assume ADU. dark_data = CCDData.read(filename, unit='adu') dark_dict[dark_data.header['totalexp']] = dark_data if combine_type and combine_type not in ('MEAN', 'MEDIAN'): raise ValueError( "combine_type must be 'MEAN' or 'MEDIAN', got '{}''".format( combine_type)) fitspath = Path(fitspath) if fitspath.is_file(): # FITS path points to a single file, turn into a list. filenames = [ fitspath, ] elif fitspath.is_dir(): # FITS path is a directory. Find FITS file and collect values of selected FITS headers ifc = ImageFileCollection(fitspath, keywords='*') if len(ifc.files) == 0: raise RuntimeError("No FITS files found in {}".format(fitspath)) # Filter by observation type. if obstype: try: ifc = ifc.filter(obstype=obstype) except FileNotFoundError: raise RuntimeError( "No FITS files with OBSTYPE={}.".format(obstype)) # Filter by object name. if object: try: ifc = ifc.filter(object=object) except FileNotFoundError: raise RuntimeError( "No FITS files with OBJECT={}.".format(object)) filenames = [ Path(ifc.location).joinpath(filename) for filename in ifc.files ] else: raise ValueError( "fitspath '{}' is not an accessible file or directory.".format( fitspath)) # Load image(s) and process them. images = [] for filename in filenames: try: ccddata = CCDData.read(filename) except ValueError: # Might be no units in FITS header. Assume ADU. ccddata = CCDData.read(filename, unit='adu') # Filtering by exposure times here because it's hard filter ImageFileCollection # with an indeterminate number of possible values. if not exposure_times or ccddata.header['totalexp'] in exposure_times: if window: ccddata = ccdproc.trim_image(ccddata[window[1]:window[3] + 1, window[0]:window[2] + 1]) if percentile: # Check percentile value is within specified range, otherwise skip to next image. percentile_value = np.percentile(ccddata.data, percentile) if percentile_value < percentile_min or percentile_value > percentile_max: continue if darks: try: ccddata = ccdproc.subtract_dark( ccddata, dark_dict[ccddata.header['totalexp']], exposure_time='totalexp', exposure_unit=u.second) except KeyError: raise RuntimeError( "No dark with matching totalexp for {}.".format( filename)) if gain: if isinstance(gain, str): egain = ccddata.header[gain] egain = egain * u.electron / u.adu elif isinstance(gain, u.Quantity): try: egain = gain.to(u.electron / u.adu) except u.UnitsError: egain = (1 / gain).to(u.electron / u.adu) else: raise ValueError( f"gain must be a string or Quantity, got {gain}.") if readnoise: if isinstance(readnoise, str): rn = ccddata.header[readnoise] rn = rn * u.electron elif isinstance(readnoise, u.Quantity): try: rn = readnoise.to(u.electron / u.pixel) except u.UnitsError: rn = (readnoise * u.pixel).to(u.electron) else: raise ValueError( f"readnoise must be a string or Quantity, got {readnoise}." ) if gain and readnoise: ccddata = ccdproc.create_deviation(ccddata, gain=egain, readnoise=rn, disregard_nan=True) if gain: ccddata = ccdproc.gain_correct(ccddata, gain=egain) if cosmic_ray: if not gain and readnoise: raise ValueError( "Cosmic ray removal required both gain & readnoise.") ccddata = ccdproc.cosmicray_lacosmic( ccddata, gain=1.0, # ccddata already gain corrected readnoise=rn, **cosmic_ray_kwargs) if normalise: ccddata = ccddata.divide(normalise_func(ccddata.data)) images.append(ccddata) n_images = len(images) if n_images == 0: msg = "No FITS files match exposure time criteria" raise RuntimeError(msg) if n_images == 1 and combine_type: warn( "Combine type '{}' selected but only 1 matching image, skipping image combination.'" ) combine_type = None if combine_type: combiner = Combiner(images) # Sigma clip data if sigma_clip: if combine_type == 'MEAN': central_func = np.ma.average else: # If not MEAN has to be MEDIAN, checked earlier that it was one or the other. central_func = np.ma.median combiner.sigma_clipping(low_thresh=low_thresh, high_thresh=high_thresh, func=central_func) # Stack images. if combine_type == 'MEAN': master = combiner.average_combine() else: master = combiner.median_combine() # Populate header of combined image with metadata about the processing. master.header['fitspath'] = str(fitspath) if obstype: master.header['obstype'] = obstype if exposure_times: if len(exposure_times) == 1: master.header['totalexp'] = float(exposure_times.pop()) else: master.header['totalexp'] = tuple(exposure_times) master.header['nimages'] = n_images master.header['combtype'] = combine_type master.header['sigclip'] = sigma_clip if sigma_clip: master.header['lowclip'] = low_thresh master.header['highclip'] = high_thresh else: # No image combination, just processing indivudal image(s) if n_images == 1: master = images[0] else: master = images return master