def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None): masks = [] processed = [] for sci in sci_list: log.info('Loading file: '+sci) log.info('Applying gain correction, dark subtraction, overscan correction, flat correction, and trimming image.') raw = CCDData.read(sci,hdu=1,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron) log.info('Exposure time of science image is '+str(red.header['EXPTIME'])) log.info('Loading correct master dark') mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron) red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)#dark_exposure=1*u.second, data_exposure=red.header['EXPTIME']*u.second, scale=True) red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3)) red = ccdproc.flat_correct(red, mflat) processed_data = ccdproc.ccd_process(red, trim=raw.header['DATASEC']) log.info('File proccessed and trimmed.') log.info('Cleaning cosmic rays and creating mask.') mask = make_source_mask(processed_data, nsigma=3, npixels=5) masks.append(mask) clean, com_mask = create_mask.create_mask(sci,processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(red.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log) processed_data.data = clean log.info('Calculating 2D background.') bkg = Background2D(processed_data, (510, 510), filter_size=(9, 9),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) log.info('Median background: '+str(np.median(bkg.background))) fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits'),bkg.background,overwrite=True) final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['EXPTIME']*u.second,propagate_uncertainties=True,handle_meta='first_found') log.info('Background subtracted and image divided by exposure time.') final.write(sci.replace('/raw/','/red/'),overwrite=True) processed.append(final) return processed, masks
def ccdproc_images_filter(list_files, image_filter=None, master_flat=None, master_bias=None, fits_section=None, gain=None, readnoise=None, error=False, sky=True, dout=None, cosmic=False, mbox=15, rbox=15, gbox=11, cleantype="medmask", cosmic_method='lacosmic', sigclip=5, key_filter='filter', dfilter={'imagetyp':'LIGHT'}, mask=None, key_find='find', invert_find=False, **kwargs): if error and (gain is None or readnoise is None): print ('WARNING: You need to provide "gain" and "readnoise" to compute the error!') return if gain is not None and not isinstance(gain, u.Quantity): gain = gain * u.electron / u.adu if readnoise is not None and not isinstance(readnoise, u.Quantity): readnoise = readnoise * u.electron if dfilter is not None and key_filter is not None and image_filter is not None: dfilter = addKeysListDict(dfilter, {key_filter: image_filter}) list_files = getListFiles(list_files, dfilter, mask, key_find=key_find, invert_find=invert_find) dccd = {} for filename in list_files: ccd = CCDData.read(filename, unit= u.adu) nccd = ccdproc.ccd_process(ccd, trim=fits_section, gain=gain, master_bias=master_bias, master_flat=master_flat, readnoise=readnoise, error=error) for key in ccd.header: if not key in nccd.header: nccd.header[key] = ccd.header[key] # Better get rid of the cosmic rays BEFORE subtracting the global sky background if cosmic: nccd = cleanCosmic(nccd, mbox=mbox, rbox=rbox, gbox=gbox, sigclip=sigclip, cleantype=cleantype, cosmic_method=cosmic_method) if sky: nccd = subtract_sky_ccd(nccd, **kwargs) addKeyHdr(nccd.header, 'MBIAS', getFilename(master_bias)) addKeyHdr(nccd.header, 'MFLAT', getFilename(master_flat)) filename = 'c%s' % os.path.basename(filename) dccd[filename] = nccd filename = join_path(filename, dout) nccd.header['FILENAME'] = os.path.basename(filename) nccd.header['CCDVER'] = VERSION nccd.header = ammendHeader(nccd.header) nccd.write(filename, clobber=True) return dccd
def create_flat(flat_list,fil,red_path,mbias=None,log=None): log.info('Processing files for filter: '+fil) log.info(str(len(flat_list))+' files found.') flats = [] flat_scale = [] for flat in flat_list: log.info('Loading file: '+flat) raw = CCDData.read(flat,hdu=1,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron) log.info('Exposure time of image is '+str(red.header['EXPTIME'])) log.info('Loading correct master dark') mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron) red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second) red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3)) mask = make_source_mask(red, nsigma=3, npixels=5) bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) masked = np.array(red) masked[mask] = bkg.background[mask] log.info('Median flat level: '+str(np.median(masked))) norm = 1/np.median(masked[500:1500,500:1500]) log.info('Flat normalization: '+str(norm)) flat_scale.append(norm) flats.append(CCDData(masked,meta=red.header,unit=u.electron)) mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True) log.info('Created master flat for filter: '+fil) mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True) log.info('Master flat written to mflat_'+fil+'.fits') return
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None): masks = [] processed = [] for sci in sci_list: log.info('Loading file: '+sci) log.info('Applying gain correction and flat correction.') with fits.open(sci) as hdr: header = hdr[0].header data = hdr[0].data data[np.isnan(data)] = np.nanmedian(data) raw = CCDData(data,meta=header,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron) log.info('Exposure time of science image is '+str(red.header['TRUITIME']*red.header['COADDONE'])) flat = np.array(ccdproc.flat_correct(red, mflat)) flat[np.isnan(flat)] = np.nanmedian(flat) processed_data = CCDData(flat,unit=u.electron,header=red.header,wcs=red.wcs) log.info('File proccessed.') log.info('Cleaning cosmic rays and creating mask.') mask = make_source_mask(processed_data, nsigma=3, npixels=5) masks.append(mask) clean, com_mask = create_mask.create_mask(sci.replace('.gz',''),processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(raw.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log) processed_data.data = clean log.info('Calculating 2D background.') bkg = Background2D(processed_data, (128, 128), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) log.info('Median background: '+str(np.median(bkg.background))) fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits').replace('.gz',''),bkg.background,overwrite=True) final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['TRUITIME']*red.header['COADDONE']*u.second,propagate_uncertainties=True,handle_meta='first_found') log.info('Background subtracted and image divided by exposure time.') final.write(sci.replace('/raw/','/red/').replace('.gz',''),overwrite=True) processed.append(final) return processed, masks
def create_flat(flat_list,fil,red_path,mbias=None,log=None): log.info('Processing files for filter: '+fil) log.info(str(len(flat_list))+' files found.') flats = [] masks = [] flat_scale = [] for flat in flat_list: log.info('Loading file: '+flat) raw = CCDData.read(flat,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron) log.info('Exposure time of image is '+str(red.header['TRUITIME']*red.header['COADDONE'])) mask = make_source_mask(red, nsigma=3, npixels=5) bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) masked = np.array(red) masked[mask] = bkg.background[mask] log.info('Median flat level: '+str(np.median(masked))) norm = 1/np.median(masked[224:1824,224:1824]) log.info('Flat normalization: '+str(norm)) flat_scale.append(norm) flats.append(CCDData(masked,meta=red.header,unit=u.electron)) mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True) log.info('Created master flat for filter: '+fil) mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True) log.info('Master flat written to mflat_'+fil+'.fits') return
def m4kproc( fitsfd ): """m4kproc merges and trims the M4k Images """ dummy_name = "dummy.fits" if os.path.exists( dummy_name ): os.remove( dummy_name ) fitsfd.writeto( dummy_name ) #CRVALs are reversed. This will probably be fixe soon. # and this will need to be removed. for ext in [1,2]: ra=fitsfd[ext].header['CRVAL2'] dec=fitsfd[ext].header['CRVAL1'] fitsfd[ext].header['CRVAL2']=dec fitsfd[ext].header['CRVAL1']=ra amp1 = ccdproc.fits_ccddata_reader( dummy_name, hdu=1 ) amp2 = ccdproc.fits_ccddata_reader( dummy_name, hdu=2 ) procdata1 = ccdproc.ccd_process( amp1, oscan=fitsfd[1].header['biassec'], trim=fitsfd[1].header['TRIMSEC'] ) procdata2 = ccdproc.ccd_process( amp2, oscan=fitsfd[2].header['biassec'], trim=fitsfd[2].header['TRIMSEC'] ) os.remove( dummy_name ) return procdata1, procdata2
def reduce(ccd, master_dark, master_flat, readnoise, gain): """ reduce raw data frame for science. Steps: make dark, make flat, then complete reduction using ccdproc.ccd_process which dark subtracts and flat correct Args: path: path to calibration frames directory to be combined ccd: CCDData to be reduced readnoise: Quantity, read noise from CCD gain: Quantity, gain from CCD Returns: reduced frame """ # get exposure times for dark and data dark_exposure = master_dark.header['EXPTIME'] * u.s data_exposure = ccd.header['EXPTIME'] * u.s # assign units to gain and readnoise gain = gain * u.electron / u.adu readnoise = readnoise * u.electron # Gain correct dark and flat before processing master_dark = ccdproc.gain_correct(master_dark, gain) master_flat = ccdproc.gain_correct(master_flat, gain) reduced_ccd = ccdproc.ccd_process(ccd, oscan=None, trim=None, error=True, master_bias=None, dark_frame=master_dark, master_flat=master_flat, bad_pixel_mask=None, gain=gain, readnoise=readnoise, oscan_median=True, oscan_model=None, min_value=None, dark_exposure=dark_exposure, data_exposure=data_exposure, exposure_key=None, exposure_unit=u.s, dark_scale=True) return reduced_ccd
def tiled_process(bads: List[CCDData], flat: CCDData, images: List[CCDData]) -> List[CCDData]: """ Do the ccdproc operation on a list of images. includes some extra logic for NOTCAM images to get the gain and readnoise out of the headers :param bads: :param flat: :param images: :return: """ if bads: bad = reduce(lambda x, y: x.astype(bool) | y.astype(bool), (i.data for i in bads)) # combine bad pixel masks else: bad = None reduceds = [] for image in images: image.mask = bad # TODO this is really sketchy as it does 4x the work # Tiling: http://www.not.iac.es/instruments/notcam/guide/observe.html#reductions # 0-> LL, 1->LR, 2->UR, 3->UL tile_table = [ None, s_[512:, 0:512], s_[512:, 512:], s_[0:512, 512:], s_[0:512, 0:512] ] reduced = image.copy() for idx in range(1, 5): gain = image.header['GAIN' + str(idx)] readnoise = image.header['RDNOISE' + str(idx)] tile = ccdproc.ccd_process( image, oscan=None, error=True, gain=gain * u.electron / u.count, # TODO check if this is right or counts->adu required readnoise=readnoise * u.electron, dark_frame=None, master_flat=flat, bad_pixel_mask=bad) image.data[tile_table[idx]] = tile.data[tile_table[idx]] reduced.header = tile.header reduced.wcs = tile.wcs reduced.unit = tile.unit reduceds.append(reduced) return reduceds
def realtimeRed(storePath, analyPath, masterDark): neos = ccdproc.ImageFileCollection(location=analyPath) neoList = [] for neo, fname in neos.hdus(return_fname=True): meta = neo.header meta['filename'] = fname neoList.append(ccdproc.CCDData(data=neo.data, header=meta, unit="adu")) masterBias_e = ccdproc.gain_correct(masterBias, gain=1 * u.electron / u.adu) masterDark_e = ccdproc.gain_correct(masterDark, gain=1 * u.electron / u.adu) masterFlat_e = ccdproc.gain_correct(masterFlat, gain=1 * u.electron / u.adu) for neo in neoList: neo_red = ccdproc.ccd_process(neo, master_bias=masterBias_e, dark_frame=masterDark_e, master_flat=masterFlat_e , gain=1 * u.electron / u.adu, readnoise=readnoise, min_value=1. , dark_exposure=darkExp * u.second, data_exposure=neo.header['exptime'] * u.second , exposure_unit=u.second, dark_scale=True) baseName = os.path.basename(neo.header['filename']) fits.writeto("{}{}_red.fits".format(storePath, baseName.split('.')[0]), neo_red.data, header=neo_red.header, overwrite=False)
def create_dark(cal_list,cal,mbias,red_path,log): images = [] processed = [] for dark in cal_list[cal]: log.info('Creating master dark with exposure time: '+cal.split('_')[1]) log.info('Loading file: '+dark) try: raw = CCDData.read(dark, hdu=1, unit='adu') images.append(dark) except astropy.io.registry.IORegistryError: log.error('File not recognized.') red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron, master_bias=mbias) processed.append(red) log.info('Creating master dark.') mdark = ccdproc.combine(processed,method='median') log.info('Master dark created.') mdark_hdu = fits.PrimaryHDU(mdark) mdark_hdu.header['VER'] = (__version__, 'Version of telescope parameter file used.') for i,im in enumerate(images): mdark_hdu.header['FILE'+str(i+1)] = im mdark_hdu.header['EXPTIME'] = (np.float(cal.split('_')[1]), 'Exposure time of master dark (sec).') mdark_hdu.writeto(red_path+cal+'.fits',overwrite=True) log.info('Master dark written to '+cal+'.fits') return
def single_reduction(image, bad, flat): image.mask = bad if "GAIN" in image.header: # ALF gain = image.header['GAIN'] readnoise = image.header['RDNOISE'] else: # NOTCAM # TODO that's from the quicklook-package, probably would want to do this individually for every sensor area gain = (image.header['GAIN1'] + image.header['GAIN2'] + image.header['GAIN3'] + image.header['GAIN4']) / 4 readnoise = (image.header['RDNOISE1'] + image.header['RDNOISE2'] + image.header['RDNOISE3'] + image.header['RDNOISE4']) / 4 reduced = ccdproc.ccd_process( image, oscan=None, error=True, gain=gain * u.electron / u.count, # TODO check if this is right or counts->adu required readnoise=readnoise * u.electron, dark_frame=None, master_flat=flat, bad_pixel_mask=bad, add_keyword=None) return reduced
infiles = args.infile print infiles if args.bias: mbias = CCDData.read(args.bias, unit = u.adu) else: mbias = None if args.flat: raise Exception('Flat fielding is not currently implemented') if args.dark: raise Exception('Dark correction is not currently implemented') for infile in infiles: ccd = CCDData.read(infile, unit = u.adu) ccd = ccdproc.ccd_process(ccd, oscan='[1117:1181, 1:330]', oscan_median=True, trim='[17:1116,1:330]', master_bias=mbias, error=True, gain=1.0 * u.electron/u.adu, readnoise=5.0 * u.electron) if args.cray: ccd = ccdproc.cosmicray_lacosmic(ccd, sigclip=4.5, sigfrac=0.3, objlim=5.0, gain=1.0, readnoise=6.5, satlevel=65536.0, pssl=0.0, niter=4, sepmed=True, cleantype='meanmask', fsmode='median', psfmodel='gauss', psffwhm=2.5, psfsize=7, psfk=None, psfbeta=4.765, verbose=False) ccd.write('p'+os.path.basename(infile), clobber=True)
#%% f_name_bias = 'bias-median.fits' f_name_dark = 'dark-median.fits' f_name_flat0 = 'flat0-median.fits' #%% # (1) Open master bias, dark, and flat bias = CCDData.read(dir_name + f_name_bias, unit='adu') dark = CCDData.read(dir_name + f_name_dark, unit='adu') flat0 = CCDData.read(dir_name + f_name_flat0, unit='adu') # Bias NOT subtracted #%% # (2) Reduce each object image separately. # Then save it with prefix 'p_' which means "preprocessed" for objname in file_list: obj = CCDData.read(objname, unit='adu') reduced = ccd_process( obj, # The input image master_bias=bias, # Master bias dark_frame=dark, # dark master_flat=flat0, # non-calibrated flat min_value=30000, # flat.min() should be 30000 exposure_key='exptime', # exposure time keyword exposure_unit=u.s, # exposure time unit dark_scale=True) # whether to scale dark frame reduced.data = np.array(reduced.data, dtype=np.uint16) print('reduced', reduced.data.dtype, reduced.data.min(), reduced.data.max(), reduced) reduced.write(objname[:-5] + '_p.fits', overwrite=True)
def create_bias(cal_list): for i, bias_file in enumerate(cal_list['BIAS4']): with fits.open(bias_file) as hdr: header = hdr[0].header bias_raw = [ CCDData.read(bias_file, hdu=x + 1, unit='adu') for x in range(4) ] bias_processed = [ ccdproc.ccd_process(x, oscan=x[:, 0:13], oscan_model=models.Chebyshev1D(3), trim='[13:2060,1:2601]', gain=gain * u.electron / u.adu, readnoise=rdnoise * u.electron) for j, x in enumerate(bias_raw) ] bias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)]) for x in bias_processed: bias_hdu.append(fits.ImageHDU(x.data, header=x.header)) bias_hdu.writeto(bias_file.replace(cal_path, red_path), overwrite=True) cal_list['BIAS4'][i] = bias_file.replace(cal_path, red_path) if len(cal_list['BIAS4']) != 0: mbias4 = [ ccdproc.combine(cal_list['BIAS4'], hdu=x + 1, unit=u.electron) for x in range(4) ] mbias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)]) for x in mbias4: mbias_hdu.append(fits.ImageHDU(x.data, header=x.header)) mbias_hdu.writeto(red_path + 'mbias4.fits', overwrite=True) for bias in cal_list['BIAS4']: os.remove(bias) else: mbias4 = None for i, bias_file in enumerate(cal_list['BIAS8']): with fits.open(bias_file) as hdr: header = hdr[0].header bias_raw = [ CCDData.read(bias_file, hdu=x + 1, unit='adu') for x in range(8) ] bias_processed = [ ccdproc.ccd_process(x, oscan=x[:, 0:13], oscan_model=models.Chebyshev1D(3), trim='[13:1036,1:2601]', gain=gain * u.electron / u.adu, readnoise=rdnoise * u.electron) for j, x in enumerate(bias_raw) ] bias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)]) for x in bias_processed: bias_hdu.append(fits.ImageHDU(x.data, header=x.header)) bias_hdu.writeto(bias_file.replace(cal_path, red_path), overwrite=True) cal_list['BIAS8'][i] = bias_file.replace(cal_path, red_path) if len(cal_list['BIAS8']) != 0: mbias8 = [ ccdproc.combine(cal_list['BIAS8'], hdu=x + 1, unit=u.electron) for x in range(8) ] mbias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)]) for x in mbias8: mbias_hdu.append(fits.ImageHDU(x.data, header=x.header)) mbias_hdu.writeto(red_path + 'mbias8.fits', overwrite=True) for bias in cal_list['BIAS8']: os.remove(bias) else: mbias8 = None return mbias4, mbias8
break scale = [] for i, flat_file in enumerate(cal_list[cal]): with fits.open(flat_file) as hdr: header = hdr[0].header data = hdr[int(int(cal[-1]) / 2)].data scale.append(1 / np.median(data[1200:2000, 800:1600])) flat_raw = [ CCDData.read(flat_file, hdu=x + 1, unit='adu') for x in range(int(cal[-1])) ] flat_processed = [ ccdproc.ccd_process(x, oscan=x[:, 0:13], oscan_model=models.Chebyshev1D(3), trim=x.header['DATASEC'], gain=gain * u.electron / u.adu, readnoise=rdnoise * u.electron, master_bias=mbias[k], gain_corrected=True) for k, x in enumerate(flat_raw) ] flat_hdu = fits.HDUList([fits.PrimaryHDU(header=header)]) for x in flat_processed: flat_hdu.append(fits.ImageHDU(x.data, header=x.header)) flat_hdu.writeto(flat_file.replace(cal_path, red_path), overwrite=True) cal_list[cal][i] = flat_file.replace(cal_path, red_path) mflat = [ ccdproc.combine(cal_list[cal], hdu=x + 1, unit=u.electron,
def main(night_path, skip_list_file, mask_file, overwrite=False, plot=False): """ See argparse block at bottom of script for description of parameters. """ night_path = path.realpath(path.expanduser(night_path)) if not path.exists(night_path): raise IOError("Path '{}' doesn't exist".format(night_path)) logger.info("Reading data from path: {}".format(night_path)) base_path, night_name = path.split(night_path) data_path, run_name = path.split(base_path) output_path = path.realpath( path.join(data_path, 'processed', run_name, night_name)) os.makedirs(output_path, exist_ok=True) logger.info("Saving processed files to path: {}".format(output_path)) if plot: # if we're making plots plot_path = path.realpath(path.join(output_path, 'plots')) logger.debug("Will make and save plots to: {}".format(plot_path)) os.makedirs(plot_path, exist_ok=True) else: plot_path = None # check for files to skip (e.g., saturated or errored exposures) if skip_list_file is not None: # a file containing a list of filenames to skip with open(skip_list_file, 'r') as f: skip_list = [x.strip() for x in f if x.strip()] else: skip_list = None # look for pixel mask file if mask_file is not None: with open( mask_file, 'r' ) as f: # load YAML file specifying pixel masks for nearby sources pixel_mask_spec = yaml.load(f.read()) else: pixel_mask_spec = None # generate the raw image file collection to process ic = GlobImageFileCollection(night_path, skip_filenames=skip_list) logger.info("Frames to process:") logger.info("- Bias frames: {}".format( len(ic.files_filtered(imagetyp='BIAS')))) logger.info("- Flat frames: {}".format( len(ic.files_filtered(imagetyp='FLAT')))) logger.info("- Comparison lamp frames: {}".format( len(ic.files_filtered(imagetyp='COMP')))) logger.info("- Object frames: {}".format( len(ic.files_filtered(imagetyp='OBJECT')))) # HACK: ic = GlobImageFileCollection(night_path, skip_filenames=skip_list) # ============================ # Create the master bias frame # ============================ # overscan region of the CCD, using FITS index notation oscan_fits_section = "[{}:{},:]".format(oscan_idx, oscan_idx + oscan_size) master_bias_file = path.join(output_path, 'master_bias.fits') if not os.path.exists(master_bias_file) or overwrite: # get list of overscan-subtracted bias frames as 2D image arrays bias_list = [] for hdu, fname in ic.hdus(return_fname=True, imagetyp='BIAS'): logger.debug('Processing Bias frame: {0}'.format(fname)) ccd = CCDData.read(path.join(ic.location, fname), unit='adu') ccd = ccdproc.gain_correct(ccd, gain=ccd_gain) ccd = ccdproc.subtract_overscan(ccd, overscan=ccd[:, oscan_idx:]) ccd = ccdproc.trim_image(ccd, fits_section="[1:{},:]".format(oscan_idx)) bias_list.append(ccd) # combine all bias frames into a master bias frame logger.info("Creating master bias frame") master_bias = ccdproc.combine(bias_list, method='average', clip_extrema=True, nlow=1, nhigh=1, error=True) master_bias.write(master_bias_file, overwrite=True) else: logger.info("Master bias frame file already exists: {}".format( master_bias_file)) master_bias = CCDData.read(master_bias_file) if plot: # TODO: this assumes vertical CCD assert master_bias.shape[0] > master_bias.shape[1] aspect_ratio = master_bias.shape[1] / master_bias.shape[0] fig, ax = plt.subplots(1, 1, figsize=(10, 12 * aspect_ratio)) vmin, vmax = zscaler.get_limits(master_bias.data) cs = ax.imshow(master_bias.data.T, origin='bottom', cmap=cmap, vmin=max(0, vmin), vmax=vmax) ax.set_title('master bias frame [zscale]') fig.colorbar(cs) fig.tight_layout() fig.savefig(path.join(plot_path, 'master_bias.png')) plt.close(fig) # ============================ # Create the master flat field # ============================ # HACK: ic = GlobImageFileCollection(night_path, skip_filenames=skip_list) master_flat_file = path.join(output_path, 'master_flat.fits') if not os.path.exists(master_flat_file) or overwrite: # create a list of flat frames flat_list = [] for hdu, fname in ic.hdus(return_fname=True, imagetyp='FLAT'): logger.debug('Processing Flat frame: {0}'.format(fname)) ccd = CCDData.read(path.join(ic.location, fname), unit='adu') ccd = ccdproc.gain_correct(ccd, gain=ccd_gain) ccd = ccdproc.ccd_process(ccd, oscan=oscan_fits_section, trim="[1:{},:]".format(oscan_idx), master_bias=master_bias) flat_list.append(ccd) # combine into a single master flat - use 3*sigma sigma-clipping logger.info("Creating master flat frame") master_flat = ccdproc.combine(flat_list, method='average', sigma_clip=True, low_thresh=3, high_thresh=3) master_flat.write(master_flat_file, overwrite=True) # TODO: make plot if requested? else: logger.info("Master flat frame file already exists: {}".format( master_flat_file)) master_flat = CCDData.read(master_flat_file) if plot: # TODO: this assumes vertical CCD assert master_flat.shape[0] > master_flat.shape[1] aspect_ratio = master_flat.shape[1] / master_flat.shape[0] fig, ax = plt.subplots(1, 1, figsize=(10, 12 * aspect_ratio)) vmin, vmax = zscaler.get_limits(master_flat.data) cs = ax.imshow(master_flat.data.T, origin='bottom', cmap=cmap, vmin=max(0, vmin), vmax=vmax) ax.set_title('master flat frame [zscale]') fig.colorbar(cs) fig.tight_layout() fig.savefig(path.join(plot_path, 'master_flat.png')) plt.close(fig) # ===================== # Process object frames # ===================== # HACK: ic = GlobImageFileCollection(night_path, skip_filenames=skip_list) logger.info("Beginning object frame processing...") for hdu, fname in ic.hdus(return_fname=True, imagetyp='OBJECT'): new_fname = path.join(output_path, 'p_{}'.format(fname)) # ------------------------------------------- # First do the simple processing of the frame # ------------------------------------------- logger.debug("Processing '{}' [{}]".format(hdu.header['OBJECT'], fname)) if path.exists(new_fname) and not overwrite: logger.log(1, "\tAlready processed! {}".format(new_fname)) ext = SourceCCDExtractor(filename=path.join( ic.location, new_fname), plot_path=plot_path, zscaler=zscaler, cmap=cmap, **ccd_props) nccd = ext.ccd # HACK: F**K this is a bad hack ext._filename_base = ext._filename_base[2:] else: # process the frame! ext = SourceCCDExtractor(filename=path.join(ic.location, fname), plot_path=plot_path, zscaler=zscaler, cmap=cmap, unit='adu', **ccd_props) _pix_mask = pixel_mask_spec.get( fname, None) if pixel_mask_spec is not None else None nccd = ext.process_raw_frame(pixel_mask_spec=_pix_mask, master_bias=master_bias, master_flat=master_flat) nccd.write(new_fname, overwrite=overwrite) # ------------------------------------------- # Now do the 1D extraction # ------------------------------------------- fname_1d = path.join(output_path, '1d_{0}'.format(fname)) if path.exists(fname_1d) and not overwrite: logger.log(1, "\tAlready extracted! {}".format(fname_1d)) continue else: logger.debug("\tExtracting to 1D") # first step is to fit a voigt profile to a middle-ish row to determine LSF lsf_p = ext.get_lsf_pars() # MAGIC NUMBER try: tbl = ext.extract_1d(lsf_p) except Exception as e: logger.error('Failed! {}: {}'.format(e.__class__.__name__, str(e))) continue hdu0 = fits.PrimaryHDU(header=nccd.header) hdu1 = fits.table_to_hdu(tbl) hdulist = fits.HDUList([hdu0, hdu1]) hdulist.writeto(fname_1d, overwrite=overwrite) del ext # ============================== # Process comparison lamp frames # ============================== # HACK: ic = GlobImageFileCollection(night_path, skip_filenames=skip_list) logger.info("Beginning comp. lamp frame processing...") for hdu, fname in ic.hdus(return_fname=True, imagetyp='COMP'): new_fname = path.join(output_path, 'p_{}'.format(fname)) logger.debug("\tProcessing '{}'".format(hdu.header['OBJECT'])) if path.exists(new_fname) and not overwrite: logger.log(1, "\tAlready processed! {}".format(new_fname)) ext = CompCCDExtractor(filename=path.join(ic.location, new_fname), plot_path=plot_path, zscaler=zscaler, cmap=cmap, **ccd_props) nccd = ext.ccd # HACK: F**K this is a bad hack ext._filename_base = ext._filename_base[2:] else: # process the frame! ext = CompCCDExtractor(filename=path.join(ic.location, fname), plot_path=plot_path, unit='adu', **ccd_props) _pix_mask = pixel_mask_spec.get( fname, None) if pixel_mask_spec is not None else None nccd = ext.process_raw_frame( pixel_mask_spec=_pix_mask, master_bias=master_bias, master_flat=master_flat, ) nccd.write(new_fname, overwrite=overwrite) # ------------------------------------------- # Now do the 1D extraction # ------------------------------------------- fname_1d = path.join(output_path, '1d_{0}'.format(fname)) if path.exists(fname_1d) and not overwrite: logger.log(1, "\tAlready extracted! {}".format(fname_1d)) continue else: logger.debug("\tExtracting to 1D") try: tbl = ext.extract_1d() except Exception as e: logger.error('Failed! {}: {}'.format(e.__class__.__name__, str(e))) continue hdu0 = fits.PrimaryHDU(header=nccd.header) hdu1 = fits.table_to_hdu(tbl) hdulist = fits.HDUList([hdu0, hdu1]) hdulist.writeto(fname_1d, overwrite=overwrite)
if args.skip_red == 'True' or args.skip_red == 'yes': if os.path.exists(red_path + 'mdark.fits'): mdark = CCDData.read(red_path + 'mdark.fits', unit=u.electron) process_bias = False else: log.error('No master dark found, creating master dark.') if process_dark: processed = [] for dark in dark_list: try: raw = CCDData.read(dark, hdu=1, unit='adu') except astropy.io.registry.IORegistryError: log.error('File ' + dark + ' not recognized.') processed.append( ccdproc.ccd_process( raw, gain=raw.header['GAIN'] * u.electron / u.adu, readnoise=raw.header['RDNOISE'] * u.electron)) mdark = ccdproc.combine(processed, method='median') mdark.write(red_path + 'mdark.fits', overwrite=True) else: con = input( 'No darks present. Continue without dark subtraction? (True or False) ' ) if con == 'True': log.error('No darks present, continuing without dark subtraction.') mdark = None else: log.critical('No darks present, check data before rerunning.') logging.shutdown() sys.exit(-1)
sys.exit(-1) process_bias = True if args.skip_red == 'True' or args.skip_red == 'yes': if os.path.exists(red_path+'mbias_blue.fits') and os.path.exists(red_path+'mbias_red.fits'): mbias_blue = [CCDData.read(red_path+'mbias_blue.fits', hdu=x+1, unit=u.electron) for x in range(4)] mbias_red = [CCDData.read(red_path+'mbias_red.fits', hdu=x+1, unit=u.electron) for x in range(4)] process_bias = False else: log.error('No bias master bias found, creating master bias.') if process_bias: for i, bias_file in enumerate(cal_list['BIAS_blue']): with fits.open(bias_file) as hdr: header = hdr[0].header bias_raw = [CCDData.read(bias_file, hdu=x+1, unit='adu') for x in range(4)] bias_processed = [ccdproc.ccd_process(x, oscan=x[:,0:51], oscan_model=models.Chebyshev1D(3), trim=x.header['DATASEC'], gain=gain_blue[j]*u.electron/u.adu, readnoise=rdnoise_blue[j]*u.electron) for j,x in enumerate(bias_raw)] bias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)]) for x in bias_processed: bias_hdu.append(fits.ImageHDU(x.data,header=x.header)) bias_hdu.writeto(bias_file.replace(cal_path,red_path),overwrite=True) cal_list['BIAS_blue'][i] = bias_file.replace(cal_path,red_path) mbias_blue = [ccdproc.combine(cal_list['BIAS_blue'],hdu=x+1,unit=u.electron) for x in range(4)] mbias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)]) for x in mbias_blue: mbias_hdu.append(fits.ImageHDU(x.data,header=x.header)) mbias_hdu.writeto(red_path+'mbias_blue.fits',overwrite=True) for bias in cal_list['BIAS_blue']: os.remove(bias) for i, bias_file in enumerate(cal_list['BIAS_red']): with fits.open(bias_file) as hdr: header = hdr[0].header bias_raw = [CCDData.read(bias_file, hdu=x+1, unit='adu') for x in range(4)] bias_processed = [ccdproc.ccd_process(x, oscan=x[:,0:7], oscan_model=models.Chebyshev1D(3), trim=x.header['DATASEC'], gain=gain_red[j]*u.electron/u.adu, readnoise=rdnoise_red[j]*u.electron) for j,x in enumerate(bias_raw)] bias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)])
def process_science(sci_list, fil, red_path, mbias=None, mflat=None, proc=None, log=None): masks = [] processed = [] flat_left = mflat[0] flat_right = mflat[1] left_list = [] right_list = [] if proc: for j, sci in enumerate(sci_list): log.info('Loading file: ' + sci) log.info('Applying flat correction and trimming left image.') left = CCDData.read(sci, hdu=1, unit=u.electron) left = ccdproc.flat_correct(left, flat_left) left = ccdproc.ccd_process(left, trim=left.header['DATASEC']) log.info('Left image proccessed and trimmed.') log.info('Cleaning cosmic rays and creating mask.') mask = make_source_mask(left, nsigma=3, npixels=5) clean, com_mask = create_mask.create_mask(sci, left, '_mask_left.fits', static_mask(proc)[0], mask, saturation(left.header), binning(proc, 'left'), rdnoise(left.header), cr_clean_sigclip(), cr_clean_sigcfrac(), cr_clean_objlim(), log) left.data = clean log.info('Calculating 2D background.') bkg = Background2D(left, (120, 120), filter_size=(3, 3), sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) log.info('Median background for left iamge: ' + str(np.median(bkg.background))) fits.writeto(sci.replace('/raw/', '/red/').replace('.fits', '_bkg_left.fits'), bkg.background, overwrite=True) left = left.subtract(CCDData(bkg.background, unit=u.electron), propagate_uncertainties=True, handle_meta='first_found') log.info('Exposure time of left image is ' + str(left.header['EXPTIME'])) left = left.divide(left.header['EXPTIME'], propagate_uncertainties=True, handle_meta='first_found') log.info( 'Background subtracted and image divided by exposure time.') left.header['DATASEC'] = '[1:' + str( np.shape(left)[1]) + ',1:' + str(np.shape(left)[0]) + ']' left_list.append(left) log.info('Applying flat correction and trimming right image.') right = CCDData.read(sci, hdu=2, unit=u.electron) right = ccdproc.flat_correct(right, flat_right) right = ccdproc.ccd_process(right, trim=right.header['DATASEC']) log.info('Right image proccessed and trimmed.') log.info('Cleaning cosmic rays and creating mask.') mask = make_source_mask(right, nsigma=3, npixels=5) clean, com_mask = create_mask.create_mask(sci, right, '_mask_right.fits', static_mask(proc)[1], mask, saturation(right.header), binning(proc, 'right'), rdnoise(right.header), cr_clean_sigclip(), cr_clean_sigcfrac(), cr_clean_objlim(), log) right.data = clean log.info('Calculating 2D background.') bkg = Background2D(right, (120, 120), filter_size=(3, 3), sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) log.info('Median background for right image : ' + str(np.median(bkg.background))) fits.writeto(sci.replace('/raw/', '/red/').replace('.fits', '_bkg_right.fits'), bkg.background, overwrite=True) right = right.subtract(CCDData(bkg.background, unit=u.electron), propagate_uncertainties=True, handle_meta='first_found') log.info('Exposure time of right image is ' + str(right.header['EXPTIME'])) right = right.divide(right.header['EXPTIME'], propagate_uncertainties=True, handle_meta='first_found') log.info( 'Background subtracted and image divided by exposure time.') right.header['DATASEC'] = '[1:' + str( np.shape(right)[1]) + ',1:' + str(np.shape(right)[0]) + ']' right_list.append(right) else: for j, sci in enumerate(sci_list): log.info('Loading file: ' + sci) log.info( 'Applying gain correction, overscan correction, flat correction, and trimming image.' ) with fits.open(sci) as hdr: header_left = hdr[1].header header_right = hdr[6].header data_list = [] for i in range(8): data = ccdproc.CCDData.read(sci, hdu=i + 1, unit=u.adu) red = ccdproc.ccd_process(data, oscan=data[:, 0:50], oscan_model=models.Chebyshev1D(3), trim='[1200:2098,210:2056]', gain=gain()[i] * u.electron / u.adu, readnoise=4 * u.electron) data_list.append(np.asarray(red).astype(np.float32)) top_left = np.concatenate( [data_list[0], np.fliplr(data_list[1])], axis=1) bot_left = np.flipud( np.concatenate( [data_list[3], np.fliplr(data_list[2])], axis=1)) left = CCDData(np.concatenate([top_left, bot_left]), unit=u.electron, header=header_left) left = ccdproc.flat_correct(left, flat_left[209:3903, 1149:2947]) log.info('Left image proccessed and trimmed.') log.info('Cleaning cosmic rays and creating mask.') mask = make_source_mask(left, nsigma=3, npixels=5) # clean, com_mask = create_mask.create_mask(sci,left,static_mask(proc)[0],mask,saturation(left.header),binning(proc,'left'),rdnoise(left.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log) # processed_data.data = clean log.info('Calculating 2D background.') bkg = Background2D(left, (120, 120), filter_size=(3, 3), sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) log.info('Median background for left image : ' + str(np.median(bkg.background))) fits.writeto(sci.replace('/raw/', '/red/').replace('.fits', '_bkg_left.fits'), bkg.background, overwrite=True) left = left.subtract(CCDData(bkg.background, unit=u.electron), propagate_uncertainties=True, handle_meta='first_found') log.info('Exposure time of left image is ' + str(left.header['EXPTIME'])) left = left.divide(left.header['EXPTIME'] * u.second, propagate_uncertainties=True, handle_meta='first_found') log.info( 'Background subtracted and image divided by exposure time.') left.header['DATASEC'] = '[1:1798,1:3694]' left.header['RADECSYS'] = 'ICRS' left.header['CUNIT1'] = 'deg' left.header['CUNIT2'] = 'deg' left.header['CTYPE1'] = 'RA---TAN' left.header['CTYPE2'] = 'DEC--TAN' left.header['CRPIX1'] = 2301 left.header['CRPIX2'] = 1846 coord = util.parse_coord(left.header['RA'], left.header['DEC']) left.header['CRVAL1'] = coord.ra.deg left.header['CRVAL2'] = coord.dec.deg left.header['PC1_1'] = -pixscale() / 3600 * np.sin( np.pi / 180. * (left.header['POSANG'] + 90)) left.header['PC1_2'] = pixscale() / 3600 * np.cos( np.pi / 180. * (left.header['POSANG'] + 90)) left.header['PC2_1'] = -pixscale() / 3600 * np.cos( np.pi / 180. * (left.header['POSANG'] + 90)) left.header['PC2_2'] = pixscale() / 3600 * np.sin( np.pi / 180. * (left.header['POSANG'] + 90)) left.write(sci.replace('/raw/', '/red/').replace('.fits', '_left.fits'), overwrite=True) left_list.append(left) top_right = np.concatenate( [data_list[6], np.fliplr(data_list[7])], axis=1) bot_right = np.flipud( np.concatenate( [data_list[5], np.fliplr(data_list[4])], axis=1)) right = CCDData(np.concatenate([top_right, bot_right]), unit=u.electron, header=header_right) right = ccdproc.flat_correct(right, flat_right[209:3903, 1149:2947]) log.info('Right image proccessed and trimmed.') log.info('Cleaning cosmic rays and creating mask.') mask = make_source_mask(right, nsigma=3, npixels=5) # clean, com_mask = create_mask.create_mask(sci,right,static_mask(proc)[1],mask,saturation(right.header),binning(proc,'right'),rdnoise(right.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log) # processed_data.data = clean log.info('Calculating 2D background.') bkg = Background2D(right, (120, 120), filter_size=(3, 3), sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) log.info('Median background for right image : ' + str(np.median(bkg.background))) fits.writeto(sci.replace('/raw/', '/red/').replace('.fits', '_bkg_right.fits'), bkg.background, overwrite=True) right = right.subtract(CCDData(bkg.background, unit=u.electron), propagate_uncertainties=True, handle_meta='first_found') log.info('Exposure time of right image is ' + str(right.header['EXPTIME'])) right = right.divide(right.header['EXPTIME'] * u.second, propagate_uncertainties=True, handle_meta='first_found') log.info( 'Background subtracted and image divided by exposure time.') right.header['DATASEC'] = '[1:1798,1:3694]' right.header['RADECSYS'] = 'ICRS' right.header['CUNIT1'] = 'deg' right.header['CUNIT2'] = 'deg' right.header['CTYPE1'] = 'RA---TAN' right.header['CTYPE2'] = 'DEC--TAN' right.header['CRPIX1'] = -504 right.header['CRPIX2'] = 1845 coord = util.parse_coord(right.header['RA'], right.header['DEC']) right.header['CRVAL1'] = coord.ra.deg right.header['CRVAL2'] = coord.dec.deg right.header['PC1_1'] = -pixscale() / 3600 * np.sin( np.pi / 180. * (right.header['POSANG'] + 90)) right.header['PC1_2'] = pixscale() / 3600 * np.cos( np.pi / 180. * (right.header['POSANG'] + 90)) right.header['PC2_1'] = -pixscale() / 3600 * np.cos( np.pi / 180. * (right.header['POSANG'] + 90)) right.header['PC2_2'] = pixscale() / 3600 * np.sin( np.pi / 180. * (right.header['POSANG'] + 90)) right.write(sci.replace('/raw/', '/red/').replace('.fits', '_right.fits'), overwrite=True) right_list.append(right) return [left_list, right_list], None
def ccdred_list(image_path_list, mbias_path, mflat_path): """ Given a list of FITS files process it by applying master calibrations and overscan. Arguments --------- image_file : list like with strings Path to the files to process mbias_path : str Path to the master bias mflat_path : str Path to the master flats File transformations -------------------- Re-write FITS file, with overscan, bias and flat corrections and updated header. Returns ------- None """ # Load masters master_bias = fits.open(mbias_path) master_flat = fits.open(mflat_path) image_indices = _check_image_extensions(master_bias) for image_file in image_path_list: # Looping over extensions for i in image_indices: # Get CCDData object for this extension ccd = CCDData.read(image_file, unit="adu", hdu=i) mbias_ccd = CCDData(data=master_bias[i].data, header=master_bias[i].header, unit="adu") mflat_ccd = CCDData(data=master_flat[i].data, header=master_flat[i].header, unit="adu") if 'CCDPROC' in ccd.header: print( f"Skipping image: {image_file:1s}[{i:1.0f}] - Already processed." ) continue # Overscan section variables biassec = None trimsec = None if "BIASSEC" in ccd.header: biassec = ccd.header["BIASSEC"] if "TRIMSEC" in ccd.header: trimsec = ccd.header["TRIMSEC"] # Applying corrections ccd = ccdproc.ccd_process(ccd, oscan=biassec, trim=trimsec, master_bias=mbias_ccd, master_flat=mbias_ccd) ## Can add more information on the function above for the pixel by ## pixel error calculation (e.g. gain, read noise). # Updating existing image ccd.header["CCDPROC"] = True # Added processed flag fits.update(image_file, ccd.data.astype(np.float32), header=ccd.header, ext=i) print(f"Processed image: {image_file}") master_bias.close() master_flat.close()
################################################# ######## FIND DARK WITH CLOSEST EXPOSURE TIME ################################################# # find dark with closest exposure time sci_exptime = header['EXPTIME'] # find dark with closest exposure time delta_t = abs(sci_exptime - dark_exptimes_set) closest_dark = dark_exptimes_set[delta_t == min(delta_t)] # open the appropriate dark hdu1 = fits.open('dark-combined-'+str(int(closest_dark[0]))+'.fits') dheader = hdu1[0].header gaincorrected_dark = CCDData(hdu1[0].data, unit=u.electron, meta=dheader) hdu1.close() newccd = ccdproc.ccd_process(ccd,error=True, gain=gain, readnoise=rdnoise, dark_frame=gaincorrected_dark, exposure_key='exposure', exposure_unit=u.second,master_flat = gaincorrected_master_flat, gain_corrected=True) header['HISTORY'] = '= Processing by ccdproc: dark, flat ' #fits.writeto('fdb'+f,newccd,header, overwrite=True) fits.writeto('fd'+f,newccd,header, overwrite=True) hdu1.close() else: print('skipping science frames') ''' # then move on to running scamp and swarp to solve WCS # and make mosaics # let's start 01-28-2019 data '''
dark_list, # ccdproc does not accept numpy.ndarray, but only python list. method='median', # default is average so I specified median. unit='adu') #%% #save fits file dark0.write(dir_name + f_name_dark0, overwrite=True) print('dark0', dark0.data.min(), dark0.data.max(), dark0) # This dark isn't bias subtracted yet, so let's subtract bias: #%% # (1) Open master bias bias = CCDData.read(dir_name + f_name_bias, unit='adu') print('bias', bias.data.min(), bias.data.max(), bias.data) # `unit='adu'` does not necessarily be set if you have header keyword for it. #%% # (2) Subtract bias: dark = ccd_process(dark0, master_bias=bias) print('dark', dark.data.min(), dark.data.max(), dark.data) # This automatically does "dark-bias" # You can do it by the function "subtract_bias", or just normal pythonic arithmetic. #%% # (3) Sigma clip the dark dark_clip = sigma_clip(dark) print('dark_clip', dark_clip.data.min(), dark_clip.data.max(), dark_clip.data) # by default, astropy.stats.sigma_clip does "3-sigma, 10-iterations" clipping. # You can tune the parameters by optional keywords (e.g., sigma, iters, ...). # dark_clip is "numpy masked array". It contains data, mask, and filled_value. # filled_value is the value which is used to fill the data which are masked (rejected). # I will change filled_value to be the median value. #%% # (4) For rejected pixels, insert the median value dark_clip.fill_value = np.median(dark_clip.data)
def imreduce(target, bias=None, dark=None, flat=None, expkey='EXPTIME', expunit=u.s, normkey='NORMALZD', biaskey='BIASSUB', maskfile=None, fixpix=False): # must read bias and dark in here since CCDData not picklable bias = CCDData.read(bias, unit='adu') if bias else None dark = CCDData.read(dark, unit='adu') if dark else None if dark: # if dark is already normalized, don't normalize again if normkey in dark.header and dark.header[normkey]: print(dark.header[normkey]) dark_exposure = None dark_scale = False else: dark_exposure = dark.header[expkey] * expunit dark_scale = True # if bias frame, bias subtract darks first if bias: # if dark frame is already bias subtracted, skip if biaskey in dark.header and dark.header[biaskey]: pass else: dark = subtract_bias(dark, bias) else: dark_exposure = None dark_scale = False if flat: if isinstance(flat, str): # flat passed in as string, must read to CCDData ffile = flat flat = CCDData.read(ffile, unit='adu') flat.header['OFNAME'] = ffile # ccdproc has issues with nans when flatfielding # -> replace nans with 1 mask = np.isnan(flat.data) flat.data[mask] = 1. if isinstance(target, str): # target passed in as string, must read to CCDData tfile = target target = CCDData.read(tfile, unit='adu') target.header['OFNAME'] = tfile data_exposure = target.header[expkey] * expunit # read in bad pixel mask if specified if maskfile: target.header.add_history('%s - masked points from %s' % (os.path.basename(__file__), maskfile)) maskfile = fits.getdata(maskfile) # do the ccd reduction! ccd = ccd_process( target, master_bias=bias, dark_frame=dark, master_flat=flat, #exposure_key=expkey,exposure_unit=expunit, data_exposure=data_exposure, dark_exposure=dark_exposure, dark_scale=dark_scale, bad_pixel_mask=maskfile) # fix bad pixels through grid interpolation if fixpix: if maskfile is None: warn(RuntimeWarning('No maskfile specified: --fixpix ignored')) else: from photutils.utils import interpolate_masked_data from photutils.utils import ShepardIDWInterpolator as idw from scipy.interpolate import griddata mask = np.asarray(maskfile, dtype=bool) goodpoints = np.where(~mask) goodvals = ccd.data[goodpoints] badpoints = np.where(mask) # interpolate and replace values res = griddata(goodpoints, goodvals, badpoints) ccd.data[badpoints] = res ''' mask = np.asarray(maskfile,dtype=bool) goodcoord = np.where(~mask) print(goodcoord,len(goodcoord)) goodval = ccd.data[goodcoord] print(len(goodval));exit() iz = idw(goodcoord,goodval) fixed = iz(np.where(mask)) ccd.data[mask] = fixed #fixed,_,_ = interpolate_masked_data(ccd.data,np.asarray(maskfile,dtype=bool)) #ccd.data = fixed ''' ccd.header.add_history('%s - interpolated over bad pixels' % os.path.basename(__file__)) return ccd.to_hdu(hdu_mask=None, hdu_uncertainty=None)
else: #default to use science files for master flat creation flat_type = 'sky' log.info('Using science files to create master flat') flat_list = sky_list[fil] flat_scale = lambda arr: 1.0 / np.ma.median( arr[224:1824, 224:1824]) #scaling function for normalization if len(flat_list) > 31: random.seed(4) flat_list = random.sample(flat_list, k=31) image_list = [] for image in flat_list: sci_raw = CCDData.read(image, unit=u.adu) image_list.append( ccdproc.ccd_process( sci_raw, gain=sci_raw.header['SYSGAIN'] * u.electron / u.adu, readnoise=readnoise[sci_raw.header['NUMREADS']] * u.electron)) mask = [ make_source_mask(x, nsigma=1, npixels=5) for x in image_list ] #####make into list weights = np.ones([len(image_list), 2048, 2048]) for i, n in enumerate(image_list): weights[i] *= np.median( n[224:1824, 224:1824]) #flats weighted by median in flat master_flat = ccdproc.combine( image_list, method='median', weights=weights, scale=flat_scale,
if args.dark_sigmaclip == 1: dark_sigmaclip = True else: dark_sigmaclip = False darkscales = np.array(darkscales) #print(darkscales) dark = combine(darklist_real, method=args.dark_method, unit='adu', sigma_clip=dark_sigmaclip, scale=darkscales) if bias != None: dark = ccd_process(dark, master_bias=bias) dark.write(mdark) hdulist = fits.open(mdark, mode='update') hdulist[0].header.set(args.fits_header_exptime, args.dark_exptime) hdulist[0].header.set(args.fits_header_ccdtemp, temperature) darkexp = args.dark_exptime hdulist.flush() hdulist.close() elif args.masterdark != None: darkcal = True mdark = args.masterdark if os.path.isfile(mdark) == False:
right.write( red_path + os.path.basename(sci).replace('.fits', '_right.fits'), overwrite=True) right_list.append(right) else: with fits.open(sci) as hdr: header_left = hdr[1].header header_right = hdr[6].header data_list = [] for i in range(8): data = ccdproc.CCDData.read(sci, hdu=i + 1, unit=u.adu) red = ccdproc.ccd_process( data, oscan=data[:, 0:50], oscan_model=models.Chebyshev1D(3), trim='[1200:2098,210:2056]', gain=gain[i] * u.electron / u.adu, readnoise=4 * u.electron) data_list.append(np.asarray(red).astype(np.float32)) top_left = np.concatenate( [data_list[0], np.fliplr(data_list[1])], axis=1) bot_left = np.flipud( np.concatenate( [data_list[3], np.fliplr(data_list[2])], axis=1)) left = CCDData(np.concatenate([top_left, bot_left]), unit=u.electron, header=header_left, wcs=wcs.WCS(header_left)) left = ccdproc.flat_correct(left, flat_left[209:3903, 1199:2997])
for x in range(12) ] process_bias = False else: log.error('No bias master bias found, creatging master bias.') if process_bias: for i, bias_file in enumerate(cal_list['BIAS']): with fits.open(bias_file) as hdr: header = hdr[0].header bias_raw = [ CCDData.read(bias_file, hdu=x + 1, unit='adu') for x in range(12) ] bias_processed = [ ccdproc.ccd_process(x, oscan=x.header['BIASSEC'], oscan_model=models.Chebyshev1D(3), trim=x.header['DATASEC'], gain=x.header['GAIN'] * u.electron / u.adu, readnoise=x.header['RDNOISE'] * u.electron) for x in bias_raw ] for x in bias_processed: x.header['DATASEC'] = ('[1:256,1:2112]', 'updated DATASEC') bias_hdu = fits.HDUList([fits.PrimaryHDU(header=header)]) for x in bias_processed: bias_hdu.append(fits.ImageHDU(x.data, header=x.header)) bias_hdu.writeto(bias_file.replace(cal_path, red_path).replace( '.fits.bz2', '.fits'), overwrite=True) cal_list['BIAS'][i] = bias_file.replace(cal_path, red_path).replace( '.fits.bz2', '.fits') mbias = [
def t120_makecosmetic(work_dir=t120.t120_data_path, orig_dir_root=t120.t120_orig_dir, reduc_dir_root=t120.t120_redu_dir, offset_dir_root=t120.t120_ofst_dir, flat_dir_root=t120.t120_flat_dir, dark_dir_root=t120.t120_dark_dir, common_dir_root=t120.t120_common_root, scampahead_file=t120.t120_scamp_ahead, master_offset_name=t120.t120_master_name): orig_dir = work_dir + orig_dir_root reduc_dir = work_dir + reduc_dir_root offset_dir = work_dir + offset_dir_root flat_dir = work_dir + flat_dir_root dark_dir = work_dir + dark_dir_root master_offset_file = offset_dir + t120.t120_master_name # now check existence of useful files and directories for subdir in [ work_dir, work_dir, orig_dir, offset_dir, flat_dir, t120.t120_common_dir ]: if not os.path.isdir(subdir): msg = '*** FATAL ERROR: directory ' + subdir + ' does not exist' t120.log.error(msg) raise IOError(msg) try: master_offset = fits_ccddata_reader(master_offset_file) except: msg = '*** FATAL ERROR while reading ' + master_offset_file t120.log.error(msg) raise IOError(msg) # read scamp ahead file #scamp_header = read_scamp_ahead(scampahead_file) # loop over imagess listremove = [] listimg = ImageFileCollection( orig_dir + '/') #,glob_include='*-c.fits',glob_exclude='*.fit') for ccd, fit_file in listimg.ccds( ccd_kwargs={'unit': 'adu'}, return_fname=True, save_location=reduc_dir ): #,save_with_name='-c',save_location=reduc_dir+'/'): t120.log.info('now treating file: ' + orig_dir + fit_file) filter_name = ccd.header['FILTER'] flat_name = flat_dir + '/master-' + filter_name + '.fits' master_flat = fits_ccddata_reader(flat_name, unit=u.adu) hdu = fits.open(orig_dir + fit_file) exp_time = hdu[0].header['EXPTIME'] strexptime = "%3.1f" % exp_time t120.log.info('exp_time=' + strexptime) dark_name = dark_dir + t120.t120_master_name.replace( '.fits', '') + '-' + strexptime + '.fits' master_dark = fits_ccddata_reader(dark_name, unit=u.adu) t120.log.info('Flat: ' + flat_name) t120.log.info('Dark: ' + dark_name) master_dark.header['EXPOSURE'] = ccd.header[ 'EXPOSURE'] # for dark subtraction master_offset.header['EXPOSURE'] = ccd.header[ 'EXPOSURE'] # for dark subtraction ccd_corr = ccd_process(ccd, exposure_key='EXPOSURE', exposure_unit=u.second, dark_frame=master_dark, master_flat=master_flat) #ccd_corr.header = put_scamp_header(ccd_corr.header,scamp_header) # update header skycoo = SkyCoord(ccd_corr.header['OBJCTRA'] + ' ' + ccd_corr.header['OBJCTDEC'], unit=(u.hourangle, u.deg)) ccd_corr.header['CRVAL1'] = ( skycoo.ra.to('deg').value, 'Reference Right ascencion in decimal deg') ccd_corr.header['CRVAL2'] = (skycoo.dec.to('deg').value, 'Reference Declination in decimal deg') out_fit_file = reduc_dir + '/' + os.path.splitext( fit_file)[0] + '-c.fits' if os.path.exists(out_fit_file): os.system('rm ' + out_fit_file) t120.log.info('File ' + out_fit_file + ' has been removed') # make primary HDU hducorrlist = ccd_corr.to_hdu() ccd_tosave = CCDData(hducorrlist[0].data, unit=u.adu) ccd_tosave.header = hducorrlist[0].header fits_ccddata_writer(ccd_tosave, out_fit_file) t120.log.info('corrected image saved in ' + out_fit_file) copy_fit_file = reduc_dir + fit_file t120.log.info('copy_fit_file ' + copy_fit_file) listremove.append(copy_fit_file) # remove copy of original fits files for file2remove in listremove: t120.log.info('now removing file ' + file2remove) if os.path.exists(file2remove): os.system('rm ' + file2remove) t120.log.info('File ' + file2remove + ' has been removed') else: t120.log.info('File ' + file2remove + ' does not exist') return