def combine_stack(self, stack_list, method="median"): """Combine a series of aligned object frames into a master stack via CCDPROC""" if method == "median": print("Combining stack by median") stack = ccdproc.combine(stack_list, method="median", unit="adu", mem_limit=2e9) elif method == "mean": print("Combining stack by mean") stack = ccdproc.combine(stack_list, method="mean", unit="adu", mem_limit=6e9) elif method == "sum": print("Combining stack by sum") stack = ccdproc.combine(stack_list, method="sum", unit="adu", mem_limit=6e9) else: print("Combining stack by median") stack = ccdproc.combine(stack_list, method="median", unit="adu", mem_limit=6e9) return stack
def combine_darks(self, dark_list, method="median"): """Combine a series of dark frames into a master dark via CCDPROC""" if method == "median": print("Combining darks by median") master_dark = ccdproc.combine(dark_list, method="median", unit="adu", mem_limit=6e9) elif method == "mean": print("Combining darks by mean") master_dark = ccdproc.combine(dark_list, method="mean", unit="adu", mem_limit=6e9) else: print("Combining darks by median") master_dark = ccdproc.combine(dark_list, method="median", unit="adu", mem_limit=6e9) return master_dark
def combine_flats(self, flat_list, master_dark, method="median"): """Combine and reduce a series of flat frames into a normalized flatfield via CCDPROC""" if method == "median": print("Combining flats by median") combined_flat = ccdproc.combine(flat_list, method="median", unit="adu", mem_limit=6e9) elif method == "mean": print("Combining flats by mean") combined_flat = ccdproc.combine(flat_list, method="mean", unit="adu", mem_limit=6e9) else: print("Combining flats by median") combined_flat = ccdproc.combine(flat_list, method="median", unit="adu", mem_limit=6e9) print("Subtracting master dark from combined flat") master_flat = ccdproc.subtract_dark(combined_flat, master_dark, data_exposure=combined_flat.header["exposure"]*u.second, dark_exposure=master_dark.header["exposure"]*u.second, scale=True) print("Reading master flat data") master_flat_data = np.asarray(master_flat) print("Creating normalized flatfield") flatfield_data = master_flat_data / np.mean(master_flat_data) print("Converting flatfield data to CCDData") flatfield = ccdproc.CCDData(flatfield_data, unit="adu") return flatfield
def combine_flats(refresh='2', method='2'): if method == '1': meta = 'med' source = 'Trimmed_Flat/subflatsmed' dest = 'Master_Files/mflat_median.fits' elif method == '2': meta = 'sig' source = 'Trimmed_Flat/subflatssig' dest = 'Master_Files/mflat.fits' subflatcollection = ImageFileCollection(source) combtime = 0 if refresh == '1': print('found', len(subflatcollection.values('file')), 'subflats') start = time.time() if method == '1': mflat = ccdp.combine(subflatcollection.files_filtered( imtype='subflat', include_path=True), method='median') mflat.meta['flatcom'] = 'median' combtime = time.time() - start print('combination took', combtime, 'seconds') elif method == '2': mflat = ccdp.combine(subflatcollection.files_filtered( imtype='subflat', include_path=True), sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.nanmedian, sigma_clip_dev_func=mad_std) mflat.meta['flatcom'] = 'sigma' combtime = time.time() - start print('combination took', combtime, 'seconds') mflat.meta['normmed'] = (np.nanmedian(mflat), 'nanmedian of the master flat') mflat.meta['subflats'] = meta mflat.write(dest[0:-5] + '_' + meta + '.fits', overwrite=True) else: try: if method == '1': mflat = CCDData.read('Master_Files/mflat_median_med.fits', unit='adu') elif method == '2': mflat = CCDData.read('Master_Files/mflat_sig.fits', unit='adu') except: print('can\'t locate master flat, create or check directory') sys.exit() return subflatcollection, mflat, dest, combtime
def combine_BiasDark(file_list, c_method, base_dir_name, master_file_dir_name, current_dir_name): try: combine_result = combine( file_list, # ccdproc does not accept numpy.ndarray, but only python list. method=c_method, # default is average so I specified median. unit='adu') combine_result.data = np.array(combine_result.data, dtype=np.float32) combine_result.write('{0}/{1}{2}_master_{3}_float32.fit'\ .format(base_dir_name, master_file_dir_name, current_dir_name, c_method), overwrite =True, format='fits') ##### fits header update with fits.open('{0}/{1}{2}_master_{3}_float32.fit'\ .format(base_dir_name, master_file_dir_name, current_dir_name, c_method), mode='update') as hdul: hdul[0].header.append( ('COMMENT', ', '.join(file_list), 'combine file list')) hdul[0].header.append('COMMENT', '{0}'.format(len(file_list)), 'combine file number') except Exception as err: print('{5} ::: {4} with {0}/{1}{2}_master_{3}_float32.fit ...'\ .format(base_dir_name, master_file_dir_name, current_dir_name, c_method, err, datetime.now())) return 0
def create_master_bias(list_files, fitsfile=None, fits_section=None, gain=None, method='median', dfilter={'imagetyp':'bias'}, mask=None, key_find='find', invert_find=False, sjoin=','): if gain is not None and not isinstance(gain, u.Quantity): gain = gain * u.electron / u.adu lbias = [] list_files = getListFiles(list_files, dfilter, mask, key_find=key_find, invert_find=invert_find) for filename in list_files: ccd = CCDData.read(filename, unit= u.adu) trimmed = True if fits_section is not None else False ccd = ccdproc.trim_image(ccd, fits_section=fits_section, add_keyword={'trimmed': trimmed}) if gain is not None: ccd = ccdproc.gain_correct(ccd, gain) lbias.append(ccd) combine = ccdproc.combine(lbias, method=method) if gain is not None and not 'GAIN' in combine.header: combine.header.set('GAIN', gain.value, gain.unit) combine.header['CGAIN'] = True if gain is not None else False combine.header['IMAGETYP'] = 'BIAS' combine.header['CMETHOD'] = method combine.header['CCDVER'] = VERSION if sjoin is not None: combine.header['LBIAS'] = sjoin.join([os.path.basename(fits) for fits in list_files]) combine.header['NBIAS'] = len(list_files) if fitsfile is not None: combine.header['FILENAME'] = os.path.basename(fitsfile) combine.write(fitsfile, clobber=True) return combine
def makeMasterFlat(images, master_bias): """ Flats are corrected for their bias level (if master_bias) TODO: Finish docstring """ try: fitsfile = 'master_flat.fits' master_flat = CCDData.read(fitsfile, unit=u.adu) return master_flat except FileNotFoundError: # empty list for the flats flat_list = [] # create the master flat field print('Reducing flats') for f in images.files_filtered(imagetyp=FLAT_KEYWORD): print(f) with fits.open(f) as fitsfile: data_exp = fitsfile[0].header[EXPTIME_KEYWORD] ccd = CCDData.read(f, unit=u.adu) if master_bias: ccd = subtract_bias(ccd, master_bias) else: print('No master bias, skipping correction...') flat_list.append(ccd) try: master_flat = combine(flat_list, method='median') master_flat.write('master_flat.fits', clobber=True) return master_flat except IndexError: print('There are no flats, skipping...') master_flat = None
def make_flat_master(): """ for each filter do: stack all flat exposures save fits to file add entry in database for each file (stack) generated""" import ccdproc nb = models.NightBundle.get(telescope_night_bundle_id=1) nb_dir = nb.directory_path flat_list_q = nb.exposures.select( lambda d: d.exposure_type == models.EXP_TYPE_CODES["FLAT"]) flat_list = [os.path.join(nb_dir, f.filename) for f in flat_list_q] # Create dark master and save to file master_flat = ccdproc.combine(flat_list, method="average", unit="adu") flat_hdu = fits.PrimaryHDU(master_flat) flat_hdu.header["IMAGETYP"] = "FLATM" flat_hdu.header["EXPTIME"] = 60.0 file_path = os.path.join(nb_dir, "products", "flat_master.fits") makedirs = os.path.dirname(file_path) if makedirs: os.makedirs(makedirs, exist_ok=True) flat_hdu.writeto(file_path) # Add entry to database flat_comb = models.ExposureCombination( night_bundle=nb, filename="flat_master.fits", combination_type=models.COMB_TYPE_CODES["FLATM"], exposures=flat_list_q, )
def loadbias(self): """ Loads the bias information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. If multiple matching files are found, they are combined into a single master bias frame by ccdproc. """ #master bias frame #Search for bias and load it into data object namelist = self.loadauxname('bias', multi=True) if (len(namelist) == 0): self.log.error('Bias calibration frame not found.') raise RuntimeError('No bias file loaded') self.log.debug('Creating master bias frame...') #if there is just one, use it as biasfile or else combine all to make a master bias if (len(namelist) == 1): self.bias = ccdproc.CCDData.read(namelist[0], unit='adu', relax=True) else: self.bias = ccdproc.combine(namelist, method='median', unit='adu', add_keyword=False) # Finish up self.biasloaded = True self.biasname = namelist[0] self.log.debug('LoadBias: done')
def create_flat(flat_list,fil,red_path,mbias=None,log=None): log.info('Processing files for filter: '+fil) log.info(str(len(flat_list))+' files found.') flats = [] flat_scale = [] for flat in flat_list: log.info('Loading file: '+flat) raw = CCDData.read(flat,hdu=1,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron) log.info('Exposure time of image is '+str(red.header['EXPTIME'])) log.info('Loading correct master dark') mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron) red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second) red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3)) mask = make_source_mask(red, nsigma=3, npixels=5) bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) masked = np.array(red) masked[mask] = bkg.background[mask] log.info('Median flat level: '+str(np.median(masked))) norm = 1/np.median(masked[500:1500,500:1500]) log.info('Flat normalization: '+str(norm)) flat_scale.append(norm) flats.append(CCDData(masked,meta=red.header,unit=u.electron)) mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True) log.info('Created master flat for filter: '+fil) mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True) log.info('Master flat written to mflat_'+fil+'.fits') return
def _perform(self): """ Returns an Argument() with the parameters that depends on this operation. """ self.log.info(f"Running {self.__class__.__name__} action") filedate = max([d.obstime() for d in self.darks[self.exptime]]) filedate_str = filedate.strftime('%Y%m%dUT') calibrated_darks = [d.pixeldata[0] for d in self.darks[self.exptime]] combined_dark = ccdproc.combine( calibrated_darks, method='average', sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.ma.median, sigma_clip_dev_func=stats.mad_std, ) self.log.info(f" Combined.") combined_dark.meta['combined'] = True combined_dark.meta['ncomb'] = len(calibrated_darks) combined_dark_filename = f'MasterDark_{self.exptime:03d}s_{filedate_str}.fits' combined_dark_filepath = Path( self.cfg['Calibrations'].get('DirectoryForMasters')) combined_dark_file = combined_dark_filepath.joinpath( combined_dark_filename) if combined_dark_file.exists() is True: self.log.debug(f" Deleting existing: {combined_dark_file.name}") combined_dark_file.unlink() self.log.info(f" Saving: {combined_dark_file.name}") combined_dark.write(combined_dark_file) return self.action.args
def combine_master_flat_file(file_list, c_method, base_dir_name, master_file_dir_name, current_dir_name, chl): try: combine_result = combine( file_list, # ccdproc does not accept numpy.ndarray, but only python list. method=c_method, # default is average so I specified median. sigma_clip=True, sigma_clip_low_thresh=3, sigma_clip_high_thresh=3, unit='adu') # unit is required: it's ADU in our case. combine_result.data = np.array(combine_result.data, dtype=np.float32) combine_result.write('{0}{1}{2}_master_{3}_{4}_float32.fit'\ .format(base_dir_name, master_file_dir_name, current_dir_name, c_method, chl), overwrite =True, format='fits') ##### fits header update with fits.open('{0}{1}{2}_master_{3}_{4}_float32.fit'\ .format(base_dir_name, master_file_dir_name, current_dir_name, c_method, chl), mode='update') as hdul: hdul[0].header.append( ('COMMENT', ', '.join(file_list), 'combine file list')) hdul[0].header.append('COMMENT', '{0}'.format(len(file_list)), 'combine file number') except Exception as err: print('{6} ::: {5} with {0}{1}{2}_master_{3}_{4}_float32.fit ...'\ .format(base_dir_name, master_file_dir_name, current_dir_name, c_method, chl, err, datetime.now())) return 0
def combinedark(cal_dir="../Data/20181207/cals", mast_dir=".", filt=[]): #Generates master dark file from calibration directory - uses max exptime dark images #Get master bias file if os.path.isfile(mast_dir + "/master/master_bias.FIT") != True: print "No master bias file" return False master_bias = CCDData.read(mast_dir + "/master/master_bias.FIT") #Generate image list imagelist = gen.getimages(cal_dir, filt=filt) full_dark_list = [] for img in imagelist: ccd = CCDData.read(cal_dir + '/' + img, unit=u.adu) if ccd.header["IMAGETYP"].strip() == "Dark Frame": img_exptime = ccd.header["EXPTIME"] ccd = ccdproc.subtract_bias(ccd, master_bias) full_dark_list.append([ccd, img_exptime]) print img #Select only those with max exptime exptime = max(full_dark_list, key=lambda x: x[1])[1] dark_list = [x[0] for x in full_dark_list if x[1] == exptime] if len(dark_list) == 0: print "ERROR: no dark files" return False #Generate master file master_dark = ccdproc.combine(dark_list, method='median', dtype="float32") master_dark.write(mast_dir + "/master/master_dark.FIT", overwrite=True) ''' print "Created master_dark file with " + str(exptime) + " exptime" ''' return True
def create_flat(flat_list,fil,red_path,mbias=None,log=None): log.info('Processing files for filter: '+fil) log.info(str(len(flat_list))+' files found.') flats = [] masks = [] flat_scale = [] for flat in flat_list: log.info('Loading file: '+flat) raw = CCDData.read(flat,unit=u.adu) red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron) log.info('Exposure time of image is '+str(red.header['TRUITIME']*red.header['COADDONE'])) mask = make_source_mask(red, nsigma=3, npixels=5) bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80) masked = np.array(red) masked[mask] = bkg.background[mask] log.info('Median flat level: '+str(np.median(masked))) norm = 1/np.median(masked[224:1824,224:1824]) log.info('Flat normalization: '+str(norm)) flat_scale.append(norm) flats.append(CCDData(masked,meta=red.header,unit=u.electron)) mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True) log.info('Created master flat for filter: '+fil) mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True) log.info('Master flat written to mflat_'+fil+'.fits') return
def combine_dark(day, chip, exptime, data_chip): #function to combine the darks for certain chip at certain exposure time output_name = 'Dark_' + exptime + '_' + chip + '.fits' if os.path.isfile(main_path + day + '/darks/' + output_name): cmd = 'rm ' + main_path + day + '/darks/' + output_name os.system(cmd) print "file %s deleted" % (main_path + day + '/darks/' + output_name) print main_path + day + '/darks/' + output_name mdark = ccd.combine(data_chip, output_file=main_path + day + '/darks/' + output_name, method=u'average', mem_limit=1600000000000, sigma_clip=True, sigma_clip_low_thresh=3, sigma_clip_high_thresh=3, sigma_clip_func=np.ma.median, sigma_clip_dev_func=np.ma.std, dtype=np.float64) #tmdark=ccd.trim_image(mdark,fits_section='[5:595,5:2395]') #ccd.fits_ccddata_writer(mdark,main_path+day+'/darks/'+output_name) #cmd='rm '+main_path+day+'/darks/nt_'+output_name #os.system(cmd) del mdark #del tmdark #cmd='mv '+output_name+' '+main_path+day+'/darks' #os.system(cmd) print "masterdark %s created for chip %s for epoch %s" % (exptime, chip, day) return (output_name)
def run(self): assert isinstance(self.input_list, list) if len(self.input_list) == 0: return if self.output_filename is not None: assert isinstance(self.output_filename, str) header = pyfits.getheader(self.input_list[0]) bx, by = header['CCDSUM'].strip().split() # Parameter obtained from PySOAR, written by Luciano Fraga try: master_bias = ccdproc.combine(self.input_list, method='average', mem_limit=6.4e7, minmax_clip=True, unit='adu') except ZeroDivisionError: raise RuntimeError('CCDProc.combine raised an error. ' 'Try again with a different number of input ' 'files') if self.output_filename is None: self.output_filename = "0Zero{}x{}".format(bx, by) pyfits.writeto(self.output_filename, master_bias.data, header)
def image_integration(self, flat=False, confirm=True): print( f'{Style.BRIGHT}Integrating {len(self.collection.files)} files.{Style.RESET_ALL}' ) if confirm: hlp.prompt() scale = None if flat: def inv_median(a): return 1 / np.median(a) scale = inv_median print('Integrating...') stack = ccdp.combine(self.collection.files_filtered(include_path=True), method='average', scale=scale, sigma_clip=True, sigma_clip_low_thresh=cfg.SIGMA_LOW, sigma_clip_high_thresh=cfg.SIGMA_HIGH, sigma_clip_func=np.ma.median, sigma_clip_dev_func=mad_std, mem_limit=600e7) return stack
def median_combine(path): """ Median combine with from a list within a directory. Args: path: path to directory of fits files to be combined Returns: a median combined CCDData object """ ic1 = ccdproc.ImageFileCollection(location=path) validate_units(ic1) framelist = ic1.files_filtered(BUNIT='adu', include_path=True) median_frame = ccdproc.combine(framelist, method='median', sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.ma.median, sigma_clip_dev_func=mad_std, mem_limit=350e6) return median_frame
def flat_combine(chip, chip_flats, chip_means): #function to combine the flat for a certain chip output_name = 'Flat_' + period + '_' + chip + '.fits' #nn=np.where(flat_chip==chip) #chip_flats=flat_list[nn] #chip_means=flat_mean[nn] if os.path.isfile(main_path + 'MasterFlats/' + period + '/' + output_name): os.system('rm ' + main_path + 'MasterFlats/' + period + '/' + output_name) print "image deleted %s" % (main_path + 'MasterFlats/' + period + '/' + output_name) mean_counts = np.mean(chip_means) mm = np.where((chip_means > mean_counts) & (chip_means < 15000)) flat_selected = chip_flats[mm] print "###########", output_name, len(flat_selected), mean_counts if len(flat_selected) > 0: data_chip = read_data(chip, flat_selected) print "combining chip=%s" % (chip) mflat = ccd.combine(data_chip, output_file=main_path + 'MasterFlats/' + period + '/' + output_name, method=u'median', mem_limit=1600000000000, sigma_clip=True, sigma_clip_low_thresh=3, sigma_clip_high_thresh=3, sigma_clip_func=np.ma.median, sigma_clip_dev_func=np.ma.std, dtype=np.float64) del mflat print "masterflat %s created for chip %s for epoch %s" % (output_name, chip, period) else: print "chip = %s without normal masterdark " % (chip) a = pf.open(main_path + chip_flats[0]) head = a[0].header dat = a[0].data fake_data = np.ones(dat.shape) pf.writeto(main_path + 'MasterFlats/' + period + '/' + output_name, data=fake_data, header=head) print "fake masterflat %s created for chip %s for epoch %s" % ( output_name, chip, period) return (output_name)
def flat_combine(textlist_files, band, outfile='mflat'): """ Combines multiple flats for a given input files list Args: textlist_files : A python list object with paths/names to the individual files. band : Filter name associated with the flat files outfile : Master flat name (default outfile + band + '.fits') Returns: None """ new_list = [ CCDData.read(filename, unit=u.adu) for filename in textlist_files ] combined_flat = ccdp.combine(img_list=new_list, method='average', scale=inv_median, sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.ma.median, sigma_clip_dev_func=mad_std) combined_flat.meta['combined'] = True file_name = outfile + band + '.fits' combined_flat.data = combined_flat.data.astype('float32') combined_flat.write(file_name, hdu_mask=None, hdu_uncertainty=None)
def _action_for_one_group(self, filter_dict=None): combined_dict = self.apply_to.copy() if filter_dict is not None: combined_dict.update(filter_dict) file_list = [ os.path.join(self.image_source.location, f) for f in self.image_source.files_filtered(**combined_dict) ] combine_keyword_args = { 'minmax_clip': self._clipping_widget.min_max, 'sigma_clip': self._clipping_widget.sigma_clip, } if self._combine_method.method == 'Average': combine_keyword_args['method'] = 'average' elif self._combine_method.method == 'Median': combine_keyword_args['method'] = 'median' if combine_keyword_args['minmax_clip']: combine_keyword_args['minmax_clip_min'] = \ self._clipping_widget.min_max.min combine_keyword_args['minmax_clip_max'] = \ self._clipping_widget.min_max.max if combine_keyword_args['sigma_clip']: combine_keyword_args['sigma_clip_low_thresh'] = \ self._clipping_widget.sigma_clip.min combine_keyword_args['sigma_clip_low_thresh'] = \ self._clipping_widget.sigma_clip.min combine_keyword_args['sigma_clip_func'] = np.ma.median combine_keyword_args['sigma_clip_dev_func'] = \ median_absolute_deviation if self._combine_method.scaling_func: combine_keyword_args['scale'] = self._combine_method.scaling_func combined = ccdproc.combine(file_list, mem_limit=DEFAULT_MEMORY_LIMIT, **combine_keyword_args) sample_image = ccdproc.CCDData.read(file_list[0]) combined.header = sample_image.header combined.header['master'] = True if combined.data.dtype != sample_image.dtype: combined.data = np.array(combined.data, dtype=sample_image.dtype) try: if isinstance(combined.uncertainty.array, np.ma.masked_array): combined.uncertainty.array = np.array( combined.uncertainty.array) except AttributeError: pass # Do not keep the mask or uncertainty if the data has neither if sample_image.mask is None and sample_image.uncertainty is None: combined.mask = None combined.uncertainty = None return combined
def produce_master(t, method="median", imagetp='FLAT', slc=None, ccdread_unit='adu'): """ a method to process master frames Parameters ---------- t: astropy.table.Table the table of SONG observation method: the method adopted when combining frames imagetp: string {'BIAS', 'FLAT', 'FLATI2', 'THAR', 'THARI2'} slc: slice to denote the fraction of being used ccdread_unit: string/unit default is 'adu' Returns ------- mst: ccdproc.CCDData the (combined) master frame """ assert method in {'average', 'median'} # 1. produce ind of master frames ind_mst = np.where(t['IMAGETYP'] == imagetp)[0] # check for positive number of master frames try: assert len(ind_mst) > 0 except AssertionError: raise IOError("There is no image of type %s!" % imagetp) # in default, combine all masters available if slc is not None: # combine a fraction of masters available assert isinstance(slc, slice) ind_mst = ind_mst[slc] # check for positive number of master frames try: assert len(ind_mst) > 0 except AssertionError: raise IOError("There is no image of type %s! (slice is bad)" % imagetp) # 2. read master frames print("@SONG: trying really hard to produce the final %s frame!" % imagetp) for _ in t['fps'][ind_mst]: print("+ %s" % _) fps = ','.join(t['fps'][ind_mst]) if len(ind_mst) == 1: mst = ccdproc.CCDData.read(fps, unit=ccdread_unit) else: mst = ccdproc.combine(fps, unit=ccdread_unit, method=method) return mst
def make_master(self, output_master_file): """Combine the raw calibration files into a master file and write it to the specified file name. """ # Recommended settings comb_method = 'average' do_sig_clip = True sig_clip_lothresh = 5 sig_clip_hithresh = 5 max_ram_bytes = 5e8 data_units = 'adu' # Calculate the header keywords to update kw_dict = self._generate_final_keywords() cal_type = kw_dict['IMAGETYP'][0] nfiles = len(self._files.summary) # Combine files msg = ( f'About to combine {nfiles} {cal_type} files, method={comb_method}' f' sigma_clip={do_sig_clip} sig_clip_lothresh={sig_clip_lothresh}' f' sig_clip_hithresh={sig_clip_hithresh} max_ram_bytes={max_ram_bytes:.3e}.' ) self._logger.debug(msg) master = ccdp.combine(self._files.files_filtered(include_path=True), method=comb_method, sigma_clip=do_sig_clip, sigma_clip_low_thresh=sig_clip_lothresh, sigma_clip_high_thresh=sig_clip_hithresh, sigma_clip_func=np.ma.median, sigma_clip_dev_func=mad_std, mem_limit=max_ram_bytes, unit=data_units, output_verify='ignore') # Calculate the header keywords to update, and update them master.meta['combined'] = True master.unit = 'adu' # Remove... kw_to_remove_list = [ 'UT', 'TIME-OBS', 'SWOWNER', 'SWCREATE', 'SBSTDVER' ] for kw in kw_to_remove_list: if kw in master.header: del master.header[kw] # Update/add for kw, val in kw_dict.items(): master.header[kw] = val # Write file master.write(output_master_file, overwrite=True, output_verify='ignore') self._logger.info( f'Wrote combined calibration file: {output_master_file}') return
def make_master_bias(self, list_of_biasfiles): bias_list = [0]*len(list_of_biasfiles) for ii, kk in enumerate(list_of_biasfiles): fitsfile = fits.open(kk) bias = ccdproc.CCDData(data=fitsfile[1].data, meta=fitsfile[1].header, unit="adu") bias_list[ii] = ccdproc.subtract_overscan(bias, fits_section='[5:35, :]') self.master_bias = ccdproc.combine(bias_list, method="median") self.master_bias.write(self.output_dir+"/"+self.filename+"_masterbias.fits", overwrite=True)
def loaddark(self): """ Loads the dark information for the instrument settings described in the header of self.datain. If an appropriate file can not be found or the file is invalid various warnings and errors are returned. If multiple matching files are found, they are combined into a single master dark frame by ccdproc. Also bias corrects dark files if not already done. """ #master dark frame dark_is_bias_corrected = False dark_bias = None namelist = self.loadauxname('dark', multi=True) if (len(namelist) == 0): self.log.error('Dark calibration frame(s) not found.') raise RuntimeError('No dark file loaded') darks = None for name in namelist: #is (any) dark file bias corrected? header = fits.getheader(name) if (header.get('BIAS') != None): dark_is_bias_corrected = True dark_bias = header.get('BIAS') elif (header.get('BIASCORR') != None): dark_is_bias_corrected = True dark_bias = header.get('BIASCORR') if (darks): darks += ',' + name else: darks = name self.log.debug('Creating master dark frame...') #if there is just one, use it as darkfile or else combine all to make a master dark if (len(namelist) == 1): self.dark = ccdproc.CCDData.read(namelist[0], unit='adu', relax=True) else: self.dark = ccdproc.combine(darks, method='median', unit='adu', add_keyword=False, **{'verify': 'ignore'}) #bias correct, if necessary if (not dark_is_bias_corrected): #Subtracting master bias frame from master dark frame self.dark = ccdproc.subtract_bias(self.dark, self.bias, add_keyword=False) else: self.log.debug( 'Master dark frame is *already* bias corrected (%s).' % dark_bias) # Finish up self.darkloaded = True self.darkname = namelist[0] self.log.debug('LoadDark: done')
def _action_for_one_group(self, filter_dict=None): combined_dict = self.apply_to.copy() if filter_dict is not None: combined_dict.update(filter_dict) file_list = [os.path.join(self.image_source.location, f) for f in self.image_source.files_filtered(**combined_dict)] combine_keyword_args = { 'minmax_clip': self._clipping_widget.min_max, 'sigma_clip': self._clipping_widget.sigma_clip, } if self._combine_method.method == 'Average': combine_keyword_args['method'] = 'average' elif self._combine_method.method == 'Median': combine_keyword_args['method'] = 'median' if combine_keyword_args['minmax_clip']: combine_keyword_args['minmax_clip_min'] = \ self._clipping_widget.min_max.min combine_keyword_args['minmax_clip_max'] = \ self._clipping_widget.min_max.max if combine_keyword_args['sigma_clip']: combine_keyword_args['sigma_clip_low_thresh'] = \ self._clipping_widget.sigma_clip.min combine_keyword_args['sigma_clip_low_thresh'] = \ self._clipping_widget.sigma_clip.min combine_keyword_args['sigma_clip_func'] = np.ma.median combine_keyword_args['sigma_clip_dev_func'] = \ median_absolute_deviation if self._combine_method.scaling_func: combine_keyword_args['scale'] = self._combine_method.scaling_func combined = ccdproc.combine(file_list, mem_limit=DEFAULT_MEMORY_LIMIT, **combine_keyword_args) sample_image = ccdproc.CCDData.read(file_list[0]) combined.header = sample_image.header combined.header['master'] = True if combined.data.dtype != sample_image.dtype: combined.data = np.array(combined.data, dtype=sample_image.dtype) try: if isinstance(combined.uncertainty.array, np.ma.masked_array): combined.uncertainty.array = np.array(combined.uncertainty.array) except AttributeError: pass # Do not keep the mask or uncertainty if the data has neither if sample_image.mask is None and sample_image.uncertainty is None: combined.mask = None combined.uncertainty = None return combined
def bias_combine(refresh='2', method='2'): tbiascollection = ImageFileCollection('Trimmed_Bias') combtime = 0 if refresh == '1': print('found', len(tbiascollection.values('file')), 'trimmed biases') start = time.time() if method == '1': combined_bias = ccdp.combine(tbiascollection.files_filtered( imtype='trimmed bias', include_path=True), method='median') combbiaspath = 'Master_Files/mbias_median.fits' combined_bias.meta['combined'] = 'median' combtime = time.time() - start print('combination took', combtime, 'seconds') combined_bias.write(combbiaspath, overwrite=True) elif method == '2': combined_bias = ccdp.combine(tbiascollection.files_filtered( imtype='trimmed bias', include_path=True), sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.nanmedian, sigma_clip_dev_func=mad_std) combbiaspath = 'Master_Files/mbias.fits' combined_bias.meta['combined'] = 'sigma_clip average' combtime = time.time() - start print('combination took', combtime, 'seconds') combined_bias.write(combbiaspath, overwrite=True) else: try: if method == '1': combined_bias = CCDData.read('Master_Files/mbias_median.fits', unit='adu') combbiaspath = 'Master_Files/mbias_median.fits' elif method == '2': combined_bias = CCDData.read('Master_Files/mbias.fits', unit='adu') combbiaspath = 'Master_Files/mbias.fits' except: print('can\'t locate master bias, create or check directory') sys.exit() return tbiascollection, combined_bias, combbiaspath, combtime
def flat_creation(bdf_files, calibrated_path,calibrated_images, output_path, master_images, args): set(bdf_files.summary['exptime'][bdf_files.summary['imagetyp'] == 'Flat Field']) if args.cal_bias: combined_bias = list(master_images.ccds(combined=True, imagetyp='Bias Frame'))[0] if args.cal_dark: combined_dark = CCDData.read(master_images.files_filtered(imagetyp='Dark Frame', combined=True, include_path=True)[0]) print('list of the flat files') for a_flat, f_name in bdf_files.ccds(imagetyp='Flat Field', return_fname=True, ccd_kwargs={'unit': 'adu'}): print(f_name) if args.cal_bias: print('BIAS') a_flat = ccdp.subtract_bias(a_flat, combined_bias) print('DONE BIAS') if args.cal_dark: print('DARK') a_flat = ccdp.subtract_dark(a_flat, combined_dark, exposure_time='EXPOSURE', exposure_unit=u.s, scale=True) print('done dark') a_flat.write(calibrated_path / f_name, overwrite=True) calibrated_images.refresh() flats = calibrated_images.summary['imagetyp'] == 'Flat Field' flat_filters = set(calibrated_images.summary['filter'][flats]) print('flat filters:', flat_filters) for filtr in sorted(flat_filters): calibrated_flats = calibrated_images.files_filtered(imagetyp='Flat Field', filter=filtr, include_path=True) print(len(calibrated_flats)) combined_flat = ccdp.combine(calibrated_flats, method='median', scale=inv_median, sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.ma.median, signma_clip_dev_func=mad_std, mem_limit=350e6) combined_flat.meta['combined'] = True flat_file_name = f'combined_flat_{filtr}.fit' combined_flat.write(output_path / flat_file_name, overwrite=True) calibrated_images.refresh() master_images.refresh() return flat_filters
def run(self): """ Runs the combining algorithm. The self.datain is run through the code, the result is in self.dataout. """ # Find master bias to subtract from master dark biaslist = self.loadauxname('bias', multi=False) if (len(biaslist) == 0): self.log.error('No bias calibration frames found.') self.bias = ccdproc.CCDData.read(biaslist, unit='adu', relax=True) # Create empy list for filenames of loaded frames filelist = [] for fin in self.datain: self.log.debug("Input filename = %s" % fin.filename) filelist.append(fin.filename) # Make a dummy dataout self.dataout = DataFits(config=self.config) if len(self.datain) == 0: self.log.error('Flat calibration frame not found.') raise RuntimeError('No flat file(s) loaded') self.log.debug('Creating master flat frame...') # Create master frame: if there is just one file, turn it into master bias or else combine all to make master bias if (len(filelist) == 1): self.dark = ccdproc.CCDData.read(filelist[0], unit='adu', relax=True) self.dark = ccdproc.subtract_bias(self.dark, self.bias, add_keyword=False) else: darklist = [] for i in filelist: dark = ccdproc.CCDData.read(i, unit='adu', relax=True) darksubbias = ccdproc.subtract_bias(dark, self.bias, add_keyword=False) darklist.append(darksubbias) self.dark = ccdproc.combine(darklist, method=self.getarg('combinemethod'), unit='adu', add_keyword=True) # set output header, put image into output self.dataout.header = self.datain[0].header self.dataout.imageset(self.dark) # rename output filename outputfolder = self.getarg('outputfolder') if outputfolder != '': outputfolder = os.path.expandvars(outputfolder) self.dataout.filename = os.path.join(outputfolder, os.path.split(filelist[0])[1]) else: self.dataout.filename = filelist[0] # Add history self.dataout.setheadval('HISTORY', 'MasterDark: %d files used' % len(filelist))
def _combine_calib_images( images: List[Image], bias: Optional[Image] = None, normalize: bool = False, method: str = "average" ) -> Image: """Combine a list of given images. Args: images: List of images to combine. bias: If given, subtract from images before combining them. normalize: If True, images are normalized to median of 1 before and after combining them. method: Method for combining images. """ import ccdproc # get CCDData objects data = [image.to_ccddata() for image in images] # subtract bias? if bias is not None: bias_data = bias.to_ccddata() data = [ccdproc.subtract_bias(d, bias_data) for d in data] # normalize? if normalize: data = [d.divide(np.median(d.data), handle_meta="first_found") for d in data] # combine image combined = ccdproc.combine( data, method=method, sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, mem_limit=350e6, unit="adu", combine_uncertainty_function=np.ma.std, ) # normalize? if normalize: combined = combined.divide(np.median(combined.data), handle_meta="first_found") # to Image and copy header image = Image.from_ccddata(combined) # add history for i, src in enumerate(images, 1): basename = src.header["FNAME"].replace(".fits.fz", "").replace(".fits", "") image.header["L1AVG%03d" % i] = (basename, "Image used for average") image.header["RLEVEL"] = (1, "Reduction level") # finished return image
def bias_combine(refresh=0): tbiascollection = ImageFileCollection('Trimmed_Bias') print('found', len(tbiascollection.values('file')), 'trimmed biases') combined_bias = ccdp.combine(tbiascollection.files_filtered( imtype='trimmed bias', include_path=True), sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.nanmedian, sigma_clip_dev_func=mad_std) combined_bias.meta['combined'] = True combined_bias.write('Master_Files/mbias.fits', overwrite=True) return tbiascollection
def align_combine_images(list_files, fitsfile, ref_image_fits=None, precision=100, dout=None, hexp='EXPTIME', force=False, minmax_clip=False, minmax_clip_min=0.0, sigma_clip=True, date=None, clip_extrema=False, func=np.ma.median, sigclip=5, cosmic=False, mbox=15, rbox=15, gbox=11, cleantype="medmask", cosmic_method='lacosmic', sky=False, dict_sky={}, dict_combine={}, suffix=None, hfilter='FILTER', key_file='file', hobj='OBJECT', ext=0, method='median', align=True, **kwargs): if ref_image_fits is None: tobj = getTimeTable(list_files, key_time=hexp, key_file=key_file, ext=ext, abspath=True, mask=-1, sort=True, clean=True) ref_image_fits, list_files = tobj[key_file][-1], tobj[key_file][:-1].data.tolist() ref_image = CCDData.read(ref_image_fits) ref_image = ccdproc.cosmicray_lacosmic(ref_image, sigclip=sigclip) lccd = [ref_image] lexp = [ref_image.header[hexp]] all_images = [os.path.basename(ref_image_fits)] for img in list_files: image = fits2CCDData(img, single=True) image_suffix = suffix if suffix is not None else img.split('.')[-1] if 'REFIMA' in image.header and 'IMAGES' in image.header and not force: continue if cosmic: image = cleanCosmic(image, mbox=mbox, rbox=rbox, gbox=gbox, sigclip=sigclip, cleantype=cleantype, cosmic_method=cosmic_method) if align: offset_image = shiftImage(image, ref_image, precision=precision, **kwargs) img_shifted = '%s_shift.%s' % (img.split('.fit')[0], image_suffix) all_images.append(os.path.basename(img_shifted)) img_shifted = join_path(img_shifted, dout) offset_image.write(img_shifted, clobber=True) lccd.append(offset_image) else: all_images.append(os.path.basename(img)) lccd.append(image) lexp.append(image.header[hexp]) #iraf.unlearn("imshift") #iraf.imshift.interp_type = "spline3" #iraf.imshift(img, img, shift[1], shift[0]) # Combine images lexp = np.array(lexp) scale_func = None # 1. / lexp combine = ccdproc.combine(lccd, method=method, scale=scale_func, minmax_clip=minmax_clip, func=func, minmax_clip_min=minmax_clip_min, sigma_clip=sigma_clip, clip_extrema=clip_extrema, **dict_combine) if sky: combine = subtract_sky_ccd(combine, **dict_sky) combine.header['IMAGES'] = str(' | '.join(all_images)) combine.header['REFIMA'] = os.path.basename(ref_image_fits) combine.header['IMGSEXP'] = ' | '.join(map(str,lexp[1:].tolist() + [lexp[0]])) combine.header['CMETHOD'] = method dir_out = os.path.dirname(ref_image_fits) if dout is None else dout fitsfile = join_path(fitsfile, dir_out) combine.header['FILENAME'] = os.path.basename(fitsfile) combine.header['CCDVER'] = VERSION combine.write(fitsfile, clobber=True)
def run(self): list_of_data = [] for f in self.input_list: hdr = pyfits.getheader(f) data = pyfits.getdata(f) data = CCDData(data, unit=u.adu) list_of_data.append(data) # Parameter obtained from PySOAR, written by Luciano Fraga master_bias = combine(list_of_data, method='average', mem_limit=6.4e7, minmax_clip=True) master_bias.header = hdr if self.output_filename is None: master_bias.write('0ZERO.fits') else: master_bias.write(self.output_filename)
def open_files_ccdproc_combine_nochunk(kind): """ Open files indirectly as part of ccdproc.combine, ensuring that the task is not broken into chunks. """ global combo paths = sorted(list(TMPPATH.glob('**/*.' + ALLOWED_EXTENSIONS[kind]))) # We ensure there are no chunks by setting a memory limit large # enough to hold everything. with fits.open(paths[0]) as hdulist: array_size = hdulist[0].data.nbytes # Why 2x the number of files? To make absolutely sure we don't # end up chunking the job. array_size *= 2 * len(paths) combo = combine(paths)
def open_files_ccdproc_combine_chunk(kind): """ Open files indirectly as part of ccdproc.combine, ensuring that the task is broken into chunks. """ global combo paths = sorted(list(TMPPATH.glob('**/*.' + ALLOWED_EXTENSIONS[kind]))) # We want to force combine to break the task into chunks even # if the task really would fit in memory; it is in that case that # we end up with too many open files. We'll open one file, determine # the size of the data in bytes, and set the memory limit to that. # That will mean lots of chunks (however many files there are plus one), # but lots of chunks is fine. with fits.open(paths[0]) as hdulist: array_size = hdulist[0].data.nbytes combo = combine(paths, mem_limit=array_size)
def run(self): list_of_data = [] for f in self.input_list: self.debug('Processing file: {:s}'.format(f)) hdr = pyfits.getheader(f) data = pyfits.getdata(f) x_center = data.shape[1] // 2 x_bsize = int(0.05 * data.shape[1]) x1, x2 = x_center - x_bsize, x_center + x_bsize x_where = np.zeros_like(data) x_where[:, x1:x2] = 1 y_center = data.shape[0] // 2 y_bsize = int(0.05 * data.shape[0]) y1, y2 = y_center - y_bsize, y_center + y_bsize y_where = np.zeros_like(data) y_where[y1:y2, :] = 1 where = np.where(x_where * y_where == 1, True, False) norm_factor = np.median(data[where]) data /= norm_factor data = CCDData(data, unit=u.adu) list_of_data.append(data) # Parameter obtained from PySOAR, written by Luciano Fraga master_flat = combine(list_of_data, method='median', mem_limit=6.4e7, sigma_clip=True) master_flat.header = hdr if self.output_filename is None: filter_name = hdr['FILTERS'].strip() binning = int(hdr['CCDSUM'].strip().split(' ')[0]) self.debug('Binning: {:d}'.format(binning)) filename = '1NSFLAT{0:d}x{0:d}_{1:s}.fits'.format(binning, filter_name) master_flat.write(filename, overwrite=True) else: master_flat.write(self.output_filename, overwrite=True)
def makeMasterBias(images): """ Make a master bias using all biases found in images object TODO: Finish docstring """ try: master_bias = CCDData.read('master_bias.fits', unit=u.adu) return master_bias except FileNotFoundError: bias_list = [] for f in images.files_filtered(imagetyp=BIAS_KEYWORD): print(f) ccd = CCDData.read(f, unit=u.adu) bias_list.append(ccd) try: master_bias = combine(bias_list, method='median') master_bias.write('master_bias.fits', clobber=True) return master_bias except IndexError: return None
def run(self, input_files, output_file='o.fits'): from astropy.io import fits as pyfits from astropy import units as u from ccdproc import CCDData, combine list_of_data = [] for f in input_files: hdr = pyfits.getheader(f) data = pyfits.getdata(f) data = CCDData(data, unit=u.adu) list_of_data.append(data) final = combine(list_of_data, method='median', mem_limit=6.4e7, sigma_clip=True, sigma_clip_low_thresh=3.0, sigma_clip_high_thresh=3.0, scale='mode', statsec="[200:800,200:800]") final.write(output_file, clobber=True)
def run(self, input_files, output_file=None): list_of_data = [] for f in input_files: hdr = pyfits.getheader(f) data = pyfits.getdata(f) data = CCDData(data, unit=u.adu) list_of_data.append(data) master_flat = combine(list_of_data, method='median', mem_limit=6.4e7, sigma_clip=True, sigma_clip_low_thresh=3.0, sigma_clip_high_thresh=3.0, scale=self.mode) master_flat.header = hdr if output_file is None: master_flat.write('nSFLAT_{:s}.fits'.format(hdr['FILTERS']), clobber=True) else: master_flat.write(output_file, clobber=True)
def create_master_flats(self, flat_group, target_name=''): """Creates master flats Using a list of compatible flat images it combines them using median and 1-sigma clipping. Also it apply all previous standard calibrations to each image. Args: flat_group (DataFrame): :class:`~pandas.DataFrame` instance. Contains a list of compatible flat images target_name (str): Science target name. This is used in some science case uses only. Returns: The master flat :class:`~astropy.nddata.CCDData` instance and the name of under which the master flat was stored. """ flat_file_list = flat_group.file.tolist() cleaned_flat_list = [] master_flat_list = [] master_flat_name = None self.log.info('Creating Master Flat') for flat_file in flat_file_list: # print(f_file) image_full_path = os.path.join(self.args.raw_path, flat_file) ccd = read_fits(image_full_path, technique=self.technique) self.log.debug('Loading flat image: ' + image_full_path) if master_flat_name is None: master_flat_name = self.name_master_flats( header=ccd.header, group=flat_group, target_name=target_name) if self.technique == 'Spectroscopy': ccd = image_overscan(ccd, overscan_region=self.overscan_region) ccd = image_trim(ccd=ccd, trim_section=self.trim_section, trim_type='trimsec') ccd = ccdproc.subtract_bias(ccd, self.master_bias, add_keyword=False) ccd.header['GSP_BIAS'] = ( os.path.basename(self.master_bias_name), 'Master bias image') elif self.technique == 'Imaging': ccd = image_trim(ccd=ccd, trim_section=self.trim_section, trim_type='trimsec') ccd = ccdproc.subtract_bias(ccd, self.master_bias, add_keyword=False) ccd.header['GSP_BIAS'] = ( os.path.basename(self.master_bias_name), 'Master bias image') else: self.log.error('Unknown observation technique: ' + self.technique) if self._is_file_saturated(ccd=ccd): self.log.warning('Removing saturated image {:s}. ' 'Use --saturation to change saturation ' 'level'.format(flat_file)) continue else: cleaned_flat_list.append(flat_file) master_flat_list.append(ccd) if master_flat_list != []: master_flat = ccdproc.combine(master_flat_list, method='median', sigma_clip=True, sigma_clip_low_thresh=1.0, sigma_clip_high_thresh=1.0, add_keyword=False) # add name of images used to create master bias for n in range(len(cleaned_flat_list)): master_flat.header['GSP_IC{:02d}'.format(n + 1)] = ( cleaned_flat_list[n], 'Image used to create master flat') write_fits(ccd=master_flat, full_path=master_flat_name, combined=True) self.log.info('Created Master Flat: ' + master_flat_name) return master_flat, master_flat_name # print(master_flat_name) else: self.log.error('Empty flat list. Check that they do not exceed the ' 'saturation limit.') return None, None
def create_master_bias(self, bias_group): """Create Master Bias Given a :class:`~pandas.DataFrame` object that contains a list of compatible bias. This function creates the master flat using ccdproc.combine using median and 3-sigma clipping. Args: bias_group (object): :class:`~pandas.DataFrame` instance that contains a list of bias images compatible with each other. """ bias_file_list = bias_group.file.tolist() default_bias_name = os.path.join(self.args.red_path, 'master_bias.fits') search_bias_name = re.sub('.fits', '*.fits', default_bias_name) n_bias = len(glob.glob(search_bias_name)) if n_bias > 0: self.master_bias_name = re.sub('.fits', '_{:d}.fits'.format(n_bias + 1), default_bias_name) self.log.info('New name for master bias: ' + self.master_bias_name) else: self.master_bias_name = default_bias_name master_bias_list = [] self.log.info('Creating master bias') for image_file in bias_file_list: image_full_path = os.path.join(self.args.raw_path, image_file) ccd = read_fits(image_full_path, technique=self.technique) self.log.debug('Loading bias image: ' + image_full_path) if self.technique == 'Spectroscopy': self.log.debug( 'Overscan Region: {:s}'.format(str(self.overscan_region))) ccd = image_overscan(ccd=ccd, overscan_region=self.overscan_region) ccd = image_trim(ccd, trim_section=self.trim_section, trim_type='trimsec') master_bias_list.append(ccd) # combine bias for spectroscopy self.master_bias = ccdproc.combine(master_bias_list, method='median', sigma_clip=True, sigma_clip_low_thresh=3.0, sigma_clip_high_thresh=3.0, add_keyword=False) # add name of images used to create master bias for n in range(len(bias_file_list)): self.master_bias.header['GSP_IC{:02d}'.format(n + 1)] = ( bias_file_list[n], 'Image used to create master bias') write_fits(ccd=self.master_bias, full_path=self.master_bias_name, combined=True) self.log.info('Created master bias: ' + self.master_bias_name) return self.master_bias, self.master_bias_name
if not os.path.isdir(outdir): os.mkdir(outdir) os.chdir(outdir) #change this to point to your raw data directory ic1 = ImageFileCollection(indir) #create the bias frames blue_bias_list = [] for filename in ic1.files_filtered(obstype='Bias', isiarm='Blue arm'): print ic1.location + filename ccd = CCDData.read(ic1.location + filename, unit = u.adu) #this has to be fixed as the bias section does not include the whole section that will be trimmed ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]') ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] ) blue_bias_list.append(ccd) master_bias_blue = ccdproc.combine(blue_bias_list, method='median') master_bias_blue.write('master_bias_blue.fits', clobber=True) red_bias_list = [] for filename in ic1.files_filtered(obstype='Bias', isiarm='Red arm'): print ic1.location + filename ccd = CCDData.read(ic1.location + filename, unit = u.adu) #this has to be fixed as the bias section does not include the whole section that will be trimmed ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]') ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] ) red_bias_list.append(ccd) master_bias_red = ccdproc.combine(red_bias_list, method='median') master_bias_red.write('master_bias_red.fits', clobber=True) #create the flat fields red_flat_list = []
indices=np.where((array_ftype == f)&(array_filter == element)) if len(indices[0]) > 0: outfile = open(ftype_filter,'w') for i in indices[0]: outfile.write(fnames[i]+'\n') outfile.close() #os.remove('tempflats') for f in flat_filelist: print 'filelist = ',f flatimages = [] try: filelist = open(f,'r') except IOError: print('Problem opening file ',f) print('Hope that is ok...') for q in filelist: flatimages.append(q.rstrip()) # combine flat images using average combine, scale by median, sigma clip flat = ccdproc.combine(flatimages,scale=np.median,method='average',sigma_clip=True,unit=u.adu) #med_flat = ccdproc.combine(flatimages, method='median') # normalize flat image by dividing by mean norm_flat = flat / np.mean(flat) print 'writing fits' norm_flat.write('n'+f+'.fits', overwrite=True)
''' import glob import os import numpy as np import ccdproc import astropy.units as u import argparse ''' from pyraf import iraf iraf.noao() iraf.imred() iraf.ccdred() ''' parser = argparse.ArgumentParser(description ='Groups images by filter and creates flatfield images') parser.add_argument('--filestring', dest='filestring', default='bias', help='match string for input files (default = bias, which gets bias*.fits)') args = parser.parse_args() files = sorted(glob.glob(args.filestring+'*.fits')) zeros = ccdproc.combine(fimages,method='average',sigma_clip=True,unit=u.adu) print('writing fits') zeros.write(args.filestring+'-combined.fits',overwrite=True)
def hrsbias(rawpath, outpath, link=False, mem_limit=1e9, sdb=None, clobber=True): """hrsbias processes the HRS red and blue bias frames in a directory Parameters ---------- rawpath: string Path to raw data outpath: string Path to output result link: boolean Add symbolic link to HRS_CALS directory clobber: boolean Overwrite existing files """ if not os.path.isdir(rawpath): return image_list = ImageFileCollection(rawpath) if len(image_list.files)==0: return #make output directory if not os.path.isdir(outpath): os.mkdir(outpath) obsdate=get_obsdate(image_list.summary['file'][0]) #process the red bias frames matches = (image_list.summary['obstype'] == 'Bias') * (image_list.summary['detnam'] == 'HRDET') rbias_list = [] for fname in image_list.summary['file'][matches]: ccd = red_process(rawpath+fname) rbias_list.append(ccd) if sdb is not None: dq_ccd_insert(rawpath + fname, sdb) if rbias_list: if os.path.isfile("{0}/RBIAS_{1}.fits".format(outpath, obsdate)) and clobber: os.remove("{0}/RBIAS_{1}.fits".format(outpath, obsdate)) rbias = ccdproc.combine(rbias_list, method='median', output_file="{0}/RBIAS_{1}.fits".format(outpath, obsdate), mem_limit=mem_limit) del rbias_list #process the red bias frames matches = (image_list.summary['obstype'] == 'Bias') * (image_list.summary['detnam'] == 'HBDET') hbias_list = [] for fname in image_list.summary['file'][matches]: ccd = blue_process(rawpath+fname) hbias_list.append(ccd) if sdb is not None: dq_ccd_insert(rawpath + fname, sdb) if hbias_list: if os.path.isfile("{0}/HBIAS_{1}.fits".format(outpath, obsdate)) and clobber: os.remove("{0}/HBIAS_{1}.fits".format(outpath, obsdate)) hbias = ccdproc.combine(hbias_list, method='median', output_file="{0}/HBIAS_{1}.fits".format(outpath, obsdate), mem_limit=mem_limit) del hbias_list #provide the link to the bias frame if link: ldir = '/salt/HRS_Cals/CAL_BIAS/{0}/{1}/'.format(obsdate[0:4], obsdate[4:8]) if not os.path.isdir(ldir): os.mkdir(ldir) ldir = '/salt/HRS_Cals/CAL_BIAS/{0}/{1}/product'.format(obsdate[0:4], obsdate[4:8]) if not os.path.isdir(ldir): os.mkdir(ldir) infile="{0}/RBIAS_{1}.fits".format(outpath, obsdate) link='/salt/HRS_Cals/CAL_BIAS/{0}/{1}/product/RBIAS_{2}.fits'.format(obsdate[0:4], obsdate[4:8], obsdate) if os.path.isfile(link) and clobber: os.remove(link) os.symlink(infile, link) infile="{0}/HBIAS_{1}.fits".format(outpath, obsdate) link='/salt/HRS_Cals/CAL_BIAS/{0}/{1}/product/HBIAS_{2}.fits'.format(obsdate[0:4], obsdate[4:8], obsdate) if os.path.isfile(link) and clobber: os.remove(link) os.symlink(infile, link)
def hrsflat(rawpath, outpath, detname, obsmode, master_bias=None, f_limit=1000, first_order=53, y_start=30, y_limit=3920, smooth_length=20, smooth_fraction=0.4, filter_size=151, link=False, sdb=None, clobber=True): """hrsflat processes the HRS flatfields. It will process for a given detector and a mode Parameters ---------- rawpath: string Path to raw data outpath: string Path to output result link: boolean Add symbolic link to HRS_CALS directory sdb: sdb_user.mysql SDB object to upload data quality clobber: boolean Overwrite existing files """ if not os.path.isdir(rawpath): return image_list = ImageFileCollection(rawpath) if len(image_list.files)==0: return #make output directory if not os.path.isdir(outpath): os.mkdir(outpath) #get the observing date obsdate=get_obsdate(image_list.summary['file'][0]) #setup the instrument prefix if detname=='HRDET': prefix='R' process = red_process elif detname=='HBDET': prefix='H' process = blue_process else: raise ValueError('detname must be a valid HRS Detector name') #process the flat frames matches = (image_list.summary['obstype'] == 'Flat field') * (image_list.summary['detnam'] == detname) * (image_list.summary['obsmode'] == obsmode) flat_list = [] for fname in image_list.summary['file'][matches]: ccd = process(rawpath+fname, masterbias=master_bias) flat_list.append(ccd) if sdb is not None: dq_ccd_insert(rawpath + fname, sdb) if flat_list: outfile = "{0}/{2}FLAT_{1}_{3}.fits".format(outpath, obsdate, prefix, obsmode.replace(' ', '_')) if os.path.isfile(outfile) and clobber: os.remove(outfile) flat = ccdproc.combine(flat_list, method='median', output_file=outfile) norm = clean_flatimage(flat.data, filter_size=filter_size, flux_limit=0.3, block_size=100, percentile_low=30, median_size=5) hdu = fits.PrimaryHDU(norm) hdu.writeto(prefix+'norm.fits', clobber=True) norm[norm>0]=1 ys, xs = norm.shape xc = int(xs/2.0) if detname=='HRDET': xc = 1947 #int(xs/2.0) ndata = norm[:,xc] detect_kern = ndata[1:100] #these remove light that has bleed at the edges and may need adjusting norm[:,:20]=0 norm[:,4040:]=0 elif detname=='HBDET': ndata = norm[:,xc] detect_kern = ndata[32:110] frame = create_orderframe(norm, first_order, xc, detect_kern, smooth_length=smooth_length, smooth_fraction=smooth_fraction, y_start=y_start, y_limit=y_limit) order_file = "{0}/{2}ORDER_{1}_{3}.fits".format(outpath, obsdate, prefix, obsmode.replace(' ', '_')) hdu = fits.PrimaryHDU(frame) hdu.writeto(prefix+'order.fits', clobber=True) hdu.writeto(order_file, clobber=True) if link: link='/salt/HRS_Cals/CAL_FLAT/{0}/{1}/product/{2}'.format(obsdate[0:4], obsdate[4:8], os.path.basename(outfile)) if os.path.isfile(link) and clobber: os.remove(link) os.symlink(outfile, link) olink='/salt/HRS_Cals/CAL_FLAT/{0}/{1}/product/{2}'.format(obsdate[0:4], obsdate[4:8], os.path.basename(order_file)) if os.path.isfile(olink) and clobber: os.remove(olink) os.symlink(order_file, olink)
def imcombine(filelist, out, options, method="average", reject="none",\ lsigma=3, hsigma=3, mclip=False,\ nlow=None, nhigh=None): '''Combines images in input list with optional rejection algorithms. Args: filelist: The list of files to imcombine out: The full path to the output file method: either "average" or "median" combine options: Options dictionary bpmask: The full path to the bad pixel mask reject: none, minmax, sigclip nlow,nhigh: Parameters for minmax rejection, see iraf docs mclip: use median as the function to calculate the baseline values for sigclip rejection? lsigma, hsigma: low and high sigma rejection thresholds. Returns: None Side effects: Creates the imcombined file at location `out' ''' assert method in ['average', 'median'] if os.path.exists(out): os.remove(out) if reject == 'none': info('Combining files using ccdproc.combine task') info(' reject=none') for file in filelist: debug(' Combining: {}'.format(file)) ccdproc.combine(filelist, out, method=method,\ minmax_clip=False,\ iraf_minmax_clip=True,\ sigma_clip=False,\ unit="adu") info(' Done.') elif reject == 'minmax': ## The IRAF imcombine minmax rejection behavior is different than the ## ccdproc minmax rejection behavior. We are using the IRAF like ## behavior here. To support this a pull request for the ccdproc ## package has been made: ## https://github.com/astropy/ccdproc/pull/358 ## ## Note that the ccdproc behavior still differs slightly from the ## nominal IRAF behavior in that the rejection does not consider whether ## any of the rejected pixels have been rejected for other reasons, so ## if nhigh=1 and that pixel was masked for some other reason, the ## new ccdproc algorithm, will not mask the next highest pixel, it will ## still just mask the highest pixel even if it is already masked. ## ## From IRAF (help imcombine): ## nlow = 1, nhigh = 1 (minmax) ## The number of low and high pixels to be rejected by the ## "minmax" algorithm. These numbers are converted to fractions ## of the total number of input images so that if no rejections ## have taken place the specified number of pixels are rejected ## while if pixels have been rejected by masking, thresholding, or ## non-overlap, then the fraction of the remaining pixels, ## truncated to an integer, is used. ## ## Check that minmax rejection is possible given the number of images if nlow is None: nlow = 0 if nhigh is None: nhigh = 0 if nlow + nhigh >= len(filelist): warning('nlow + nhigh >= number of input images. Combining without rejection') nlow = 0 nhigh = 0 if ccdproc.version.major >= 1 and ccdproc.version.minor >= 1\ and ccdproc.version.release: info('Combining files using ccdproc.combine task') info(' reject=clip_extrema') info(' nlow={}'.format(nlow)) info(' nhigh={}'.format(nhigh)) for file in filelist: info(' {}'.format(file)) ccdproc.combine(filelist, out, method=method,\ minmax_clip=False,\ clip_extrema=True,\ nlow=nlow, nhigh=nhigh,\ sigma_clip=False,\ unit="adu") info(' Done.') else: ## If ccdproc does not have new rejection algorithm in: ## https://github.com/astropy/ccdproc/pull/358 ## Manually perform rejection using ccdproc.combiner.Combiner object info('Combining files using local clip_extrema rejection algorithm') info('and the ccdproc.combiner.Combiner object.') info(' reject=clip_extrema') info(' nlow={}'.format(nlow)) info(' nhigh={}'.format(nhigh)) for file in filelist: info(' {}'.format(file)) ccdlist = [] for file in filelist: ccdlist.append(ccdproc.CCDData.read(file, unit='adu', hdu=0)) c = ccdproc.combiner.Combiner(ccdlist) nimages, nx, ny = c.data_arr.mask.shape argsorted = np.argsort(c.data_arr.data, axis=0) mg = np.mgrid[0:nx,0:ny] for i in range(-1*nhigh, nlow): where = (argsorted[i,:,:].ravel(), mg[0].ravel(), mg[1].ravel()) c.data_arr.mask[where] = True if method == 'average': result = c.average_combine() elif method == 'median': result = c.median_combine() for key in list(ccdlist[0].header.keys()): header_entry = ccdlist[0].header[key] if key != 'COMMENT': result.header[key] = (header_entry, ccdlist[0].header.comments[key]) hdul = result.to_hdu() # print(hdul) # for hdu in hdul: # print(type(hdu.data)) hdul[0].writeto(out) # result.write(out) info(' Done.') elif reject == 'sigclip': info('Combining files using ccdproc.combine task') info(' reject=sigclip') info(' mclip={}'.format(mclip)) info(' lsigma={}'.format(lsigma)) info(' hsigma={}'.format(hsigma)) baseline_func = {False: np.mean, True: np.median} ccdproc.combine(filelist, out, method=method,\ minmax_clip=False,\ clip_extrema=False,\ sigma_clip=True,\ sigma_clip_low_thresh=lsigma,\ sigma_clip_high_thresh=hsigma,\ sigma_clip_func=baseline_func[mclip],\ sigma_clip_dev_func=np.std,\ ) info(' Done.') else: raise NotImplementedError('{} rejection unrecognized by MOSFIRE DRP'.format(reject))