def load_data(args): bdf_files = ccdp.ImageFileCollection(args.data_path) raw_darks = bdf_files.files_filtered(imagetyp='Dark Frame', include_path=True) if args.calib_path: calibrated_path = Path(args.calib_path) else: calibrated_path = Path(args.data_path, 'calibrated') calibrated_path.mkdir(exist_ok=True) calibrated_images = ccdp.ImageFileCollection(calibrated_path) if args.output_path: output_path = Path(args.output_path) else: output_path = Path(args.data_path, 'masters') output_path.mkdir(exist_ok=True) master_images = ccdp.ImageFileCollection(output_path) calibrated_images.refresh() return bdf_files, raw_darks, calibrated_path, calibrated_images, output_path, master_images
def load_data(args): master_images = ccdp.ImageFileCollection(args.data_path) raw_gaias = ccdp.ImageFileCollection(args.raw_images) raw_gaias_im = raw_gaias.files_filtered(imagetyp='Light Frame', include_path=True) raw_gaias_im = sorted(raw_gaias_im) if args.red_path: reduced_path = Path(args.red_path) else: reduced_path = Path(args.raw_images, 'reduced') reduced_path.mkdir(exist_ok=True) reduced_images = ccdp.ImageFileCollection(reduced_path) if args.al_path: aligned_path = Path(args.al_path) else: aligned_path = Path(args.raw_images, 'aligned') aligned_path.mkdir(exist_ok=True) aligned_images = ccdp.ImageFileCollection(aligned_path) if args.st_path: stacked_path = Path(args.st_path) else: stacked_path = Path(args.raw_images, 'stacked') stacked_path.mkdir(exist_ok=True) stacked_images = ccdp.ImageFileCollection(stacked_path) return master_images, raw_gaias, raw_gaias_im, reduced_path, reduced_images, aligned_path, aligned_images, stacked_path, stacked_images
def divide_flat(self): """Divides all LMI science frames by the appropriate flat field image This method is LMI-specific, rather than being wrapper for a more general function. Basic emulation of IRAF's ccdproc/flatcor function. :return: None """ # Load the list of master flats and bias-subtracted data frames flat_cl = ccdp.ImageFileCollection( self.path, glob_include=f'flat_bin{self.bin_factor}_*.fits') sci_cl = ccdp.ImageFileCollection( self.path, glob_include=f'{self.prefix}.*b.fits') # Check to be sure there are, indeed, flats... if flat_cl.files: # Loop through the filters present for filt in sorted(list(flat_cl.summary['filters'])): # Load in the master flat for this filter if self.debug: print(f'Dividing by master flat for filter: {filt}') master_flat, mflat_fn = next( flat_cl.ccds(ccdsum=self.binning, filters=filt, return_fname=True)) # Set up a progress bar, so we can see how the process is going sci_filt_files = sci_cl.files_filtered(filters=filt) prog_bar = tqdm(total=len(sci_filt_files), unit='frame', unit_scale=False, colour='#D8BFD8') # Loop through the science frames to correct for ccd, sci_fn in sci_cl.ccds(ccdsum=self.binning, filters=filt, return_fname=True): # Divide by master flat ccdp.flat_correct(ccd, master_flat) # Update the header ccd.header['flatcor'] = True ccd.header['HISTORY'] = PKG_NAME ccd.header['HISTORY'] = 'Flat-corrected image saved: ' + \ _savetime() ccd.header['HISTORY'] = f'Divided by flat: {mflat_fn}' ccd.header['HISTORY'] = f'Previous filename: {sci_fn}' # Save the result (suffix = 'f'); delete the input file ccd.write(f'{self.path}/{sci_fn[:-6]}f{sci_fn[-5:]}', overwrite=True) os.remove(f'{self.path}/{sci_fn}') # Update the progress bar prog_bar.update(1) # Close the progress bar, end of loop prog_bar.close()
def load_data(args): bdf_files = ccdp.ImageFileCollection(args.data_path) raw_biases = bdf_files.files_filtered(imagetyp='Bias Frame', include_path=True) print(raw_biases) if args.output_path: output_path = Path(args.output_path) else: output_path = Path(args.data_path, 'masters') output_path.mkdir(exist_ok=True) master_images = ccdp.ImageFileCollection(output_path) return raw_biases, output_path
def create_nirspec_collection(self): """Create a collection of Keck NIRSPEC echelle spectra""" keywords = [ "imagetyp", "filname", "slitname", "itime", "frameno", "object", "ut", "airmass", "dispers", "slitwidt", "slitlen", "ra", "dec", ] with warnings.catch_warnings(): warnings.simplefilter("ignore") ims = (ccdproc.ImageFileCollection( self.root_dir, keywords=keywords, glob_include="NS*.fits", ).filter(dispers="high").filter(regex_match=True, slitlen="12|24")) return ims
def _create_file_collection(self, data_dir, exclude_pattern, file_list): """Create a FITS ImageFileCollection for a directory, optionally including only the files named in fname_list. :param data_dir: Path to the directory containing the files. :param exclude_pattern: Pattern used to exclude certain files. :param file_list: None, or list of file names to read instead of reading all files in the data_dir (excluding files that match the exclude_pattern). """ if file_list is not None: msg = (f'Looking for FITS files in {data_dir},' f' including only files in the list: {file_list}') else: msg = ( f'Looking for FITS files in {data_dir},' f' excluding files matching the pattern "{exclude_pattern}"') self._logger.info(msg) file_collection = ccdp.ImageFileCollection( data_dir, keywords=self._summary_kw, glob_exclude=exclude_pattern, filenames=file_list) self._logger.info( f'Found {len(file_collection.summary)} FITS files matching the constraints.' ) return file_collection
def median_combine(path): """ Median combine with from a list within a directory. Args: path: path to directory of fits files to be combined Returns: a median combined CCDData object """ ic1 = ccdproc.ImageFileCollection(location=path) validate_units(ic1) framelist = ic1.files_filtered(BUNIT='adu', include_path=True) median_frame = ccdproc.combine(framelist, method='median', sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.ma.median, sigma_clip_dev_func=mad_std, mem_limit=350e6) return median_frame
def __init__(self, path, biassec=None, trimsec=None, prefix=None, multilamp=False): """__init__: Initialize DeVeny class. Args: path (:TYPE:`str`) Path to the directory containing the images to be reduced. biassec (:TYPE:`str`) The IRAF-style overscan region to be subtracted from each frame. If unspecified, use the values suggested in the LMI User Manual. trimsec (:TYPE:`str`) The IRAF-style image region to be retained in each frame. If unspecified, use the values suggested in the LMI User Manual. """ super().__init__(path) self.bin_factor = 1 self.binning = f'{self.bin_factor} {self.bin_factor}' self.multilamp = multilamp # Set the BIASSEC and TRIMSEC appropriately self.biassec = '[2101:2144,5:512]' if biassec is None else biassec self.trimsec = '[54: 2096,5:512]' if trimsec is None else trimsec # File prefix -- DeVeny files prefix with the UT date if prefix is None: # Look at all the 20*.fits files in this directory, and choose # Note: This will need to be updated for the year 2100 fitsfiles = glob.glob(self.path + '/' + '20*.????.fits') if fitsfiles: slashind = fitsfiles[0].rfind('/') self.prefix = fitsfiles[0][slashind + 1:slashind + 9] else: self.prefix = prefix if self.debug: print(f'Directory prefix: {self.prefix}') # Define standard filenames self.zerofn = 'bias.fits' # Define the gratings self.gratings = { "DV1": "150/5000", "DV2": "300/4000", "DV3": "300/6750", "DV4": "400/8500", "DV5": "500/5500", "DV6": "600/4900", "DV7": "600/6750", "DV8": "831/8000", "DV9": "1200/5000", "DV10": "2160/5000", "DVxx": "UNKNOWN" } self.icl = ccdp.ImageFileCollection( self.path, glob_include=f'{self.prefix}.*.fits')
def load_images(data_dir, grating): """load_images Load in the images associated with DATA_DIR and grating [extended_summary] Parameters ---------- data_dir : `str` The directory containing the data to analyze grating : `str` The grating ID to use Returns ------- `ccdproc.image_collection.ImageFileCollection` IFC of the files meeting the input criteria """ # Dictionary gratid = {'DV1': '150/5000', 'DV2': '300/4000', 'DV5': '500/5500'} # Load the images of interest in to an ImageFileCollection() icl = ccdp.ImageFileCollection(data_dir) # Clean up the telescope altitude for ccd, fn in icl.ccds(return_fname=True): ccd.header['telalt'] = np.round(ccd.header['telalt']) ccd.write(os.path.join(data_dir, fn), overwrite=True) # Return an ImageFileCollection filtered by the grating desired return icl.filter(grating=gratid[grating])
def __init__(self, path, biassec=None, trimsec=None, bin_factor=2): """__init__: Initialize LMI class. Args: path (:TYPE:`str`) Path to the directory containing the images to be reduced. biassec (:TYPE:`str`) The IRAF-style overscan region to be subtracted from each frame. If unspecified, use the values suggested in the LMI User Manual. trimsec (:TYPE:`str`) The IRAF-style image region to be retained in each frame. If unspecified, use the values suggested in the LMI User Manual. bin_factor (:TYPE:`int`) The binning factor used to create the image(s) to be processed. [Default: 2] """ super().__init__(path) self.bin_factor = int(bin_factor) self.binning = f'{self.bin_factor} {self.bin_factor}' # # Set the BIASSEC and TRIMSEC appropriately FOR 2x2 BINNING # if self.bin_factor == 2: # self.biassec = '[3100:3124, 3:3079]' if biassec is None else biassec # self.trimsec = '[30:3094, 3:3079]' if trimsec is None else trimsec # else: self.biassec = biassec self.trimsec = trimsec # Define file prefix & standard filenames self.prefix = 'lmi' self.zerofn = f'bias_bin{self.bin_factor}.fits' # Load initial ImageFileCollection self.icl = ccdp.ImageFileCollection( self.path, glob_include=f'{self.prefix}.*.fits')
def add_units(path, units='adu'): """ Check for units in fits header. If none then add Keyword BUNIT with default = adu Args: path: path to fits file units: optional, default set to adu useful options adu, photon, and electron Returns: None """ ic1 = ccdproc.ImageFileCollection(location=path) # check for units in header if 'bunit' not in ic1.summary.colnames: warnings.warn('Overwriting Original Header!!!') for hdu in ic1.hdus(overwrite=True): hdu.header['bunit'] = 'adu'
def realtimeRed(storePath, analyPath, masterDark): neos = ccdproc.ImageFileCollection(location=analyPath) neoList = [] for neo, fname in neos.hdus(return_fname=True): meta = neo.header meta['filename'] = fname neoList.append(ccdproc.CCDData(data=neo.data, header=meta, unit="adu")) masterBias_e = ccdproc.gain_correct(masterBias, gain=1 * u.electron / u.adu) masterDark_e = ccdproc.gain_correct(masterDark, gain=1 * u.electron / u.adu) masterFlat_e = ccdproc.gain_correct(masterFlat, gain=1 * u.electron / u.adu) for neo in neoList: neo_red = ccdproc.ccd_process(neo, master_bias=masterBias_e, dark_frame=masterDark_e, master_flat=masterFlat_e , gain=1 * u.electron / u.adu, readnoise=readnoise, min_value=1. , dark_exposure=darkExp * u.second, data_exposure=neo.header['exptime'] * u.second , exposure_unit=u.second, dark_scale=True) baseName = os.path.basename(neo.header['filename']) fits.writeto("{}{}_red.fits".format(storePath, baseName.split('.')[0]), neo_red.data, header=neo_red.header, overwrite=False)
def stack(self, tmp_fit_dir_p, filename_p, exposure, image_bitpix): logger.info( 'Stacking dark frames for exposure %0.1fs, gain %d, bin %d', exposure, self.gain_v.value, self.bin_v.value) if image_bitpix == 16: numpy_type = numpy.uint16 elif image_bitpix == 8: numpy_type = numpy.uint8 dark_images = ccdproc.ImageFileCollection(tmp_fit_dir_p) cal_darks = dark_images.files_filtered(imagetyp='Dark Frame', exptime=exposure, include_path=True) start = time.time() combined_dark = ccdproc.combine( cal_darks, method='average', sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=numpy.ma.median, signma_clip_dev_func=mad_std, dtype=numpy_type, mem_limit=350000000, ) elapsed_s = time.time() - start logger.info('Exposure sigma clip stacked in %0.4f s', elapsed_s) combined_dark.meta['combined'] = True combined_dark.write(filename_p)
def calibrate_collection(self, options, collection): write_path = Path(f'{os.getcwd()}/calibrated') write_path.mkdir(exist_ok=True) count = 1 for img, fname in collection.ccds(return_fname=True): print( f'\n{Style.BRIGHT}[{count}/{len(self.collection.files)}] Calibrating: {fname + Style.RESET_ALL}' ) ccd_temp = img.header['ccd-temp'] gain = img.header['gain'] offset = img.header['offset'] exptime = img.header['exptime'] filter = img.header['filter'] print(f'CCD_TEMP: {ccd_temp}') print(f'GAIN: {gain}') print(f'OFFSET: {offset}') print(f'EXPTIME: {exptime}') print(f'FILTER: {filter}') self.calibrate_image(options, img).write(write_path / fname, overwrite=True) count += 1 return ccdp.ImageFileCollection(write_path)
from astropy import units as u from astropy.io import fits import matplotlib.pyplot as plt import ccdproc as cp # The path listed below is for the directory on ShadowHQ not local. path = '/Users/JPeg/cometdata/' fileKeys = ['FILENAME','NAXIS1','NAXIS2','OBJECT','OBSTYPE', 'FILTERS' ] allfiles = cp.ImageFileCollection(path,keywords = fileKeys) gain = 2.89 * u.electron/u.adu rdnoise = 6 * u.electron # correct bias images for overscan and trim the image: biasImages=[] for filename in allfiles.files_filtered(NAXIS1=3128,NAXIS2=3080,OBSTYPE = 'BIAS'): ccd = fits.getdata(allfiles.location + filename) print(ccd) ccd = cp.CCDData(ccd, unit = u.adu) print(ccd) ccd = cp.subtract_overscan(ccd,overscan_axis = 1, overscan = ccd[:,3099:3125]) print(ccd) ccd = cp.trim_image(ccd,fits_section = '[27:3095,3:3078]') print(ccd) biasImages.append(ccd)
def calibrate_images( x_d, x_f, x_s, it_s='object', x_b='', ): """ Parameters ---------- ---------- x_b : str Path of the bias files. By default bias is not provided x_d : str Path of the dark files x_f : str Path of flat files x_s : str Path of Science files it_s : str Imagetyp of fits science file Default is set to object ----------- """ path_b = Path(x_b) path_d = Path(x_d) path_f = Path(x_f) path_s = Path(x_s) #-----Bias if x_b == '' and path_b == Path(''): print( 'Be aware: You did not provide the Bias files; the process will still continue though.' ) files_b = None elif not path_b.is_dir(): raise RuntimeError( 'The path you provided for the Bias files does not exist.') else: files_b = ccdp.ImageFileCollection(path_b) #-----Dark if x_d == '' or not path_d.is_dir(): raise RuntimeError( 'You must provide Dark files for processing.\n Or the path you provided does not exist.' ) else: files_d = ccdp.ImageFileCollection(path_d) #-----Flat if x_f == '' or not path_f.is_dir(): raise RuntimeError( 'You must provide Flatfield files for processing.\n Or the path you provided does not exist.' ) else: files_f = ccdp.ImageFileCollection(path_f) #-----Science if x_s == '' or not path_s.is_dir(): raise RuntimeError( 'You must provide Science images for processing.\n Or the path you provided does not exist.' ) else: files_s = ccdp.ImageFileCollection(path_s) #----------------------------------- # #--------Calibrating Images--------- # #----------------------------------- if files_b is not None: #------------------------------- #------Creating Master-bias----- #------------------------------- cali_bias_path = Path(path_b / 'cali_bias') cali_bias_path.mkdir(exist_ok=True) files_b_cali = files_b.files_filtered(imagetyp='bias', include_path=True) combined_bias = ccdp.combine(files_b_cali,\ method='average',\ sigma_clip=True,\ sigma_clip_low_thresh=5,\ sigma_clip_high_thresh=5,\ sigma_clip_func=np.ma.median,\ sigma_clip_dev_func=mad_std,\ mem_limit=350e6) combined_bias.meta['combined'] = True combined_bias.write(cali_bias_path / 'master_bias.fits') # Reading master bias master_bias = CCDData.read(cali_bias_path / 'master_bias.fits') else: master_bias = None #------------------------------- #-------Calibrating Darks------- #------------------------------- cali_dark_path = Path(path_d / 'cali_dark') cali_dark_path.mkdir(exist_ok=True) files_d_cali = files_d.files_filtered(imagetyp='DARK', include_path=True) for ccd, file_name in files_d.ccds(imagetyp='DARK', return_fname=True, ccd_kwargs={'unit': 'adu'}): if master_bias is not None: # Subtract bias ccd = ccdp.subtract_bias(ccd, master_bias) else: ccd = ccd # Save the result ccd.write(cali_dark_path / file_name) #-------------------------------- #------Creating Master-Dark------ #-------------------------------- red_dark = ccdp.ImageFileCollection(cali_dark_path) # Calculating exposure times of DARK images dark_times = set( red_dark.summary['exptime'][red_dark.summary['imagetyp'] == 'DARK']) for exposure in sorted(dark_times): cali_darks = red_dark.files_filtered(imagetyp = 'dark',\ exptime = exposure,\ include_path = True) combined_dark = ccdp.combine(cali_darks,\ method='average',\ sigma_clip=True,\ sigma_clip_low_thresh=5,\ sigma_clip_high_thresh=5,\ sigma_clip_func=np.ma.median,\ sigma_clip_dev_func=mad_std,\ mem_limit=350e6) combined_dark.meta['combined'] = True com_dark_name = 'combined_dark_{:6.3f}.fits'.format(exposure) combined_dark.write(cali_dark_path / com_dark_name) # Reading master dark of various exposure times red_dark = ccdp.ImageFileCollection(cali_dark_path) combined_darks = { ccd.header['exptime']: ccd for ccd in red_dark.ccds(imagetyp='DARK', combined=True) } #-------------------------------- #-------Calibrating Flats-------- #-------------------------------- cali_flat_path = Path(path_f / 'cali_flat') cali_flat_path.mkdir(exist_ok=True) files_f_cali = files_f.files_filtered(imagetyp='FLAT', include_path=True) for ccd, file_name in files_f.ccds(imagetyp='FLAT', ccd_kwargs={'unit': 'adu'}, return_fname=True): # Subtract bias if master_bias is not None: ccd = ccdp.subtract_bias(ccd, master_bias) else: ccd = ccd closest_dark = utl.find_nearest_dark_exposure(ccd, dark_times) if closest_dark is None: closest_dark1 = utl.find_nearest_dark_exposure(ccd, dark_times, tolerance=100) # Subtract scaled Dark ccd = ccdp.subtract_dark(ccd, combined_darks[closest_dark1],\ exposure_time = 'exptime',\ exposure_unit = u.second,\ scale = True) ccd.write(cali_flat_path / ('flat-' + file_name)) else: closest_dark2 = utl.find_nearest_dark_exposure(ccd, dark_times) # Subtracting Darks ccd = ccdp.subtract_dark(ccd, combined_darks[closest_dark2], exposure_time='exptime', exposure_unit=u.second) ccd.write(cali_flat_path / ('flat-' + file_name)) #-------------------------------- #-----Creating Master-Flat------- #-------------------------------- red_flats = ccdp.ImageFileCollection(cali_flat_path) cali_flats = red_flats.files_filtered(imagetyp='FLAT', include_path=True) combined_flat = ccdp.combine(cali_flats,\ method='average',\ scale = utl.inverse_median,\ sigma_clip=True,\ sigma_clip_low_thresh=5,\ sigma_clip_high_thresh=5,\ sigma_clip_func=np.ma.median,\ sigma_clip_dev_func=mad_std,\ mem_limit=350e6) combined_flat.meta['combined'] = True combined_flat.write(cali_flat_path / 'master_flat.fits') # Reading master flat red_flats = ccdp.ImageFileCollection(cali_flat_path) combined_flat = CCDData.read(cali_flat_path / 'master_flat.fits') #-------------------------------- #---Calibrating Science Images--- #-------------------------------- # Creating a list of spectrum images files_spec = files_s.summary['file', 'view_pos'] files_spec_list = np.array([]) for i in range(len(files_spec)): xxx = files_spec['view_pos'][i] if xxx[0:4] == 'open': files_spec_list = np.hstack( (files_spec_list, files_spec['file'][i])) # Sky subtracting images cali_science_path1 = Path(path_s / 'Sky_subtracted_science') cali_science_path1.mkdir(exist_ok=True) j = 0 for i in range(int(len(files_spec_list) / 2)): ccd1 = CCDData.read(x_s + files_spec_list[j], unit='adu') ccd2 = CCDData.read(x_s + files_spec_list[j + 1], unit='adu') sky_sub1 = ccd1.data - ccd2.data ss1 = CCDData(sky_sub1, unit='adu') ss1.header = ccd1.header ss1.meta['sky_sub'] = True name1 = 'sky_sub_' + files_spec_list[j] ss1.write(cali_science_path1 / name1) sky_sub2 = ccd2.data - ccd1.data ss2 = CCDData(sky_sub2, unit='adu') ss2.header = ccd2.header ss2.meta['sky_sub'] = True name2 = 'sky_sub_' + files_spec_list[j + 1] ss2.write(cali_science_path1 / name2) j = j + 2 files_s1 = ccdp.ImageFileCollection(cali_science_path1) final_calibrated = Path(path_s / 'Final_calibrated_science') final_calibrated.mkdir(exist_ok=True) # Correcting for flat for ccd, file_name in files_s1.ccds(imagetyp=it_s, ccd_kwargs={'unit': 'adu'}, return_fname=True): # Subtract scaled Dark ccd = ccdp.flat_correct(ccd, combined_flat) #['FLAT']) ccd.write(final_calibrated / file_name)
def flux_extraction(file_name, path, out_path, images=True): """ Parameters ---------- ---------- file_name : str Name of the image/telluric file from which flux has to be extracted path : str Path of the desired image file out_path : str Path of the output data and/or image file images : bool True if one wants to save visualization of flux data False if not. Default is True ---------- returns ---------- flux : data file .dat file containing the flux at various pixel values Path of this file would be similar to that of image file. ---------- """ pt = Path(path) f1 = ccdp.ImageFileCollection(pt) ccd = CCDData.read(path + file_name) # + '.fits') # Trimming the Image trimmed = ccdp.trim_image(ccd, fits_section='[1:256, 100:1000]') trimmed.meta['TRIM'] = True trimmed.header = ccd.header #trimmed.write(file_name + '_trim.fits') # Reading the data from Trimmed image data = trimmed.data # Creating a function to detect the edges of slit # For lower edge def xlow(raw_data): """ Parameters ---------- ---------- raw_data : numpy.ndarray Array containing flux at some particular wavelength ---------- returns ---------- number : float A pixel number showing the lower edge of slit ---------- """ j = 0 for i in range(int(len(raw_data) / 5)): st = np.std(raw_data[j:j + 5]) xlw = 0 if st < 2: xlw = j if xlw != 0: break j = j + 5 return xlw # For upper edge def xup(raw_data): """ Parameters ---------- ---------- raw_data : numpy.ndarray Array containing flux at some particular wavelength ---------- returns ---------- number : float A pixel number showing the upper edge of slit ---------- """ j = 255 for i in range(int(len(raw_data) / 5)): st = np.std(raw_data[j - 5:j]) xup = 0 if st < 2: xup = j if xup != 0: break j = j - 5 return xup # Defining line and inverse line def line(x, m, c): return m * x + c def inv_line(x, m, c): bc = (x - c) / m return bc # Detecting the edges of the spectrum ys = np.array([150, 300, 450, 600, 750]) xs_left = np.array([]) xs_right = np.array([]) xs_mid = np.array([]) for i in range(len(ys)): dd1 = data[ys[i]] xll = xlow(dd1) xs_left = np.hstack((xs_left, xll)) xuu = xup(dd1) xs_right = np.hstack((xs_right, xuu)) popt_l, pcov_l = cft(line, xs_left, ys) popt_r, pcov_r = cft(line, xs_right, ys) # Detecting a line where spectrum could reside for i in range(len(ys)): ran_l = inv_line(ys[i], popt_l[0], popt_l[1]) ran_r = inv_line(ys[i], popt_r[0], popt_r[1]) xd1 = data[ys[i]] xd = xd1[int(ran_l):int(ran_r)] ma = np.max(xd) ab = np.where(xd == ma) xs_mid = np.hstack((xs_mid, ab[0][0] + ran_l)) popt_m, pcov_m = cft(line, xs_mid, ys) # Finding total flux def total_flux(lam, xlim=20): ydata = data[lam] xmid = inv_line(lam, popt_m[0], popt_m[1]) xlow = xmid - xlim xup = xmid + xlim total_flux1 = 0 xdata = np.arange(int(xlow), int(xup + 1), 1) for i in range(len(xdata)): total_flux1 = total_flux1 + ydata[xdata[i]] return total_flux1 # Flux as a function of pixel flux = np.array([]) y11 = np.arange(0, 900, 1) for i in range(len(y11)): f11 = total_flux(y11[i]) flux = np.hstack((flux, f11)) # Saving the image file for flux if images == True: fig1 = plt.figure(figsize=(20, 10)) plt.plot(flux) plt.xlabel('Pixel Number') plt.ylabel('Total Flux') plt.title('Total flux for ' + file_name + ' observation') plt.grid() plt.savefig(out_path + '/' + file_name + '_flux.png') plt.close(fig1) # Saving Data file of the flux f1 = open(out_path + '/' + file_name + '_flux.dat', 'w') f1.write('#Pixel\t\tFlux\n') for i in range(len(y11)): f1.write(str(y11[i]) + '\t\t' + str(flux[i]) + '\n') f1.close()
def reduce_night(year=datetime.date.today().year, month=datetime.date.today().month, day=datetime.date.today().day, telescope="C28", config_file="config.ini"): config = get_config(config_file) log = init_log(time.strftime("%Y%m%d_%H%M%S", time.gmtime()), config_file) t = datetime.date(year, month, day) t_str = datetime.date.strftime(t, format="%Y%m%d") im_path = config.get("GENERAL", "PATH") + config.get( telescope, "PATH") + t_str + config.get(telescope, "DIR_SUFFIX") + os.sep if not os.path.isdir(im_path): log.warning(f"Folder {im_path} doesn't exist!") return log.info( f"""Creating {telescope} master calibration frames for {t_str}...""") calframes.create_masters(year, month, day, telescope, log=log, config_file=config_file) imlist = ccdproc.ImageFileCollection(im_path, keywords='*') files = imlist.files_filtered(imagetyp="LIGHT") if len(files.tolist()) == 0: log.warning(f"No science frames in {t_str}.") return imlist = ccdproc.ImageFileCollection(im_path, keywords='*', filenames=files.tolist()) save_uncertainty = config.getboolean("GENERAL", "SAVE_UNCERTAINTY") is_overwrite = config.getboolean("GENERAL", "OVERWRITE") instrument = imlist.values("instrume", True)[0] ccd_shape = utils.get_ccd_shape(imlist, telescope) reduced_path = im_path + config.get("GENERAL", "REDUCED_DIR") + os.sep # create reduced folder if not os.path.exists(reduced_path): os.makedirs(reduced_path) log.debug(f"""ccd_shape length {len(ccd_shape["x_naxis"])}""") for i in range(len(ccd_shape["x_naxis"])): ccd_set = utils.get_set_from_dict(ccd_shape, i) ccd_str = utils.get_ccd_str(ccd_shape, idx=i) filters = np.unique(imlist.summary["filter"]) log.debug(f"{filters}") for filt in filters: bias_file, dark_file, flat_file = calframes.get_calframes( year, month, day, filt, ccd_str, telescope=telescope, instrument=instrument, log=log, config_file=config_file) if (not bias_file) or (not dark_file) or (not flat_file): log.warning(f"No calibration frames found, skipping.") continue bias = ccdproc.CCDData.read(bias_file) dark = ccdproc.CCDData.read(dark_file) flat = ccdproc.CCDData.read(flat_file) kwargs = dict() kwargs[get_key_name("image_type", telescope)] = get_key_val("light", telescope) kwargs[get_key_name("filter", telescope)] = filt kwargs[get_key_name("x_naxis", telescope)] = ccd_set["x_naxis"] kwargs[get_key_name("y_naxis", telescope)] = ccd_set["y_naxis"] kwargs[get_key_name("x_subframe", telescope)] = ccd_set["x_subframe"] kwargs[get_key_name("y_subframe", telescope)] = ccd_set["y_subframe"] kwargs[get_key_name("x_bin", telescope)] = ccd_set["x_bin"] kwargs[get_key_name("y_bin", telescope)] = ccd_set["y_bin"] for im, filename in imlist.hdus(return_fname=True, **kwargs): log.debug(f"{filename}") obj = im.header[get_key_name("object", telescope)] jd = im.header[get_key_name("jd", telescope)] * u.day # fix JD for RBI flood delay (C28): if telescope == "C28": if im.header[get_key_name("readout", telescope)] == get_key_val( "rbi", telescope): jd = jd + get_key_val("rbi_delay", telescope) im.header[get_key_name("jd", telescope)] = jd.to_value() im.header[get_key_name("rbi_delay", telescope)] = "TRUE" im.header.comments[get_key_name( "rbi_delay", telescope )] = f"""Corrected JD by {get_key_val("rbi_delay", telescope)} of RBI flood delay.""" log.warning( f"""Corrected RBI flood delay of {get_key_val("rbi_delay", telescope)}.""" ) jd = str(jd.to_value()).replace(".", "_") filename = f"{obj}_{jd}_{filt}_{telescope}" file_exists = os.path.isfile(reduced_path + filename + ".fits") if (not file_exists) | (file_exists & is_overwrite): im.data = im.data.astype("float32") im = ccdproc.subtract_bias( ccdproc.CCDData(data=im.data, unit=u.adu, header=im.header), bias, add_keyword=ccdproc.Keyword("DEBIAS", value=bias_file.split( os.sep)[-1])) im = ccdproc.subtract_dark( im, dark, exposure_time=get_key_name("exptime", telescope), exposure_unit=u.s, scale=True, add_keyword=ccdproc.Keyword("DEDARK", value=dark_file.split( os.sep)[-1])) im = ccdproc.flat_correct(im, flat, add_keyword=ccdproc.Keyword( "DEFLAT", value=flat_file.split( os.sep)[-1])) if not save_uncertainty: im.uncertainty = None im.mask = None im.data = im.data.astype('float32') im.write(reduced_path + filename + ".fits", overwrite=True) close_log(log) return reduced_path
def __init__(self, data=None): self.main_path = Path(cfg.CALIBRATION_PATH) self.masters = ccdp.ImageFileCollection(self.main_path) if data: self.collection = ccdp.ImageFileCollection(filenames=data)
def flat_combine(self): """ :return: """ if self.debug: print("Combining flats...") # Load the list of bias-subtracted data frames bsub_cl = ccdp.ImageFileCollection( self.path, glob_include=f'{self.prefix}.*b.fits') # Find just the flats # NOTE: When CCDPROC filters an ImgFileCol, the resulting filenames # have the path attached. This allows for generality, but will # need to be accounted for. flats_cl = bsub_cl.filter(imagetyp="dome flat") # Check that we have any if flats_cl.files: # In case more than one grating was used (unlikely except eng) for grname in list(set(list(flats_cl.summary['grat_id']))): # Filter the ImgFileColl to include only this grating gr_cl = flats_cl.filter(grat_id=grname) # In case more than one grating tilt angle was used (possible) for gra in list(set(list(gr_cl.summary['grangle']))): # Filter the ImgFileColl to include only this tilt gra_cl = gr_cl.filter(grangle=gra) # In case more than one order-blocking filter was used (???) for filt in list(set(list(gra_cl.summary['filtrear']))): # Filter the ImgFileColl to include only this filter filt_cl = gra_cl.filter(filtrear=filt) # For engineering, possibly use different lamps for comp if self.multilamp: lamps = list(set(list(filt_cl.summary['comment']))) else: lamps = ['domelamp'] if self.debug: print(f'Flat lamps used: {lamps}') for this_lamp in lamps: if self.multilamp: lamp_cl = filt_cl.filter(comment=this_lamp) lname = '_TRING' if this_lamp[0:3] == 'Top' \ else '_FLOOD' else: lamp_cl = filt_cl lname = '' # Actually do the flat combining cflat = ccdp.combine(lamp_cl.files, method='median', sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.ma.median, sigma_clip_dev_func=mad_std, mem_limit=4e9) # Add FITS keyword NCOMBINE and HISTORY cflat.header.set( 'ncombine', len(lamp_cl.files), '# of input images in combination') cflat.header['HISTORY'] = PKG_NAME cflat.header['HISTORY'] = 'Combined flat ' + \ 'created: ' + _savetime() cflat.header['HISTORY'] = \ f'Median combined {len(lamp_cl.files)} files:' for fn in lamp_cl.files: # Note: These filenames have the path attached, # via the .filter() method of ImgFileCol. # Include just the pathless filename. cflat.header['HISTORY'] = fn[fn.rfind('/') + 1:] # Build filename, save, remove input files flat_fn = f'flat_{grname}_{gra}_{filt}{lname}.fits' if self.debug: print(f'Saving combined flat as {flat_fn}') cflat.write(f'{self.path}/{flat_fn}', overwrite=True) for fn in lamp_cl.files: # Note: These filenames have the path already # attached, via the .filter() method of # ImgFileCol. os.remove(f'{fn}') else: print("No flats to be combined.")
def imcombine(*infiles, inlist=None, outfn=None, del_input=False, combine=None, printstat=True, overwrite=True, returnccd=False): """Combine a collection of images This function (crudely) emulates the IRAF imcombine function. Pass in a list of images to be combined, and the result is written to disk with an optionally specified output filename. :param infiles: `list`: List of filenames to combine :param inlist: `str`: Filename of text file listing images to be combined :param outfn: `str`: Filename to write combined image. Default: append '_comb' to first filename in the input list. :param del_input: `bool`: Delete the input files after combination. Default: `false` :param combine: `str`: Combine method. 'median' (default), or 'mean' :param printstat: `bool`: Print image statistics to screen :param overwrite: `bool`: Overwrite the output file. Default: True :return: None (Unless returnccd=True, then it returns the combined CCDData) """ # Unpack the single-item tuple *infiles if len(infiles) > 0: files, = infiles else: files = [] # Check for inputs if len(files) > 0 and inlist is not None: print("Only one of files or inlist may be specified, not both.") raise Exception() # Read in the text list inlist, if specified if inlist is not None: with open(inlist, 'r', encoding='utf-8') as f: files = [] for line in f: files.append(line.rstrip()) # Check for proper file list if len(files) < 3: print("Combination requires at least three input images.") raise Exception() # Check that specified input files exist for f in files: if not os.path.isfile(f): print(f"File {f} does not exist.") raise Exception() # Determine combine method (default = 'median') if combine != 'median' or combine != 'mean': combine = 'median' # Create an ImgFileColl using the input files file_cl = ccdp.ImageFileCollection(filenames=files) if printstat: # Print out the statistics, for clarity for img, fn in file_cl.ccds(return_fname=True): mini, maxi, mean, stdv = mmms(img) print(f'{fn}:: Min: {mini:.2f} Max: {maxi:.2f} ' + f'Mean: {mean:.2f} Stddev: {stdv:.2f}') comb_img = ccdp.combine(file_cl.files, method=combine, sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.ma.median, sigma_clip_dev_func=mad_std, mem_limit=4e9) # Add FITS keyword NCOMBINE and add HISTORY comb_img.header.set('ncombine', len(file_cl.files), '# of input images in combination') comb_img.header['HISTORY'] = PKG_NAME comb_img.header['HISTORY'] = 'Combined image created: ' + _savetime() comb_img.header['HISTORY'] = f'{combine.title()} combined ' + \ f'{len(file_cl.files)} files:' for fn in file_cl.files: comb_img.header['HISTORY'] = fn # If returnccd is True, return now before thinking about saving. if returnccd: return comb_img # Build filename (if not specified in call), save, remove input files if outfn is None: outfn = f'{files[0][:-5]}_comb{files[0][-5:]}' print(f'Saving combined image as {outfn}') comb_img.write(f'{outfn}', overwrite=overwrite) if del_input: for f in files: os.remove(f'{f}') return None
def get_log(folder, extra_keys=[], write=True): """ Given a folder, it generate a log file with the listing of the keys given on the keys parameter. Default are the keys from OPD. Parameters ---------- folder : String Path to the night run folder with the FITS files. extra_keys : List Listing of keys to look at in order, in addition to the default ones. write : bool If True writes an output csv (default is True). Retuns ------ log_tab: DataFrame Table containing information given on keys File transformations -------------------- Write to disk a file named w/ the folder name + "_night.log" containing the log inside the folder when write is True. """ out = Path(folder) # Check if there are fles if len(list(out.glob("*.fits"))) == 0: print( "ERROR: No files found, can't create log dataframe for orientation. Returning None" ) return None out = out / f"{str(out)}_night.log" keys = ["DATE-OBS", "OBJECT", "FILTER", "EXPTIME", "AIRMASS", "COMMENT"] keys.extend(extra_keys) ifc = ccdproc.ImageFileCollection(folder, keywords=keys) df = ifc.summary.to_pandas(index="file") # Cleaning OPD comment and exptime. Then standardizing OBJECT and FILTER. def standardize(value): """ Put values on standard way for folder creation by removing certain characteres, and having it in upper case. (str -> str) """ translator = str.maketrans({" ": "", "\\": "-", "/": "-"}) return value.translate(translator).upper() cleaners = { "COMMENT": lambda value: value.split("'")[1].strip(), "EXPTIME": lambda value: int(value.split(",")[0]), "OBJECT": standardize, "FILTER": standardize } for key in cleaners: df[key] = df[key].apply(cleaners[key]) if write: df.to_csv(out) return [df, str(out)]
""" Created on Thu May 7 16:56:18 2020 @author: ambar """ import sp from astropy.io import ascii import time import ccdproc as ccdp import numpy as np a='fasfas' print(type(a)) science=ccdp.ImageFileCollection('Corrected_Science') #ascii.write(sc,'sc_summary.csv',format='csv',overwrite=True) for i,j in science.hdus(return_fname=True): print(i) #found=np.zeros(len(hdulist),dtype=type(a)) ''' start_time=time.time() for imsc, imsd in enumerate(science.data()): f=sp.find(imsd,hmin=1000,fwhm=12) found[imsc]=len(f[0]) ascii.write(found,'found.csv',format='csv',overwrite=True) '''
def flat_combine(self): """Finds and combines flat frames with the indicated binning :return: None """ # Load the list of bias-subtracted data frames -- check binning bsub_cl = ccdp.ImageFileCollection( self.path, glob_include=f'{self.prefix}.*b.fits') if not bsub_cl.files: print("Nothing to be done for flat_combine()!") return if self.debug: print("Normalizing flat field frames...") # Filter here to get # of images for progress bar flat_cl = bsub_cl.filter(ccdsum=self.binning, imagetyp='[a-z]+ flat', regex_match=True) # Set up a progress bar, so we can see how the process is going... prog_bar = tqdm(total=len(flat_cl.files), unit='frame', unit_scale=False, colour='yellow') # Normalize flat field images by the mean value for ccd, flat_fn in flat_cl.ccds(return_fname=True): # Perform the division ccd = ccd.divide(np.mean(ccd), handle_meta='first_found') # Update the header ccd.header['HISTORY'] = 'Normalized flat saved: ' + _savetime() ccd.header['HISTORY'] = f'Previous filename: {flat_fn}' # Save the result (suffix = 'n'); delete the input file ccd.write(f'{self.path}/{flat_fn[:-6]}n{flat_fn[-5:]}', overwrite=True) os.remove(f'{self.path}/{flat_fn}') # Update the progress bar prog_bar.update(1) # Close the progress bar, end of loop prog_bar.close() # Load the list of normalized flat field images norm_cl = ccdp.ImageFileCollection( self.path, glob_include=f'{self.prefix}.*n.fits') if norm_cl.files: # Create a unique list of the filter collection found in this set filters = list(norm_cl.summary['filters']) unique_filters = sorted(list(set(filters))) # Combine flat field frames for each filt in unique_filters for filt in unique_filters: flats = norm_cl.files_filtered(filters=filt, include_path=True) print(f"Combining flats for filter {filt}...") cflat = ccdp.combine(flats, method='median', sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.ma.median, sigma_clip_dev_func=mad_std, mem_limit=4e9) # Add FITS keyword NCOMBINE and HISTORY cflat.header.set('ncombine', len(flats), '# of input images in combination') cflat.header['HISTORY'] = PKG_NAME cflat.header['HISTORY'] = 'Combined flat created: ' + \ _savetime() cflat.header['HISTORY'] = 'Median combined ' + \ f'{len(flats)} files:' for fn in flats: # Remove the path portion of the filename for the HISTORY cflat.header['HISTORY'] = fn[fn.rfind('/') + 1:] # Build filename, save, remove input files flat_fn = f'flat_bin{self.bin_factor}_{filt}.fits' if self.debug: print(f'Saving combined flat as {flat_fn}') cflat.write(f'{self.path}/{flat_fn}', overwrite=True) for fn in flats: # Path name is already included os.remove(f'{fn}') else: print("No flats to be combined.")
if len(LWTdata_flatPaths) < 5: os.system("cp /LWTdata/LWT_{0}/flat/*V_Astrodon* /LWTanaly/{0}/flat/.".format(date)) else: for path in LWTdata_flatPaths[:5]: os.system("cp {} /LWTanaly/{}/flat/.".format(path, date)) gain = 1.4 * u.electron / u.adu gainValue = 1.4 readnoise = 8.2 * u.electron readnoiseValue = 8.2 biasPath = "/LWTanaly/{}/bias/".format(date) darkPath_test = "/LWTanaly/{}/dark_test/".format(date) flatPath = "/LWTanaly/{}/flat/".format(date) # Create master bias frame. biases = ccdproc.ImageFileCollection(location=biasPath) biasList = [] for bias, fname in biases.hdus(return_fname=True): meta = bias.header meta['filename'] = fname biasList.append(ccdproc.CCDData(data=bias.data, meta=meta, unit='adu')) masterBias = ccdproc.combine(biasList, output_file="/LWTanaly/{}/bias/Master_Bias.fits".format(date), method='median' , clip_extrema=True, nlow=0, nhigh=1) # Create master dark frame for each NEO. darkPaths = [i for i in glob("/LWTanaly/{}/dark_*".format(date)) if 'test' not in i] masterDark = [] for path in darkPaths: darks = ccdproc.ImageFileCollection(location=path+'/') darkList = [] for dark, fname in darks.hdus(return_fname=True):
from pathlib import Path from subprocess import run from datetime import datetime import ccdproc as ccdp from astropy.stats import mad_std import numpy as np from matplotlib import pyplot as plt from calibrate import calibrate_collection import helpers as hlp import config as cfg calibration_path = Path(cfg.CALIBRATION_PATH) calibration_masters = ccdp.ImageFileCollection(calibration_path) if __name__ == "__main__": import argparse parser = argparse.ArgumentParser( description='Master flat generation script. Uses average combination and sigma clipping pixel rejection. Generates bias-substracted and raw masters.') parser.add_argument( 'files', help='input files (FITS only)', type=str, nargs='+') parser.add_argument('-c', '--calibrated', action='store_true', help='generate bias-substracted master dark') parser.add_argument( '-s', '--sigmalow', help='sigma low threshold for pixel rejection', default=5) parser.add_argument( '-S', '--sigmahigh', help='sigma high threshold for pixel rejection', default=5)
'-R', '--radius', help='search radius around center (degrees, default=15 unless no RA or DEC: 360)', type=str, dest='radius', default=15) parser.add_argument( '-s', '--scale', help='estimated field scale (arcminutes, online solver only)', type=int, dest='scaleEst') parser.add_argument( '-e', '--error', help='estimated field scale error (percent, online solver only)', type=int, dest='scaleErr') parser.add_argument('-D', '--downsample', help='downsampling amount', type=int) parser.add_argument('-b', '--blind', action='store_true', help='blind solve') parser.add_argument('-y', '--noconfirm', action='store_true', help='skip confirmation') parser.add_argument('-o', '--online', action='store_true', help='use online solver (requires internet connection)') args = parser.parse_args() images = ccdp.ImageFileCollection(filenames=args.files) print( f'{Style.BRIGHT}Platesolving {len(images.files)} files.{Style.RESET_ALL}') if (not args.ra and not args.dec) and not args.blind: try: args.ra = ccdp.CCDData.read(args.files[0]).header['ra'] args.dec = ccdp.CCDData.read(args.files[0]).header['dec'] print( f'{Style.BRIGHT + Fore.GREEN}Found WCS in file, using as target.{Style.RESET_ALL}') except: print( f'{Style.BRIGHT + Fore.RED}No WCS found.{Style.RESET_ALL}') args.blind = True if not args.blind:
def reduce_sami(path): log.info('SAMI Data-Reduction Pipeline') log.info('Version {}'.format(version.__str__)) sami_merger = merge.SamiMerger() reduced_path = os.path.join(path, 'RED') if os.path.exists(reduced_path): log.warning('Skipping existing directory: {}'.format(reduced_path)) else: log.info( 'Creating directory for reduced data: {}'.format(reduced_path)) os.makedirs(os.path.join(path, "RED"), exist_ok=True) list_of_files = glob.glob(os.path.join(path, '*.fits')) log.info('Reading raw files') table = [] for _file in list_of_files: try: hdu = pyfits.open(_file) except OSError: log.warning("Could not read file: {}".format(_file)) continue if numpy.std(hdu[1].data.ravel()) == 0: log.warning("Bad data found on file: {}".format(_file)) continue row = { 'filename': _file, 'obstype': hdu[0].header['obstype'], 'filter_id': hdu[0].header['filters'], 'filter_name': hdu[0].header['filter1'], 'binning': [int(b) for b in hdu[1].header['ccdsum'].strip().split(' ')] } table.append(row) list_of_binning = [] for row in table: if row['binning'] not in list_of_binning: list_of_binning.append(row['binning']) log.info('Found new binning mode: {}'.format( row['binning'][0], row['binning'][1])) for binning in list_of_binning: log.info('Organizing data.') sub_table = [row for row in table if row['binning'] == binning] zero_table = [row for row in sub_table if row['obstype'] == 'ZERO'] dflat_table = [row for row in sub_table if row['obstype'] == 'DFLAT'] sflat_table = [row for row in sub_table if row['obstype'] == 'SFLAT'] obj_table = [row for row in sub_table if row['obstype'] == 'OBJECT'] log.info('Processing ZERO files') zero_files = [r['filename'] for r in zero_table] zero_files.sort() zero_list_name = os.path.join( path, "RED", "0Zero{}x{}".format(binning[0], binning[1])) with open(zero_list_name, 'w') as zero_list_buffer: for zero_file in zero_files: sami_merger.zero_file = None sami_merger.flat_file = None path, fname = os.path.split(zero_file) prefix = sami_merger.get_prefix() output_zero_file = os.path.join(path, 'RED', prefix + fname) if os.path.exists(output_zero_file): log.warning( 'Skipping existing file: {}'.format(output_zero_file)) continue log.info('Processing ZERO file: {}'.format(zero_file)) data = sami_merger.get_joined_data(zero_file) header = pyfits.getheader(zero_file) data, header, prefix = sami_merger.join_and_process( data, header) pyfits.writeto(output_zero_file, data, header) zero_list_buffer.write('{:s}\n'.format(zero_file)) log.info('Combining ZERO files.') master_zero_fname = zero_list_name + '.fits' if os.path.exists(master_zero_fname): log.warning('Skipping existing MASTER ZERO: {:s}'.format( master_zero_fname)) else: ic = ccdproc.ImageFileCollection(location=os.path.join( path, 'RED'), glob_include='*.fits', keywords=KEYWORDS) zero_combine_files = [ os.path.join(path, 'RED', f) for f in ic.files_filtered(obstype='ZERO') ] log.info("Writing master zero to: {}".format(master_zero_fname)) zero_combine = combine.ZeroCombine(input_list=zero_combine_files, output_file=master_zero_fname) zero_combine.run() log.info('Done.') log.info('Processing FLAT files (SFLAT + DFLAT)') all_flats = sflat_table + dflat_table filters_used = [] for row in all_flats: if row['filter_id'] not in filters_used: filters_used.append(row['filter_id']) log.info('Found new filter: {}'.format(row['filter_name'])) for _filter in filters_used: log.info('Processing FLATs for filter: {}'.format(_filter)) sub_table_by_filter = [ row for row in all_flats if row['filter_id'] == _filter ] flat_list_name = os.path.join( path, 'RED', "1FLAT_{}x{}_{}".format(binning[0], binning[1], _filter)) flat_files = [row['filename'] for row in sub_table_by_filter] flat_files.sort() flat_combine_files = [] with open(flat_list_name, 'w') as flat_list_buffer: for flat_file in flat_files: sami_merger.zero_file = master_zero_fname sami_merger.flat_file = None prefix = sami_merger.get_prefix() path, fname = os.path.split(flat_file) output_flat_file = os.path.join(path, 'RED', prefix + fname) if os.path.exists(os.path.join(path, output_flat_file)): log.warning('Skipping existing FLAT file: {}'.format( output_flat_file)) continue log.info('Processing FLAT file: {}'.format(flat_file)) d = sami_merger.get_joined_data(flat_file) h = pyfits.getheader(flat_file) d, h, p = sami_merger.join_and_process(d, h) pyfits.writeto(output_flat_file, d, h) flat_list_buffer.write('{:s}\n'.format(output_flat_file)) flat_combine_files.append(output_flat_file) master_flat_fname = flat_list_name + '.fits' if os.path.exists(master_flat_fname): log.warning('Skipping existing MASTER FLAT: {:s}'.format( master_flat_fname)) else: log.info('Writing master FLAT to file: {}'.format( master_flat_fname)) flat_combine = combine.FlatCombine( input_list=flat_combine_files, output_file=master_flat_fname) flat_combine.run() for _filter in filters_used: master_flat_fname = os.path.join( path, 'RED', "1FLAT_{}x{}_{}.fits".format(binning[0], binning[1], _filter)) sami_merger.zero_file = master_zero_fname sami_merger.flat_file = master_flat_fname sami_merger.cosmic_rays = True sub_table_by_filter = [ row for row in obj_table if row['filter_id'] == _filter ] log.info('Processing OBJECT files with filter: {}'.format(_filter)) obj_list_name = os.path.join( path, 'RED', "2OBJECT_{}x{}_{}".format(binning[0], binning[1], _filter)) obj_files = [row['filename'] for row in sub_table_by_filter] obj_files.sort() with open(obj_list_name, 'w') as obj_list_buffer: for obj_file in obj_files: path, fname = os.path.split(obj_file) prefix = sami_merger.get_prefix() output_obj_file = os.path.join(path, 'RED', prefix + fname) if os.path.exists(output_obj_file): log.warning('Skipping existing OBJECT file: {}'.format( output_obj_file)) continue log.info('Processing OBJECT file: {}'.format(obj_file)) d = sami_merger.get_joined_data(obj_file) h = sami_merger.get_header(obj_file) h = sami_merger.add_wcs(d, h) d, h, p = sami_merger.join_and_process(d, h) pyfits.writeto(output_obj_file, d, h) obj_list_buffer.write('\n'.format(obj_file)) log.info('All done.')
def __init__(self, data=None): if data: self.collection = ccdp.ImageFileCollection(filenames=data)
def _biascombine(self, binning=None, output="bias.fits"): """Finds and combines bias frames with the indicated binning :param binning: :param output: :return: """ if binning is None: raise InputError('Binning not set.') if self.debug: print("Trimming and combining bias frames with binning " f"{binning.replace(' ','x')} into {output}...") # First, refresh the ImageFileCollection self.icl.refresh() # Set up a progress bar, so we can see how the process is going... bias_files = self.icl.files_filtered(imagetyp='bias') prog_bar = tqdm(total=len(bias_files), unit='frame', unit_scale=False, colour='#808080') # Loop through files, for ccd, file_name in self.icl.ccds(ccdsum=binning, imagetyp='bias', bitpix=16, return_fname=True): # Fit the overscan section, subtract it, then trim the image ccd = _trim_oscan(ccd, self.biassec, self.trimsec) # Update the header ccd.header['HISTORY'] = PKG_NAME ccd.header['HISTORY'] = 'Trimmed bias saved: ' + _savetime() ccd.header['HISTORY'] = f'Original filename: {file_name}' # Save the result (suffix = 't'); delete the input file ccd.write(f'{self.path}/{file_name[:-5]}t{file_name[-5:]}', overwrite=True) os.remove(f'{self.path}/{file_name}') # Update the progress bar prog_bar.update(1) # Close the progress bar, end of loop prog_bar.close() # Collect the trimmed biases t_bias_cl = ccdp.ImageFileCollection( self.path, glob_include=f'{self.prefix}.*t.fits') # If we have a fresh list of trimmed biases to work with... if t_bias_cl.files: if self.debug: print("Doing median combine now...") comb_bias = ccdp.combine( [f'{self.path}/{fn}' for fn in t_bias_cl.files], method='median', sigma_clip=True, sigma_clip_low_thresh=5, sigma_clip_high_thresh=5, sigma_clip_func=np.ma.median, sigma_clip_dev_func=mad_std, mem_limit=4e9) # Add FITS keyword NCOMBINE and HISTORY comb_bias.header.set('ncombine', len(t_bias_cl.files), '# of input images in combination') comb_bias.header['HISTORY'] = 'Combined bias created: ' + \ _savetime() comb_bias.header['HISTORY'] = 'Median combined ' + \ f'{len(t_bias_cl.files)} files:' for f in t_bias_cl.files: comb_bias.header['HISTORY'] = f # Save the result; delete the input files comb_bias.write(f'{self.path}/{output}', overwrite=True) for f in t_bias_cl.files: os.remove(f'{self.path}/{f}')