def __init__(self, spectrum, atmosphere, telescope, disperser, fast_sim=True): """Class to simulate cross spectrum. Parameters ---------- spectrum: Spectrum Spectrum instance to load main properties before simulation. atmosphere: Atmosphere Atmosphere or AtmosphereGrid instance to make the atmospheric simulation. telescope: TelescopeTransmission Telescope transmission. disperser: Grating Disperser instance. fast_sim: bool, optional If True, do a fast simulation without integrating within the wavelength bins (default: True). Examples -------- >>> spectrum, telescope, disperser, target = SimulatorInit("./tests/data/reduc_20170530_134_spectrum.fits") >>> atmosphere = Atmosphere(airmass=1.2, pressure=800, temperature=10) >>> sim = SpectrumSimulation(spectrum, atmosphere, telescope, disperser, fast_sim=True) """ Spectrum.__init__(self) for k, v in list(spectrum.__dict__.items()): self.__dict__[k] = copy.copy(v) self.my_logger = set_logger(self.__class__.__name__) self.disperser = disperser self.telescope = telescope self.atmosphere = atmosphere self.fast_sim = fast_sim # save original pixel distances to zero order # self.disperser.grating_lambda_to_pixel(self.lambdas, x0=self.x0, order=1) # now reset data self.lambdas = None self.lambdas_order2 = None self.err = None self.model = lambda x: np.zeros_like(x) self.model_err = lambda x: np.zeros_like(x) lbdas_sed = self.target.wavelengths[0] sub = np.where((lbdas_sed > parameters.LAMBDA_MIN) & (lbdas_sed < parameters.LAMBDA_MAX)) self.lambdas_step = min(parameters.LAMBDA_STEP, np.min(lbdas_sed[sub]))
def SimulatorInit(filename, fast_load=False): """ SimulatorInit Main function to evaluate several spectra A grid of spectra will be produced for a given target, airmass and pressure """ my_logger = set_logger(__name__) my_logger.info('\n\tStart SIMULATOR initialisation') # Load data spectrum spectrum = Spectrum(filename, fast_load=fast_load) # TELESCOPE TRANSMISSION # ------------------------ telescope = TelescopeTransmission(spectrum.filter_label) # DISPERSER TRANSMISSION # ------------------------ if not isinstance(spectrum.disperser, str): disperser = spectrum.disperser else: disperser = Hologram(spectrum.disperser) # STAR SPECTRUM # ------------------------ if not isinstance(spectrum.target, str): target = spectrum.target else: target = Target(spectrum.target) return spectrum, telescope, disperser, target
def filter_data(file_names): # pragma: no cover from scipy.stats import median_absolute_deviation D = [] chi2 = [] dx = [] amplitude = [] regs = [] for name in file_names: # try: spectrum = Spectrum(name, fast_load=True) D.append(spectrum.header["D2CCD"]) dx.append(spectrum.header["PIXSHIFT"]) regs.append(np.log10(spectrum.header["PSF_REG"])) amplitude.append(np.sum(spectrum.data[300:])) if "CHI2_FIT" in spectrum.header: chi2.append(spectrum.header["CHI2_FIT"]) # except: # print(f"fail to open {name}") D = np.array(D) dx = np.array(dx) regs = np.array(regs) chi2 = np.array(chi2) k = np.arange(len(D)) plt.plot(k, amplitude) plt.show() plt.plot(k, D) # plt.plot(k, np.polyval(np.polyfit(k, reg, deg=1), k)) plt.axhline(np.median(D)) plt.axhline(np.median(D) + 3 * median_absolute_deviation(D)) plt.axhline(np.median(D) - 3 * median_absolute_deviation(D)) plt.grid() plt.title("D2CCD") plt.show() filter_indices = np.logical_and(D > np.median(D) - 3 * median_absolute_deviation(D), D < np.median(D) + 3 * median_absolute_deviation(D)) if len(chi2) > 0: filter_indices *= np.logical_and(chi2 > np.median(chi2) - 3 * median_absolute_deviation(chi2), chi2 < np.median(chi2) + 3 * median_absolute_deviation(chi2)) filter_indices *= np.logical_and(dx > np.median(dx) - 3 * median_absolute_deviation(dx), dx < np.median(dx) + 3 * median_absolute_deviation(dx)) filter_indices *= np.logical_and(regs > np.median(regs) - 3 * median_absolute_deviation(regs), regs < np.median(regs) + 3 * median_absolute_deviation(regs)) plt.plot(k, D) plt.title("D2CCD") plt.plot(k[filter_indices], D[filter_indices], "ko") plt.show() plt.plot(k, dx) plt.title("dx") plt.plot(k[filter_indices], dx[filter_indices], "ko") plt.show() plt.plot(k, regs) plt.title("regs") plt.plot(k[filter_indices], regs[filter_indices], "ko") plt.show() if len(chi2) > 0: plt.title("chi2") plt.plot(k, chi2) plt.plot(k[filter_indices], chi2[filter_indices], "ko") plt.show() return np.array(file_names)[filter_indices]
def test_fitchromaticpsf2d_test(sim_image="./tests/data/sim_20170530_134.fits", config="./config/ctio.ini"): image, lambdas_truth, amplitude_truth = load_test(sim_image, config=config) spectrum = Spectrum(sim_image.replace(".fits", "_spectrum.fits"), config=config) plot_residuals(spectrum, lambdas_truth, amplitude_truth) assert np.isclose(float(image.header['X0_T']), spectrum.target_pixcoords[0], atol=0.01) assert np.isclose(float(image.header['Y0_T']), spectrum.target_pixcoords[1], atol=0.01) assert np.isclose(float(image.header['ROTANGLE']), spectrum.rotation_angle, atol=180 / np.pi * 1 / parameters.CCD_IMSIZE) assert np.isclose(float(image.header['BKGD_LEV']), np.mean(spectrum.spectrogram_bgd), atol=2e-3) assert np.isclose(float(image.header['D2CCD_T']), spectrum.disperser.D, atol=0.05) print(spectrum.chromatic_psf.poly_params[spectrum.lambdas.size + 6:] - np.array(PSF_POLY_PARAMS_TRUTH)[3:]) print(np.std((amplitude_truth - spectrum.data) / spectrum.err))
def __init__(self, spectrum, atmosphere, telescope, disperser, with_background=True, fast_sim=True): """Class to simulate a spectrogram. Parameters ---------- spectrum: Spectrum Spectrum instance to load main properties before simulation. atmosphere: Atmosphere Atmosphere or AtmosphereGrid instance to make the atmospheric simulation. telescope: TelescopeTransmission Telescope transmission. disperser: Grating Disperser instance. """ Spectrum.__init__(self) for k, v in list(spectrum.__dict__.items()): self.__dict__[k] = copy.copy(v) self.disperser = disperser self.telescope = telescope self.atmosphere = atmosphere self.true_lambdas = None self.true_spectrum = None self.lambdas = None self.err = None self.model = lambda x, y: np.zeros((x.size, y.size)) self.psf = load_PSF(psf_type=parameters.PSF_TYPE) self.profile_params = None self.psf_cube = None self.psf_cube_order2 = None self.fix_psf_cube = False self.fast_sim = fast_sim self.with_background = with_background lbdas_sed = self.target.wavelengths[0] sub = np.where((lbdas_sed > parameters.LAMBDA_MIN) & (lbdas_sed < parameters.LAMBDA_MAX)) self.lambdas_step = min(parameters.LAMBDA_STEP, np.min(lbdas_sed[sub])) self.yy, self.xx = np.mgrid[:self.spectrogram_Ny, :self.spectrogram_Nx] self.pixels = np.asarray([self.xx, self.yy])
def __init__(self, spectrum, atmosphere, telescope, disperser, with_background=True, fast_sim=True, full_image=False, with_adr=True): """Class to simulate a spectrogram. Parameters ---------- spectrum: Spectrum Spectrum instance to load main properties before simulation. atmosphere: Atmosphere Atmosphere or AtmosphereGrid instance to make the atmospheric simulation. telescope: TelescopeTransmission Telescope transmission. disperser: Grating Disperser instance. with_background: bool, optional If True, add the background model to the simulated spectrogram (default: True). fast_sim: bool, optional If True, perform a fast simulation of the spectrum without integrated the spectrum in pixel bins (default: True). full_image: bool, optional If True, simulate the spectrogram on the full CCD size, otherwise only the cropped spectrogram (default: False). with_adr: bool, optional If True, simulate the spectrogram with ADR effect (default: True). Examples -------- >>> spectrum, telescope, disperser, target = SimulatorInit("./tests/data/reduc_20170530_134_spectrum.fits") >>> atmosphere = Atmosphere(airmass=1.2, pressure=800, temperature=10) >>> sim = SpectrogramModel(spectrum, atmosphere, telescope, disperser, with_background=True, fast_sim=True) """ Spectrum.__init__(self) for k, v in list(spectrum.__dict__.items()): self.__dict__[k] = copy.copy(v) self.disperser = disperser self.telescope = telescope self.atmosphere = atmosphere self.true_lambdas = None self.true_spectrum = None self.lambdas = None self.err = None self.model = lambda x, y: np.zeros((x.size, y.size)) self.psf = load_PSF(psf_type=parameters.PSF_TYPE) self.profile_params = None self.psf_cube = None self.psf_cube_order2 = None self.fix_psf_cube = False self.fix_atm_sim = False self.atmosphere_sim = None self.fast_sim = fast_sim self.with_background = with_background self.full_image = full_image self.with_adr = with_adr if self.full_image: self.Nx = parameters.CCD_IMSIZE self.Ny = self.spectrogram_Ny # too long if =parameters.CCD_IMSIZE self.r0 = self.x0[0] + 1j * self.spectrogram_y0 else: self.Nx = self.spectrogram_Nx self.Ny = self.spectrogram_Ny self.r0 = self.spectrogram_x0 + 1j * self.spectrogram_y0 lbdas_sed = self.target.wavelengths[0] sub = np.where((lbdas_sed > parameters.LAMBDA_MIN) & (lbdas_sed < parameters.LAMBDA_MAX)) self.lambdas_step = min(parameters.LAMBDA_STEP, np.min(lbdas_sed[sub])) self.yy, self.xx = np.mgrid[:self.Ny, :self.Nx] self.pixels = np.asarray([self.xx, self.yy])
def fullchain_run(sim_image="./tests/data/sim_20170530_134.fits"): # load test and make image simulation load_config("./config/ctio.ini") if not os.path.isfile(sim_image): make_image() image = Image(sim_image) lambdas_truth = np.fromstring(image.header['LBDAS_T'][1:-1], sep=' ') amplitude_truth = np.fromstring(image.header['AMPLIS_T'][1:-1], sep=' ', dtype=float) parameters.AMPLITUDE_TRUTH = np.copy(amplitude_truth) parameters.LAMBDA_TRUTH = np.copy(lambdas_truth) # extractor tag = os.path.basename(sim_image) tag = tag.replace('sim_', 'reduc_') logbook = LogBook(logbook="./ctiofulllogbook_jun2017_v5.csv") disperser_label, target, xpos, ypos = logbook.search_for_image(tag) parameters.PSF_POLY_ORDER = PSF_POLY_ORDER spectrum = Spectractor(sim_image, "./tests/data", guess=[xpos, ypos], target_label=target, disperser_label=disperser_label) # spectrum = Spectrum("./tests/data/sim_20170530_134_spectrum.fits") # spectrum = Spectrum("./tests/data/sim_20170530_176_spectrum.fits") # tests residuals = plot_residuals(spectrum, lambdas_truth, amplitude_truth) spectrum.my_logger.warning( f"\n\tQuantities to test:" f"\n\t\tspectrum.header['X0_T']={spectrum.header['X0_T']:.5g} vs {spectrum.x0[0]:.5g}" f"\n\t\tspectrum.header['Y0_T']={spectrum.header['Y0_T']:.5g} vs {spectrum.x0[1]:.5g}" f"\n\t\tspectrum.header['ROT_T']={spectrum.header['ROT_T']:.5g} " f"vs {spectrum.rotation_angle:.5g}" f"\n\t\tspectrum.header['BKGD_LEV']={spectrum.header['BKGD_LEV']:.5g} " f"vs {np.mean(spectrum.spectrogram_bgd):.5g}" f"\n\t\tspectrum.header['D2CCD_T']={spectrum.header['D2CCD_T']:.5g} " f"vs {spectrum.disperser.D:.5g}" f"\n\t\tspectrum.header['A2_FIT']={spectrum.header['A2_FIT']:.5g} vs {A2_T:.5g}" f"\n\t\tspectrum.header['CHI2_FIT']={spectrum.header['CHI2_FIT']:.4g}" f"\n\t\tspectrum.chromatic_psf.poly_params=" f"{spectrum.chromatic_psf.poly_params[spectrum.chromatic_psf.Nx + 2 * (PSF_POLY_ORDER + 1):-1]}" f" vs {PSF_POLY_PARAMS_TRUTH[2 * (PSF_POLY_ORDER + 1):-1]}" f"\n\t\tresiduals wrt truth: mean={np.mean(residuals[100:-100]):.5g}, " f"std={np.std(residuals[100:-100]):.5g}") assert np.isclose(float(spectrum.header['X0_T']), spectrum.x0[0], atol=0.2) assert np.isclose(float(spectrum.header['Y0_T']), spectrum.x0[1], atol=0.5) assert np.isclose(float(spectrum.header['ROT_T']), spectrum.rotation_angle, atol=180 / np.pi * 1 / parameters.CCD_IMSIZE) assert np.isclose(float(spectrum.header['BKGD_LEV']), np.mean(spectrum.spectrogram_bgd), rtol=1e-2) assert np.isclose(float(spectrum.header['D2CCD_T']), spectrum.disperser.D, atol=0.1) assert float(spectrum.header['CHI2_FIT']) < 0.65 assert np.all( np.isclose( spectrum.chromatic_psf.poly_params[spectrum.chromatic_psf.Nx + 2 * (PSF_POLY_ORDER + 1):-1], np.array(PSF_POLY_PARAMS_TRUTH)[2 * (PSF_POLY_ORDER + 1):-1], rtol=0.1, atol=0.1)) assert np.abs(np.mean(residuals[100:-100])) < 0.25 assert np.std(residuals[100:-100]) < 2 spectrum_file_name = "./tests/data/sim_20170530_134_spectrum.fits" # spectrum_file_name = "./tests/data/sim_20170530_176_spectrum.fits" assert os.path.isfile(spectrum_file_name) spectrum = Spectrum(spectrum_file_name) atmgrid_filename = sim_image.replace('sim', 'reduc').replace( '.fits', '_atmsim.fits') assert os.path.isfile(atmgrid_filename)
def convert_from_fits_to_txt(): prod_name = parameters.PROD_NAME prod_txt = parameters.PROD_TXT if os.path.exists(prod_txt) == False: os.makedirs(prod_txt) to_convert_list = [] Lsimutxt = glob.glob(prod_txt + "/sim*spectrum.txt") Lreductxt = glob.glob(prod_txt + "/reduc*spectrum.txt") Lsimufits = glob.glob(prod_name + "/sim*spectrum.fits") Lreducfits = glob.glob(prod_name + "/reduc*spectrum.fits") Ldefaut = glob.glob(prod_name + "/*20170530_201_spectrum*") + glob.glob( prod_name + "/*20170530_200_spectrum*") + glob.glob(prod_name + "/*20170530_205_spectrum*") Lsimutxt = [i for i in Lsimutxt if i not in Ldefaut] Lreductxt = [i for i in Lreductxt if i not in Ldefaut] Lsimufits = [i for i in Lsimufits if i not in Ldefaut] Lreducfits = [i for i in Lreducfits if i not in Ldefaut] if len(Lsimutxt) != len(Lsimufits) or len(Lreductxt) != len(Lreducfits): for file in Lsimufits: tag = file.split('/')[-1] fichier = os.path.join(prod_txt, tag.replace('fits', 'txt')) if fichier not in Lsimutxt: to_convert_list.append(file) for file in Lreducfits: tag = file.split('/')[-1] fichier = os.path.join(prod_txt, tag.replace('fits', 'txt')) if fichier not in Lreductxt: to_convert_list.append(file) for i in range(len(to_convert_list)): startest = to_convert_list[i] s = Spectrum(startest) airmass = s.header["AIRMASS"] TARGETX = s.header["TARGETX"] TARGETY = s.header["TARGETY"] D2CCD = s.header["D2CCD"] PIXSHIFT = s.header["PIXSHIFT"] ROTANGLE = s.header["ROTANGLE"] psf_transverse = s.chromatic_psf.table['fwhm'] PARANGLE = s.header["PARANGLE"] PSF_REG = s.header['PSF_REG'] x0 = [TARGETX, TARGETY] print(to_convert_list[i][:len(to_convert_list[i]) - 5]) disperser = s.disperser distance = disperser.grating_lambda_to_pixel(s.lambdas, x0=x0, order=1) distance += adr_calib(s.lambdas, s.adr_params, parameterss.OBS_LATITUDE, lambda_ref=s.lambda_ref) distance -= adr_calib(s.lambdas / 2, s.adr_params, parameterss.OBS_LATITUDE, lambda_ref=s.lambda_ref) lambdas_order2 = disperser.grating_pixel_to_lambda(distance, x0=x0, order=2) disperseur = s.disperser_label star = s.header['TARGET'] lambda_obs = s.lambdas intensite_obs = s.data intensite_err = s.err cov = s.cov_matrix if s.target.wavelengths == []: print('CALSPEC error') else: lambda_reel = s.target.wavelengths[0] intensite_reel = s.target.spectra[0] tag = to_convert_list[i].split('/')[-1] fichier = open( os.path.join(prod_txt, tag.replace('fits', 'txt')), 'w') fichier.write('#' + '\t' + star + '\t' + disperseur + '\t' + str(airmass) + '\t' + str(TARGETX) + '\t' + str(TARGETY) + '\t' + str(D2CCD) + '\t' + str(PIXSHIFT) + '\t' + str(ROTANGLE) + '\t' + str(PARANGLE) + '\t' + str(PSF_REG) + '\n') for j in range(len(lambda_reel)): if len(lambda_obs) > j: if len(psf_transverse) > j: fichier.write( str(lambda_reel[j]) + '\t' + str(intensite_reel[j]) + '\t' + str(lambda_obs[j]) + '\t' + str(intensite_obs[j]) + '\t' + str(intensite_err[j]) + '\t' + str(lambdas_order2[j]) + '\t' + str(psf_transverse[j]) + '\n') else: fichier.write( str(lambda_reel[j]) + '\t' + str(intensite_reel[j]) + '\t' + str(lambda_obs[j]) + '\t' + str(intensite_obs[j]) + '\t' + str(intensite_err[j]) + '\t' + str(lambdas_order2[j]) + '\n') else: fichier.write( str(lambda_reel[j]) + '\t' + str(intensite_reel[j]) + '\n') fichier.close() np.save(os.path.join(prod_txt, tag.replace('fits', 'npy')), cov) return False, Lsimutxt, Lreductxt else: print('already done') return True, Lsimutxt, Lreductxt
def run(self, exp, xpos, ypos, target, outputRoot=None, plotting=True): # run option kwargs in the original code, seems to ~always be True atmospheric_lines = True self.log.info('Starting SPECTRACTOR') # TODO: rename _makePath _makeOutputPath if outputRoot is not None: # TODO: remove post Gen3 transition self._makePath(outputRoot, plotting=plotting) # early in case this fails, as processing is slow # Upstream loads config file here # TODO: passing exact centroids seems to be causing a serious # and non-obvious problem! # this needs fixing for several reasons, mostly because if we have a # known good centroid then we want to skip the refitting entirely xpos = int(np.round(xpos)) ypos = int(np.round(ypos)) filter_label, disperser = self._getFilterAndDisperserFromExp(exp) image = self.spectractorImageFromLsstExposure(exp, target_label=target, disperser_label=disperser, filter_label=filter_label) if self.TRANSPOSE: xpos, ypos = self.transposeCentroid(xpos, ypos, image) image.target_guess = (xpos, ypos) if parameters.DEBUG: image.plot_image(scale='log10', target_pixcoords=image.target_guess) self.log.info(f"Pixel value at centroid = {image.data[int(ypos), int(xpos)]}") # XXX this needs removing or at least dealing with to not always # just run! ASAP XXX # if disperser == 'ronchi170lpmm': # TODO: add something more robust as to whether to flip! # image, xpos, ypos = self.flipImageLeftRight(image, xpos, ypos) # self.displayImage(image, centroid=(xpos, ypos)) # Use fast mode if parameters.CCD_REBIN > 1: self.log.info(f'Rebinning image with rebin of {parameters.CCD_REBIN}') # TODO: Fix bug here where the passed parameter isn't used! image.rebin() if parameters.DEBUG: image.plot_image(scale='symlog', target_pixcoords=image.target_guess) # image turning and target finding - use LSST code instead? # and if not, at least test how the rotation code compares # this part of Spectractor is certainly slow at the very least if True: # TODO: change this to be an option, at least for testing vs LSST self.log.info('Search for the target in the image...') _ = find_target(image, image.target_guess) # sets the image.target_pixcoords turn_image(image) # creates the rotated data, and sets the image.target_pixcoords_rotated # Rotate the image: several methods # Find the exact target position in the rotated image: # several methods - but how are these controlled? MFL self.log.info('Search for the target in the rotated image...') _ = find_target(image, image.target_guess, rotated=True) else: # code path for if the image is pre-rotated by LSST code raise NotImplementedError # Create Spectrum object spectrum = Spectrum(image=image, order=parameters.SPEC_ORDER) # XXX new in DM-33589 check SPEC_ORDER self.setAdrParameters(spectrum, exp) # Subtract background and bad pixels w_psf1d, bgd_model_func = extract_spectrum_from_image(image, spectrum, signal_width=parameters.PIXWIDTH_SIGNAL, ws=(parameters.PIXDIST_BACKGROUND, parameters.PIXDIST_BACKGROUND + parameters.PIXWIDTH_BACKGROUND), right_edge=parameters.CCD_IMSIZE) spectrum.atmospheric_lines = atmospheric_lines # PSF2D deconvolution if parameters.SPECTRACTOR_DECONVOLUTION_PSF2D: run_spectrogram_deconvolution_psf2d(spectrum, bgd_model_func=bgd_model_func) # Calibrate the spectrum with_adr = True if parameters.OBS_OBJECT_TYPE != "STAR": # XXX Check what this is set to, and how # likely need to be passed through with_adr = False calibrate_spectrum(spectrum, with_adr=with_adr) # not necessarily set during fit but required to be present for astropy # fits writing to work (required to be in keeping with upstream) spectrum.data_order2 = np.zeros_like(spectrum.lambdas_order2) spectrum.err_order2 = np.zeros_like(spectrum.lambdas_order2) # Full forward model extraction: # adds transverse ADR and order 2 subtraction w = None if parameters.SPECTRACTOR_DECONVOLUTION_FFM: w = FullForwardModelFitWorkspace(spectrum, verbose=parameters.VERBOSE, plot=True, live_fit=False, amplitude_priors_method="spectrum") spectrum = run_ffm_minimisation(w, method="newton", niter=2) # Save the spectrum self._ensureFitsHeader(spectrum) # SIMPLE is missing by default # Plot the spectrum parameters.DISPLAY = True if parameters.VERBOSE and parameters.DISPLAY: spectrum.plot_spectrum(xlim=None) spectrum.chromatic_psf.table['lambdas'] = spectrum.lambdas result = Spectraction() result.spectrum = spectrum result.image = image result.w = w # XXX technically this should be a pipeBase.Struct I think # change it if it matters return result
def _readFile(self, path, pytype=None): return Spectrum(path)