def test_atmosphere(): a = Atmosphere(airmass=1.2, pressure=800, temperature=5) transmission = a.simulate(ozone=400, pwv=5, aerosols=0.05) assert transmission is not None assert a.transmission(500) > 0 assert a.ozone == 400 assert a.pwv == 5 assert a.aerosols == 0.05 a.plot_transmission() a = AtmosphereGrid(image_filename='tests/data/reduc_20170605_028.fits', pwv_grid=[5, 5, 1], ozone_grid=[400, 400, 1], aerosol_grid=[0.0, 0.1, 2]) atmospheric_grid = a.compute() assert np.sum(atmospheric_grid) > 0 assert np.all(np.isclose(a.atmgrid[0, a.index_atm_data:], parameters.LAMBDAS)) assert not np.any(np.isclose(a.atmgrid[1, a.index_atm_data:], np.zeros_like(parameters.LAMBDAS), rtol=1e-6)) assert a.atmgrid.shape == (3, a.index_atm_data + len(parameters.LAMBDAS)) a.save_file(a.image_filename.replace('.fits', '_atmsim.fits')) assert os.path.isfile('tests/data/reduc_20170605_028_atmsim.fits') a.load_file(a.image_filename.replace('.fits', '_atmsim.fits')) assert np.all(a.AER_Points == np.array([0., 0.1])) assert np.all(a.PWV_Points == np.array([5.])) assert np.all(a.OZ_Points == np.array([400.])) a.plot_transmission() a.plot_transmission_image() a = AtmosphereGrid(filename='tests/data/reduc_20170530_134_atmsim.fits') lambdas = np.arange(200, 1200) transmission = a.simulate(ozone=400, pwv=5, aerosols=0.05) assert np.max(transmission(lambdas)) < 1 and np.min(transmission(lambdas)) >= 0
def test_simulator(): file_names = ['tests/data/reduc_20170530_134_spectrum.fits'] parameters.VERBOSE = True parameters.DEBUG = True load_config('config/ctio.ini') output_directory = './outputs/' for file_name in file_names: tag = file_name.split('/')[-1] spectrum_simulation = SpectrumSimulator(file_name, output_directory, pwv=5, ozone=300, aerosols=0.03, A1=1, A2=10, reso=2, D=56, shift=-1) assert np.sum(spectrum_simulation.data) > 0 assert np.sum(spectrum_simulation.data) < 1e-10 assert np.sum(spectrum_simulation.data_order2) < 1e-10 # spectrogram_simulation = SpectrogramSimulator(file_name, output_directory, pwv=5, ozone=300, aerosols=0.03, # A1=1, A2=1, D=56, shift_x=-1, shift_y=1, angle=-1) # assert np.sum(spectrogram_simulation.data) > 20 # psf_poly_params = spectrogram_simulation.chromatic_psf.from_table_to_poly_params() image_simulation = ImageSim(file_name.replace('_spectrum.fits', '.fits'), file_name, output_directory, A2=1, psf_poly_params=None, with_stars=True) SpectrumSimulatorSimGrid(file_name, output_directory, pwv_grid=[0, 10, 2], ozone_grid=[200, 400, 2], aerosol_grid=[0, 0.1, 2]) atmgrid = AtmosphereGrid(file_name, file_name.replace('spectrum', 'atmsim')) atm = Atmosphere(atmgrid.airmass, atmgrid.pressure, atmgrid.temperature) assert os.path.isfile(output_directory + tag.replace('_spectrum.fits', '_atmsim.fits')) is True assert image_simulation.data is not None # assert spectrum_simulation.data is not None # assert spectrogram_simulation.data is not None assert os.path.isfile(output_directory + tag.replace('reduc', 'sim').replace('_spectrum.fits', '.fits')) assert atm.transmission is not None
def fit_spectrum(self): nsamples = 300 def spec_calspec(): spec = open(self.prod_reduc[0], 'r') lambdas = [] data = [] for line in spec: Line = line.split() if Line[0] != '#': lambdas.append(float(Line[0])) data.append(float(Line[1])) lambdas_calspec = np.array(lambdas) data_calspec_org = np.array(data) data_calspec = data_calspec_org fluxlum_Binreel = np.zeros(len(self.Bin) - 1) interpolation_reel = sp.interpolate.interp1d(lambdas_calspec, data_calspec, kind="linear", bounds_error=False, fill_value=(0, 0)) for v in range(len(self.Bin) - 1): "On rempli les tableaux par bin de longueur d'onde" X = np.linspace(self.Bin[v], self.Bin[v + 1], int(self.binwidths * 100)) Y = interpolation_reel(X) fluxlum_Binreel[v] = integrate.simps(Y, X, dx=1) / self.binwidths data_calspec = fluxlum_Binreel return data_calspec def Ordonnee(): data_calspec = spec_calspec() if self.disperseur != parameters.DISPERSER_REF: disp = np.loadtxt(parameters.DISPERSER_EXTRACTION) else: disp = np.loadtxt(parameters.DISPERSER_REF_BANC) Tctio = np.loadtxt(parameters.THROUGHPUT_REDUC) Tctio_data = sp.interpolate.interp1d(Tctio.T[0], Tctio.T[1], bounds_error=False, fill_value="extrapolate")(self.new_lambda) disp_data = sp.interpolate.interp1d(disp.T[0], disp.T[1], bounds_error=False, fill_value="extrapolate")(self.new_lambda) return data_calspec * Tctio_data * disp_data OZONE = np.zeros(len(self.names)) AEROSOLS = np.zeros(len(self.names)) EAU = np.zeros(len(self.names)) ERR_OZONE = np.zeros(len(self.names)) ERR_AEROSOLS = np.zeros(len(self.names)) ERR_EAU = np.zeros(len(self.names)) for spec in range(len(self.names)): atm = AtmosphereGrid( filename=(os.path.join(parameters.PROD_DIRECTORY, parameters.PROD_NAME) + '/' + self.names[spec].split('/')[-1]).replace('sim', 'reduc').replace( 'spectrum.txt', 'atmsim.fits')) self.atm = atm ORDO = Ordonnee() def Atm(atm, ozone, eau, aerosols): nb_bin = len(self.data) Atmo = np.zeros(nb_bin) Lambdas = np.arange(self.Bin[0], self.Bin[-1], 0.2) Atm = atm.simulate(ozone, eau, aerosols)(Lambdas) for i in range(len(self.new_lambda)): Atmo[i] = np.mean(Atm[i * int(self.binwidths / 0.2):(i + 1) * int(self.binwidths / 0.2)]) M = np.diagflat(Atmo) return M def log_likelihood(params_fit): nb_bin = len(self.data) ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1] M = Atm(self.atm,ozone, eau, aerosols) D = self.data[:,spec] - self.order2[:,spec] mat = D - M @ ORDO chi2 = mat @ self.INVCOV[spec] @ mat n = np.random.randint(0, 100) if n > 97: print(chi2 / (nb_bin)) print(ozone, eau, aerosols) return -0.5 * chi2 def log_prior(params_fit): ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1] if 100 < ozone < 700 and 0 < eau < 10 and 0 < aerosols < 0.1: return 0 else: return -np.inf def log_probability(params_fit): lp = log_prior(params_fit) if not np.isfinite(lp): return -np.inf return lp + log_likelihood(params_fit) p_ozone = 300 p_eau = 5 p_aerosols = 0.03 p0 = np.array([p_ozone, p_eau, p_aerosols]) walker = 10 init_ozone = p0[0] + p0[0] / 5 * np.random.randn(walker) init_eau = p0[1] + p0[1] / 5 * np.random.randn(walker) init_aerosols = p0[2] + p0[2] / 5 * np.random.randn(walker) p0 = np.array([[init_ozone[i], init_eau[i], init_aerosols[i]] for i in range(walker)]) nwalkers, ndim = p0.shape sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability, threads=multiprocessing.cpu_count()) sampler.run_mcmc(p0, nsamples, progress=True) flat_samples = sampler.get_chain(discard=100, thin=1, flat=True) ozone, d_ozone = np.mean(flat_samples[:, -3]), np.std(flat_samples[:, -3]) eau, d_eau = np.mean(flat_samples[:, -2]), np.std(flat_samples[:, -2]) aerosols, d_aerosols = np.mean(flat_samples[:, -1]), np.std(flat_samples[:, -1]) print(ozone, d_ozone) print(eau, d_eau) print(aerosols, d_aerosols) OZONE[spec] = ozone AEROSOLS[spec] = aerosols EAU[spec] = eau ERR_OZONE[spec] = d_ozone ERR_EAU[spec] = d_eau ERR_AEROSOLS[spec] = d_aerosols if self.sim: chemin = parameters.OUTPUTS_FITSPECTRUM_SIM else: chemin = parameters.OUTPUTS_FITSPECTRUM_REDUC NUM = np.zeros(len(self.names)) for i in range(len(self.names)): NUM[i] = self.names[i][-16:-13] fig = plt.figure(figsize=[10, 10]) ax2 = fig.add_subplot(111) ax2.get_xaxis().set_tick_params(labelsize=12) ax2.get_yaxis().set_tick_params(labelsize=12) plt.grid(True) plt.title("Fit spectrum with bins", fontsize=22) plt.xlabel('Spectrum index', fontsize=20) plt.ylabel("Ozone [db]", fontsize=20) plt.scatter(NUM, OZONE, c='blue') plt.errorbar(NUM, OZONE, xerr=None, yerr=ERR_OZONE, fmt='none', capsize=1, ecolor='blue', zorder=2, elinewidth=2) fig.tight_layout() if os.path.exists(chemin) == False: os.makedirs(chemin) plt.savefig(chemin + 'Ozone_fit, ' + self.disperseur + ', version_' + parameters.PROD_NUM + '.pdf') fig = plt.figure(figsize=[10, 10]) ax2 = fig.add_subplot(111) ax2.get_xaxis().set_tick_params(labelsize=12) ax2.get_yaxis().set_tick_params(labelsize=12) plt.grid(True) plt.title("Fit spectrum with bins", fontsize=22) plt.xlabel('Spectrum index', fontsize=20) plt.ylabel("Aerosols [VAOD]", fontsize=20) plt.scatter(NUM, AEROSOLS, c='blue') plt.errorbar(NUM, AEROSOLS, xerr=None, yerr=ERR_AEROSOLS, fmt='none', capsize=1, ecolor='blue', zorder=2, elinewidth=2) fig.tight_layout() plt.savefig(chemin + 'Aerosols_fit, ' + self.disperseur + ', version_' + parameters.PROD_NUM + '.pdf') fig = plt.figure(figsize=[10, 10]) ax2 = fig.add_subplot(111) ax2.get_xaxis().set_tick_params(labelsize=12) ax2.get_yaxis().set_tick_params(labelsize=12) plt.grid(True) plt.title("Fit spectrum with bins", fontsize=22) plt.xlabel('Spectrum index', fontsize=20) plt.ylabel("PWV [mm]", fontsize=20) plt.scatter(NUM, EAU, c='blue') plt.errorbar(NUM, EAU, xerr=None, yerr=ERR_EAU, fmt='none', capsize=1, ecolor='blue', zorder=2, elinewidth=2) fig.tight_layout() plt.savefig(chemin + 'Eau_fit, ' + self.disperseur + ', version_' + parameters.PROD_NUM + '.pdf') plt.show()
def megafit_emcee(self): nsamples = 300 atm = [] for j in range(len(self.names)): prod_name = os.path.join(parameters.PROD_DIRECTORY, parameters.PROD_NAME) atmgrid = AtmosphereGrid( filename=(prod_name + '/' + self.names[j].split('/')[-1]).replace('sim', 'reduc').replace( 'spectrum.txt', 'atmsim.fits')) atm.append(atmgrid) def matrice_data(): y = self.data - self.order2 nb_spectre = len(self.names) nb_bin = len(self.data) D = np.zeros(nb_bin * nb_spectre) for j in range(nb_spectre): D[j * nb_bin: (j + 1) * nb_bin] = y[:, j] return D def Atm(atm, ozone, eau, aerosols): nb_spectre = len(self.names) nb_bin = len(self.data) M = np.zeros((nb_spectre, nb_bin, nb_bin)) M_p = np.zeros((nb_spectre * nb_bin, nb_bin)) for j in range(nb_spectre): Atmo = np.zeros(len(self.new_lambda)) Lambdas = np.arange(self.Bin[0],self.Bin[-1],0.2) Atm = atm[j].simulate(ozone, eau, aerosols)(Lambdas) for i in range(len(self.new_lambda)): Atmo[i] = np.mean(Atm[i*int(self.binwidths/0.2):(i+1)*int(self.binwidths/0.2)]) a = np.diagflat(Atmo) M[j, :, :] = a M_p[nb_bin * j:nb_bin * (j+1),:] = a return M, M_p def log_likelihood(params_fit, atm): nb_spectre = len(self.names) nb_bin = len(self.data) ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1] D = matrice_data() M, M_p = Atm(atm, ozone, eau, aerosols) prod = np.zeros((nb_bin, nb_spectre * nb_bin)) for spec in range(nb_spectre): prod[:,spec * nb_bin : (spec+1) * nb_bin] = M[spec] @ self.INVCOV[spec] COV = inv(prod @ M_p) A = COV @ prod @ D chi2 = 0 for spec in range(nb_spectre): mat = D[spec * nb_bin : (spec+1) * nb_bin] - M[spec] @ A chi2 += mat @ self.INVCOV[spec] @ mat n = np.random.randint(0, 100) if n > 97: print(chi2 / (nb_spectre * nb_bin)) print(ozone, eau, aerosols) return -0.5 * chi2 def log_prior(params_fit): ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1] if 100 < ozone < 700 and 0 < eau < 10 and 0 < aerosols < 0.1: return 0 else: return -np.inf def log_probability(params_fit, atm): lp = log_prior(params_fit) if not np.isfinite(lp): return -np.inf return lp + log_likelihood(params_fit, atm) if self.sim: filename = "sps/" + self.disperseur + "_"+ "sim_"+ parameters.PROD_NUM + "_emcee.h5" else: filename = "sps/" + self.disperseur + "_" + "reduc_" + parameters.PROD_NUM + "_emcee.h5" p_ozone = 300 p_eau = 5 p_aerosols = 0.03 p0 = np.array([p_ozone, p_eau, p_aerosols]) walker = 10 init_ozone = p0[0] + p0[0] / 5 * np.random.randn(walker) init_eau = p0[1] + p0[1] / 5 * np.random.randn(walker) init_aerosols = p0[2] + p0[2] / 5 * np.random.randn(walker) p0 = np.array([[init_ozone[i], init_eau[i], init_aerosols[i]] for i in range(walker)]) nwalkers, ndim = p0.shape backend = emcee.backends.HDFBackend(filename) try: pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability, args=(atm,), pool=pool, backend=backend) if backend.iteration > 0: p0 = backend.get_last_sample() if nsamples - backend.iteration > 0: sampler.run_mcmc(p0, nsteps=max(0, nsamples - backend.iteration), progress=True) pool.close() except ValueError: sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability, args=(atm,), threads=multiprocessing.cpu_count(), backend=backend) if backend.iteration > 0: p0 = sampler.get_last_sample() for _ in sampler.sample(p0, iterations=max(0, nsamples - backend.iteration), progress=True, store=True): continue flat_samples = sampler.get_chain(discard=100, thin=1, flat=True) ozone, d_ozone = np.mean(flat_samples[:, -3]), np.std(flat_samples[:, -3]) eau, d_eau = np.mean(flat_samples[:, -2]), np.std(flat_samples[:, -2]) aerosols, d_aerosols = np.mean(flat_samples[:, -1]), np.std(flat_samples[:, -1]) print(ozone, d_ozone) print(eau, d_eau) print(aerosols, d_aerosols) self.params_atmo = np.array([ozone, eau, aerosols]) self.err_params_atmo = np.array([d_ozone, d_eau, d_aerosols]) nb_spectre = len(self.names) nb_bin = len(self.data) M, M_p = Atm(atm, ozone, eau, aerosols) prod = np.zeros((nb_bin, nb_spectre * nb_bin)) chi2 = 0 for spec in range(nb_spectre): prod[:, spec * nb_bin: (spec + 1) * nb_bin] = M[spec] @ self.INVCOV[spec] COV = inv(prod @ M_p) D = matrice_data() Tinst = COV @ prod @ D Tinst_err = np.array([np.sqrt(COV[i,i]) for i in range(len(Tinst))]) if self.disperseur == 'HoloAmAg' and self.sim == False: a, b = np.argmin(abs(self.new_lambda - 537.5)), np.argmin(abs(self.new_lambda - 542.5)) Tinst_err[a], Tinst_err[b] = Tinst_err[a-1], Tinst_err[b+1] for spec in range(nb_spectre): mat = D[spec * nb_bin: (spec + 1) * nb_bin] - M[spec] @ Tinst chi2 += mat @ self.INVCOV[spec] @ mat print(chi2 / (nb_spectre * nb_bin)) err = np.zeros_like(D) for j in range(len(self.names)): for i in range(len(self.data)): if self.disperseur == 'HoloAmAg' and self.sim == False: if self.new_lambda[i] == 537.5 or self.new_lambda[i] == 542.5: err[j * len(self.data) + i] = 1 else: err[j * len(self.data) + i] = np.sqrt(self.cov[j][i, i]) else: err[j * len(self.data) + i] = np.sqrt(self.cov[j][i, i]) model = M_p @ Tinst Err = (D - model) / err if parameters.plot_residuals: self.Plot_residuals(model, D, Err, COV) return Tinst, Tinst_err
parameters.DEBUG = True parameters.VERBOSE = True file_names = args.input load_config(args.config) logbook = LogBook(logbook=args.logbook) for file_name in file_names: tag = file_name.split('/')[-1] disperser_label, target, xpos, ypos = logbook.search_for_image(tag) if target is None or xpos is None or ypos is None: continue spectrum_file_name = args.output_directory + '/' + tag.replace( '.fits', '_spectrum.fits') atmgrid = AtmosphereGrid(file_name) SpectrumSimulatorSimGrid(spectrum_file_name, args.output_directory) image = ImageSim(file_name, spectrum_file_name, args.output_directory, A1=1, A2=1, pwv=5, ozone=300, aerosols=0.03, psf_poly_params=None, with_stars=True) sim_file_name = args.output_directory + tag.replace('reduc_', 'sim_') Spectractor(sim_file_name, args.output_directory, target, [xpos, ypos], disperser_label, args.config)
def archi_mega_fit(self): nsamples = 300 OZONE = np.zeros(len(self.names)) AEROSOLS = np.zeros(len(self.names)) EAU = np.zeros(len(self.names)) ERR_OZONE = np.zeros(len(self.names)) ERR_AEROSOLS = np.zeros(len(self.names)) ERR_EAU = np.zeros(len(self.names)) for spec in range(len(self.names)): atm = AtmosphereGrid(filename=( os.path.join(parameters.PROD_DIRECTORY, parameters.PROD_NAME) + '/' + self.names[spec].split('/')[-1] ).replace('sim', 'reduc').replace('spectrum.txt', 'atmsim.fits')) self.atm = atm ORDO = self.Ordonnee() def Atm(atm, ozone, eau, aerosols): nb_bin = len(self.data_mag) Atmo = np.zeros(nb_bin) Lambdas = np.arange(self.Bin[0], self.Bin[-1], 0.2) Atm = atm.simulate(ozone, eau, aerosols)(Lambdas) for i in range(len(self.new_lambda)): Atmo[i] = np.mean( Atm[i * int(self.binwidths / 0.2):(i + 1) * int(self.binwidths / 0.2)]) M = np.diagflat(Atmo) return M def log_likelihood(params_fit): nb_bin = len(self.data_mag) ozone, eau, aerosols = params_fit[-3], params_fit[ -2], params_fit[-1] M = Atm(self.atm, ozone, eau, aerosols) D = np.exp(self.data_mag[:, spec]) - 0 * self.order2[:, spec] mat = D - M @ ORDO chi2 = mat @ self.INVCOV[spec] @ mat n = np.random.randint(0, 100) if n > 97: print(chi2 / (nb_bin)) print(ozone, eau, aerosols) return -0.5 * chi2 def log_prior(params_fit): ozone, eau, aerosols = params_fit[-3], params_fit[ -2], params_fit[-1] if 100 < ozone < 700 and 0 < eau < 10 and 0 < aerosols < 0.1: # and 0<A2<5: return 0 else: return -np.inf def log_probability(params_fit): lp = log_prior(params_fit) if not np.isfinite(lp): return -np.inf return lp + log_likelihood(params_fit) p_ozone = 300 p_eau = 5 p_aerosols = 0.03 p0 = np.array([p_ozone, p_eau, p_aerosols]) walker = 10 init_ozone = p0[0] + p0[0] / 5 * np.random.randn(walker) init_eau = p0[1] + p0[1] / 5 * np.random.randn(walker) init_aerosols = p0[2] + p0[2] / 5 * np.random.randn(walker) p0 = np.array([[init_ozone[i], init_eau[i], init_aerosols[i]] for i in range(walker)]) nwalkers, ndim = p0.shape sampler = emcee.EnsembleSampler( nwalkers, ndim, log_probability, threads=multiprocessing.cpu_count()) sampler.run_mcmc(p0, nsamples, progress=True) flat_samples = sampler.get_chain(discard=100, thin=1, flat=True) ozone, d_ozone = np.mean(flat_samples[:, -3]), np.std( flat_samples[:, -3]) eau, d_eau = np.mean(flat_samples[:, -2]), np.std(flat_samples[:, -2]) aerosols, d_aerosols = np.mean(flat_samples[:, -1]), np.std( flat_samples[:, -1]) print(ozone, d_ozone) print(eau, d_eau) print(aerosols, d_aerosols) OZONE[spec] = ozone AEROSOLS[spec] = aerosols EAU[spec] = eau ERR_OZONE[spec] = d_ozone ERR_EAU[spec] = d_eau ERR_AEROSOLS[spec] = d_aerosols NUM = np.zeros(len(self.names)) for i in range(len(self.names)): NUM[i] = self.names[i][-16:-13] fig = plt.figure(figsize=[10, 10]) ax2 = fig.add_subplot(111) ax2.get_xaxis().set_tick_params(labelsize=12) ax2.get_yaxis().set_tick_params(labelsize=12) plt.grid(True) plt.title("Fit spectrum with bins", fontsize=22) plt.xlabel('Spectrum index', fontsize=20) plt.ylabel("Ozone [db]", fontsize=20) plt.scatter(NUM, OZONE, c='blue') plt.errorbar(NUM, OZONE, xerr=None, yerr=ERR_OZONE, fmt='none', capsize=1, ecolor='blue', zorder=2, elinewidth=2) fig.tight_layout() plt.savefig('Ozone_fit.pdf') fig = plt.figure(figsize=[10, 10]) ax2 = fig.add_subplot(111) ax2.get_xaxis().set_tick_params(labelsize=12) ax2.get_yaxis().set_tick_params(labelsize=12) plt.grid(True) plt.title("Fit spectrum with bins", fontsize=22) plt.xlabel('Spectrum index', fontsize=20) plt.ylabel("Aerosols [VAOD]", fontsize=20) plt.scatter(NUM, AEROSOLS, c='blue') plt.errorbar(NUM, AEROSOLS, xerr=None, yerr=ERR_AEROSOLS, fmt='none', capsize=1, ecolor='blue', zorder=2, elinewidth=2) fig.tight_layout() plt.savefig('Aerosols_fit.pdf') fig = plt.figure(figsize=[10, 10]) ax2 = fig.add_subplot(111) ax2.get_xaxis().set_tick_params(labelsize=12) ax2.get_yaxis().set_tick_params(labelsize=12) plt.grid(True) plt.title("Fit spectrum with bins", fontsize=22) plt.xlabel('Spectrum index', fontsize=20) plt.ylabel("PWV [mm]", fontsize=20) plt.scatter(NUM, EAU, c='blue') plt.errorbar(NUM, EAU, xerr=None, yerr=ERR_EAU, fmt='none', capsize=1, ecolor='blue', zorder=2, elinewidth=2) fig.tight_layout() plt.savefig('Eau_fit.pdf') plt.show()
def megafit_emcee(self): nsamples = 300 atm = [] for j in range(len(self.names)): prod_name = os.path.join(parameters.PROD_DIRECTORY, parameters.PROD_NAME) atmgrid = AtmosphereGrid(filename=( prod_name + '/' + self.names[j].split('/')[-1] ).replace('sim', 'reduc').replace('spectrum.txt', 'atmsim.fits')) atm.append(atmgrid) #D = self.matrice_data() """ def f_tinst_atm(Tinst, ozone, eau, aerosols, atm): model = np.zeros((len(self.data_mag), len(self.names))) for j in range(len(self.names)): a = atm[j].simulate(ozone, eau, aerosols) model[:, j] = Tinst * a(self.new_lambda) return model """ def Atm(atm, ozone, eau, aerosols): nb_spectre = len(self.names) nb_bin = len(self.data_mag) M = np.zeros((nb_spectre, nb_bin, nb_bin)) M_p = np.zeros((nb_spectre * nb_bin, nb_bin)) for j in range(nb_spectre): Atmo = np.zeros(len(self.new_lambda)) Lambdas = np.arange(self.Bin[0], self.Bin[-1], 0.2) Atm = atm[j].simulate(ozone, eau, aerosols)(Lambdas) for i in range(len(self.new_lambda)): #step = int(self.binwidths * 10) #X = np.linspace(self.Bin[i], self.Bin[i + 1] + step, step) Atmo[i] = np.mean( Atm[i * int(self.binwidths / 0.2):(i + 1) * int(self.binwidths / 0.2)]) a = np.diagflat(Atmo) M[j, :, :] = a M_p[nb_bin * j:nb_bin * (j + 1), :] = a return M, M_p def log_likelihood(params_fit, atm): nb_spectre = len(self.names) nb_bin = len(self.data_mag) ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[ -1] A2 = 0 D = self.matrice_data(A2) M, M_p = Atm(atm, ozone, eau, aerosols) #A = np.zeros(nb_bin) #COV = np.zeros((nb_bin, nb_bin)) prod = np.zeros((nb_bin, nb_spectre * nb_bin)) for spec in range(nb_spectre): prod[:, spec * nb_bin:(spec + 1) * nb_bin] = M[spec] @ self.INVCOV[spec] COV = inv(prod @ M_p) A = COV @ prod @ D chi2 = 0 for spec in range(nb_spectre): mat = D[spec * nb_bin:(spec + 1) * nb_bin] - M[spec] @ A chi2 += mat @ self.INVCOV[spec] @ mat n = np.random.randint(0, 100) if n > 97: print(chi2 / (nb_spectre * nb_bin)) print(A2, ozone, eau, aerosols) return -0.5 * chi2 def log_prior(params_fit): ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[ -1] if 100 < ozone < 700 and 0 < eau < 10 and 0 < aerosols < 0.1: # and 0<A2<5: return 0 else: return -np.inf def log_probability(params_fit, atm): lp = log_prior(params_fit) if not np.isfinite(lp): return -np.inf return lp + log_likelihood(params_fit, atm) if self.sim: filename = "sps/" + self.disperseur + "_" + "sim_new_" + parameters.PROD_NUM + "_emcee.h5" #+ "sim_new_" else: filename = "sps/" + self.disperseur + "_" + "reduc_new_" + parameters.PROD_NUM + "_emcee.h5" if os.path.exists(filename): slope, ord, err_slope, err_ord = self.bouguer_line() else: slope, ord, err_slope, err_ord, A2, A2_err = self.bouguer_line_order2( ) p_ozone = 300 p_eau = 5 p_aerosols = 0.03 #p_A2 = 1 p0 = np.array([p_ozone, p_eau, p_aerosols]) walker = 10 #init_A2 = p0[0] + p0[0] / 5 * np.random.randn(walker) init_ozone = p0[0] + p0[0] / 5 * np.random.randn(walker) init_eau = p0[1] + p0[1] / 5 * np.random.randn(walker) init_aerosols = p0[2] + p0[2] / 5 * np.random.randn(walker) p0 = np.array([[init_ozone[i], init_eau[i], init_aerosols[i]] for i in range(walker)]) nwalkers, ndim = p0.shape """ plt.errorbar(self.new_lambda, (np.exp(self.data_mag) - self.order2)[:,10], yerr=(self.err_mag * np.exp(self.data_mag))[:,10]) plt.show() """ backend = emcee.backends.HDFBackend(filename) try: pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability, args=(atm, ), pool=pool, backend=backend) if backend.iteration > 0: p0 = backend.get_last_sample() if nsamples - backend.iteration > 0: sampler.run_mcmc(p0, nsteps=max(0, nsamples - backend.iteration), progress=True) pool.close() except ValueError: sampler = emcee.EnsembleSampler( nwalkers, ndim, log_probability, args=(atm, ), threads=multiprocessing.cpu_count(), backend=backend) if backend.iteration > 0: p0 = sampler.get_last_sample() for _ in sampler.sample(p0, iterations=max( 0, nsamples - backend.iteration), progress=True, store=True): continue flat_samples = sampler.get_chain(discard=100, thin=1, flat=True) #A2, A2_err = np.mean(flat_samples[:, -4]), np.std(flat_samples[:, -4]) ozone, d_ozone = np.mean(flat_samples[:, -3]), np.std(flat_samples[:, -3]) eau, d_eau = np.mean(flat_samples[:, -2]), np.std(flat_samples[:, -2]) aerosols, d_aerosols = np.mean(flat_samples[:, -1]), np.std( flat_samples[:, -1]) print(ozone, d_ozone) print(eau, d_eau) print(aerosols, d_aerosols) #print(A2, A2_err) nb_spectre = len(self.names) nb_bin = len(self.data_mag) M, M_p = Atm(atm, ozone, eau, aerosols) prod = np.zeros((nb_bin, nb_spectre * nb_bin)) chi2 = 0 for spec in range(nb_spectre): prod[:, spec * nb_bin:(spec + 1) * nb_bin] = M[spec] @ self.INVCOV[spec] COV = inv(prod @ M_p) A2 = 0 D = self.matrice_data(A2) Tinst = COV @ prod @ D Tinst_err = np.array([np.sqrt(COV[i, i]) for i in range(len(Tinst))]) a, b = np.argmin(abs(self.new_lambda - 537.5)), np.argmin( abs(self.new_lambda - 542.5)) Tinst_err[a], Tinst_err[b] = 1e-16, 1e-16 A = COV @ prod @ D for spec in range(nb_spectre): mat = D[spec * nb_bin:(spec + 1) * nb_bin] - M[spec] @ A chi2 += mat @ self.INVCOV[spec] @ mat print(chi2 / (nb_spectre * nb_bin)) def compute_correlation_matrix(cov): rho = np.zeros_like(cov) for i in range(cov.shape[0]): for j in range(cov.shape[1]): rho[i, j] = cov[i, j] / np.sqrt(cov[i, i] * cov[j, j]) return rho def plot_correlation_matrix_simple(ax, rho, axis_names, ipar=None): if ipar is None: ipar = np.arange(rho.shape[0]).astype(int) im = plt.imshow(rho[ipar[:, None], ipar], interpolation="nearest", cmap='bwr', vmin=-1, vmax=1) ax.set_title("Correlation matrix") names = [axis_names[ip] for ip in ipar] plt.xticks(np.arange(ipar.size), names, rotation='vertical', fontsize=11) plt.yticks(np.arange(ipar.size), names, fontsize=11) cbar = plt.colorbar(im) cbar.ax.tick_params(labelsize=9) plt.gcf().tight_layout() def plot_err(err, ipar=None): rho = np.zeros((len(self.names), len(self.data_mag))) test = [ int(self.names[i][-16:-13]) for i in range(len(self.names)) ] print(test) test2 = test.copy() for Test in test: C = 0 for Test2 in test: if Test > Test2: C += 1 test2[C] = Test print(test2) axis_names_vert = [] for i in range(rho.shape[0]): k = np.argmin(abs(np.array(test) - test2[i])) axis_names_vert.append(str(test[k])) for j in range(rho.shape[1]): rho[i, j] = err[k * len(self.data_mag) + j] if ipar is None: vert = np.arange(rho.shape[0]).astype(int) hor = np.arange(rho.shape[1]).astype(int) """ gs_kw = dict(height_ratios=[1], width_ratios=[5,1]) fig, ax = plt.subplots(1, 2, figsize=[15, 15], constrained_layout=True, gridspec_kw=gs_kw) ax1, ax2 = ax[0], ax[1] """ fig = plt.figure(figsize=[15, 7]) ax = fig.add_subplot(111) axis_names_hor = [ str(self.new_lambda[i]) for i in range(len(self.new_lambda)) ] norm = matplotlib.colors.SymLogNorm(vmin=-np.max(abs(rho)), vmax=np.max(abs(rho)), linthresh=10) im = plt.imshow(rho[vert[:, None], hor], interpolation="nearest", cmap='bwr', vmin=-5, vmax=5) if self.sim: #ax.set_title("Résidus: (data - model) / err sur des simulations du "+self.disperseur+" version "+parameters.PROD_NUM) ax.set_title(self.disperseur, fontsize=21) else: ax.set_title(self.disperseur, fontsize=21) print(np.mean(rho)) print(np.std(rho)) names_vert = [axis_names_vert[ip] for ip in vert] names_hor = [axis_names_hor[ip] for ip in hor] plt.xticks(np.arange(0, hor.size, 3), names_hor[::3], rotation='vertical', fontsize=14) plt.yticks(np.arange(0, vert.size, 3), names_vert[::3], fontsize=14) plt.xlabel('$\lambda$ [nm]', fontsize=17) plt.ylabel('Spectrum index', fontsize=17) cbar = plt.colorbar(im, orientation='horizontal') cbar.set_label('Residuals in #$\sigma$', fontsize=20) cbar.ax.tick_params(labelsize=13) # cbar.ax.set_yticklabels(['{:.0f}'.format(x) for x in np.linspace(np.min(rho), np.max(rho), 10)]) #fontsize=16, weight='bold') plt.gcf().tight_layout() fig.tight_layout() if self.sim and 1 == 1: plt.savefig(parameters.OUTPUTS_THROUGHPUT_SIM + 'throughput_simb, ' + self.disperseur + ',résidus, version_' + parameters.PROD_NUM + '.pdf') elif 1 == 1: plt.savefig(parameters.OUTPUTS_THROUGHPUT_REDUC + 'throughput_reduc, ' + self.disperseur + ',résidus, version_' + parameters.PROD_NUM + '.pdf') fig = plt.figure(figsize=[15, 10]) ax = fig.add_subplot(111) axis_names = [str(i) for i in range(len(COV))] plot_correlation_matrix_simple(ax, compute_correlation_matrix(COV), axis_names, ipar=None) err = np.zeros_like(D) for j in range(len(self.names)): for i in range(len(self.data_mag)): #if self.new_lambda[i]==537.5 or self.new_lambda[i]==542.5: # err[j * len(self.data_mag) + i] = 1 #else: err[j * len(self.data_mag) + i] = np.sqrt(self.cov[j][i, i]) model = M_p @ Tinst Err = (D - model) / err plot_err(Err) fig = plt.figure(figsize=[15, 15]) ax = fig.add_subplot(111) ax.hist(Err, bins=np.arange(-10, 10, 1)) ax.set_title( "Histogramme des résidus: (data - model) / err sur des données du HoloAmAg version " + parameters.PROD_NUM) plt.xlabel('Ecart aux données', fontsize=14) plt.grid(True) test = [int(self.names[i][-16:-13]) for i in range(len(self.names))] Kplot = [181, 186, 191, 196] for i in range(len(Kplot)): k = np.argmin(abs(np.array(test) - Kplot[i])) fig = plt.figure(figsize=[15, 15]) ax = fig.add_subplot(111) plt.plot(self.new_lambda, model[k * len(self.new_lambda):(k + 1) * len(self.new_lambda)], c='red', label='model') plt.plot(self.new_lambda, D[k * len(self.new_lambda):(k + 1) * len(self.new_lambda)], c='blue', label='data') plt.title("spectrum :" + str(Kplot[i])) plt.grid(True) plt.legend() plt.show() return Tinst, Tinst_err
def droites_Bouguer(method, sim, fileday, star, disperseur, binwidth, lambda_min, lambda_max, sigma_max, mag_diffmax, filtre2_order, filtre3_order, filtre2avg_window, filtre2_window, filtre3avg_window, filtre3_window, lambda_mid, DEBUG, sigma1, window1, LAMBDA_MIN, LAMBDA_MAX, trigger, filtre1_window, filtre1_order, moy_raies, demi_taille_max, filtre1avg_window, filtre_global, order_global, debut_filtre_global, debord_raies, seuilEAU_1, seuilEAU_2, bord_droit, bord_gauche, filtre4avg_window, save): """Fonction: effectue pour toute les photos d'une même etoile, le trace de magabs par bin de longueur d'onde Entree: 37 params utilent pour une fonction interne cf Plot_magabsorbe_star Sortie: Trace des droites de Bouguer. Trace de la magnitude absorbee en fonction de la masse d'air. Tableau des coefficients ordonnees à l'origine et pente (il s'agit d'une matrice de taille nombre de bin x 2) Tableaux des bins, des incertitudes, et de l'intensite par bin tabulee""" "Recuperation des tableaux renvoyes par Plot_magabsorbe_star" airmass, magabs, magabs_err, Bin, fluxlum_Binreel = Plot_magabsorbe_star(method, sim, fileday, star, disperseur, binwidth, lambda_min, lambda_max, sigma_max, mag_diffmax, filtre2_order, filtre3_order, filtre2avg_window, filtre2_window, filtre3avg_window, filtre3_window, lambda_mid, DEBUG, sigma1, window1, LAMBDA_MIN, LAMBDA_MAX, trigger, filtre1_window, filtre1_order, moy_raies, demi_taille_max, filtre1avg_window, filtre_global, order_global, debut_filtre_global, debord_raies, seuilEAU_1, seuilEAU_2, bord_droit, bord_gauche, filtre4avg_window) "Definition de la fonction lineaire qu'on va utiliser pour tracer les droites de Bouguer." def f(x, a, b): return a * x + b Z = np.linspace(0, 2.2, 1000) # nombre de points pour le trace des droites coeff = np.zeros((len(magabs), 2)) # on initialise la liste des coefficients à la liste vide. err = np.zeros(len(magabs)) # on initialise la liste des erreurs sur les ordonnees à l'origine à la liste vide. err2 = np.zeros(len(magabs)) fig = plt.figure(figsize=[15, 10]) ax = fig.add_subplot(111) new_lambda = 0.5 * (Bin[1:] + Bin[:-1]) "On trace les droites et on recupère l'ordonnee à l'origine pour chaque bin de longueur d'onde" for i in range(len(magabs)): popt, pcov = sp.optimize.curve_fit(f, airmass[i], magabs[i], sigma=magabs_err[i]) # fit propageant les incertitudes "On recupère les coefficients et les incertitudes types sur l'ordonnee à l'origine" coeff[i][0], coeff[i][1] = popt[0], popt[1] err[i] = np.sqrt(pcov[1][1]) err2[i] = np.sqrt(pcov[0][0]) "Affichage" MAG = f(Z, *popt) MAG_sup = f(Z, popt[0] + np.sqrt(pcov[0][0]), popt[1] - np.sqrt(pcov[1][1])) MAG_inf = f(Z, popt[0] - np.sqrt(pcov[0][0]), popt[1] + np.sqrt(pcov[1][1])) if i % 10 == 0: plt.plot(Z, MAG, c=wavelength_to_rgb(new_lambda[i])) plt.plot(Z, MAG_sup, c=wavelength_to_rgb(new_lambda[i]), linestyle=':') plt.plot(Z, MAG_inf, c=wavelength_to_rgb(new_lambda[i]), linestyle=':') plt.fill_between(Z, MAG_sup, MAG_inf, color=[wavelength_to_rgb(new_lambda[i])]) "la commande ci-dessus grise donne la bonne couleur la zone où se trouve la droite de Bouguer" plt.scatter(airmass[i], magabs[i], c=[wavelength_to_rgb(new_lambda[i])], label=f'{Bin[i]}-{Bin[i + 1]} nm', marker='o', s=30) plt.errorbar(airmass[i], magabs[i], xerr=None, yerr=magabs_err[i], fmt='none', capsize=1, ecolor=(wavelength_to_rgb(new_lambda[i])), zorder=2, elinewidth=2) if sim: plt.title('Droites de Bouguer, simulation, ' + disperseur, fontsize=18) else: plt.title('Droites de Bouguer, données, ' + disperseur, fontsize=18) ax.get_xaxis().set_tick_params(labelsize=17) ax.get_yaxis().set_tick_params(labelsize=14) plt.xlabel('$\lambda$ (nm)', fontsize=17) plt.ylabel('ln(flux)', fontsize=15) plt.grid(True) plt.legend(prop={'size': 12}, loc='upper right') if save: if sim == False: plt.savefig('droites_bouguer_prod63/' + disperseur + '.png') else: plt.savefig('droites_bouguer_prod63_simu/' + disperseur + '.png') fig = plt.figure(figsize=[15, 10]) Lambda = np.arange(new_lambda[0], new_lambda[-1], 1) atmgrid = AtmosphereGrid(filename="../reduc_20170530_060_atmsim.fits") b = atmgrid.simulate(300, 5, 0.03) if sim: plt.title('Transmission atmosphérique, simulation, ' + disperseur, fontsize=18) else: plt.title('Transmission atmosphérique, données, ' + disperseur, fontsize=18) plt.plot(Lambda, np.exp(np.log(b(Lambda)) / 1.047), color='blue', label='transmission libradtran typique') plt.plot(new_lambda, np.exp(coeff.T[0]), color='red', label='transmission atmosphérique droites bouguer') plt.errorbar(new_lambda, np.exp(coeff.T[0]), xerr=None, yerr=err2 * np.exp(coeff.T[0]), fmt='none', capsize=1, ecolor='red', zorder=2, elinewidth=2) plt.fill_between(new_lambda, np.exp(coeff.T[0]) + err2 * np.exp(coeff.T[0]), np.exp(coeff.T[0]) - err2 * np.exp(coeff.T[0]), color='red') ax.get_xaxis().set_tick_params(labelsize=17) ax.get_yaxis().set_tick_params(labelsize=14) plt.xlabel('$\lambda$ (nm)', fontsize=17) plt.ylabel('Transmission atmosphérique', fontsize=15) plt.grid(True) plt.legend(prop={'size': 15}, loc='lower right') if save: if sim == False: plt.savefig('droites_bouguer_prod63/' + disperseur + '_atm.png') else: plt.savefig('droites_bouguer_prod63_simu/' + disperseur + '_atm.png') fig = plt.figure(figsize=[15, 10]) plt.plot(new_lambda, np.exp(coeff.T[1]), c='black') return (coeff, Bin, err, fluxlum_Binreel)