def test_atmosphere():
a = Atmosphere(airmass=1.2, pressure=800, temperature=5)
transmission = a.simulate(ozone=400, pwv=5, aerosols=0.05)
assert transmission is not None
assert a.transmission(500) > 0
assert a.ozone == 400
assert a.pwv == 5
assert a.aerosols == 0.05
a.plot_transmission()
a = AtmosphereGrid(image_filename='tests/data/reduc_20170605_028.fits', pwv_grid=[5, 5, 1],
ozone_grid=[400, 400, 1], aerosol_grid=[0.0, 0.1, 2])
atmospheric_grid = a.compute()
assert np.sum(atmospheric_grid) > 0
assert np.all(np.isclose(a.atmgrid[0, a.index_atm_data:], parameters.LAMBDAS))
assert not np.any(np.isclose(a.atmgrid[1, a.index_atm_data:], np.zeros_like(parameters.LAMBDAS), rtol=1e-6))
assert a.atmgrid.shape == (3, a.index_atm_data + len(parameters.LAMBDAS))
a.save_file(a.image_filename.replace('.fits', '_atmsim.fits'))
assert os.path.isfile('tests/data/reduc_20170605_028_atmsim.fits')
a.load_file(a.image_filename.replace('.fits', '_atmsim.fits'))
assert np.all(a.AER_Points == np.array([0., 0.1]))
assert np.all(a.PWV_Points == np.array([5.]))
assert np.all(a.OZ_Points == np.array([400.]))
a.plot_transmission()
a.plot_transmission_image()
a = AtmosphereGrid(filename='tests/data/reduc_20170530_134_atmsim.fits')
lambdas = np.arange(200, 1200)
transmission = a.simulate(ozone=400, pwv=5, aerosols=0.05)
assert np.max(transmission(lambdas)) < 1 and np.min(transmission(lambdas)) >= 0
def test_simulator():
file_names = ['tests/data/reduc_20170530_134_spectrum.fits']
parameters.VERBOSE = True
parameters.DEBUG = True
load_config('config/ctio.ini')
output_directory = './outputs/'
for file_name in file_names:
tag = file_name.split('/')[-1]
spectrum_simulation = SpectrumSimulator(file_name, output_directory, pwv=5, ozone=300, aerosols=0.03,
A1=1, A2=10, reso=2, D=56, shift=-1)
assert np.sum(spectrum_simulation.data) > 0
assert np.sum(spectrum_simulation.data) < 1e-10
assert np.sum(spectrum_simulation.data_order2) < 1e-10
# spectrogram_simulation = SpectrogramSimulator(file_name, output_directory, pwv=5, ozone=300, aerosols=0.03,
# A1=1, A2=1, D=56, shift_x=-1, shift_y=1, angle=-1)
# assert np.sum(spectrogram_simulation.data) > 20
# psf_poly_params = spectrogram_simulation.chromatic_psf.from_table_to_poly_params()
image_simulation = ImageSim(file_name.replace('_spectrum.fits', '.fits'), file_name, output_directory, A2=1,
psf_poly_params=None, with_stars=True)
SpectrumSimulatorSimGrid(file_name, output_directory, pwv_grid=[0, 10, 2], ozone_grid=[200, 400, 2],
aerosol_grid=[0, 0.1, 2])
atmgrid = AtmosphereGrid(file_name, file_name.replace('spectrum', 'atmsim'))
atm = Atmosphere(atmgrid.airmass, atmgrid.pressure, atmgrid.temperature)
assert os.path.isfile(output_directory + tag.replace('_spectrum.fits', '_atmsim.fits')) is True
assert image_simulation.data is not None
# assert spectrum_simulation.data is not None
# assert spectrogram_simulation.data is not None
assert os.path.isfile(output_directory + tag.replace('reduc', 'sim').replace('_spectrum.fits', '.fits'))
assert atm.transmission is not None
def fit_spectrum(self):
nsamples = 300
def spec_calspec():
spec = open(self.prod_reduc[0], 'r')
lambdas = []
data = []
for line in spec:
Line = line.split()
if Line[0] != '#':
lambdas.append(float(Line[0]))
data.append(float(Line[1]))
lambdas_calspec = np.array(lambdas)
data_calspec_org = np.array(data)
data_calspec = data_calspec_org
fluxlum_Binreel = np.zeros(len(self.Bin) - 1)
interpolation_reel = sp.interpolate.interp1d(lambdas_calspec, data_calspec, kind="linear",
bounds_error=False,
fill_value=(0, 0))
for v in range(len(self.Bin) - 1):
"On rempli les tableaux par bin de longueur d'onde"
X = np.linspace(self.Bin[v], self.Bin[v + 1], int(self.binwidths * 100))
Y = interpolation_reel(X)
fluxlum_Binreel[v] = integrate.simps(Y, X, dx=1) / self.binwidths
data_calspec = fluxlum_Binreel
return data_calspec
def Ordonnee():
data_calspec = spec_calspec()
if self.disperseur != parameters.DISPERSER_REF:
disp = np.loadtxt(parameters.DISPERSER_EXTRACTION)
else:
disp = np.loadtxt(parameters.DISPERSER_REF_BANC)
Tctio = np.loadtxt(parameters.THROUGHPUT_REDUC)
Tctio_data = sp.interpolate.interp1d(Tctio.T[0], Tctio.T[1], bounds_error=False,
fill_value="extrapolate")(self.new_lambda)
disp_data = sp.interpolate.interp1d(disp.T[0], disp.T[1], bounds_error=False,
fill_value="extrapolate")(self.new_lambda)
return data_calspec * Tctio_data * disp_data
OZONE = np.zeros(len(self.names))
AEROSOLS = np.zeros(len(self.names))
EAU = np.zeros(len(self.names))
ERR_OZONE = np.zeros(len(self.names))
ERR_AEROSOLS = np.zeros(len(self.names))
ERR_EAU = np.zeros(len(self.names))
for spec in range(len(self.names)):
atm = AtmosphereGrid(
filename=(os.path.join(parameters.PROD_DIRECTORY, parameters.PROD_NAME) + '/' + self.names[spec].split('/')[-1]).replace('sim', 'reduc').replace(
'spectrum.txt', 'atmsim.fits'))
self.atm = atm
ORDO = Ordonnee()
def Atm(atm, ozone, eau, aerosols):
nb_bin = len(self.data)
Atmo = np.zeros(nb_bin)
Lambdas = np.arange(self.Bin[0], self.Bin[-1], 0.2)
Atm = atm.simulate(ozone, eau, aerosols)(Lambdas)
for i in range(len(self.new_lambda)):
Atmo[i] = np.mean(Atm[i * int(self.binwidths / 0.2):(i + 1) * int(self.binwidths / 0.2)])
M = np.diagflat(Atmo)
return M
def log_likelihood(params_fit):
nb_bin = len(self.data)
ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1]
M = Atm(self.atm,ozone, eau, aerosols)
D = self.data[:,spec] - self.order2[:,spec]
mat = D - M @ ORDO
chi2 = mat @ self.INVCOV[spec] @ mat
n = np.random.randint(0, 100)
if n > 97:
print(chi2 / (nb_bin))
print(ozone, eau, aerosols)
return -0.5 * chi2
def log_prior(params_fit):
ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1]
if 100 < ozone < 700 and 0 < eau < 10 and 0 < aerosols < 0.1:
return 0
else:
return -np.inf
def log_probability(params_fit):
lp = log_prior(params_fit)
if not np.isfinite(lp):
return -np.inf
return lp + log_likelihood(params_fit)
p_ozone = 300
p_eau = 5
p_aerosols = 0.03
p0 = np.array([p_ozone, p_eau, p_aerosols])
walker = 10
init_ozone = p0[0] + p0[0] / 5 * np.random.randn(walker)
init_eau = p0[1] + p0[1] / 5 * np.random.randn(walker)
init_aerosols = p0[2] + p0[2] / 5 * np.random.randn(walker)
p0 = np.array([[init_ozone[i], init_eau[i], init_aerosols[i]] for i in range(walker)])
nwalkers, ndim = p0.shape
sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability,
threads=multiprocessing.cpu_count())
sampler.run_mcmc(p0, nsamples, progress=True)
flat_samples = sampler.get_chain(discard=100, thin=1, flat=True)
ozone, d_ozone = np.mean(flat_samples[:, -3]), np.std(flat_samples[:, -3])
eau, d_eau = np.mean(flat_samples[:, -2]), np.std(flat_samples[:, -2])
aerosols, d_aerosols = np.mean(flat_samples[:, -1]), np.std(flat_samples[:, -1])
print(ozone, d_ozone)
print(eau, d_eau)
print(aerosols, d_aerosols)
OZONE[spec] = ozone
AEROSOLS[spec] = aerosols
EAU[spec] = eau
ERR_OZONE[spec] = d_ozone
ERR_EAU[spec] = d_eau
ERR_AEROSOLS[spec] = d_aerosols
if self.sim:
chemin = parameters.OUTPUTS_FITSPECTRUM_SIM
else:
chemin = parameters.OUTPUTS_FITSPECTRUM_REDUC
NUM = np.zeros(len(self.names))
for i in range(len(self.names)):
NUM[i] = self.names[i][-16:-13]
fig = plt.figure(figsize=[10, 10])
ax2 = fig.add_subplot(111)
ax2.get_xaxis().set_tick_params(labelsize=12)
ax2.get_yaxis().set_tick_params(labelsize=12)
plt.grid(True)
plt.title("Fit spectrum with bins", fontsize=22)
plt.xlabel('Spectrum index', fontsize=20)
plt.ylabel("Ozone [db]", fontsize=20)
plt.scatter(NUM, OZONE, c='blue')
plt.errorbar(NUM, OZONE, xerr=None, yerr=ERR_OZONE, fmt='none', capsize=1,
ecolor='blue', zorder=2, elinewidth=2)
fig.tight_layout()
if os.path.exists(chemin) == False:
os.makedirs(chemin)
plt.savefig(chemin + 'Ozone_fit, ' + self.disperseur + ', version_' + parameters.PROD_NUM + '.pdf')
fig = plt.figure(figsize=[10, 10])
ax2 = fig.add_subplot(111)
ax2.get_xaxis().set_tick_params(labelsize=12)
ax2.get_yaxis().set_tick_params(labelsize=12)
plt.grid(True)
plt.title("Fit spectrum with bins", fontsize=22)
plt.xlabel('Spectrum index', fontsize=20)
plt.ylabel("Aerosols [VAOD]", fontsize=20)
plt.scatter(NUM, AEROSOLS, c='blue')
plt.errorbar(NUM, AEROSOLS, xerr=None, yerr=ERR_AEROSOLS, fmt='none', capsize=1,
ecolor='blue', zorder=2, elinewidth=2)
fig.tight_layout()
plt.savefig(chemin + 'Aerosols_fit, ' + self.disperseur + ', version_' + parameters.PROD_NUM + '.pdf')
fig = plt.figure(figsize=[10, 10])
ax2 = fig.add_subplot(111)
ax2.get_xaxis().set_tick_params(labelsize=12)
ax2.get_yaxis().set_tick_params(labelsize=12)
plt.grid(True)
plt.title("Fit spectrum with bins", fontsize=22)
plt.xlabel('Spectrum index', fontsize=20)
plt.ylabel("PWV [mm]", fontsize=20)
plt.scatter(NUM, EAU, c='blue')
plt.errorbar(NUM, EAU, xerr=None, yerr=ERR_EAU, fmt='none', capsize=1,
ecolor='blue', zorder=2, elinewidth=2)
fig.tight_layout()
plt.savefig(chemin + 'Eau_fit, ' + self.disperseur + ', version_' + parameters.PROD_NUM + '.pdf')
plt.show()
def megafit_emcee(self):
nsamples = 300
atm = []
for j in range(len(self.names)):
prod_name = os.path.join(parameters.PROD_DIRECTORY, parameters.PROD_NAME)
atmgrid = AtmosphereGrid(
filename=(prod_name + '/' + self.names[j].split('/')[-1]).replace('sim', 'reduc').replace(
'spectrum.txt', 'atmsim.fits'))
atm.append(atmgrid)
def matrice_data():
y = self.data - self.order2
nb_spectre = len(self.names)
nb_bin = len(self.data)
D = np.zeros(nb_bin * nb_spectre)
for j in range(nb_spectre):
D[j * nb_bin: (j + 1) * nb_bin] = y[:, j]
return D
def Atm(atm, ozone, eau, aerosols):
nb_spectre = len(self.names)
nb_bin = len(self.data)
M = np.zeros((nb_spectre, nb_bin, nb_bin))
M_p = np.zeros((nb_spectre * nb_bin, nb_bin))
for j in range(nb_spectre):
Atmo = np.zeros(len(self.new_lambda))
Lambdas = np.arange(self.Bin[0],self.Bin[-1],0.2)
Atm = atm[j].simulate(ozone, eau, aerosols)(Lambdas)
for i in range(len(self.new_lambda)):
Atmo[i] = np.mean(Atm[i*int(self.binwidths/0.2):(i+1)*int(self.binwidths/0.2)])
a = np.diagflat(Atmo)
M[j, :, :] = a
M_p[nb_bin * j:nb_bin * (j+1),:] = a
return M, M_p
def log_likelihood(params_fit, atm):
nb_spectre = len(self.names)
nb_bin = len(self.data)
ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1]
D = matrice_data()
M, M_p = Atm(atm, ozone, eau, aerosols)
prod = np.zeros((nb_bin, nb_spectre * nb_bin))
for spec in range(nb_spectre):
prod[:,spec * nb_bin : (spec+1) * nb_bin] = M[spec] @ self.INVCOV[spec]
COV = inv(prod @ M_p)
A = COV @ prod @ D
chi2 = 0
for spec in range(nb_spectre):
mat = D[spec * nb_bin : (spec+1) * nb_bin] - M[spec] @ A
chi2 += mat @ self.INVCOV[spec] @ mat
n = np.random.randint(0, 100)
if n > 97:
print(chi2 / (nb_spectre * nb_bin))
print(ozone, eau, aerosols)
return -0.5 * chi2
def log_prior(params_fit):
ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1]
if 100 < ozone < 700 and 0 < eau < 10 and 0 < aerosols < 0.1:
return 0
else:
return -np.inf
def log_probability(params_fit, atm):
lp = log_prior(params_fit)
if not np.isfinite(lp):
return -np.inf
return lp + log_likelihood(params_fit, atm)
if self.sim:
filename = "sps/" + self.disperseur + "_"+ "sim_"+ parameters.PROD_NUM + "_emcee.h5"
else:
filename = "sps/" + self.disperseur + "_" + "reduc_" + parameters.PROD_NUM + "_emcee.h5"
p_ozone = 300
p_eau = 5
p_aerosols = 0.03
p0 = np.array([p_ozone, p_eau, p_aerosols])
walker = 10
init_ozone = p0[0] + p0[0] / 5 * np.random.randn(walker)
init_eau = p0[1] + p0[1] / 5 * np.random.randn(walker)
init_aerosols = p0[2] + p0[2] / 5 * np.random.randn(walker)
p0 = np.array([[init_ozone[i], init_eau[i], init_aerosols[i]] for i in range(walker)])
nwalkers, ndim = p0.shape
backend = emcee.backends.HDFBackend(filename)
try:
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability,
args=(atm,), pool=pool, backend=backend)
if backend.iteration > 0:
p0 = backend.get_last_sample()
if nsamples - backend.iteration > 0:
sampler.run_mcmc(p0, nsteps=max(0, nsamples - backend.iteration), progress=True)
pool.close()
except ValueError:
sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability,
args=(atm,),
threads=multiprocessing.cpu_count(), backend=backend)
if backend.iteration > 0:
p0 = sampler.get_last_sample()
for _ in sampler.sample(p0, iterations=max(0, nsamples - backend.iteration), progress=True, store=True):
continue
flat_samples = sampler.get_chain(discard=100, thin=1, flat=True)
ozone, d_ozone = np.mean(flat_samples[:, -3]), np.std(flat_samples[:, -3])
eau, d_eau = np.mean(flat_samples[:, -2]), np.std(flat_samples[:, -2])
aerosols, d_aerosols = np.mean(flat_samples[:, -1]), np.std(flat_samples[:, -1])
print(ozone, d_ozone)
print(eau, d_eau)
print(aerosols, d_aerosols)
self.params_atmo = np.array([ozone, eau, aerosols])
self.err_params_atmo = np.array([d_ozone, d_eau, d_aerosols])
nb_spectre = len(self.names)
nb_bin = len(self.data)
M, M_p = Atm(atm, ozone, eau, aerosols)
prod = np.zeros((nb_bin, nb_spectre * nb_bin))
chi2 = 0
for spec in range(nb_spectre):
prod[:, spec * nb_bin: (spec + 1) * nb_bin] = M[spec] @ self.INVCOV[spec]
COV = inv(prod @ M_p)
D = matrice_data()
Tinst = COV @ prod @ D
Tinst_err = np.array([np.sqrt(COV[i,i]) for i in range(len(Tinst))])
if self.disperseur == 'HoloAmAg' and self.sim == False:
a, b = np.argmin(abs(self.new_lambda - 537.5)), np.argmin(abs(self.new_lambda - 542.5))
Tinst_err[a], Tinst_err[b] = Tinst_err[a-1], Tinst_err[b+1]
for spec in range(nb_spectre):
mat = D[spec * nb_bin: (spec + 1) * nb_bin] - M[spec] @ Tinst
chi2 += mat @ self.INVCOV[spec] @ mat
print(chi2 / (nb_spectre * nb_bin))
err = np.zeros_like(D)
for j in range(len(self.names)):
for i in range(len(self.data)):
if self.disperseur == 'HoloAmAg' and self.sim == False:
if self.new_lambda[i] == 537.5 or self.new_lambda[i] == 542.5:
err[j * len(self.data) + i] = 1
else:
err[j * len(self.data) + i] = np.sqrt(self.cov[j][i, i])
else:
err[j * len(self.data) + i] = np.sqrt(self.cov[j][i, i])
model = M_p @ Tinst
Err = (D - model) / err
if parameters.plot_residuals:
self.Plot_residuals(model, D, Err, COV)
return Tinst, Tinst_err
parameters.DEBUG = True
parameters.VERBOSE = True
file_names = args.input
load_config(args.config)
logbook = LogBook(logbook=args.logbook)
for file_name in file_names:
tag = file_name.split('/')[-1]
disperser_label, target, xpos, ypos = logbook.search_for_image(tag)
if target is None or xpos is None or ypos is None:
continue
spectrum_file_name = args.output_directory + '/' + tag.replace(
'.fits', '_spectrum.fits')
atmgrid = AtmosphereGrid(file_name)
SpectrumSimulatorSimGrid(spectrum_file_name, args.output_directory)
image = ImageSim(file_name,
spectrum_file_name,
args.output_directory,
A1=1,
A2=1,
pwv=5,
ozone=300,
aerosols=0.03,
psf_poly_params=None,
with_stars=True)
sim_file_name = args.output_directory + tag.replace('reduc_', 'sim_')
Spectractor(sim_file_name, args.output_directory, target, [xpos, ypos],
disperser_label, args.config)
def archi_mega_fit(self):
nsamples = 300
OZONE = np.zeros(len(self.names))
AEROSOLS = np.zeros(len(self.names))
EAU = np.zeros(len(self.names))
ERR_OZONE = np.zeros(len(self.names))
ERR_AEROSOLS = np.zeros(len(self.names))
ERR_EAU = np.zeros(len(self.names))
for spec in range(len(self.names)):
atm = AtmosphereGrid(filename=(
os.path.join(parameters.PROD_DIRECTORY, parameters.PROD_NAME) +
'/' + self.names[spec].split('/')[-1]
).replace('sim', 'reduc').replace('spectrum.txt', 'atmsim.fits'))
self.atm = atm
ORDO = self.Ordonnee()
def Atm(atm, ozone, eau, aerosols):
nb_bin = len(self.data_mag)
Atmo = np.zeros(nb_bin)
Lambdas = np.arange(self.Bin[0], self.Bin[-1], 0.2)
Atm = atm.simulate(ozone, eau, aerosols)(Lambdas)
for i in range(len(self.new_lambda)):
Atmo[i] = np.mean(
Atm[i * int(self.binwidths / 0.2):(i + 1) *
int(self.binwidths / 0.2)])
M = np.diagflat(Atmo)
return M
def log_likelihood(params_fit):
nb_bin = len(self.data_mag)
ozone, eau, aerosols = params_fit[-3], params_fit[
-2], params_fit[-1]
M = Atm(self.atm, ozone, eau, aerosols)
D = np.exp(self.data_mag[:, spec]) - 0 * self.order2[:, spec]
mat = D - M @ ORDO
chi2 = mat @ self.INVCOV[spec] @ mat
n = np.random.randint(0, 100)
if n > 97:
print(chi2 / (nb_bin))
print(ozone, eau, aerosols)
return -0.5 * chi2
def log_prior(params_fit):
ozone, eau, aerosols = params_fit[-3], params_fit[
-2], params_fit[-1]
if 100 < ozone < 700 and 0 < eau < 10 and 0 < aerosols < 0.1: # and 0<A2<5:
return 0
else:
return -np.inf
def log_probability(params_fit):
lp = log_prior(params_fit)
if not np.isfinite(lp):
return -np.inf
return lp + log_likelihood(params_fit)
p_ozone = 300
p_eau = 5
p_aerosols = 0.03
p0 = np.array([p_ozone, p_eau, p_aerosols])
walker = 10
init_ozone = p0[0] + p0[0] / 5 * np.random.randn(walker)
init_eau = p0[1] + p0[1] / 5 * np.random.randn(walker)
init_aerosols = p0[2] + p0[2] / 5 * np.random.randn(walker)
p0 = np.array([[init_ozone[i], init_eau[i], init_aerosols[i]]
for i in range(walker)])
nwalkers, ndim = p0.shape
sampler = emcee.EnsembleSampler(
nwalkers,
ndim,
log_probability,
threads=multiprocessing.cpu_count())
sampler.run_mcmc(p0, nsamples, progress=True)
flat_samples = sampler.get_chain(discard=100, thin=1, flat=True)
ozone, d_ozone = np.mean(flat_samples[:, -3]), np.std(
flat_samples[:, -3])
eau, d_eau = np.mean(flat_samples[:, -2]), np.std(flat_samples[:,
-2])
aerosols, d_aerosols = np.mean(flat_samples[:, -1]), np.std(
flat_samples[:, -1])
print(ozone, d_ozone)
print(eau, d_eau)
print(aerosols, d_aerosols)
OZONE[spec] = ozone
AEROSOLS[spec] = aerosols
EAU[spec] = eau
ERR_OZONE[spec] = d_ozone
ERR_EAU[spec] = d_eau
ERR_AEROSOLS[spec] = d_aerosols
NUM = np.zeros(len(self.names))
for i in range(len(self.names)):
NUM[i] = self.names[i][-16:-13]
fig = plt.figure(figsize=[10, 10])
ax2 = fig.add_subplot(111)
ax2.get_xaxis().set_tick_params(labelsize=12)
ax2.get_yaxis().set_tick_params(labelsize=12)
plt.grid(True)
plt.title("Fit spectrum with bins", fontsize=22)
plt.xlabel('Spectrum index', fontsize=20)
plt.ylabel("Ozone [db]", fontsize=20)
plt.scatter(NUM, OZONE, c='blue')
plt.errorbar(NUM,
OZONE,
xerr=None,
yerr=ERR_OZONE,
fmt='none',
capsize=1,
ecolor='blue',
zorder=2,
elinewidth=2)
fig.tight_layout()
plt.savefig('Ozone_fit.pdf')
fig = plt.figure(figsize=[10, 10])
ax2 = fig.add_subplot(111)
ax2.get_xaxis().set_tick_params(labelsize=12)
ax2.get_yaxis().set_tick_params(labelsize=12)
plt.grid(True)
plt.title("Fit spectrum with bins", fontsize=22)
plt.xlabel('Spectrum index', fontsize=20)
plt.ylabel("Aerosols [VAOD]", fontsize=20)
plt.scatter(NUM, AEROSOLS, c='blue')
plt.errorbar(NUM,
AEROSOLS,
xerr=None,
yerr=ERR_AEROSOLS,
fmt='none',
capsize=1,
ecolor='blue',
zorder=2,
elinewidth=2)
fig.tight_layout()
plt.savefig('Aerosols_fit.pdf')
fig = plt.figure(figsize=[10, 10])
ax2 = fig.add_subplot(111)
ax2.get_xaxis().set_tick_params(labelsize=12)
ax2.get_yaxis().set_tick_params(labelsize=12)
plt.grid(True)
plt.title("Fit spectrum with bins", fontsize=22)
plt.xlabel('Spectrum index', fontsize=20)
plt.ylabel("PWV [mm]", fontsize=20)
plt.scatter(NUM, EAU, c='blue')
plt.errorbar(NUM,
EAU,
xerr=None,
yerr=ERR_EAU,
fmt='none',
capsize=1,
ecolor='blue',
zorder=2,
elinewidth=2)
fig.tight_layout()
plt.savefig('Eau_fit.pdf')
plt.show()
def megafit_emcee(self):
nsamples = 300
atm = []
for j in range(len(self.names)):
prod_name = os.path.join(parameters.PROD_DIRECTORY,
parameters.PROD_NAME)
atmgrid = AtmosphereGrid(filename=(
prod_name + '/' + self.names[j].split('/')[-1]
).replace('sim', 'reduc').replace('spectrum.txt', 'atmsim.fits'))
atm.append(atmgrid)
#D = self.matrice_data()
"""
def f_tinst_atm(Tinst, ozone, eau, aerosols, atm):
model = np.zeros((len(self.data_mag), len(self.names)))
for j in range(len(self.names)):
a = atm[j].simulate(ozone, eau, aerosols)
model[:, j] = Tinst * a(self.new_lambda)
return model
"""
def Atm(atm, ozone, eau, aerosols):
nb_spectre = len(self.names)
nb_bin = len(self.data_mag)
M = np.zeros((nb_spectre, nb_bin, nb_bin))
M_p = np.zeros((nb_spectre * nb_bin, nb_bin))
for j in range(nb_spectre):
Atmo = np.zeros(len(self.new_lambda))
Lambdas = np.arange(self.Bin[0], self.Bin[-1], 0.2)
Atm = atm[j].simulate(ozone, eau, aerosols)(Lambdas)
for i in range(len(self.new_lambda)):
#step = int(self.binwidths * 10)
#X = np.linspace(self.Bin[i], self.Bin[i + 1] + step, step)
Atmo[i] = np.mean(
Atm[i * int(self.binwidths / 0.2):(i + 1) *
int(self.binwidths / 0.2)])
a = np.diagflat(Atmo)
M[j, :, :] = a
M_p[nb_bin * j:nb_bin * (j + 1), :] = a
return M, M_p
def log_likelihood(params_fit, atm):
nb_spectre = len(self.names)
nb_bin = len(self.data_mag)
ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[
-1]
A2 = 0
D = self.matrice_data(A2)
M, M_p = Atm(atm, ozone, eau, aerosols)
#A = np.zeros(nb_bin)
#COV = np.zeros((nb_bin, nb_bin))
prod = np.zeros((nb_bin, nb_spectre * nb_bin))
for spec in range(nb_spectre):
prod[:, spec * nb_bin:(spec + 1) *
nb_bin] = M[spec] @ self.INVCOV[spec]
COV = inv(prod @ M_p)
A = COV @ prod @ D
chi2 = 0
for spec in range(nb_spectre):
mat = D[spec * nb_bin:(spec + 1) * nb_bin] - M[spec] @ A
chi2 += mat @ self.INVCOV[spec] @ mat
n = np.random.randint(0, 100)
if n > 97:
print(chi2 / (nb_spectre * nb_bin))
print(A2, ozone, eau, aerosols)
return -0.5 * chi2
def log_prior(params_fit):
ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[
-1]
if 100 < ozone < 700 and 0 < eau < 10 and 0 < aerosols < 0.1: # and 0<A2<5:
return 0
else:
return -np.inf
def log_probability(params_fit, atm):
lp = log_prior(params_fit)
if not np.isfinite(lp):
return -np.inf
return lp + log_likelihood(params_fit, atm)
if self.sim:
filename = "sps/" + self.disperseur + "_" + "sim_new_" + parameters.PROD_NUM + "_emcee.h5" #+ "sim_new_"
else:
filename = "sps/" + self.disperseur + "_" + "reduc_new_" + parameters.PROD_NUM + "_emcee.h5"
if os.path.exists(filename):
slope, ord, err_slope, err_ord = self.bouguer_line()
else:
slope, ord, err_slope, err_ord, A2, A2_err = self.bouguer_line_order2(
)
p_ozone = 300
p_eau = 5
p_aerosols = 0.03
#p_A2 = 1
p0 = np.array([p_ozone, p_eau, p_aerosols])
walker = 10
#init_A2 = p0[0] + p0[0] / 5 * np.random.randn(walker)
init_ozone = p0[0] + p0[0] / 5 * np.random.randn(walker)
init_eau = p0[1] + p0[1] / 5 * np.random.randn(walker)
init_aerosols = p0[2] + p0[2] / 5 * np.random.randn(walker)
p0 = np.array([[init_ozone[i], init_eau[i], init_aerosols[i]]
for i in range(walker)])
nwalkers, ndim = p0.shape
"""
plt.errorbar(self.new_lambda, (np.exp(self.data_mag) - self.order2)[:,10], yerr=(self.err_mag * np.exp(self.data_mag))[:,10])
plt.show()
"""
backend = emcee.backends.HDFBackend(filename)
try:
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
sampler = emcee.EnsembleSampler(nwalkers,
ndim,
log_probability,
args=(atm, ),
pool=pool,
backend=backend)
if backend.iteration > 0:
p0 = backend.get_last_sample()
if nsamples - backend.iteration > 0:
sampler.run_mcmc(p0,
nsteps=max(0, nsamples - backend.iteration),
progress=True)
pool.close()
except ValueError:
sampler = emcee.EnsembleSampler(
nwalkers,
ndim,
log_probability,
args=(atm, ),
threads=multiprocessing.cpu_count(),
backend=backend)
if backend.iteration > 0:
p0 = sampler.get_last_sample()
for _ in sampler.sample(p0,
iterations=max(
0, nsamples - backend.iteration),
progress=True,
store=True):
continue
flat_samples = sampler.get_chain(discard=100, thin=1, flat=True)
#A2, A2_err = np.mean(flat_samples[:, -4]), np.std(flat_samples[:, -4])
ozone, d_ozone = np.mean(flat_samples[:, -3]), np.std(flat_samples[:,
-3])
eau, d_eau = np.mean(flat_samples[:, -2]), np.std(flat_samples[:, -2])
aerosols, d_aerosols = np.mean(flat_samples[:, -1]), np.std(
flat_samples[:, -1])
print(ozone, d_ozone)
print(eau, d_eau)
print(aerosols, d_aerosols)
#print(A2, A2_err)
nb_spectre = len(self.names)
nb_bin = len(self.data_mag)
M, M_p = Atm(atm, ozone, eau, aerosols)
prod = np.zeros((nb_bin, nb_spectre * nb_bin))
chi2 = 0
for spec in range(nb_spectre):
prod[:, spec * nb_bin:(spec + 1) *
nb_bin] = M[spec] @ self.INVCOV[spec]
COV = inv(prod @ M_p)
A2 = 0
D = self.matrice_data(A2)
Tinst = COV @ prod @ D
Tinst_err = np.array([np.sqrt(COV[i, i]) for i in range(len(Tinst))])
a, b = np.argmin(abs(self.new_lambda - 537.5)), np.argmin(
abs(self.new_lambda - 542.5))
Tinst_err[a], Tinst_err[b] = 1e-16, 1e-16
A = COV @ prod @ D
for spec in range(nb_spectre):
mat = D[spec * nb_bin:(spec + 1) * nb_bin] - M[spec] @ A
chi2 += mat @ self.INVCOV[spec] @ mat
print(chi2 / (nb_spectre * nb_bin))
def compute_correlation_matrix(cov):
rho = np.zeros_like(cov)
for i in range(cov.shape[0]):
for j in range(cov.shape[1]):
rho[i, j] = cov[i, j] / np.sqrt(cov[i, i] * cov[j, j])
return rho
def plot_correlation_matrix_simple(ax, rho, axis_names, ipar=None):
if ipar is None:
ipar = np.arange(rho.shape[0]).astype(int)
im = plt.imshow(rho[ipar[:, None], ipar],
interpolation="nearest",
cmap='bwr',
vmin=-1,
vmax=1)
ax.set_title("Correlation matrix")
names = [axis_names[ip] for ip in ipar]
plt.xticks(np.arange(ipar.size),
names,
rotation='vertical',
fontsize=11)
plt.yticks(np.arange(ipar.size), names, fontsize=11)
cbar = plt.colorbar(im)
cbar.ax.tick_params(labelsize=9)
plt.gcf().tight_layout()
def plot_err(err, ipar=None):
rho = np.zeros((len(self.names), len(self.data_mag)))
test = [
int(self.names[i][-16:-13]) for i in range(len(self.names))
]
print(test)
test2 = test.copy()
for Test in test:
C = 0
for Test2 in test:
if Test > Test2:
C += 1
test2[C] = Test
print(test2)
axis_names_vert = []
for i in range(rho.shape[0]):
k = np.argmin(abs(np.array(test) - test2[i]))
axis_names_vert.append(str(test[k]))
for j in range(rho.shape[1]):
rho[i, j] = err[k * len(self.data_mag) + j]
if ipar is None:
vert = np.arange(rho.shape[0]).astype(int)
hor = np.arange(rho.shape[1]).astype(int)
"""
gs_kw = dict(height_ratios=[1], width_ratios=[5,1])
fig, ax = plt.subplots(1, 2, figsize=[15, 15], constrained_layout=True, gridspec_kw=gs_kw)
ax1, ax2 = ax[0], ax[1]
"""
fig = plt.figure(figsize=[15, 7])
ax = fig.add_subplot(111)
axis_names_hor = [
str(self.new_lambda[i]) for i in range(len(self.new_lambda))
]
norm = matplotlib.colors.SymLogNorm(vmin=-np.max(abs(rho)),
vmax=np.max(abs(rho)),
linthresh=10)
im = plt.imshow(rho[vert[:, None], hor],
interpolation="nearest",
cmap='bwr',
vmin=-5,
vmax=5)
if self.sim:
#ax.set_title("Résidus: (data - model) / err sur des simulations du "+self.disperseur+" version "+parameters.PROD_NUM)
ax.set_title(self.disperseur, fontsize=21)
else:
ax.set_title(self.disperseur, fontsize=21)
print(np.mean(rho))
print(np.std(rho))
names_vert = [axis_names_vert[ip] for ip in vert]
names_hor = [axis_names_hor[ip] for ip in hor]
plt.xticks(np.arange(0, hor.size, 3),
names_hor[::3],
rotation='vertical',
fontsize=14)
plt.yticks(np.arange(0, vert.size, 3),
names_vert[::3],
fontsize=14)
plt.xlabel('$\lambda$ [nm]', fontsize=17)
plt.ylabel('Spectrum index', fontsize=17)
cbar = plt.colorbar(im, orientation='horizontal')
cbar.set_label('Residuals in #$\sigma$', fontsize=20)
cbar.ax.tick_params(labelsize=13)
# cbar.ax.set_yticklabels(['{:.0f}'.format(x) for x in np.linspace(np.min(rho), np.max(rho), 10)]) #fontsize=16, weight='bold')
plt.gcf().tight_layout()
fig.tight_layout()
if self.sim and 1 == 1:
plt.savefig(parameters.OUTPUTS_THROUGHPUT_SIM +
'throughput_simb, ' + self.disperseur +
',résidus, version_' + parameters.PROD_NUM +
'.pdf')
elif 1 == 1:
plt.savefig(parameters.OUTPUTS_THROUGHPUT_REDUC +
'throughput_reduc, ' + self.disperseur +
',résidus, version_' + parameters.PROD_NUM +
'.pdf')
fig = plt.figure(figsize=[15, 10])
ax = fig.add_subplot(111)
axis_names = [str(i) for i in range(len(COV))]
plot_correlation_matrix_simple(ax,
compute_correlation_matrix(COV),
axis_names,
ipar=None)
err = np.zeros_like(D)
for j in range(len(self.names)):
for i in range(len(self.data_mag)):
#if self.new_lambda[i]==537.5 or self.new_lambda[i]==542.5:
# err[j * len(self.data_mag) + i] = 1
#else:
err[j * len(self.data_mag) + i] = np.sqrt(self.cov[j][i, i])
model = M_p @ Tinst
Err = (D - model) / err
plot_err(Err)
fig = plt.figure(figsize=[15, 15])
ax = fig.add_subplot(111)
ax.hist(Err, bins=np.arange(-10, 10, 1))
ax.set_title(
"Histogramme des résidus: (data - model) / err sur des données du HoloAmAg version "
+ parameters.PROD_NUM)
plt.xlabel('Ecart aux données', fontsize=14)
plt.grid(True)
test = [int(self.names[i][-16:-13]) for i in range(len(self.names))]
Kplot = [181, 186, 191, 196]
for i in range(len(Kplot)):
k = np.argmin(abs(np.array(test) - Kplot[i]))
fig = plt.figure(figsize=[15, 15])
ax = fig.add_subplot(111)
plt.plot(self.new_lambda,
model[k * len(self.new_lambda):(k + 1) *
len(self.new_lambda)],
c='red',
label='model')
plt.plot(self.new_lambda,
D[k * len(self.new_lambda):(k + 1) *
len(self.new_lambda)],
c='blue',
label='data')
plt.title("spectrum :" + str(Kplot[i]))
plt.grid(True)
plt.legend()
plt.show()
return Tinst, Tinst_err
def droites_Bouguer(method, sim, fileday, star, disperseur, binwidth, lambda_min, lambda_max, sigma_max, mag_diffmax,
filtre2_order, filtre3_order, filtre2avg_window, filtre2_window, filtre3avg_window, filtre3_window,
lambda_mid, DEBUG, sigma1, window1, LAMBDA_MIN, LAMBDA_MAX, trigger, filtre1_window, filtre1_order,
moy_raies, demi_taille_max, filtre1avg_window, filtre_global, order_global, debut_filtre_global,
debord_raies, seuilEAU_1, seuilEAU_2, bord_droit, bord_gauche, filtre4avg_window, save):
"""Fonction: effectue pour toute les photos d'une même etoile, le trace de magabs par bin de longueur d'onde
Entree:
37 params utilent pour une fonction interne cf Plot_magabsorbe_star
Sortie:
Trace des droites de Bouguer.
Trace de la magnitude absorbee en fonction de la masse d'air.
Tableau des coefficients ordonnees à l'origine et pente (il s'agit d'une matrice de taille nombre de bin x 2)
Tableaux des bins, des incertitudes, et de l'intensite par bin tabulee"""
"Recuperation des tableaux renvoyes par Plot_magabsorbe_star"
airmass, magabs, magabs_err, Bin, fluxlum_Binreel = Plot_magabsorbe_star(method, sim, fileday, star, disperseur,
binwidth, lambda_min, lambda_max,
sigma_max, mag_diffmax, filtre2_order,
filtre3_order, filtre2avg_window,
filtre2_window, filtre3avg_window,
filtre3_window, lambda_mid, DEBUG, sigma1,
window1, LAMBDA_MIN, LAMBDA_MAX, trigger,
filtre1_window, filtre1_order, moy_raies,
demi_taille_max, filtre1avg_window,
filtre_global, order_global,
debut_filtre_global, debord_raies,
seuilEAU_1, seuilEAU_2, bord_droit,
bord_gauche, filtre4avg_window)
"Definition de la fonction lineaire qu'on va utiliser pour tracer les droites de Bouguer."
def f(x, a, b):
return a * x + b
Z = np.linspace(0, 2.2, 1000) # nombre de points pour le trace des droites
coeff = np.zeros((len(magabs), 2)) # on initialise la liste des coefficients à la liste vide.
err = np.zeros(len(magabs)) # on initialise la liste des erreurs sur les ordonnees à l'origine à la liste vide.
err2 = np.zeros(len(magabs))
fig = plt.figure(figsize=[15, 10])
ax = fig.add_subplot(111)
new_lambda = 0.5 * (Bin[1:] + Bin[:-1])
"On trace les droites et on recupère l'ordonnee à l'origine pour chaque bin de longueur d'onde"
for i in range(len(magabs)):
popt, pcov = sp.optimize.curve_fit(f, airmass[i], magabs[i],
sigma=magabs_err[i]) # fit propageant les incertitudes
"On recupère les coefficients et les incertitudes types sur l'ordonnee à l'origine"
coeff[i][0], coeff[i][1] = popt[0], popt[1]
err[i] = np.sqrt(pcov[1][1])
err2[i] = np.sqrt(pcov[0][0])
"Affichage"
MAG = f(Z, *popt)
MAG_sup = f(Z, popt[0] + np.sqrt(pcov[0][0]), popt[1] - np.sqrt(pcov[1][1]))
MAG_inf = f(Z, popt[0] - np.sqrt(pcov[0][0]), popt[1] + np.sqrt(pcov[1][1]))
if i % 10 == 0:
plt.plot(Z, MAG, c=wavelength_to_rgb(new_lambda[i]))
plt.plot(Z, MAG_sup, c=wavelength_to_rgb(new_lambda[i]), linestyle=':')
plt.plot(Z, MAG_inf, c=wavelength_to_rgb(new_lambda[i]), linestyle=':')
plt.fill_between(Z, MAG_sup, MAG_inf, color=[wavelength_to_rgb(new_lambda[i])])
"la commande ci-dessus grise donne la bonne couleur la zone où se trouve la droite de Bouguer"
plt.scatter(airmass[i], magabs[i], c=[wavelength_to_rgb(new_lambda[i])], label=f'{Bin[i]}-{Bin[i + 1]} nm',
marker='o', s=30)
plt.errorbar(airmass[i], magabs[i], xerr=None, yerr=magabs_err[i], fmt='none', capsize=1,
ecolor=(wavelength_to_rgb(new_lambda[i])), zorder=2, elinewidth=2)
if sim:
plt.title('Droites de Bouguer, simulation, ' + disperseur, fontsize=18)
else:
plt.title('Droites de Bouguer, données, ' + disperseur, fontsize=18)
ax.get_xaxis().set_tick_params(labelsize=17)
ax.get_yaxis().set_tick_params(labelsize=14)
plt.xlabel('$\lambda$ (nm)', fontsize=17)
plt.ylabel('ln(flux)', fontsize=15)
plt.grid(True)
plt.legend(prop={'size': 12}, loc='upper right')
if save:
if sim == False:
plt.savefig('droites_bouguer_prod63/' + disperseur + '.png')
else:
plt.savefig('droites_bouguer_prod63_simu/' + disperseur + '.png')
fig = plt.figure(figsize=[15, 10])
Lambda = np.arange(new_lambda[0], new_lambda[-1], 1)
atmgrid = AtmosphereGrid(filename="../reduc_20170530_060_atmsim.fits")
b = atmgrid.simulate(300, 5, 0.03)
if sim:
plt.title('Transmission atmosphérique, simulation, ' + disperseur, fontsize=18)
else:
plt.title('Transmission atmosphérique, données, ' + disperseur, fontsize=18)
plt.plot(Lambda, np.exp(np.log(b(Lambda)) / 1.047), color='blue', label='transmission libradtran typique')
plt.plot(new_lambda, np.exp(coeff.T[0]), color='red', label='transmission atmosphérique droites bouguer')
plt.errorbar(new_lambda, np.exp(coeff.T[0]), xerr=None, yerr=err2 * np.exp(coeff.T[0]), fmt='none', capsize=1,
ecolor='red', zorder=2, elinewidth=2)
plt.fill_between(new_lambda, np.exp(coeff.T[0]) + err2 * np.exp(coeff.T[0]),
np.exp(coeff.T[0]) - err2 * np.exp(coeff.T[0]), color='red')
ax.get_xaxis().set_tick_params(labelsize=17)
ax.get_yaxis().set_tick_params(labelsize=14)
plt.xlabel('$\lambda$ (nm)', fontsize=17)
plt.ylabel('Transmission atmosphérique', fontsize=15)
plt.grid(True)
plt.legend(prop={'size': 15}, loc='lower right')
if save:
if sim == False:
plt.savefig('droites_bouguer_prod63/' + disperseur + '_atm.png')
else:
plt.savefig('droites_bouguer_prod63_simu/' + disperseur + '_atm.png')
fig = plt.figure(figsize=[15, 10])
plt.plot(new_lambda, np.exp(coeff.T[1]), c='black')
return (coeff, Bin, err, fluxlum_Binreel)