def GP(kernel, kernel_params, white=False):
'''
'''
if kernel == 'Basic':
w, a, t = kernel_params
if white:
if OLDGEORGE:
return george.GP(
WhiteKernel(w**2) + a**2 * Matern32Kernel(t**2))
else:
return george.GP(a**2 * Matern32Kernel(t**2),
white_noise=np.log(w**2),
fit_white_noise=True)
else:
return george.GP(a**2 * Matern32Kernel(t**2))
elif kernel == 'QuasiPeriodic':
w, a, g, p = kernel_params
if white:
if OLDGEORGE:
return george.GP(
WhiteKernel(w**2) + a**2 * ExpSine2Kernel(g, p))
else:
return george.GP(a**2 * ExpSine2Kernel(g, p),
white_noise=np.log(w**2),
fit_white_noise=True)
else:
return george.GP(a**2 * ExpSine2Kernel(g, p))
else:
raise ValueError('Invalid value for `kernel`.')
def GP(kernel, kernel_params, white = False):
if kernel == 'Basic':
w, a, t = kernel_params
if white:
return george.GP(WhiteKernel(w ** 2) + a ** 2 * Matern32Kernel(t ** 2))
else:
return george.GP(a ** 2 * Matern32Kernel(t ** 2))
elif kernel == 'QuasiPeriodic':
w, a, g, p = kernel_params
if white:
return george.GP(WhiteKernel(w ** 2) + a ** 2 * ExpSine2Kernel(g, p))
else:
return george.GP(a ** 2 * ExpSine2Kernel(g, p))
else:
raise ValueError('Invalid value for `kernel`.')
def create_fake_data(add_sin=False):
# fake 1D lightcurve.
# data is a constant, plus and airmass term, plus a random cloud
x = np.linspace(0, 8, 101)
# model cloud using Gaussian Process
kernel = 2 * Matern32Kernel(1.0)
gp = george.GP(kernel)
gp.compute(x, 0.1 * np.ones_like(x))
cloud = np.fabs(gp.sample(x))
# const + airmass + cloud
ybase = 20.0 + 5 * np.sin(np.pi * x / 12.) - cloud
#plt.plot(ybase)
#plt.show()
# stack many times to get 2D array [stars,hours]
nstars = 20
data = np.repeat(ybase[:, np.newaxis], nstars, axis=1).T
# add gaussian noise (amplitude around 1)
data += np.random.normal(loc=0, scale=0.8, size=data.shape)
# add sine wave to star 11 if required
if add_sin:
data[10, :] += 40.0 * np.sin(2.0 * np.pi * x / 3.5)
return data
def plot_gp_one_season(data):
"""
Plot DES light curve as a single season.
Parameters
----------
data : `pandas.DataFrame`
DataFrame object contaning a full DES light curve. This should be
obtained though [NAME OF LOCAL QUERY FUNC] or [NAME OF EASYACCESS
WRAPPER]
Returns
-------
fig : `plt.figure`
PyPlot canvas object, if originally provided as a parameter the
original object will be returned, otherwise new object is created.
"""
flux_arr = []
gdf = data.groupby('band')
for group, obs in gdf:
plt.errorbar(obs['mjd'],
obs['flux'],
yerr=obs['fluxerr'],
fmt='o',
c=band_colour(group),
label=group)
flux_norm = obs['flux'].mean()
obs['flux'] /= flux_norm
obs['fluxerr'] /= flux_norm
gp = george.GP(Matern32Kernel(np.exp(10)))
gp.compute(obs['mjd'], obs['fluxerr'])
p0 = gp.kernel.vector
opt.minimize(ll, p0, jac=grad_ll, args=(gp, obs))
t = np.linspace(data['mjd'].min(), data['mjd'].max(), 500)
mu, cov = gp.predict(obs['flux'], t)
std = np.sqrt(np.diag(cov))
mu *= flux_norm
std *= flux_norm
flux_arr.append(mu)
plt.fill_between(t,
mu + std,
mu - std,
color=band_colour(group),
alpha=0.4)
plt.plot(t, mu, color=band_colour(group))
plt.tight_layout()
plt.legend(fontsize=16, loc='best')
def gp_des_lc(data):
df = pd.DataFrame({
'mjd': [],
'flux': [],
'season': [],
'band': [],
'snid': []
})
gdf = data.groupby(('season', 'band'))
for group, obs in gdf:
mask = ((edges['season'] == group[0]) &
(edges['field'] == obs['field'].values[0]))
edge_query = edges[mask]
min_edge = edge_query['min_mjd'].values[0]
if (int(obs['mjd'].min()) > min_edge or int(obs['mjd'].max()) <
(min_edge + 149)):
print(min_edge, min_edge + 149)
print(obs['mjd'].min(), obs['mjd'].max())
return
flux_norm = obs['flux'].mean()
obs['flux'] /= flux_norm
obs['fluxerr'] /= flux_norm
if flux_norm == 0.0000:
print('Flux avarage is 0')
return
gp = george.GP(Matern32Kernel(np.exp(10)))
gp.compute(obs['mjd'], obs['fluxerr'])
p0 = gp.kernel.vector
opt.minimize(ll, p0, jac=grad_ll, args=(gp, obs))
# t = np.linspace(0, 149, 23) + min_edge
t = np.linspace(0, 149, 46) + min_edge
try:
mu, cov = gp.predict(obs['flux'], t)
except:
print('Could not interpolate LC')
return
mu *= flux_norm
mu *= (10**1.56)
temp_df = pd.DataFrame({'mjd': t, 'flux': mu})
temp_df['season'] = group[0]
temp_df['band'] = group[1]
temp_df['snid'] = obs['snid'].values[0]
df = pd.concat((df, temp_df))
df.to_sql('real_nonia_interp_46', engine, if_exists='append', index=False)
def __init__(self, lc, dist_factor=10.0, time_factor=0.1, tau_frac=0.25):
self.time = lc.time
mu = np.median(lc.flux)
self.flux = lc.flux / mu - 1.0
self.ferr = lc.ferr / mu
# Convert to parts per thousand.
self.flux *= 1e3
self.ferr *= 1e3
# Estimate the kernel parameters.
tau = tau_frac * estimate_tau(self.time, self.flux)
self.kernel = np.var(self.flux) * Matern32Kernel(tau**2)
self.gp = george.GP(self.kernel, solver=george.HODLRSolver)
self.K_0 = self.gp.get_matrix(self.time)
self.gp.compute(self.time, self.ferr, seed=1234)
self.alpha = self.gp.solver.apply_inverse(self.flux)
# Compute the likelihood of the null model.
self.ll0 = self.lnlike()
# vae = load_model(ModelDir + 'fullAE_' + fileOut + '.hdf5')
encoder = load_model(ModelDir + 'EncoderP' + str(num_para) + ClID + '_' +
fileOut + '.hdf5')
decoder = load_model(ModelDir + 'DecoderP' + str(num_para) + ClID + '_' +
fileOut + '.hdf5')
history = np.loadtxt(ModelDir + 'TrainingHistoryP' + str(num_para) + ClID +
'_' + fileOut + '.txt')
import george
from george.kernels import Matern32Kernel # , ConstantKernel, WhiteKernel, Matern52Kernel
# kernel = ConstantKernel(0.5, ndim=num_para) * Matern52Kernel(0.9, ndim=num_para) + WhiteKernel( 0.1, ndim=num_para)
#kernel = Matern32Kernel(1000, ndim=num_para)
# kernel = Matern32Kernel( [1000,2000,2000,1000,1000], ndim=num_para)
kernel = Matern32Kernel([1000, 4000, 3000, 1000, 2000], ndim=num_para)
#kernel = Matern32Kernel( [1,0.5,1,1.4,0.5], ndim=num_para)
#kernel = Matern32Kernel(ndim=num_para)
# This kernel (and more importantly its subclasses) computes
# the distance between two samples in an arbitrary metric and applies a radial function to this distance.
# metric: The specification of the metric. This can be a float, in which case the metric is considered isotropic
# with the variance in each dimension given by the value of metric.
# Alternatively, metric can be a list of variances for each dimension. In this case, it should have length ndim.
# The fully general not axis aligned metric hasn't been implemented yet
# PLOTTING y_train and y_test
# import pandas as pd
# plt.figure(431)
def mangle_spectrum_function(self, show=False):
if (not hasattr(self, 'mangledspec_path')):
self.create_mangledspec_folder()
elif (not hasattr(self, 'raw_spec')):
self.load_raw_spec()
ratios = []
ratios_err = []
fitted_phot_list = []
fitted_photerr_list = []
wls_eff = []
used_filters = []
outMJD_ratios = []
outMJD_ratios_err = []
outMJD_fitted_phot_list = []
outMJD_fitted_photerr_list = []
outMJD_wls_eff = []
outMJD_used_filters = []
outwls_filters_wls_eff = []
outwls_filters = []
outwls_fitted_phot_list = []
outwls_fitted_photerr_list = []
for filt in self.avail_filters:
fitted_phot = self.phot4mangling['%s_fitflux' % filt].values[0]
fitted_phot_err = self.phot4mangling['%s_fitfluxerr' %
filt].values[0]
inMJDrange_pnt = self.phot4mangling['%s_inrange' %
filt].values[0]
lam_avg, lam_eff, raw_phot, raw_phot_err, min_wls, max_wls = self.band_flux(
filt, type_spec2use='raw_spec')
if filt in ['Bessell_R', 'sdss_z', "sdss_z'"]:
condition = (lam_avg > min(
self.raw_spec['wls'])) & (lam_avg < max(
self.raw_spec['wls'])) & (fitted_phot > 0.)
else:
condition = (lam_avg > min(
self.raw_spec['wls'])) & (lam_avg < max(
self.raw_spec['wls'])) & (fitted_phot > 0.)
if (not inMJDrange_pnt):
outMJD_fitted_phot_list.append(fitted_phot)
outMJD_fitted_photerr_list.append(fitted_phot_err)
outMJD_ratios.append(fitted_phot / raw_phot)
outMJD_ratios_err.append(
prop_err_div(fitted_phot, raw_phot, fitted_phot_err,
0.))
outMJD_wls_eff.append(lam_eff)
outMJD_used_filters.append(filt)
else:
if condition:
fitted_phot_list.append(fitted_phot)
fitted_photerr_list.append(fitted_phot_err)
ratios.append(fitted_phot / raw_phot)
ratios_err.append(
prop_err_div(fitted_phot, raw_phot,
fitted_phot_err, 0.))
wls_eff.append(lam_eff)
used_filters.append(filt)
else:
if (fitted_phot > 0.):
outwls_filters.append(filt)
outwls_filters_wls_eff.append(lam_avg)
outwls_fitted_phot_list.append(fitted_phot)
outwls_fitted_photerr_list.append(fitted_phot_err)
ratios = np.array(ratios)
wls_eff = np.array(wls_eff)
ratios_err = np.array(ratios_err)
used_filters = np.array(used_filters)
if len(ratios) < 1:
print('Impossible to mangle ', self.spec_file, 'used',
used_filters, '\nUnused', outMJD_used_filters)
return None
else:
if len(self.raw_spec['wls']) > 10**4:
# GP struggle to handle such a big number of points
int_fraction = int(len(self.raw_spec['wls']) / 5000.)
print(
'This spectrum has a huge amount of data points(%i), Im chopping a %i th of them'
% (len(self.raw_spec['wls']), int_fraction))
full_wls = self.raw_spec['wls'][::int_fraction]
else:
full_wls = self.raw_spec['wls']
norm_wls = np.median(full_wls)
full_wls_normed = full_wls / norm_wls
wls_eff_normed = np.array(wls_eff) / norm_wls
offset = 1.
norm = np.mean(ratios)
ratios_normed = np.array(ratios) / norm - offset
ratios_err_normed = np.array(ratios_err) / norm
def ll(p):
gp.set_parameter_vector(p)
scale = np.exp(
gp.get_parameter_dict()['kernel:k2:metric:log_M_0_0'])
if scale < 0.09:
return np.inf
else:
return -gp.lnlikelihood(ratios_normed, quiet=False) #
def grad_ll(p):
gp.set_parameter_vector(p)
return -gp.grad_lnlikelihood(ratios_normed, quiet=False)
k = np.var(ratios_normed) * Matern32Kernel(0.2)
wls_eff_normedT = np.atleast_2d(wls_eff_normed).T
gp = george.GP(k)
gp.compute(wls_eff_normedT, (ratios_err_normed))
p0 = gp.get_parameter_vector()
results = opt.minimize(ll, p0, jac=grad_ll)
# print ('SCALE', np.exp(results.x))
# print ('#######', wls_eff,'\n', ratios,'\n', ratios_err,'\n', ratios_normed,'\n', norm)
gp.set_parameter_vector(results.x)
mu, cov = gp.predict(ratios_normed, full_wls_normed)
std = np.sqrt(np.diag(cov))
spec_number = self.phot4mangling.spec_mjd.values[
0] # self.phot4mangling.index.values[0]
fig = plt.figure(1, figsize=(14, 6))
plt.rc('font', family='serif')
plt.rc('xtick', labelsize=13)
plt.rc('ytick', labelsize=13)
ax1 = plt.subplot2grid((5, 1), (0, 0), rowspan=2)
for f, w, r, rerr in zip(used_filters, wls_eff, ratios,
ratios_err):
if 'swift' in f:
flabel = f.split('_')[1] + '(Swift)'
else:
flabel = f.split('_')[1]
ax1.errorbar(w, r, yerr=rerr, marker=mark_dict[f], ms=8,
mfc=color_dict[f], mec=color_dict[f], linestyle='None', \
ecolor=color_dict[f], label='%s' % flabel)
ax1.errorbar(full_wls,
norm * (mu + offset),
color='orange',
label='Mangling\nfunction')
ax1.fill_between(full_wls,
norm * ((mu + offset) - std),
norm * ((mu + offset) + std),
color='orange',
alpha=0.3)
ax1.set_ylabel('Photometric Flux/\nSynthetic Flux',
fontsize=13)
ax1.set_title('Example of mangling using a spectrum of %s' %
self.snname,
fontsize=15)
ax1.set_ylim(min(ratios) * (0.9), max(ratios) * (1.1))
ax1.set_xlim(1600., 10300.)
plt.tick_params(axis='x', labelbottom=False)
ax1.legend(ncol=2, fontsize=12, loc='best')
if len(self.raw_spec['wls']) > 10**4:
# GP struggle to handle such a big number of points
mu_full = np.interp(self.raw_spec['wls'],
self.raw_spec['wls'][::int_fraction],
mu)
std_full = np.interp(self.raw_spec['wls'],
self.raw_spec['wls'][::int_fraction],
std)
else:
mu_full = mu
std_full = std
mangled_spec = self.raw_spec['flux'] * norm * (mu_full +
offset)
mangled_spec_err = (
(norm * (std_full) * self.raw_spec['flux'])**2 +
((norm *
(mu_full + offset)) * self.raw_spec['fluxerr'])**2)**0.5
self.mangled_spec = np.array([
a for a in zip(self.raw_spec['wls'], mangled_spec,
mangled_spec_err)
],
dtype=[('wls', '<f8'),
('flux', '<f8'),
('fluxerr', '<f8')])
self.fitted_phot_dict = {
'eff_wls': wls_eff,
'fitted_phot': fitted_phot_list,
'fitted_phot_err': fitted_photerr_list,
'used_filters': used_filters
}
self.mangling_mask = norm * (mu_full + offset)
mangled_phot_list = []
for filt in used_filters:
lam_eff, mangled_phot = self.band_flux(
filt, type_spec2use='mangled')
mangled_phot_list.append(mangled_phot)
self.magled_photometry_dict = {
'eff_wls': wls_eff,
'fitted_phot': mangled_phot_list,
'used_filters': used_filters
}
ax2 = plt.subplot2grid((5, 1), (2, 0), rowspan=3)
ax2.errorbar(self.mangled_spec['wls'],
self.mangled_spec['flux'],
lw=0.9,
color='k',
label='Mangled Spectrum')
mask_neg = self.mangled_spec['flux'] < 0.
ax2.errorbar(self.mangled_spec['wls'][mask_neg],
self.mangled_spec['flux'][mask_neg],
lw=3.9,
color='m',
label='Mangled Spectrum')
ax2.errorbar(self.raw_spec['wls'],
self.raw_spec['flux'] * norm,
lw=0.6,
color='r',
alpha=1,
label='Uncalibrated Spectrum\n+ Offset')
ax2.errorbar(np.concatenate([self.fitted_phot_dict['eff_wls'], outwls_filters_wls_eff]),
np.concatenate([self.fitted_phot_dict['fitted_phot'], outwls_fitted_phot_list]), \
yerr=np.concatenate([self.fitted_phot_dict['fitted_phot_err'],
outwls_fitted_photerr_list]),
marker='o', mfc='grey', mec='grey', ms=7,
ecolor='grey', linestyle='None', label='Photometric flux\nfrom LC fitting')
ax2.errorbar(self.magled_photometry_dict['eff_wls'],
self.magled_photometry_dict['fitted_phot'],
marker='^',
mfc='r',
mec='r',
linestyle='None',
ms=7,
label='Flux synthesized\nfrom mangled spec')
ax2.fill_between(
self.mangled_spec['wls'],
self.mangled_spec['flux'] - self.mangled_spec['fluxerr'],
self.mangled_spec['flux'] + self.mangled_spec['fluxerr'],
color='grey',
alpha=0.3)
ax2.set_ylabel(
r'Flux ($\mathrm{erg}$ $\mathrm{s^{-1} cm^{-2}} \mathrm{\AA} $)',
fontsize=13)
ax2.set_xlabel(r'Wavelength ($\mathrm{\AA}$)', fontsize=13)
ax2.set_xlim(1600., 10300.)
ax2.legend(ncol=1, fontsize=13, loc='upper right')
plt.subplots_adjust(hspace=0.3, wspace=0.1)
fig.savefig(self.mangledspec_path +
'%.2f_mangled_spec.pdf' % spec_number,
bbox_inches='tight')
if show: plt.show()
plt.close(fig)
self.save_mangled_spectrum()
return {'wls': self.raw_spec['wls'], 'flux': mangled_spec}
def run_2DGP_GRID(GP2DIM_Class, y_data_nonan, y_data_nonan_err, x1_data_norm, x2_data_norm,\
kernel_wls_scale, kernel_time_scale, extrap_mjds, prior=False, points=np.nan, values=np.nan):
""" ## for NUV extention: extrap_mjds = grid_ext_columns
## for spectra augmentation:
extrap_mjds = grid_ext.columns.values
if (len(extrap_mjds)>200):
extrap_mjds = grid_ext.columns.values[:200]
if (max(extrap_mjds-min(extrap_mjds))>200):
extrap_mjds = extrap_mjds[extrap_mjds-min(extrap_mjds)<200]
tot_iteration = int(len(extrap_mjds)/slot_size+1)
print (tot_iteration)"""
# TRAINING: X, y, terr
norm1 = GP2DIM_Class.grid_norm_info['norm1']
norm2 = GP2DIM_Class.grid_norm_info['norm2']
if prior:
from george.modeling import Model
class Model_2dim(Model):
parameter_names = ()
def get_value(self, t):
points_eval = np.array([tup for tup in zip(t[:, 0], t[:, 1])])
grid_z1 = griddata(points,
values,
points_eval,
method='nearest')
grid_z1[np.isnan(grid_z1)] = 0.
#plt.plot(t[:,0]*norm1, grid_z1, '-b', label='PRIOR')
return grid_z1
mean_model = Model_2dim()
X = np.vstack((x1_data_norm, x2_data_norm)).T
y = y_data_nonan
yerr = y_data_nonan_err
kernel_mix = Matern32Kernel([kernel_wls_scale, kernel_time_scale], ndim=2)
kernel2dim = np.var(y) * kernel_mix #+ 0.3*np.var(y)*kernel2*kernel1
if prior:
gp = george.GP(
kernel2dim,
mean=mean_model) #, fit_mean=True, fit_white_noise=True)
else:
gp = george.GP(kernel2dim)
gp.compute(X, yerr)
wls_normed_range = np.sort(
np.concatenate((np.arange(1600., 3000., 40), np.arange(
3000., 9000., 10), np.arange(
9000., 10350., 40)))) / GP2DIM_Class.grid_norm_info['norm1']
mu_fill_resh = []
std_fill_resh = []
slot_size = 3
tot_iteration = int(len(extrap_mjds) / slot_size + 1)
for j in range(int(len(extrap_mjds) / slot_size + 1)):
mjd_normed_range = ((extrap_mjds[j * slot_size:(j + 1) * slot_size]) -
GP2DIM_Class.grid_norm_info['offset2']
) / GP2DIM_Class.grid_norm_info['norm2']
x1_fill = [] #np.random.permutation(np.linspace(0,1., N))
x2_fill = [] #np.random.permutation(np.linspace(0,1., N))
for i in wls_normed_range:
for k in mjd_normed_range:
x1_fill.append(i)
x2_fill.append(k)
x1_fill = np.array(x1_fill)
x2_fill = np.array(x2_fill)
X_fill = np.vstack((x1_fill, x2_fill)).T
if GP2DIM_Class.verbose:
print(j, 'of', int(len(extrap_mjds) / slot_size + 1))
frac_tot_iteration = int(20. * (j + 1) / tot_iteration)
#print('[','*'*frac_tot_iteration,' '*(20-frac_tot_iteration),']' + ' %i of %i'%(slot_size*(j+1),slot_size*tot_iteration)+' spec extrapolated', end='\r')
mu_iter, cov_iter = (gp.predict(y, X_fill, return_cov=True))
std_iter = np.sqrt(np.diag(cov_iter))
count = 0
for mj in mjd_normed_range:
fig = plt.figure(figsize=(8, 2))
plt.subplot(1, slot_size, count + 1)
plt.plot(norm1 * x1_fill[x2_fill == mj],
mu_iter[x2_fill == mj],
'-k',
label='PREDICTION')
if prior:
points_eval = np.array([
tup for tup in zip(x1_fill[x2_fill == mj], x2_fill[x2_fill
== mj])
])
grid_z1 = griddata(points,
values,
points_eval,
method='nearest')
grid_z1[np.isnan(grid_z1)] = 0.
plt.plot(norm1 * x1_fill[x2_fill == mj],
grid_z1,
'-b',
label='PRIOR')
plt.legend()
plt.show()
plt.close(fig)
count = count + 1
mu_resh_iter = mu_iter.reshape(len(wls_normed_range),
len(mjd_normed_range))
std_resh_iter = std_iter.reshape(len(wls_normed_range),
len(mjd_normed_range))
if mu_fill_resh == []:
mu_fill_resh = np.copy(mu_resh_iter)
std_fill_resh = np.copy(std_resh_iter)
else:
mu_fill_resh = np.concatenate([mu_fill_resh, mu_resh_iter], axis=1)
std_fill_resh = np.concatenate([std_fill_resh, std_resh_iter],
axis=1)
print(
'[', '*' * frac_tot_iteration, ' ' * (20 - frac_tot_iteration),
']' + '%i of %i' % (slot_size * (j + 1), slot_size * tot_iteration) +
'spec extrapolated')
mu_fill = mu_fill_resh.reshape(len(wls_normed_range) * len(extrap_mjds))
std_fill = std_fill_resh.reshape(len(wls_normed_range) * len(extrap_mjds))
mjd_normed_range = (extrap_mjds - GP2DIM_Class.grid_norm_info['offset2']
) / GP2DIM_Class.grid_norm_info['norm2']
x1_fill = [] #np.random.permutation(np.linspace(0,1., N))
x2_fill = [] #np.random.permutation(np.linspace(0,1., N))
for i in wls_normed_range:
for k in mjd_normed_range:
x1_fill.append(i)
x2_fill.append(k)
x1_fill = np.array(x1_fill)
x2_fill = np.array(x2_fill)
print('EXTENDING SPECTRA BETWEEN:')
print('WLS:', min(x1_fill * GP2DIM_Class.grid_norm_info['norm1']),
max(x1_fill * GP2DIM_Class.grid_norm_info['norm1']))
print('MJD:', min(x2_fill * GP2DIM_Class.grid_norm_info['norm2']),
max(x2_fill * GP2DIM_Class.grid_norm_info['norm2']))
return (x1_fill, x2_fill, mu_fill, std_fill)
def GP_interpolation_mangle(self,
wls_eff,
ratios,
ratios_err,
min_scale,
optimization=True):
if len(self.ext_spec['wls']) > 10**4:
# GP struggle to handle such a big number of points
int_fraction = int(len(self.ext_spec['wls']) / 5000.)
print(
'This spectrum has a huge amount of data points(%i), Im chopping a %i th of them'
% (len(self.ext_spec['wls']), int_fraction))
full_wls = self.ext_spec['wls'][::int_fraction]
else:
full_wls = self.ext_spec['wls']
norm_wls = np.median(full_wls)
full_wls_normed = full_wls / norm_wls
wls_eff_normed = np.array(wls_eff) / norm_wls
offset = 1.
norm = np.mean(ratios)
ratios_normed = np.array(ratios) / norm - offset
ratios_err_normed = np.array(ratios_err) / norm
if len(ratios_normed) < 1:
return np.ones(len(full_wls_normed)) * np.nan, np.ones(
len(full_wls_normed)) * np.nan
else:
def ll(p):
# print (np.exp(p))
if (np.exp(p)[1] < 5 * 10**-3): # |(np.exp(p)[1]>10**5):
return np.inf
else:
gp.set_parameter_vector(p)
return -gp.lnlikelihood(ratios_normed, quiet=False) #
def grad_ll(p):
gp.set_parameter_vector(p)
return -gp.grad_lnlikelihood(ratios_normed, quiet=False)
k = np.var(ratios_normed) * Matern32Kernel(0.3)
wls_eff_normedT = np.atleast_2d(wls_eff_normed).T
gp = george.GP(k)
gp.compute(wls_eff_normedT, (ratios_err_normed))
if optimization:
try:
p0 = gp.get_parameter_vector()
results = opt.minimize(ll, p0, jac=grad_ll)
print('SCALE:', '%.4f' % np.exp(results.x[1]))
except:
pass
print('*** GP optimization failed ***' * 10)
# print ('results', np.exp(results.x))
mu, cov = gp.predict(ratios_normed, full_wls_normed)
std = np.sqrt(np.diag(cov))
if len(self.ext_spec['wls']) > 10**4:
# GP struggle to handle such a big number of points
mu_full = np.interp(self.ext_spec['wls'],
self.ext_spec['wls'][::int_fraction],
mu)
std_full = np.interp(self.ext_spec['wls'],
self.ext_spec['wls'][::int_fraction],
std)
else:
mu_full = mu
std_full = std
return norm * (mu_full + offset), np.abs(norm * (std_full))
def main():
"""
Meat of the code
"""
insn = sys.argv[1]
inband = sys.argv[2]
par = np.loadtxt(pt + 'tmax_dm15.dat', dtype='string')
lc = read_inp_lc('../files/SDSS_SN00013689.DAT')
print lc[0]
#return 0
lc = np.loadtxt('../files/SN13689_data.DAT')
mag1 = lc[:, 1]
ph1 = lc[:, 0]
mag = lc[:, 1]
magerr = lc[:, 2]
magerr /= max(mag)
mag /= max(mag)
# Set up the Gaussian process.
param = float(sys.argv[3])
kernel = RationalQuadraticKernel(param, 100) + Matern32Kernel(
param) #ExpSquaredKernel(param)
gp = george.GP(kernel)
#lc=set_arr(insn, inband)
def nll(p):
# Update the kernel parameters and compute the likelihood.
gp.kernel[:] = p
ll = gp.lnlikelihood(mag, quiet=True)
# The scipy optimizer doesn't play well with infinities.
return -ll if np.isfinite(ll) else 1e25
# And the gradient of the objective function.
def grad_nll(p):
# Update the kernel parameters and compute the likelihood.
gp.kernel[:] = p
return -gp.grad_lnlikelihood(mag, quiet=True)
#ph=lc['MJD']-tbmax
#condition for second maximum
##TODO: GUI for selecting region
#cond=(ph>=10.0) & (ph<=40.0)
#define the data in the region of interest
"""
ph1=ph[cond]
mag=lc[inband][cond]
magerr=lc['e_'+inband][cond]
"""
print "Fitting with george"
#print max(mag)
# Pre-compute the factorization of the matrix.
gp.compute(ph1, magerr)
print gp.lnlikelihood(mag), gp.grad_lnlikelihood(mag)
gp.compute(ph1, magerr)
if sys.argv[4] == 'mle':
p0 = gp.kernel.vector
results = op.minimize(nll, p0, jac=grad_nll)
gp.kernel[:] = results.x
print gp.lnlikelihood(mag), gp.kernel.value
#vertically stack it for no apparent reason
arr = np.vstack([ph1, mag, magerr]).T
#define x array for applying the fit
t = np.linspace(ph1.min(), ph1.max(), 500)
#print t.min()#gp.lnlikelihood(mag)
mu, cov = gp.predict(mag, t)
#condition for peak
mpeak = (mu == min(mu))
#calculate the standard deviation from covariance matrix
std = np.sqrt(np.diag(cov))
#as a check for parameters, print the array at max out
print t[mpeak][0], max(mu), std[mpeak][0]
#return 0
arr = np.vstack([t, mu, std]).T
np.savetxt("mle_gpfit_SDSS_new.txt", arr)
plt.errorbar(ph1, mag, magerr, fmt=".k", capsize=2, label='data')
plt.plot(t, mu, 'k:', label='best fit')
plt.fill_between(t, mu - std, mu + std, alpha=0.3)
plt.legend(loc=0)
#plt.ylim(plt.ylim()[::-1])
plt.xlabel("MJD")
plt.ylabel("flux")
plt.savefig("mle_SDSS_gpfit_george.pdf")
plt.show()
def LCfit_withGP_xfilter(self, filt, minMJD=None, maxMJD=None):
if not hasattr(self, "phot"):
self.load()
def ll(p):
gp.set_parameter_vector(p)
return -gp.log_likelihood(flux_norm,
quiet=False) # OFEK: changed from deprecated -gp.lnlikelihood(flux_norm, quiet=False)
# gradient of the liklihood for optimisation of the kernel size
def grad_ll(p):
gp.set_parameter_vector(p)
return -gp.grad_log_likelihood(flux_norm,
quiet=False) # OFEK: changed from deprecated -gp.grad_lnlikelihood(flux_norm, quiet=False)
mjd_peak = self.get_mjdpeak()
mjd_spectra = self.get_spec_mjd()
if minMJD is None:
# minMJD= min([mjd_peak-15.,np.min(self.clipped_phot['MJD'])]) #np.min(self.clipped_phot['MJD'])
minMJD = min([min(mjd_spectra), np.min(self.clipped_phot['MJD'])]) # np.min(self.clipped_phot['MJD'])
if maxMJD is None:
# maxMJD= min([max(mjd_spectra),np.max(self.phot['MJD'])])
maxMJD = min([minMJD + 300., np.max(self.phot['MJD'])])
print('MJD range', maxMJD - minMJD)
new_mjd = np.arange(minMJD - 1., maxMJD + 1., 0.05)
LC_filt_extended = self.get_singlefilter(filt, extended_clipped=True)
mjd = LC_filt_extended['MJD']
mjdT = np.atleast_2d(mjd).T
orig_flux = (LC_filt_extended['Flux'])
orig_flux_err = LC_filt_extended['Flux_err']
if TRY_LOG:
flux_gp = np.log(LC_filt_extended['Flux'])
flux_err_gp = LC_filt_extended['Flux_err'] / LC_filt_extended['Flux']
else:
flux_gp = (LC_filt_extended['Flux'])
flux_err_gp = LC_filt_extended['Flux_err']
keep = ~np.isnan(flux_gp) * ~np.isnan(flux_err_gp)
mjdT, flux_gp, flux_err_gp = mjdT[keep], flux_gp[keep], flux_err_gp[
keep] # OFEK: kick nans (from t==t_explosion)
norm = np.max(flux_gp) # np.median(flux_gp)
flux_norm = flux_gp / norm
err_flux_norm = flux_err_gp / norm
if (self.snname in list(kernel_settings_dict.keys())):
if filt in list(kernel_settings_dict[self.snname].keys()):
set_scale = kernel_settings_dict[self.snname][filt]['scale']
set_optimization = kernel_settings_dict[self.snname][filt]['opt']
set_mean = kernel_settings_dict[self.snname][filt]['mean']
else:
set_scale, set_optimization, set_mean = set_default_kernel_settings(filt)
else:
set_scale, set_optimization, set_mean = set_default_kernel_settings(filt)
if set_mean:
set_fit_mean = True
else:
set_fit_mean = False
k = np.var(flux_norm) * Matern32Kernel(set_scale)
gp = george.GP(k, mean=set_mean, fit_mean=set_fit_mean) # ,
# white_noise=10**-5, fit_white_noise=True)
gp.compute(mjdT, err_flux_norm)
if set_optimization:
p0 = gp.get_parameter_vector()
results = opt.minimize(ll, p0, jac=grad_ll)
print('results ', filt, np.exp(gp.get_parameter_vector()))
mu_gp, cov = gp.predict(flux_norm, new_mjd)
std_gp = np.sqrt(np.diag(cov))
if TRY_LOG:
mu = np.exp(mu_gp * norm)
std = np.abs(mu * std_gp * norm)
else:
mu = mu_gp * norm
std = std_gp * norm
# if filt != 'Bessell_V':
mu_mjdspec, cov_mjdspec = gp.predict(flux_norm, mjd_spectra)
std_mjdspec = np.sqrt(np.diag(cov_mjdspec))
mu_mjdspec = (mu_mjdspec * norm)
std_mjdspec = (std_mjdspec * norm)
sudo_pts = LC_filt_extended['FilterSet'] == 'SUDO_PTS'
self.fitted_phot[filt] = {'clipped_extended_data': [mjd, orig_flux, orig_flux_err, sudo_pts],
'fit_highcadence': [new_mjd, mu, std],
'fit_mjdspec': [mjd_spectra, mu_mjdspec, std_mjdspec]}
return None
gp = george.GP(kernel)
# Here we draw plot to show what kind of function we have when the gaussian
# process have no knowledge of our objective value of our function. We draw
# 10 samples below:
fig, ax = plt.subplots()
for pred in gp.sample(t, 10):
ax.plot(t, pred)
ax.set_ylim(-9, 7)
ax.set_xlabel("x")
ax.set_ylabel("y")
plt.show()
# If we choose a different kernel, the functions we sample will end up looking
# different.
matern_kernel = Matern32Kernel(1.0)
matern_gp = george.GP(matern_kernel)
fig, ax = plt.subplots()
for pred in matern_gp.sample(t, 10):
ax.plot(t, pred)
ax.set_ylim(-9, 7)
ax.set_xlabel("x")
ax.set_ylabel("y")
plt.show()
#
# Adding Knowledge
#
# Generate some fake, noisy data. In real world, this is the objective value
# from the function we are trying to model.
x = 10 * np.sort(np.random.rand(10))
d = os.path.dirname
sys.path.insert(0, d(d(os.path.abspath(__file__))))
import george
from george.kernels import ExpSquaredKernel, Matern32Kernel, CosineKernel
np.random.seed(12345)
experiments = [
("exponential squared", [
("-k", "$l=0.5$", ExpSquaredKernel(0.5)),
("--k", "$l=1$", ExpSquaredKernel(1.0)),
(":k", "$l=2$", ExpSquaredKernel(2.0)),
]),
("quasi-periodic", [
("-k", "$l=2,\,P=3$", Matern32Kernel(2.0) * CosineKernel(3.0)),
("--k", "$l=3,\,P=3$", Matern32Kernel(3.0) * CosineKernel(3.0)),
(":k", "$l=3,\,P=1$", Matern32Kernel(3.0) * CosineKernel(1.0)),
])
]
t = np.linspace(0, 10, 500)
h, w = len(experiments) * 4, 6
fig, axes = pl.subplots(len(experiments), 1, figsize=(w, h), sharex=True)
fig.subplots_adjust(left=0.1,
bottom=0.1,
right=0.96,
top=0.98,
wspace=0.0,
hspace=0.05)
for ax, (name, runs) in zip(axes, experiments):
def plot_gp_all_seasons(data):
"""
Plot a horizontal multi-panel light curve plot for DES objects.
Parameters
----------
data : `pandas.DataFrame`
DataFrame object contaning a full DES light curve. This should be
obtained though [NAME OF LOCAL QUERY FUNC] or [NAME OF EASYACCESS
WRAPPER]
Returns
-------
fig : `plt.figure`
PyPlot canvas object, if originally provided as a parameter the
original object will be returned, otherwise new object is created.
"""
fig, ax = plt.subplots(1, 4, figsize=(20, 5), sharey=True)
gdf = data.groupby(('season', 'band'))
for group, obs in gdf:
i = int(group[0]) - 1
if i == 3:
label = group[1]
else:
label = None
ax[i].errorbar(obs['mjd'],
obs['flux'],
yerr=obs['fluxerr'],
fmt='o',
c=band_colour(group[1]),
label=label)
flux_norm = obs['flux'].mean()
obs['flux'] /= flux_norm
obs['fluxerr'] /= flux_norm
gp = george.GP(Matern32Kernel(np.exp(10)))
gp.compute(obs['mjd'], obs['fluxerr'])
p0 = gp.kernel.vector
opt.minimize(ll, p0, jac=grad_ll, args=(gp, obs))
t = np.linspace(obs['mjd'].min(), obs['mjd'].max(), 500)
mu, cov = gp.predict(obs['flux'], t)
std = np.sqrt(np.diag(cov))
mu *= flux_norm
std *= flux_norm
ax[i].fill_between(t,
mu + std,
mu - std,
color=band_colour(group[1]),
alpha=0.4)
ax[i].plot(t, mu, color=band_colour(group[1]))
fig.tight_layout()
plt.legend(fontsize=16, loc='best')
return ax
# kernels = [1.0 * Matern(length_scale=length_scaleParameter, length_scale_bounds=(length_scaleBoundMin, length_scaleBoundMax),
# nu=1.5)]
# from george import kernels
# k1 = 66.0**2 * kernels.ExpSquaredKernel(67.0**2)
# k2 = 2.4**2 * kernels.ExpSquaredKernel(90**2) * kernels.ExpSine2Kernel(2.0 / 1.3**2, 1.0)
# k3 = 0.66**2 * kernels.RationalQuadraticKernel(0.78, 1.2**2)
# k4 = 0.18**2 * kernels.ExpSquaredKernel(1.6**2) + kernels.WhiteKernel(0.19)
# kernel = k1 + k2 + k3 + k4
# kernel = k1
from george.kernels import Matern32Kernel #, ConstantKernel, WhiteKernel
# kernel = ConstantKernel(0.5, ndim=5) * Matern32Kernel(0.5, ndim=5) + WhiteKernel(0.1, ndim=5)
kernel = Matern32Kernel(0.5, ndim=4)
# hmf = np.loadtxt('Data/HMF_5Para.txt')
# ----------------------------- i/o ------------------------------------------
###################### PARAMETERS ##############################
original_dim = params.original_dim # 2549
#intermediate_dim3 = params.intermediate_dim3 # 1600
intermediate_dim2 = params.intermediate_dim2 # 1024
intermediate_dim1 = params.intermediate_dim1 # 512
intermediate_dim0 = params.intermediate_dim0 # 256
intermediate_dim = params.intermediate_dim # 256
latent_dim = params.latent_dim # 10
if a > (x[mp] + 29.97) and a < (x[mp] + 30.03):
print 'Phase, Mag, Mag-err:', a - x[mp], b, c
# Plot the function, the prediction and the 95% confidence interval based on
# the MSE
############# george part #########
#print flux, dyf, ph400
Yg = np.array(flux)
Y_err = np.array(dyfg)
Xg = np.array(ph400)
norm = Yg.max()
Yg /= norm
Y_err /= norm
gp = george.GP(Matern32Kernel(500)) # + WhiteKernel(0.001))
gp.compute(Xg, Y_err)
p0 = gp.get_parameter_vector()
def ll(p):
gp.set_parameter_vector(p)
return -gp.lnlikelihood(Yg, quiet=True)
def grad_ll(p):
gp.set_parameter_vector(p)
return -gp.grad_lnlikelihood(Yg, quiet=True)
results = opt.minimize(ll, p0, jac=grad_ll)
