def run_signal_scan(
box, btags, sms, # analysis region
num_mr_bins, mr_max, # fit range
k_ell=200, k_alpha=200, # kernel parameters
steps=1000, lr=0.001, # fit parameters
mu_min=-1.0, mu_max=5.0, mu_step=0.05, mu_true=1.0,
verbose=False):
"""
Scans the signal strength from mu_min to mu_max.
At each point, performs a fit and gets the resulting GP.
"""
data = get_data(box, btags, num_mr_bins, mr_max)
data_sig = get_data(box, btags, num_mr_bins, mr_max,
proc=sms)
U = data['u']
# fix the random signal realization for all runs
S_mean = data_sig['y']
true_signal = np.random.poisson(S_mean.numpy() * mu_true)
Y = data['y'] + torch.Tensor(true_signal)
kernel = gp.SquaredExponentialKernel(k_ell, k_alpha, fixed=True)
Gs = []
for mu in np.arange(mu_min, mu_max, mu_step):
if verbose:
print("Fitting GP with signal = {:.3f}".format(mu))
G = gp.PoissonGPWithSignal(kernel, U, Y, S_mean,
num_true_signal=true_signal.sum())
G.signal = torch.nn.Parameter(torch.Tensor([mu]), requires_grad=False)
G.fit(num_steps=steps, lr=lr, verbose=False,
parameters=[G.g])
Gs.append(G)
return Gs
def fit_interpolation(
box, btags, # analysis region
num_mr_bins, mr_max, # fit range
interp_bins, # bins to predict
k_ell=200, k_alpha=200, # kernel parameters
steps=1000, lr=0.001, # fit parameters
hmc_epsilon=0.0001, hmc_L_max=10, # HMC parameters
num_samples=40000, verbose=False):
"""
Fits the GP model (with no signal) on the chosen analysis box
and interpolates to make predictions for blinded bins
"""
kernel = gp.SquaredExponentialKernel(k_ell, k_alpha)
data = get_data(box, btags, num_mr_bins, mr_max)
U = data['u']
Y = data['y']
fit_index = [i for i in range(U.size(0)) if i not in interp_bins]
fit_U = U[fit_index]
fit_Y = Y[fit_index]
# perform initial fit
G = gp.PoissonLikelihoodGP(kernel, fit_U, fit_Y,
hmc_epsilon=hmc_epsilon, hmc_L_max=hmc_L_max)
G.fit(num_steps=steps, lr=lr, verbose=verbose)
G.sample(num_samples=num_samples, verbose=verbose)
return G
def fit_annealer(box, btags, num_mr_bins, mr_max,
sms, mu_true=1.0, mu_test=1.0,
k_ell=200, k_alpha=200,
par_scheduler=None, last_beta=None,
num_runs=1, num_hmc_steps=1, num_beta=100,
verbose=False, best_params=None):
data = get_data(box, btags, num_mr_bins, mr_max)
data_sig = get_data(box, btags, num_mr_bins, mr_max, sms)
kernel = gp.SquaredExponentialKernel(k_ell, k_alpha)
U = data['u']
Y = data['y']
S_mean = data_sig['y']
true_signal = np.random.poisson(S_mean.numpy() * mu_true)
Y = Y + torch.Tensor(true_signal)
# parameter scheduler based on best params from BayesOpt
if par_scheduler is None:
if best_params is None:
raise ValueError(
"Please provide HMC parameters or a scheduler function")
betas = sorted([b for b in best_params])
def scheduler(beta):
if beta <= 0:
return best_params[0]
if beta >= 1:
return best_params[1]
# find the two beta nodes we are between
beta_index = bisect.bisect_left(betas, beta)
beta_low = betas[beta_index - 1]
beta_high = betas[beta_index]
frac = (beta - beta_low) / (beta_high - beta_low)
# interpolate between parameters
log_eps_low, K_max_low = best_params[beta_low]
log_eps_high, K_max_high = best_params[beta_high]
log_eps = log_eps_low + frac * (
log_eps_high - log_eps_low)
K_max = K_max_low + frac * (K_max_high - K_max_low)
#ret = float(log_eps_low), int(K_max_low)
ret = float(log_eps), int(K_max)
return ret
par_scheduler = scheduler
G = gp.AnnealingPoissonGP(kernel, U, Y, S_mean, mu=mu_test,
hmc_par_scheduler=par_scheduler,
num_true_signal=true_signal.sum())
G.sample(num_runs, num_hmc_steps, num_beta,
verbose=verbose, print_every=100, last_beta=last_beta)
return G
def fit(
box, btags, # analysis region
num_mr_bins, mr_max, # fit range
k_ell=200, k_alpha=200, # kernel parameters
steps=1000, lr=0.001, # fit parameters
hmc_epsilon=0.0001, hmc_L_max=10, chains=1, # HMC parameters
num_samples=40000, verbose=False, best_g=None,
return_g=False): # specify return_g=True to return best value of g
"""
Fits the GP model (with no signal) on the chosen analysis box
and returns samples from the GP posterior.
"""
if best_g is None:
# perform initial fit
data = get_data(box, btags, num_mr_bins, mr_max)
U = data['u']
Y = data['y']
kernel = gp.SquaredExponentialKernel(k_ell, k_alpha)
G = gp.PoissonLikelihoodGP(kernel, U, Y,
hmc_epsilon=hmc_epsilon, hmc_L_max=hmc_L_max)
G.fit(num_steps=steps, lr=lr, verbose=verbose)
best_g = G.g.data
if return_g: # we just want to fit and not sample
return best_g
n = int(num_samples / chains)
args = [box, btags, num_mr_bins, mr_max, k_ell, k_alpha,
hmc_epsilon, hmc_L_max, n, best_g]
if chains == 1:
t0 = time.time()
results = _do_sample(-1, *args)
t1 = time.time()
print("Total time: {}".format(t1 - t0))
return results
else:
# Sample in parallel.
# We have to use Pool.starmap to pass multiple arguments
# to the sampling function.
args_list = [[i] + args for i in range(chains)]
p = mp.Pool(chains)
t0 = time.time()
results = p.starmap(_do_sample, args_list)
t1 = time.time()
# Note: when I tested this I did not see any speedup
# from parallelization.
print("Total time: {}".format(t1 - t0))
return np.concatenate(results)
def fit_signal(
box, btags, sms, # analysis region
num_mr_bins, mr_max, # fit range
k_ell=200, k_alpha=200, # kernel parameters
steps=1000, lr=0.001, # fit parameters
hmc_epsilon=0.0001, hmc_L_max=10, # HMC parameters
num_samples=40000,
mu_true=1.0, # signal strength
best_pars=None, return_pars=False,
verbose=False,
kernel_gp=None, scale=1.0, adam=False):
"""
Performs a signal + background fit using a fixed signal shape.
Samples from the GP posterior and returns the GP object.
Access the samples via G.preds_from_samples().
(note: gave up on multiprocessing for now.)
"""
data = get_data(box, btags, num_mr_bins, mr_max, scale=scale)
data_sig = get_data(box, btags, num_mr_bins, mr_max,
proc=sms, scale=scale)
U = data['u']
Y = data['y']
# inject signal into the data
S_mean = data_sig['y']
true_signal = np.random.poisson(S_mean.numpy() * mu_true)
Y = Y + torch.Tensor(true_signal)
if kernel_gp is None:
kernel = gp.SquaredExponentialKernel(k_ell, k_alpha, fixed=True)
else:
kernel = kernel_gp
G = gp.PoissonGPWithSignal(kernel, U, Y, S_mean,
hmc_epsilon=hmc_epsilon, hmc_L_max=hmc_L_max,
num_true_signal=true_signal.sum())
if best_pars is None:
G.fit(num_steps=steps, lr=lr, verbose=verbose, adam=adam)
best_pars = (G.g.data, G.signal.data)
if return_pars:
return best_pars
G.g = torch.nn.Parameter(best_pars[0].clone())
G.signal = torch.nn.Parameter(best_pars[1].clone())
G.sample(num_samples, verbose=verbose)
return G
def _do_sample(seed, box, btags, num_mr_bins, mr_max, k_ell, k_alpha,
hmc_epsilon, hmc_L_max, num_samples, g, verbose=True):
if seed >= 0:
print("Sampling with seed {}".format(seed))
np.random.seed(seed)
torch.manual_seed(seed)
data = get_data(box, btags, num_mr_bins, mr_max)
U = data['u']
Y = data['y']
kernel = gp.SquaredExponentialKernel(k_ell, k_alpha)
G = gp.PoissonLikelihoodGP(kernel, U, Y,
hmc_epsilon=hmc_epsilon, hmc_L_max=hmc_L_max)
# input g should be a torch Tensor
G.g = torch.nn.Parameter(g.clone())
samples = G.sample(num_samples=num_samples, verbose=verbose)
if seed >= 0:
print("Seed {} done".format(seed))
return samples
def optim():
values = []
evals = []
gps = []
nj = 100
for j in range(0, nj):
evals.append([])
gps.append([])
print optime, j
fail = False
#kernel = GP.SquaredExponentialKernel([2.], 1.)#, [1e-2, 1e-2])
#gp = GP.GaussianProcess(kernel, orig_bounds, noise=[1e-4, [1e-5]], noiseGP=True)
#gp.explore(2, objective)
kernel = GP.SquaredExponentialKernel([2., 0.1], 1.) #, [1e-2, 1e-2])
def constraint(x):
return sum(x) - 30.6
gp = GP.GaussianProcess(kernel,
orig_bounds,
noise=[1e-4, [1e-5, 1e-3]],
noiseGP=True,
cons=constraint)
gp.explore(4, objective)
#orig_bounds = np.array([[-3., 3], [-3, 3.]])
#kernel = GP.SquaredExponentialKernel([1., 1], 100.)#, [1e-2, 1e-2])
#gp = GP.GaussianProcess(kernel, orig_bounds, noise=[sig**2, sig**2])
#gp.explore(4, test_func)
x1 = np.linspace(gp.bounds[0, 0], gp.bounds[0, 1], 40)
xs = x1.reshape(-1, 1)
#x2 = np.linspace(gp.bounds[1,0], gp.bounds[1,1], 40)
#x1, x2 = np.meshgrid(x1, x2)
#xs = np.vstack((x1.flatten(), x2.flatten())).T
ei = GP.ExpectedImprovement(gp)
# acquisition function improvements:
# * noise: using EI with std for now
# * batch
# * finite budget
print
for i in range(0, 100):
#x = ei.optimize()
try:
x = ei.optimize()
except:
fail = True
break
evals[-1].append(gp.data_min())
gps[-1].append(gp.posterior_min())
print 'ei choice:', i, x
#eix = ei.evaluate(xs)
#plt.plot(x1, eix.reshape(x1.shape))
#plt.contourf(x1, x2, eix.reshape(x1.shape), 100)
#plt.colorbar()
##plt.savefig('ei_{}.pdf'.format(i))
##plt.clf()
#plt.show()
#mu, cov = gp.evaluate(xs)
##mu, cov = gp.noiseGP[1].exponential(xs)
##plt.plot(x1, mu*gp.noise[0])
##plt.show()
##std = np.diag(cov)**0.5
#plt.contourf(x1, x2, mu.reshape(x1.shape), 100)
#plt.colorbar()
###plt.savefig('mu_{}.pdf'.format(i))
###plt.clf()
#plt.show()
#plt.contourf(x1, x2, std.reshape(x1.shape), 1000)
#plt.colorbar()
#plt.savefig('cov_{}.pdf'.format(i))
#plt.clf()
y, yd, yn, ydn = objective_single(x)
#y, yd, yn, ydn = test_func(x)
gp.train(x, y, yd, yn, ydn)
print
if not fail:
values.append(gp.y)
else:
evals = evals[:-1]
gps = gps[:-1]
with open('{}.pkl'.format(optime), 'w') as f:
pkl.dump([evals, gps, values], f)
def optim():
orig_bounds = np.array([[0., 1.], [0, 1], [0, 1], [0, 1]])
L, sigma = 0.5, 0.001
#L, sigma = 0.3, 0.003
mean = 0.0092
if grad:
kernel = GP2.SquaredExponentialKernel([L, L, L, L], sigma)
gp = GP2.GaussianProcess(kernel,
orig_bounds,
noise=[5e-9, [1e-9, 1e-7, 1e-8, 1e-7]],
noiseGP=True,
cons=constraint)
ei = GP2.ExpectedImprovement(gp)
else:
kernel = GP.SquaredExponentialKernel([L, L, L, L], sigma)
gp = GP.GaussianProcess(kernel,
orig_bounds,
noise=5e-9,
noiseGP=True,
cons=constraint)
ei = GP.ExpectedImprovement(gp)
kernel = GP2.SquaredExponentialKernel([L, L, L, L], sigma)
gp2 = GP2.GaussianProcess(kernel,
orig_bounds,
noise=[5e-9, [1e-9, 1e-7, 1e-8, 1e-7]],
noiseGP=True,
cons=constraint)
#ei = GP2.ExpectedImprovement(gp)
assert os.path.exists(stateFile)
state = load_state()
#for index in range(0, len(state['state'])):
# if get_state(state, index) != 'DONE':
# x = state['points'][index]
# res = evaluate(x, state, index)
# state['evals'].append(res)
# update_state(state, 'DONE', index)
# exit(1)
n = len(state['points'])
if len(state['evals']) < len(state['points']):
n -= 1
m = 8
x = state['points'][:n]
y = [res[0] - mean for res in state['evals']][:n]
yd = [res[2:6] for res in state['evals']][:n]
yn = [res[1] for res in state['evals']][:n]
ydn = [res[6:10] for res in state['evals']][:n]
#print yd
#print ydn
#print yd
#print ydn
#exit(1)
gp2.train(x[:m], y[:m], yd[:m], yn[:m], ydn[:m])
for i in range(m, n):
gp2.train(x[i], y[i], yd[i], yn[i], ydn[i])
# GRAD based optim
#pmin = gp2.posterior_min()
#print pmin[1] + mean, pmin[2]
#exit(1)
# GRAD based optim
x = x[:m]
y = y[:m]
yn = yn[:m]
yd = yd[:m]
ydn = ydn[:m]
if grad:
gp.train(x, y, yd, yn, ydn)
else:
gp.train(x, y, yn)
for i in range(m, 25):
dmin = gp.data_min()
pmin = gp.posterior_min()
#print 'data min:', dmin[0], dmin[1] + mean
#print 'posterior min:', pmin[0], pmin[1] + mean
print '{},{},{},{}'.format(i, ','.join(np.char.mod('%f', pmin[0])),
pmin[1] + mean, pmin[2])
x = ei.optimize()
if grad:
y, _ = gp2.evaluate_grad(x)
y, yd = y[0], y[1:]
yn, ydn = gp2.get_noise(x)
yn = yn[0]
ydn = np.array(ydn).flatten()
z = np.random.randn() * np.sqrt(yn) + y
zd = np.random.randn(ydn.shape[0]) * np.sqrt(ydn) + yd
gp.train(x, z, zd, yn, ydn)
else:
y, _ = gp2.evaluate(x)
y = y[0]
yn = gp2.get_noise(x)[0]
z = np.random.randn() * np.sqrt(yn) + y
gp.train(x, z, yn)
def optim():
values = []
evals = []
gps = []
#nj = 1500
#ne = 35
nj = 1000
ne = 50
for j in range(0, nj):
evals.append([])
gps.append([])
print j
fail = False
#kernel = GP.SquaredExponentialKernel([2.], 1.)#, [1e-2, 1e-2])
#gp = GP.GaussianProcess(kernel, orig_bounds, noise=[1e-4, [1e-5]], noiseGP=True)
#gp.explore(2, objective)
#kernel = GP.SquaredExponentialKernel([2., 0.1], 1.)#, [1e-2, 1e-2])
#def constraint(x):
# return sum(x) - 30.6
#gp = GP.GaussianProcess(kernel, orig_bounds, noise=[1e-4, [1e-5, 1e-3]], noiseGP=True, cons=constraint)
#gp.explore(4, objective)
#kernel = GP.SquaredExponentialKernel([1., 1.], 100)#, [1e-2, 1e-2])
kernel = GP.SquaredExponentialKernel([1., 1.], 50.)#, [1e-2, 1e-2])
gp = GP.GaussianProcess(kernel, orig_bounds, noise=[sig**2, [sig**2, sig**2]], noiseGP=True)
#gp = GP.GaussianProcess(kernel, orig_bounds, noise=[sig**2, sig**2], noiseGP=False)
gp.explore(4, objective_single)
#gp.explore(100, objective_single)
x1 = np.linspace(gp.bounds[0,0], gp.bounds[0,1], 40)
xs = x1.reshape(-1,1)
x2 = np.linspace(gp.bounds[1,0], gp.bounds[1,1], 40)
x1, x2 = np.meshgrid(x1, x2)
xs = np.vstack((x1.flatten(), x2.flatten())).T
ei = GP.ExpectedImprovement(gp)
# acquisition function improvements:
# * noise: using EI with std for now
# * batch
# * finite budget
#print
for i in range(0, ne):
print j, i
x = ei.optimize()
#try:
# x = ei.optimize()
#except:
# fail = True
# break
#x = ei.optimize()
evals[-1].append(gp.data_min())
gps[-1].append(gp.posterior_min())
#print 'ei choice:', i, x
#print 'data min', evals[-1][-1]
#print 'posterior min', gps[-1][-1]
#eix = ei.evaluate(xs)
#plt.contourf(x1, x2, eix.reshape(x1.shape), 100)
#plt.colorbar()
#plt.savefig('debug/ei_{}.pdf'.format(i))
#plt.clf()
#plt.show()
#mu, cov = gp.evaluate(xs)
#plt.contourf(x1, x2, mu.reshape(x1.shape), 100)
#plt.colorbar()
#plt.savefig('debug/mu_{}.pdf'.format(i))
#plt.clf()
#plt.show()
y, yd, yn, ydn = objective_single(x)
#y, yd, yn, ydn = test_func(x)
gp.train(x, y, yd, yn, ydn)
#print
if not fail:
values.append(gp.y)
else:
evals = evals[:-1]
gps = gps[:-1]
with open('{}.pkl'.format(optime), 'w') as f:
pkl.dump([evals, gps, values], f)
def optim():
orig_bounds = np.array([[0., 1.], [0, 1], [0, 1], [0, 1]])
L, sigma = 0.5, 0.001
#L, sigma = 0.3, 0.003
mean = 0.0092
if grad:
kernel = GP2.SquaredExponentialKernel([L, L, L, L], sigma)
gp = GP2.GaussianProcess(kernel,
orig_bounds,
noise=[5e-9, [1e-9, 1e-7, 1e-8, 1e-7]],
noiseGP=True,
cons=constraint)
ei = GP2.ExpectedImprovement(gp)
else:
kernel = GP.SquaredExponentialKernel([L, L, L, L], sigma)
gp = GP.GaussianProcess(kernel,
orig_bounds,
noise=5e-9,
noiseGP=True,
cons=constraint)
ei = GP.ExpectedImprovement(gp)
kernel = GP2.SquaredExponentialKernel([L, L, L, L], sigma)
gp2 = GP2.GaussianProcess(kernel,
orig_bounds,
noise=[5e-9, [1e-9, 1e-7, 1e-8, 1e-7]],
noiseGP=True,
cons=constraint)
#ei = GP2.ExpectedImprovement(gp)
assert os.path.exists(stateFile)
state = load_state()
#for index in range(0, len(state['state'])):
# if get_state(state, index) != 'DONE':
# x = state['points'][index]
# res = evaluate(x, state, index)
# state['evals'].append(res)
# update_state(state, 'DONE', index)
# exit(1)
n = len(state['points'])
if len(state['evals']) < len(state['points']):
n -= 1
m = 8
x = state['points'][:n]
y = [res[0] - mean for res in state['evals']][:n]
yd = [res[2:6] for res in state['evals']][:n]
yn = [res[1] for res in state['evals']][:n]
ydn = [res[6:10] for res in state['evals']][:n]
#print yd
#print ydn
#print yd
#print ydn
#exit(1)
gp2.train(x[:m], y[:m], yd[:m], yn[:m], ydn[:m])
mus = []
varis = []
for i in range(m, n):
mu, vari = gp2.evaluate(x[i])
mus.append(mu[0] + mean)
varis.append(vari[0, 0]**0.5)
gp2.train(x[i], y[i], yd[i], yn[i], ydn[i])
# GRAD based optim
#pmin = gp2.posterior_min()
#print pmin[1] + mean, pmin[2]
mus = np.array(mus)
varis = np.array(varis)
cm = plt.cm.get_cmap('RdYlBu')
z = np.arange(0, len(mus))
plt.locator_params(axis='x', numticks=4)
plt.locator_params(axis='y', numticks=4)
sc = plt.scatter(varis, mus, c=z, s=100)
for i, x in enumerate(z):
plt.annotate(x, (varis[i] + 1e-5, mus[i]), fontsize=16)
plt.colorbar(sc, ticks=z + 1)
d = 0.0001
plt.xlim([varis.min() - d, varis.max() + d])
plt.ylim([mus.min() - d, mus.max() + d])
plt.xlabel('standard deviation of GP at evaluation')
plt.ylabel('mean of GP at evaluation')
plt.show()
exit(1)
# GRAD based optim
x = x[:m]
y = y[:m]
yn = yn[:m]
yd = yd[:m]
ydn = ydn[:m]
if grad:
gp.train(x, y, yd, yn, ydn)
else:
gp.train(x, y, yn)
for i in range(m, 25):
dmin = gp.data_min()
pmin = gp.posterior_min()
#print 'data min:', dmin[0], dmin[1] + mean
#print 'posterior min:', pmin[0], pmin[1] + mean
print '{},{},{},{}'.format(i, ','.join(np.char.mod('%f', pmin[0])),
pmin[1] + mean, pmin[2])
x = ei.optimize()
if grad:
y, _ = gp2.evaluate_grad(x)
y, yd = y[0], y[1:]
yn, ydn = gp2.get_noise(x)
yn = yn[0]
ydn = np.array(ydn).flatten()
z = np.random.randn() * np.sqrt(yn) + y
zd = np.random.randn(ydn.shape[0]) * np.sqrt(ydn) + yd
gp.train(x, z, zd, yn, ydn)
else:
y, _ = gp2.evaluate(x)
y = y[0]
yn = gp2.get_noise(x)[0]
z = np.random.randn() * np.sqrt(yn) + y
gp.train(x, z, yn)
def optim():
orig_bounds = np.array([[0., 1.], [0, 1], [0, 1], [0, 1]])
L, sigma = 0.5, 0.001
#L, sigma = 0.3, 0.003
mean = 0.0092
kernel = GP.SquaredExponentialKernel([L, L, L, L], sigma)
gp = GP.GaussianProcess(kernel,
orig_bounds,
noise=[5e-9, [1e-9, 1e-7, 1e-8, 1e-7]],
noiseGP=True,
cons=constraint)
ei = GP.ExpectedImprovement(gp)
assert os.path.exists(stateFile)
state = load_state()
for index in range(0, len(state['state'])):
if get_state(state, index) != 'DONE':
x = state['points'][index]
res = evaluate(x, state, index)
state['evals'].append(res)
update_state(state, 'DONE', index)
exit(1)
print state
x = state['points']
y = [res[0] - mean for res in state['evals']]
yd = [res[2:6] for res in state['evals']]
yn = [res[1] for res in state['evals']]
ydn = [res[6:10] for res in state['evals']]
#print yd
#print ydn
#print yd
#print ydn
#exit(1)
gp.train(x, y, yd, yn, ydn)
for i in range(len(state['points']), 100):
dmin = gp.data_min()
pmin = gp.posterior_min()
print 'data min:', dmin[0], dmin[1] + mean
print 'posterior min:', pmin[0], pmin[1] + mean
x = ei.optimize()
print 'ei choice:', i, x
#exit(1)
state['points'].append(x)
state['state'].append('BEGIN')
save_state(state)
res = evaluate(x, state)
print 'result:', res
state['evals'].append(res)
update_state(state, 'DONE')
exit(1)
resm = [x for x in res]
resm[0] = res[0] - mean
gp.train(x, *resm)
#eix = ei.evaluate(xs)
#plt.plot(x1, eix.reshape(x1.shape))
#plt.contourf(x1, x2, eix.reshape(x1.shape), 100)
#plt.colorbar()
##plt.savefig('ei_{}.pdf'.format(i))
##plt.clf()
#plt.show()
#mu, cov = gp.evaluate(xs)
##mu, cov = gp.noiseGP[1].exponential(xs)
##plt.plot(x1, mu*gp.noise[0])
##plt.show()
##std = np.diag(cov)**0.5
#plt.contourf(x1, x2, mu.reshape(x1.shape), 100)
#plt.colorbar()
###plt.savefig('mu_{}.pdf'.format(i))
###plt.clf()
#plt.show()
#plt.contourf(x1, x2, std.reshape(x1.shape), 1000)
#plt.colorbar()
#plt.savefig('cov_{}.pdf'.format(i))
#plt.clf()
print