def cuda_curve_fit_sync(*args, **kwargs):
x = fitter_torch.curve_fit(*args, **kwargs)
torch.cuda.synchronize()
return x
for i in range(nfits):
p, q = scipy.optimize.curve_fit(data.testfunc_lin_np, data_lin.x,
data_lin.y)
t1_stop = time.time()
comp_time_lin.append(t1_stop - t1_start)
print(p)
print(q)
print(t1_stop - t1_start)
#pytorch CPU
t1_start = time.time()
x = torch.from_numpy(data_lin.x)
y = torch.from_numpy(data_lin.y)
for i in range(nfits):
p, q, j = fitter_torch.curve_fit(
data.testfunc_lin_torch, x, y,
[torch.ones(2, requires_grad=True, dtype=torch.float64)])
t1_stop = time.time()
comp_time_lin.append(t1_stop - t1_start)
print(p)
print(q)
print(j)
print(t1_stop - t1_start)
#pytorch GPU
t1_start = time.time()
if torch.cuda.is_available():
x = torch.from_numpy(data_lin.x).cuda()
y = torch.from_numpy(data_lin.y).cuda()
t1_start = time.time()
for i in range(nfits):
def benchmark_epx(pointlist):
params = []
covs = []
params_true = []
optimizers = []
npoints = []
weights_index = []
for idx, el in enumerate(pointlist):
comp_time_exp = []
#weights = np.random.rand(nfits,el)*2
#weights = np.random.poisson(lam=1., size=(nfits, el))
weights = bootstrap_weights(nfits, el) + eps
weights[0] = np.ones(el)
exp_fitter = bfgsfitter(data.testfunc_exp)
data_exp.setxy(el, sigma0)
# bfgsfitter
t1_start = time.time()
for i in range(nfits):
p, q = exp_fitter.curve_fit(data_exp.x, data_exp.y)
t1_stop = time.time()
comp_time_exp.append(t1_stop - t1_start)
print(p.numpy())
print(q.numpy())
weights_index.append(0)
params.append(p.numpy())
covs.append(np.diag(q.numpy()))
params_true.append(data_exp.params)
optimizers.append("Tensorflow_BFGS")
npoints.append(el)
# second fit after initializiation
t1_start = time.time()
for i in range(nfits):
p, q = exp_fitter.curve_fit(data_exp.x, data_exp.y)
t1_stop = time.time()
comp_time_exp.append(t1_stop - t1_start)
# scipy
t1_start = time.time()
for i in range(nfits):
p, q = scipy.optimize.curve_fit(data.testfunc_exp_np,
data_exp.x,
data_exp.y,
sigma=sigma0 / weights[i])
params.append(p)
covs.append(np.diag(q))
params_true.append(data_exp.params)
optimizers.append("Scipy_LM")
npoints.append(el)
weights_index.append(i)
chisq.append(
np.sum(((data.testfunc_exp_np(data_exp.x, *p) - data_exp.y) /
sigma0)**2))
chisq_transformed.append(
np.sum(weights[i] *
((data.testfunc_exp_np(data_exp.x, *p) - data_exp.y) /
sigma0)**2))
t1_stop = time.time()
comp_time_exp.append(t1_stop - t1_start)
print(p)
print(q)
# pytorch CPU
t1_start = time.time()
x = torch.from_numpy(data_exp.x)
y = torch.from_numpy(data_exp.y)
for i in range(nfits):
p, q = fitter_torch.curve_fit(
data.testfunc_exp_torch,
x,
y, [torch.ones(2, requires_grad=True, dtype=torch.float64)],
weights=torch.from_numpy(weights[i] / sigma0**2))
params.append(p[0].detach().numpy())
covs.append(np.diag(q.numpy()))
params_true.append(data_exp.params)
optimizers.append("PyTorch_LBFGS")
npoints.append(el)
weights_index.append(i)
chisq.append(
np.sum((
(data.testfunc_lin_np(data_exp.x, *p[0].detach().numpy()) -
data_exp.y) / sigma0)**2))
chisq_transformed.append(
np.sum(weights[i] * (
(data.testfunc_exp_np(data_exp.x, *p[0].detach().numpy()) -
data_exp.y) / sigma0)**2))
t1_stop = time.time()
comp_time_exp.append(t1_stop - t1_start)
print(p[0].detach().numpy())
print(q.numpy())
print(t1_stop - t1_start)
explist.append(comp_time_exp)
name = "exp"
df = create_benchmark_df(name, optimizers, params_true, params, covs,
npoints, weights_index)
return explist, df
def benchmark_linear(pointList):
params = []
covs = []
params_true = []
optimizers = []
npoints = []
weights_index = []
for idx, el in enumerate(pointList):
comp_time_lin = []
lin_fitter = bfgsfitter(data.testfunc_lin)
data_lin.setxy(el, sigma0)
lin_parameter_list_org.append(data_lin.params)
#weights = np.random.rand(nfits,el)*2
weights = bootstrap_weights(nfits, el) + eps
weights[0] = np.ones(el)
p0 = np.random.normal(data_lin.params, lin_initial_guess_sigma,
[nfits, 2])
# bfgsfitter
t1_start = time.time()
for i in range(nfits):
p, q = lin_fitter.curve_fit(
data_lin.x,
data_lin.y,
weights=tf.constant(weights[i] / sigma0**2),
initial_parameters=tf.cast(p0[i], tf.float32))
weights_index.append(i)
params.append(p.numpy())
covs.append(np.diag(q.numpy()))
params_true.append(data_lin.params)
optimizers.append("Tensorflow_BFGS")
npoints.append(el)
chisq.append(
np.sum(((data.testfunc_lin_np(data_lin.x, *p.numpy()) -
data_lin.y) / sigma0)**2))
chisq_transformed.append(
np.sum(weights[i] *
((data.testfunc_lin_np(data_lin.x, *p.numpy()) -
data_lin.y) / sigma0)**2))
t1_stop = time.time()
comp_time_lin.append(t1_stop - t1_start)
print(p)
print(q)
print(t1_stop - t1_start)
# second fit after initializiation
t1_start = time.time()
for i in range(nfits):
p, q = lin_fitter.curve_fit(
data_lin.x,
data_lin.y,
weights=tf.constant(weights[i] / sigma0**2),
initial_parameters=tf.cast(p0[i], tf.float32))
t1_stop = time.time()
comp_time_lin.append(t1_stop - t1_start)
print(t1_stop - t1_start)
# scipy
t1_start = time.time()
for i in range(nfits):
p, q = scipy.optimize.curve_fit(data.testfunc_lin_np,
data_lin.x,
data_lin.y,
sigma=sigma0 / np.sqrt(weights[i]),
p0=p0[i])
params.append(p)
covs.append(np.diag(q))
params_true.append(data_lin.params)
optimizers.append("Scipy_LM")
npoints.append(el)
weights_index.append(i)
chisq.append(
np.sum(((data.testfunc_lin_np(data_lin.x, *p) - data_lin.y) /
sigma0)**2))
chisq_transformed.append(
np.sum(weights[i] *
((data.testfunc_lin_np(data_lin.x, *p) - data_lin.y) /
sigma0)**2))
t1_stop = time.time()
comp_time_lin.append(t1_stop - t1_start)
print(p)
print(q)
print(t1_stop - t1_start)
# pytorch CPU
t1_start = time.time()
x = torch.from_numpy(data_lin.x)
y = torch.from_numpy(data_lin.y)
for i in range(nfits):
p, q = fitter_torch.curve_fit(
data.testfunc_lin_torch,
x,
y, [torch.ones(2, requires_grad=True, dtype=torch.float64)],
weights=torch.from_numpy(weights[i] / sigma0**2))
params.append(p[0].detach().numpy())
covs.append(np.diag(q.numpy()))
params_true.append(data_lin.params)
optimizers.append("PyTorch_LBFGS")
npoints.append(el)
weights_index.append(i)
chisq.append(
np.sum((
(data.testfunc_lin_np(data_lin.x, *p[0].detach().numpy()) -
data_lin.y) / sigma0)**2))
chisq_transformed.append(
np.sum(weights[i] * (
(data.testfunc_lin_np(data_lin.x, *p[0].detach().numpy()) -
data_lin.y) / sigma0)**2))
t1_stop = time.time()
comp_time_lin.append(t1_stop - t1_start)
print(p)
print(q)
print(t1_stop - t1_start)
# pytorch GPU
"""
t1_start = time.time()
if torch.cuda.is_available():
x = torch.from_numpy(data_lin.x).cuda()
y = torch.from_numpy(data_lin.y).cuda()
t1_start = time.time()
for i in range(nfits):
p, q, j = fitter_torch.curve_fit(data.testfunc_lin_torch, x, y, [
torch.ones(2, requires_grad=True, dtype=torch.float64, device="cuda:0")])
t1_stop = time.time()
comp_time_lin.append(t1_stop - t1_start)
"""
linlist.append(comp_time_lin)
name = "lin"
df = create_benchmark_df(name, optimizers, params_true, params, covs,
npoints, weights_index)
return linlist, df
def benchmark_bootstrap(npoints,
nfits,
nbootstrap,
testfunc,
sigma_data,
sigma_initial_guess,
generate_params,
nparams,
xmin=-1,
xmax=1,
weights=None,
testfunc_tf=None,
testfunc_torch=None):
frames = []
params = []
errors = []
params_true = []
fit_idx = []
fitter_name = []
bs_mean = []
bs_median = []
bs_std = []
bs_rmsexp = []
bs_rmsexp_filtered = []
bs_mean_filtered = []
bs_median_filtered = []
bs_std_filtered = []
bs_number_accepted = []
bs_number_valid = []
chisq = []
number_points = []
t = []
if weights is None:
weights = bootstrap_weights(nbootstrap, npoints)
if testfunc_tf is None:
testfunc_tf = testfunc
if testfunc_torch is None:
testfunc_torch = testfunc
x = np.linspace(xmin, xmax, npoints).astype(np.float32)
fitterTF = bfgsfitter(testfunc_tf)
for ifit in range(nfits):
print("Fit ", ifit)
params_true_0 = generate_params()
y = np.random.normal(testfunc(x, *params_true_0),
sigma_data).astype(np.float32)
p0 = np.random.normal(params_true_0, sigma_initial_guess,
[nbootstrap, nparams]).astype(np.float32)
if "Tensorflow_BFGS" in fitters:
p, q = fitterTF.curve_fit(x,
y,
initial_parameters=p0[0],
weights=1 / sigma_data**2)
#print(p.numpy()); print(q.numpy())
pn = p.numpy()
params.append(pn)
errors.append(np.sqrt(np.diag(q.numpy())))
chisq.append(
np.sum((testfunc(x, *pn) - y)**2) / sigma_data**2 /
(npoints - nparams))
params_true.append(params_true_0)
number_points.append(npoints)
fit_idx.append(ifit)
fitter_name.append("Tensorflow_BFGS")
t0 = time.time()
df0, mean_filtered, median_filtered, std_filtered, _ = fitterTF.curve_fit_BS(
x,
y,
weights=weights,
init_params=p0,
sigma0=sigma_data,
nbootstrap=nbootstrap)
t1 = time.time()
frames.append(df0)
df0["fit_idx"] = ifit
df0["time"] = (t1 - t0) / nbootstrap
t.append(t1 - t0)
mean = []
median = []
std = []
rmsexp = []
rmsexp_filtered = []
df0_filtered = df0.query("is_accepted")
for a, b in enumerate(params_true_0):
df0[str.format("params_true_{}", a)] = b
mean.append(df0[str.format("params_{}", a)].mean())
median.append(df0[str.format("params_{}", a)].median())
std.append(df0[str.format("params_{}", a)].std())
rmsexp.append(df0[str.format("errors_{}", a)].mean())
rmsexp_filtered.append(df0_filtered[str.format("errors_{}",
a)].mean())
mean = np.array(mean)
median = np.array(median)
std = np.array(std)
rmsexp = np.array(rmsexp)
bs_mean.append(mean)
bs_median.append(median)
bs_std.append(std)
bs_rmsexp.append(rmsexp)
bs_mean_filtered.append(mean_filtered)
bs_median_filtered.append(median_filtered)
bs_std_filtered.append(std_filtered)
bs_rmsexp_filtered.append(np.array(rmsexp_filtered))
bs_number_accepted.append(df0["is_accepted"].sum())
bs_number_valid.append(df0["is_valid"].sum())
if "Scipy_LM" in fitters:
try:
p, q = scipy.optimize.curve_fit(testfunc,
x,
y,
sigma=sigma_data *
np.ones_like(y),
p0=p0[0])
except (RuntimeError):
p = np.full(nparams, np.nan)
q = np.full([nparams, nparams], np.nan)
#print(p); print(q)
params.append(p)
errors.append(np.sqrt(np.diag(q)))
chisq.append(
np.sum((testfunc(x, *p) - y)**2) / sigma_data**2 /
(npoints - nparams))
params_true.append(params_true_0)
number_points.append(npoints)
fit_idx.append(ifit)
fitter_name.append("Scipy_LM")
t0 = time.time()
df0, mean_filtered, median_filtered, std_filtered, _ = bootstrap_scipy(
x,
y,
testfunc,
init_params=p0,
weights=weights,
sigma0=sigma_data,
nbootstrap=nbootstrap,
bootstrap_options={'chisq_cut': 4})
t1 = time.time()
frames.append(df0)
df0["fit_idx"] = ifit
df0["time"] = (t1 - t0) / nbootstrap
t.append(t1 - t0)
mean = []
median = []
std = []
rmsexp = []
rmsexp_filtered = []
df0_filtered = df0.query("is_accepted")
for a, b in enumerate(params_true_0):
df0[str.format("params_true_{}", a)] = b
mean.append(df0[str.format("params_{}", a)].mean())
median.append(df0[str.format("params_{}", a)].median())
std.append(df0[str.format("params_{}", a)].std())
rmsexp.append(df0[str.format("errors_{}", a)].mean())
rmsexp_filtered.append(df0_filtered[str.format("errors_{}",
a)].mean())
mean = np.array(mean)
median = np.array(median)
std = np.array(std)
rmsexp = np.array(rmsexp)
bs_mean.append(mean)
bs_median.append(median)
bs_std.append(std)
bs_rmsexp.append(rmsexp)
bs_mean_filtered.append(mean_filtered)
bs_median_filtered.append(median_filtered)
bs_std_filtered.append(std_filtered)
bs_rmsexp_filtered.append(np.array(rmsexp_filtered))
bs_number_accepted.append(df0["is_accepted"].sum())
bs_number_valid.append(df0["is_valid"].sum())
if "Pytorch_LBFGS" in fitters:
p, q, optim_state, fit_status = fitter_torch.curve_fit(
testfunc_torch,
torch.from_numpy(x),
torch.from_numpy(y),
[torch.tensor(i, requires_grad=True) for i in p0[0]],
sigma=sigma_data,
full_output=True)
#print(p[0].detach().numpy()); print(q.numpy())
if fit_status["is_valid"]:
pn = np.hstack([j.detach().cpu().numpy() for j in p])
params.append(pn)
errors.append(np.sqrt(np.diag(q.cpu().numpy())))
chisq.append(fit_status['fval'] / (npoints - nparams))
else:
params.append(np.full(nparams, np.nan))
errors.append(np.full(nparams, np.nan))
chisq.append(np.nan)
params_true.append(params_true_0)
number_points.append(npoints)
fit_idx.append(ifit)
fitter_name.append("Pytorch_LBFGS")
t0 = time.time()
df0, mean_filtered, median_filtered, std_filtered, _ = fitter_torch.curve_fit_BS(
x,
y,
testfunc_torch,
init_params=p0,
weights=weights,
sigma0=sigma_data,
nbootstrap=nbootstrap,
bootstrap_options={'chisq_cut': 4})
t1 = time.time()
frames.append(df0)
df0["fit_idx"] = ifit
df0["time"] = (t1 - t0) / nbootstrap
t.append(t1 - t0)
mean = []
median = []
std = []
rmsexp = []
rmsexp_filtered = []
df0_filtered = df0.query("is_accepted")
for a, b in enumerate(params_true_0):
df0[str.format("params_true_{}", a)] = b
mean.append(df0[str.format("params_{}", a)].mean())
median.append(df0[str.format("params_{}", a)].median())
std.append(df0[str.format("params_{}", a)].std())
rmsexp.append(df0[str.format("errors_{}", a)].mean())
rmsexp_filtered.append(df0_filtered[str.format("errors_{}",
a)].mean())
mean = np.array(mean)
median = np.array(median)
std = np.array(std)
rmsexp = np.array(rmsexp)
bs_mean.append(mean)
bs_median.append(median)
bs_std.append(std)
bs_rmsexp.append(rmsexp)
bs_mean_filtered.append(mean_filtered)
bs_median_filtered.append(median_filtered)
bs_std_filtered.append(std_filtered)
bs_rmsexp_filtered.append(np.array(rmsexp_filtered))
bs_number_accepted.append(df0["is_accepted"].sum())
bs_number_valid.append(df0["is_valid"].sum())
if torch.cuda.is_available() and "Pytorch_LBFGS_CUDA" in fitters:
p, q, optim_state, fit_status = fitter_torch.curve_fit(
testfunc_torch,
torch.from_numpy(x).cuda(),
torch.from_numpy(y).cuda(), [
torch.tensor(i, requires_grad=True, device="cuda:0")
for i in p0[0]
],
sigma=sigma_data,
full_output=True)
#print(p[0].detach().numpy()); print(q.numpy())
if fit_status["is_valid"]:
pn = np.hstack([j.detach().cpu().numpy() for j in p])
params.append(pn)
errors.append(np.sqrt(np.diag(q.cpu().numpy())))
chisq.append(fit_status['fval'] / (npoints - nparams))
else:
params.append(np.full(nparams, np.nan))
errors.append(np.full(nparams, np.nan))
chisq.append(np.nan)
params_true.append(params_true_0)
number_points.append(npoints)
fit_idx.append(ifit)
fitter_name.append("Pytorch_LBFGS_CUDA")
t0 = time.time()
df0, mean_filtered, median_filtered, std_filtered, _ = fitter_torch.curve_fit_BS(
x,
y,
testfunc_torch,
init_params=p0,
weights=weights,
sigma0=sigma_data,
nbootstrap=nbootstrap,
device="cuda:0",
fitter_name="Pytorch_LBFGS_CUDA",
bootstrap_options={'chisq_cut': 4})
torch.cuda.synchronize()
t1 = time.time()
frames.append(df0)
df0["fit_idx"] = ifit
df0["time"] = (t1 - t0) / nbootstrap
t.append(t1 - t0)
mean = []
median = []
std = []
rmsexp = []
rmsexp_filtered = []
df0_filtered = df0.query("is_accepted")
for a, b in enumerate(params_true_0):
df0[str.format("params_true_{}", a)] = b
mean.append(df0[str.format("params_{}", a)].mean())
median.append(df0[str.format("params_{}", a)].median())
std.append(df0[str.format("params_{}", a)].std())
rmsexp.append(df0[str.format("errors_{}", a)].mean())
rmsexp_filtered.append(df0_filtered[str.format("errors_{}",
a)].mean())
mean = np.array(mean)
median = np.array(median)
std = np.array(std)
rmsexp = np.array(rmsexp)
bs_mean.append(mean)
bs_median.append(median)
bs_std.append(std)
bs_rmsexp.append(rmsexp)
bs_mean_filtered.append(mean_filtered)
bs_median_filtered.append(median_filtered)
bs_std_filtered.append(std_filtered)
bs_rmsexp_filtered.append(np.array(rmsexp_filtered))
bs_number_accepted.append(df0["is_accepted"].sum())
bs_number_valid.append(df0["is_valid"].sum())
if "iminuit" in fitters:
p, q, status = fitter_minuit.curve_fit(testfunc,
x,
y,
weights=1 / sigma_data**2,
p0=p0[0],
full_output=True)
#print(p[0].detach().numpy()); print(q.numpy())
if status.is_valid:
params.append(p)
errors.append(np.sqrt(np.diag(q)))
chisq.append(status.fval / (npoints - nparams))
else:
params.append(np.full(nparams, np.nan))
errors.append(np.full(nparams, np.nan))
chisq.append(np.nan)
params_true.append(params_true_0)
number_points.append(npoints)
fit_idx.append(ifit)
fitter_name.append("Minuit")
t0 = time.time()
df0, mean_filtered, median_filtered, std_filtered, _ = fitter_minuit.curve_fit_BS(
x,
y,
testfunc,
init_params=p0,
weights=weights,
sigma0=sigma_data,
nbootstrap=nbootstrap,
bootstrap_options={'chisq_cut': 4})
t1 = time.time()
frames.append(df0)
df0["fit_idx"] = ifit
df0["time"] = (t1 - t0) / nbootstrap
t.append(t1 - t0)
mean = []
median = []
std = []
rmsexp = []
rmsexp_filtered = []
df0_filtered = df0.query("is_accepted")
for a, b in enumerate(params_true_0):
df0[str.format("params_true_{}", a)] = b
mean.append(df0[str.format("params_{}", a)].mean())
median.append(df0[str.format("params_{}", a)].median())
std.append(df0[str.format("params_{}", a)].std())
rmsexp.append(df0[str.format("errors_{}", a)].mean())
rmsexp_filtered.append(df0_filtered[str.format("errors_{}",
a)].mean())
mean = np.array(mean)
median = np.array(median)
std = np.array(std)
rmsexp = np.array(rmsexp)
bs_mean.append(mean)
bs_median.append(median)
bs_std.append(std)
bs_rmsexp.append(rmsexp)
bs_mean_filtered.append(mean_filtered)
bs_median_filtered.append(median_filtered)
bs_std_filtered.append(std_filtered)
bs_rmsexp_filtered.append(np.array(rmsexp_filtered))
bs_number_accepted.append(df0["is_accepted"].sum())
bs_number_valid.append(df0["is_valid"].sum())
bs_mean = np.stack(bs_mean)
bs_median = np.stack(bs_median)
bs_std = np.stack(bs_std)
bs_rmsexp = np.stack(bs_rmsexp)
bs_mean_filtered = np.stack(bs_mean_filtered)
bs_median_filtered = np.stack(bs_median_filtered)
bs_std_filtered = np.stack(bs_std_filtered)
bs_rmsexp_filtered = np.stack(bs_rmsexp_filtered)
params = np.stack(params)
errors = np.stack(errors)
params_true = list(zip(*params_true))
d = {
"fitter_name": fitter_name,
"fit_idx": fit_idx,
"number_points": number_points,
"time": t,
"chisq": chisq,
"nbootstrap": nbootstrap,
'bs_number_valid': bs_number_valid,
'bs_number_accepted': bs_number_accepted
}
d.update({
str.format("params_{}", i): params[:, i]
for i in range(params.shape[1])
})
d.update({
str.format("errors_{}", i): errors[:, i]
for i in range(errors.shape[1])
})
d.update({
str.format("bs_mean_{}", i): bs_mean[:, i]
for i in range(bs_mean.shape[1])
})
d.update({
str.format("bs_median_{}", i): bs_median[:, i]
for i in range(bs_median.shape[1])
})
d.update({
str.format("bs_std_{}", i): bs_std[:, i]
for i in range(bs_std.shape[1])
})
d.update({
str.format("bs_rmsexp_{}", i): bs_rmsexp[:, i]
for i in range(bs_rmsexp.shape[1])
})
d.update({
str.format("bs_mean_filtered_{}", i): bs_mean_filtered[:, i]
for i in range(bs_mean.shape[1])
})
d.update({
str.format("bs_median_filtered_{}", i): bs_median_filtered[:, i]
for i in range(bs_median.shape[1])
})
d.update({
str.format("bs_std_filtered_{}", i): bs_std_filtered[:, i]
for i in range(bs_std.shape[1])
})
d.update({
str.format("bs_rmsexp_filtered_{}", i): bs_rmsexp_filtered[:, i]
for i in range(bs_rmsexp_filtered.shape[1])
})
d.update({
str.format("params_true_{}", idx): el
for idx, el in enumerate(params_true)
})
df1 = pd.DataFrame(d)
df2 = pd.concat(frames)
return df1, df2