def run_batches(shift_data=0,
n_batches=20,
batch_size=20,
lr=1.e-1,
iterations=200,
zero_grad=True,
verbose=False,
plot=True,
seed=1):
m = n_batches * batch_size
x, y, w_true = generate_data(size=m,
shift_data=shift_data,
plot=False,
seed=seed)
w_guess = nn.Parameter(tensor(-1., 1))
losses, weights_1, weights_2 = [], [], []
batch_idx = -1
for t in range(iterations):
# -- batch limits --
batch_idx += 1
batch_idx = batch_idx % n_batches
start = batch_idx * batch_size
end = start + batch_size
# ------------------
x_b = x[start:end].clone() # clone makes sure we do not override
y_b = y[start:end] # no need to clone this
loss, w_guess = update_batch(x_b,
y_b,
w_guess,
lr=lr,
zero_grad=zero_grad)
losses.append(float(loss.detach().numpy()))
weights_1.append(w_guess.detach().numpy()[0])
weights_2.append(w_guess.detach().numpy()[1])
if (t % (iterations // 10) == 0) & verbose:
print(f'MSE {losses[-1]}')
if plot:
plot_summary(x,
y,
x @ w_guess.detach().numpy(),
losses,
weights_1,
weights_2,
shift_data=shift_data,
title='mini batches')
return np.array(losses)
def run_batch(shift_data=0,
size=400,
lr=1.e-1,
iterations=200,
zero_grad=True,
verbose=False,
plot=True,
seed=1):
x, y, w_true = generate_data(size=size,
shift_data=shift_data,
plot=False,
seed=seed)
w_guess = nn.Parameter(tensor(-1., 1))
losses, weights_1, weights_2 = [], [], []
for t in range(iterations):
loss, w_guess = update_batch(x, y, w_guess, lr=lr, zero_grad=zero_grad)
losses.append(float(loss.detach().numpy()))
weights_1.append(w_guess.detach().numpy()[0])
weights_2.append(w_guess.detach().numpy()[1])
if (t % (iterations // 10) == 0) & verbose:
print(f'MSE {losses[-1]}')
if plot:
plot_summary(x,
y,
x @ w_guess.detach().numpy(),
losses,
weights_1,
weights_2,
shift_data=shift_data,
title='one batch')
return np.array(losses)
#print(df1)
with torch.no_grad():
vertLatents.sub_(lr * vertLatents.grad)
vertLatents.grad.zero_()
horiLatents.sub_(lr * horiLatents.grad)
horiLatents.grad.zero_()
return loss.item()
vecLatents = 10
shape = (20, 14)
# random large block of data
blockData = tensor(numpy.random.random_sample(shape))
horiLatents = \
nn.Parameter(tensor(numpy.random.random_sample((vecLatents,shape[1]))))
vertLatents = \
nn.Parameter(tensor(numpy.random.random_sample((shape[0],vecLatents))))
lr = 1e-1
horiLatents.requires_grad_(True)
vertLatents.requires_grad_(True)
lossDict = {}
for t in range(10001):
lossDict[t] = update(horiLatents, vertLatents)
pandas.DataFrame.from_dict(lossDict, orient='index').to_csv('lossDict.csv',
index=False)
pandas.DataFrame(horiLatents.data.tolist()).to_csv('horiLatents.csv',
index=False)
pandas.DataFrame(vertLatents.data.tolist()).to_csv('vertLatents.csv',
index=False)
def run_batch_norm(shift_data=0,
momentum=0.9,
n_batches=10,
batch_size=20,
lr=1.e-1,
iterations=100,
zero_grad=True,
verbose=False,
epsilon=1.e-7,
plot=True,
title='batch norm - momentum ',
seed=1):
m = n_batches * batch_size
x, y, w_true = generate_data(size=m,
shift_data=shift_data,
plot=False,
seed=seed)
w_guess = nn.Parameter(tensor(-1., 1))
gamma = nn.Parameter(tensor(-1., 1.))
beta = nn.Parameter(tensor(-1., 1.))
losses, weights_1, weights_2 = [], [], []
batch_idx = -1
for t in range(iterations): #(batch_size * n_batches // 2):
# -- batch limits --
batch_idx += 1
batch_idx = batch_idx % n_batches
start = batch_idx * batch_size
end = start + batch_size
# ------------------
x_b = x[start:end].clone() # clone makes sure we do not override
y_b = y[start:end] # no need to clone this
if not momentum:
mu, var = None, None
if (t == 0):
title += '0'
else:
if (t == 0):
mu = x_b.mean(axis=0)[0]
var = x_b.var(axis=0)[0]
title += f'{momentum}'
loss, w_guess, mu, var = update_batch_norm(x_b,
y_b,
w_guess,
mu,
var,
gamma,
beta,
momentum=momentum,
lr=lr,
zero_grad=zero_grad,
epsilon=epsilon)
w1_guess, w2_guess = _params_to_weights(gamma.detach().numpy(),
beta.detach().numpy(),
w_guess.detach().numpy(),
mu.detach().numpy(),
var.detach().numpy())
losses.append(float(loss.detach().numpy()))
weights_1.append(w1_guess)
weights_2.append(w2_guess)
#print(gamma.detach().numpy(), beta.detach().numpy(), w_guess.detach().numpy(), mu.detach().numpy(), var.detach().numpy())
if (t % 10 == 0) & verbose:
print(f'MSE {losses[-1]}')
x_ = x.clone()
x_[:, 0].sub_(mu).div_((var + epsilon)**0.5)
z_ = x_ * gamma + beta
if plot:
plot_summary(x,
y, (z_ @ w_guess).detach().numpy(),
losses,
weights_1,
weights_2,
shift_data=shift_data,
no_weights=False,
title=title)
return np.array(losses)