def __call__(self, model: NNLayer, tx, y, **kwargs):
"""
Performs Stochastic Gradient Descent.
:param tx: sample
:param y: labels
:param batch_size: size of the batches
:param num_batches: number of batches to learn
:param loss: loss function
:param lr: learning rate
:param epoch: number of times to go over the dataset
"""
batch_size = kwargs['batch_size'] if 'batch_size' in kwargs else 1
num_batches = min(kwargs['num_batches'],
tx.shape[0]) if 'num_batches' in kwargs else 1
loss = kwargs['loss'] if 'loss' in kwargs else LogCosh()
lr = kwargs['lr'] if 'lr' in kwargs else .01
epochs = kwargs['epochs'] if 'epochs' in kwargs else 100
activation = model.get_activation_function()
for epoch in range(epochs):
running_loss = 0
for batch_y, batch_tx in batch_iter(y, tx, batch_size,
num_batches):
txw = np.dot(batch_tx, model.get_params())
model.set_param(model.get_params() - lr * np.dot(
np.transpose(batch_tx, (1, 0)),
loss.gradient(activation(batch_tx), batch_y) *
activation.gradient(batch_tx)))
print(loss(activation(txw), batch_y))
print(running_loss)
def __call__(self, model: LinearModel, tx, y, **kwargs):
"""
Performs Stochastic Gradient Descent.
:param tx: sample
:param y: labels
:param max_iter: number of batches to learn
:param loss: loss function
:param lr: learning rate
"""
loss = kwargs['loss'] if 'loss' in kwargs else LogCosh()
lr = kwargs['lr'] if 'lr' in kwargs else .01
epochs = kwargs['epochs'] if 'epochs' in kwargs else 100
for epoch in range(epochs):
gradient = np.dot(np.transpose(tx, (1, 0)),
loss.gradient(model(tx), y))
model.set_param(model.get_params() - lr * gradient)
if np.sum(
np.abs(gradient)) < lr * 10**-2 / model.get_params().size:
break
def __call__(self, model: LinearModel, tx, y, **kwargs):
"""
Performs Stochastic Gradient Descent.
:param tx: sample
:param y: labels
:param batch_size: size of the batches
:param num_batches: number of batches to learn
:param loss: loss function
:param lr: learning rate
:param epoch: number of times to go over the dataset
"""
batch_size = kwargs['batch_size'] if 'batch_size' in kwargs else 1
num_batches = min(kwargs['num_batches'],
tx.shape[0]) if 'num_batches' in kwargs else 1000
loss = kwargs['loss'] if 'loss' in kwargs else LogCosh()
lr = kwargs['lr'] if 'lr' in kwargs else .01
epochs = kwargs['epochs'] if 'epochs' in kwargs else 100
epoch_step = kwargs['epoch_step'] if 'epoch_step' in kwargs else (50,
0.75)
i = 0
running_loss = 0
for step in range(int(epochs / epoch_step[0])):
for epoch_iter in range(epoch_step[0]):
running_loss = 0
acc_grad = 0
for batch_y, batch_tx in batch_iter(y, tx, batch_size,
num_batches):
out = model(batch_tx)
running_loss += loss(out, batch_y)
grad = np.dot(np.transpose(batch_tx, (1, 0)),
loss.gradient(model(batch_tx), batch_y))
model.set_param(model.get_params() - lr * grad)
acc_grad += np.sum(np.abs(grad))
if acc_grad < lr * 10**-2 / model.get_params().size:
return
i += 1
lr *= epoch_step[1]
def __call__(self, model: NNLayer, tx, y, **kwargs):
"""
Performs Gradient Descent.
:param tx: sample
:param y: labels
:param epochs: number of timesto go through the dataset
:param loss: loss function
:param lr: learning rate
"""
loss = kwargs['loss'] if 'loss' in kwargs else LogCosh()
lr = kwargs['lr'] if 'lr' in kwargs else .01
epochs = kwargs['epochs'] if 'epochs' in kwargs else 1000
activation = model.get_activation_function()
for i in range(epochs):
txw = np.dot(tx, model.get_params())
grad = lr * np.dot(
np.transpose(tx, (1, 0)),
loss.gradient(activation(txw), y) * activation.gradient(txw))
model.set_param(model.get_params() - lr * np.dot(
np.transpose(tx, (1, 0)),
loss.gradient(activation(txw), y) * activation.gradient(txw)))
print(loss(activation(txw), y))
spine.set_visible(False)
ax.set_xticks(np.arange(data.shape[1] + 1) - .5, minor=True)
ax.set_yticks(np.arange(data.shape[0] + 1) - .5, minor=True)
ax.grid(which="minor", color="w", linestyle='-', linewidth=3)
ax.tick_params(which="minor", bottom=False, left=False)
return im, cbar
if __name__ == "__main__":
path = os.path.split(
os.path.split(os.path.dirname(os.path.abspath(__file__)))[0])[0]
data = np.load(file=path + '\\resources\\' + 'train.npy')
loss = LogCosh()
model = LinearModel((3, 1))
kwargs = {
'batch_size': 25,
'loss': loss,
'lr': 10**-1,
'epochs': 1000,
'epoch_step': (100, .75)
}
optimizer = LinearSGD()
n_models = 1
for i in range(32, data.shape[1]):
for j in range(i + 1, data.shape[1]):
if i in [8, 19, 28] or j in [8, 19, 28]:
continue
n_features = np.sum(mask)
models = [LinearModel((n_features + 1, 1)),
LinearModel((n_features + 1, 1)),
Logistic(n_features + 1),
LinearModel((n_features + 1, 1)),
LinearModel((n_features + 1, 1))]
optimizers = [Ridge(),
LS(),
LinearSGD(),
LinearGD(),
LogisticSGD()]
optimizer_kwargs = [[{'lambda_': 0.5}],
[{}],
[{'batch_size': 25, 'loss': LogCosh(), 'lr': 10**-1, 'epochs': 1000, 'regularize': r}
for r in range(2)],
[{'batch_size': 25, 'loss': LogCosh(), 'lr': 10**-1, 'epochs': 1000}],
[]]
normalizers = [
MinMaxNormalizer()
# , GaussianNormalizer()
# , DecimalScaling()
]
lrf = LinearRegressionFilling(data[:, mask], epochs=100)
lrf.load(path + '/src/preconditioning/regression_filler_params.npy')
filling_data = [
MeanFilling(data[:, mask]),