def train(dataset, args):
n_weights = np.size(dataset.x, 1)
model = LinearRegression(n_weights=n_weights)
optimizer = Optimizer(dataset)
losses = []
for _ in range(args.epochs):
losses.append(mse(model, dataset))
optimizer.step(model, args.learning_rate)
print ('Model:', model, '\n')
print ('Loss:', mse(model, dataset))
sklearn_model = SklearnLinearRegression().fit(dataset.x, dataset.y)
print ('Loss with sklearn:', mean_squared_error(dataset.y, sklearn_model.predict(dataset.x)), '\n')
if args.plot:
plot(model, n_weights, losses, dataset)
return model
def display_model_metrics(model, dataset):
print('--- Model\'s metrics ---')
print(model)
print('Mean square error:', mse(model, dataset))
print('Mean absolute error:', mae(model, dataset))
print('Mean percentage error: ', mpe(model, dataset))
print('Mean absolute percentage error: ', mape(model, dataset), '\n')
def add_loss_layer(self, layer_name, prediction_layer_id,
ground_truth_layer_id, loss_type):
"""
Adds a layer corresponding to the loss function
:param layer_name: The name of the layer. Type=string
:param prediction_layer_id: The identifier for the prediction layer
:param ground_truth_layer_id: The identifier for the ground truth layer
:param loss_type: The loss function to use. Available options defined by LossTypes.
:return: None
"""
layer_id = self._get_layer_id(layer_name)
assert self._layer_verifier(
layer_id), 'Invalid: This layer is already present.'
assert not self._layer_verifier(
prediction_layer_id), 'Invalid: Output layer id is invalid.'
assert not self._layer_verifier(
ground_truth_layer_id
), 'Invalid: Ground truth layer id is invalid.'
output = self.layers[prediction_layer_id]
ground_truth = self.layers[ground_truth_layer_id]
if loss_type == LossTypes.mse:
self.layers[layer_id] = mse(ground_truth, output)
elif loss_type == LossTypes.cross_entropy:
self.layers[layer_id] = cross_entropy(ground_truth, output)
else:
raise ValueError('The type of loss can only be one of ["mse"]')
def test_edge_cases_mse(self):
"""`loss.mse`: Edge Case Validator.
Tests the behavior of `mse` with edge cases.
Raises:
Exception: If at least one `Exception` raised is not of the expected
kind.
"""
with self.assertRaises(_InvalidObservationSetError):
# Empty observation matrix.
mse(_np.matrix([[]]), _random_matrix((self.n, 1)))
with self.assertRaises(_InvalidObservationSetError):
# Empty prediction matrix.
mse(_random_matrix((self.n, 1)), _np.matrix([[]]))
def evaluate_batch(self,
images: torch.tensor,
labels: torch.tensor,
print_labels=False) -> tuple:
""" evaluations for batch """
self.eval()
with torch.no_grad():
labels_pred = self.forward(images)
if print_labels:
for i, (prediction,
target) in enumerate(zip(labels_pred, labels)):
print(
"target\t\t",
np.around(target[self.considered_label_indices].cpu(),
3))
print("\033[1mprediction\t",
np.around(
prediction[self.considered_label_indices].cpu(),
3),
end="\033[0m\n")
if i >= 2:
break
print(
"<target>\t",
np.around(
torch.mean(labels[:, self.considered_label_indices],
dim=0).cpu(), 3))
print(
"<target>\t",
np.around(
torch.std(labels[:, self.considered_label_indices],
dim=0).cpu(), 3))
print("\033[1m<prediction>\t",
np.around(
torch.mean(
labels_pred[:, self.considered_label_indices],
dim=0).cpu(), 3),
end="\033[0m\n")
print("\033[1m<prediction>\t",
np.around(
torch.std(labels_pred[:,
self.considered_label_indices],
dim=0).cpu(), 3),
end="\033[0m\n")
loss_regression = torch.sqrt(
mse(labels_pred[:, self.considered_label_indices],
labels[:, self.considered_label_indices])).item()
loss_variance = self.weight_loss_sample_variance * \
loss_sample_variance(labels_pred[:,self.considered_label_indices],
threshold=self.sample_variance_threshold
).item()
accs = measure_accuracy_classifier(
labels_pred,
labels,
considered_groups=self.considered_groups.considered_groups)
variance = get_sample_variance(
labels_pred[:, self.considered_label_indices]).item()
return loss_regression, loss_variance, accs, variance
def train_step(self, images: torch.tensor, labels: torch.tensor) -> float:
self.train()
labels_pred = self.forward(images, train=True)
loss_regression = mse(labels_pred[:, self.considered_label_indices],
labels[:, self.considered_label_indices])
loss_variance = self.weight_loss_sample_variance * \
loss_sample_variance(labels_pred[:,self.considered_label_indices],
threshold=self.sample_variance_threshold)
loss = loss_regression + loss_variance
self.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(self.parameters(), max_norm=1)
self.optimizer.step()
self.iteration += 1
return loss.item()
def test_random_mse(self):
"""`loss.mse`: Randomized Validator.
Tests the behavior of `mse` by feeding it randomly generated arguments.
Raises:
AssertionError: If `mse` needs debugging.
"""
for i in range(self.n_tests):
Y = _random_matrix((self.n, 1), max_val=self.max_mean)
"""float: Random-valued observations."""
Y_hat = _random_matrix((self.n, 1), max_val=self.max_mean)
"""float: Random-valued predictions."""
delta_Y = abs(Y - Y_hat)
"""float: Distance between predictions and observations."""
squared_sum_delta_Y = _np.linalg.norm(delta_Y[1:, 0])**2
"""float: Sum of the squares of all `delta_Y` values."""
# To ensure that the coercion does not result in the square of a
# negative number, we can use the mean of the upper-bound
# `squared_sum_delta_Y` as insurance that the computation will only
# work with positive numbers.
err = _uniform((squared_sum_delta_Y + 1.0) / self.n,
(squared_sum_delta_Y + self.max_mean) / self.n)
"""float: MSE to coerce."""
# Coerce MSE by changing the first prediction to a strategic choice
# and mathematically guaranteeing
Y_hat[0, 0] = (_np.sqrt(self.n * err - squared_sum_delta_Y) -
Y[0, 0]) * -1.0
result = mse(Y, Y_hat)
"""float: Test input."""
self.assertAlmostEqual(result, err)
def train_classifier_distributed(rank, world_size, optimizer=optimizer):
# initialize distributed process group
os.environ['MASTER_ADDR'] = 'localhost'
os.environ['MASTER_PORT'] = '12355'
dist.init_process_group(rank=rank, world_size=world_size, backend="nccl")
# prepare splitted dataloader
make_data_loader = MakeDataLoader()
sampler_train = DistributedSampler(make_data_loader.dataset_train,
num_replicas=world_size,
rank=rank,
shuffle=True,
drop_last=True)
data_loader_train = make_data_loader.get_data_loader_train(
batch_size=batch_size,
num_workers=0,
sampler=sampler_train,
shuffle=False)
data_loader_valid = make_data_loader.get_data_loader_valid(
batch_size=batch_size, num_workers=0)
# prepare model
classifier = Classifier(seed=seed_parameter).to(rank)
ddp_classifier = DDP(classifier,
device_ids=list(range(world_size)),
output_device=rank,
find_unused_parameters=True)
optimizer = optimizer(ddp_classifier.parameters(), lr=learning_rate_init)
scheduler = MultiStepLR(optimizer, milestones=[292, 373], gamma=gamma)
for epoch in range(epochs):
data_loader_train.sampler.set_epoch(epoch)
ddp_classifier.train()
for image, label in data_loader_train:
image = image.to(rank)
label = label.to(rank)
optimizer.zero_grad(set_to_none=True)
prediction = ddp_classifier(image, train=True)
loss_train = mse(prediction, label)
loss_train.mean().backward()
torch.nn.utils.clip_grad_norm_(ddp_classifier.parameters(),
max_norm=1)
optimizer.step()
if rank == 0:
accs_train = measure_accuracy_classifier(prediction, label)
ddp_classifier.eval()
loss_valid = 0
accs_valid = Counter({group: 0 for group in range(1, 12)})
with torch.no_grad():
for N_test, (image, label) in enumerate(data_loader_valid):
image = image.to(rank)
label = label.to(rank)
prediction = ddp_classifier(image, train=False)
loss_valid += mse(predction, label).item()
accs_valid.update(
measure_accuracy_classifier(prediction, label))
for group in accs.keys():
accs[group] /= N_test + 1
logs = {
"loss_train": np.sqrt(loss_train.item()),
"loss_valid": np.sqrt(loss_valid)
}
logs.update({
f"accuracy_Q{group}_train": acc
for group, acc in accs_train.items()
})
logs.update({
f"accuracy_Q{group}_valid": acc
for group, acc in accs_valid.items()
})
wandb.log(logs)
# safe full model for later use
if not epoch % 100:
torch.save(ddp_classifier.model,
folder_results + f"classifier_model_epoch{epoch}.pth")
# end distributed process
dist.destroy_process_group()
def test_invalid_args_mse(self):
"""`loss.mse`: Argument Validator.
Tests the behavior of `mse` with invalid argument counts and values.
Raises:
Exception: If at least one `Exception` raised is not of the expected
kind.
"""
with self.assertRaises(TypeError):
# No arguments.
mse()
with self.assertRaises(TypeError):
# Only one arguments.
mse(123)
with self.assertRaises(TypeError):
# More than two arguments.
mse(123, 123, 123, 1123)
with self.assertRaises(_InvalidObservationSetError):
# List of an empty list instead of observation matrix.
mse([[]], _random_matrix((self.n, 1)))
with self.assertRaises(_InvalidObservationSetError):
# `None` instead of prediction matrix.
mse(_random_matrix((self.n, 1)), [[]])
with self.assertRaises(_IncompatibleDataSetsError):
# Incompatible observation and prediction matrices.
mse(_random_matrix((self.n, 1)), _random_matrix((self.n + 1, 1)))
