def compare_classifiers(data_folder,
model,
batch_size,
to_exclude=None,
use_magnitude=False,
valid_size=0.1,
use_lstm=False,
dev="cpu"):
if to_exclude is None:
sub_folder = "acc_magnitude" if use_magnitude else "all_sensors"
else:
sub_folder = "without_" + "_".join(to_exclude)
# If needed create dataset from session files in data_folder
if need_train_test_folder(os.path.join(data_folder, sub_folder)):
data_helper_transportation.create_train_test_folders(
data_folder,
sub_folder=sub_folder,
to_exclude=to_exclude,
acc_magnitude=use_magnitude,
smoothing=True)
error_msg = "[!] valid_size should be in the range [0, 1]."
assert ((valid_size >= 0) and (valid_size <= 1)), error_msg
# load dataset
dataset = transportation_dataset(data_path=os.path.join(
data_folder, sub_folder),
train=True,
use_magnitude=use_magnitude)
test_dataset = transportation_dataset(data_path=os.path.join(
data_folder, sub_folder),
train=False,
use_magnitude=use_magnitude)
# Split the data into training and validation set
num_train = len(dataset)
split_valid = int(np.floor(valid_size * num_train))
split_train = num_train - split_valid
train_dataset, valid_dataset = random_split(dataset,
[split_train, split_valid])
# Test dataset
# normalize dataset (using scaler trained on training set)
# get mean and std of trainset (for every feature)
mean_train = torch.mean(train_dataset.dataset.data[train_dataset.indices],
dim=0)
std_train = torch.std(train_dataset.dataset.data[train_dataset.indices],
dim=0)
# Dummy classifiers don't need train+val set
data_for_dummy = (dataset.data - mean_train) / std_train
test_dataset.data = (test_dataset.data - mean_train) / std_train
test_dl = DataLoader(test_dataset,
batch_size,
shuffle=False,
num_workers=0,
pin_memory=True)
###### Dummy Classifier 1 #######
myStratifiedClassifier = DummyClassifier(strategy="stratified")
myStratifiedClassifier.fit(data_for_dummy, dataset.targets)
class_report = classification_report(myStratifiedClassifier.predict(
test_dataset.data),
test_dataset.targets,
target_names=list(classes.keys()),
output_dict=True,
zero_division=0)
metrics_str_stratClass = " ".join([
f"{key} {class_report[key]['precision']:.2f}|{class_report[key]['recall']:.2f}|{class_report[key]['f1-score']:.2f}"
for key in classes.keys()
])
print(f"StratDummy: Test_acc: {class_report['accuracy']:.2f}"
f", Class-metrics (Precision|Recall|F1): {metrics_str_stratClass}")
latex_str = " & ".join([
f"{class_report[key]['precision']:.2f} & {class_report[key]['recall']:.2f} & {class_report[key]['f1-score']:.2f}"
for key in classes.keys()
])
print(f"StratDummy latex: {latex_str}")
###### Dummy Classifier 2 ######
myFrequentClassifier = DummyClassifier(strategy="most_frequent")
myFrequentClassifier.fit(data_for_dummy, dataset.targets)
class_report = classification_report(myFrequentClassifier.predict(
test_dataset.data),
test_dataset.targets,
target_names=list(classes.keys()),
output_dict=True,
zero_division=0)
metrics_str_freqClass = " ".join([
f"{key} {class_report[key]['precision']:.2f}|{class_report[key]['recall']:.2f}|{class_report[key]['f1-score']:.2f}"
for key in classes.keys()
])
print(f"FrequencyDummy: Test_acc: {class_report['accuracy']:.2f}"
f", Class-metrics (Precision|Recall|F1): {metrics_str_freqClass}")
latex_str = " & ".join([
f"{class_report[key]['precision']:.2f} & {class_report[key]['recall']:.2f} & {class_report[key]['f1-score']:.2f}"
for key in classes.keys()
])
print(f"FrequencyDummy latex: {latex_str}")
###### CNN ######
if use_lstm:
test_dataset = transportation_dataset(data_path=os.path.join(
data_folder, sub_folder),
train=False,
use_magnitude=use_magnitude,
use_lstm=use_lstm)
test_dataset.data = (test_dataset.data - mean_train.T) / std_train.T
test_dl = DataLoader(test_dataset,
batch_size,
shuffle=False,
num_workers=0,
pin_memory=True)
model_name = "LSTM"
else:
model_name = "CNN"
loss_func = nn.CrossEntropyLoss().to(dev)
# Use best NN-model for Test dataset:
conf_mat = ConfusionMatrix(num_classes=6, normalized=False)
test_loss, class_report = eval_model(model,
loss_func,
test_dl,
conf_mat,
dev=dev)
# Use conf_mat to create metrics
conf_mat = conf_mat.value()
metrics_str_CNN = " ".join([
f"{key} {class_report[key]['precision']:.2f}|{class_report[key]['recall']:.2f}|{class_report[key]['f1-score']:.2f}"
for key in classes.keys()
])
print(f"{model_name}: Test_acc: {class_report['accuracy']:.2f}"
f", Class-metrics (Precision|Recall|F1): {metrics_str_CNN}")
latex_str = " & ".join([
f"{class_report[key]['precision']:.2f} & {class_report[key]['recall']:.2f} & {class_report[key]['f1-score']:.2f}"
for key in classes.keys()
])
print(f"{model_name} latex: {latex_str}")
class IoU(metric.Metric):
"""Computes the intersection over union (IoU) per class and corresponding
mean (mIoU).
Intersection over union (IoU) is a common evaluation metric for semantic
segmentation. The predictions are first accumulated in a confusion matrix
and the IoU is computed from it as follows:
IoU = true_positive / (true_positive + false_positive + false_negative).
Keyword arguments:
- num_classes (int): number of classes in the classification problem
- normalized (boolean, optional): Determines whether or not the confusion
matrix is normalized or not. Default: False.
- ignore_index (int or iterable, optional): Index of the classes to ignore
when computing the IoU. Can be an int, or any iterable of ints.
"""
def __init__(self, num_classes, normalized=False, ignore_index=None):
super().__init__()
self.conf_metric = ConfusionMatrix(num_classes, normalized)
if ignore_index is None:
self.ignore_index = None
elif isinstance(ignore_index, int):
self.ignore_index = (ignore_index,)
else:
try:
self.ignore_index = tuple(ignore_index)
except TypeError:
raise ValueError("'ignore_index' must be an int or iterable")
def reset(self):
self.conf_metric.reset()
def add(self, predicted, target):
"""Adds the predicted and target pair to the IoU metric.
Keyword arguments:
- predicted (Tensor): Can be a (N, K, H, W) tensor of
predicted scores obtained from the model for N examples and K classes,
or (N, H, W) tensor of integer values between 0 and K-1.
- target (Tensor): Can be a (N, K, H, W) tensor of
target scores for N examples and K classes, or (N, H, W) tensor of
integer values between 0 and K-1.
"""
# Dimensions check
assert predicted.size(0) == target.size(0), \
'number of targets and predicted outputs do not match'
assert predicted.dim() == 3 or predicted.dim() == 4, \
"predictions must be of dimension (N, H, W) or (N, K, H, W)"
assert target.dim() == 3 or target.dim() == 4, \
"targets must be of dimension (N, H, W) or (N, K, H, W)"
# If the tensor is in categorical format convert it to integer format
if predicted.dim() == 4:
_, predicted = predicted.max(1)
if target.dim() == 4:
_, target = target.max(1)
self.conf_metric.add(predicted.view(-1), target.view(-1))
def value(self):
"""Computes the IoU and mean IoU.
The mean computation ignores NaN elements of the IoU array.
Returns:
Tuple: (IoU, mIoU). The first output is the per class IoU,
for K classes it's numpy.ndarray with K elements. The second output,
is the mean IoU.
"""
conf_matrix = self.conf_metric.value()
if self.ignore_index is not None:
for index in self.ignore_index:
conf_matrix[:, self.ignore_index] = 0
conf_matrix[self.ignore_index, :] = 0
true_positive = np.diag(conf_matrix)
false_positive = np.sum(conf_matrix, 0) - true_positive
false_negative = np.sum(conf_matrix, 1) - true_positive
# Just in case we get a division by 0, ignore/hide the error
with np.errstate(divide='ignore', invalid='ignore'):
iou = true_positive / (true_positive + false_positive + false_negative)
return iou, np.nanmean(iou)
def test_model(data_folder,
model,
batch_size,
to_exclude=None,
use_magnitude=False,
valid_size=0.1,
dev="cpu"):
sub_folder = "acc_magnitude" if use_magnitude else "all_sensors"
# If needed create dataset from session files in data_folder
if need_train_test_folder(os.path.join(data_folder, sub_folder)):
data_helper_transportation.create_train_test_folders(
data_folder,
sub_folder=sub_folder,
to_exclude=to_exclude,
acc_magnitude=use_magnitude,
smoothing=True)
error_msg = "[!] valid_size should be in the range [0, 1]."
assert ((valid_size >= 0) and (valid_size <= 1)), error_msg
# load dataset
dataset = transportation_dataset(data_path=os.path.join(
data_folder, sub_folder),
train=True,
use_magnitude=use_magnitude)
test_dataset = transportation_dataset(data_path=os.path.join(
data_folder, sub_folder),
train=False,
use_magnitude=use_magnitude)
# Split the data into training and validation set
num_train = len(dataset)
split_valid = int(np.floor(valid_size * num_train))
split_train = num_train - split_valid
train_dataset, valid_dataset = random_split(dataset,
[split_train, split_valid])
# Test dataset
# normalize dataset (using scaler trained on training set)
# get mean and std of trainset (for every feature)
mean_train = torch.mean(train_dataset.dataset.data[train_dataset.indices],
dim=0)
std_train = torch.std(train_dataset.dataset.data[train_dataset.indices],
dim=0)
test_dataset.data = (test_dataset.data - mean_train) / std_train
test_dl = DataLoader(test_dataset,
batch_size,
shuffle=False,
num_workers=0,
pin_memory=True)
loss_func = nn.CrossEntropyLoss().to(dev)
# Use best model for Test dataset:
conf_mat = ConfusionMatrix(num_classes=6, normalized=False)
test_loss, class_report = eval_model(model,
loss_func,
test_dl,
conf_mat,
dev=dev)
# Use conf_mat to create metrics
conf_mat = conf_mat.value()
metrics_str = " ".join([
f"{key} {class_report[key]['precision']:.2f}|{class_report[key]['recall']:.2f}|{class_report[key]['f1-score']:.2f}"
for key in classes.keys()
])
print(
f"Test_loss: {test_loss:2.10f}, Test_acc: {class_report['accuracy']:.2f}"
f", Class-metrics (Precision|Recall|F1): {metrics_str}")
def train_model(data_folder,
epochs,
n_classes,
batch_size,
learning_rate,
valid_size=0.1,
to_exclude=None,
use_magnitude=True,
earlystopping=None,
lr_scheduler=True,
save_every=None,
use_lstm=False,
dev="cpu"):
if to_exclude is None:
sub_folder = "acc_magnitude" if use_magnitude else "all_sensors"
else:
sub_folder = "without_" + "_".join(to_exclude)
# If needed create dataset from session files in data_folder
if need_train_test_folder(os.path.join(data_folder, sub_folder)):
data_helper_transportation.create_train_test_folders(
data_folder,
sub_folder=sub_folder,
to_exclude=to_exclude,
acc_magnitude=use_magnitude,
smoothing=True)
error_msg = "[!] valid_size should be in the range [0, 1]."
assert ((valid_size >= 0) and (valid_size <= 1)), error_msg
# load dataset
dataset = transportation_dataset(data_path=os.path.join(
data_folder, sub_folder),
train=True,
use_magnitude=use_magnitude,
use_lstm=use_lstm)
test_dataset = transportation_dataset(data_path=os.path.join(
data_folder, sub_folder),
train=False,
use_magnitude=use_magnitude,
use_lstm=use_lstm)
# Split the data into training and validation set
num_train = len(dataset)
split_valid = int(np.floor(valid_size * num_train))
split_train = num_train - split_valid
train_dataset, valid_dataset = random_split(dataset,
[split_train, split_valid])
# Test dataset
# normalize dataset (using scaler trained on training set)
# get mean and std of trainset (for every feature)
mean_train = torch.mean(train_dataset.dataset.data[train_dataset.indices],
dim=0)
std_train = torch.std(train_dataset.dataset.data[train_dataset.indices],
dim=0)
train_dataset.dataset.data[train_dataset.indices] = (
train_dataset.dataset.data[train_dataset.indices] -
mean_train) / std_train
valid_dataset.dataset.data[valid_dataset.indices] = (
valid_dataset.dataset.data[valid_dataset.indices] -
mean_train) / std_train
test_dataset.data = (test_dataset.data - mean_train) / std_train
# get the dataloaders (with the datasets)
train_dl = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=0,
pin_memory=True)
valid_dl = DataLoader(valid_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=0,
pin_memory=True)
test_dl = DataLoader(test_dataset,
batch_size,
shuffle=False,
num_workers=0,
pin_memory=True)
# load the classification model
input_channels = 1 if use_magnitude else dataset.data.shape[1 +
int(use_lstm)]
if use_lstm:
model = TransportationLSTM(input_channels,
128,
output_size=n_classes,
bidirectional=True)
else:
model = TransportationCNN(in_channels=input_channels,
n_classes=n_classes,
activation_function="elu",
alpha=0.1)
# Print the model and parameter count
print("Trainable parameters: ", count_parameters(model))
# summary(model, (input_channels, 512), device="cpu")
model.to(dev)
# define optimizers and loss function
# weight_decay is L2 weight normalization (used in paper), but I dont know how much
opt = torch.optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=0.001)
if lr_scheduler:
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt,
'min',
factor=0.1,
patience=5)
loss_func = nn.CrossEntropyLoss().to(dev)
# fit the model
tensorboard = False
#### Training ####
if tensorboard:
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(
comment=f"transportation_{sub_folder}_{model.__class__.__name__}")
start_time = time.time()
best_val_loss = 1e300
earlystopping_counter = 0
for epoch in tqdm(range(epochs), desc="Epochs", leave=True, position=0):
model.train()
train_loss = 0.0
for i, (xb,
yb) in enumerate(tqdm(train_dl, desc="Batches", leave=False)):
# for i, (xb, yb) in enumerate(train_dl):
loss = loss_func(model(xb.to(dev)), yb.to(dev))
opt.zero_grad()
loss.backward()
opt.step()
train_loss += loss.item()
# if i > 100:
# break
train_loss /= len(train_dl)
# Calc validation loss
conf_mat = ConfusionMatrix(num_classes=n_classes, normalized=False)
val_loss, class_report = eval_model(model,
loss_func,
valid_dl,
conf_mat,
dev=dev)
# Use conf_mat to create metrics
conf_mat = conf_mat.value()
# per_class_acc = np.nan_to_num(conf_mat.diagonal()/conf_mat.sum(1))
# Reduce learning rate after epoch
if lr_scheduler:
scheduler.step(val_loss)
# Save the model with the best validation loss
if val_loss < best_val_loss:
os.makedirs("models/checkpoints", exist_ok=True)
torch.save(
{
'model': model,
'epoch': epoch,
'state_dict': model.state_dict(),
'optimizer_state_dict': opt.state_dict(),
'val_loss': val_loss,
'train_loss': train_loss
},
f"models/checkpoints/best_val_loss_model_{model.__class__.__name__}_{sub_folder}.pt"
)
best_val_loss = val_loss
earlystopping_counter = 0
else:
if earlystopping is not None:
earlystopping_counter += 1
if earlystopping_counter >= earlystopping:
print(
f"Stopping early --> val_loss has not decreased over {earlystopping} epochs"
)
break
metrics_str = " ".join([
f"{key} {class_report[key]['precision']:.2f}|{class_report[key]['recall']:.2f}|{class_report[key]['f1-score']:.2f}"
for key in classes.keys()
])
print(
f"Epoch: {epoch:5d}, Time: {(time.time() - start_time) / 60:.3f} min, "
f"Train_loss: {train_loss:2.10f}, Val_loss: {val_loss:2.10f}, Val_acc: {class_report['accuracy']:.2f}"
f", Class-metrics (Precision|Recall|F1): {metrics_str}"
f", Early stopping counter: {earlystopping_counter}/{earlystopping}"
if earlystopping is not None else "")
if tensorboard:
# add to tensorboard
writer.add_scalar('Loss/train', train_loss, epoch)
writer.add_scalar('Loss/val', val_loss, epoch)
# TODO add confusion-matrix to tensorboard?
if save_every is not None:
if epoch % save_every == 0:
# save model
torch.save(
{
'model': model,
'epoch': epoch,
'state_dict': model.state_dict(),
'optimizer_state_dict': opt.state_dict(),
# 'scheduler_state_dict': scheduler.state_dict(),
'val_loss': val_loss,
'train_loss': train_loss
},
f"models/checkpoints/model_{model.__class__.__name__}_epoch_{epoch}_{sub_folder}.pt"
)
# Save best model
load_best_val_model = torch.load(
f"models/checkpoints/best_val_loss_model_{model.__class__.__name__}_{sub_folder}.pt"
)
os.makedirs("models/trained_models", exist_ok=True)
torch.save(
{
'model': load_best_val_model['model'],
'state_dict': load_best_val_model['state_dict']
}, f"models/trained_models/{model.__class__.__name__}_{sub_folder}.pt")
# Use best model for Test dataset:
model = load_best_val_model["model"]
model.load_state_dict(load_best_val_model["state_dict"])
conf_mat = ConfusionMatrix(num_classes=n_classes, normalized=False)
test_loss, class_report = eval_model(model,
loss_func,
test_dl,
conf_mat,
dev=dev)
# Use conf_mat to create metrics
conf_mat = conf_mat.value()
metrics_str = " ".join([
f"{key} {class_report[key]['precision']:.2f}|{class_report[key]['recall']:.2f}|{class_report[key]['f1-score']:.2f}"
for key in classes.keys()
])
print(
f"Test_loss: {test_loss:2.10f}, Test_acc: {class_report['accuracy']:.2f}"
f", Class-metrics (Precision|Recall|F1): {metrics_str}")
class IoU(metric.Metric):
"""计算每个类的并集(IoU)和相应的均值(mIoU)
联合交叉(IoU)是语义的通用评估度量分割,预测首先在混淆矩阵中累积
IoU的计算方法如下:
IoU = true_positive /(true_positive + false_positive + false_negative)
Args:
num_classes(int): 分类问题中的类数
normalized(boolean,optional): 确定是否混淆,矩阵是否归一化, 默认值: False
ignore_index(int或iterable，optional): 要忽略的类的索引,在计算IoU时, 可以是int,也可以是任何可迭代的int。
"""
def __init__(self, num_classes, normalized=False, ignore_index=None):
super().__init__()
self.conf_metric = ConfusionMatrix(num_classes, normalized)
if ignore_index is None:
self.ignore_index = None
elif isinstance(ignore_index, int):
self.ignore_index = (ignore_index, )
else:
try:
self.ignore_index = tuple(ignore_index)
except TypeError:
raise ValueError("'ignore_index' must be an int or iterable")
def reset(self):
self.conf_metric.reset()
def add(self, predicted, target):
"""将predicted和target添加到IoU metric.混淆矩阵
Args:
predicted (Tensor): 可以是(N, K, H, W) tensor，从N个样本的K类的类别得分
或者是(N, H, W) tensor，值在0到K-1
target (Tensor): 可以是N个样本和K类的目标分数的(N,K,H,W)张量,或者(N,H,W) tensor 值在0到K-1
"""
# Dimensions check
assert predicted.size(0) == target.size(0), \
'number of targets and predicted outputs do not match'
assert predicted.dim() == 3 or predicted.dim() == 4, \
"predictions must be of dimension (N, H, W) or (N, K, H, W)"
assert target.dim() == 3 or target.dim() == 4, \
"targets must be of dimension (N, H, W) or (N, K, H, W)"
# If the tensor is in categorical format convert it to integer format
if predicted.dim() == 4:
_, predicted = predicted.max(1)
if target.dim() == 4:
_, target = target.max(1)
self.conf_metric.add(predicted.view(-1), target.view(-1))
def value(self):
"""计算 IoU 和 mean IoU.
平均计算忽略IoU阵列的NaN元素。
Returns:
Tuple: (IoU, mIoU). 他的第一个输出是每个类IoU，对于K类，它是带有K个元素的numpy.ndarray 第二个输出是平均IoU。
"""
conf_matrix = self.conf_metric.value()
if self.ignore_index is not None:
for index in self.ignore_index:
conf_matrix[:, self.ignore_index] = 0
conf_matrix[self.ignore_index, :] = 0
true_positive = np.diag(conf_matrix)
false_positive = np.sum(conf_matrix, 0) - true_positive
false_negative = np.sum(conf_matrix, 1) - true_positive
# Just in case we get a division by 0, ignore/hide the error
with np.errstate(divide='ignore', invalid='ignore'):
iou = true_positive / (true_positive + false_positive +
false_negative)
return iou, np.nanmean(iou)