def part_b():
print("\n--- Part B ---\n")
print("Reading Data")
train_y, train_x = read_data("train")
test_y, test_x = read_data("test")
print("Normalizing")
# train_x = np.asarray(train_x)
# test_x = np.asarray(test_x)
train_x = normalize(train_x)
test_x = normalize(test_x)
train_y = np.asarray(train_y)
test_y = np.asarray(test_y)
# Build 10_C_2 classifiers
print("Building Classifiers")
classifiers = pegasos_train(train_x, train_y)
with open("models/svm-model-1", "wb") as m:
pickle.dump(classifiers, m)
print("Finding Accuracy")
predictions_test = pegasos_predict(test_x, classifiers)
acc = accuracy(test_y.tolist(), predictions_test)
print("Testing Accuracy: %.2f" % (acc * 100))
predictions_train = pegasos_predict(train_x, classifiers)
acc = accuracy(train_y.tolist(), predictions_train)
print("Training Accuracy: %.2f" % (acc * 100))
def performance(self, train_fold, val_fold, test_fold=None):
with torch.no_grad():
# Expected output class indices
train_expected = torch.Tensor([ d['y'] for d in train_fold ])
val_expected = torch.Tensor([ d['y'] for d in val_fold ]) if val_fold else None
test_expected = torch.Tensor([ d['y'] for d in test_fold ]) if test_fold else None
for m in self.models:
m.eval()
train_outs = [ m.forward_in_batches(train_fold, self.batch_size, return_probs=True) for m in self.models ]
val_outs = [ m.forward_in_batches(val_fold, self.batch_size, return_probs=True) for m in self.models ] if val_fold else None
test_outs = [ m.forward_in_batches(test_fold, self.batch_size, return_probs=True) for m in self.models ] if test_fold else None
train_out = (torch.stack(train_outs).mean(dim=0) > 0)
val_out = (torch.stack(val_outs).mean(dim=0) > 0) if val_fold else None
test_out = (torch.stack(test_outs).mean(dim=0) > 0) if test_fold else None
# Compute performance measures
train_accuracy = common.accuracy(train_out, train_expected)
val_accuracy = common.accuracy(val_out, val_expected) if val_fold else 0
test_accuracy = common.accuracy(test_out, test_expected) if test_fold else 0
return train_accuracy, val_accuracy, test_accuracy
def performance(self, train_fold, val_fold, test_fold=None):
with torch.no_grad():
# Actual output class indices
self.eval()
train_out = self.cached_train_out
val_out = self.forward_in_batches(
val_fold, self.batch_size) if val_fold else None
test_out = self.forward_in_batches(
test_fold, self.batch_size) if test_fold else None
# Expected output class indices
train_expected = torch.Tensor([d['y'] for d in train_fold])
val_expected = torch.Tensor([d['y'] for d in val_fold
]) if val_fold else None
test_expected = torch.Tensor([d['y'] for d in test_fold
]) if test_fold else None
save_raw_predictions = False
if save_raw_predictions:
raw_preds_filename = '/home/disarli/tmp/predictions.pt'
raw_train_out = self.forward_in_batches(train_fold,
self.batch_size,
return_probs=True)
raw_val_out = self.forward_in_batches(
val_fold, self.batch_size,
return_probs=True) if val_fold else None
raw_test_out = self.forward_in_batches(
test_fold, self.batch_size,
return_probs=True) if test_fold else None
try:
saved = torch.load(raw_preds_filename)
except FileNotFoundError:
saved = []
saved.append({
'train_out':
raw_train_out.cpu(),
'train_expected':
train_expected.cpu(),
'val_out':
raw_val_out.cpu() if val_fold else None,
'val_expected':
val_expected.cpu() if val_fold else None,
'test_out':
raw_test_out.cpu() if test_fold else None,
'test_expected':
test_expected.cpu() if test_fold else None,
})
torch.save(saved, raw_preds_filename)
# Compute performance measures
train_accuracy = common.accuracy(train_out, train_expected)
val_accuracy = common.accuracy(val_out,
val_expected) if val_fold else 0
test_accuracy = common.accuracy(test_out,
test_expected) if test_fold else 0
return train_accuracy, val_accuracy, test_accuracy
def xgb():
print("Training an XGB Classifier")
params = {
"max_depth": 8,
"n_estimators": 400,
"learning_rate": 0.05,
"n_jobs": -1,
"subsample": 0.8,
"nthread": 4,
}
trX_, tvX_, trY_, tvY_ = train_test_split(trX, trYi, test_size=0.3)
gbm = XGBClassifier(**params)
print(gbm.get_xgb_params())
gbm.fit(trX_, trY_, eval_set=[(tvX_, tvY_)], verbose=True)
# Find training accuracy
trP = classes[gbm.predict(trX)]
print("Training Accuracy: ", 100 * accuracy(trY, trP))
# Dump test labels
tsP = classes[gbm.predict(tsX)]
write_csv("xgb_d5_n150.csv", tsP)
def train(epoch):
train_sampler.set_epoch(epoch)
model.train()
losses = AverageMeter()
top1 = AverageMeter()
global best_pred, acclist_train
for batch_idx, (data, target) in enumerate(train_loader):
scheduler(optimizer, batch_idx, epoch, best_pred)
if not args.mixup:
data, target = data.cuda(args.gpu), target.cuda(args.gpu)
optimizer.zero_grad()
output = model(data)
loss = criterion(output, target)
loss.backward()
optimizer.step()
if not args.mixup:
acc1 = accuracy(output, target, topk=(1,))
top1.update(acc1[0], data.size(0))
losses.update(loss.item(), data.size(0))
if batch_idx % 100 == 0 and args.gpu == 0:
if args.mixup:
print('Batch: %d| Loss: %.3f'%(batch_idx, losses.avg))
else:
print('Batch: %d| Loss: %.3f | Top1: %.3f'%(batch_idx, losses.avg, top1.avg))
acclist_train += [top1.avg]
def performance_from_out(self, output, expected):
"""
Given a tensor of network outputs and a tensor of expected outputs, returns the performance
:param output:
:param expected:
:return:
"""
output = output.argmax(dim=1).cpu()
return common.accuracy(output, expected)
def test_mlp(devs, kv_type):
# guarantee the same weight init for each run
mx.random.seed(0)
logging.basicConfig(level=logging.DEBUG)
(train, val) = common.mnist(batch_size=100, input_shape=(784,))
# train
model = mx.model.FeedForward.create(
symbol=common.mlp(), ctx=devs, X=train, num_epoch=4, learning_rate=0.1, wd=0.0004, momentum=0.9, kvstore=kv_type
)
return common.accuracy(model, val)
def find_best_alpha(self, train_fold, val_fold, batch_size):
"""
Fit the model while searching for the best regularization parameter. The best regularization
parameter is then assigned to self.alpha.
:param train_fold:
:param val_fold:
:param batch_size:
:return:
"""
# Collect the states without computing the readout
train_states = self.forward_in_batches(train_fold,
batch_size,
return_states=True)
train_expected = torch.Tensor([d['y'] for d in train_fold])
val_states = self.forward_in_batches(val_fold,
batch_size,
return_states=True)
val_expected = torch.Tensor([d['y'] for d in val_fold])
#possible_alpha_exponents = [-6, -5, -4, -3, -2, -1, 0, 1, 0.5, 0.8, 2, 2.5, 2.8, 3, 4, 5, 6]
#possible_alpha_exponents = [0, 1, 0.5, 0.8, 2, 2.5, 2.8, 3, 4, 5, 6]
possible_alpha_exponents = [
0, 1, 0.5, 0.8, 2, 2.2, 2.5, 2.6, 2.7, 2.8, 2.9, 3, 3.1, 3.2, 3.3,
3.4, 3.5, 3.6, 3.8, 4, 5, 6
]
possible_alphas = [10**v for v in possible_alpha_exponents]
best_alpha = None
best_val_perf = -math.inf
for a in possible_alphas:
# Fit with alpha = a
self.alpha = a
self._fit_from_states(train_states, train_expected)
# Check the validation accuracy for this alpha
self.eval()
val_out = self._predict_from_states(val_states)
val_perf = common.accuracy(val_out, val_expected)
# Update the best alpha
if val_perf > best_val_perf:
best_val_perf = val_perf
best_alpha = a
self.alpha = best_alpha
return best_alpha
def test_lenet(devs, kv_type):
# guarantee the same weight init for each run
mx.random.seed(0)
logging.basicConfig(level=logging.DEBUG)
# (train, val) = cifar10(batch_size = 128, input_shape=(3,28,28))
(train, val) = mnist(batch_size=100, input_shape=(1, 28, 28))
model = mx.model.FeedForward.create(ctx=devs,
kvstore=kv_type,
symbol=lenet,
X=train,
num_round=3,
learning_rate=0.1,
momentum=0.9,
wd=0.00001)
return accuracy(model, val)
def test_mlp(devs, kv_type):
# guarantee the same weight init for each run
mx.random.seed(0)
logging.basicConfig(level=logging.DEBUG)
(train, val) = mnist(batch_size=102, input_shape=(784, ))
# train
model = mx.model.FeedForward.create(symbol=softmax,
ctx=devs,
X=train,
num_round=2,
learning_rate=0.1,
wd=0.0004,
momentum=0.9,
kvstore=kv_type)
return accuracy(model, val)
def part_a(max_iter=300):
print()
print("Training Kmeans (max_iter=%d)" % max_iter)
kmeans = KMeans(n_init=10,
n_clusters=20,
max_iter=max_iter,
random_state=0).fit(trX)
labels = cluster_labels(kmeans, trY)
# Find training accuracy
trP = labels[kmeans.predict(trX)]
print("Training Accuracy: ", 100 * accuracy(trY, trP))
# Dump test labels
# Test accuracy can only be calculated by uploading to Kaggle
tsP = labels[kmeans.predict(tsX)]
write_csv("kmeans_%d.csv" % max_iter, tsP)
def test_inception(devs, kv_type):
# guarantee the same weight init for each run
mx.random.seed(0)
logging.basicConfig(level=logging.DEBUG)
(train, val) = common.cifar10(batch_size=128, input_shape=(3, 28, 28))
model = mx.model.FeedForward.create(ctx=devs,
symbol=common.inception(),
X=train,
eval_data=val,
kvstore=kv_type,
num_round=10,
learning_rate=0.1,
momentum=0.9,
wd=0.00001,
initializer=mx.init.Uniform(0.07))
return common.accuracy(model, val)
def test_lenet(devs, kv_type):
# guarantee the same weight init for each run
mx.random.seed(0)
logging.basicConfig(level=logging.DEBUG)
# (train, val) = common.cifar10(batch_size = 128, input_shape=(3,28,28))
(train, val) = common.mnist(batch_size = 100, input_shape=(1,28,28))
model = mx.model.FeedForward.create(
ctx = devs,
kvstore = kv_type,
symbol = common.lenet(),
X = train,
num_round = 3,
learning_rate = 0.1,
momentum = 0.9,
wd = 0.00001)
return common.accuracy(model, val)
def validate(epoch):
model.eval()
top1 = AverageMeter()
top5 = AverageMeter()
global best_pred, acclist_train, acclist_val
is_best = False
for batch_idx, (data, target) in enumerate(val_loader):
data, target = data.cuda(args.gpu), target.cuda(args.gpu)
with torch.no_grad():
output = model(data)
acc1, acc5 = accuracy(output, target, topk=(1, 5))
top1.update(acc1[0], data.size(0))
top5.update(acc5[0], data.size(0))
# sum all
sum1, cnt1, sum5, cnt5 = torch_dist_sum(args.gpu, top1.sum, top1.count, top5.sum, top5.count)
if args.eval:
if args.gpu == 0:
top1_acc = sum(sum1) / sum(cnt1)
top5_acc = sum(sum5) / sum(cnt5)
print('Validation: Top1: %.3f | Top5: %.3f'%(top1_acc, top5_acc))
return
if args.gpu == 0:
top1_acc = sum(sum1) / sum(cnt1)
top5_acc = sum(sum5) / sum(cnt5)
print('Validation: Top1: %.3f | Top5: %.3f'%(top1_acc, top5_acc))
# save checkpoint
acclist_val += [top1_acc]
if top1_acc > best_pred:
best_pred = top1_acc
is_best = True
save_checkpoint({
'epoch': epoch,
'state_dict': model.module.state_dict(),
'optimizer': optimizer.state_dict(),
'best_pred': best_pred,
'acclist_train':acclist_train,
'acclist_val':acclist_val,
}, args=args, is_best=is_best)
def test_inception(devs, kv_type):
# guarantee the same weight init for each run
mx.random.seed(0)
logging.basicConfig(level=logging.DEBUG)
(train, val) = common.cifar10(batch_size = 128, input_shape=(3,28,28))
model = mx.model.FeedForward.create(
ctx = devs,
symbol = common.inception(),
X = train,
eval_data = val,
kvstore = kv_type,
num_epoch = 10,
learning_rate = 0.1,
momentum = 0.9,
wd = 0.00001,
initializer = mx.init.Uniform(0.07))
return common.accuracy(model, val)
def score(self, data):
"""Find accuracy of dtree over data."""
predictions = [self._predict(self.root, x) for x in data]
return 100 * accuracy(data[:, 0], predictions)
#!/usr/bin/env python
import mxnet as mx
import logging
import common
mx.random.seed(0)
logging.basicConfig(level=logging.DEBUG)
kv = mx.kvstore.create('dist_async')
(train, val) = common.mnist(num_parts = kv.num_workers,
part_index = kv.rank,
batch_size = 100,
input_shape = (784,))
# train
model = mx.model.FeedForward.create(
symbol = common.mlp(),
ctx = mx.cpu(),
X = train,
num_round = 4,
learning_rate = 0.05,
wd = 0.0004,
momentum = 0.9,
kvstore = kv)
common.accuracy(model, val)
#!/usr/bin/env python
import common
import mxnet as mx
import logging
mx.random.seed(0)
logging.basicConfig(level=logging.DEBUG)
kv = mx.kvstore.create('dist_async')
(train, val) = common.mnist(num_parts=kv.num_workers,
part_index=kv.rank,
batch_size=100,
input_shape=(1, 28, 28))
model = mx.model.FeedForward.create(ctx=mx.gpu(kv.rank),
kvstore=kv,
symbol=common.lenet(),
X=train,
num_epoch=10,
learning_rate=0.05,
momentum=0.9,
wd=0.00001)
common.accuracy(model, val)
def ensemble_performance(predictions, expected):
out = torch.stack(predictions).mean(dim=0)
out = out.argmax(dim=1)
return common.accuracy(out, expected)
def finetune_stage1(net,
train_loader,
val_loader,
fold=None,
weight_path='finetune_stage_1_weights',
epochs=80):
mkdir(weight_path)
log = open('log{}.txt'.format(fold), 'a+')
net.load_state_dict(
torch.load('stage_1_weights/Stage_1_Fold_{}'.format(fold)))
print('\nModel weights from stage_1_weights/Stage_1_Fold_{} Loaded'.format(
fold))
log.write(
'\nModel weights from stage_1_weights/Stage_1_Fold_{} Loaded\n'.format(
fold))
stat = pd.DataFrame(columns=[
'Training Loss', 'Training Acc', 'Training IoU', 'Valid Loss',
'Valid Acc', 'Valid IoU'
])
best_val_iou = .0
train_loss_record, train_acc_record, train_iou_record = [], [], []
val_loss_record, val_acc_record, val_iou_record = [], [], []
start = time.time()
optimizer = optim.SGD(net.parameters(),
lr=0.005,
momentum=0.9,
weight_decay=0.0001)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='max',
factor=0.5,
patience=8)
print('\nFold-{} Finetune Training Overview'.format(fold))
print('| Train | Val |')
print('|-----------------------------------------------------|')
print('| Loss | Acc | IoU | Loss | Acc | IoU |')
log.write('\nFold-{} Finetune Training Overview\n'.format(fold))
log.write('| Train | Val |\n')
log.write('|-----------------------------------------------------|\n')
log.write('| Loss | Acc | IoU | Loss | Acc | IoU |\n')
early_stop_counter = 0
for epoch in range(epochs):
if early_stop_counter > 15:
print('\nEarly stop at epoch {}!'.format(epoch))
log.write('\nEarly stop at epoch {}!\n'.format(epoch))
break
epoch_loss = 0
epoch_acc = 0
epoch_iou = 0
for batch_idx, batch in enumerate(train_loader):
data, target = batch['image'].to(device), batch['mask'].to(device)
output = net(data)
# add sigmoid to the logits
loss = net.criterion(output, target)
acc = accuracy(output, target)
iou_ = iou(output, target.int())
loss.backward()
optimizer.step()
optimizer.zero_grad()
epoch_loss += loss.item() / len(train_loader)
epoch_acc += acc.item() / len(train_loader)
epoch_iou += iou_.item() / len(train_loader)
print('|{:9.4f}|{:8.4f}|{:7.4f}'.format(epoch_loss, epoch_acc,
epoch_iou),
end='')
log.write('|{:9.4f}|{:8.4f}|{:7.4f}'.format(epoch_loss, epoch_acc,
epoch_iou))
train_loss_record.append(epoch_loss)
train_acc_record.append(epoch_acc)
train_iou_record.append(epoch_iou)
with torch.no_grad():
epoch_val_loss = 0
epoch_val_acc = 0
epoch_val_iou = 0
for batch_idx, batch in enumerate(val_loader):
data, target = batch['image'].to(device), batch['mask'].to(
device)
output = net(data)
loss = net.criterion(output, target)
acc = accuracy(output, target)
iou_ = iou(output, target.int())
epoch_val_loss += loss.item() / len(val_loader)
epoch_val_acc += acc.item() / len(val_loader)
epoch_val_iou += iou_.item() / len(val_loader)
# Learning rate decay step
scheduler.step(epoch_val_iou)
print('|{:9.4f}|{:8.4f}|{:7.4f}|'.format(epoch_val_loss,
epoch_val_acc,
epoch_val_iou))
log.write('|{:9.4f}|{:8.4f}|{:7.4f}|\n'.format(
epoch_val_loss, epoch_val_acc, epoch_val_iou))
val_loss_record.append(epoch_val_loss)
val_acc_record.append(epoch_val_acc)
val_iou_record.append(epoch_val_iou)
if epoch_val_iou > best_val_iou:
early_stop_counter = 0
best_val_iou = epoch_val_iou
torch.save(net.state_dict(),
'{}/Stage_2_Fold_{}'.format(weight_path, fold))
else:
early_stop_counter += 1
# stat['Epoch'] = [_ for _ in range(1, epochs + 1)]
# stat['Stage'] = 1
# stat['Fold'] = fold
# stat['Training Loss'] = train_loss_record
# stat['Training Acc'] = train_acc_record
# stat['Training IoU'] = train_iou_record
# stat['Valid Loss'] = val_loss_record
# stat['Valid Acc'] = val_acc_record
# stat['Valid IoU'] = val_iou_record
print('Best --> {}'.format(best_val_iou))
log.write('Best --> {}\n'.format(best_val_iou))
print('Time used', time.time() - start)
return net, stat
loss = loss_fn(pred, label)
else:
# CutMix : generate mixed sample
lam = np.random.beta(cutmix_alpha, cutmix_alpha)
rand_index = torch.randperm(x.size()[0]).cuda()
target_a = label
target_b = label[rand_index]
bbx1, bby1, bbx2, bby2 = rand_bbox(x.size(), lam)
x[:, :, bbx1:bbx2, bby1:bby2] = x[rand_index, :, bbx1:bbx2, bby1:bby2]
pred = model(x)
lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.size()[-1] * x.size()[-2]))
loss = loss_fn(pred, target_a) * lam + loss_fn(pred, target_b) * (1. - lam)
w.start(tag='step3')
top1, top5 = accuracy(pred, label, (1, 5))
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), C.get()['optimizer'].get('clip', 5))
optimizer.step()
w.pause(tag='step3')
metrics.add_dict({
'loss': loss.item() * len(x),
'top1': top1.item() * len(x),
'top5': top5.item() * len(x),
})
cnt += len(x)
model.eval()
postfix = metrics / cnt
"""
Find accuracy from two files that contain a label per line.
"""
import sys
from common import accuracy
if __name__ == '__main__':
with open(sys.argv[1], "r") as f:
actual = list(map(lambda line: line.strip(), f.readlines()))
with open(sys.argv[2], "r") as f:
predicted = list(map(lambda line: line.strip(), f.readlines()))
print("Actual labels: ", sys.argv[1])
print("Predicted labels: ", sys.argv[2])
print("Accuracy: ", accuracy(actual, predicted) * 100, "%")
def finetune_stage(net,
train_loader,
val_loader,
fold=None,
weight_path='finetune_weights',
n_cycle=6,
initial_lr=0.01,
epochs_per_cycle=50):
mkdir(f'{weight_path}_fold{fold}')
log = open('finetune-log{}.txt'.format(fold), 'a+')
net.load_state_dict(torch.load(f'stage_2_weights/NoDepth_Fold-{fold}'))
print(f'\nModel weights from stage_2_weights/NoDepth_Fold-{fold} Loaded')
log.write(
f'\nModel weights from stage_2_weights/NoDepth_Fold-{fold} Loaded\n')
lr_record = []
train_loss_record, train_acc_record, train_iou_record = [], [], []
val_loss_record, val_acc_record, val_iou_record = [], [], []
optimizer = optim.SGD(net.parameters(),
lr=initial_lr,
momentum=0.9,
weight_decay=0.0001)
for cycle in range(n_cycle):
start = time.time()
print(f'\nFold # {fold} Snapshot # {cycle} Overview')
print('| Train | Val |')
print('|-----------------------------------------------------|')
print('| Loss | Acc | IoU | Loss | Acc | IoU |')
log.write(f'\nFold # {fold} Snapshot # {cycle} Overview\n')
log.write('| Train | Val |\n')
log.write('|-----------------------------------------------------|\n')
log.write('| Loss | Acc | IoU | Loss | Acc | IoU |\n')
best_val_iou = .0
for epoch in range(epochs_per_cycle):
epoch_loss = 0
epoch_acc = 0
epoch_iou = 0
lr = cos_annealing_lr(initial_lr, epoch, epochs_per_cycle)
if lr < 0.001:
break
lr_record.append(lr)
optimizer.state_dict()['param_groups'][0]['lr'] = lr
for batch_idx, batch in enumerate(train_loader):
data, target = batch['image'].to(device), batch['mask'].to(
device)
output = net(data)
loss = net.criterion(output, target.long())
acc = accuracy(output, target)
iou_ = iou(output, target.int())
loss.backward()
optimizer.step()
optimizer.zero_grad()
epoch_loss += loss.item() / len(train_loader)
epoch_acc += acc.item() / len(train_loader)
epoch_iou += iou_.item() / len(train_loader)
print(f'|{epoch_loss:9.4f}|{epoch_acc:8.4f}|{epoch_iou:7.4f}',
end='')
log.write(f'|{epoch_loss:9.4f}|{epoch_acc:8.4f}|{epoch_iou:7.4f}')
train_loss_record.append(epoch_loss)
train_acc_record.append(epoch_acc)
train_iou_record.append(epoch_iou)
with torch.no_grad():
epoch_val_loss = 0
epoch_val_acc = 0
epoch_val_iou = 0
for batch_idx, batch in enumerate(val_loader):
data, target = batch['image'].to(device), batch['mask'].to(
device)
output = net(data)
loss = net.criterion(output, target.long())
acc = accuracy(output, target)
iou_ = iou(output, target.int())
epoch_val_loss += loss.item() / len(val_loader)
epoch_val_acc += acc.item() / len(val_loader)
epoch_val_iou += iou_.item() / len(val_loader)
print(
f'|{epoch_val_loss:9.4f}|{epoch_val_acc:8.4f}|{epoch_val_iou:7.4f}|'
)
log.write(
f'|{epoch_val_loss:9.4f}|{epoch_val_acc:8.4f}|{epoch_val_iou:7.4f}|{epoch}\n'
)
val_loss_record.append(epoch_val_loss)
val_acc_record.append(epoch_val_acc)
val_iou_record.append(epoch_val_iou)
if epoch_val_iou > best_val_iou:
best_val_iou = epoch_val_iou
torch.save(
net.state_dict(),
f'{weight_path}_fold{fold}/cycle_{cycle}_{epoch_val_iou}'
)
print(f'Best Result: {best_val_iou}')
log.write(f'Best Result: {best_val_iou}\n')
print('Time used', time.time() - start)
def train_stage1(net,
optimizer,
train_loader,
val_loader,
fold=None,
weight_path='stage_1_weights',
epochs=30):
mkdir(weight_path)
log = open('log{}.txt'.format(fold), 'a+')
stat = pd.DataFrame(columns=[
'Training Loss', 'Training Acc', 'Training IoU', 'Valid Loss',
'Valid Acc', 'Valid IoU'
])
best_val_iou = .0
train_loss_record, train_acc_record, train_iou_record = [], [], []
val_loss_record, val_acc_record, val_iou_record = [], [], []
# criterion = torch.nn.BCELoss()
criterion = FocalLoss2d()
start = time.time()
print(f'\nFold-{fold} Warm-up Training Overview')
print('| Train | Val |')
print('|-----------------------------------------------------|')
print('| Loss | Acc | IoU | Loss | Acc | IoU |')
log.write(f'\nFold-{fold} Warm-up Training Overview\n')
log.write('| Train | Val |\n')
log.write('|-----------------------------------------------------|\n')
log.write('| Loss | Acc | IoU | Loss | Acc | IoU |\n')
for epoch in range(epochs):
epoch_loss = 0
epoch_acc = 0
epoch_iou = 0
for batch_idx, batch in enumerate(train_loader):
data, target = batch['image'].to(device), batch['mask'].to(device)
output = net(data)
# add sigmoid to the logits
loss = criterion(nn.Sigmoid()(output), target)
acc = accuracy(output, target)
iou_ = iou(output, target.int())
loss.backward()
optimizer.step()
optimizer.zero_grad()
epoch_loss += loss.item() / len(train_loader)
epoch_acc += acc.item() / len(train_loader)
epoch_iou += iou_.item() / len(train_loader)
print(f'|{epoch_loss:9.4f}|{epoch_acc:8.4f}|{epoch_iou:7.4f}', end='')
log.write(f'|{epoch_loss:9.4f}|{epoch_acc:8.4f}|{epoch_iou:7.4f}')
train_loss_record.append(epoch_loss)
train_acc_record.append(epoch_acc)
train_iou_record.append(epoch_iou)
with torch.no_grad():
epoch_val_loss = 0
epoch_val_acc = 0
epoch_val_iou = 0
for batch_idx, batch in enumerate(val_loader):
data, target = batch['image'].to(device), batch['mask'].to(
device)
output = net(data)
loss = criterion(nn.Sigmoid()(output), target)
acc = accuracy(output, target)
iou_ = iou(output, target.int())
epoch_val_loss += loss.item() / len(val_loader)
epoch_val_acc += acc.item() / len(val_loader)
epoch_val_iou += iou_.item() / len(val_loader)
print(
f'|{epoch_val_loss:9.4f}|{epoch_val_acc:8.4f}|{epoch_val_iou:7.4f}|'
)
log.write(
f'|{epoch_val_loss:9.4f}|{epoch_val_acc:8.4f}|{epoch_val_iou:7.4f}|\n'
)
val_loss_record.append(epoch_val_loss)
val_acc_record.append(epoch_val_acc)
val_iou_record.append(epoch_val_iou)
if epoch_val_iou > best_val_iou:
best_val_iou = epoch_val_iou
torch.save(net.state_dict(),
f'{weight_path}/Stage-1_Fold-{fold}')
stat['Epoch'] = [_ for _ in range(1, epochs + 1)]
stat['Stage'] = 1
stat['Fold'] = fold
stat['Training Loss'] = train_loss_record
stat['Training Acc'] = train_acc_record
stat['Training IoU'] = train_iou_record
stat['Valid Loss'] = val_loss_record
stat['Valid Acc'] = val_acc_record
stat['Valid IoU'] = val_iou_record
print('Time used', time.time() - start)
return net, stat
def score(self, X, y):
"""Calculate accuracy of this net on data."""
return accuracy(y, self.predict(X))
def full_grad_descent(X,
y,
Xv,
yv,
Xt,
yt,
alpha,
reg,
lossType,
epochs,
err_tolerance=1e-7):
"""Run Batched Gradient Descent.
:param X: training input
:param y: training labels
:param Xv: validation input
:param yv: validation labels
:param Xt: test input
:param yt: test labels
:param alpha: learning rate
:parma reg: Regularization param(lambda)
:param lossType: 'ce' or 'mse', loss function type
:param epochs: training epochs
:returns: w, loss data
"""
w = np.zeros(d)
perf_logger = PerfLogger([
'train_loss', 'train_accuracy', 'valid_loss', 'valid_accuracy',
'test_loss', 'test_accuracy'
])
if lossType == 'ce':
loss_func = crossEntropyLoss
elif lossType == 'mse':
loss_func = MSE
else:
sys.stderr.write('Invalid lossType: %s' % lossType)
sys.exit()
last_loss = -1
stop = False
for i in range(1, epochs + 1):
w, loss = grad_descent_step(w, X, y, alpha, reg, lossType=lossType)
if last_loss != -1 and abs(last_loss - loss) < err_tolerance:
stop = True
# To accelerate learning, we only compute valid/test loss every 10 epochs
if i == 1 or i % 10 == 0 or i == epochs or stop:
perf_logger.append(i, {
'train_loss': loss,
'train_accuracy': accuracy(w, X, y),
'valid_loss': loss_func(w, Xv, yv, reg),
'valid_accuracy': accuracy(w, Xv, yv),
'test_loss': loss_func(w, Xt, yt, reg),
'test_accuracy': accuracy(w, Xt, yt),
},
print_log=True)
if stop:
break
last_loss = loss
return w, perf_logger
params = {
'reg': reg,
}
model_file, loss_file, time_file = make_filenames(
path, [alg], params)
perf_logger = PerfLogger([
'train_loss', 'train_accuracy', 'valid_loss', 'valid_accuracy',
'test_loss', 'test_accuracy'
])
time_start = time.time()
with TimeThis(time_file, params):
w = linear_regression_normal_equation(X, y, reg)
perf_logger.append(
1, {
'train_loss': MSE(w, X, y, reg),
'train_accuracy': accuracy(w, X, y),
'valid_loss': MSE(w, Xv, yv, reg),
'valid_accuracy': accuracy(w, Xv, yv),
'test_loss': MSE(w, Xt, yt, reg),
'test_accuracy': accuracy(w, Xt, yt),
})
# Save model and loss data
save_model(model_file, w)
perf_logger.save(loss_file)
