class AutoEncoderTrainer(object):
def __init__(self, args,
trainRegressionDataLoader, trainRegressionClassificationLoader,
testDataLoader, trainRainFallLoader,
means, std):
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
self.trainRegressionDataLoader = trainRegressionDataLoader
self.trainRegressionClassificationLoader = trainRegressionClassificationLoader
self.testDataLoader = testDataLoader
self.classificationLoader = trainRainFallLoader
self.run_datetime = datetime.datetime.now()
self.out_path = args.out
self.sigma = args.sigma
self.beta = args.beta
self.earlyStop = args.earlyStop
self.nClass = args.nClass
self.noiseMean = torch.zeros(args.batch_size, args.featureNums, 17, 17)
self.noiseStd = 1e-3
self.model = AutoencoderBN(self.noiseMean, self.noiseStd).to(self.device)
self.regressionModel = Regression(self.nClass).to(self.device)
self.classificationModel = regressionClassification(self.nClass).to(self.device)
self.rainFallClassifierModel = rainFallClassification().to(self.device)
self.meanStdNormalizer = MeanVarianceNormalizer(means, std).to(self.device)
self.meanvarLoss = MeanVarLoss(self.nClass).to(self.device)
self.normaliedLoss = NormalizerLoss(std).to(self.device)
self.focalLoss = FocalLoss(self.nClass, alpha=0.25, gamma=2).to(self.device)
self.rainFocalLoss = FocalLoss(2, alpha=0.25, gamma=2).to(self.device)
self.regressionOptim = torch.optim.Adam([
{'params': self.regressionModel.parameters(), 'lr': args.lr,
'weight_decay': args.weight_decay},
{'params': self.model.parameters(), 'lr': args.lr,
'weight_decay': args.weight_decay},
],
lr=args.lr * 10, weight_decay=args.weight_decay * 10)
self.classificationOptim = torch.optim.Adam(self.classificationModel.parameters(), lr=args.lr * 100)
self.rainFallOptim = torch.optim.Adam(self.rainFallClassifierModel.parameters(), lr=args.lr * 10)
# self.reconstructOptim = torch.optim.Adam(self.model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
self.scheduler = torch.optim.lr_scheduler.StepLR(self.regressionOptim, step_size=750 * 2)
self.criterion = nn.MSELoss()
self.classificationCriterion = nn.CrossEntropyLoss()
if not os.path.exists(self.out_path):
os.makedirs(self.out_path)
self.logger = Logger(self.out_path)
with open(os.path.join(self.out_path, "para.json"), "w") as f:
json.dump(args.__dict__, f)
self.epoch = 0
self.iteration = 0
self.classificationIteration = 0
self.rainfallclassificationIteration = 0
self.test_step = 0
self.max_epoch = args.epochs
self.val_interval = args.interval
self.res = 0
self.bestConstructLoss = 1e7
self.bestConstructEpoch = 0
self.best_error = 1e7;
self.best_res_epoch = 0
def mask_norm(self, mask, threshold=0.5):
mask_ = mask * (mask > threshold).float()
mask_ = mask_ / mask_.sum(1).unsqueeze(-1)
return mask_
def generateOneHot(self, softmax):
maxIdxs = torch.argmax(softmax, dim=1, keepdim=True).cpu().long()
oneHotMask = torch.zeros(softmax.shape, dtype=torch.float32)
oneHotMask = oneHotMask.scatter_(1, maxIdxs, 1.0)
oneHotMask = oneHotMask.unsqueeze(-2)
return oneHotMask
def validate_one_epoch(self):
self.model.eval()
self.regressionModel.eval()
self.classificationModel.eval()
self.rainFallClassifierModel.eval()
self.test_step += 1
tsthreas = [0.1, 1, 10]
tp = [0] * len(tsthreas) # true positive
tn = [0] * len(tsthreas) # true negetive
fp = [0] * len(tsthreas) # false positve
fn = [0] * len(tsthreas) # false negetive
ts = [0] * len(tsthreas)
totalRegressionLoss = []
totalReconstructLoss = []
totalClassificationLoss = []
totalRClassificationLoss = []
total_error = 0
total_count = 0
p_error = 0
ps_error = 0
p_count = 0
pxErrorList = [0] * (self.nClass)
pxsErrorList = [0] * (self.nClass)
pxCountList = [0] * (self.nClass)
pxAverageError = [0] * (self.nClass)
pxsAverageError = [0] * (self.nClass)
classCorrect = [0] * (self.nClass)
classCounnt = [0] * (self.nClass)
accuray = [0] * (self.nClass + 1)
rainCorrect = [0] * 2
rainCount = [0] * 2
rainAccuracy = [0] * 3
for batch_idx, (data, target, rainClass, rainMask, regressionClass, regressionMask) in tqdm.tqdm(
enumerate(self.testDataLoader), total=len(self.testDataLoader),
desc='Test Test Data :', ncols=80,
leave=False):
rainNumpy = rainClass.numpy()
regressionNumpy = regressionClass.numpy()
one_hot_mask = regressionMask.numpy()
gt_micaps = target.numpy()
data = data.to(device=self.device)
target = target.to(device=self.device)
rainClass = rainClass.to(device=self.device)
# rainMask = rainMask.to(device=self.device)
regressionClass = regressionClass.to(device=self.device)
regressionMask = regressionMask.to(device=self.device).unsqueeze(-2)
with torch.no_grad():
encoder, decoder = self.model(data)
predictValues = self.regressionModel(encoder)
rainPreds = self.rainFallClassifierModel(data)
regressionPreds = self.classificationModel(data)
rainPredsSoftMax = F.softmax(rainPreds, dim=1)
regressionPredsSoftmax = F.softmax(regressionPreds, dim=1)
# if predict class belong to the last class , the output will be set zero
rainOneHotMask = self.generateOneHot(rainPredsSoftMax).to(self.device)
regressionOneHotMask = self.generateOneHot(regressionPredsSoftmax).to(self.device)
predictValues = self.meanStdNormalizer(predictValues).unsqueeze(-1)
# print(predictValues[0])
regressionValues = torch.matmul(regressionOneHotMask, predictValues).squeeze(-1)
# print(regressionValues[0])
zeros = torch.zeros(regressionValues.size()).to(self.device)
regressionValues = torch.matmul(rainOneHotMask,
torch.cat([zeros, regressionValues], dim=1).unsqueeze(-1)).squeeze(-1)
# print("res: ",regressionValues[:10])
# print("resSum: ",regressionValues.mean())
# print("target: ",target[:10])
# print(regressionValues[0])
# print(target[0])
# Three loss reconstruct loss , regression Loss and classification Loss
regressionLoss = self.criterion(regressionValues, target)
reconstructLoss = self.criterion(decoder, data)
rainClassificationLoss = self.classificationCriterion(rainPreds, rainClass)
regressionClassificationLoss = self.classificationCriterion(regressionPreds, regressionClass)
rainPredicted = torch.argmax(rainPredsSoftMax, dim=1).cpu().numpy()
predicted = torch.argmax(regressionPredsSoftmax, dim=1).cpu().numpy()
for i in range(self.nClass):
classCorrect[i] += np.sum((predicted == i) * (regressionNumpy == i) * (rainNumpy == 1))
classCounnt[i] += np.sum((regressionNumpy == i) * (rainNumpy == 1))
for i in range(2):
rainCorrect[i] += np.sum((rainPredicted == i) * (rainNumpy == i))
rainCount[i] += np.sum(rainNumpy == i)
predictNumpy = regressionValues.cpu().numpy()
# biasNumpy = resValues.cpu().numpy()
# labelsIndex = predicted.cpu().numpy()
# predictNumpy = np.array([[biasNumpy[i,0]*(idx<self.nClass) + self.center[idx]] for i,idx in enumerate(labelsIndex)])
totalRegressionLoss.append(regressionLoss.item())
totalReconstructLoss.append(reconstructLoss.item())
totalClassificationLoss.append(regressionClassificationLoss.item())
totalRClassificationLoss.append(rainClassificationLoss.item())
gapValues = np.abs(predictNumpy - gt_micaps)
total_error += np.sum(gapValues)
total_count += gapValues.shape[0]
# print(gt_micaps[:10])
# print(one_hot_mask[:10])
p_ae = (gt_micaps > 0.05) * gapValues
p_error += np.sum(p_ae)
ps_error += np.sum(p_ae ** 2)
p_count += np.sum(gt_micaps > 0.05)
for i in range(self.nClass):
ae = one_hot_mask[:, i].reshape(-1, 1) * gapValues
pxErrorList[i] += np.sum(ae)
pxsErrorList[i] += np.sum(ae ** 2)
pxCountList[i] += np.sum(one_hot_mask[:, i])
for i, threas in enumerate(tsthreas):
tp[i] += np.sum((gt_micaps >= threas) * (predictNumpy >= threas))
tn[i] += np.sum((gt_micaps < threas) * (predictNumpy < threas))
fp[i] += np.sum((gt_micaps < threas) * (predictNumpy >= threas))
fn[i] += np.sum((gt_micaps >= threas) * (predictNumpy < threas))
for i, _ in enumerate(tsthreas):
ts[i] += round(tp[i] / (tp[i] + fp[i] + fn[i]), 5)
totalAverageError = round(total_error / total_count, 5)
pAverageError = round(p_error / p_count, 5)
psAverageError = round(ps_error / p_count - pAverageError ** 2, 5)
for i in range(self.nClass):
pxAverageError[i] += round(pxErrorList[i] / pxCountList[i], 5)
pxsAverageError[i] += round(pxsErrorList[i] / pxCountList[i] - pxAverageError[i] ** 2, 5)
totalLoss = np.mean(totalRegressionLoss)
totalRLoss = np.mean(totalReconstructLoss)
totalCLoss = np.mean(totalClassificationLoss)
for i in range(self.nClass):
accuray[i] += round(classCorrect[i] / classCounnt[i], 5)
accuray[self.nClass] += round(sum(classCorrect) / sum(classCounnt), 5)
for i in range(2):
rainAccuracy[i] += round(rainCorrect[i] / rainCount[i], 5)
rainAccuracy[2] += round(sum(rainCorrect) / sum(rainCount), 5)
info = {"test_regression_loss": totalLoss,
"test_reconstruct_loss": totalRLoss,
"test_classification_loss": totalCLoss,
"aver_gap": totalAverageError,
"aver_p_gap": pAverageError,
"aver_ps_gap": psAverageError,
"p_num": p_count,
}
tsDisplay = list(zip(tp, tn, fp, fn, ts))
classStatistics = {
"average_p_gap": pxAverageError,
"aver_p_s_gap": pxsAverageError,
"p_count": pxCountList,
"ts_score": tsDisplay,
"test_rain_classification_accuracy": rainAccuracy,
"test_classification_accuracy": accuray,
}
print(info)
print(classStatistics)
if totalAverageError < self.best_error:
self.best_error = totalAverageError
self.best_res_epoch = self.epoch
info["epoch"] = self.epoch
info["modelParam"] = self.model.state_dict()
info["regressionParam"] = self.regressionModel.state_dict()
info["optimParam"] = self.regressionOptim.state_dict()
torch.save(info, os.path.join(self.out_path, str(self.epoch) + "_checkpoints.pth"))
def train_one_epoch_for_rainFall(self):
classCorrect = [0] * 2
classCounnt = [0] * 2
accuray = [0] * 3
self.rainFallClassifierModel.train()
for batch_idx, (data, target, rainClass, rainMask, regressionClass, regressionMask) in tqdm.tqdm(
enumerate(self.classificationLoader), total=len(self.classificationLoader),
desc='Train RainFall Classification epoch=%d' % self.epoch, ncols=100, leave=False):
iter_idx = batch_idx + self.epoch * len(self.classificationLoader)
self.rainfallclassificationIteration = iter_idx
assert self.rainFallClassifierModel.train
self.rainFallOptim.zero_grad()
logitNumpy = rainClass.numpy()
data = data.to(device=self.device)
logitsFloat = rainClass.float().to(device=self.device)
logits = rainClass.to(device=self.device)
preds = self.rainFallClassifierModel(data)
predsSoftmax = F.softmax(preds, dim=1)
classificationLoss = self.rainFocalLoss(preds, logits)
# classificationLoss = self.classificationCriterion(preds, logits)
classificationLoss.backward()
self.rainFallOptim.step()
classificationLossCpu = classificationLoss.item()
predicted = torch.argmax(predsSoftmax, dim=1).cpu().numpy()
for i in range(2):
classCorrect[i] += np.sum((predicted == i) * (logitNumpy == i))
classCounnt[i] += np.sum(logitNumpy == i)
self.logger.scalar_summary("train_rainfall_classification_loss", classificationLossCpu,
self.rainfallclassificationIteration + 1)
for i in range(2):
accuray[i] += round(classCorrect[i] / classCounnt[i], 5)
accuray[2] += round(sum(classCorrect) / sum(classCounnt), 5)
print("Train Rain Fall Classification Accuracy : ", accuray)
def train_one_epoch_for_classification(self):
classCorrect = [0] * self.nClass
classCounnt = [0] * self.nClass
accuray = [0] * (self.nClass + 1)
self.classificationModel.train()
for batch_idx, (data, target, rainClass, rainMask, regressionClass, regressionMask) in tqdm.tqdm(
enumerate(self.trainRegressionClassificationLoader),
total=len(self.trainRegressionClassificationLoader),
desc='Train Classification epoch=%d' % self.epoch, ncols=100, leave=False):
iter_idx = batch_idx + self.epoch * len(self.trainRegressionClassificationLoader)
self.classificationIteration = iter_idx
assert self.classificationModel.train
self.classificationOptim.zero_grad()
logitNumpy = regressionClass.numpy()
data = data.to(device=self.device)
logitsFloat = regressionClass.float().to(device=self.device)
logits = regressionClass.to(device=self.device)
preds = self.classificationModel(data)
predsSoftmax = F.softmax(preds, dim=1)
classificationLoss = self.focalLoss(preds, logits)
# classificationLoss = self.classificationCriterion(preds, logits)
meanLoss, varLoss = self.meanvarLoss(predsSoftmax, logitsFloat.unsqueeze(-1))
loss = classificationLoss + 1 * meanLoss + 0.5 * varLoss
loss.backward()
self.classificationOptim.step()
classificationLossCpu = loss.item()
predicted = torch.argmax(predsSoftmax, dim=1).cpu().numpy()
for i in range(self.nClass):
classCorrect[i] += np.sum((predicted == i) * (logitNumpy == i))
classCounnt[i] += np.sum(logitNumpy == i)
self.logger.scalar_summary("train_classification_loss", classificationLossCpu,
self.classificationIteration + 1)
for i in range(self.nClass):
accuray[i] += round(classCorrect[i] / classCounnt[i], 5)
accuray[self.nClass] += round(sum(classCorrect) / sum(classCounnt), 5)
print("Train classification Accuracy : ", accuray)
def train_one_epoch_for_regression(self):
self.model.train()
self.regressionModel.train()
for batch_idx, (data, target, rainClass, rainMask, regressionClass, regressionMask) in tqdm.tqdm(
enumerate(self.trainRegressionDataLoader), total=len(self.trainRegressionDataLoader),
desc='Train Regression epoch=%d' % self.epoch, ncols=100, leave=False):
iter_idx = batch_idx + self.epoch * len(self.trainRegressionDataLoader)
# if (self.iteration != 0) and (iter_idx - 1) != self.iteration:
# continue
self.iteration = iter_idx
assert self.regressionModel.training
self.regressionOptim.zero_grad()
# noise = torch.randn(data.size()).to(device=self.device)
noise = torch.normal(mean=self.noiseMean, std=self.noiseStd).to(self.device)
# noise = torch.normal(mean=self.noiseMean, std=self.noiseStd).to(device=self.device)
data = data.to(device=self.device)
rainMask = rainMask.to(device=self.device)
regressionMask = regressionMask.to(device=self.device)
noisedData = data + noise
target = target.to(device=self.device)
encoder, decoder = self.model(noisedData)
predictValues = self.regressionModel(encoder)
# thresholdMask = labels.narrow(1, 0, self.nClass).view(-1, 1, self.nClass)
predictValues = self.meanStdNormalizer(predictValues)
# resValues = torch.matmul(thresholdMask, predictValues).squeeze(-1)
regressionLoss = self.normaliedLoss(predictValues, target, rainMask, regressionMask)
# regressionLoss = self.criterion(resValues, target)
constructLoss = self.criterion(decoder, data)
# classificationLoss = self.classificationCriterion(preds, logits)
# meanLoss, varLoss = self.meanvarLoss(predictClassesSoftmax, logitsFloat.unsqueeze(-1))
loss = constructLoss + self.sigma * regressionLoss
# loss = constructLoss + self.sigma* regressionLoss
loss.backward()
# for param in self.model.parameters():
# print(param.grad.data.sum())
self.regressionOptim.step()
constructLossCpu = constructLoss.item()
regressionLossCpu = regressionLoss.item()
self.logger.scalar_summary("train_construct_loss", constructLossCpu, self.iteration + 1)
self.logger.scalar_summary("train_regression_loss", regressionLossCpu, self.iteration + 1)
def run(self):
for epoch in tqdm.trange(self.epoch, self.max_epoch,
desc='Experiments ', ncols=100):
self.epoch = epoch
self.train_one_epoch_for_rainFall()
self.train_one_epoch_for_classification()
if self.epoch % args.interval == 0:
self.validate_one_epoch()
self.train_one_epoch_for_regression()
def regression():
# ********************* load the dataset and divide to X&y ***********************
from sklearn.datasets import make_blobs
X, Y = make_blobs(cluster_std=0.9,
random_state=20,
n_samples=1000,
centers=10,
n_features=10)
from Algorithms.ML_.helper.data_helper import split_train_val_test
X, Xv, y, Yv, Xt, Yt = split_train_val_test(X, Y)
print(X.shape, y.shape, Xv.shape, Yv.shape, Xt.shape, Yt.shape)
# ********************* build model ***********************
from model import Regression
from layer import Layer, Dense
from activation import Activation, Softmax, Sigmoid, ReLU
from regularization import Regularization, L1, L2, L12
from optimizer import Vanilla
model = Regression()
input_size = X.shape[1]
hidden_size = 50
num_classes = 10
learning_rate, reg_rate = 1e-3, 0.5
model = Regression([
Dense(hidden_size,
input_shape=(input_size, ),
activation=ReLU(),
alpha=learning_rate,
lambda_=reg_rate),
])
model += Dense(num_classes,
activation=Softmax(),
alpha=learning_rate,
lambda_=reg_rate) # add layer with +=
model.compile()
model.describe()
# ********************* train ***********************
loss_train, loss_val = model.train(X,
y,
val=(Xv, Yv),
iter_=5000,
batch=32,
return_loss=True,
verbose=True)
import matplotlib.pyplot as plt
plt.plot(range(len(loss_train)), loss_train)
plt.plot(range(len(loss_val)), loss_val)
plt.legend(['train', 'val'])
plt.xlabel('Iteration')
plt.ylabel('Training loss')
plt.title('Training Loss history')
plt.show()
# ********************* predict ***********************
pred_train = model.predict(X)
pred_val = model.predict(Xv)
pred_test = model.predict(Xt)
import metrics
print('train accuracy=', metrics.accuracy(y, pred_train))
print('val accuracy=', metrics.accuracy(Yv, pred_val))
print('test accuracy=', metrics.accuracy(Yt, pred_test))
print('null accuracy=', metrics.null_accuracy(y))
import metrics
metrics.print_metrics(Yt, pred_test)