fname = "%scdlc-mlp-batch-loss-values-%s-%s-%d.bin" % (path,lrStr,momentumStr,maxEpoch) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(lossVal, fh) fh.close() return classifier # Set the random seed manually for reproducibility. torch.manual_seed(1111) if torch.cuda.is_available(): torch.cuda.manual_seed(1111) torch.cuda.is_available(),torch.cuda.device_count() path = './' wrdpath = '../wordvectors/' compositionalMethod = 'additive' lrStrMdl = mres.floatToStr("%f" , 1e-3) mmtStrMdl = mres.floatToStr("%f" , 0.2) epochMdl = 100 batchSizeMdl = 100 prmStrMdl = "%s-%s-%d-%d" % (lrStrMdl, mmtStrMdl, epochMdl, batchSizeMdl) fullModelNamePck = wrdpath + compositionalMethod + '-' + 'model1000EnTr-' + prmStrMdl + '.pck' fullModelNamePth = wrdpath + compositionalMethod + '-' + 'model1000EnTr-' + prmStrMdl + '.pth' fullVocabFilePri = wrdpath + compositionalMethod + '-' + 'english.1000EnTr-' + prmStrMdl + '.vocab' fullVocabFileSec = wrdpath + compositionalMethod + '-' + 'turkish.1000EnTr-' + prmStrMdl + '.vocab' embedding_dim = 64 batch_size = 100 max_epoch = 100 # 300 learning_rate_decay = 0.01 threshold = 100 folds = 20 # 10 test_per = 0.25
def trainClassifier(allDataTrain,targetDataTrain,allDataTest,targetDataTest,learning_rate,momentum,maxEpoch,saveModel,results): classifier = buildClassifierModel(embedding_dim, 1).cuda() for param in classifier.parameters(): init.uniform(param, -1 * 0.0000001, 0.0000001) loss_function = nn.BCELoss(size_average=True).cuda() optimizer = optim.RMSprop(classifier.parameters(), lr=learning_rate, alpha=0.99, eps=1e-08, weight_decay=0,momentum=momentum, centered=False) lossVal = mres.LossValues() errors = [] lrStr = mres.floatToStr("%2.15f",learning_rate) momentumStr = mres.floatToStr("%2.15f",momentum) epc = 0 numberOfDoc = math.floor((allDataTrain.size()[0]/batch_size)*batch_size) for fold in range(folds): for epoch in range(maxEpoch): print("class : %s fold %d epoch %d" % (classname,fold,epoch)) epc += 1 inds = torch.range(1, numberOfDoc, batch_size).long() shuffle = torch.randperm(inds.size()[0]) lr=optimizer.param_groups[0]['lr'] lrStr = mres.floatToStr("%2.15f",lr) for i in range(int(numberOfDoc/batch_size)): start = inds[shuffle[i]] - 1 endd = inds[shuffle[i]] + batch_size - 1 inp = autograd.Variable(allDataTrain[start:endd].data.cuda(), requires_grad=False) target = autograd.Variable(torch.Tensor(batch_size).copy_(targetDataTrain[start:endd]).cuda(), requires_grad=False) classifier.zero_grad() pred = classifier.forward(inp) loss = loss_function(pred,target) print("fold %d epoch %d lr %s - pred %f target %f loss %f " % (fold,epoch,lrStr,pred.data[0][0],target.data[0], loss.data[0])) loss.backward() optimizer.step() errors.append(loss.data[0]) lossVal.y.append(loss.data[0]) mean = torch.mean(torch.Tensor(errors)) lossVal.mean.append(mean) if epoch % 50 == 0 and epoch != 0: trainresults = mres.testClassifier(classifier,allDataTrain,targetDataTrain) res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, epoch+1,trainresults.correct, trainresults.all,trainresults.trueNegatives,trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives,trainresults.posRate) results.append(res) testresults = mres.testClassifier(classifier, allDataTest, targetDataTest) res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, epoch+1,testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate) results.append(res) trainresults = mres.testClassifier(classifier, allDataTrain, targetDataTrain) res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, maxEpoch, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate) results.append(res) testresults = mres.testClassifier(classifier, allDataTest, targetDataTest) res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, maxEpoch, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate) results.append(res) if saveModel == True: lossVal.x = range(folds * maxEpoch * int(numberOfDoc/batch_size)) lrStr = mres.floatToStr("%2.15f",learning_rate) fname = "%scdlc-mlp-batch-loss-values-%s-%s-%d.bin" % (path,lrStr,momentumStr,maxEpoch) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(lossVal, fh) fh.close() return classifier
def trainClassifierLSTM(allDataTrain, allDataTest, targetsTrain, targetsTest, embeddingDim, initWeight, learningRate, momentum, maxEpoch, batchSize, folds, lstmDepth, saveModelStats, results): classifier = LSTMClassifier(embeddingDim, learningRate, momentum, lstmDepth).cuda() classifier.initHidden(initWeight) lrStr = mres.floatToStr("%2.15f", learningRate) momentumStr = mres.floatToStr("%2.15f", momentum) lossVal = mres.LossValues() errors = [] #results = [] totWndCount = allDataTrain.size()[0] wndBatch = math.floor(totWndCount / batchSize) #if totWndCount % batchSize != 0: # wndBatch += 1 allCnt = 0 for fold in range(folds): #errors = [] for epoch in range(maxEpoch): print("class : %s fold %d epoch %d" % (classname, fold + 1, epoch + 1)) shuffle = np.random.permutation(totWndCount) for btcCnt in range(wndBatch): index = torch.from_numpy( shuffle[btcCnt * batchSize:(btcCnt + 1) * batchSize]).cuda() inp = torch.index_select(allDataTrain, 0, index) targets = autograd.Variable(torch.index_select( targetsTrain, 0, index).float(), requires_grad=False) cx = autograd.Variable( torch.zeros(batchSize, embeddingDim).cuda()) hx = autograd.Variable( torch.zeros(batchSize, embeddingDim).cuda()) hidden = [hx, cx] allCnt += 1 classifier.zero_grad() preds, hx, cx = classifier.forward(inp, hidden, lstmDepth) loss = ((preds - targets) * (preds - targets)).mean() print( "fold %d epoch %d lstmDepth %d lr %s mmt %s btcCnt %d - loss %f " % (fold + 1, epoch + 1, lstmDepth, lrStr, momentumStr, btcCnt + 1, loss.data[0])) loss.backward() classifier.optimizer.step() errors.append(loss.data[0]) lossVal.y.append(loss.data[0]) mean = torch.mean(torch.Tensor(errors)) lossVal.mean.append(mean) if epoch % 50 == 0 and epoch != 0: trainresults = testClassifierLSTMNew(classifier, allDataTrain, targetsTrain, embeddingDim, lstmDepth) res = "train - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, epoch + 1, lstmDepth, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate) results.append(res) testresults = testClassifierLSTMNew(classifier, allDataTest, targetsTest, embeddingDim, lstmDepth) res = "test - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, epoch + 1, lstmDepth, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate) results.append(res) trainresults = testClassifierLSTMNew(classifier, allDataTrain, targetsTrain, embeddingDim, lstmDepth) res = "train - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, maxEpoch, lstmDepth, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate) results.append(res) testresults = testClassifierLSTMNew(classifier, allDataTest, targetsTest, embeddingDim, lstmDepth) res = "test - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, maxEpoch, lstmDepth, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate) results.append(res) #for res in results: # print(res) if saveModelStats == True: lossVal.x = range(allCnt) lrStr = mres.floatToStr("%2.15f", learningRate) initStr = mres.floatToStr("%2.15f", initWeight) fname = "%s%s-lstm-docwindow-cdlc-loss-values-%s-%s-%s-%d-%d-%d.bin" % ( path, compositionalMethod, lrStr, momentumStr, initStr, batchSize, maxEpoch, lstmDepth) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(lossVal, fh) fh.close() return classifier
def trainClassifier(allDataTrain, targetDataTrain, allDataTest, targetDataTest, learning_rate, momentum, maxEpoch, saveModel, results): maxCol = allDataTest.size()[1] if allDataTrain.size()[1] < allDataTest.size()[1]: maxCol = allDataTrain.size()[1] allDataTrain = allDataTrain.narrow(1, 0, maxCol) allDataTest = allDataTest.narrow(1, 0, maxCol) classifier = MLPClassifier(maxCol, 100, 2, learning_rate, momentum).cuda() lrStr = mres.floatToStr("%2.15f", learning_rate) momentumStr = mres.floatToStr("%2.15f", momentum) lossVal = mres.LossValues() errors = [] for fold in range(folds): for epoch in range(maxEpoch): print("class : %s fold %d epoch %d" % (classname, fold, epoch)) shuffle = np.random.permutation(allDataTrain.size()[0]) lr = classifier.optimizer.param_groups[0]['lr'] lrStr = mres.floatToStr("%2.15f", lr) for i in range(math.floor(allDataTrain.size()[0] * train_per)): inp = autograd.Variable(allDataTrain[shuffle[i]].data.resize_( 1, maxCol).cuda(), requires_grad=False) target = autograd.Variable(torch.Tensor(1).fill_( targetDataTrain[shuffle[i]]).long().cuda(), requires_grad=False) classifier.zero_grad() pred = classifier.forward(inp) loss = classifier.criterion(pred, target) print( "fold %d epoch %d lr %s - pred %f/%f target %f loss %f " % (fold, epoch, lrStr, pred.data[0][0], pred.data[0][1], target.data[0], loss.data[0])) loss.backward() classifier.optimizer.step() errors.append(loss.data[0]) lossVal.y.append(loss.data[0]) mean = torch.mean(torch.Tensor(errors)) lossVal.mean.append(mean) if epoch % 50 == 0 and epoch != 0: trainresults = mres.testClassifier(classifier, allDataTrain, targetDataTrain) res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, epoch + 1, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate) results.append(res) testresults = mres.testClassifier(classifier, allDataTest, targetDataTest) res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, epoch + 1, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate) results.append(res) trainresults = mres.testClassifier(classifier, allDataTrain, targetDataTrain) res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, maxEpoch, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate) results.append(res) testresults = mres.testClassifier(classifier, allDataTest, targetDataTest) res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, maxEpoch, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate) results.append(res) if saveModel == True: lossVal.x = range(folds * max_epoch * math.floor(allDataTrain.size()[0] * train_per)) lrStr = mres.floatToStr("%2.15f", learning_rate) fname = "%s%s-cdlc-mlp-relu-loss-values-%s-%s-%d.bin" % ( path, compositionalMethod, lrStr, momentumStr, maxEpoch) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(lossVal, fh) fh.close() return classifier, maxCol
def trainClassifierNew(classifier, optimizer, allDataTrain, targetDataTrain, allDataTest, targetDataTest, mdlPrms, saveModel): #loss_func = torch.nn.CrossEntropyLoss() # the target label is NOT an one-hotted loss_func = torch.nn.BCELoss(size_average=True).cuda() for param in classifier.parameters(): init.uniform(param, -1 * 0, mdlPrms.initWeight) lossVal = mres.LossValues() trainStats = mres.ModelStats() testStats = mres.ModelStats() errors = [] lrStr = mres.floatToStr("%2.15f", mdlPrms.lr) initStr = mres.floatToStr("%2.15f", mdlPrms.initWeight) momentumStr = mres.floatToStr("%2.15f", mdlPrms.momentum) epc = 0 reportFlag = False wndBatch = math.floor(allDataTrain.size()[0] / mdlPrms.batchSize) if allDataTrain.size()[0] % mdlPrms.batchSize != 0: wndBatch += 1 for fold in range(mdlPrms.maxFold): for epoch in range(mdlPrms.maxEpoch): print("class : %s fold %d epoch %d" % (classname, fold + 1, epoch + 1)) epc += 1 shuffle = np.random.permutation(allDataTrain.size()[0]) for btcCnt in range(wndBatch): index = torch.from_numpy( shuffle[btcCnt * mdlPrms.batchSize:(btcCnt + 1) * mdlPrms.batchSize]).cuda() inp = torch.index_select(allDataTrain.data, 0, index) inp = autograd.Variable(inp) #target = autograd.Variable(torch.index_select(targetDataTrain.long().cuda(), 0, index)) target = autograd.Variable( torch.index_select(targetDataTrain.cuda(), 0, index)) optimizer.zero_grad() pred = classifier(inp) loss = loss_func(pred, target) #print("%s hidden %d opt %s fold %d epoch %d lr %s mmt %s init %s - pred %f/%f target %f loss %f " % (classname,mdlPrms.hLayerNum,mdlPrms.optimName,fold,epoch,lrStr,momentumStr,initStr,pred.data[0][0],pred.data[0][1],target.data[0], loss.data[0])) print( "class %s opt %s fold %d epoch %d lr %s mmt %s init %s - pred %f target %f loss %f " % (classname, mdlPrms.optimName, fold, epoch, lrStr, momentumStr, initStr, pred.data[0][0], target.data[0], loss.data[0])) loss.backward() optimizer.step() errors.append(loss.data[0]) lossVal.y.append(loss.data[0]) mean = torch.mean(torch.Tensor(errors)) lossVal.mean.append(mean) if ( epoch + 1 ) % mdlPrms.threshold == 0 and epoch != 0 and reportFlag == True: trainresults = mres.testClassifier2(classifier, allDataTrain, targetDataTrain) trainresults.calculateScores() res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold, lrStr, momentumStr, initStr, mdlPrms.maxEpoch, epoch + 1, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate, trainresults.precision, trainresults.recall, trainresults.f1Score, trainresults.score, trainresults.accuracy, trainresults.errorRate) mdlPrms.results['train'].append(res) trainStats.addStat(fold + 1, epoch + 1, trainresults) testresults = mres.testClassifier2(classifier, allDataTest, targetDataTest) testresults.calculateScores() res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold, lrStr, momentumStr, initStr, mdlPrms.maxEpoch, epoch + 1, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate, testresults.precision, testresults.recall, testresults.f1Score, testresults.score, testresults.accuracy, testresults.errorRate) mdlPrms.results['test'].append(res) testStats.addStat(fold + 1, epoch + 1, testresults) trainresults = mres.testClassifier2(classifier, allDataTrain, targetDataTrain) trainresults.calculateScores() res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold, lrStr, momentumStr, initStr, mdlPrms.maxEpoch, mdlPrms.maxEpoch, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate, trainresults.precision, trainresults.recall, trainresults.f1Score, trainresults.score, trainresults.accuracy, trainresults.errorRate) mdlPrms.results['train'].append(res) trainStats.addStat(fold + 1, mdlPrms.maxEpoch, trainresults) testresults = mres.testClassifier2(classifier, allDataTest, targetDataTest) testresults.calculateScores() res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold, lrStr, momentumStr, initStr, mdlPrms.maxEpoch, mdlPrms.maxEpoch, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate, testresults.precision, testresults.recall, testresults.f1Score, testresults.score, testresults.accuracy, testresults.errorRate) mdlPrms.results['test'].append(res) testStats.addStat(fold + 1, mdlPrms.maxEpoch, testresults) if saveModel == True: lossVal.x = range( mdlPrms.maxFold * mdlPrms.maxEpoch * math.floor(allDataTrain.size()[0] * mdlPrms.trainPer)) lrStr = mres.floatToStr("%2.15f", mdlPrms.lr) fname = "%scdlc-%s-%s-%s-mlp-simple-loss-values-%s-%s-%s-%d-%d-%d.bin" % ( path, classname, modelPrms.wordVecsCmpMtd, modelPrms.docVecsCmpMtd, mdlPrms.optimName, lrStr, momentumStr, mdlPrms.maxEpoch, mdlPrms.maxFold, mdlPrms.hLayerNum) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(lossVal, fh) fh.close() mdlStats = {"train": trainStats, "test": testStats} fname = "%scdlc-%s-%s-%s-mlp-simple-stat-values-%s-%s-%s-%d-%d-%d.bin" % ( path, classname, modelPrms.wordVecsCmpMtd, modelPrms.docVecsCmpMtd, mdlPrms.optimName, lrStr, momentumStr, mdlPrms.maxEpoch, mdlPrms.maxFold, mdlPrms.hLayerNum) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(mdlStats, fh) fh.close() return classifier
def trainClassifier(allDataTrain,targetDataTrain,allDataTest,targetDataTest,learning_rate,momentum,maxEpoch,init_weight,saveModel,optimName,results): classifier = buildClassifierModel(embedding_dim,128,128,1).cuda() #classifier = MLPClassifier(embedding_dim,hidden_dim,2,learning_rate,momentum).cuda() for param in classifier.parameters(): init.uniform(param, -1 * init_weight, init_weight) loss_function = nn.BCELoss(size_average=True).cuda() #loss_function = nn.L1Loss().cuda() optimizer = selectOptimizer(classifier,optimName,learning_rate,momentum) lossVal = mres.LossValues() errors = [] lrStr = mres.floatToStr("%2.15f",learning_rate) initStr = mres.floatToStr("%2.15f", init_weight) momentumStr = mres.floatToStr("%2.15f",momentum) epc = 0 wndBatch = math.floor(allDataTrain.size()[0] / batchSize) if allDataTrain.size()[0] % batchSize != 0: wndBatch += 1 for fold in range(folds): for epoch in range(maxEpoch): print("class : %s fold %d epoch %d" % (classname,fold,epoch)) epc += 1 shuffle = np.random.permutation(allDataTrain.size()[0]) lr=optimizer.param_groups[0]['lr'] #lrStr = mres.floatToStr("%2.15f",lr) #for i in range(math.floor(allDataTrain.size()[0] * train_per)): for btcCnt in range(wndBatch): index = torch.from_numpy(shuffle[btcCnt * batchSize:(btcCnt + 1) * batchSize]).cuda() #inp = scaleVector(torch.index_select(allDataTrain.data, 0, index)) inp = torch.index_select(allDataTrain.data, 0, index) inp = autograd.Variable(inp) target = autograd.Variable(torch.index_select(targetDataTrain.cuda(), 0, index)) #inp = scaleVector(allDataTrain[shuffle[i]].data) #inp = autograd.Variable(inp.resize_(1,embedding_dim).cuda(), requires_grad=False) #target = autograd.Variable(torch.Tensor(1).fill_(targetDataTrain[shuffle[i]]).long().cuda(), requires_grad=False) optimizer.zero_grad() pred = classifier.forward(inp) #loss = -(target * torch.log(pred) + (1 - target)*torch.log(1 - pred)) loss = loss_function(pred, target) #loss = classifier.criterion(pred, target) pr = torch.max(F.softmax(pred), 1)[1] print("opt %s fold %d epoch %d lr %s mmt %s init %s - pred %f target %f loss %f " % (optimName,fold,epoch,lrStr,momentumStr,initStr,pred.data[0][0],target.data[0], loss.data[0])) loss.backward() #nn.utils.clip_grad_norm(classifier.parameters(),0.001) optimizer.step() errors.append(loss.data[0]) lossVal.y.append(loss.data[0]) mean = torch.mean(torch.Tensor(errors)) lossVal.mean.append(mean) if (epoch+1) % threshold == 0 and epoch != 0: trainresults = mres.testClassifier(classifier,allDataTrain,targetDataTrain) trainresults.calculateScores() res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold,lrStr, momentumStr, initStr,maxEpoch, epoch+1,trainresults.correct, trainresults.all,trainresults.trueNegatives,trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives,trainresults.posRate, trainresults.precision, trainresults.recall, trainresults.f1Score, trainresults.score, trainresults.accuracy, trainresults.errorRate ) results['train'].append(res) testresults = mres.testClassifier(classifier, allDataTest, targetDataTest) testresults.calculateScores() res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold,lrStr, momentumStr,initStr, maxEpoch, epoch+1,testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate, testresults.precision, testresults.recall, testresults.f1Score, testresults.score, testresults.accuracy, testresults.errorRate) results['test'].append(res) trainresults = mres.testClassifier(classifier, allDataTrain, targetDataTrain) trainresults.calculateScores() res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold, lrStr, momentumStr, initStr,maxEpoch, maxEpoch, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate, trainresults.precision, trainresults.recall, trainresults.f1Score, trainresults.score, trainresults.accuracy, trainresults.errorRate) results['train'].append(res) testresults = mres.testClassifier(classifier, allDataTest, targetDataTest) testresults.calculateScores() res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold, lrStr, momentumStr,initStr, maxEpoch, maxEpoch, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate, testresults.precision, testresults.recall, testresults.f1Score, testresults.score, testresults.accuracy, testresults.errorRate) results['test'].append(res) if saveModel == True: lossVal.x = range(folds * maxEpoch * math.floor(allDataTrain.size()[0] * train_per)) lrStr = mres.floatToStr("%2.15f" ,learning_rate) fname = "%scdlc-mlp-simple-loss-values-%s-%s-%s-%s-%d.bin" % (path,optimName,lrStr,momentumStr,initStr,maxEpoch) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(lossVal, fh) fh.close() return classifier
def trainClassifierNew(allDataTrain,targetDataTrain,allDataTest,targetDataTest,learning_rate,momentum,maxEpoch,init_weight,saveModel,optimName,results): classifier = Net(n_feature=128, n_hidden=128, n_output=2).cuda() # define the network print(classifier) # net architecture #optimizer = torch.optim.SGD(classifier.parameters(), lr=learning_rate) optimizer = selectOptimizer(classifier,optimName,learning_rate,momentum) loss_func = torch.nn.CrossEntropyLoss() # the target label is NOT an one-hotted lossVal = mres.LossValues() trainStats = mres.ModelStats() testStats = mres.ModelStats() errors = [] lrStr = mres.floatToStr("%2.15f",learning_rate) initStr = mres.floatToStr("%2.15f", init_weight) momentumStr = mres.floatToStr("%2.15f",momentum) epc = 0 wndBatch = math.floor(allDataTrain.size()[0] / batchSize) if allDataTrain.size()[0] % batchSize != 0: wndBatch += 1 for fold in range(folds): for epoch in range(maxEpoch): print("class : %s fold %d epoch %d" % (classname,fold,epoch)) epc += 1 shuffle = np.random.permutation(allDataTrain.size()[0]) lr=optimizer.param_groups[0]['lr'] for btcCnt in range(wndBatch): index = torch.from_numpy(shuffle[btcCnt * batchSize:(btcCnt + 1) * batchSize]).cuda() #inp = scaleVector(torch.index_select(allDataTrain.data, 0, index)) inp = torch.index_select(allDataTrain.data, 0, index) inp = autograd.Variable(inp) target = autograd.Variable(torch.index_select(targetDataTrain.long().cuda(), 0, index)) optimizer.zero_grad() pred = classifier(inp) loss = loss_func(pred, target) print("opt %s fold %d epoch %d lr %s mmt %s init %s - pred %f/%f target %f loss %f " % (optimName,fold,epoch,lrStr,momentumStr,initStr,pred.data[0][0],pred.data[0][1],target.data[0], loss.data[0])) loss.backward() optimizer.step() errors.append(loss.data[0]) lossVal.y.append(loss.data[0]) mean = torch.mean(torch.Tensor(errors)) lossVal.mean.append(mean) if (epoch+1) % threshold == 0 and epoch != 0: trainresults = mres.testClassifier2(classifier,allDataTrain,targetDataTrain) trainresults.calculateScores() res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold,lrStr, momentumStr, initStr,maxEpoch, epoch+1,trainresults.correct, trainresults.all,trainresults.trueNegatives,trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives,trainresults.posRate, trainresults.precision, trainresults.recall, trainresults.f1Score, trainresults.score, trainresults.accuracy, trainresults.errorRate ) results['train'].append(res) trainStats.addStat(fold,epoch+1,trainresults) testresults = mres.testClassifier2(classifier, allDataTest, targetDataTest) testresults.calculateScores() res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold,lrStr, momentumStr,initStr, maxEpoch, epoch+1,testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate, testresults.precision, testresults.recall, testresults.f1Score, testresults.score, testresults.accuracy, testresults.errorRate) results['test'].append(res) testStats.addStat(fold, epoch + 1, testresults) trainresults = mres.testClassifier2(classifier, allDataTrain, targetDataTrain) trainresults.calculateScores() res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold, lrStr, momentumStr, initStr,maxEpoch, maxEpoch, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate, trainresults.precision, trainresults.recall, trainresults.f1Score, trainresults.score, trainresults.accuracy, trainresults.errorRate) results['train'].append(res) trainStats.addStat(fold, maxEpoch, trainresults) testresults = mres.testClassifier2(classifier, allDataTest, targetDataTest) testresults.calculateScores() res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % ( fold, lrStr, momentumStr,initStr, maxEpoch, maxEpoch, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate, testresults.precision, testresults.recall, testresults.f1Score, testresults.score, testresults.accuracy, testresults.errorRate) results['test'].append(res) testStats.addStat(fold, maxEpoch, testresults) if saveModel == True: lossVal.x = range(folds * maxEpoch * math.floor(allDataTrain.size()[0] * train_per)) lrStr = mres.floatToStr("%2.15f" ,learning_rate) fname = "%scdlc-%s-mlp-simple-minmax-loss-values-%s-%s-%s-%s-%d.bin" % (path,classname,optimName,lrStr,momentumStr,initStr,maxEpoch) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(lossVal, fh) fh.close() mdlStats = {"train" : trainStats,"test":testStats} fname = "%scdlc-%s-mlp-simple-minmax-4-hidden-stat-values-%s-%s-%s-%s-%d.bin" % (path,classname,optimName,lrStr,momentumStr,initStr,maxEpoch) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(mdlStats, fh) fh.close() return classifier
def trainClassifierLSTM(allDataTrain,allDataTest,embeddingDim,hiddenSize,initWeight,learningRate,momentum,maxEpoch,batchSize,folds,lstmDepth,saveModelStats,results): classifier = LSTMClassifier(embeddingDim,hiddenSize,learningRate, momentum,lstmDepth).cuda() classifier.initHidden(initWeight) lrStr = mres.floatToStr("%2.15f", learningRate) momentumStr = mres.floatToStr("%2.15f", momentum) lossVal = mres.LossValues() posValues = list(allDataTrain[1].values()) negValues = list(allDataTrain[0].values()) allPositives=len(posValues) allNegatives=len(negValues) totFileCount = allPositives + allNegatives errors = [] #results = [] docDiv = math.floor(totFileCount / batchSize) if totFileCount % batchSize != 0: docDiv += 1 for fold in range(folds): #errors = [] for epoch in range(maxEpoch): print("class : %s fold %d epoch %d" % (classname,fold+1,epoch+1)) shuffle = np.random.permutation(totFileCount) for sCnt in range(docDiv): if shuffle[sCnt] < allPositives: docInp = [posValues[shuffle[sCnt]].data.clone()] target = autograd.Variable(torch.Tensor(1).fill_(1).float().cuda(), requires_grad=False) else: docInp = [negValues[shuffle[sCnt] - allPositives].data.clone()] target = autograd.Variable(torch.Tensor(1).fill_(0).float().cuda(), requires_grad=False) cx = autograd.Variable(torch.zeros(batchSize, hiddenSize).cuda()) hx = autograd.Variable(torch.zeros(batchSize, hiddenSize).cuda()) hidden = [hx, cx] inp = list(docInp) #inp = autograd.Variable(inp.resize_(inp.size()[0],1,inp.size()[1]).cuda()) classifier.zero_grad() pred = classifier.forward(inp,hidden,inp[0].size()[0]) loss = ((pred - target) * (pred - target)).mean() print("fold %d epoch %d lstmDepth %d lr %s mmt %s doc num %d - pred %f target %f loss %f " % (fold+1,epoch+1,lstmDepth,lrStr,momentumStr,sCnt+1,pred.data[0][0],target.data[0], loss.data[0])) loss.backward() classifier.optimizer.step() errors.append(loss.data[0]) lossVal.y.append(loss.data[0]) mean = torch.mean(torch.Tensor(errors)) lossVal.mean.append(mean) if epoch % 50 == 0 and epoch != 0: trainresults = mres.testClassifierLSTM(classifier,allDataTrain,hiddenSize) res = "train - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, epoch+1,lstmDepth, trainresults.correct, trainresults.all,trainresults.trueNegatives,trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives,trainresults.posRate) results.append(res) testresults = mres.testClassifierLSTM(classifier, allDataTest, hiddenSize) res = "test - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, epoch+1,lstmDepth, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate) results.append(res) trainresults = mres.testClassifierLSTM(classifier, allDataTrain, hiddenSize) res = "train - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, maxEpoch ,lstmDepth, trainresults.correct, trainresults.all,trainresults.trueNegatives,trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives,trainresults.posRate) results.append(res) testresults = mres.testClassifierLSTM(classifier, allDataTest, hiddenSize) res = "test - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, maxEpoch, maxEpoch ,lstmDepth, testresults.correct, testresults.all,testresults.trueNegatives,testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives,testresults.posRate) results.append(res) #for res in results: # print(res) if saveModelStats == True: lossVal.x = range(folds * maxEpoch * docDiv) lrStr = mres.floatToStr("%2.15f", learningRate) fname = "%s%s-lstm-cdlc-loss-values-%s-%s-%d-%d-%d.bin" % (path,compositionalMethod,lrStr,momentumStr,batchSize,maxEpoch,lstmDepth) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(lossVal, fh) fh.close() return classifier
def trainClassifier(allDataTrain, targetDataTrain, allDataTest, targetDataTest, prms, results): classifier = buildClassifierModel(allDataTrain.size()[1], 1).cuda() #classifier = MLPClassifier(embedding_dim,embedding_dim,2,learning_rate,momentum).cuda() for param in classifier.parameters(): init.uniform(param, -1 * prms.initWeight, prms.initWeight) loss_function = nn.BCELoss(size_average=True).cuda() #optimizer = optim.SGD(classifier.parameters(), lr=prms.learningRate, momentum=prms.momentum) optimizer = optim.RMSprop(classifier.parameters(), lr=prms.learningRate, alpha=0.99, eps=1e-08, weight_decay=0, momentum=prms.momentum, centered=False) lossVal = mres.LossValues() errors = [] lrStr = mres.floatToStr("%2.15f", prms.learningRate) momentumStr = mres.floatToStr("%2.15f", prms.momentum) epc = 0 wndBatch = math.floor(allDataTrain.size()[0] / prms.batchSize) if allDataTrain.size()[0] % prms.batchSize != 0: wndBatch += 1 for fold in range(prms.folds): for epoch in range(prms.maxEpoch): print("class : %s fold %d epoch %d" % (classname, fold, epoch)) epc += 1 shuffle = np.random.permutation(allDataTrain.size()[0]) lr = optimizer.param_groups[0]['lr'] lrStr = mres.floatToStr("%2.15f", lr) for btcCnt in range(wndBatch): index = torch.from_numpy( shuffle[btcCnt * prms.batchSize:(btcCnt + 1) * prms.batchSize]).cuda() inp = autograd.Variable( torch.index_select(allDataTrain.data, 0, index)) target = autograd.Variable(torch.index_select( targetDataTrain.cuda(), 0, index), requires_grad=False) classifier.zero_grad() pred = classifier.forward(inp) loss = loss_function(pred, target) #loss = classifier.criterion(pred, target) print( "fold %d epoch %d lr %s mmt %s- pred %f target %f loss %f " % (fold, epoch, lrStr, momentumStr, pred.data[0][0], target.data[0], loss.data[0])) loss.backward() optimizer.step() errors.append(loss.data[0]) lossVal.y.append(loss.data[0]) mean = torch.mean(torch.Tensor(errors)) lossVal.mean.append(mean) if epoch % modelPrms.threshold == 0 and epoch != 0: trainresults = mres.testClassifier(classifier, allDataTrain, targetDataTrain) res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, prms.maxEpoch, epoch + 1, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate) results.append(res) testresults = mres.testClassifier(classifier, allDataTest, targetDataTest) res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, prms.maxEpoch, epoch + 1, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate) results.append(res) trainresults = mres.testClassifier(classifier, allDataTrain, targetDataTrain) res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, prms.maxEpoch, prms.maxEpoch, trainresults.correct, trainresults.all, trainresults.trueNegatives, trainresults.allNegatives, trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate) results.append(res) testresults = mres.testClassifier(classifier, allDataTest, targetDataTest) res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % ( lrStr, momentumStr, prms.maxEpoch, prms.maxEpoch, testresults.correct, testresults.all, testresults.trueNegatives, testresults.allNegatives, testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate) results.append(res) if prms.saveModel == True: lossVal.x = range(prms.folds * prms.maxEpoch * wndBatch) lrStr = mres.floatToStr("%2.15f", prms.learningRate) fname = "%scdlc-mlp-simple-flat-loss-values-%s-%s-%d.bin" % ( path, lrStr, momentumStr, prms.maxEpoch) fh = open(fname, 'wb') # Save model file as pickle pickle.dump(lossVal, fh) fh.close() return classifier