fname = "%scdlc-mlp-batch-loss-values-%s-%s-%d.bin" % (path,lrStr,momentumStr,maxEpoch)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(lossVal, fh)
fh.close()
return classifier
# Set the random seed manually for reproducibility.
torch.manual_seed(1111)
if torch.cuda.is_available():
torch.cuda.manual_seed(1111)
torch.cuda.is_available(),torch.cuda.device_count()
path = './'
wrdpath = '../wordvectors/'
compositionalMethod = 'additive'
lrStrMdl = mres.floatToStr("%f" , 1e-3)
mmtStrMdl = mres.floatToStr("%f" , 0.2)
epochMdl = 100
batchSizeMdl = 100
prmStrMdl = "%s-%s-%d-%d" % (lrStrMdl, mmtStrMdl, epochMdl, batchSizeMdl)
fullModelNamePck = wrdpath + compositionalMethod + '-' + 'model1000EnTr-' + prmStrMdl + '.pck'
fullModelNamePth = wrdpath + compositionalMethod + '-' + 'model1000EnTr-' + prmStrMdl + '.pth'
fullVocabFilePri = wrdpath + compositionalMethod + '-' + 'english.1000EnTr-' + prmStrMdl + '.vocab'
fullVocabFileSec = wrdpath + compositionalMethod + '-' + 'turkish.1000EnTr-' + prmStrMdl + '.vocab'
embedding_dim = 64
batch_size = 100
max_epoch = 100 # 300
learning_rate_decay = 0.01
threshold = 100
folds = 20 # 10
test_per = 0.25
def trainClassifier(allDataTrain,targetDataTrain,allDataTest,targetDataTest,learning_rate,momentum,maxEpoch,saveModel,results):
classifier = buildClassifierModel(embedding_dim, 1).cuda()
for param in classifier.parameters():
init.uniform(param, -1 * 0.0000001, 0.0000001)
loss_function = nn.BCELoss(size_average=True).cuda()
optimizer = optim.RMSprop(classifier.parameters(), lr=learning_rate, alpha=0.99, eps=1e-08, weight_decay=0,momentum=momentum, centered=False)
lossVal = mres.LossValues()
errors = []
lrStr = mres.floatToStr("%2.15f",learning_rate)
momentumStr = mres.floatToStr("%2.15f",momentum)
epc = 0
numberOfDoc = math.floor((allDataTrain.size()[0]/batch_size)*batch_size)
for fold in range(folds):
for epoch in range(maxEpoch):
print("class : %s fold %d epoch %d" % (classname,fold,epoch))
epc += 1
inds = torch.range(1, numberOfDoc, batch_size).long()
shuffle = torch.randperm(inds.size()[0])
lr=optimizer.param_groups[0]['lr']
lrStr = mres.floatToStr("%2.15f",lr)
for i in range(int(numberOfDoc/batch_size)):
start = inds[shuffle[i]] - 1
endd = inds[shuffle[i]] + batch_size - 1
inp = autograd.Variable(allDataTrain[start:endd].data.cuda(), requires_grad=False)
target = autograd.Variable(torch.Tensor(batch_size).copy_(targetDataTrain[start:endd]).cuda(), requires_grad=False)
classifier.zero_grad()
pred = classifier.forward(inp)
loss = loss_function(pred,target)
print("fold %d epoch %d lr %s - pred %f target %f loss %f " % (fold,epoch,lrStr,pred.data[0][0],target.data[0], loss.data[0]))
loss.backward()
optimizer.step()
errors.append(loss.data[0])
lossVal.y.append(loss.data[0])
mean = torch.mean(torch.Tensor(errors))
lossVal.mean.append(mean)
if epoch % 50 == 0 and epoch != 0:
trainresults = mres.testClassifier(classifier,allDataTrain,targetDataTrain)
res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, epoch+1,trainresults.correct, trainresults.all,trainresults.trueNegatives,trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives, trainresults.allPositives,trainresults.posRate)
results.append(res)
testresults = mres.testClassifier(classifier, allDataTest, targetDataTest)
res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, epoch+1,testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate)
results.append(res)
trainresults = mres.testClassifier(classifier, allDataTrain, targetDataTrain)
res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, maxEpoch, trainresults.correct, trainresults.all,
trainresults.trueNegatives, trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate)
results.append(res)
testresults = mres.testClassifier(classifier, allDataTest, targetDataTest)
res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, maxEpoch, testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate)
results.append(res)
if saveModel == True:
lossVal.x = range(folds * maxEpoch * int(numberOfDoc/batch_size))
lrStr = mres.floatToStr("%2.15f",learning_rate)
fname = "%scdlc-mlp-batch-loss-values-%s-%s-%d.bin" % (path,lrStr,momentumStr,maxEpoch)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(lossVal, fh)
fh.close()
return classifier
def trainClassifierLSTM(allDataTrain, allDataTest, targetsTrain, targetsTest,
embeddingDim, initWeight, learningRate, momentum,
maxEpoch, batchSize, folds, lstmDepth, saveModelStats,
results):
classifier = LSTMClassifier(embeddingDim, learningRate, momentum,
lstmDepth).cuda()
classifier.initHidden(initWeight)
lrStr = mres.floatToStr("%2.15f", learningRate)
momentumStr = mres.floatToStr("%2.15f", momentum)
lossVal = mres.LossValues()
errors = []
#results = []
totWndCount = allDataTrain.size()[0]
wndBatch = math.floor(totWndCount / batchSize)
#if totWndCount % batchSize != 0:
# wndBatch += 1
allCnt = 0
for fold in range(folds):
#errors = []
for epoch in range(maxEpoch):
print("class : %s fold %d epoch %d" %
(classname, fold + 1, epoch + 1))
shuffle = np.random.permutation(totWndCount)
for btcCnt in range(wndBatch):
index = torch.from_numpy(
shuffle[btcCnt * batchSize:(btcCnt + 1) *
batchSize]).cuda()
inp = torch.index_select(allDataTrain, 0, index)
targets = autograd.Variable(torch.index_select(
targetsTrain, 0, index).float(),
requires_grad=False)
cx = autograd.Variable(
torch.zeros(batchSize, embeddingDim).cuda())
hx = autograd.Variable(
torch.zeros(batchSize, embeddingDim).cuda())
hidden = [hx, cx]
allCnt += 1
classifier.zero_grad()
preds, hx, cx = classifier.forward(inp, hidden, lstmDepth)
loss = ((preds - targets) * (preds - targets)).mean()
print(
"fold %d epoch %d lstmDepth %d lr %s mmt %s btcCnt %d - loss %f "
% (fold + 1, epoch + 1, lstmDepth, lrStr, momentumStr,
btcCnt + 1, loss.data[0]))
loss.backward()
classifier.optimizer.step()
errors.append(loss.data[0])
lossVal.y.append(loss.data[0])
mean = torch.mean(torch.Tensor(errors))
lossVal.mean.append(mean)
if epoch % 50 == 0 and epoch != 0:
trainresults = testClassifierLSTMNew(classifier, allDataTrain,
targetsTrain,
embeddingDim, lstmDepth)
res = "train - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, epoch + 1, lstmDepth,
trainresults.correct, trainresults.all,
trainresults.trueNegatives, trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives,
trainresults.allPositives, trainresults.posRate)
results.append(res)
testresults = testClassifierLSTMNew(classifier, allDataTest,
targetsTest, embeddingDim,
lstmDepth)
res = "test - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, epoch + 1, lstmDepth,
testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives,
testresults.allPositives, testresults.posRate)
results.append(res)
trainresults = testClassifierLSTMNew(classifier, allDataTrain,
targetsTrain, embeddingDim, lstmDepth)
res = "train - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, maxEpoch, lstmDepth,
trainresults.correct, trainresults.all, trainresults.trueNegatives,
trainresults.allNegatives, trainresults.negRate,
trainresults.truePositives, trainresults.allPositives,
trainresults.posRate)
results.append(res)
testresults = testClassifierLSTMNew(classifier, allDataTest, targetsTest,
embeddingDim, lstmDepth)
res = "test - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, maxEpoch, lstmDepth, testresults.correct,
testresults.all, testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives,
testresults.allPositives, testresults.posRate)
results.append(res)
#for res in results:
# print(res)
if saveModelStats == True:
lossVal.x = range(allCnt)
lrStr = mres.floatToStr("%2.15f", learningRate)
initStr = mres.floatToStr("%2.15f", initWeight)
fname = "%s%s-lstm-docwindow-cdlc-loss-values-%s-%s-%s-%d-%d-%d.bin" % (
path, compositionalMethod, lrStr, momentumStr, initStr, batchSize,
maxEpoch, lstmDepth)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(lossVal, fh)
fh.close()
return classifier
def trainClassifier(allDataTrain, targetDataTrain, allDataTest, targetDataTest,
learning_rate, momentum, maxEpoch, saveModel, results):
maxCol = allDataTest.size()[1]
if allDataTrain.size()[1] < allDataTest.size()[1]:
maxCol = allDataTrain.size()[1]
allDataTrain = allDataTrain.narrow(1, 0, maxCol)
allDataTest = allDataTest.narrow(1, 0, maxCol)
classifier = MLPClassifier(maxCol, 100, 2, learning_rate, momentum).cuda()
lrStr = mres.floatToStr("%2.15f", learning_rate)
momentumStr = mres.floatToStr("%2.15f", momentum)
lossVal = mres.LossValues()
errors = []
for fold in range(folds):
for epoch in range(maxEpoch):
print("class : %s fold %d epoch %d" % (classname, fold, epoch))
shuffle = np.random.permutation(allDataTrain.size()[0])
lr = classifier.optimizer.param_groups[0]['lr']
lrStr = mres.floatToStr("%2.15f", lr)
for i in range(math.floor(allDataTrain.size()[0] * train_per)):
inp = autograd.Variable(allDataTrain[shuffle[i]].data.resize_(
1, maxCol).cuda(),
requires_grad=False)
target = autograd.Variable(torch.Tensor(1).fill_(
targetDataTrain[shuffle[i]]).long().cuda(),
requires_grad=False)
classifier.zero_grad()
pred = classifier.forward(inp)
loss = classifier.criterion(pred, target)
print(
"fold %d epoch %d lr %s - pred %f/%f target %f loss %f " %
(fold, epoch, lrStr, pred.data[0][0], pred.data[0][1],
target.data[0], loss.data[0]))
loss.backward()
classifier.optimizer.step()
errors.append(loss.data[0])
lossVal.y.append(loss.data[0])
mean = torch.mean(torch.Tensor(errors))
lossVal.mean.append(mean)
if epoch % 50 == 0 and epoch != 0:
trainresults = mres.testClassifier(classifier, allDataTrain,
targetDataTrain)
res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, epoch + 1,
trainresults.correct, trainresults.all,
trainresults.trueNegatives, trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives,
trainresults.allPositives, trainresults.posRate)
results.append(res)
testresults = mres.testClassifier(classifier, allDataTest,
targetDataTest)
res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, epoch + 1,
testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives,
testresults.allPositives, testresults.posRate)
results.append(res)
trainresults = mres.testClassifier(classifier, allDataTrain,
targetDataTrain)
res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, maxEpoch, trainresults.correct,
trainresults.all, trainresults.trueNegatives,
trainresults.allNegatives, trainresults.negRate,
trainresults.truePositives, trainresults.allPositives,
trainresults.posRate)
results.append(res)
testresults = mres.testClassifier(classifier, allDataTest,
targetDataTest)
res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, maxEpoch, testresults.correct,
testresults.all, testresults.trueNegatives,
testresults.allNegatives, testresults.negRate,
testresults.truePositives, testresults.allPositives,
testresults.posRate)
results.append(res)
if saveModel == True:
lossVal.x = range(folds * max_epoch *
math.floor(allDataTrain.size()[0] * train_per))
lrStr = mres.floatToStr("%2.15f", learning_rate)
fname = "%s%s-cdlc-mlp-relu-loss-values-%s-%s-%d.bin" % (
path, compositionalMethod, lrStr, momentumStr, maxEpoch)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(lossVal, fh)
fh.close()
return classifier, maxCol
def trainClassifierNew(classifier, optimizer, allDataTrain, targetDataTrain,
allDataTest, targetDataTest, mdlPrms, saveModel):
#loss_func = torch.nn.CrossEntropyLoss() # the target label is NOT an one-hotted
loss_func = torch.nn.BCELoss(size_average=True).cuda()
for param in classifier.parameters():
init.uniform(param, -1 * 0, mdlPrms.initWeight)
lossVal = mres.LossValues()
trainStats = mres.ModelStats()
testStats = mres.ModelStats()
errors = []
lrStr = mres.floatToStr("%2.15f", mdlPrms.lr)
initStr = mres.floatToStr("%2.15f", mdlPrms.initWeight)
momentumStr = mres.floatToStr("%2.15f", mdlPrms.momentum)
epc = 0
reportFlag = False
wndBatch = math.floor(allDataTrain.size()[0] / mdlPrms.batchSize)
if allDataTrain.size()[0] % mdlPrms.batchSize != 0:
wndBatch += 1
for fold in range(mdlPrms.maxFold):
for epoch in range(mdlPrms.maxEpoch):
print("class : %s fold %d epoch %d" %
(classname, fold + 1, epoch + 1))
epc += 1
shuffle = np.random.permutation(allDataTrain.size()[0])
for btcCnt in range(wndBatch):
index = torch.from_numpy(
shuffle[btcCnt * mdlPrms.batchSize:(btcCnt + 1) *
mdlPrms.batchSize]).cuda()
inp = torch.index_select(allDataTrain.data, 0, index)
inp = autograd.Variable(inp)
#target = autograd.Variable(torch.index_select(targetDataTrain.long().cuda(), 0, index))
target = autograd.Variable(
torch.index_select(targetDataTrain.cuda(), 0, index))
optimizer.zero_grad()
pred = classifier(inp)
loss = loss_func(pred, target)
#print("%s hidden %d opt %s fold %d epoch %d lr %s mmt %s init %s - pred %f/%f target %f loss %f " % (classname,mdlPrms.hLayerNum,mdlPrms.optimName,fold,epoch,lrStr,momentumStr,initStr,pred.data[0][0],pred.data[0][1],target.data[0], loss.data[0]))
print(
"class %s opt %s fold %d epoch %d lr %s mmt %s init %s - pred %f target %f loss %f "
% (classname, mdlPrms.optimName, fold, epoch, lrStr,
momentumStr, initStr, pred.data[0][0], target.data[0],
loss.data[0]))
loss.backward()
optimizer.step()
errors.append(loss.data[0])
lossVal.y.append(loss.data[0])
mean = torch.mean(torch.Tensor(errors))
lossVal.mean.append(mean)
if (
epoch + 1
) % mdlPrms.threshold == 0 and epoch != 0 and reportFlag == True:
trainresults = mres.testClassifier2(classifier, allDataTrain,
targetDataTrain)
trainresults.calculateScores()
res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold, lrStr, momentumStr, initStr, mdlPrms.maxEpoch,
epoch + 1, trainresults.correct, trainresults.all,
trainresults.trueNegatives, trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives,
trainresults.allPositives, trainresults.posRate,
trainresults.precision, trainresults.recall,
trainresults.f1Score, trainresults.score,
trainresults.accuracy, trainresults.errorRate)
mdlPrms.results['train'].append(res)
trainStats.addStat(fold + 1, epoch + 1, trainresults)
testresults = mres.testClassifier2(classifier, allDataTest,
targetDataTest)
testresults.calculateScores()
res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold, lrStr, momentumStr, initStr, mdlPrms.maxEpoch,
epoch + 1, testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives,
testresults.allPositives, testresults.posRate,
testresults.precision, testresults.recall,
testresults.f1Score, testresults.score,
testresults.accuracy, testresults.errorRate)
mdlPrms.results['test'].append(res)
testStats.addStat(fold + 1, epoch + 1, testresults)
trainresults = mres.testClassifier2(classifier, allDataTrain,
targetDataTrain)
trainresults.calculateScores()
res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold, lrStr, momentumStr, initStr, mdlPrms.maxEpoch,
mdlPrms.maxEpoch, trainresults.correct, trainresults.all,
trainresults.trueNegatives, trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives,
trainresults.allPositives, trainresults.posRate,
trainresults.precision, trainresults.recall, trainresults.f1Score,
trainresults.score, trainresults.accuracy, trainresults.errorRate)
mdlPrms.results['train'].append(res)
trainStats.addStat(fold + 1, mdlPrms.maxEpoch, trainresults)
testresults = mres.testClassifier2(classifier, allDataTest,
targetDataTest)
testresults.calculateScores()
res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold, lrStr, momentumStr, initStr, mdlPrms.maxEpoch,
mdlPrms.maxEpoch, testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives,
testresults.allPositives, testresults.posRate,
testresults.precision, testresults.recall, testresults.f1Score,
testresults.score, testresults.accuracy, testresults.errorRate)
mdlPrms.results['test'].append(res)
testStats.addStat(fold + 1, mdlPrms.maxEpoch, testresults)
if saveModel == True:
lossVal.x = range(
mdlPrms.maxFold * mdlPrms.maxEpoch *
math.floor(allDataTrain.size()[0] * mdlPrms.trainPer))
lrStr = mres.floatToStr("%2.15f", mdlPrms.lr)
fname = "%scdlc-%s-%s-%s-mlp-simple-loss-values-%s-%s-%s-%d-%d-%d.bin" % (
path, classname, modelPrms.wordVecsCmpMtd, modelPrms.docVecsCmpMtd,
mdlPrms.optimName, lrStr, momentumStr, mdlPrms.maxEpoch,
mdlPrms.maxFold, mdlPrms.hLayerNum)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(lossVal, fh)
fh.close()
mdlStats = {"train": trainStats, "test": testStats}
fname = "%scdlc-%s-%s-%s-mlp-simple-stat-values-%s-%s-%s-%d-%d-%d.bin" % (
path, classname, modelPrms.wordVecsCmpMtd, modelPrms.docVecsCmpMtd,
mdlPrms.optimName, lrStr, momentumStr, mdlPrms.maxEpoch,
mdlPrms.maxFold, mdlPrms.hLayerNum)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(mdlStats, fh)
fh.close()
return classifier
def trainClassifier(allDataTrain,targetDataTrain,allDataTest,targetDataTest,learning_rate,momentum,maxEpoch,init_weight,saveModel,optimName,results):
classifier = buildClassifierModel(embedding_dim,128,128,1).cuda()
#classifier = MLPClassifier(embedding_dim,hidden_dim,2,learning_rate,momentum).cuda()
for param in classifier.parameters():
init.uniform(param, -1 * init_weight, init_weight)
loss_function = nn.BCELoss(size_average=True).cuda()
#loss_function = nn.L1Loss().cuda()
optimizer = selectOptimizer(classifier,optimName,learning_rate,momentum)
lossVal = mres.LossValues()
errors = []
lrStr = mres.floatToStr("%2.15f",learning_rate)
initStr = mres.floatToStr("%2.15f", init_weight)
momentumStr = mres.floatToStr("%2.15f",momentum)
epc = 0
wndBatch = math.floor(allDataTrain.size()[0] / batchSize)
if allDataTrain.size()[0] % batchSize != 0:
wndBatch += 1
for fold in range(folds):
for epoch in range(maxEpoch):
print("class : %s fold %d epoch %d" % (classname,fold,epoch))
epc += 1
shuffle = np.random.permutation(allDataTrain.size()[0])
lr=optimizer.param_groups[0]['lr']
#lrStr = mres.floatToStr("%2.15f",lr)
#for i in range(math.floor(allDataTrain.size()[0] * train_per)):
for btcCnt in range(wndBatch):
index = torch.from_numpy(shuffle[btcCnt * batchSize:(btcCnt + 1) * batchSize]).cuda()
#inp = scaleVector(torch.index_select(allDataTrain.data, 0, index))
inp = torch.index_select(allDataTrain.data, 0, index)
inp = autograd.Variable(inp)
target = autograd.Variable(torch.index_select(targetDataTrain.cuda(), 0, index))
#inp = scaleVector(allDataTrain[shuffle[i]].data)
#inp = autograd.Variable(inp.resize_(1,embedding_dim).cuda(), requires_grad=False)
#target = autograd.Variable(torch.Tensor(1).fill_(targetDataTrain[shuffle[i]]).long().cuda(), requires_grad=False)
optimizer.zero_grad()
pred = classifier.forward(inp)
#loss = -(target * torch.log(pred) + (1 - target)*torch.log(1 - pred))
loss = loss_function(pred, target)
#loss = classifier.criterion(pred, target)
pr = torch.max(F.softmax(pred), 1)[1]
print("opt %s fold %d epoch %d lr %s mmt %s init %s - pred %f target %f loss %f " % (optimName,fold,epoch,lrStr,momentumStr,initStr,pred.data[0][0],target.data[0], loss.data[0]))
loss.backward()
#nn.utils.clip_grad_norm(classifier.parameters(),0.001)
optimizer.step()
errors.append(loss.data[0])
lossVal.y.append(loss.data[0])
mean = torch.mean(torch.Tensor(errors))
lossVal.mean.append(mean)
if (epoch+1) % threshold == 0 and epoch != 0:
trainresults = mres.testClassifier(classifier,allDataTrain,targetDataTrain)
trainresults.calculateScores()
res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold,lrStr, momentumStr, initStr,maxEpoch, epoch+1,trainresults.correct, trainresults.all,trainresults.trueNegatives,trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives, trainresults.allPositives,trainresults.posRate,
trainresults.precision, trainresults.recall, trainresults.f1Score, trainresults.score, trainresults.accuracy, trainresults.errorRate )
results['train'].append(res)
testresults = mres.testClassifier(classifier, allDataTest, targetDataTest)
testresults.calculateScores()
res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold,lrStr, momentumStr,initStr, maxEpoch, epoch+1,testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate,
testresults.precision, testresults.recall, testresults.f1Score, testresults.score,
testresults.accuracy, testresults.errorRate)
results['test'].append(res)
trainresults = mres.testClassifier(classifier, allDataTrain, targetDataTrain)
trainresults.calculateScores()
res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold, lrStr, momentumStr, initStr,maxEpoch, maxEpoch, trainresults.correct, trainresults.all,
trainresults.trueNegatives, trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate,
trainresults.precision, trainresults.recall, trainresults.f1Score, trainresults.score,
trainresults.accuracy, trainresults.errorRate)
results['train'].append(res)
testresults = mres.testClassifier(classifier, allDataTest, targetDataTest)
testresults.calculateScores()
res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold, lrStr, momentumStr,initStr, maxEpoch, maxEpoch, testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate,
testresults.precision, testresults.recall, testresults.f1Score, testresults.score,
testresults.accuracy, testresults.errorRate)
results['test'].append(res)
if saveModel == True:
lossVal.x = range(folds * maxEpoch * math.floor(allDataTrain.size()[0] * train_per))
lrStr = mres.floatToStr("%2.15f" ,learning_rate)
fname = "%scdlc-mlp-simple-loss-values-%s-%s-%s-%s-%d.bin" % (path,optimName,lrStr,momentumStr,initStr,maxEpoch)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(lossVal, fh)
fh.close()
return classifier
def trainClassifierNew(allDataTrain,targetDataTrain,allDataTest,targetDataTest,learning_rate,momentum,maxEpoch,init_weight,saveModel,optimName,results):
classifier = Net(n_feature=128, n_hidden=128, n_output=2).cuda() # define the network
print(classifier) # net architecture
#optimizer = torch.optim.SGD(classifier.parameters(), lr=learning_rate)
optimizer = selectOptimizer(classifier,optimName,learning_rate,momentum)
loss_func = torch.nn.CrossEntropyLoss() # the target label is NOT an one-hotted
lossVal = mres.LossValues()
trainStats = mres.ModelStats()
testStats = mres.ModelStats()
errors = []
lrStr = mres.floatToStr("%2.15f",learning_rate)
initStr = mres.floatToStr("%2.15f", init_weight)
momentumStr = mres.floatToStr("%2.15f",momentum)
epc = 0
wndBatch = math.floor(allDataTrain.size()[0] / batchSize)
if allDataTrain.size()[0] % batchSize != 0:
wndBatch += 1
for fold in range(folds):
for epoch in range(maxEpoch):
print("class : %s fold %d epoch %d" % (classname,fold,epoch))
epc += 1
shuffle = np.random.permutation(allDataTrain.size()[0])
lr=optimizer.param_groups[0]['lr']
for btcCnt in range(wndBatch):
index = torch.from_numpy(shuffle[btcCnt * batchSize:(btcCnt + 1) * batchSize]).cuda()
#inp = scaleVector(torch.index_select(allDataTrain.data, 0, index))
inp = torch.index_select(allDataTrain.data, 0, index)
inp = autograd.Variable(inp)
target = autograd.Variable(torch.index_select(targetDataTrain.long().cuda(), 0, index))
optimizer.zero_grad()
pred = classifier(inp)
loss = loss_func(pred, target)
print("opt %s fold %d epoch %d lr %s mmt %s init %s - pred %f/%f target %f loss %f " % (optimName,fold,epoch,lrStr,momentumStr,initStr,pred.data[0][0],pred.data[0][1],target.data[0], loss.data[0]))
loss.backward()
optimizer.step()
errors.append(loss.data[0])
lossVal.y.append(loss.data[0])
mean = torch.mean(torch.Tensor(errors))
lossVal.mean.append(mean)
if (epoch+1) % threshold == 0 and epoch != 0:
trainresults = mres.testClassifier2(classifier,allDataTrain,targetDataTrain)
trainresults.calculateScores()
res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold,lrStr, momentumStr, initStr,maxEpoch, epoch+1,trainresults.correct, trainresults.all,trainresults.trueNegatives,trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives, trainresults.allPositives,trainresults.posRate,
trainresults.precision, trainresults.recall, trainresults.f1Score, trainresults.score, trainresults.accuracy, trainresults.errorRate )
results['train'].append(res)
trainStats.addStat(fold,epoch+1,trainresults)
testresults = mres.testClassifier2(classifier, allDataTest, targetDataTest)
testresults.calculateScores()
res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold,lrStr, momentumStr,initStr, maxEpoch, epoch+1,testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate,
testresults.precision, testresults.recall, testresults.f1Score, testresults.score,
testresults.accuracy, testresults.errorRate)
results['test'].append(res)
testStats.addStat(fold, epoch + 1, testresults)
trainresults = mres.testClassifier2(classifier, allDataTrain, targetDataTrain)
trainresults.calculateScores()
res = "train - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold, lrStr, momentumStr, initStr,maxEpoch, maxEpoch, trainresults.correct, trainresults.all,
trainresults.trueNegatives, trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives, trainresults.allPositives, trainresults.posRate,
trainresults.precision, trainresults.recall, trainresults.f1Score, trainresults.score,
trainresults.accuracy, trainresults.errorRate)
results['train'].append(res)
trainStats.addStat(fold, maxEpoch, trainresults)
testresults = mres.testClassifier2(classifier, allDataTest, targetDataTest)
testresults.calculateScores()
res = "test - fold:%d,lr:%s,mmt:%s,init:%s,maxepoch:%d,epoch:%d,score:%d/%d,trueNegPred/allNeg:%d/%d=%f,truePosPred/allPos:%d/%d=%f,precision:%f,recall:%f,f1:%f,score:%f,accuracy:%f,error_rate:%f" % (
fold, lrStr, momentumStr,initStr, maxEpoch, maxEpoch, testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate,
testresults.precision, testresults.recall, testresults.f1Score, testresults.score,
testresults.accuracy, testresults.errorRate)
results['test'].append(res)
testStats.addStat(fold, maxEpoch, testresults)
if saveModel == True:
lossVal.x = range(folds * maxEpoch * math.floor(allDataTrain.size()[0] * train_per))
lrStr = mres.floatToStr("%2.15f" ,learning_rate)
fname = "%scdlc-%s-mlp-simple-minmax-loss-values-%s-%s-%s-%s-%d.bin" % (path,classname,optimName,lrStr,momentumStr,initStr,maxEpoch)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(lossVal, fh)
fh.close()
mdlStats = {"train" : trainStats,"test":testStats}
fname = "%scdlc-%s-mlp-simple-minmax-4-hidden-stat-values-%s-%s-%s-%s-%d.bin" % (path,classname,optimName,lrStr,momentumStr,initStr,maxEpoch)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(mdlStats, fh)
fh.close()
return classifier
def trainClassifierLSTM(allDataTrain,allDataTest,embeddingDim,hiddenSize,initWeight,learningRate,momentum,maxEpoch,batchSize,folds,lstmDepth,saveModelStats,results):
classifier = LSTMClassifier(embeddingDim,hiddenSize,learningRate, momentum,lstmDepth).cuda()
classifier.initHidden(initWeight)
lrStr = mres.floatToStr("%2.15f", learningRate)
momentumStr = mres.floatToStr("%2.15f", momentum)
lossVal = mres.LossValues()
posValues = list(allDataTrain[1].values())
negValues = list(allDataTrain[0].values())
allPositives=len(posValues)
allNegatives=len(negValues)
totFileCount = allPositives + allNegatives
errors = []
#results = []
docDiv = math.floor(totFileCount / batchSize)
if totFileCount % batchSize != 0:
docDiv += 1
for fold in range(folds):
#errors = []
for epoch in range(maxEpoch):
print("class : %s fold %d epoch %d" % (classname,fold+1,epoch+1))
shuffle = np.random.permutation(totFileCount)
for sCnt in range(docDiv):
if shuffle[sCnt] < allPositives:
docInp = [posValues[shuffle[sCnt]].data.clone()]
target = autograd.Variable(torch.Tensor(1).fill_(1).float().cuda(), requires_grad=False)
else:
docInp = [negValues[shuffle[sCnt] - allPositives].data.clone()]
target = autograd.Variable(torch.Tensor(1).fill_(0).float().cuda(), requires_grad=False)
cx = autograd.Variable(torch.zeros(batchSize, hiddenSize).cuda())
hx = autograd.Variable(torch.zeros(batchSize, hiddenSize).cuda())
hidden = [hx, cx]
inp = list(docInp)
#inp = autograd.Variable(inp.resize_(inp.size()[0],1,inp.size()[1]).cuda())
classifier.zero_grad()
pred = classifier.forward(inp,hidden,inp[0].size()[0])
loss = ((pred - target) * (pred - target)).mean()
print("fold %d epoch %d lstmDepth %d lr %s mmt %s doc num %d - pred %f target %f loss %f " %
(fold+1,epoch+1,lstmDepth,lrStr,momentumStr,sCnt+1,pred.data[0][0],target.data[0], loss.data[0]))
loss.backward()
classifier.optimizer.step()
errors.append(loss.data[0])
lossVal.y.append(loss.data[0])
mean = torch.mean(torch.Tensor(errors))
lossVal.mean.append(mean)
if epoch % 50 == 0 and epoch != 0:
trainresults = mres.testClassifierLSTM(classifier,allDataTrain,hiddenSize)
res = "train - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, epoch+1,lstmDepth, trainresults.correct, trainresults.all,trainresults.trueNegatives,trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives, trainresults.allPositives,trainresults.posRate)
results.append(res)
testresults = mres.testClassifierLSTM(classifier, allDataTest, hiddenSize)
res = "test - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, epoch+1,lstmDepth, testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives, testresults.allPositives, testresults.posRate)
results.append(res)
trainresults = mres.testClassifierLSTM(classifier, allDataTrain, hiddenSize)
res = "train - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, maxEpoch ,lstmDepth, trainresults.correct, trainresults.all,trainresults.trueNegatives,trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives, trainresults.allPositives,trainresults.posRate)
results.append(res)
testresults = mres.testClassifierLSTM(classifier, allDataTest, hiddenSize)
res = "test - lr %s mmt %s maxepoch %d epoch %d lstmDepth %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, maxEpoch, maxEpoch ,lstmDepth, testresults.correct, testresults.all,testresults.trueNegatives,testresults.allNegatives,
testresults.negRate, testresults.truePositives, testresults.allPositives,testresults.posRate)
results.append(res)
#for res in results:
# print(res)
if saveModelStats == True:
lossVal.x = range(folds * maxEpoch * docDiv)
lrStr = mres.floatToStr("%2.15f", learningRate)
fname = "%s%s-lstm-cdlc-loss-values-%s-%s-%d-%d-%d.bin" % (path,compositionalMethod,lrStr,momentumStr,batchSize,maxEpoch,lstmDepth)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(lossVal, fh)
fh.close()
return classifier
def trainClassifier(allDataTrain, targetDataTrain, allDataTest, targetDataTest,
prms, results):
classifier = buildClassifierModel(allDataTrain.size()[1], 1).cuda()
#classifier = MLPClassifier(embedding_dim,embedding_dim,2,learning_rate,momentum).cuda()
for param in classifier.parameters():
init.uniform(param, -1 * prms.initWeight, prms.initWeight)
loss_function = nn.BCELoss(size_average=True).cuda()
#optimizer = optim.SGD(classifier.parameters(), lr=prms.learningRate, momentum=prms.momentum)
optimizer = optim.RMSprop(classifier.parameters(),
lr=prms.learningRate,
alpha=0.99,
eps=1e-08,
weight_decay=0,
momentum=prms.momentum,
centered=False)
lossVal = mres.LossValues()
errors = []
lrStr = mres.floatToStr("%2.15f", prms.learningRate)
momentumStr = mres.floatToStr("%2.15f", prms.momentum)
epc = 0
wndBatch = math.floor(allDataTrain.size()[0] / prms.batchSize)
if allDataTrain.size()[0] % prms.batchSize != 0:
wndBatch += 1
for fold in range(prms.folds):
for epoch in range(prms.maxEpoch):
print("class : %s fold %d epoch %d" % (classname, fold, epoch))
epc += 1
shuffle = np.random.permutation(allDataTrain.size()[0])
lr = optimizer.param_groups[0]['lr']
lrStr = mres.floatToStr("%2.15f", lr)
for btcCnt in range(wndBatch):
index = torch.from_numpy(
shuffle[btcCnt * prms.batchSize:(btcCnt + 1) *
prms.batchSize]).cuda()
inp = autograd.Variable(
torch.index_select(allDataTrain.data, 0, index))
target = autograd.Variable(torch.index_select(
targetDataTrain.cuda(), 0, index),
requires_grad=False)
classifier.zero_grad()
pred = classifier.forward(inp)
loss = loss_function(pred, target)
#loss = classifier.criterion(pred, target)
print(
"fold %d epoch %d lr %s mmt %s- pred %f target %f loss %f "
% (fold, epoch, lrStr, momentumStr, pred.data[0][0],
target.data[0], loss.data[0]))
loss.backward()
optimizer.step()
errors.append(loss.data[0])
lossVal.y.append(loss.data[0])
mean = torch.mean(torch.Tensor(errors))
lossVal.mean.append(mean)
if epoch % modelPrms.threshold == 0 and epoch != 0:
trainresults = mres.testClassifier(classifier, allDataTrain,
targetDataTrain)
res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, prms.maxEpoch, epoch + 1,
trainresults.correct, trainresults.all,
trainresults.trueNegatives, trainresults.allNegatives,
trainresults.negRate, trainresults.truePositives,
trainresults.allPositives, trainresults.posRate)
results.append(res)
testresults = mres.testClassifier(classifier, allDataTest,
targetDataTest)
res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, prms.maxEpoch, epoch + 1,
testresults.correct, testresults.all,
testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives,
testresults.allPositives, testresults.posRate)
results.append(res)
trainresults = mres.testClassifier(classifier, allDataTrain,
targetDataTrain)
res = "train - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, prms.maxEpoch, prms.maxEpoch, trainresults.correct,
trainresults.all, trainresults.trueNegatives,
trainresults.allNegatives, trainresults.negRate,
trainresults.truePositives, trainresults.allPositives,
trainresults.posRate)
results.append(res)
testresults = mres.testClassifier(classifier, allDataTest, targetDataTest)
res = "test - lr %s mmt %s maxepoch %d epoch %d score %d/%d - trueNegPred/allNeg:%d/%d=%f truePosPred/allPos:%d/%d=%f" % (
lrStr, momentumStr, prms.maxEpoch, prms.maxEpoch, testresults.correct,
testresults.all, testresults.trueNegatives, testresults.allNegatives,
testresults.negRate, testresults.truePositives,
testresults.allPositives, testresults.posRate)
results.append(res)
if prms.saveModel == True:
lossVal.x = range(prms.folds * prms.maxEpoch * wndBatch)
lrStr = mres.floatToStr("%2.15f", prms.learningRate)
fname = "%scdlc-mlp-simple-flat-loss-values-%s-%s-%d.bin" % (
path, lrStr, momentumStr, prms.maxEpoch)
fh = open(fname, 'wb') # Save model file as pickle
pickle.dump(lossVal, fh)
fh.close()
return classifier
