def trainMegaSolarCNN():
# paths in the local machine
filelocationspath = args.filelocationspath
filename = filelocationspath +"/"+args.positivefilename
trainPosLocations = readtxt.ReadLocations(filename);
filename = filelocationspath +"/"+args.negativefilename
trainNegLocations = readtxt.ReadLocations(filename);
model = cnnmodel.MegaSolarBN()
optimizer = optimizers.MomentumSGD(lr=0.001, momentum=0.9)
optimizer.setup(model)
model.to_gpu()
negbatchsize = 120
posbatchsize = 8
negdatasize = len(trainNegLocations)
posdatasize = len(trainPosLocations)
datamean=[]
datastd=[]
datamean,datastd=get_coordinate_from_csv(args.meanstdfilepath)
x_train_all_positive = bl.loadbatchfiles(trainPosLocations,range(posdatasize),datamean,datastd)
x_train_all_negative = bl.loadbatchfiles(trainNegLocations,range(negdatasize),datamean,datastd)
NumNegTrainData = 72720
print('Number of Negative samples %d' % (negdatasize))
print('Number of Positive samples %d' % (posdatasize))
print('----------------------------------------------------------')
print('Number of Negative samples at each epoch %d' % (NumNegTrainData))
print('Number of Positive samples at each epoch %d' % (posdatasize))
print('----------------------------------------------------------')
batchlooplen = len(range(0, NumNegTrainData, negbatchsize));
datasize = batchlooplen*(posbatchsize+negbatchsize)
NUMEPOCH = 2000
for epoch in range(NUMEPOCH):
print('epoch %d' % epoch)
sum_loss = 0
sum_accuracy = 0
mean_loss = np.zeros(NUMEPOCH)
mean_accuracy = np.zeros(NUMEPOCH)
negindexes = np.random.permutation(negdatasize)
posindexes = np.random.permutation(posdatasize)
posindexes = np.tile(posindexes,(10*8))
for i in range(0, NumNegTrainData, negbatchsize):
x_train_positive = x_train_all_positive[posindexes[(i/negbatchsize)*posbatchsize:(i/negbatchsize)*posbatchsize+posbatchsize]]
y_train_positive = np.int32(np.ones(posbatchsize))
x_train_negative = x_train_all_negative[negindexes[i:i+negbatchsize]]
y_train_negative = np.int32(np.zeros(negbatchsize))
x_train = np.append(x_train_positive,x_train_negative,axis=0)
y_train = np.append(y_train_positive,y_train_negative)
indexes = np.random.permutation(posbatchsize+negbatchsize)
x = Variable(cuda.to_gpu(x_train[indexes]))
t = Variable(cuda.to_gpu(y_train[indexes]))
optimizer.update(model,x,t)
loss = model(x, t)
sum_loss += loss.data * (posbatchsize+negbatchsize)
sum_accuracy += model.accuracy.data * (posbatchsize+negbatchsize)
print "EPOCH MEAN TRAIN LOSS and ACCURACY VALUES: "
mean_loss[epoch] = sum_loss / datasize
mean_accuracy[epoch] = sum_accuracy / datasize
print mean_loss[epoch]
print mean_accuracy[epoch]
modelFileName = str(args.modelFileNamePath)+str(epoch)+"_iteration.model"
stateFileName = str(args.modelFileNamePath)+str(epoch)+"_iteration.state"
serializers.save_npz(modelFileName, model)
serializers.save_npz(stateFileName, optimizer)
return model,optimizer,loss,mean_loss,mean_accuracy
def testMegaSolarCNN():
# paths in the local machine
filelocationspath = args.filelocationspath
filename = filelocationspath + "/" + args.positivefilename
testPosLocations = readtxt.ReadLocations(filename)
filename = filelocationspath + "/" + args.negativefilename
testNegLocations = readtxt.ReadLocations(filename)
negbatchsize = 500
posdatasize = len(testPosLocations)
negdatasize_part1 = len(
testNegLocations) - len(testNegLocations) % negbatchsize
negdatasize = len(testNegLocations)
datamean = []
datastd = []
datamean, datastd = get_coordinate_from_csv(args.meanstdfilepath)
x_test_all_positive = bl.loadbatchfiles(testPosLocations,
range(posdatasize), datamean,
datastd)
x_test_all_negative = bl.loadbatchfiles(testNegLocations,
range(negdatasize), datamean,
datastd)
print('Number of Negative samples %d' % (negdatasize))
print('Number of Positive samples %d' % (posdatasize))
print('----------------------------------------------------------')
model = cnnmodel.MegaSolarBN()
MAX_EPOCH = 5000
accuracy_pos = []
accuracy_neg = []
iou_all = []
my_pos_thresh = 0.999
for epoch in range(0, MAX_EPOCH, 1):
print('EPOCH %d' % epoch)
modelFileNamePath = str(args.modelFileNamePath)
modelFileName = modelFileNamePath + str(epoch) + "_iteration.model"
serializers.load_npz(modelFileName, model)
model.to_gpu()
model.train = False
print "___________________________________________"
print "Epoch MEAN TEST ACCURACY Positive Samples: "
x_test_positive = x_test_all_positive
y_test_positive = np.int32(np.ones(len(testPosLocations)))
x = Variable(cuda.to_gpu(x_test_positive))
t = Variable(cuda.to_gpu(y_test_positive))
respos = cnnForward.forward(model, x)
cnn_prediction = np.float32(
cuda.to_cpu(respos.data[:, 1]) > my_pos_thresh)
cnn_acc = cnn_prediction.sum() / posdatasize
accuracy_pos = np.append(accuracy_pos, cnn_acc)
print cnn_acc
x_pos_test = x
t_pos_test = t
print "___________________________________________"
print "Epoch MEAN TEST ACCURACY Negative Samples: "
sum_accuracy = 0
for ti in range(0, negdatasize_part1, negbatchsize):
x_test_negative = x_test_all_negative[ti:ti + negbatchsize]
y_test_negative = np.int32(np.zeros(negbatchsize))
x = Variable(cuda.to_gpu(x_test_negative))
t = Variable(cuda.to_gpu(y_test_negative))
resneg = cnnForward.forward(model, x)
cnn_prediction = np.float32(
cuda.to_cpu(resneg.data[:, 1]) <= my_pos_thresh)
cnn_acc = cnn_prediction.sum() / negbatchsize
sum_accuracy += cnn_acc * (negbatchsize)
x_test_negative = x_test_all_negative[negdatasize_part1:negdatasize]
y_test_negative = np.int32(
np.zeros(len(range(negdatasize_part1, negdatasize))))
x = Variable(cuda.to_gpu(x_test_negative))
t = Variable(cuda.to_gpu(y_test_negative))
resneg = cnnForward.forward(model, x)
cnn_prediction = np.float32(
cuda.to_cpu(resneg.data[:, 1]) <= my_pos_thresh)
cnn_acc = cnn_prediction.sum() / len(y_test_negative)
sum_accuracy += cnn_acc * len(y_test_negative)
test_negative_accuracy = sum_accuracy / negdatasize
accuracy_neg = np.append(accuracy_neg, test_negative_accuracy)
print test_negative_accuracy
x_neg_test = x
t_neg_test = t
TP = np.round(accuracy_pos[-1] * posdatasize)
FN = posdatasize - TP
FP = np.round(negdatasize - accuracy_neg[-1] * negdatasize)
TN = negdatasize - FP
iou_all = np.append(iou_all, TP / (TP + FN + FP))
print "___________________________________________"
print "Epoch MEAN TEST Intersection over Union: "
print iou_all[-1]
print "TP:" + str(TP) + " FN:" + str(FN)
print "FP:" + str(FP) + " TN:" + str(TN)
return model, iou_all, x_pos_test, t_pos_test, x_neg_test, t_neg_test
mean = []
std = []
for channel in range(inputchanel):
data = np.array(image[channel])
mean.append(np.mean(data))
std.append(np.std(data))
return mean, std
if __name__ == "__main__":
inputchanel = 7
pfilename = args.filelocationspath + "/" + args.positivefilename
trainPosLocations = readtxt.ReadLocations(pfilename)
nfilename = args.filelocationspath + "/" + args.negativefilename
trainNegLocations = readtxt.ReadLocations(nfilename)
negdatasize = len(trainNegLocations)
posdatasize = len(trainPosLocations)
mean, std = mathdatameanstd(trainPosLocations,
range(posdatasize), trainNegLocations,
range(negdatasize), inputchanel)
file1 = open(str(args.meanstdfilepath) + ".csv", 'a')
for channel in range(inputchanel):
file1.write(str(mean[channel]) + "," + str(std[channel]) + '\n')
file1.close()