def __init__(self, modelFilename, inputImages, nw, nBins):
self.reset()
self.modelFilename = modelFilename
self.inputImages = inputImages
self.nw = nw
self.nBins = nBins
self.imWidthReduc = 8
self.imWidth = 32 #Image size and width
self.x, self.y_, self.keep_prob, self.y_conv = convnet_variables(
self.imWidth, self.imWidthReduc, self.nw, self.nBins)
self.saver = tf.train.Saver()
def train_model(dataDirName, overwrite=False, numTrainBatches=500000):
""" Train model. Unzip and overwrite exisiting training set if overwrite is True"""
# Open training data files
trainingSet = os.path.join(dataDirName, 'training_set.zip')
extractedFolder = os.path.join(dataDirName, 'training_set')
# zipRef = zipfile.ZipFile(trainingSet, 'r')
# zipRef.extractall(extractedFolder)
# zipRef.close()
if os.path.exists(extractedFolder) and overwrite:
shutil.rmtree(extractedFolder, ignore_errors=True)
os.system("unzip %s -d %s" % (trainingSet, extractedFolder))
elif not os.path.exists(extractedFolder):
os.system("unzip %s -d %s" % (trainingSet, extractedFolder))
else:
pass
npyFiles = {}
fileList = os.listdir(extractedFolder)
for filename in fileList:
f = os.path.join(extractedFolder, filename)
if filename.endswith('.gz'):
# # npyFiles[filename.strip('.npy.gz')] = gzip.GzipFile(f, 'r')
# gzFile = gzip.open(f, "rb")
# unCompressedFile = open(f.strip('.gz'), "wb")
# decoded = gzFile.read()
# unCompressedFile.write(decoded)
# gzFile.close()
# unCompressedFile.close()
npyFiles[filename.strip('.npy.gz')] = f.strip('.gz')
os.system("gzip -d %s" % f) # "gzip -dk %s" % f
elif filename.endswith('.npy'):
npyFiles[filename.strip('.npy')] = f
trainImages = np.load(npyFiles['trainImages'], mmap_mode='r')
trainLabels = np.load(npyFiles['trainLabels'], mmap_mode='r')
testImages = np.load(npyFiles['testImages'], mmap_mode='r')
testLabelsIndexes = np.load(npyFiles['testLabels'], mmap_mode='r')
typeNamesList = np.load(npyFiles['typeNamesList'])
testTypeNames = np.load(npyFiles['testTypeNames'])
# testImages = np.load(npyFiles['testImagesNoGal'], mmap_mode='r')
# testLabelsIndexes = np.load(npyFiles['testLabelsNoGal'], mmap_mode='r')
print("Completed creatingArrays")
print(len(trainImages))
nLabels = len(typeNamesList)
N = 1024
overSampling = OverSampling(nLabels, N, images=trainImages, labels=trainLabels)
trainArrays = overSampling.over_sample_arrays(smote=False)
trainImages, trainLabels = trainArrays['images'], trainArrays['labels']
# # Delete temporary memory mapping files
# for filename in glob.glob('shuffled*.dat') + glob.glob('oversampled*.dat'):
# if not os.path.samefile(filename, trainImages.filename) and not os.path.samefile(filename, trainLabels.filename):
# os.remove(filename)
# Set up the convolutional network architecture
imWidth = 32 # Image size and width
imWidthReduc = 8
a = []
x, y_, keep_prob, y_conv = convnet_variables(imWidth, imWidthReduc, N, nLabels)
with tf.Session() as sess: # config=tf.ConfigProto(inter_op_parallelism_threads=1, intra_op_parallelism_threads=1)) as sess:
# TRAIN AND EVALUATE MODEL
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_conv + 1e-8), reduction_indices=[1]))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
sess.run(tf.global_variables_initializer())
testLabels = labels_indexes_to_arrays(testLabelsIndexes[0:400], nLabels)
testImages = testImages[0:400]
# testLabelsWithGal = labels_indexes_to_arrays(testLabelsIndexesWithGal[0:200], nLabels)
# testImagesWithGal = testImagesWithGal[0:200]
trainImagesCycle = itertools.cycle(trainImages)
trainLabelsCycle = itertools.cycle(trainLabels)
for i in range(numTrainBatches):
batch_xs = np.array(list(itertools.islice(trainImagesCycle, 50 * i, 50 * i + 50)))
batch_ys = labels_indexes_to_arrays(list(itertools.islice(trainLabelsCycle, 50 * i, 50 * i + 50)), nLabels)
train_step.run(feed_dict={x: batch_xs, y_: batch_ys, keep_prob: 0.5})
if i % 100 == 0:
train_accuracy = accuracy.eval(feed_dict={x: batch_xs, y_: batch_ys, keep_prob: 1.0})
print("step %d, training accuracy %g" % (i, train_accuracy))
testAcc = accuracy.eval(feed_dict={x: testImages, y_: testLabels, keep_prob: 1.0})
print("test accuracy %g" % testAcc)
a.append(testAcc)
# if i % 1000 == 0:
# testWithGalAcc = accuracy.eval(feed_dict={x: testImagesWithGal, y_: testLabelsWithGal, keep_prob: 1.0})
# print("test With Gal accuracy %g" % testWithGalAcc)
print("test accuracy %g" % accuracy.eval(feed_dict={x: testImages, y_: testLabels, keep_prob: 1.0}))
yy = y_conv.eval(feed_dict={x: testImages, y_: testLabels, keep_prob: 1.0})
cp = correct_prediction.eval(feed_dict={x: testImages, y_: testLabels, keep_prob: 1.0})
print(cp)
for i in range(len(cp)):
if cp[i] == False:
predictedIndex = np.argmax(yy[i])
print(i, testTypeNames[i], typeNamesList[predictedIndex])
# SAVE THE MODEL
saveFilename = os.path.join(dataDirName, "tensorflow_model.ckpt")
saver = tf.train.Saver()
save_path = saver.save(sess, saveFilename)
print("Model saved in file: %s" % save_path)
modelFilenames = [saveFilename + '.index', saveFilename + '.meta', saveFilename + '.data-00000-of-00001']
try:
import matplotlib.pyplot as plt
plt.plot(a)
plt.xlabel("Number of Epochs")
plt.ylabel("Testing accuracy")
plt.savefig(os.path.join(dataDirName, "testing_accuracy.png"))
np.savetxt(os.path.join(dataDirName, "testing_accuracy.txt"), np.array(a))
except Exception as e:
print(e)
try:
calc_model_statistics(saveFilename, testImages, testTypeNames, typeNamesList)
except Exception as e:
print(e)
return modelFilenames
def train_model(dataDirName):
# Open training data files
scriptDirectory = os.path.dirname(os.path.abspath(__file__))
trainingSet = dataDirName + 'training_set.zip'
extractedFolder = os.path.join(dataDirName, 'training_set')
# zipRef = zipfile.ZipFile(trainingSet, 'r')
# zipRef.extractall(extractedFolder)
# zipRef.close()
os.system("unzip %s -d %s" % (trainingSet, extractedFolder))
npyFiles = {}
fileList = os.listdir(extractedFolder)
for filename in fileList:
if filename.endswith('.gz'):
f = os.path.join(scriptDirectory, extractedFolder, filename)
# # npyFiles[filename.strip('.npy.gz')] = gzip.GzipFile(f, 'r')
# gzFile = gzip.open(f, "rb")
# unCompressedFile = open(f.strip('.gz'), "wb")
# decoded = gzFile.read()
# unCompressedFile.write(decoded)
# gzFile.close()
# unCompressedFile.close()
npyFiles[filename.strip('.npy.gz')] = f.strip('.gz')
os.system("gzip -dk %s" % f)
trainImages = np.load(npyFiles['trainImages'], mmap_mode='r')
trainLabels = np.load(npyFiles['trainLabels'], mmap_mode='r')
testImages = np.load(npyFiles['testImages'], mmap_mode='r')
testLabelsIndexes = np.load(npyFiles['testLabels'], mmap_mode='r')
typeNamesList = np.load(npyFiles['typeNamesList'])
testTypeNames = np.load(npyFiles['testTypeNames'])
# testImages = np.load(npyFiles['testImagesNoGal'], mmap_mode='r')
# testLabelsIndexes = np.load(npyFiles['testLabelsNoGal'], mmap_mode='r')
print("Completed creatingArrays")
print(len(trainImages))
nLabels = len(typeNamesList)
# Set up the convolutional network architecture
N = 1024
imWidth = 32 # Image size and width
imWidthReduc = 8
a = []
x, y_, keep_prob, y_conv = convnet_variables(imWidth, imWidthReduc, N,
nLabels)
with tf.Session(
) as sess: # config=tf.ConfigProto(inter_op_parallelism_threads=1, intra_op_parallelism_threads=1)) as sess:
# TRAIN AND EVALUATE MODEL
cross_entropy = tf.reduce_mean(
-tf.reduce_sum(y_ * tf.log(y_conv + 1e-8), reduction_indices=[1]))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
sess.run(tf.initialize_all_variables())
testLabels = labels_indexes_to_arrays(testLabelsIndexes[0:400],
nLabels)
testImages = testImages[0:400]
# testLabelsWithGal = labels_indexes_to_arrays(testLabelsIndexesWithGal[0:200], nLabels)
# testImagesWithGal = testImagesWithGal[0:200]
trainImagesCycle = itertools.cycle(trainImages)
trainLabelsCycle = itertools.cycle(trainLabels)
for i in range(400000):
batch_xs = np.array(
list(itertools.islice(trainImagesCycle, 50 * i, 50 * i + 50)))
batch_ys = labels_indexes_to_arrays(
list(itertools.islice(trainLabelsCycle, 50 * i, 50 * i + 50)),
nLabels)
train_step.run(feed_dict={
x: batch_xs,
y_: batch_ys,
keep_prob: 0.5
})
if i % 100 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batch_xs,
y_: batch_ys,
keep_prob: 1.0
})
print("step %d, training accuracy %g" % (i, train_accuracy))
testAcc = accuracy.eval(feed_dict={
x: testImages,
y_: testLabels,
keep_prob: 1.0
})
print("test accuracy %g" % testAcc)
a.append(testAcc)
# if i % 1000 == 0:
# testWithGalAcc = accuracy.eval(feed_dict={x: testImagesWithGal, y_: testLabelsWithGal, keep_prob: 1.0})
# print("test With Gal accuracy %g" % testWithGalAcc)
print("test accuracy %g" % accuracy.eval(feed_dict={
x: testImages,
y_: testLabels,
keep_prob: 1.0
}))
yy = y_conv.eval(feed_dict={
x: testImages,
y_: testLabels,
keep_prob: 1.0
})
cp = correct_prediction.eval(feed_dict={
x: testImages,
y_: testLabels,
keep_prob: 1.0
})
print(cp)
for i in range(len(cp)):
if cp[i] == False:
predictedIndex = np.argmax(yy[i])
print(i, testTypeNames[i], typeNamesList[predictedIndex])
# SAVE THE MODEL
saveFilename = dataDirName + "tensorflow_model.ckpt"
saver = tf.train.Saver()
save_path = saver.save(sess, saveFilename)
print("Model saved in file: %s" % save_path)
modelFilenames = [
saveFilename + '.index', saveFilename + '.meta',
saveFilename + '.data-00000-of-00001'
]
# ACTUAL ACCURACY, broadTYPE ACCURACY, AGE ACCURACY
typeAndAgeCorrect = 0
typeCorrect = 0
broadTypeCorrect = 0
broadTypeAndAgeCorrect = 0
typeAndNearAgeCorrect = 0
broadTypeAndNearAgeCorrect = 0
for i in range(len(testTypeNames)):
predictedIndex = np.argmax(yy[i])
testBroadType = testTypeNames[i][0:2]
actualBroadType = typeNamesList[predictedIndex][0:2]
if testTypeNames[i][0:3] == 'IIb':
testBroadType = 'Ib'
if typeNamesList[predictedIndex][0:3] == 'IIb':
actualBroadType = 'Ib'
testType = testTypeNames[i].split(': ')[0]
actualType = typeNamesList[predictedIndex].split(': ')[0]
testAge = testTypeNames[i].split(': ')[1]
actualAge = typeNamesList[predictedIndex].split(': ')[1]
nearTestAge = testAge.split(' to ')
if testTypeNames[i] == typeNamesList[predictedIndex]:
typeAndAgeCorrect += 1
if testType == actualType: # correct type
typeCorrect += 1
if (nearTestAge[0] in actualAge) or (
nearTestAge[1] in actualAge
): # check if the age is in the neigbouring bin
typeAndNearAgeCorrect += 1 # all correct except nearby bin
if testBroadType == actualBroadType: # correct broadtype
broadTypeCorrect += 1
if testAge == actualAge:
broadTypeAndAgeCorrect += 1
if (nearTestAge[0] in actualAge) or (
nearTestAge[1] in actualAge
): # check if the age is in the neigbouring bin
broadTypeAndNearAgeCorrect += 1 # Broadtype and nearby bin
typeAndAgeAccuracy = float(typeAndAgeCorrect) / len(testTypeNames)
typeAccuracy = float(typeCorrect) / len(testTypeNames)
broadTypeAccuracy = float(broadTypeCorrect) / len(testTypeNames)
broadTypeAndAgeAccuracy = float(broadTypeAndAgeCorrect) / len(
testTypeNames)
typeAndNearAgeAccuracy = float(typeAndNearAgeCorrect) / len(testTypeNames)
broadTypeAndNearAgeAccuracy = float(broadTypeAndNearAgeCorrect) / len(
testTypeNames)
print("typeAndAgeAccuracy : " + str(typeAndAgeAccuracy))
print("typeAccuracy : " + str(typeAccuracy))
print("broadTypeAccuracy : " + str(broadTypeAccuracy))
print("broadTypeAndAgeAccuracy: " + str(broadTypeAndAgeAccuracy))
print("typeAndNearAgeAccuracy : " + str(typeAndNearAgeAccuracy))
print("broadTypeAndNearAgeAccuracy : " + str(broadTypeAndNearAgeAccuracy))
return modelFilenames
def calc_model_statistics(modelFilename,
testLabels,
testImages,
testTypeNames,
typeNamesList,
typeNamesBroad=None):
tf.reset_default_graph()
nw = len(testImages[0])
nBins = len(typeNamesList)
imWidthReduc = 8
imWidth = 32 # Image size and width
x, y_, keep_prob, y_conv = convnet_variables(imWidth, imWidthReduc, nw,
nBins)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, modelFilename)
yy = y_conv.eval(feed_dict={x: testImages, keep_prob: 1.0})
# CONFUSION MATRIX
predictedLabels = []
for i, name in enumerate(testTypeNames):
predictedLabels.append(np.argmax(yy[i]))
predictedLabels = np.array(predictedLabels)
confMatrix = tf.confusion_matrix(testLabels, predictedLabels).eval()
if typeNamesBroad is not None:
broadTypeLabels = np.arange(0, nBins + 1,
int(nBins / len(typeNamesBroad)))
testLabelsBroad = np.digitize(testLabels, broadTypeLabels) - 1
predictedLabelsBroad = np.digitize(predictedLabels,
broadTypeLabels) - 1
confMatrixBroad = tf.confusion_matrix(testLabelsBroad,
predictedLabelsBroad).eval()
np.set_printoptions(precision=2)
print(confMatrixBroad)
plot_confusion_matrix(confMatrixBroad,
classes=typeNamesBroad,
normalize=True,
title='Normalized confusion matrix',
fig_dir='.',
name='broad')
# plt.show()
# np.set_printoptions(precision=2)
# print(confMatrix)
# plot_confusion_matrix(confMatrix, classes=typeNamesList, normalize=True, title='Normalized confusion matrix', fig_dir='.', name='all')
# ACTUAL ACCURACY, broadTYPE ACCURACY, AGE ACCURACY
typeAndAgeCorrect = 0
typeCorrect = 0
broadTypeCorrect = 0
broadTypeAndAgeCorrect = 0
typeAndNearAgeCorrect = 0
broadTypeAndNearAgeCorrect = 0
for i in range(len(testTypeNames)):
predictedIndex = np.argmax(yy[i])
classification = testTypeNames[i].split(': ')
if len(classification) == 2:
testType, testAge = classification
else:
testGalType, testType, testAge = classification
actual = typeNamesList[predictedIndex].split(': ')
if len(actual) == 2:
actualType, actualAge = actual
else:
actualGalType, actualType, actualAge = actual
testBroadType = testType[0:2]
actualBroadType = actualType[0:2]
if testType[0:3] == 'IIb':
testBroadType = 'Ib'
if actualType[0:3] == 'IIb':
actualBroadType = 'Ib'
nearTestAge = testAge.split(' to ')
if testTypeNames[i] == typeNamesList[predictedIndex]:
typeAndAgeCorrect += 1
if testType == actualType: # correct type
typeCorrect += 1
if (nearTestAge[0] in actualAge) or (
nearTestAge[1] in actualAge
): # check if the age is in the neigbouring bin
typeAndNearAgeCorrect += 1 # all correct except nearby bin
if testBroadType == actualBroadType: # correct broadtype
broadTypeCorrect += 1
if testAge == actualAge:
broadTypeAndAgeCorrect += 1
if (nearTestAge[0] in actualAge) or (
nearTestAge[1] in actualAge
): # check if the age is in the neigbouring bin
broadTypeAndNearAgeCorrect += 1 # Broadtype and nearby bin
typeAndAgeAccuracy = float(typeAndAgeCorrect) / len(testTypeNames)
typeAccuracy = float(typeCorrect) / len(testTypeNames)
broadTypeAccuracy = float(broadTypeCorrect) / len(testTypeNames)
broadTypeAndAgeAccuracy = float(broadTypeAndAgeCorrect) / len(
testTypeNames)
typeAndNearAgeAccuracy = float(typeAndNearAgeCorrect) / len(testTypeNames)
broadTypeAndNearAgeAccuracy = float(broadTypeAndNearAgeCorrect) / len(
testTypeNames)
print("typeAndAgeAccuracy : " + str(typeAndAgeAccuracy))
print("typeAccuracy : " + str(typeAccuracy))
print("broadTypeAccuracy : " + str(broadTypeAccuracy))
print("broadTypeAndAgeAccuracy: " + str(broadTypeAndAgeAccuracy))
print("typeAndNearAgeAccuracy : " + str(typeAndNearAgeAccuracy))
print("broadTypeAndNearAgeAccuracy : " + str(broadTypeAndNearAgeAccuracy))
def calc_model_statistics(modelFilename, testImages, testTypeNames,
typeNamesList):
tf.reset_default_graph()
nw = len(testImages[0])
nBins = len(typeNamesList)
imWidthReduc = 8
imWidth = 32 # Image size and width
x, y_, keep_prob, y_conv = convnet_variables(imWidth, imWidthReduc, nw,
nBins)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, modelFilename)
yy = y_conv.eval(feed_dict={x: testImages, keep_prob: 1.0})
# ACTUAL ACCURACY, broadTYPE ACCURACY, AGE ACCURACY
typeAndAgeCorrect = 0
typeCorrect = 0
broadTypeCorrect = 0
broadTypeAndAgeCorrect = 0
typeAndNearAgeCorrect = 0
broadTypeAndNearAgeCorrect = 0
for i in range(len(testTypeNames)):
predictedIndex = np.argmax(yy[i])
classification = testTypeNames[i].split(': ')
if len(classification) == 2:
testType, testAge = classification
else:
testGalType, testType, testAge = classification
actual = typeNamesList[predictedIndex].split(': ')
if len(actual) == 2:
actualType, actualAge = actual
else:
actualGalType, actualType, actualAge = actual
testBroadType = testType[0:2]
actualBroadType = actualType[0:2]
if testType[0:3] == 'IIb':
testBroadType = 'Ib'
if actualType[0:3] == 'IIb':
actualBroadType = 'Ib'
nearTestAge = testAge.split(' to ')
if testTypeNames[i] == typeNamesList[predictedIndex]:
typeAndAgeCorrect += 1
if testType == actualType: # correct type
typeCorrect += 1
if (nearTestAge[0] in actualAge) or (
nearTestAge[1] in actualAge
): # check if the age is in the neigbouring bin
typeAndNearAgeCorrect += 1 # all correct except nearby bin
if testBroadType == actualBroadType: # correct broadtype
broadTypeCorrect += 1
if testAge == actualAge:
broadTypeAndAgeCorrect += 1
if (nearTestAge[0] in actualAge) or (
nearTestAge[1] in actualAge
): # check if the age is in the neigbouring bin
broadTypeAndNearAgeCorrect += 1 # Broadtype and nearby bin
typeAndAgeAccuracy = float(typeAndAgeCorrect) / len(testTypeNames)
typeAccuracy = float(typeCorrect) / len(testTypeNames)
broadTypeAccuracy = float(broadTypeCorrect) / len(testTypeNames)
broadTypeAndAgeAccuracy = float(broadTypeAndAgeCorrect) / len(
testTypeNames)
typeAndNearAgeAccuracy = float(typeAndNearAgeCorrect) / len(testTypeNames)
broadTypeAndNearAgeAccuracy = float(broadTypeAndNearAgeCorrect) / len(
testTypeNames)
print("typeAndAgeAccuracy : " + str(typeAndAgeAccuracy))
print("typeAccuracy : " + str(typeAccuracy))
print("broadTypeAccuracy : " + str(broadTypeAccuracy))
print("broadTypeAndAgeAccuracy: " + str(broadTypeAndAgeAccuracy))
print("typeAndNearAgeAccuracy : " + str(typeAndNearAgeAccuracy))
print("broadTypeAndNearAgeAccuracy : " + str(broadTypeAndNearAgeAccuracy))