def loadData(path, ):
# Vengono caricati dai rispettivi file i training set ed i test set
emnistLoader = MNIST(path)
trainingData, trainingLabels = emnistLoader.load(path + '/emnist-byclass-train-images-idx3-ubyte',
path + '/emnist-byclass-train-labels-idx1-ubyte')
testData, testLabels = emnistLoader.load(path + '/emnist-byclass-test-images-idx3-ubyte',
path + '/emnist-byclass-test-labels-idx1-ubyte')
# Si convertono i valori ACII in caratteri
mapping = []
with open(path + '/emnist-byclass-mapping.txt') as f:
for line in f:
mapping.append(chr(int(line.split()[1])))
trainingData = np.array(trainingData)
trainingLabels = np.array(trainingLabels)
testData = np.array(testData)
testLabels = np.array(testLabels)
trainingData = normalize(trainingData)
testData = normalize(testData)
trainingData = reshape(trainingData)
testData = reshape(testData)
trainingLabels = preprocess_labels(trainingLabels, len(mapping))
testLabels = preprocess_labels(testLabels, len(mapping))
return trainingData, trainingLabels, testData, testLabels, mapping
def load_train_and_test_dataset(data_path='data/gzip'):
data = MNIST(data_path)
X_train, y_train = data.load(
os.path.join(data_path, 'emnist-byclass-train-images-idx3-ubyte'),
os.path.join(data_path, 'emnist-byclass-train-labels-idx1-ubyte'))
X_test, y_test = data.load(
os.path.join(data_path, 'emnist-byclass-test-images-idx3-ubyte'),
os.path.join(data_path, 'emnist-byclass-test-labels-idx1-ubyte'))
# Normalização
X_train = np.array(X_train) / 255.0
y_train = np.array(y_train)
X_test = np.array(X_test) / 255.0
y_test = np.array(y_test)
# Formatação
X_train = X_train.reshape(X_train.shape[0], 28, 28)
X_test = X_test.reshape(X_test.shape[0], 28, 28)
X_train = X_train.reshape(X_train.shape[0], 784, 1)
X_test = X_test.reshape(X_test.shape[0], 784, 1)
return {
"X_train": X_train,
"y_train": y_train,
"X_test": X_test,
"y_test": y_test,
}
def load_data(path, ):
"""Load data from the EMNIST dataset.
All the data files should be using the original file names as given by
the EMNIST website (https://www.nist.gov/itl/iad/image-group/emnist-dataset).
Args:
path (str): Directory containing all data files.
Returns:
Train and test data arrays with their respective label arrays
and label mapping.
"""
# Read all EMNIST test and train data
mndata = MNIST(path)
X_train, y_train = mndata.load(
path + '/emnist-byclass-train-images-idx3-ubyte',
path + '/emnist-byclass-train-labels-idx1-ubyte')
X_test, y_test = mndata.load(
path + '/emnist-byclass-test-images-idx3-ubyte',
path + '/emnist-byclass-test-labels-idx1-ubyte')
# Read mapping of the labels and convert ASCII values to chars
mapping = []
with open(path + '/emnist-byclass-mapping.txt') as f:
for line in f:
mapping.append(chr(int(line.split()[1])))
X_train = np.array(X_train)
y_train = np.array(y_train)
X_test = np.array(X_test)
y_test = np.array(y_test)
X_train = normalize(X_train)
X_test = normalize(X_test)
X_train = reshape_for_cnn(X_train)
X_test = reshape_for_cnn(X_test)
y_train = preprocess_labels(y_train, len(mapping))
y_test = preprocess_labels(y_test, len(mapping))
return X_train, y_train, X_test, y_test, mapping
def getData():
# This loads in the character datasets that we want to interpret
mndata = MNIST('data')
x_train, y_train = mndata.load(
'data/emnist-letters-train-images-idx3-ubyte',
'data/emnist-letters-train-labels-idx1-ubyte')
x_test, y_test = mndata.load('data/emnist-letters-test-images-idx3-ubyte',
'data/emnist-letters-test-labels-idx1-ubyte')
print("Finished loading data.\n")
# Normalizes our data from 0 to 255 to 0 to 1
x_train = np.array(x_train) / 255.0
x_test = np.array(x_test) / 255.0
# The data given is from 1 to 26, so this 0 indexes it
y_train = np.array(y_train) - 1
y_test = np.array(y_test) - 1
# This puts it into the form that we want to pass into keras (samples, rows, columns, channels)
# Channels is just 1 because our images are gray-scale
# x_train.shape[0] tells how many samples we have
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
# Rotates the image and reflects it so it looks "normal" to us
x_train = np.rot90(x_train, axes=(1, 2), k=3)
x_test = np.rot90(x_test, axes=(1, 2), k=3)
x_train = x_train[:, :, ::-1, :]
x_test = x_test[:, :, ::-1, :]
# This just shows how some of our samples look like, comment it out after trying it out
for z in range(0, 10):
fig = plt.figure()
plt.imshow(255 * x_train[z].reshape(28, 28),
interpolation="nearest",
cmap="gray")
fig.suptitle(chr(y_train[z] + 65), fontsize=20)
plt.show()
# This one-hot encodes our data. This post explains why we need to do it, if you're interested
# https://hackernoon.com/what-is-one-hot-encoding-why-and-when-do-you-have-to-use-it-e3c6186d008f
y_train = np_utils.to_categorical(y_train, 26)
y_test = np_utils.to_categorical(y_test, 26)
return x_train, y_train, x_test, y_test
def load_preprocess_EMNIST():
emndata = MNIST('emnist_data')
X_train, y_train = emndata.load(
'emnist_data/emnist-byclass-train-images-idx3-ubyte',
'emnist_data/emnist-byclass-train-labels-idx1-ubyte')
X_test, y_test = emndata.load(
'emnist_data/emnist-byclass-test-images-idx3-ubyte',
'emnist_data/emnist-byclass-test-labels-idx1-ubyte')
X_train = np.array(X_train)
y_train = np.array(y_train)
X_test = np.array(X_test)
y_test = np.array(y_test)
X_train = X_train.astype('float32') / (255 / 2) - 1 # for tanh
X_test = X_test.astype('float32') / (255 / 2) - 1
y_train = to_categorical(y_train, n_label)
y_test = to_categorical(y_test, n_label)
return X_train, y_train, X_test, y_test
def load_data(path, ):
mndata = MNIST(path)
X_train, y_train = mndata.load(
path + '/emnist-bymerge-train-images-idx3-ubyte',
path + '/emnist-bymerge-train-labels-idx1-ubyte')
X_test, y_test = mndata.load(
path + '/emnist-bymerge-test-images-idx3-ubyte',
path + '/emnist-bymerge-test-labels-idx1-ubyte')
X_train = np.array(X_train)
y_train = np.array(y_train)
X_test = np.array(X_test)
y_test = np.array(y_test)
X_train = normalize(X_train)
X_test = normalize(X_test)
y_train = preprocess_labels(y_train, 47)
y_test = preprocess_labels(y_test, 47)
return X_train, y_train, X_test, y_test
#import other libraries
import numpy as np
import matplotlib.pyplot as plt
from fg import freeze_graph
from mnist import MNIST
mndata = MNIST('data')
#Data Sets
EMNIST_TRAINING_IMAGES = "data/emnist-letters-train-images-idx3-ubyte"
EMNIST_TRAINING_LABELS = "data/emnist-letters-train-labels-idx1-ubyte"
EMNIST_TEST_IMAGES = "data/emnist-letters-test-images-idx3-ubyte"
EMNIST_TEST_LABELS = "data/emnist-letters-test-labels-idx1-ubyte"
#Load data
X_train, y_train = mndata.load(EMNIST_TRAINING_IMAGES, EMNIST_TRAINING_LABELS)
X_test, y_test = mndata.load(EMNIST_TEST_IMAGES, EMNIST_TEST_LABELS)
#Convert data to numpy arrays
#Also normalize the images so they become between [0, 1]
X_train = np.array(X_train) / 255.0
y_train = np.array(y_train)
X_test = np.array(X_test) / 255.0
y_test = np.array(y_test)
#Resize the images to 28*28 for pre-processing
X_train = X_train.reshape(X_train.shape[0], 28, 28)
X_test = X_test.reshape(X_test.shape[0], 28, 28)
#for train data
for t in range(112800):
cross_entropy = -tf.reduce_sum(Y_ * tf.log(Y))
#% of correct answers found in batch
is_correct = tf.equal(tf.argmax(Y, 1), tf.argmax(Y_, 1))
accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))
#define how much distance is minimized per training iteration
optimizer = tf.train.GradientDescentOptimizer(0.003)
train_step = optimizer.minimize(cross_entropy)
#start session (instantiate network)
sess = tf.Session()
sess.run(init)
#load training and test set
mn_full_data = MNIST.load('train-images.idx3-ubyte', 'train-labels.idx1-ubyte')
mn_train_data = MNIST.load('t10k-images.idx3-ubyte', 't10k-labels.idx1-ubyte')
idx = 0
for i in range(100):
#load a batch of images and correct answers
batch_X, batch_Y, idx = next_batch(100, idx, mn_train_data)
train_data = {X: batch_X, Y_: batch_Y}
#train
sess.run(train_step, feed_dict=train_data)
#determine training success
a, c = sess.run([accuracy, cross_entropy], feed=train_data)
#Run full test
from PIL import Image
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
# Import the modules
from sklearn.externals import joblib
from sklearn import datasets
from skimage.feature import hog
from sklearn.svm import LinearSVC
import numpy as np
# Read all EMNIST test and train data
mndata = MNIST('C:/Users/manoj/Documents/py_workspace/ocr/EMNIST/data')
X_train, y_train = mndata.load('C:/Users/manoj/Documents/py_workspace/ocr/EMNIST/data/emnist-letters-train-images-idx3-ubyte',
'C:/Users/manoj/Documents/py_workspace/ocr/EMNIST/data/emnist-letters-train-labels-idx1-ubyte')
'''
X_train, y_train = mndata.load('C:/Users/manoj/Documents/py_workspace/ocr/EMNIST/data/emnist-letters-test-images-idx3-ubyte',
'C:/Users/manoj/Documents/py_workspace/ocr/EMNIST/data/emnist-letters-test-labels-idx1-ubyte')
'''
# Read mapping of the labels and convert ASCII values to chars
print("Read mapping of the labels and convert ASCII values to chars")
mapping = []
with open('C:/Users/manoj/Documents/py_workspace/ocr/EMNIST/data/emnist-letters-mapping.txt') as f:
for line in f:
mapping.append(chr(int(line.split()[1])))
print("Convert data to numpy arrays and normalize images to the interval [0, 1]")
# Convert data to numpy arrays and normalize images to the interval [0, 1]
print(len(X_train))
def load_data(self, get_first: bool = False):
"""Load all the unpacked data into numpy arrays.
Args:
get_first (bool): Download and unpack data first.
"""
if get_first:
self.get_data()
dtype = 'balanced' if self.balanced else 'byclass'
idx_paths = [
(
os.path.join(self.data_path, 'gzip',
f'emnist-{dtype}-train-images-idx3-ubyte'),
os.path.join(self.data_path, 'gzip',
f'emnist-{dtype}-train-labels-idx1-ubyte'),
),
(
os.path.join(self.data_path, 'gzip',
f'emnist-{dtype}-test-images-idx3-ubyte'),
os.path.join(self.data_path, 'gzip',
f'emnist-{dtype}-test-labels-idx1-ubyte'),
)
]
mapping = []
with open(os.path.join(self.data_path, 'gzip',
f'emnist-{dtype}-mapping.txt'),
mode='r') as lm:
for line in lm:
mapping.append(chr(int(line.split()[1])))
mndata = MNIST(os.path.join(self.data_path, 'gzip'))
mndata.gz = True
print("\n[~] Loading dataset...")
x_train, y_train = mndata.load(idx_paths[0][0], idx_paths[0][1])
x_test, y_test = mndata.load(idx_paths[1][0], idx_paths[1][1])
x_train = np.array(x_train)
y_train = np.array(y_train)
x_test = np.array(x_test)
y_test = np.array(y_test)
# Normalize an array with data in an interval of [0, 255] to [0, 1]
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
# Reshape the image to be used in a CNN
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
# Perform one-hot encoding on label array
y_train = tf.keras.utils.to_categorical(y_train, len(mapping))
y_test = tf.keras.utils.to_categorical(y_test, len(mapping))
print("[!] Dataset loaded!")
return x_train, y_train, x_test, y_test, mapping
