def load_english_hand(base_dir,
samples=55,
width=20,
retain=0.8,
classes=62,
thresh=None,
colors=256):
"""
Load english handwritten characters (using pc tablet) from
http://www.ee.surrey.ac.uk/CVSSP/demos/chars74k/
Samples (55 of each; total 3410 samples)
0-10:
Numbers 0-9
11-36:
Uppercase
37-62:
Lowercase
Args:
base_dir (str): Directory containing sample data.
samples (Optional[int]): Number of samples for each class.
width (Optional[int]): Width and height of each sample.
Defaults to 20.
retain (Optional[float]): Percentage of sample data to retain as
training data. The rest is used as test data. Defaults to 0.8.
classes (Optional[int]): Number of classes. Defaults to 62.
thresh (Optional[int]): Background threshold.
colors (Optional[int]): Number of colors. Defaults to 256.
Returns:
numpy.ndarray: Training data.
numpy.ndarray: Training labels.
numpy.ndarray: Test data.
numpy.ndarray: Test labels.
"""
import time
start = time.time()
X = [None] * samples * classes
y = [c for c in ALPHA_NUMERIC for x in xrange(samples)]
default_thresh = thresh
with open(os.path.join(base_dir, "all.txt~"), "r") as samples:
for i, sample in enumerate(samples):
sample = sample.strip()
filename = os.path.join(base_dir, sample)
img = cv2.imread(filename, 0)
if default_thresh is None:
thresh = default_background_threshold(img)
else:
thresh = default_thresh
vec = transform(img, width, width, thresh)
X[i] = vec.astype(np.float32) / colors
LOGGER.debug("Loaded: {}".format(i))
LOGGER.info("Loading training set: {} seconds".format(time.time() - start))
return split_data(X, y, retain, classes)
def load_english_hand(base_dir, samples=55, width=20, retain=0.8,
classes=62, thresh=None, colors=256):
"""
Load english handwritten characters (using pc tablet) from
http://www.ee.surrey.ac.uk/CVSSP/demos/chars74k/
Samples (55 of each; total 3410 samples)
0-10:
Numbers 0-9
11-36:
Uppercase
37-62:
Lowercase
Args:
base_dir (str): Directory containing sample data.
samples (Optional[int]): Number of samples for each class.
width (Optional[int]): Width and height of each sample.
Defaults to 20.
retain (Optional[float]): Percentage of sample data to retain as
training data. The rest is used as test data. Defaults to 0.8.
classes (Optional[int]): Number of classes. Defaults to 62.
thresh (Optional[int]): Background threshold.
colors (Optional[int]): Number of colors. Defaults to 256.
Returns:
numpy.ndarray: Training data.
numpy.ndarray: Training labels.
numpy.ndarray: Test data.
numpy.ndarray: Test labels.
"""
import time
start = time.time()
X = [None] * samples * classes
y = [c for c in ALPHA_NUMERIC for x in xrange(samples)]
default_thresh = thresh
with open(os.path.join(base_dir, "all.txt~"), "r") as samples:
for i, sample in enumerate(samples):
sample = sample.strip()
filename = os.path.join(base_dir, sample)
img = cv2.imread(filename, 0)
if default_thresh is None:
thresh = default_background_threshold(img)
else:
thresh = default_thresh
vec = transform(img, width, width, thresh)
X[i] = vec.astype(np.float32) / colors
LOGGER.debug("Loaded: {}".format(i))
LOGGER.info("Loading training set: {} seconds".format(time.time() - start))
return split_data(X, y, retain, classes)
def load_digits(filename,
rows=50,
cols=100,
width=20,
colors=256,
classes=10,
retain=0.8):
"""
Load training data from file.
The file contains the mnist digits, but together in one collage that must
be split apart. This image was obtained from the opencv sample image data.
Args:
filename (str): File containing mnist digits.
rows (Optional[int]): Number of rows in file.
cols (Optional[int]): Number of cols in file.
width (Optional[int]): Width and height of each sample.
Defaults to 20.
colors (Optional[int]): Number of colors. Defaults to 256.
classes (Optional[int]): Number of classes. Defaults to 10.
retain (Optional[float]): Percentage of sample data to retain as
training data. The rest is used as test data. Defaults to 0.8.
Returns:
numpy.ndarray: Training data.
numpy.ndarray: Training labels.
numpy.ndarray: Test data.
numpy.ndarray: Test labels.
"""
gray = cv2.imread(filename, 0)
# Now we split the image to 5000 cells, each 20x20 size
cells = [np.hsplit(row, cols) for row in np.vsplit(gray, rows)]
# Make it into a Numpy array. It size will be (50,100,20,20)
x = np.array(cells)
# Training data
X = [
np.reshape(x[y][x_], (width**2, )).astype(np.float32) / colors
for x_ in xrange(cols) for y in xrange(rows)
]
# Expected
y = [y for y in xrange(classes) for x_ in xrange(len(X) / classes)]
assert len(X) == len(y)
return split_data(X, y, retain, classes)
def load_handwritten(root, width, height, classes=62, thresh=None, retain=0.8):
"""
Load typed training data.
Args:
root (str): Root directory containing training data.
width (int): Desired width of each sample.
height (int): Desired height of each sample.
classes (Optional[int]): Number of classes. Defaults to 62.
thresh (Optional[int]): Background image threshold. Defaults to None.
retain (Optional[float]): Percentage of sample data to retain as
training data. The rest is used as test data. Defaults to 0.8.
Returns:
numpy.ndarray: Training data.
numpy.ndarray: Training labels.
numpy.ndarray: Test data.
numpy.ndarray: Test labels.
"""
train = []
labels = []
default_thresh = thresh
for dirname in os.listdir(root):
label = dirname[0]
full_dirname = os.path.join(root, dirname)
vector = []
for filename in os.listdir(full_dirname):
sample = os.path.join(full_dirname, filename)
img = cv2.imread(sample, 0)
if default_thresh is None:
ravel = img.ravel()
avg = np.mean(ravel)
thresh = avg
else:
thresh = default_thresh
vec = transform(img, width, height, thresh)
train.append(vec)
labels.append(label)
#show_img(img)
#img2 = grayscale_to_black_and_white(img, thresh)
#show_img(img2)
#img3 = resize(img2, width, height, thresh)
#show_img(img3)
return split_data(train, labels, retain, classes)
def load_handwritten(root, width, height, classes=62, thresh=None, retain=0.8):
"""
Load typed training data.
Args:
root (str): Root directory containing training data.
width (int): Desired width of each sample.
height (int): Desired height of each sample.
classes (Optional[int]): Number of classes. Defaults to 62.
thresh (Optional[int]): Background image threshold. Defaults to None.
retain (Optional[float]): Percentage of sample data to retain as
training data. The rest is used as test data. Defaults to 0.8.
Returns:
numpy.ndarray: Training data.
numpy.ndarray: Training labels.
numpy.ndarray: Test data.
numpy.ndarray: Test labels.
"""
train = []
labels = []
default_thresh = thresh
for dirname in os.listdir(root):
label = dirname[0]
full_dirname = os.path.join(root, dirname)
vector = []
for filename in os.listdir(full_dirname):
sample = os.path.join(full_dirname, filename)
img = cv2.imread(sample, 0)
if default_thresh is None:
ravel = img.ravel()
avg = np.mean(ravel)
thresh = avg
else:
thresh = default_thresh
vec = transform(img, width, height, thresh)
train.append(vec)
labels.append(label)
#show_img(img)
#img2 = grayscale_to_black_and_white(img, thresh)
#show_img(img2)
#img3 = resize(img2, width, height, thresh)
#show_img(img3)
return split_data(train, labels, retain, classes)
def load_typed(root,
width,
height,
samples,
classes=62,
digits=3,
digits2=5,
thresh=128,
retain=0.8):
"""
Load typed training data.
Args:
root (str): Root directory containing training data.
width (int): Desired width of each sample.
height (int): Desired height of each sample.
samples (int): Number of samples to use for each class.
classes (Optional[int]): Number of classes. Defaults to 62.
digits (Optional[int]): Number of digits in the sample number.
digits2 (Optional[int]): Number of digits in the image number.
thresh (Optional[int]): Background image threshold. Defaults to 128.
retain (Optional[float]): Percentage of sample data to retain as
training data. The rest is used as test data. Defaults to 0.8.
Returns:
numpy.ndarray: Training data.
numpy.ndarray: Training labels.
numpy.ndarray: Test data.
numpy.ndarray: Test labels.
"""
data = []
labels = []
for i in xrange(1, classes + 1):
dirname = os.path.join(root, "Sample" + str(i).zfill(digits))
for j in xrange(1, samples + 1):
filename = os.path.join(
dirname, "img{}-{}.png".format(
str(i).zfill(digits),
str(j).zfill(digits2)))
img = cv2.imread(filename, 0)
vec = transform(img, width, height, thresh)
data.append(vec)
labels.append(ALPHA_NUMERIC[i - 1])
LOGGER.debug("Loaded {}".format(filename))
return split_data(data, labels, retain, classes)
def load_shrinked_imgs(dirname,
width=20,
samples=55,
classes=62,
retain=0.8,
colors=256):
"""
Same as load_english_hand, but for an already shrinked data set that does
not need to be resized.
Args:
dirname (str): Directory containing sample data.
width (Optional[int]): Width and height of each sample.
Defaults to 20.
samples (Optional[int]): Number of samples for each class.
classes (Optional[int]): Number of classes. Defaults to 62.
retain (Optional[float]): Percentage of sample data to retain as
training data. The rest is used as test data. Defaults to 0.8.
colors (Optional[int]): Number of colors. Defaults to 256.
Returns:
numpy.ndarray: Training data.
numpy.ndarray: Training labels.
numpy.ndarray: Test data.
numpy.ndarray: Test labels.
"""
X = [None] * samples * classes
y = [c for c in ALPHA_NUMERIC for x in xrange(samples)]
i = 0
for sample in os.listdir(dirname):
if not sample.startswith("Sample"):
continue
dirpath = os.path.join(dirname, sample)
for j, imgname in enumerate(os.listdir(dirpath)):
if j >= samples:
break
filepath = os.path.join(dirpath, imgname)
x = cv2.imread(filepath, 0)
X[i] = np.array(np.reshape(x,
(width**2, ))).astype(np.float32) / 256
LOGGER.debug("Loaded {} as {}:".format(filepath, y[i]))
i += 1
return split_data(X, y, retain, classes)
def load_digits(filename, rows=50, cols=100, width=20, colors=256,
classes=10, retain=0.8):
"""
Load training data from file.
The file contains the mnist digits, but together in one collage that must
be split apart. This image was obtained from the opencv sample image data.
Args:
filename (str): File containing mnist digits.
rows (Optional[int]): Number of rows in file.
cols (Optional[int]): Number of cols in file.
width (Optional[int]): Width and height of each sample.
Defaults to 20.
colors (Optional[int]): Number of colors. Defaults to 256.
classes (Optional[int]): Number of classes. Defaults to 10.
retain (Optional[float]): Percentage of sample data to retain as
training data. The rest is used as test data. Defaults to 0.8.
Returns:
numpy.ndarray: Training data.
numpy.ndarray: Training labels.
numpy.ndarray: Test data.
numpy.ndarray: Test labels.
"""
gray = cv2.imread(filename, 0)
# Now we split the image to 5000 cells, each 20x20 size
cells = [np.hsplit(row, cols) for row in np.vsplit(gray, rows)]
# Make it into a Numpy array. It size will be (50,100,20,20)
x = np.array(cells)
# Training data
X = [np.reshape(x[y][x_], (width**2, )).astype(np.float32) / colors
for x_ in xrange(cols) for y in xrange(rows)]
# Expected
y = [y for y in xrange(classes) for x_ in xrange(len(X) / classes)]
assert len(X) == len(y)
return split_data(X, y, retain, classes)
def load_shrinked_imgs(dirname, width=20, samples=55, classes=62, retain=0.8,
colors=256):
"""
Same as load_english_hand, but for an already shrinked data set that does
not need to be resized.
Args:
dirname (str): Directory containing sample data.
width (Optional[int]): Width and height of each sample.
Defaults to 20.
samples (Optional[int]): Number of samples for each class.
classes (Optional[int]): Number of classes. Defaults to 62.
retain (Optional[float]): Percentage of sample data to retain as
training data. The rest is used as test data. Defaults to 0.8.
colors (Optional[int]): Number of colors. Defaults to 256.
Returns:
numpy.ndarray: Training data.
numpy.ndarray: Training labels.
numpy.ndarray: Test data.
numpy.ndarray: Test labels.
"""
X = [None] * samples * classes
y = [c for c in ALPHA_NUMERIC for x in xrange(samples)]
i = 0
for sample in os.listdir(dirname):
if not sample.startswith("Sample"):
continue
dirpath = os.path.join(dirname, sample)
for j, imgname in enumerate(os.listdir(dirpath)):
if j >= samples:
break
filepath = os.path.join(dirpath, imgname)
x = cv2.imread(filepath, 0)
X[i] = np.array(np.reshape(x, (width**2, ))).astype(np.float32) / 256
LOGGER.debug("Loaded {} as {}:".format(filepath, y[i]))
i += 1
return split_data(X, y, retain, classes)
def load_typed(root, width, height, samples, classes=62, digits=3,
digits2=5, thresh=128, retain=0.8):
"""
Load typed training data.
Args:
root (str): Root directory containing training data.
width (int): Desired width of each sample.
height (int): Desired height of each sample.
samples (int): Number of samples to use for each class.
classes (Optional[int]): Number of classes. Defaults to 62.
digits (Optional[int]): Number of digits in the sample number.
digits2 (Optional[int]): Number of digits in the image number.
thresh (Optional[int]): Background image threshold. Defaults to 128.
retain (Optional[float]): Percentage of sample data to retain as
training data. The rest is used as test data. Defaults to 0.8.
Returns:
numpy.ndarray: Training data.
numpy.ndarray: Training labels.
numpy.ndarray: Test data.
numpy.ndarray: Test labels.
"""
data = []
labels = []
for i in xrange(1, classes + 1):
dirname = os.path.join(root, "Sample" + str(i).zfill(digits))
for j in xrange(1, samples + 1):
filename = os.path.join(
dirname,
"img{}-{}.png".format(str(i).zfill(digits), str(j).zfill(digits2)))
img = cv2.imread(filename, 0)
vec = transform(img, width, height, thresh)
data.append(vec)
labels.append(ALPHA_NUMERIC[i-1])
LOGGER.debug("Loaded {}".format(filename))
return split_data(data, labels, retain, classes)