def average_data_set(image_batch_file, sample_approx_n_images):
image_batch_file = open(data_path("raw", image_batch_file), "rb")
image_file_header = read_image_file_header(image_batch_file)
single_image_size = image_file_header["num_rows"] * image_file_header["num_columns"]
total_images = image_file_header["num_items"]
keep_chance = max(min(float(sample_approx_n_images) / float(total_images), 1.0), 0.0)
images_to_avg = []
for i in xrange(total_images):
image_data = image_batch_file.read(single_image_size)
if random.random() <= keep_chance:
image = PIL.Image.frombytes("L", (image_file_header["num_rows"], image_file_header["num_columns"]), image_data)
images_to_avg.append(image)
avg = numpy.zeros((images_to_avg[0].size[0], images_to_avg[0].size[1]), numpy.float)
n = len(images_to_avg)
for image in images_to_avg:
imarr = numpy.array(image, dtype=numpy.float)
avg += imarr / n
return numpy.average(avg)
def create_lmdb(lmdb_filename, image_batch_file, label_batch_file):
image_batch_file = open(data_path("raw", image_batch_file), "rb")
image_file_header = read_image_file_header(image_batch_file)
single_image_size = image_file_header["num_rows"] * image_file_header["num_columns"]
label_batch_file = open(data_path("raw", label_batch_file), "rb")
label_file_header = read_label_file_header(label_batch_file)
num_images = image_file_header["num_items"]
# We need to prepare the database for the size. We'll set it 10 times
# greater than what we theoretically need. There is little drawback to
# setting this too big. If you still run into problem after raising
# this, you might want to try saving fewer entries in a single
# transaction.
map_size = single_image_size * num_images * 10
env = lmdb.open(data_path("processed", lmdb_filename), map_size=map_size)
size_x = image_file_header["num_columns"]
size_y = image_file_header["num_rows"]
with env.begin(write=True) as txn:
# txn is a Transaction object
for i in xrange(num_images):
print (i + 1), "/", num_images
image_data = image_batch_file.read(single_image_size)
label = ord(label_batch_file.read(1))
datum = caffe.proto.caffe_pb2.Datum()
datum.channels = 1
datum.height = size_y
datum.width = size_x
datum.data = image_data
datum.label = label
str_id = '{:08}'.format(i)
# The encode is only essential in Python 3
txn.put(str_id.encode('ascii'), datum.SerializeToString())
def convert_to_image_files(image_batch_file, label_batch_file, skip_first_n_images=0, save_n_images=100, step=1):
image_batch_file = open(data_path("raw", image_batch_file), "rb")
image_file_header = read_image_file_header(image_batch_file)
single_image_size = image_file_header["num_rows"] * image_file_header["num_columns"]
label_batch_file = open(data_path("raw", label_batch_file), "rb")
label_file_header = read_label_file_header(label_batch_file)
image_batch_file.seek(skip_first_n_images * single_image_size, 1)
label_batch_file.seek(skip_first_n_images, 1)
mkdir_recursive(temp_path("vis"))
for i in xrange(save_n_images):
image_data = image_batch_file.read(single_image_size)
label = ord(label_batch_file.read(1))
image = PIL.Image.frombytes("L", (image_file_header["num_rows"], image_file_header["num_columns"]), image_data)
# image = PIL.ImageOps.invert(image)
if i % step == 0:
image.save(
open(temp_path("vis", "image{}-label{}.png".format(i, label)), "wb"),
"png"
)
def main():
# train()
show_saved_net_accuracy()
test_on_image_directory(data_path("my_samples"))
from util.paths import data_path
DEBUG_CHECKS = False
BATCH_SIZE = 64
TEST_BATCH_SIZE = 1000
EPOCHS = 3
#WEIGHT_DECAY = 0.005
BASE_LEARNING_RATE = 0.01
LEARNING_RATE_DECAY_PER_EPOCH = 0.01
LEARNING_MOMENTUM = 0.5
LOG_INTERVAL = 10
SCRIPT_PATH = os.path.dirname(__file__)
DATA_PATH = data_path()
MODEL_FILE = os.path.join(SCRIPT_PATH, "net.pt")
class SimpleConvNet(nn.Module):
def __init__(self, debug=False):
super(SimpleConvNet, self).__init__()
self.debug = debug
self.layers = OrderedDict([
("conv1", nn.Conv2d(in_channels=1, out_channels=4, kernel_size=3)),
("conv1-relu", nn.ReLU()),
("conv2", nn.Conv2d(in_channels=4, out_channels=8, kernel_size=3)),
("conv2-relu", nn.ReLU()),
("pool1", nn.MaxPool2d(kernel_size=2)),
("conv3", nn.Conv2d(in_channels=8, out_channels=16,
kernel_size=3)),
# print image.shape, transformed_image.shape
# print transformed_image
# break
# copy the image data into the memory allocated for the net
net.blobs['data'].data[...] = transformed_image
### perform classification
output = net.forward()
# print output
cur_predicted = output["fc8"][0].argmax()
print os.path.basename(file_path) + " - " + str(cur_predicted)
total += 1
if cur_correct == cur_predicted:
correct += 1
# output_prob = output['fc8'][0] # the output probability vector for the first image in the batch
# print output
# print 'predicted class is:', output_prob.argmax() + 1
print "Identified {} of {}, {:.2%}".format(correct, total,
float(correct) / float(total))
if __name__ == "__main__":
# test_path(temp_path("vis"))
test_path(os.path.join(data_path("my_samples")))
# print LABEL_REGEX.findall("/Users/eranshirazi/Home/Dev/cnn-classify-digits/util/../temp/vis/image8900-label9.png")