#if update: # stop = raw_input("Loading from folder "+folder+" : Hit enter to proceed or ctrl+C to cancel") #else: # print "Initializing in folder "+folder """Load the train/test split information if update, else split and write out which images are in which dataset""" trainFs, testFs = helperFuncs.getTrainTestSplit(False, folder, numEx, trainTestSplit, ld) trainL = len(trainFs) testL = len(testFs) print "number of examples: ", numEx print "training examples : ", trainL print "test examples : ", testL features, labels = helperFuncs.getTargets( "justbonds") #get the target vector for each CID outsize = helperFuncs.getOutSize(features) """DEFINE THE MODEL HERE""" model = Sequential() model.add(Convolution2D(8, 8, 8, input_shape=(1, size, size))) model.add(Activation('relu')) model.add(Convolution2D(8, 5, 5)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(3, 3))) model.add(Dropout(0.25)) model.add(Convolution2D(8, 5, 5))
ydim = image.shape[0] - wSize + 1 xdim = image.shape[1] - wSize + 1 output = [] xran = np.linspace(0,xdim,(xdim/stride)) yran = np.linspace(0,ydim,(ydim/stride)) for y in yran: for x in xran: output.append(canvas[y:y+wSize,x:x+wSize]) return np.reshape(np.array(output),(len(output),1,wSize,wSize)) #Process the arguments given (see helperFuncs.py for details) size, indir, binarize, blur, padding, targetType = helperFuncs.dataProcessorArgs(sys.argv[1:]) targets, labels = helperFuncs.getTargets(targetType) outsize = helperFuncs.getOutSize(targets) #wait until the folder exists while not isdir(indir+"tempTrain/"): time.sleep(10.) print "I'm sleeping", isdir(indir), indir, " \r", if not isdir(indir+"tempTrainNP/"): mkdir(indir+"tempTrainNP/") mkdir(indir+"tempTestNP/") #define folders trainFolder = indir+"tempTrain/" trainNPfolder = indir+"tempTrainNP/"
"""Define the folder where the model will be stored based on the input arguments""" folder = helperFuncs.defineFolder(True,outType,size,run) print folder trainDirect = folder+"tempTrain/" trainNP = folder+"tempTrainNP/" testDirect = folder+"tempTest/" testNP = folder+"tempTestNP/" """Load the train/test split information""" trainFs, testFs = helperFuncs.getTrainTestSplit(True,folder) trainL = len(trainFs) testL = len(testFs) features,labels = helperFuncs.getTargets(outType) #get the OCR vector for each CID outsize = len(features[features.keys()[0]]) #this it the size of the target (# of OCRfeatures) means,stds = helperFuncs.getMeansStds(features) """load model""" model = helperFuncs.loadModel(folder+"wholeModel") while not isfile(testNP+"Xtest.h5"): print "sleeping because Test folder empty \r", time.sleep(1.) print "" print "Loading np test arrays"
#if update: # stop = raw_input("Loading from folder "+folder+" : Hit enter to proceed or ctrl+C to cancel") #else: # print "Initializing in folder "+folder """Load the train/test split information if update, else split and write out which images are in which dataset""" trainFs, testFs = helperFuncs.getTrainTestSplit(False, folder, numEx, trainTestSplit, ld) trainL = len(trainFs) testL = len(testFs) print "number of examples: ", numEx print "training examples : ", trainL print "test examples : ", testL features, labels = helperFuncs.getTargets( "ocr") #get the target vector for each CID outsize = len(features[ features.keys()[0]]) #this it the size of the target (# of OCRfeatures) """DEFINE THE MODEL HERE""" model = Sequential() model.add(Convolution2D(16, 8, 8, input_shape=(1, size, size))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Convolution2D(32, 5, 5)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2)))
"""Load the train/test split information if update, else split and write out which images are in which dataset""" trainFs, testFs = helperFuncs.getTrainTestSplit(False,folder,numEx,trainTestSplit,ld) trainL = len(trainFs) testL = len(testFs) print "number of examples: ", numEx print "training examples : ", trainL print "test examples : ", testL features,labels = helperFuncs.getTargets("ocr") #get the target vector for each CID outsize = len(features[features.keys()[0]]) #this it the size of the target (# of OCRfeatures) """DEFINE THE MODEL HERE""" model = Sequential() model.add(Convolution2D(8, 1, 5, 5, border_mode='full')) model.add(Activation('relu')) model.add(MaxPooling2D(poolsize=(2,2))) lastLayerSize = getLastSize(model) print lastLayerSize print model.layers[-1] model.add(BatchNormalization())
from os import mkdir from os.path import isfile from random import shuffle import sys import time import numpy as np import subprocess import cPickle import helperFuncs import h5py """Processes images and pickles numpy arrays of them for training/testing""" size, indir, binarize, blur, padding, targetType = helperFuncs.dataProcessorArgs(sys.argv[1:]) targets, labels = helperFuncs.getTargets(targetType) outsize = len(targets[targets.keys()[0]]) while not isdir(indir+"tempTrain/"): time.sleep(10.) print "I'm sleeping", isdir(indir), indir, " \r", if not isdir(indir+"tempTrainNP/"): mkdir(indir+"tempTrainNP/") mkdir(indir+"tempTestNP/") #print "reading Train/Test files" #train = [x for x in file(indir+"traindata.csv").read().split("\n") if x != ''] #test = [x for x in file(indir+"testdata.csv").read().split("\n") if x != ''] trainFolder = indir+"tempTrain/"
"""Define the folder where the model will be stored based on the input arguments""" folder = helperFuncs.defineFolder(True,outType,size,run) print folder trainDirect = folder+"tempTrain/" trainNP = folder+"tempTrainNP/" testDirect = folder+"tempTest/" testNP = folder+"tempTestNP/" """Load the train/test split information""" trainFs, testFs = helperFuncs.getTrainTestSplit(True,folder) trainL = len(trainFs) testL = len(testFs) features,labels = helperFuncs.getTargets(outType) #get the OCR vector for each CID outsize = len(features[features.keys()[0]]) #this it the size of the target (# of OCRfeatures) means,stds = helperFuncs.getMeansStds() """load model""" #with open(folder+"bestModel.pickle",'rb') as f: # model = cPickle.load(f) model = helperFuncs.loadModel(folder+"wholeModel") while not isfile(testNP+"Xtest.h5"): print "sleeping because Test folder empty \r", time.sleep(1.) print ""