Exemplo n.º 1
0
                costP,_ = self.costAndGrad(data)
                L[i,j] -= epsilon
                numGrad = (costP - cost)/epsilon
                err = np.abs(dL[j][i] - numGrad)
                err2+=err
                count+=1

        if 1e-5 > err2 / count:
            print "Grad Check Passed for dL"
        else:
            print "Grad Check Failed for dL: Sum of Error = %.9f" % (err2/count)

if __name__ == '__main__':
    import tree as treeM
    train = treeM.load_trees()
    numW = len(treeM.load_word_to_index_map())

    wvecDim = 4
    outputDim = 42

    rnn = RNN(wvecDim, outputDim, numW, mbSize=4)
    rnn.initParams()

    mbData = train[:4]
    
    print "Numerical gradient check..."
    rnn.check_grad(mbData)



Exemplo n.º 2
0
def run(args=None):
    usage = "usage : %prog [options]"
    parser = optparse.OptionParser(usage=usage)

    parser.add_option("--test",action="store_true",dest="test",default=False)

    # Optimizer
    parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
    parser.add_option("--optimizer",dest="optimizer",type="string",
        default="adagrad")
    parser.add_option("--epochs",dest="epochs",type="int",default=50)
    parser.add_option("--step",dest="step",type="float",default=1e-2)


    parser.add_option("--middleDim",dest="middleDim",type="int",default=10)
    parser.add_option("--outputDim",dest="outputDim",type="int",default=5)
    parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)

    
    parser.add_option("--outFile",dest="outFile",type="string",
        default="models/test.bin")
    parser.add_option("--inFile",dest="inFile",type="string",
        default="models/test.bin")
    parser.add_option("--data",dest="data",type="string",default="train")

    parser.add_option("--model",dest="model",type="string",default="RNN")

    (opts, args) = parser.parse_args(args)


    # make this false if you dont care about your accuracies per epoch, makes things faster!
    evaluate_accuracy_while_training = True

    # Testing
    if opts.test:
        test(opts.inFile, opts.data, opts.model)
        return
    
    print "Loading data..."
    train_accuracies = []
    dev_accuracies = []

    # load training data
    trees = tr.load_trees(TRAIN_DATA_FILE)
    opts.numWords = len(tr.load_word_to_index_map())

    if (opts.model=='RNTN'):
        nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
    elif(opts.model=='RNN'):
        nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
    elif(opts.model=='RNN2'):
        nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
    else:
        raise '%s is not a valid neural network so far only RNTN, RNN, RNN2' % opts.model
    
    nn.initParams()

    sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer)

    dev_trees = tr.load_trees(DEV_DATA_FILE)
    for e in range(opts.epochs):
        start = time.time()
        print "Running epoch %d" % e
        sgd.run(trees)
        end = time.time()
        print "Time per epoch : %f" % (end-start)

        # save the net to the output file
        #f = open(opts.outFile, 'wb')
        #pickle.dump(opts, f, -1)
        #pickle.dump(sgd.costt, f, -1)
        #pickle.dump(nn.stack, f, -1)
        #np.save(f, nn.stack)
        #f.close()
        joblib.dump(opts, opts.outFile + "_opts")
        joblib.dump(sgd.costt, opts.outFile + "_cost")
        joblib.dump(nn.stack, opts.outFile + "_stack")

        if evaluate_accuracy_while_training:
            print "testing on training set..."
            train_accuracies.append(test(opts.outFile, "train", opts.model, trees))
            
            print "testing on dev set..."
            dev_accuracies.append(test(opts.outFile, "dev", opts.model, dev_trees))
            
            # clear the fprop flags in trees and dev_trees
            for tree in trees:
                tr.traverse(tree.root, func=tr.clear_fprop)
            for tree in dev_trees:
                tr.traverse(tree.root, func=tr.clear_fprop)
            print "fprop in trees cleared"


    if False: # don't do this for now
    #if evaluate_accuracy_while_training:
        #print train_accuracies
        #print dev_accuracies
        # Plot train/dev_accuracies here
        x = range(opts.epochs)
        figure(figsize=(6,4))
        plot(x, train_accuracies, color='b', marker='o', linestyle='-', label="training")
        plot(x, dev_accuracies, color='g', marker='o', linestyle='-', label="dev")
        title("Accuracy vs num epochs.")
        xlabel("Epochs")
        ylabel("Accuracy")
        #ylim(ymin=0, ymax=max(1.1*max(train_accuracies),3*min(train_accuracies)))
        legend()
        savefig("train_dev_acc.png")
Exemplo n.º 3
0
                L[i, j] += epsilon
                costP, _ = self.costAndGrad(data)
                L[i, j] -= epsilon
                numGrad = (costP - cost) / epsilon
                err = np.abs(dL[j][i] - numGrad)
                err2 += err
                count += 1

        if 1e-5 > err2 / count:
            print "Grad Check Passed for dL"
        else:
            print "Grad Check Failed for dL: Sum of Error = %.9f" % (err2 /
                                                                     count)


if __name__ == '__main__':
    import tree as treeM
    train = treeM.load_trees()
    numW = len(treeM.load_word_to_index_map())

    wvecDim = 4
    outputDim = 42

    rnn = RNN(wvecDim, outputDim, numW, mbSize=4)
    rnn.initParams()

    mbData = train[:4]

    print "Numerical gradient check..."
    rnn.check_grad(mbData)