Ejemplo n.º 1
0
    def build_feed_dict(self, in_node, is_training=True):
        nodes_list = []
        tr.leftTraverse(in_node, lambda node, args: args.append(node), nodes_list)

        if is_training:
            feed_dict = {
              self.is_a_leaf   : [ n.isLeaf for n in nodes_list ],
              self.left_child : [ nodes_list.index(n.left) if not n.isLeaf else -1 for n in nodes_list ],
              self.right_child : [ nodes_list.index(n.right) if not n.isLeaf else -1 for n in nodes_list ],
              self.word_index  : [ self.vocab.encode(n.word) if n.word else -1 for n in nodes_list ],
              self.labelholder : [ n.label for n in nodes_list ],
              self.learning_rate : self.config.lr,
              self.l2_reg : self.config.l2
            }
        else:
            feed_dict = {
              self.is_a_leaf   : [ n.isLeaf for n in nodes_list ],
              self.left_child : [ nodes_list.index(n.left) if not n.isLeaf else -1 for n in nodes_list ],
              self.right_child : [ nodes_list.index(n.right) if not n.isLeaf else -1 for n in nodes_list ],
              self.word_index  : [ self.vocab.encode(n.word) if n.word else -1 for n in nodes_list ],
              self.labelholder : [ n.label for n in nodes_list ],
              self.learning_rate : self.config.lr,
              self.l2_reg : 0.
            }
        return feed_dict
Ejemplo n.º 2
0
  def build_feed_dict(self, batch_data):

    is_leaf = []
    left_children = []
    right_children = []
    node_word_indices = []
    labels = []
    tree_size = []

    for _, atree in enumerate(batch_data):

      nodes_list = []
      tree.leftTraverse(atree.root, lambda node, args: args.append(node), nodes_list)
      node_to_index = OrderedDict()
      for i in range(len(nodes_list)):
        node_to_index[nodes_list[i]] = i

      nodes_list += (self.config.max_tree_nodes-len(nodes_list))*[None]

      is_leaf.append([False if node is None else node.isLeaf for node in nodes_list])
      left_children.append([-1 if node is None or node.isLeaf else node_to_index[node.left]  for node in nodes_list])
      right_children.append([-1 if node is None or node.isLeaf else node_to_index[node.right]  for node in nodes_list])
      node_word_indices.append([-1 if node is None or not node.word else self.vocab.encode(node.word) for node in nodes_list])
      labels.append([-1 if node is None else node.label for node in nodes_list])
      tree_size.append(atree.num_nodes)

    feed_dict = {
        self.is_leaf_placeholder: is_leaf,
        self.left_children_placeholder: left_children, 
        self.right_children_placeholder: right_children,
        self.node_word_indices_placeholder: node_word_indices,
        self.labels_placeholder: labels,
        self.tree_size_placeholder: tree_size
    }
    return feed_dict
Ejemplo n.º 3
0
    def build_feed_dict(self, in_node):
        nodes_list = []
        tr.leftTraverse(in_node, lambda node, args: args.append(node), nodes_list)
        node_to_index = OrderedDict()
        for idx, i in enumerate(nodes_list):
            node_to_index[i] = idx

        feed_dict = {
          self.is_a_leaf   : [ n.isLeaf for n in nodes_list ],
          self.left_child  : [ node_to_index[n.left] if not n.isLeaf else -1 for n in nodes_list ],
          self.right_child : [ node_to_index[n.right] if not n.isLeaf else -1 for n in nodes_list ],
          self.word_index  : [ self.vocab.encode(n.word) if n.word else -1 for n in nodes_list ],
          self.labelholder : [ n.label for n in nodes_list ]
        }
        return feed_dict
    def build_feed_dict(self, in_node, is_training=True):
        nodes_list = []
        tr.leftTraverse(in_node, lambda node, args: args.append(node),
                        nodes_list)

        if is_training:
            feed_dict = {
                self.is_a_leaf: [n.isLeaf for n in nodes_list],
                self.left_child: [
                    nodes_list.index(n.left) if not n.isLeaf else -1
                    for n in nodes_list
                ],
                self.right_child: [
                    nodes_list.index(n.right) if not n.isLeaf else -1
                    for n in nodes_list
                ],
                self.word_index: [
                    self.vocab.encode(n.word) if n.word else -1
                    for n in nodes_list
                ],
                self.labelholder: [n.label for n in nodes_list],
                self.learning_rate:
                self.config.lr,
                self.l2_reg:
                self.config.l2
            }
        else:
            feed_dict = {
                self.is_a_leaf: [n.isLeaf for n in nodes_list],
                self.left_child: [
                    nodes_list.index(n.left) if not n.isLeaf else -1
                    for n in nodes_list
                ],
                self.right_child: [
                    nodes_list.index(n.right) if not n.isLeaf else -1
                    for n in nodes_list
                ],
                self.word_index: [
                    self.vocab.encode(n.word) if n.word else -1
                    for n in nodes_list
                ],
                self.labelholder: [n.label for n in nodes_list],
                self.learning_rate:
                self.config.lr,
                self.l2_reg:
                0.
            }
        return feed_dict
 def build_feed_dict(self, node):
   nodes_list = []
   tree.leftTraverse(node, lambda node, args: args.append(node), nodes_list)
   node_to_index = OrderedDict()
   for i in range(len(nodes_list)):
     node_to_index[nodes_list[i]] = i
   feed_dict = {
       self.is_leaf_placeholder: [node.isLeaf for node in nodes_list],
       self.left_children_placeholder: [node_to_index[node.left] if
                                        not node.isLeaf else -1
                                        for node in nodes_list],
       self.right_children_placeholder: [node_to_index[node.right] if
                                         not node.isLeaf else -1
                                         for node in nodes_list],
       self.node_word_indices_placeholder: [self.vocab.encode(node.word) if
                                            node.word else -1
                                            for node in nodes_list],
       self.labels_placeholder: [node.label for node in nodes_list]
   }
   return feed_dict
Ejemplo n.º 6
0
    def build_feed_dict(self, trees):
        batch_node_lists = []
        for tree_instance in trees:
            tree_root = tree_instance.root
            nodes_list = []
            tree.leftTraverse(tree_root,
                              lambda tree_root, args: args.append(tree_root),
                              nodes_list)
            batch_node_lists.extend(nodes_list)

        node_to_index = OrderedDict()
        num_nodes = len(batch_node_lists)
        for i in xrange(num_nodes):
            node_to_index[batch_node_lists[i]] = i

        feed_dict = {
            self.node_level_placeholder:
            [node.level for node in batch_node_lists],
            self.root_indeces_placeholder:
            [node_to_index[node] for node in batch_node_lists if node.isRoot],
            self.node_word_indices_placeholder: [
                self.vocab.encode(node.word) if node.word else -1
                for node in batch_node_lists
            ],
            self.left_children_placeholder: [
                node_to_index[node.left] if node.left else -1
                for node in batch_node_lists
            ],
            self.right_children_placeholder: [
                node_to_index[node.right] if node.right else -1
                for node in batch_node_lists
            ],
            self.labels_placeholder: [node.label for node in batch_node_lists],
            self.number_of_examples_placeholder:
            len(trees)
        }
        return feed_dict
Ejemplo n.º 7
0
def run(args=None):
    usage = "usage : %prog [options]"
    parser = optparse.OptionParser(usage=usage)

    parser.add_option("--test",
                      action="store_true",
                      dest="test",
                      default=False)

    # Optimizer
    parser.add_option("--minibatch", dest="minibatch", type="int", default=30)
    parser.add_option("--optimizer",
                      dest="optimizer",
                      type="string",
                      default="adagrad")
    parser.add_option("--epochs", dest="epochs", type="int", default=50)
    parser.add_option("--step", dest="step", type="float", default=1e-2)

    parser.add_option("--middleDim", dest="middleDim", type="int", default=10)
    parser.add_option("--outputDim", dest="outputDim", type="int", default=5)
    parser.add_option("--wvecDim", dest="wvecDim", type="int", default=30)

    # for DCNN only
    parser.add_option("--ktop", dest="ktop", type="int", default=5)
    parser.add_option("--m1", dest="m1", type="int", default=10)
    parser.add_option("--m2", dest="m2", type="int", default=7)
    parser.add_option("--n1", dest="n1", type="int", default=6)
    parser.add_option("--n2", dest="n2", type="int", default=12)

    parser.add_option("--outFile",
                      dest="outFile",
                      type="string",
                      default="models/test.bin")
    parser.add_option("--inFile",
                      dest="inFile",
                      type="string",
                      default="models/test.bin")
    parser.add_option("--data", dest="data", type="string", default="train")

    parser.add_option("--model", dest="model", type="string", default="RNN")

    (opts, args) = parser.parse_args(args)

    # make this false if you dont care about your accuracies per epoch, makes things faster!
    evaluate_accuracy_while_training = True

    # Testing
    if opts.test:
        test(opts.inFile, opts.data, opts.model)
        return

    print "Loading data..."
    train_accuracies = []
    dev_accuracies = []
    # load training data
    trees = tr.loadTrees('train')
    opts.numWords = len(tr.loadWordMap())

    if (opts.model == 'RNTN'):
        nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch)
    elif (opts.model == 'RNN'):
        nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch)
    elif (opts.model == 'RNN2'):
        nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords,
                  opts.minibatch)
    elif (opts.model == 'RNN3'):
        nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords,
                  opts.minibatch)
    elif (opts.model == 'DCNN'):
        nn = DCNN(opts.wvecDim,
                  opts.ktop,
                  opts.m1,
                  opts.m2,
                  opts.n1,
                  opts.n2,
                  0,
                  opts.outputDim,
                  opts.numWords,
                  2,
                  opts.minibatch,
                  rho=1e-4)
        trees = cnn.tree2matrix(trees)
    else:
        raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model

    nn.initParams()

    sgd = optimizer.SGD(nn,
                        alpha=opts.step,
                        minibatch=opts.minibatch,
                        optimizer=opts.optimizer)

    dev_trees = tr.loadTrees("dev")
    for e in range(opts.epochs):
        start = time.time()
        print "Running epoch %d" % e
        sgd.run(trees)
        end = time.time()
        print "Time per epoch : %f" % (end - start)

        with open(opts.outFile, 'w') as fid:
            pickle.dump(opts, fid)
            pickle.dump(sgd.costt, fid)
            nn.toFile(fid)
        if evaluate_accuracy_while_training:
            print "testing on training set real quick"
            train_accuracies.append(
                test(opts.outFile, "train", opts.model, trees))
            print "testing on dev set real quick"
            dev_accuracies.append(
                test(opts.outFile, "dev", opts.model, dev_trees))
            # clear the fprop flags in trees and dev_trees
            for tree in trees:
                tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
            for tree in dev_trees:
                tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
            print "fprop in trees cleared"

    if evaluate_accuracy_while_training:
        pdb.set_trace()
        print train_accuracies
        print dev_accuracies
Ejemplo n.º 8
0
def run( args = None):
    usage = "usage : %prog [options]"
    parser = optparse.OptionParser(usage=usage)

    parser.add_option("--test",action="store_true",dest="test",default=False)

    # Optimizer
    parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
    parser.add_option("--optimizer",dest="optimizer",type="string",default="adagrad")
    parser.add_option("--epochs",dest="epochs",type="int",default=50)
    parser.add_option("--step",dest="step",type="float",default=1e-2)


    parser.add_option("--middleDim",dest="middleDim",type="int",default=10)
    parser.add_option("--outputDim",dest="outputDim",type="int",default=5)
    parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)

    # for DCNN only
    parser.add_option("--ktop",dest="ktop",type="int",default=5)
    parser.add_option("--m1",dest="m1",type="int",default=10)
    parser.add_option("--m2",dest="m2",type="int",default=7)
    parser.add_option("--n1",dest="n1",type="int",default=6)
    parser.add_option("--n2",dest="n2",type="int",default=12)
    
    parser.add_option("--outFile",dest="outFile",type="string",default="models/test.bin")
    parser.add_option("--inFile",dest="inFile",type="string",default="models/test.bin")
    parser.add_option("--data",dest="data",type="string",default="train")

    parser.add_option("--model",dest="model",type="string",default="RNN")

    (opts,args)=parser.parse_args(args)


    # make this false if you dont care about your accuracies per epoch, makes things faster!
    evaluate_accuracy_while_training = True

    # Testing
    if opts.test:
        test(opts.inFile, opts.data, opts.model)
        return
    
    print "Loading data..."
    train_accuracies = []
    dev_accuracies = []
    # load training data
    trees = tr.loadTrees('train')
    opts.numWords = len(tr.loadWordMap())

    if (opts.model=='RNTN'):
        nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
    elif(opts.model=='RNN'):
        nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
    elif(opts.model=='RNN2'):
        nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
    elif(opts.model=='RNN3'):
        nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
    elif(opts.model=='DCNN'):
        nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4)
        trees = cnn.tree2matrix(trees)
    else:
        raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model
    
    nn.initParams()

    sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer)


    dev_trees = tr.loadTrees("dev")
    for e in range(opts.epochs):
        start = time.time()
        print "Running epoch %d"%e
        sgd.run(trees)
        end = time.time()
        print "Time per epoch : %f" %(end-start)

        with open(opts.outFile,'w') as fid:
            pickle.dump(opts,fid)
            pickle.dump(sgd.costt,fid)
            nn.toFile(fid)
        if evaluate_accuracy_while_training:
            print "testing on training set real quick"
            train_accuracies.append(test(opts.outFile,"train",opts.model,trees))
            print "testing on dev set real quick"
            dev_accuracies.append(test(opts.outFile,"dev",opts.model,dev_trees))
            # clear the fprop flags in trees and dev_trees
            for tree in trees:
                tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
            for tree in dev_trees:
                tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
            print "fprop in trees cleared"


    if evaluate_accuracy_while_training:
        pdb.set_trace()
        print train_accuracies
        print dev_accuracies
Ejemplo n.º 9
0
def run(args=None):
    usage = "usage : %prog [options]"
    parser = optparse.OptionParser(usage=usage)

    parser.add_option("--test",
                      action="store_true",
                      dest="test",
                      default=False)

    # Optimizer
    parser.add_option("--minibatch", dest="minibatch", type="int", default=30)
    parser.add_option("--optimizer",
                      dest="optimizer",
                      type="string",
                      default="adagrad")
    parser.add_option("--epochs", dest="epochs", type="int", default=50)
    parser.add_option("--step", dest="step", type="float", default=1e-2)

    parser.add_option("--middleDim", dest="middleDim", type="int", default=10)
    parser.add_option("--outputDim", dest="outputDim", type="int", default=5)
    parser.add_option("--wvecDim", dest="wvecDim", type="int", default=30)

    # By @tiagokv, just to ease the first assignment test
    parser.add_option("--wvecDimBatch",
                      dest="wvecDimBatch",
                      type="string",
                      default="")

    # for DCNN only
    parser.add_option("--ktop", dest="ktop", type="int", default=5)
    parser.add_option("--m1", dest="m1", type="int", default=10)
    parser.add_option("--m2", dest="m2", type="int", default=7)
    parser.add_option("--n1", dest="n1", type="int", default=6)
    parser.add_option("--n2", dest="n2", type="int", default=12)

    parser.add_option("--outFile",
                      dest="outFile",
                      type="string",
                      default="models/test.bin")
    parser.add_option("--inFile",
                      dest="inFile",
                      type="string",
                      default="models/test.bin")
    parser.add_option("--data", dest="data", type="string", default="train")

    parser.add_option("--model", dest="model", type="string", default="RNN")

    (opts, args) = parser.parse_args(args)

    # make this false if you dont care about your accuracies per epoch, makes things faster!
    evaluate_accuracy_while_training = True

    # Testing
    if opts.test:
        test(opts.inFile, opts.data, opts.model)
        return

    print "Loading data..."
    train_accuracies = []
    dev_accuracies = []
    # load training data
    trees = tr.loadTrees('train')
    opts.numWords = len(tr.loadWordMap())

    if (opts.model == 'RNTN'):
        nn = RNTN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch)
    elif (opts.model == 'RNN'):
        nn = RNN(opts.wvecDim, opts.outputDim, opts.numWords, opts.minibatch)
    elif (opts.model == 'RNN2'):
        nn = RNN2(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords,
                  opts.minibatch)
    elif (opts.model == 'RNN3'):
        nn = RNN3(opts.wvecDim, opts.middleDim, opts.outputDim, opts.numWords,
                  opts.minibatch)
    elif (opts.model == 'DCNN'):
        nn = DCNN(opts.wvecDim,
                  opts.ktop,
                  opts.m1,
                  opts.m2,
                  opts.n1,
                  opts.n2,
                  0,
                  opts.outputDim,
                  opts.numWords,
                  2,
                  opts.minibatch,
                  rho=1e-4)
        trees = cnn.tree2matrix(trees)
    else:
        raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN' % opts.model

    nn.initParams()

    sgd = optimizer.SGD(nn,
                        alpha=opts.step,
                        minibatch=opts.minibatch,
                        optimizer=opts.optimizer)

    # assuring folder for plots exists
    if (os.path.isdir('plots') == False): os.makedirs('test')
    if (os.path.isdir('plots/' + opts.model) == False):
        os.makedirs('plots/' + opts.model)

    dev_trees = tr.loadTrees("dev")
    for e in range(opts.epochs):
        start = time.time()
        print "Running epoch %d" % e
        sgd.run(trees)
        end = time.time()
        print "Time per epoch : %f" % (end - start)

        with open(opts.outFile, 'w') as fid:
            pickle.dump(opts, fid)
            pickle.dump(sgd.costt, fid)
            nn.toFile(fid)
        if evaluate_accuracy_while_training:
            print "testing on training set real quick"
            train_accuracies.append(
                test(opts.outFile, "train", opts.model, trees))
            print "testing on dev set real quick"
            dev_accuracies.append(
                test(opts.outFile, "dev", opts.model, dev_trees))
            # clear the fprop flags in trees and dev_trees
            for tree in trees:
                tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
            for tree in dev_trees:
                tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
            print "fprop in trees cleared"

    if evaluate_accuracy_while_training:
        #pdb.set_trace()

        plt.figure()
        #Lets set up the plot
        plt.title('Accuracy in set per epochs')
        plt.plot(range(opts.epochs), train_accuracies, label='train')
        plt.plot(range(opts.epochs), dev_accuracies, label='dev')

        with open('dev_accu' + opts.model, 'a') as fid:
            fid.write(
                str(opts.wvecDim) + ',' + str(opts.middleDim) + ',' +
                str(dev_accuracies[-1]) + ';')

        #plt.axis([0,opts.epochs,0,1])
        plt.xlabel('epochs')
        plt.ylabel('accuracy')
        plt.legend(loc=2, borderaxespad=0.)

        #always save with middleDim, even if it's a one-layer RNN
        plt.savefig('plots/' + opts.model + '/accuracy_wvec_' +
                    str(opts.wvecDim) + '_middleDim_' + str(opts.middleDim) +
                    ' .png')

        print 'image saved at %s' % os.getcwd()
Ejemplo n.º 10
0
def run(args=None):
    usage = "usage : %prog [options]"
    parser = optparse.OptionParser(usage=usage)

    parser.add_option("--test",action="store_true",dest="test",default=False)

    # Optimizer
    parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
    parser.add_option("--optimizer",dest="optimizer",type="string",
        default="adagrad")
    parser.add_option("--epochs",dest="epochs",type="int",default=50)
    parser.add_option("--step",dest="step",type="float",default=1e-2)


    parser.add_option("--middleDim",dest="middleDim",type="int",default=10)
    parser.add_option("--outputDim",dest="outputDim",type="int",default=5)
    parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)

    # By @tiagokv, just to ease the first assignment test
    parser.add_option("--wvecDimBatch",dest="wvecDimBatch",type="string",default="")

    # for DCNN only
    parser.add_option("--ktop",dest="ktop",type="int",default=5)
    parser.add_option("--m1",dest="m1",type="int",default=10)
    parser.add_option("--m2",dest="m2",type="int",default=7)
    parser.add_option("--n1",dest="n1",type="int",default=6)
    parser.add_option("--n2",dest="n2",type="int",default=12)

    parser.add_option("--outFile",dest="outFile",type="string",
        default="models/test.bin")
    parser.add_option("--inFile",dest="inFile",type="string",
        default="models/test.bin")
    parser.add_option("--data",dest="data",type="string",default="train")

    parser.add_option("--model",dest="model",type="string",default="RNN")

    (opts,args)=parser.parse_args(args)


    # make this false if you dont care about your accuracies per epoch, makes things faster!
    evaluate_accuracy_while_training = True

    # Testing
    if opts.test:
        test(opts.inFile,opts.data,opts.model)
        return

    print "Loading data..."
    train_accuracies = []
    dev_accuracies = []
    # load training data
    trees = tr.loadTrees('train')
    opts.numWords = len(tr.loadWordMap())

    if (opts.model=='RNTN'):
        nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
    elif(opts.model=='RNN'):
        nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
    elif(opts.model=='RNN2'):
        nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
    elif(opts.model=='RNN3'):
        nn = RNN3(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
    elif(opts.model=='DCNN'):
        nn = DCNN(opts.wvecDim,opts.ktop,opts.m1,opts.m2, opts.n1, opts.n2,0, opts.outputDim,opts.numWords, 2, opts.minibatch,rho=1e-4)
        trees = cnn.tree2matrix(trees)
    else:
        raise '%s is not a valid neural network so far only RNTN, RNN, RNN2, RNN3, and DCNN'%opts.model

    nn.initParams()

    sgd = optimizer.SGD(nn,alpha=opts.step,minibatch=opts.minibatch,
        optimizer=opts.optimizer)

    # assuring folder for plots exists
    if( os.path.isdir('plots') == False ): os.makedirs('test')
    if( os.path.isdir('plots/' + opts.model ) == False ): os.makedirs('plots/' + opts.model)

    dev_trees = tr.loadTrees("dev")
    for e in range(opts.epochs):
        start = time.time()
        print "Running epoch %d"%e
        sgd.run(trees)
        end = time.time()
        print "Time per epoch : %f"%(end-start)

        with open(opts.outFile,'w') as fid:
            pickle.dump(opts,fid)
            pickle.dump(sgd.costt,fid)
            nn.toFile(fid)
        if evaluate_accuracy_while_training:
            print "testing on training set real quick"
            train_accuracies.append(test(opts.outFile,"train",opts.model,trees))
            print "testing on dev set real quick"
            dev_accuracies.append(test(opts.outFile,"dev",opts.model,dev_trees))
            # clear the fprop flags in trees and dev_trees
            for tree in trees:
                tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
            for tree in dev_trees:
                tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
            print "fprop in trees cleared"


    if evaluate_accuracy_while_training:
        #pdb.set_trace()

        plt.figure()
        #Lets set up the plot
        plt.title('Accuracy in set per epochs')
        plt.plot(range(opts.epochs),train_accuracies,label='train')
        plt.plot(range(opts.epochs),dev_accuracies,label='dev')

        with open('dev_accu' + opts.model,'a') as fid:
            fid.write(str(opts.wvecDim) + ',' + str(opts.middleDim) + ',' + str(dev_accuracies[-1]) + ';')

        #plt.axis([0,opts.epochs,0,1])
        plt.xlabel('epochs')
        plt.ylabel('accuracy')
        plt.legend(loc=2, borderaxespad=0.)

        #always save with middleDim, even if it's a one-layer RNN
        plt.savefig('plots/' + opts.model + '/accuracy_wvec_' + str(opts.wvecDim) + '_middleDim_' + str(opts.middleDim) + ' .png')

        print 'image saved at %s' % os.getcwd()
Ejemplo n.º 11
0
def run(args=None):
    usage = "usage : %prog [options]"
    parser = optparse.OptionParser(usage=usage)

    parser.add_option("--test",action="store_true",dest="test",default=False)

    # Optimizer
    parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
    parser.add_option("--optimizer",dest="optimizer",type="string",
        default="adagrad")
    parser.add_option("--epochs",dest="epochs",type="int",default=50)
    parser.add_option("--step",dest="step",type="float",default=1e-2)
    parser.add_option("--init",dest="init",type="float",default=0.01)

    parser.add_option("--outputDim",dest="outputDim",type="int",default=5)
    parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)

    parser.add_option("--rho",dest="rho",type="float",default=1e-6)

    parser.add_option("--outFile",dest="outFile",type="string",
        default="models/test.bin")
    parser.add_option("--inFile",dest="inFile",type="string",
        default="models/test.bin")
    parser.add_option("--data",dest="data",type="string",default="train")

    parser.add_option("--model",dest="model",type="string",default="RNTN")

    parser.add_option("--maxTrain",dest="maxTrain", type="int", default=-1)
    parser.add_option("--activation",dest="acti", type="string", default="tanh")

    parser.add_option("--partial",action="store_true",dest="partial",default=False)
    parser.add_option("--w2v",dest="w2vmodel", type="string")

    (opts,args)=parser.parse_args(args)


    # make this false if you dont care about your accuracies per epoch, makes things faster!
    evaluate_accuracy_while_training = True

    # Testing
    if opts.test:
        cmfile = opts.inFile + ".confusion_matrix-" + opts.data + ".png"
        test(opts.inFile,opts.data,opts.model,acti=opts.acti)
        return
    
    print "Loading data..."

    embedding = None
    wordMap = None
    if opts.w2vmodel is not None:
        print "Loading pre-trained word2vec model from %s" % opts.w2vmodel
        w2v = models.Word2Vec.load(opts.w2vmodel)
        embedding, wordMap = readW2v(w2v,opts.wvecDim)

    train_accuracies = []
    train_rootAccuracies = []
    dev_accuracies = []
    dev_rootAccuracies = []
    # load training data
    trees = tr.loadTrees('train',wordMap=wordMap)[:opts.maxTrain] #train.full.15
    if opts.maxTrain > -1:
        print "Training only on %d trees" % opts.maxTrain
    opts.numWords = len(tr.loadWordMap())


    if opts.partial==True:
        print "Only partial feedback"

    if (opts.model=='RNTN'):
        nn = RNTN(wvecDim=opts.wvecDim,outputDim=opts.outputDim,numWords=opts.numWords,
                  mbSize=opts.minibatch,rho=opts.rho, acti=opts.acti, init=opts.init, partial=opts.partial)
    else:
        raise '%s is not a valid neural network so far only RNTN, RNN'%opts.model
    
    nn.initParams(embedding=embedding)

    sgd = optimizer.SGD(nn,alpha=opts.step,minibatch=opts.minibatch,
        optimizer=opts.optimizer)


    dev_trees = tr.loadTrees("dev") #dev.full.15
    for e in range(opts.epochs):
        start = time.time()
        print "Running epoch %d"%e
        sgd.run(trees)
        end = time.time()
        print "Time per epoch : %f"%(end-start)

        with open(opts.outFile,'w') as fid:
            pickle.dump(opts,fid)
            pickle.dump(sgd.costt,fid)
            nn.toFile(fid)
        if evaluate_accuracy_while_training:
            print "testing on training set"
            acc, sacc = test(opts.outFile,"train",opts.model,trees, acti=opts.acti)
            train_accuracies.append(acc)
            train_rootAccuracies.append(sacc)
            print "testing on dev set"
            dacc, dsacc = test(opts.outFile,"dev",opts.model,dev_trees, acti=opts.acti)
            dev_accuracies.append(dacc)
            dev_rootAccuracies.append(dsacc)
            # clear the fprop flags and dev_trees
            for tree in trees:
                tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
            for tree in dev_trees:
                tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
            print "fprop in trees cleared"


    if evaluate_accuracy_while_training:
        pdb.set_trace()
        print train_accuracies
        print dev_accuracies

        print "on sentence-level:"
        print train_rootAccuracies
        print dev_rootAccuracies

        # Plot train/dev_accuracies
        plt.figure()
        plt.plot(range(len(train_accuracies)), train_accuracies, label='Train')
        plt.plot(range(len(dev_accuracies)), dev_accuracies, label='Dev')
        plt.xlabel("Epoch")
        plt.ylabel("Accuracy")
        plt.legend()
        # plot.show()
        plt.savefig(opts.outFile + ".accuracy_plot.png")

          # Plot train/dev_accuracies
        plt.figure()
        plt.plot(range(len(train_rootAccuracies)), train_rootAccuracies, label='Train')
        plt.plot(range(len(dev_rootAccuracies)), dev_rootAccuracies, label='Dev')
        plt.xlabel("Epoch")
        plt.ylabel("Accuracy")
        plt.legend()
        # plot.show()
        plt.savefig(opts.outFile + ".sent.accuracy_plot.png")
Ejemplo n.º 12
0
def run():
    print "Loading data..."
    model = "RNN"
    trees = tr.loadTrees('train')
    dev_trees = tr.loadTrees('dev')
    wvecDimList = [5, 15, 25, 35, 45]
    #wvecDimList = [10,20,40]
    accuracy_per_wvecDim = []
    epochs = 100
    outFileText = "./param/%s/%s_cost_and_acc" % (model, model)
    f = open(outFileText, 'w')
    for wvecDim in wvecDimList:
        nn = RNN(wvecDim, 5, len(tr.loadWordMap()), 30)
        nn.initParams()
        sgd = optimizer.SGD(nn, alpha=0.01, minibatch=30, optimizer="adagrad")
        outFile = "./param/%s/%s_wvecDim_%d_epochs_%d_step_001.bin" % (
            model, model, wvecDim, epochs)

        train_cost = []
        train_acc = []
        dev_cost = []
        dev_acc = []
        cost = 0
        accuracy = 0
        for e in range(epochs):
            start = time.time()
            sgd.run(trees)
            end = time.time()
            print "Time per epoch : %f" % (end - start)
            with open(outFile, 'w') as fid:
                hyperparam = {}
                hyperparam['alpha'] = 0.01
                hyperparam['minibatch'] = 30
                hyperparam['wvecDim'] = wvecDim
                pickle.dump(hyperparam, fid)
                nn.toFile(fid)

            cost, accuracy = test(nn, trees)
            train_cost.append(cost)
            train_acc.append(accuracy)

            cost, accuracy = test(nn, dev_trees)
            dev_cost.append(cost)
            dev_acc.append(accuracy)

            for tree in trees:
                tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
            for tree in dev_trees:
                tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
            print "fprop in trees cleared"

        plot_cost_acc(
            train_cost, dev_cost,
            "./figures/%s/%s_Cost_Figure_%d" % (model, model, wvecDim), epochs)
        plot_cost_acc(
            train_acc, dev_acc,
            "./figures/%s/%s_Accuracy_Figure_%d" % (model, model, wvecDim),
            epochs)

        anwser = "Cost = %f, Acc= %f" % (cost, accuracy)
        f.write(anwser)
        accuracy_per_wvecDim.append(accuracy)

    f.close()
    plt.figure(figsize=(6, 4))
    plt.title(r"Accuracies and vector Dimension")
    plt.xlabel("vector Dimension")
    plt.ylabel(r"Accuracy")
    plt.ylim(ymin=min(accuracy_per_wvecDim) * 0.8,
             ymax=max(accuracy_per_wvecDim) * 1.2)
    plt.plot(wvecDimList,
             accuracy_per_wvecDim,
             color='b',
             marker='o',
             linestyle='-')
    plt.savefig("./figures/%s/%s_Accuracy_and_vectorDimsension.png" %
                (model, model))
    plt.close()
def run(args=None):
    usage = "usage : %prog [options]"
    parser = optparse.OptionParser(usage=usage)

    parser.add_option("--test",action="store_true",dest="test",default=False)

    # Optimizer
    parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
    parser.add_option("--optimizer",dest="optimizer",type="string",
        default="adagrad")
    parser.add_option("--epochs",dest="epochs",type="int",default=50)
    parser.add_option("--step",dest="step",type="float",default=1e-2)


    parser.add_option("--middleDim",dest="middleDim",type="int",default=10)
    parser.add_option("--outputDim",dest="outputDim",type="int",default=3)
    parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)

    # for DCNN only
    parser.add_option("--ktop",dest="ktop",type="int",default=5)
    parser.add_option("--m1",dest="m1",type="int",default=10)
    parser.add_option("--m2",dest="m2",type="int",default=7)
    parser.add_option("--n1",dest="n1",type="int",default=6)
    parser.add_option("--n2",dest="n2",type="int",default=12)

    parser.add_option("--outFile",dest="outFile",type="string",
        default="models/test.bin")
    parser.add_option("--inFile",dest="inFile",type="string",
        default="models/test.bin")
    parser.add_option("--data",dest="data",type="string",default="train")

    parser.add_option("--model",dest="model",type="string",default="RNN")

    (opts,args)=parser.parse_args(args)

    # make this false if you dont care about your accuracies per epoch, makes things faster!
    evaluate_accuracy_while_training = True

    # Testing
    if opts.test:
        cmfile = opts.inFile + ".confusion_matrix-" + opts.data
        test(opts.inFile,opts.data,None,opts.model,confusion_matrix_file=cmfile,full=True)
        return

    print "Loading data..."
    train_accuracies = []
    dev_accuracies = []
    # load training data
    trees = tr.loadTrees('train')
    opts.numWords = len(tr.loadWordMap())

    #Load word embeddings
    L = tr.loadWordEmbedding()

    if(opts.model=='RNN2'):
        nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
    else:
        raise '%s is not a valid neural network, only RNN2'%opts.model

    nn.initParams(L)

    sgd = optimizer.SGD(nn,alpha=opts.step,minibatch=opts.minibatch,
        optimizer=opts.optimizer)


    dev_trees = tr.loadTrees("dev")
    for e in range(opts.epochs):
        start = time.time()
        print "Running epoch %d"%e
        sgd.run(trees)
        end = time.time()
        print "Time per epoch : %f"%(end-start)

        with open(opts.outFile,'w') as fid:
            pickle.dump(opts,fid)
            pickle.dump(sgd.costt,fid)
            nn.toFile(fid)
        if evaluate_accuracy_while_training:
            print "testing on training set real quick"
            train_accuracies.append(test(opts.outFile,"train",L,opts.model,trees))
            print "testing on dev set real quick"
            dev_accuracies.append(test(opts.outFile,"dev",L,opts.model,dev_trees))
            # clear the fprop flags in trees and dev_trees
            for tree in trees:
                tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
            for tree in dev_trees:
                tr.leftTraverse(tree.root,nodeFn=tr.clearFprop)
            print "fprop in trees cleared"


    if evaluate_accuracy_while_training:
        # pdb.set_trace()
        print train_accuracies
        print dev_accuracies
        # Plot train/dev_accuracies here?
        plt.figure()
        plt.plot(range(len(train_accuracies)), train_accuracies, label='Train')
        plt.plot(range(len(dev_accuracies)), dev_accuracies, label='Dev')
        plt.xlabel("Epoch")
        plt.ylabel("Accuracy")
        plt.legend()
        # plot.show()
        plt.savefig(opts.outFile + ".accuracy_plot.png")
Ejemplo n.º 14
0
def run(args=None):
    usage = "usage : %prog [options]"
    parser = optparse.OptionParser(usage=usage)

    parser.add_option("--test",
                      action="store_true",
                      dest="test",
                      default=False)

    # Optimizer
    parser.add_option("--minibatch", dest="minibatch", type="int", default=30)
    parser.add_option("--optimizer",
                      dest="optimizer",
                      type="string",
                      default="adagrad")
    parser.add_option("--epochs", dest="epochs", type="int", default=50)
    parser.add_option("--step", dest="step", type="float", default=1e-2)
    parser.add_option("--init", dest="init", type="float", default=0.01)

    parser.add_option("--outputDim", dest="outputDim", type="int", default=5)
    parser.add_option("--wvecDim", dest="wvecDim", type="int", default=30)

    parser.add_option("--rho", dest="rho", type="float", default=1e-6)

    parser.add_option("--outFile",
                      dest="outFile",
                      type="string",
                      default="models/test.bin")
    parser.add_option("--inFile",
                      dest="inFile",
                      type="string",
                      default="models/test.bin")
    parser.add_option("--data", dest="data", type="string", default="train")

    parser.add_option("--model", dest="model", type="string", default="RNTN")

    parser.add_option("--maxTrain", dest="maxTrain", type="int", default=-1)
    parser.add_option("--activation",
                      dest="acti",
                      type="string",
                      default="tanh")

    parser.add_option("--partial",
                      action="store_true",
                      dest="partial",
                      default=False)
    parser.add_option("--w2v", dest="w2vmodel", type="string")

    (opts, args) = parser.parse_args(args)

    # make this false if you dont care about your accuracies per epoch, makes things faster!
    evaluate_accuracy_while_training = True

    # Testing
    if opts.test:
        cmfile = opts.inFile + ".confusion_matrix-" + opts.data + ".png"
        test(opts.inFile, opts.data, opts.model, acti=opts.acti)
        return

    print "Loading data..."

    embedding = None
    wordMap = None
    if opts.w2vmodel is not None:
        print "Loading pre-trained word2vec model from %s" % opts.w2vmodel
        w2v = models.Word2Vec.load(opts.w2vmodel)
        embedding, wordMap = readW2v(w2v, opts.wvecDim)

    train_accuracies = []
    train_rootAccuracies = []
    dev_accuracies = []
    dev_rootAccuracies = []
    # load training data
    trees = tr.loadTrees('train',
                         wordMap=wordMap)[:opts.maxTrain]  #train.full.15
    if opts.maxTrain > -1:
        print "Training only on %d trees" % opts.maxTrain
    opts.numWords = len(tr.loadWordMap())

    if opts.partial == True:
        print "Only partial feedback"

    if (opts.model == 'RNTN'):
        nn = RNTN(wvecDim=opts.wvecDim,
                  outputDim=opts.outputDim,
                  numWords=opts.numWords,
                  mbSize=opts.minibatch,
                  rho=opts.rho,
                  acti=opts.acti,
                  init=opts.init,
                  partial=opts.partial)
    else:
        raise '%s is not a valid neural network so far only RNTN, RNN' % opts.model

    nn.initParams(embedding=embedding)

    sgd = optimizer.SGD(nn,
                        alpha=opts.step,
                        minibatch=opts.minibatch,
                        optimizer=opts.optimizer)

    dev_trees = tr.loadTrees("dev")  #dev.full.15
    for e in range(opts.epochs):
        start = time.time()
        print "Running epoch %d" % e
        sgd.run(trees)
        end = time.time()
        print "Time per epoch : %f" % (end - start)

        with open(opts.outFile, 'w') as fid:
            pickle.dump(opts, fid)
            pickle.dump(sgd.costt, fid)
            nn.toFile(fid)
        if evaluate_accuracy_while_training:
            print "testing on training set"
            acc, sacc = test(opts.outFile,
                             "train",
                             opts.model,
                             trees,
                             acti=opts.acti)
            train_accuracies.append(acc)
            train_rootAccuracies.append(sacc)
            print "testing on dev set"
            dacc, dsacc = test(opts.outFile,
                               "dev",
                               opts.model,
                               dev_trees,
                               acti=opts.acti)
            dev_accuracies.append(dacc)
            dev_rootAccuracies.append(dsacc)
            # clear the fprop flags and dev_trees
            for tree in trees:
                tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
            for tree in dev_trees:
                tr.leftTraverse(tree.root, nodeFn=tr.clearFprop)
            print "fprop in trees cleared"

    if evaluate_accuracy_while_training:
        pdb.set_trace()
        print train_accuracies
        print dev_accuracies

        print "on sentence-level:"
        print train_rootAccuracies
        print dev_rootAccuracies

        # Plot train/dev_accuracies
        plt.figure()
        plt.plot(range(len(train_accuracies)), train_accuracies, label='Train')
        plt.plot(range(len(dev_accuracies)), dev_accuracies, label='Dev')
        plt.xlabel("Epoch")
        plt.ylabel("Accuracy")
        plt.legend()
        # plot.show()
        plt.savefig(opts.outFile + ".accuracy_plot.png")

        # Plot train/dev_accuracies
        plt.figure()
        plt.plot(range(len(train_rootAccuracies)),
                 train_rootAccuracies,
                 label='Train')
        plt.plot(range(len(dev_rootAccuracies)),
                 dev_rootAccuracies,
                 label='Dev')
        plt.xlabel("Epoch")
        plt.ylabel("Accuracy")
        plt.legend()
        # plot.show()
        plt.savefig(opts.outFile + ".sent.accuracy_plot.png")