Exemple #1
0
def write_graph(loss):
    with open("graph.dot", "w") as o:
        o.write(c.build_computational_graph((loss, )).dump())
    with open("graph.wo_split.dot", "w") as o:
        g = c.build_computational_graph((loss, ), remove_split=True)
        o.write(g.dump())
    print('graph generated')
def train_loop():
    # Trainer
    graph_generated = False
    data = np.ndarray((args.batchsize, 3, 224, 224), dtype=np.float32)
    data.fill(33333)
    x = xp.asarray(data)
    
    label = np.ndarray((args.batchsize), dtype=np.int32)
    label.fill(1)
    y = xp.asarray(label)

    

    while True:

        optimizer.zero_grads()
        loss, accuracy = model.forward(x, y)
        loss.backward()
        optimizer.update()

        if not graph_generated:
            with open('graph.dot', 'w') as o:
                o.write(c.build_computational_graph((loss,), False).dump())
            with open('graph.wo_split.dot', 'w') as o:
                o.write(c.build_computational_graph((loss,), True).dump())
            print('generated graph')
            graph_generated = True

        
        del loss, accuracy, x, y
    def setUp(self):
        self.x = variable.Variable(np.zeros((1, 2)).astype(np.float32))
        self.y = 2 * self.x

        self.x_splitter = self.x.splitter()
        self.x_clone = self.x_splitter.outputs[0]()
        self.f = self.y.creator

        self.g1 = c.build_computational_graph((self.y,), False)
        self.g2 = c.build_computational_graph((self.y,), True)
def train_loop():
    # Trainer
    graph_generated = False
    while True:
        while data_q.empty():
            time.sleep(0.1)
        inp = data_q.get()
        if inp == 'end':  # quit
            res_q.put('end')
            break
        elif inp == 'train':  # restart training
            res_q.put('train')
            train = True
            continue
        elif inp == 'val':  # start validation
            res_q.put('val')
            pickle.dump(model, open(folder+'model', 'wb'), -1)
            train = False
            continue

        x = xp.asarray(inp[0])
        y = xp.asarray(inp[1])

        if train:
            optimizer.zero_grads()
            loss, accuracy = model.forward(x, y)
            loss.backward()
            optimizer.update()

            if not graph_generated:
                with open('graph.dot', 'w') as o:
                    o.write(c.build_computational_graph((loss,), False).dump())
                with open('graph.wo_split.dot', 'w') as o:
                    o.write(c.build_computational_graph((loss,), True).dump())
                print('generated graph')
                graph_generated = True

        else:
            loss, accuracy = model.forward(x, y, train=False)
        if epoch_count % 2 == 0:
            print ('save model')
            model.to_cpu()
            with open(folder + 'model_' + str(epoch_count), 'wb') as o:
                pickle.dump(model, o)
            model.to_gpu()#もう一度GPUに戻すのか?
            optimizer.setup(model)
            
        res_q.put((float(loss.data),
                   float(accuracy.data)))
        del loss, accuracy, x, y
    def learn_as_autoencoder(self, x_train, x_test=None):
        optimizer = self.optimizer
        train_size = x_train.shape[0]
        train_data_size = x_train.shape[1]
        #self.add_dummy_output_link(train_data_size)
        for epoch in six.moves.range(self.epoch):
            perm = np.random.permutation(train_size)
            train_loss = 0
            test_loss = None
            test_accuracy = 0
            for i in range(0, train_size, self.batch_size):
                x = Variable(x_train[perm[i:i+self.batch_size]])
                self.zerograds()
                loss = self.loss_function(self[0](x), x)
                loss.backward()
                self.optimizer.update()
                train_loss += loss.data * self.batch_size
            train_loss /= train_size

            if len(x_test):
                x = Variable(x_test)
                test_loss = self.loss_function(self[0](x), x).data
        if test_loss is not None:
            print('Pre-training test loss: ' + str(test_loss))
        if self.visualize:
            import chainer.computational_graph as c
            g = c.build_computational_graph((loss,))
            with open('child_graph.dot', 'w') as o:
                o.write(g.dump())
        del self.optimizer
 def test_tail_node(self):
     edges = c.build_computational_graph((self.y,), False)
     self.assertEqual(len(edges), 4)
     self.assertTrue((self.x, self.x_splitter) in edges)
     self.assertTrue((self.x_splitter, self.x_clone) in edges)
     self.assertTrue((self.x_clone, self.f) in edges)
     self.assertTrue((self.f, self.y) in edges)
Exemple #7
0
def train_loop(model, output_dir, xp, optimizer, res_q, data_q):
    graph_generated = False
    while True:
        while data_q.empty():
            time.sleep(0.1)
        inp = data_q.get()
        if inp == 'end':
            res_q.put('end')
            break
        elif inp == 'train':
            res_q.put('train')
            model.train = True
            continue
        elif inp == 'val':
            res_q.put('val')
            model.train = False
            continue
        volatile = 'off' if model.train else 'on'
        x = chainer.Variable(xp.asarray(inp[0]), volatile=volatile)
        t = chainer.Variable(xp.asarray(inp[1]), volatile=volatile)
        if model.train:
            optimizer.update(model, x, t)
            if not graph_generated:
                with open('graph.dot', 'w') as o:
                    o.write(computational_graph.build_computational_graph((model.loss,)).dump())
                print('generated graph')
                graph_generated = True
        else:
            model(x, t)
            
        serializers.save_hdf5(output_dir + os.sep + 'model%04d'%inp[2], model)
        #serializers.save_hdf5(output_dir + os.sep + 'optimizer%04d'%inp[2], optimizer)
        res_q.put((float(model.loss.data), float(model.accuracy.data), inp[2]))
        del x, t
Exemple #8
0
def draw_computational_graph(*args, **kwargs):
    """Draw computational graph.

    @param output: output ps file.
    """
    from chainer.computational_graph import build_computational_graph
    output = kwargs.pop('output')
    if len(args) > 2:
        variable_style = args[2]
    else:
        variable_style = kwargs.get(
            'variable_style',
            {'shape': 'octagon', 'fillcolor': '#E0E0E0', 'style': 'filled'},
        )
        kwargs['variable_style'] = variable_style
    if len(args) > 3:
        function_style = args[3]
    else:
        function_style = kwargs.get(
            'function_style',
            {'shape': 'record', 'fillcolor': '#6495ED', 'style': 'filled'},
        )
        kwargs['function_style'] = function_style
    dotfile = tempfile.mktemp()
    with open(dotfile, 'w') as f:
        f.write(build_computational_graph(*args, **kwargs).dump())
    ext = osp.splitext(output)[-1][1:]  # ex) .ps -> ps
    cmd = 'dot -T{0} {1} > {2}'.format(ext, dotfile, output)
    subprocess.call(cmd, shell=True)
    def test_backward(self):
        rpn_in_ch = 512
        rpn_out_ch = 512
        feat_stride = 16
        anchor_ratios = [0.5, 1, 2]
        anchor_scales = [8, 16, 32]
        num_classes = 21
        model = FasterRCNN(
            self.trunk, rpn_in_ch, rpn_out_ch, feat_stride,
            anchor_ratios, anchor_scales, num_classes)
        model.rpn_train, model.rcnn_train = self.train
        if self.device >= 0:
            model.to_gpu(self.device)
            self.x.to_gpu(self.device)
            self.im_info.to_gpu(self.device)
            self.gt_boxes.to_gpu(self.device)
            self.assertIs(model.xp, cp)
            self.assertIs(model.trunk.xp, cp)
        opt = optimizers.Adam()
        opt.setup(model)

        if model.rpn_train:
            st = time.time()
            rpn_loss = model(self.x, self.im_info, self.gt_boxes)
            model.cleargrads()
            rpn_loss.backward()
            opt.update()
            print('Backward rpn device:{}, ({}, train:{}): {} sec'.format(
                self.device, self.trunk.__name__, self.train, time.time() - st))

            rpn_cg = cg.build_computational_graph([rpn_loss])
            with open('tests/rpn_cg.dot', 'w') as fp:
                fp.write(rpn_cg.dump())

        elif model.rcnn_train:
            st = time.time()
            loss_rcnn = model(self.x, self.im_info, self.gt_boxes)
            model.cleargrads()
            loss_rcnn.backward()
            opt.update()
            print('Backward rcnn device:{}, ({}, train:{}): {} sec'.format(
                self.device, self.trunk.__name__, self.train, time.time() - st))

            loss_rcnn_cg = cg.build_computational_graph([loss_rcnn])
            with open('tests/loss_rcnn_cg.dot', 'w') as fp:
                fp.write(loss_rcnn_cg.dump())
Exemple #10
0
def train_loop():
    graph_generated = False
    while True:
        while data_q.empty():
            time.sleep(0.1)
        inp = data_q.get()
        if inp == 'end':  # quit
            res_q.put('end')
            break
        elif inp == 'train':  # restart training
            res_q.put('train')
            train = True
            continue
        elif inp == 'val':  # start validation
            res_q.put('val')
            pickle.dump(model, open('model', 'wb'), -1)
            train = False
            continue

        x, y = inp
        if args.gpu >= 0:
            x = cuda.to_gpu(x)
            y = cuda.to_gpu(y)

        if train:
            optimizer.zero_grads()
            loss, accuracy = model.forward(x, y)
            loss.backward()
            optimizer.update()

            if not graph_generated:
                with open('graph.dot', 'w') as o:
                    o.write(c.build_computational_graph((loss,), False).dump())
                with open('graph.wo_split.dot', 'w') as o:
                    o.write(c.build_computational_graph((loss,), True).dump())
                print('generated graph')
                graph_generated = True

        else:
            loss, accuracy = model.forward(x, y, train=False)

        res_q.put((float(cuda.to_cpu(loss.data)),
                   float(cuda.to_cpu(accuracy.data))))
        del loss, accuracy, x, y
Exemple #11
0
def train_nnet(model, optimizer, train_data_resource, opts):
    if opts.gpu >= 0:
        cuda.check_cuda_available()
        model.to_gpu(opts.gpu)
    accum_loss = 0

    i = 0
    train_loss = 0
    prev_dev_loss = 100000
    prev_percentage = 0
    dump_graph = True
    for train_idx, x_batch, y_batch, epoch, percentage, eos in train_data_resource:
        if train_idx is None:  # Done one epoch
            if opts.fname_dev:
                dev_loss, _, _ = evaluation(model, opts.fname_dev)
                if xp == cuda.cupy:
                    model.to_gpu()
                print(' dev loss: %.3f' % dev_loss, end='')
                if optimizer.lr < opts.lr_stop:
                    break
                if prev_dev_loss - dev_loss < opts.start_decay:
                    optimizer.lr *= opts.lr_decay
                    print('\n...reducing lr to %.6f' % optimizer.lr)
                prev_dev_loss = dev_loss
            print('')
            continue
        x = Variable(xp.asarray(x_batch))
        t = Variable(xp.asarray(y_batch))
        loss_i = model(x, t)
        accum_loss += loss_i
        if dump_graph:
            print('Dump graph')
            with open('graph.dot', 'w') as o:
                o.write(c.build_computational_graph((loss_i, )).dump())
            dump_graph = False
        if train_idx >= 1:
            train_loss = (train_loss * (train_idx - 1) + loss_i.data) / train_idx

        if eos and opts.forget_on_new_utt:
            model.predictor.forget_history()

        if eos or (i + 1) % opts.bprop_len == 0:
            model.zerograds()
            accum_loss.backward()
            accum_loss.unchain_backward()
            accum_loss = 0
            optimizer.update()
            i = 0

        if percentage != prev_percentage:
            prev_percentage = percentage
            print_stats(percentage, epoch, optimizer.lr, train_loss)
            sys.stdout.flush()
        i += 1
 def setUp(self):
     self.x1 = variable.Variable(np.zeros((1, 2)).astype(np.float32))
     self.x2 = variable.Variable(np.zeros((1, 2)).astype(np.float32))
     self.y = self.x1 + self.x2
     self.f = self.y.creator
     self.variable_style = {'label': 'variable_0', 'shape': 'octagon',
                            'style': 'filled', 'fillcolor': '#E0E0E0'}
     self.function_style = {'label': 'function_0', 'shape': 'record',
                            'style': 'filled', 'fillcolor': '#6495ED'}
     self.g = c.build_computational_graph(
         (self.y,), variable_style=self.variable_style,
         function_style=self.function_style)
    def dump_graph(trainer):
        var = trainer.observation[root_name]
        if not isinstance(var, variable.Variable):
            raise TypeError('root value is not a Variable')
        cg = computational_graph.build_computational_graph(
            [var],
            variable_style=variable_style,
            function_style=function_style
        ).dump()

        out_path = os.path.join(trainer.out, out_name)
        # TODO(beam2d): support outputting images by the dot command
        with open(out_path, 'w') as f:
            f.write(cg)
Exemple #14
0
    def __call__(self, trainer):
        try:
            var = trainer.observation[self._root_name]
            if not isinstance(var, variable.Variable):
                raise TypeError('root value is not a Variable')
            cg = computational_graph.build_computational_graph(
                [var],
                variable_style=self._variable_style,
                function_style=self._function_style
            ).dump()

            filename = os.path.join(trainer.out, self._filename)
            with open(filename, 'w') as f:
                f.write(cg)
            if is_graphviz_available():
                img_fn = os.path.splitext(self._filename)[0] + '.png'
                image_filename = os.path.join(trainer.out, img_fn)
                subprocess.check_call(
                    ['dot', '-Tpng', filename, '-o', image_filename])
        finally:
            configuration.config.keep_graph_on_report = self._original_flag
Exemple #15
0
def train_loop():
    # Trainer
    graph_generated = False
    while True:
        while data_q.empty():
            time.sleep(0.1)
        inp = data_q.get()
        if inp == 'end':  # quit
            res_q.put('end')
            break
        elif inp == 'train':  # restart training
            res_q.put('train')
            model.train = True
            continue
        elif inp == 'val':  # start validation
            res_q.put('val')
            serializers.save_npz(args.out, model)
            serializers.save_npz(args.outstate, optimizer)
            model.train = False
            continue

        volatile = 'off' if model.train else 'on'
        x = chainer.Variable(xp.asarray(inp[0]), volatile=volatile)
        t = chainer.Variable(xp.asarray(inp[1]), volatile=volatile)

        if model.train:
            optimizer.update(model, x, t)
            if not graph_generated:
                with open('graph.dot', 'w') as o:
                    o.write(computational_graph.build_computational_graph(
                        (model.loss,)).dump())
                print('generated graph', file=sys.stderr)
                graph_generated = True
        else:
            model(x, t)

        res_q.put((float(model.loss.data), float(model.accuracy.data)))
        del x, t
Exemple #16
0
	def backprop(self,t):
		x=Tensor.context


		self.optimizer.lr = Deel.optimizer_lr

		self.optimizer.zero_grads()
		loss,accuracy = self.func.getLoss(x.content,t.content)

		loss.backward()
		self.optimizer.update()
		

		if not self.graph_generated:
			#with open('graph.dot', 'w') as o:
			#	o.write(c.build_computational_graph((loss,), False).dump())
			with open('graph.wo_split.dot', 'w') as o:
				o.write(c.build_computational_graph((loss,), True).dump())
			print('generated graph')
			self.graph_generated = True


		return loss.data,accuracy.data
def _check(self, outputs, node_num, edge_num):
    g = c.build_computational_graph(outputs)
    self.assertEqual(len(g.nodes), node_num)
    self.assertEqual(len(g.edges), edge_num)
 def setUp(self):
     self.x1 = variable.Variable(np.zeros((1, 2)).astype(np.float32))
     self.x2 = variable.Variable(np.zeros((1, 2)).astype(np.float32))
     self.y = self.x1 + self.x2
     self.f = self.y.creator
     self.g = c.build_computational_graph((self.y,))
Exemple #19
0
    sum_loss = 0
    for i in six.moves.range(0, N, batchsize):
        x_batch = x_train[perm[i:i + batchsize]]
        y_batch = y_train[perm[i:i + batchsize]]
        if args.gpu >= 0:
            x_batch = cuda.to_gpu(x_batch)
            y_batch = cuda.to_gpu(y_batch)

        optimizer.zero_grads()
        loss, acc = forward(x_batch, y_batch)
        loss.backward()
        optimizer.update()

        if epoch == 1 and i == 0:
            with open("graph.dot", "w") as o:
                o.write(c.build_computational_graph((loss, )).dump())
            with open("graph.wo_split.dot", "w") as o:
                g = c.build_computational_graph((loss, ),
                                                remove_split=True)
                o.write(g.dump())
            print('graph generated')

        sum_loss += float(cuda.to_cpu(loss.data)) * len(y_batch)
        sum_accuracy += float(cuda.to_cpu(acc.data)) * len(y_batch)

    print('train mean loss={}, accuracy={}'.format(
        sum_loss / N, sum_accuracy / N))

    # evaluation
    sum_accuracy = 0
    sum_loss = 0
Exemple #20
0
    serializers.load_npz(args.resume, optimizer)

# Learning loop
for epoch in six.moves.range(1, n_epoch + 1):
    print('epoch', epoch)

    # training
    perm = np.random.permutation(N)
    sum_loss = 0       # total loss
    sum_rec_loss = 0   # reconstruction loss
    for i in six.moves.range(0, N, batchsize):
        x = chainer.Variable(xp.asarray(x_train[perm[i:i + batchsize]]))
        optimizer.update(model.get_loss_func(), x)
        if epoch == 1 and i == 0:
            with open('graph.dot', 'w') as o:
                g = computational_graph.build_computational_graph(
                    (model.loss, ))
                o.write(g.dump())
            print('graph generated')

        sum_loss += float(model.loss.data) * len(x.data)
        sum_rec_loss += float(model.rec_loss.data) * len(x.data)

    print('train mean loss={}, mean reconstruction loss={}'
          .format(sum_loss / N, sum_rec_loss / N))

    # evaluation
    sum_loss = 0
    sum_rec_loss = 0
    with chainer.no_backprop_mode():
        for i in six.moves.range(0, N_test, batchsize):
            x = chainer.Variable(xp.asarray(x_test[i:i + batchsize]))
def main():
    
    w2vec_model = None
    w_vocab = {}
    t_vocab = {}
    if (opts.fname_test or opts.fname_decode) and not opts.fname_in_model:
        print('Please specify a model in testing mode')
        exit(0)
    
    if opts.fname_w2vec:
        print('Loading w2vec model')
        w2vec_model = Word2VecModel()
        w2vec_model.load(opts.fname_w2vec)
        w_vocab = w2vec_model.vocab

    if opts.fname_in_model:
        model = pickle.load(open(opts.fname_in_model, 'rb'))

        w_vocab = model.w_vocab
        t_vocab = model.t_vocab
        if opts.fname_train:
            trainset, _ = load_data(opts.fname_train, w_vocab, t_vocab, update_dict=False)
    else:
        update_dict = False if opts.fname_w2vec else True
        trainset, _ = load_data(opts.fname_train, w_vocab, t_vocab, update_dict=update_dict)
        model = setup_model(len(w_vocab), len(t_vocab), nlayer=opts.nlayer,
                layer_size=opts.layer_size,
                use_gpu=opts.gpu, dropout=opts.dropout,
                w2vec_model=w2vec_model,
                seed=opts.random_seed)
        setattr(model, 'w_vocab', w_vocab)
        setattr(model, 't_vocab', t_vocab)
    
    if opts.fname_test:
        test_mode(model, opts.fname_test)
        exit(0)
    
    if opts.fname_decode:
        test_mode(model, opts.fname_decode, decode=True)
        exit(0)

    devset = None
    if opts.fname_dev:
        devset, oov = load_data(opts.fname_dev, w_vocab, t_vocab, update_dict=False)

    optimizer = optimizers.SGD(lr=opts.lr)
    optimizer.setup(model)
    
    # Calculating training parameters
    whole_len = trainset['input'].shape[0]
    jump = whole_len // opts.batchsize
    cur_log_perp = 0
    epoch = 0
    cur_loss = 0
    start_at = time.time()
    progress_idx = 0
    state = make_initial_state(batchsize=opts.batchsize, nlayer=opts.nlayer,
            layer_size=model.layer_size
            )
    accum_loss = chainer.Variable(xp.zeros((), dtype=np.float32))
    print('going to train {} iterations'.format(jump * opts.n_epoch))
    
    prev_dev_loss = 1000
    curr_dev_loss = 0
    for i in six.moves.range(jump * opts.n_epoch):
        x_batch = xp.array([trainset['input'][(jump * j + i) % whole_len]
                   for j in six.moves.range(opts.batchsize)])
        y_batch = xp.array([trainset['target'][(jump * j + i) % whole_len]
                   for j in six.moves.range(opts.batchsize)])
        state, loss_i, _ = forward_one_step(model, x_batch, y_batch, state)
        cur_loss += loss_i.data
        if i == 0:
            with open('graph.dot', 'w') as o:
                o.write(c.build_computational_graph((loss_i, )).dump())
        accum_loss += loss_i
        cur_log_perp += loss_i.data

        if (i + 1) % opts.bprop_len == 0:
            optimizer.zero_grads()
            accum_loss.backward()
            accum_loss.unchain_backward()
            optimizer.clip_grads(opts.grad_clip)
            optimizer.update()
            accum_loss = chainer.Variable(xp.zeros((), dtype=np.float32))

        if i >= 1:
            training_loss = cur_loss / i
            progress_idx += 1
            progress = int(progress_idx * 100 / float(jump))
            print('\r#Epoch {0}, lr: {1} [{2}] {3}% Training loss: {4}'.format(
                epoch, "%.6f" % optimizer.lr, '#'*(progress/5),
                progress, "%.2f" % training_loss), end='')
            sys.stdout.flush()

        if (i + 1) % jump == 0:
            progress_idx = 0
            epoch += 1
            if devset:
                # print('')
                curr_dev_loss, _ = evaluate(model, devset, report_progress=False)
                improvment = prev_dev_loss - curr_dev_loss
                print(", Dev lost:", curr_dev_loss)
                if improvment < opts.lr_start_decay:
                    print("... reducing learning rate")
                    optimizer.lr /= opts.lr_decay

                if optimizer.lr < opts.lr_stop:
                    break
                prev_dev_loss = curr_dev_loss
            else:
                print('')
    
    if opts.fname_out_model:
        model = model.to_cpu()
        print('... saving model to', opts.fname_out_model)
        pickle.dump(model, open(opts.fname_out_model, 'wb'), -1)
def CifarAnalysis(folderName=None,n_epoch=1,batchsize = 1000, **kwd):
    id_gpu  = 0

    OutStr = ""
    OutStr += 'GPU: {}\n'.format(id_gpu)
    OutStr += 'Minibatch-size: {}\n'.format(batchsize) 
    OutStr += 'epoch: {}\n'.format(n_epoch) 
    OutStr += 'kwd: {}\n'.format(kwd) 
    OutStr += '' 

    print OutStr

    fOutput = None
    fInfo   = None
    if folderName:
        if not os.path.exists(folderName):
            os.makedirs(folderName)
        fOutput = open(os.path.join(folderName,"output.dat"),"w")
        fInfo   = open(os.path.join(folderName,"info.dat"),"w")
        shutil.copyfile(__file__,os.path.join(folderName,os.path.basename(__file__)))

    if fInfo: fInfo.write(OutStr)

# Prepare dataset
    InDataBatch = []

    data_tr  = np.zeros((50000,3*32*32),dtype=np.float32)
    data_ev  = np.zeros((10000,3*32*32),dtype=np.float32)
    label_tr = np.zeros((50000),dtype=np.int32)
    label_ev = np.zeros((10000),dtype=np.int32)

    for i in range(1,5+1):
        with open("data_cifar10/data_batch_%d"%i,"r") as f:
            tmp = pickle.load(f)
            data_tr [(i-1)*10000:i*10000] = tmp["data"]
            label_tr[(i-1)*10000:i*10000] = tmp["labels"]
    with open("data_cifar10/test_batch","r") as f:
        tmp = pickle.load(f)
        data_ev  [:] = tmp["data"]
        label_ev [:] = tmp["labels"]

## Prep
    print "Normalizing data ..."
    def Normalize(x):
        avg  = np.average(x,axis=1).reshape((len(x),1))
        std  = np.sqrt(np.sum(x*x,axis=1) - np.sum(x,axis=1)).reshape((len(x),1))
        y    = (x - avg) / std
        return y
    data_tr = Normalize(data_tr)
    data_ev = Normalize(data_ev)

    x_tr = data_tr.reshape((len(data_tr),3,32,32))
    x_ev = data_ev.reshape((len(data_ev),3,32,32))
    y_tr = label_tr
    y_ev = label_ev
    N_tr = len(data_tr) # 50000
    N_ev = len(data_ev) # 10000

## Define analisis
    Resume = None
    if "Resume" in kwd:
        Resume = kwd["Resume"]
        del kwd["Resume"]

    model = L.Classifier(ImageProcessNetwork(I_colors=3, I_Xunit=32, I_Yunit=32, F_unit = 10, **kwd))
    if id_gpu >= 0:
        cuda.get_device(id_gpu).use()
        model.to_gpu()
    xp = np if id_gpu < 0 else cuda.cupy

# Setup optimizer
    optimizer = optimizers.Adam()
    optimizer.setup(model)

# Init/Resume
    if Resume:
        print('Load optimizer state from', Resume)
        serializers.load_hdf5(Resume+".state", optimizer)
        serializers.load_hdf5(Resume+".model", model)

# Learning loop
    if fOutput: fOutput.write("epoch,mode,loss,accuracy\n")
    for epoch in six.moves.range(1, n_epoch + 1):
        print 'epoch %d'%epoch 

        # training
        perm = np.random.permutation(N_tr)
        sum_accuracy = 0
        sum_loss = 0
        start = time.time()
        for i in six.moves.range(0, N_tr, batchsize):
            x = chainer.Variable(xp.asarray(x_tr[perm[i:i + batchsize]]))
            t = chainer.Variable(xp.asarray(y_tr[perm[i:i + batchsize]]))

            # Pass the loss function (Classifier defines it) and its arguments
            model.predictor.setTrainMode(True)
            optimizer.update(model, x, t)

            if (epoch == 1 and i == 0) and folderName:
                with open(os.path.join(folderName,'graph.dot'), 'w') as o:
                    g = computational_graph.build_computational_graph(
                        (model.loss, ))
                    o.write(g.dump())
                print 'graph generated' 

            sum_loss += float(model.loss.data) * len(t.data)
            sum_accuracy += float(model.accuracy.data) * len(t.data)
        end = time.time()
        elapsed_time = end - start
        throughput = N_tr / elapsed_time
        print 'train mean loss=%.3f, accuracy=%.1f%%, throughput=%.0f images/sec'%(sum_loss / N_tr, sum_accuracy / N_tr * 100., throughput) 
        if fOutput: fOutput.write("%d,Train,%e,%e\n"%(epoch,sum_loss/N_tr,sum_accuracy/N_tr))

        # evaluation
        perm = np.random.permutation(N_ev)
        sum_accuracy = 0
        sum_loss = 0
        for i in six.moves.range(0, N_ev, batchsize):
            x = chainer.Variable(xp.asarray(x_ev[perm[i:i + batchsize]]),volatile='on')
            t = chainer.Variable(xp.asarray(y_ev[perm[i:i + batchsize]]),volatile='on')
            model.predictor.setTrainMode(False)
            loss = model(x, t)
            sum_loss += float(loss.data) * len(t.data)
            sum_accuracy += float(model.accuracy.data) * len(t.data)

        print 'test  mean loss=%.3f, accuracy=%.1f%%'%(sum_loss / N_ev, sum_accuracy / N_ev * 100, ) 
        if fOutput: fOutput.write("%d,Test,%e,%e\n"%(epoch,sum_loss/N_ev,sum_accuracy/N_ev))

        if folderName and (epoch%10 == 0 or epoch==n_epoch):
            # Save the model and the optimizer
            if epoch == n_epoch:
                myFname = os.path.join(folderName,'mlp_final')
            else:
                myFname = os.path.join(folderName,'mlp_%d'%n_epoch)

            #print 'save the model' 
            serializers.save_hdf5(myFname+".model", model)
            serializers.save_hdf5(myFname+".state", optimizer)

    if fOutput: fOutput.close()
    if fInfo  : fInfo.close()
Exemple #23
0
    x_batch[i] = image
    y_batch[i] = label
    i += 1

    if i == args.batchsize:
        x_data = xp.asarray(x_batch)
        y_data = xp.asarray(y_batch)

        x = chainer.Variable(x_data, volatile=True)
        t = chainer.Variable(y_data, volatile=True)

        loss, accuracy = forward(x, t)

        import chainer.computational_graph as c
        with open('data/graph.dot', 'w') as o:
            o.write(c.build_computational_graph((loss,)).dump())

        accum_loss += float(loss.data) * args.batchsize
        accum_accuracy += float(accuracy.data) * args.batchsize
        del x, t, loss, accuracy

        count += args.batchsize
        print('{} / {}'.format(count, len(dataset)), end='\r', file=sys.stderr)
        sys.stderr.flush()

        i = 0


print('mean loss:     {}'.format(accum_loss / count))
print('mean accuracy: {}'.format(accum_accuracy / count))
    # training
    perm = np.random.permutation(N)
    sum_accuracy = 0
    sum_loss = 0
    net.train = True
    for i in six.moves.range(0, N, batchsize):

        x = chainer.Variable(xp.asarray(x_train[perm[i:i + batchsize]]))
        t = chainer.Variable(xp.asarray(y_train[perm[i:i + batchsize]]))

        # Pass the loss function (Classifier defines it) and its arguments
        optimizer.update(model, x, t)

        if epoch == 1 and i == 0:
            with open('graph.dot', 'w') as o:
                g = computational_graph.build_computational_graph(
                    (model.loss, ), remove_split=True)
                o.write(g.dump())
            print('graph generated')

        sum_loss += float(model.loss.data) * len(t.data)
        sum_accuracy += float(model.accuracy.data) * len(t.data)

    print('train mean loss={}, accuracy={}'.format(
        sum_loss / N, sum_accuracy / N))

    # evaluation
    sum_accuracy = 0
    sum_loss = 0
#    net.train = False
    for i in six.moves.range(0, N_test, batchsize):
        # these volatile='on' but current chainer has bug on batch normalization
 def setUp(self):
     self.x1 = variable.Variable(np.zeros((1, 2)).astype('f'))
     self.x2 = variable.Variable(np.zeros((1, 2)).astype('f'))
     self.y = self.x1 + self.x2
     self.f = self.y.creator
     self.g = c.build_computational_graph((self.y,), remove_variable=True)
 def test_randir(self):
     for rankdir in ['TB', 'BT', 'LR', 'RL']:
         g = c.build_computational_graph((self.y,), rankdir=rankdir)
         self.assertIn('rankdir=%s' % rankdir, g.dump())
 def test_dont_show_name(self):
     g = c.build_computational_graph(
         (self.x1, self.x2, self.y), show_name=False)
     dotfile_content = g.dump()
     for var in [self.x1, self.x2, self.y]:
         self.assertNotIn('label="%s:' % var.name, dotfile_content)
Exemple #28
0
def train(
        iterator, gpu, encoder, decoder, enc_optim, dec_optim, rel_send, rel_rec, edge_types,
        temp, prediction_steps, var, out, benchmark, lr_decay, gamma):
    iter_i = 0
    edge_accuracies = []
    node_mses = []
    nll_train = []
    kl_train = []

    logger = logging.getLogger(__name__)

    while True:
        inputs = iterator.next()
        node_features, edge_labels = dataset.concat_examples(inputs, device=gpu)

        # logits: [batch_size, num_edges, edge_types]
        logits = encoder(node_features, rel_send, rel_rec)  # inverse func. of softmax
        edges = F.gumbel_softmax(logits, tau=temp, axis=2)
        edge_probs = F.softmax(logits, axis=2)
        # edges, edge_probs: [batch_size, num_edges, edge_types]

        if isinstance(decoder, decoders.MLPDecoder):
            output = decoder(
                node_features, edges, rel_rec, rel_send, prediction_steps)
        elif isinstance(decoder, decoders.RNNDecoder):
            output = decoder(
                node_features, edges, rel_rec, rel_send, 100,
                burn_in=True,
                burn_in_steps=args.timesteps - args.prediction_steps)

        target = node_features[:, :, 1:, :]
        num_nodes = node_features.shape[1]

        loss_nll = get_nll_gaussian(output, target, var)
        loss_kl = get_kl_categorical_uniform(edge_probs, num_nodes, edge_types)

        loss = loss_nll + loss_kl

        nll_train.append(float(loss_nll.array))
        kl_train.append(float(loss_kl.array))

        edge_accuracy = get_edge_accuracy(logits.array, edge_labels)
        edge_accuracies.append(edge_accuracy)

        node_mse = float(F.mean_squared_error(output, target).array)
        node_mses.append(node_mse)

        encoder.cleargrads()
        decoder.cleargrads()
        loss.backward()
        enc_optim.update()
        dec_optim.update()

        # Exit after 10 iterations when benchmark mode is ON
        iter_i += 1
        if benchmark:
            put_log(iterator.epoch, np.mean(nll_train), np.mean(kl_train),
                    np.mean(edge_accuracies), np.mean(node_mses))
            if iter_i == 10:
                exit()

        if iterator.is_new_epoch:
            break

    if not os.path.exists(os.path.join(out, 'graph.dot')):
        with open(os.path.join(out, 'graph.dot'), 'w') as o:
            g = computational_graph.build_computational_graph([loss])
            o.write(g.dump())

    if iterator.is_new_epoch:
        put_log(iterator.epoch, np.mean(nll_train), np.mean(kl_train), np.mean(edge_accuracies), np.mean(node_mses))
        serializers.save_npz(os.path.join(out, 'encoder_epoch-{}.npz'.format(iterator.epoch)), encoder)
        serializers.save_npz(os.path.join(out, 'decoder_epoch-{}.npz'.format(iterator.epoch)), decoder)
        serializers.save_npz(os.path.join(out, 'enc_optim_epoch-{}.npz'.format(iterator.epoch)), enc_optim)
        serializers.save_npz(os.path.join(out, 'dec_optim_epoch-{}.npz'.format(iterator.epoch)), dec_optim)

        if iterator.epoch % lr_decay == 0:
            enc_optim.alpha *= gamma
            dec_optim.alpha *= gamma
            logger.info('alpha of enc_optim: {}'.format(enc_optim.alpha))
            logger.info('alpha of dec_optim: {}'.format(dec_optim.alpha))
def save_model_graph(loss, filepath, remove_split=False):
   with open(filepath, 'w') as o:
      o.write(c.build_computational_graph((loss,), remove_split).dump())
def dump_comp_graph(filename, vs):
    g = C.build_computational_graph(vs)
    with open(filename, 'w') as o:
        o.write(g.dump())