def check_forward(self, x1_data, x2_data, x3_data): xp = self.link.xp x1 = chainer.Variable(x1_data) if self.input_variable else x1_data h1 = self.link(x1) with cuda.get_device(x1_data): c0 = chainer.Variable(xp.zeros((len(self.x1), self.out_size), dtype=self.x1.dtype)) c1_expect, h1_expect = functions.lstm(c0, self.link.upward(x1)) testing.assert_allclose(h1.data, h1_expect.data) testing.assert_allclose(self.link.h.data, h1_expect.data) testing.assert_allclose(self.link.c.data, c1_expect.data) batch = len(x2_data) x2 = chainer.Variable(x2_data) if self.input_variable else x2_data h1_in, h1_rest = functions.split_axis( self.link.h.data, [batch], axis=0) y2 = self.link(x2) with cuda.get_device(x1): c2_expect, y2_expect = \ functions.lstm(c1_expect, self.link.upward(x2) + self.link.lateral(h1_in)) testing.assert_allclose(y2.data, y2_expect.data) testing.assert_allclose(self.link.h.data[:batch], y2_expect.data) testing.assert_allclose(self.link.h.data[batch:], h1_rest.data) x3 = chainer.Variable(x3_data) if self.input_variable else x3_data h2_rest = self.link.h y3 = self.link(x3) c3_expect, y3_expect = \ functions.lstm(c2_expect, self.link.upward(x3)) testing.assert_allclose(y3.data, y3_expect.data) testing.assert_allclose(self.link.h.data, h2_rest.data)
def _finetune(self, X, y): utils.disp('*** finetune ***', self.verbose) # construct model and setup optimizer params = {'l{}'.format(layer + 1): dA.encoder for layer, dA in enumerate(self.dAs)} params.update({'l{}'.format(len(self.dAs) + 1): F.Linear(self.dAs[-1].n_hidden, self.n_output)}) self.model = FunctionSet(**params) self.optimizer.setup(self.model) if self.gpu >= 0: cuda.get_device(self.gpu).use() self.model.to_gpu() xp = cuda.cupy if self.gpu >= 0 else np n = len(X) for epoch in range(self.n_epoch_finetune): utils.disp('epoch: {}'.format(epoch + 1), self.verbose) perm = np.random.permutation(n) sum_loss = 0 for i in range(0, n, self.batch_size): X_batch = xp.asarray(X[perm[i: i + self.batch_size]]) y_batch = xp.asarray(y[perm[i: i + self.batch_size]]) self.optimizer.zero_grads() y_var = self._forward(X_batch) loss = self._loss_func(y_var, Variable(y_batch)) loss.backward() self.optimizer.update() sum_loss += float(loss.data) * len(X_batch) utils.disp('fine tune mean loss={}'.format(sum_loss / n), self.verbose)
def get_model_optimizer(result_dir, args): model_fn = os.path.basename(args.model) model_name = model_fn.split('.')[0] module = imp.load_source(model_fn.split('.')[0], args.model) Net = getattr(module, model_name) dst = '%s/%s' % (result_dir, model_fn) if not os.path.exists(dst): shutil.copy(args.model, dst) dst = '%s/%s' % (result_dir, os.path.basename(__file__)) if not os.path.exists(dst): shutil.copy(__file__, dst) # prepare model model = Net() if args.gpu >= 0: cuda.get_device(args.gpu).use() if args.restart_from is not None: model = pickle.load(open(args.restart_from, 'rb')) if args.gpu >= 0: model.to_gpu() # prepare optimizer if args.opt == 'AdaGrad': optimizer = optimizers.AdaGrad(lr=0.0005) elif args.opt == 'MomentumSGD': optimizer = optimizers.MomentumSGD(lr=0.0005, momentum=0.9) elif args.opt == 'Adam': optimizer = optimizers.Adam() else: raise Exception('No optimizer is selected') optimizer.setup(model) return model, optimizer
def inspect(image_path, mean, model_path, label, network, gpu=-1): network = network.split(os.sep)[-1] model_name = re.sub(r"\.py$", "", network) model_module = load_module(os.path.dirname(model_path), model_name) mean_image = pickle.load(open(mean, 'rb')) model = model_module.Network() serializers.load_hdf5(model_path, model) if gpu >= 0: cuda.check_cuda_available() cuda.get_device(gpu).use() model.to_gpu() cropwidth = 256 - model.insize img = read_image(image_path, model, mean_image, cropwidth) x = np.ndarray((1, 3, model.insize, model.insize), dtype=np.float32) x[0] = img if gpu >= 0: x = cuda.to_gpu(x) score = model.predict(x) score = cuda.to_cpu(score.data) categories = np.loadtxt(label, str, delimiter="\t") top_k = 20 prediction = zip(score[0].tolist(), categories) prediction.sort(cmp=lambda x, y:cmp(x[0], y[0]), reverse=True) ret = [] for rank, (score, name) in enumerate(prediction[:top_k], start=1): ret.append({"rank": rank, "name": name, "score": "{0:4.1f}%".format(score*100)}) return ret
def __init__(self, gpu, model, model_type, out_dim): self.gpu = gpu self.model = 'bvlc_alexnet.caffemodel' self.model_type = 'alexnet' self.batchsize = 1 self.out_dim = out_dim if self.gpu >= 0: cuda.check_cuda_available() print('Loading Caffe model file %s...' % self.model, file = sys.stderr) self.func = caffe.CaffeFunction(self.model) print('Loaded', file=sys.stderr) if self.gpu >= 0: cuda.get_device(self.gpu).use() self.func.to_gpu() if self.model_type == 'alexnet': self.in_size = 227 mean_image = np.load('ilsvrc_2012_mean.npy') del self.func.layers[15:23] self.outname = 'pool5' #del self.func.layers[13:23] #self.outname = 'conv5' cropwidth = 256 - self.in_size start = cropwidth // 2 stop = start + self.in_size self.mean_image = mean_image[:, start:stop, start:stop].copy()
def main(args): if args.gpu is not None: cuda.get_device(args.gpu).use() # load model vae = Vae.load(args.model) label_dim = vae.encoder.rnn.feature_dim # load vocabulary model_base_path = os.path.dirname(args.model) vocab = vocabulary.Vocab.load(os.path.join(model_base_path, 'vocab')) while True: try: # process input in_str = raw_input('> ').decode('utf-8') es = in_str.split() label_id = int(es[0]) ids = vocab.convert_ids(es[1:]) # create input xs = map(lambda d: Variable(np.asarray([d], dtype=np.int32), volatile='on'), ids) label_in = _create_label_var([label_id], label_dim) label_out = _create_label_var(range(label_dim), label_dim) score_ids, debug_info = vae.generate(xs, label_in, label_out, no_var=args.no_var, sample=args.sample, temp=args.temp, max_len=args.max_len) mu = debug_info['mu'] ln_var = debug_info['ln_var'] z = debug_info['z'] for ids, score in score_ids: print u'{}\t{}'.format(score, u' '.join(vocab.convert_words(ids))).encode('utf-8') except Exception as ex: print 'Usage: <label ID> <space-separated tokens>'
def generate(img): parser = argparse.ArgumentParser(description='Real-time style transfer image generator') #parser.add_argument('input') args = parser.parse_args() model = FastStyleNet() serializers.load_npz('models/seurat.model', model) if -1 >= 0: cuda.get_device(-1).use() model.to_gpu() xp = np if -1 < 0 else cuda.cupy start = time.time() image = xp.asarray(Image.open(img).convert('RGB'), dtype=xp.float32).transpose(2, 0, 1) image = image.reshape((1,) + image.shape) x = Variable(image) y = model(x) result = cuda.to_cpu(y.data) result = result.transpose(0, 2, 3, 1) result = result.reshape((result.shape[1:])) result = np.uint8(result) print(time.time() - start, 'sec') # Image.fromarray(result).save(args.out) return Image.fromarray(result)
def set_library(args): if args.gpu >= 0: XP.__lib = cuda.cupy cuda.get_device(args.gpu).use() else: XP.__lib = numpy XP.train = args.mode == 'train'
def __call__(self, x, test=False, finetune=False): """Invokes the forward propagation of BatchNormalization. BatchNormalization accepts additional arguments, which controls three different running mode. Args: x (Variable): Input variable. test (bool): If ``True``, BatchNormalization runs in testing mode; it normalizes the input using pre-computed statistics. finetune (bool): If ``finetune`` is ``True`` and ``test`` is ``False``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. If ``test`` is ``False``, then BatchNormalization runs in training mode; it computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. """ if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device(self._device_id): gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype), volatile='auto') if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device(self._device_id): beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype), volatile='auto') # Var is always ones with cuda.get_device(self._device_id): self.one_var = self.xp.ones(self.avg_mean.shape, dtype=x.dtype) if not test: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalizationFunction( self.eps, self.avg_mean, self.avg_var, True, decay, self.use_cudnn) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean, volatile='auto') #var = variable.Variable(self.avg_var, volatile='auto') var = variable.Variable(self.one_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, self.use_cudnn) return ret
def __init__(self, vocab_size, embed_size, hidden_size, choose_model, use_gpu, gpu_id): # gpu Setting model = Alex_RNNLM(hidden_size) if choose_model == "Alex_RNNLM": model = Alex_RNNLM(hidden_size) if choose_model == "AlexBn_RNNLM": model = AlexBn_RNNLM(hidden_size) if use_gpu: cuda.get_device(gpu_id).use() model.to_gpu() # Setting Model super(EncoderDecoderAttention, self).__init__( enc=model, im1 = links.Linear(IM_SIZE, RESIZE_IM_SIZE), im2 = links.Linear(IM_SIZE, RESIZE_IM_SIZE), im3 = links.Linear(IM_SIZE, RESIZE_IM_SIZE), att=Attention(hidden_size, RESIZE_IM_SIZE), outay = links.Linear(hidden_size, hidden_size), dec=Decoder(vocab_size, embed_size, hidden_size), ) self.vocab_size = vocab_size self.embed_size = embed_size self.hidden_size = hidden_size self.common_function = CommonFunction() self.use_gpu = use_gpu self.gpu_id = gpu_id self.choose_model = choose_model self.__set_gpu()
def main(description, gpu, output): logging.basicConfig(level=logging.INFO) logging.info('fetch MNIST dataset') mnist = fetch_mldata(description) mnist.data = mnist.data.astype(numpy.float32) mnist.data /= 255 mnist.target = mnist.target.astype(numpy.int32) data_train, data_test, target_train, target_test = train_test_split(mnist.data, mnist.target) data = data_train, data_test target = target_train, target_test start_time = time.time() if gpu >= 0: cuda.check_cuda_available() cuda.get_device(gpu).use() logging.info("Using gpu device {}".format(gpu)) else: logging.info("Not using gpu device") mlp = MLP(data=data, target=target, gpu=gpu) mlp.train_and_test(n_epoch=1) end_time = time.time() logging.info("time = {} min".format((end_time - start_time) / 60.0)) logging.info('saving trained mlp into {}'.format(output)) with open(output, 'wb') as fp: pickle.dump(mlp, fp)
def __init__(self, vocab_size, embed_size, hidden_size, choose_model, use_gpu, gpu_id): # gpu Setting model = Alex_RNNLM(hidden_size) if choose_model == "Alex_RNNLM": model = Alex_RNNLM(hidden_size) if choose_model == "AlexBn_RNNLM": model = AlexBn_RNNLM(hidden_size) if use_gpu: cuda.get_device(gpu_id).use() model.to_gpu() # Setting Model super(EncoderDecoder, self).__init__( enc=model, dec=Decoder(vocab_size, embed_size, hidden_size), ) self.vocab_size = vocab_size self.embed_size = embed_size self.hidden_size = hidden_size self.common_function = CommonFunction() self.use_gpu = use_gpu self.gpu_id = gpu_id self.choose_model = choose_model self.__set_gpu()
def __init__(self, model, GPU_ID=0, cont=False): self.model = model self.cont = cont # moving model to default GPU cuda.get_device(GPU_ID).use() model.to_gpu()
def set_library(args): if args.use_gpu: XP.__lib = cuda.cupy cuda.get_device(args.gpu_device).use() else: XP.__lib = numpy XP.__train == args.mode == 'train'
def main(args): # load model encdec = util.load_model(args.model) if args.gpu is not None: cuda.get_device(args.gpu).use() encdec.to_gpu() # load data model_base_path = os.path.dirname(args.model) if os.path.exists(os.path.join(model_base_path, 'vocab_src')): vocab_src_path = os.path.join(model_base_path, 'vocab_src') vocab_trg_path = os.path.join(model_base_path, 'vocab_trg') vocab_src = Vocab.load(vocab_src_path) vocab_trg = Vocab.load(vocab_trg_path) else: vocab_path = os.path.join(model_base_path, 'vocab') vocab = Vocab.load(vocab_path) vocab_src = vocab vocab_trg = vocab data = util.load_sentences(args.data, vocab_src) is_beam_search = args.beam > 1 # create batches if is_beam_search: batch_size = 1 else: batch_size = args.batch batches = util.create_batches_src(data, batch_size, args.bucket_step) # generate res = defaultdict(list) for idx_lst, xs_data in batches: if args.gpu is not None: xs_data = cuda.to_gpu(xs_data) xs = procedure.create_variables(xs_data, volatile='on') if is_beam_search: decoded = encdec.generate_beam(xs, beam_size=args.beam, max_len=args.max_len) if not args.n_best: # only show best result decoded = decoded[:1] assert len(idx_lst) == 1 idx = idx_lst[0] for ids in decoded: words = map(vocab_trg.get_word, ids) res[idx].append(words) else: ids_batch, ys, ws = encdec.generate(xs, max_len=args.max_len, sample=args.sample, temp=args.temp) assert len(idx_lst) == len(ids_batch) for idx, ids in zip(idx_lst, ids_batch): words = map(vocab_trg.get_word, ids) res[idx].append(words) for idx, (idx2, cands) in enumerate(sorted(res.items(), key=lambda k_v: k_v[0])): for words in cands: assert idx == idx2 print(' '.join(words))
def __call__(self, c, h, x): """Returns new cell state and updated output of LSTM. Args: c (~chainer.Variable): Cell states of LSTM units. h (~chainer.Variable): Output at the previous time step. x (~chainer.Variable): A new batch from the input sequence. Returns: tuple of ~chainer.Variable: Returns ``(c_new, h_new)``, where ``c_new`` represents new cell state, and ``h_new`` is updated output of LSTM units. """ if self.upward.has_uninitialized_params: in_size = x.size // x.shape[0] with cuda.get_device(self._device_id): self.upward._initialize_params(in_size) self._initialize_params() lstm_in = self.upward(x) if h is not None: lstm_in += self.lateral(h) if c is None: xp = self.xp with cuda.get_device(self._device_id): c = variable.Variable( xp.zeros((x.shape[0], self.state_size), dtype=x.dtype), volatile='auto') return lstm.lstm(c, lstm_in)
def load_states(stage_cnt, joint_idx): ''' Load model, optimizer, and losses ''' _, model, optimizer, _, _ = setup_initial_states(stage_cnt) modif = create_modifier(stage_cnt, joint_idx) # Alexnet model filename = settings.RESUME_MODEL % modif logger.info('Load model from %s', filename) serializers.load_npz(filename, model) if settings.GPU >= 0: # GPU setup cuda.get_device(settings.GPU).use() model.to_gpu() # Optimizer optimizer.setup(model) filename = settings.RESUME_OPTIMIZER % modif logger.info('Load optimizer from %s', filename) serializers.load_npz(filename, optimizer) # Losses filename = settings.RESUME_LOSS % modif logger.info('Load loss history from %s', filename) loss_data = np.load(filename) train_losses = loss_data['train'].tolist() test_losses = loss_data['test'].tolist() assert(len(train_losses) == len(test_losses)) # Epoch count epoch_cnt = len(train_losses) logger.info('Resume from epoch %d', epoch_cnt) return epoch_cnt, model, optimizer, train_losses, test_losses
def test_forward_gpu_multi(self): with cuda.get_device(0): self.link.to_gpu() c = cuda.to_gpu(self.c) h = cuda.to_gpu(self.h) x = cuda.to_gpu(self.x) with cuda.get_device(1): self.check_forward(c, h, x)
def test_forward_gpu_multi(self): with cuda.get_device(0): self.link.to_gpu() x1 = cuda.to_gpu(self.x1) x2 = cuda.to_gpu(self.x2) x3 = cuda.to_gpu(self.x3) with cuda.get_device(1): self.check_forward(x1, x2, x3)
def test_addgrad_gpu_to_another_gpu(self): cp = cuda.cupy with cuda.get_device(1): a = cp.full(3, 10, dtype=np.float32) with cuda.get_device(0): b = cp.full(3, 20, dtype=np.float32) c = cp.full(3, 30, dtype=np.float32) self.check_addgrad(a, b, c)
def test_copydata_gpu_to_another_gpu(self): cp = cuda.cupy with cuda.get_device(0): data1 = cp.zeros(3, dtype=np.float32) expect = cp.ones(3, dtype=np.float32) with cuda.get_device(1): data2 = cp.ones(3, dtype=np.float32) self.check_copydata(data1, data2, expect)
def set_library(args): if args.gpu >= 0: XP.__lib = cuda.cupy cuda.get_device(args.gpu).use() XP.xp = XP.__lib else: XP.__lib = np XP.xp = XP.__lib
def setup_gpu(self): if self.gpu >= 0: cuda.check_cuda_available() cuda.get_device(self.gpu).use() self.model.to_gpu() self.mod = cuda.cupy else: self.mod = np
def load_models(self, models): models = models.split(',') availiable_path = 'models_availiable' update_path = 'models_upload' all_models = 'models' models_availiable = os.listdir(availiable_path) models_upload = os.listdir(update_path) self.models = {m.split('.')[0].split('_')[0]:availiable_path +'/'+ m for m in models_availiable} for m in models_upload: model_name = m.split('.')[0].split('_')[0] if not self.models.has_key(model_name): cmd = 'mv {update}/{update_name} {availiable}/'.format( update=update_path, update_name=m, availiable=availiable_path) (status, output) = commands.getstatusoutput(cmd) self.models[model_name] = availiable_path + '/' + m else: cmd = 'mv {availiable} {all_models}'.format( availiable=self.models[model_name], all_models=all_models) (status, output) = commands.getstatusoutput(cmd) cmd = 'mv {update}/{update_name} {availiable}/'.format( update=update_path, update_name=m, availiable=availiable_path) (status, output) = commands.getstatusoutput(cmd) self.models[model_name] = availiable_path + '/' + m # init denoise file_name = 'denoise.txt' if os.path.exists(file_name): with open(file_name, 'r') as pf: for line in pf.readlines(): line = line.strip().split(' ') self.denoise[line[0].strip()] = float(line[-1].strip()) # init model for model_name,value in self.models.items(): # model_name = m.split('/')[-1].split('.')[0] handle_model = FastStyleNet() serializers.load_npz(value, handle_model) if self.args.gpu >= 0: cuda.get_device(self.args.gpu).use() handle_model.to_gpu() self.models[model_name] = handle_model load_all_models = ', '.join(self.models.keys()) logging.info('loading models : ' + load_all_models) logging.info('load success')
def __init__(self, discretize=10, gpu=-1): self.discretize = discretize self.gpu = gpu if self.gpu >= 0: from chainer import cuda self.xp = cuda.cupy cuda.get_device(self.gpu).use() else: self.xp = np
def __init__(self, meanpath, model, gpu=-1): self.mean = loader.load_mean(meanpath) self.model = model self.gpu = gpu self.insize = model.insize if gpu >= 0: cuda.check_cuda_available() cuda.get_device(gpu).use() self.model.to_gpu()
def test_addgrad_gpu_to_another_gpu_none_dst_dev1(self): cp = cuda.cupy with cuda.get_device(1): a = cp.full(3, 20, dtype=np.float32) with cuda.get_device(0): b = cp.full(3, 10, dtype=np.float32) c = cp.full(3, 20, dtype=np.float32) with cuda.get_device(1): self.check_addgrad(a, b, c, clear_dst_grad=True)
def __init__(self): super(MLP, self).__init__( l1=L.Linear(784, 100), l2=L.Linear(100, 100), l3=L.Linear(100, 10), ) if cuda.available: cuda.get_device(0).use() self.to_gpu()
def set_net(self, net): self.source_net = deepcopy(net) self.target_net = deepcopy(net) if self.gpu: cuda.get_device(0).use() self.source_net.to_gpu() self.target_net.to_gpu() self.optimizer.setup(self.source_net) self.target_net.train = False
def test_zerograds_fill_multi_gpu(self): cupy = cuda.cupy with cuda.get_device(1): a = chainer.Variable(cupy.empty(3, dtype=np.float32)) a.grad = cupy.empty_like(a.data) a.zerograd() self.assertEqual(int(a.grad.device), 1) with cuda.get_device(1): g_expect = cupy.zeros_like(a.data) cupy.testing.assert_array_equal(a.grad, g_expect)
def __init__(self, model, actor_optimizer, critic_optimizer, replay_buffer, gamma, explorer, gpu=None, replay_start_size=50000, minibatch_size=32, update_interval=1, target_update_interval=10000, phi=lambda x: x, target_update_method='hard', soft_update_tau=1e-2, n_times_update=1, average_q_decay=0.999, average_loss_decay=0.99, episodic_update=False, episodic_update_len=None, logger=getLogger(__name__), batch_states=batch_states): self.model = model if gpu is not None and gpu >= 0: cuda.get_device(gpu).use() self.model.to_gpu(device=gpu) self.xp = self.model.xp self.replay_buffer = replay_buffer self.gamma = gamma self.explorer = explorer self.gpu = gpu self.target_update_interval = target_update_interval self.phi = phi self.target_update_method = target_update_method self.soft_update_tau = soft_update_tau self.logger = logger self.average_q_decay = average_q_decay self.average_loss_decay = average_loss_decay self.actor_optimizer = actor_optimizer self.critic_optimizer = critic_optimizer if episodic_update: update_func = self.update_from_episodes else: update_func = self.update self.replay_updater = ReplayUpdater( replay_buffer=replay_buffer, update_func=update_func, batchsize=minibatch_size, episodic_update=episodic_update, episodic_update_len=episodic_update_len, n_times_update=n_times_update, replay_start_size=replay_start_size, update_interval=update_interval, ) self.batch_states = batch_states self.t = 0 self.last_state = None self.last_action = None self.target_model = copy.deepcopy(self.model) disable_train(self.target_model['q_function']) disable_train(self.target_model['policy']) self.average_q = 0 self.average_actor_loss = 0.0 self.average_critic_loss = 0.0 # Aliases for convenience self.q_function = self.model['q_function'] self.policy = self.model['policy'] self.target_q_function = self.target_model['q_function'] self.target_policy = self.target_model['policy'] self.sync_target_network()
def test_get_dummy_device_for_empty_array(self): x = cuda.cupy.array([]).reshape((0, 10)) self.assertIs(cuda.get_device(x), cuda.DummyDevice)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--fcn16s', required=True) parser.add_argument('--gpu', type=int, default=0) parser.add_argument('--out', required=True) parser.add_argument('--dataset', default='v2', choices=['v1', 'v2']) args = parser.parse_args() fcn16s_path = args.fcn16s gpu = args.gpu out = args.out if args.dataset == 'v1': dataset_class = datasets.APC2016DatasetV1 else: dataset_class = datasets.APC2016DatasetV2 if not osp.exists(out): os.makedirs(out) # 1. dataset dataset_train = dataset_class('train') dataset_val = dataset_class('val') iter_train = chainer.iterators.SerialIterator(dataset_train, batch_size=1) iter_val = chainer.iterators.SerialIterator(dataset_val, batch_size=1, repeat=False, shuffle=False) # 2. model n_class = len(dataset_train.label_names) fcn16s = fcn.models.FCN16s(n_class=n_class) chainer.serializers.load_hdf5(fcn16s_path, fcn16s) model = fcn.models.FCN8s(n_class=n_class) model.train = True fcn.utils.copy_chainermodel(fcn16s, model) if gpu >= 0: cuda.get_device(gpu).use() model.to_gpu() # 3. optimizer optimizer = chainer.optimizers.Adam(alpha=1e-5) optimizer.setup(model) # training loop trainer = fcn.Trainer( device=gpu, model=model, optimizer=optimizer, iter_train=iter_train, iter_val=iter_val, out=out, ) trainer.train( max_iter=150000, interval_eval=5000, )
def model_to_gpu(model, gpu_num): cuda.get_device(gpu_num).use() model.to_gpu() model.init_state()
def main(): args = parse_args() args.seed = init_rand(seed=args.seed) _, log_file_exist = initialize_logging( logging_dir_path=args.save_dir, logging_file_name=args.logging_file_name, script_args=args, log_packages=args.log_packages, log_pip_packages=args.log_pip_packages) num_gpus = args.num_gpus if num_gpus > 0: cuda.get_device(0).use() batch_size = args.batch_size net = prepare_model(model_name=args.model, use_pretrained=args.use_pretrained, pretrained_model_file_path=args.resume.strip(), num_gpus=num_gpus) num_classes = net.classes if hasattr(net, 'classes') else 1000 input_image_size = net.in_size[0] if hasattr( net, 'in_size') else args.input_size train_iter, val_iter = get_data_iterators( data_dir=args.data_dir, batch_size=batch_size, num_workers=args.num_workers, num_classes=num_classes, input_image_size=input_image_size, resize_inv_factor=args.resize_inv_factor) trainer = prepare_trainer(net=net, optimizer_name=args.optimizer_name, lr=args.lr, momentum=args.momentum, num_epochs=args.num_epochs, train_iter=train_iter, val_iter=val_iter, logging_dir_path=args.save_dir, num_gpus=num_gpus) # if args.save_dir and args.save_interval: # lp_saver = TrainLogParamSaver( # checkpoint_file_name_prefix='imagenet_{}'.format(args.model), # last_checkpoint_file_name_suffix="last", # best_checkpoint_file_name_suffix=None, # last_checkpoint_dir_path=args.save_dir, # best_checkpoint_dir_path=None, # last_checkpoint_file_count=2, # best_checkpoint_file_count=2, # checkpoint_file_save_callback=save_params, # checkpoint_file_exts=['.npz', '.states'], # save_interval=args.save_interval, # num_epochs=args.num_epochs, # param_names=['Val.Top1', 'Train.Top1', 'Val.Top5', 'Train.Loss', 'LR'], # acc_ind=2, # # bigger=[True], # # mask=None, # score_log_file_path=os.path.join(args.save_dir, 'score.log'), # score_log_attempt_value=args.attempt, # best_map_log_file_path=os.path.join(args.save_dir, 'best_map.log')) # else: # lp_saver = None trainer.run()
def main(args): source_dataset, target_dataset, vocab, vocab_inv = read_data_and_vocab( args.source_train, args.target_train, args.source_dev, args.target_dev, args.source_test, args.target_test, reverse_source=True) save_vocab(args.model_dir, vocab, vocab_inv) source_dataset_train, source_dataset_dev, source_dataset_test = source_dataset target_dataset_train, target_dataset_dev, target_dataset_test = target_dataset vocab_source, vocab_target = vocab vocab_inv_source, vocab_inv_target = vocab_inv # split into buckets source_buckets_train, target_buckets_train = make_buckets( source_dataset_train, target_dataset_train) if args.buckets_slice is not None: source_buckets_train = source_buckets_train[:args.buckets_slice + 1] target_buckets_train = target_buckets_train[:args.buckets_slice + 1] # development dataset source_buckets_dev = None if len(source_dataset_dev) > 0: source_buckets_dev, target_buckets_dev = make_buckets( source_dataset_dev, target_dataset_dev) if args.buckets_slice is not None: source_buckets_dev = source_buckets_dev[:args.buckets_slice + 1] target_buckets_dev = target_buckets_dev[:args.buckets_slice + 1] # test dataset source_buckets_test = None if len(source_dataset_test) > 0: source_buckets_test, target_buckets_test = make_buckets( source_dataset_test, target_dataset_test) if args.buckets_slice is not None: source_buckets_test = source_buckets_test[:args.buckets_slice + 1] target_buckets_test = target_buckets_test[:args.buckets_slice + 1] # show log dump_dataset( source_dataset, vocab, (source_buckets_train, source_buckets_dev, source_buckets_test)) # to maintain equilibrium required_interations = [] for data in source_buckets_train: itr = len(data) // args.batchsize + 1 required_interations.append(itr) total_iterations = sum(required_interations) buckets_distribution = np.asarray(required_interations, dtype=float) / total_iterations # init model = load_model(args.model_dir) if model is None: model = seq2seq(len(vocab_source), len(vocab_target), args.ndim_embedding, args.ndim_h, args.num_layers, pooling=args.pooling, dropout=args.dropout, zoneout=args.zoneout, weightnorm=args.weightnorm, wgain=args.wgain, densely_connected=args.densely_connected, attention=args.attention) if args.gpu_device >= 0: cuda.get_device(args.gpu_device).use() model.to_gpu() # setup an optimizer optimizer = get_optimizer(args.optimizer, args.learning_rate, args.momentum) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(args.grad_clip)) optimizer.add_hook(chainer.optimizer.WeightDecay(args.weight_decay)) final_learning_rate = 1e-5 total_time = 0 indices_train = [] for bucket_idx, bucket in enumerate(source_buckets_train): indices = np.arange(len(bucket)) np.random.shuffle(indices) indices_train.append(indices) def mean(l): return sum(l) / len(l) # training for epoch in range(1, args.epoch + 1): print("Epoch", epoch) start_time = time.time() with chainer.using_config("train", True): for itr in range(total_iterations): bucket_idx = int( np.random.choice(np.arange(len(source_buckets_train)), size=1, p=buckets_distribution)) source_bucket = source_buckets_train[bucket_idx] target_bucket = target_buckets_train[bucket_idx] # sample minibatch source_batch = source_bucket[:args.batchsize] target_batch = target_bucket[:args.batchsize] skip_mask = source_batch != ID_PAD target_batch_input, target_batch_output = make_source_target_pair( target_batch) # to gpu if args.gpu_device >= 0: skip_mask = cuda.to_gpu(skip_mask) source_batch = cuda.to_gpu(source_batch) target_batch_input = cuda.to_gpu(target_batch_input) target_batch_output = cuda.to_gpu(target_batch_output) # compute loss model.reset_state() if args.attention: last_hidden_states, last_layer_outputs = model.encode( source_batch, skip_mask) y_batch = model.decode(target_batch_input, last_hidden_states, last_layer_outputs, skip_mask) else: last_hidden_states = model.encode(source_batch, skip_mask) y_batch = model.decode(target_batch_input, last_hidden_states) loss = softmax_cross_entropy(y_batch, target_batch_output, ignore_label=ID_PAD) # update parameters optimizer.update(lossfun=lambda: loss) # show log printr("iteration {}/{}".format(itr + 1, total_iterations)) source_buckets_train[bucket_idx] = np.roll(source_bucket, -args.batchsize, axis=0) # shift target_buckets_train[bucket_idx] = np.roll(target_bucket, -args.batchsize, axis=0) # shift # shuffle for bucket_idx in range(len(source_buckets_train)): indices = indices_train[bucket_idx] np.random.shuffle(indices) source_buckets_train[bucket_idx] = source_buckets_train[ bucket_idx][indices] target_buckets_train[bucket_idx] = target_buckets_train[ bucket_idx][indices] # serialize save_model(args.model_dir, model) # clear console printr("") # show log with chainer.using_config("train", False): if epoch % args.interval == 0: printb("translate (train)") dump_random_source_target_translation(model, source_buckets_train, target_buckets_train, vocab_inv_source, vocab_inv_target, num_translate=5, beam_width=1) if source_buckets_dev is not None: printb("translate (dev)") dump_random_source_target_translation(model, source_buckets_dev, target_buckets_dev, vocab_inv_source, vocab_inv_target, num_translate=5, beam_width=1) if source_buckets_dev is not None: printb("WER (dev)") wer_dev = compute_error_rate_buckets(model, source_buckets_dev, target_buckets_dev, len(vocab_inv_target), beam_width=1) print(mean(wer_dev), wer_dev) elapsed_time = (time.time() - start_time) / 60. total_time += elapsed_time print("done in {} min, lr = {:.4f}, total {} min".format( int(elapsed_time), get_current_learning_rate(optimizer), int(total_time))) # decay learning rate decay_learning_rate(optimizer, args.lr_decay_factor, final_learning_rate)
def main(): logging.basicConfig( format='%(asctime)s : %(threadName)s : %(levelname)s : %(message)s', level=logging.INFO) import argparse parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', default=-1, type=int, help='GPU ID (negative value indicates CPU)') parser.add_argument('--batchsize', dest='batchsize', type=int, default=32, help='learning minibatch size') parser.add_argument('--batchsize_semi', dest='batchsize_semi', type=int, default=64, help='learning minibatch size') parser.add_argument('--n_epoch', dest='n_epoch', type=int, default=30, help='n_epoch') parser.add_argument('--pretrained_model', dest='pretrained_model', type=str, default='', help='pretrained_model') parser.add_argument('--use_unlabled_to_vocab', dest='use_unlabled_to_vocab', type=int, default=1, help='use_unlabled_to_vocab') parser.add_argument('--use_rational', dest='use_rational', type=int, default=0, help='use_rational') parser.add_argument('--save_name', dest='save_name', type=str, default='sentiment_model', help='save_name') parser.add_argument('--n_layers', dest='n_layers', type=int, default=1, help='n_layers') parser.add_argument('--alpha', dest='alpha', type=float, default=0.001, help='alpha') parser.add_argument('--alpha_decay', dest='alpha_decay', type=float, default=0.0, help='alpha_decay') parser.add_argument('--clip', dest='clip', type=float, default=5.0, help='clip') parser.add_argument('--debug_mode', dest='debug_mode', type=int, default=0, help='debug_mode') parser.add_argument('--use_exp_decay', dest='use_exp_decay', type=int, default=1, help='use_exp_decay') parser.add_argument('--load_trained_lstm', dest='load_trained_lstm', type=str, default='', help='load_trained_lstm') parser.add_argument('--freeze_word_emb', dest='freeze_word_emb', type=int, default=0, help='freeze_word_emb') parser.add_argument('--dropout', dest='dropout', type=float, default=0.50, help='dropout') parser.add_argument('--use_adv', dest='use_adv', type=int, default=0, help='use_adv') parser.add_argument('--xi_var', dest='xi_var', type=float, default=1.0, help='xi_var') parser.add_argument('--xi_var_first', dest='xi_var_first', type=float, default=1.0, help='xi_var_first') parser.add_argument('--lower', dest='lower', type=int, default=1, help='lower') parser.add_argument('--nl_factor', dest='nl_factor', type=float, default=1.0, help='nl_factor') parser.add_argument('--min_count', dest='min_count', type=int, default=1, help='min_count') parser.add_argument('--ignore_unk', dest='ignore_unk', type=int, default=0, help='ignore_unk') parser.add_argument('--use_semi_data', dest='use_semi_data', type=int, default=0, help='use_semi_data') parser.add_argument('--add_labeld_to_unlabel', dest='add_labeld_to_unlabel', type=int, default=1, help='add_labeld_to_unlabel') parser.add_argument('--norm_sentence_level', dest='norm_sentence_level', type=int, default=1, help='norm_sentence_level') parser.add_argument('--dataset', default='imdb', choices=['imdb', 'elec', 'rotten', 'dbpedia', 'rcv1']) parser.add_argument('--eval', dest='eval', type=int, default=0, help='eval') parser.add_argument('--emb_dim', dest='emb_dim', type=int, default=256, help='emb_dim') parser.add_argument('--hidden_dim', dest='hidden_dim', type=int, default=1024, help='hidden_dim') parser.add_argument('--hidden_cls_dim', dest='hidden_cls_dim', type=int, default=30, help='hidden_cls_dim') parser.add_argument('--adaptive_softmax', dest='adaptive_softmax', type=int, default=1, help='adaptive_softmax') parser.add_argument('--random_seed', dest='random_seed', type=int, default=1234, help='random_seed') parser.add_argument('--n_class', dest='n_class', type=int, default=2, help='n_class') parser.add_argument('--word_only', dest='word_only', type=int, default=0, help='word_only') # iVAT parser.add_argument('--use_attn_d', dest='use_attn_d', type=int, default=0, help='use_attn_d') parser.add_argument('--nn_k', dest='nn_k', type=int, default=10, help='nn_k') parser.add_argument('--nn_k_offset', dest='nn_k_offset', type=int, default=1, help='nn_k_offset') parser.add_argument('--online_nn', dest='online_nn', type=int, default=0, help='online_nn') parser.add_argument('--use_limit_vocab', dest='use_limit_vocab', type=int, default=1, help='use_limit_vocab') parser.add_argument('--batchsize_nn', dest='batchsize_nn', type=int, default=10, help='batchsize_nn') parser.add_argument('--update_nearest_epoch', dest='update_nearest_epoch', type=int, default=1, help='update_nearest_epoch') args = parser.parse_args() batchsize = args.batchsize batchsize_semi = args.batchsize_semi print(args) random.seed(args.random_seed) np.random.seed(args.random_seed) os.environ["CHAINER_SEED"] = str(args.random_seed) os.makedirs("models", exist_ok=True) if args.debug_mode: chainer.set_debug(True) use_unlabled_to_vocab = args.use_unlabled_to_vocab lower = args.lower == 1 n_char_vocab = 1 n_class = 2 if args.dataset == 'imdb': vocab_obj, dataset, lm_data, t_vocab = utils.load_dataset_imdb( include_pretrain=use_unlabled_to_vocab, lower=lower, min_count=args.min_count, ignore_unk=args.ignore_unk, use_semi_data=args.use_semi_data, add_labeld_to_unlabel=args.add_labeld_to_unlabel) (train_x, train_x_len, train_y, dev_x, dev_x_len, dev_y, test_x, test_x_len, test_y) = dataset vocab, vocab_count = vocab_obj n_class = 2 # TODO: add other dataset code if args.use_semi_data: semi_train_x, semi_train_x_len = lm_data print('train_vocab_size:', t_vocab) vocab_inv = dict([(widx, w) for w, widx in vocab.items()]) print('vocab_inv:', len(vocab_inv)) xp = cuda.cupy if args.gpu >= 0 else np if args.gpu >= 0: cuda.get_device(args.gpu).use() xp.random.seed(args.random_seed) n_vocab = len(vocab) model = nets.uniLSTM_iVAT(n_vocab=n_vocab, emb_dim=args.emb_dim, hidden_dim=args.hidden_dim, use_dropout=args.dropout, n_layers=args.n_layers, hidden_classifier=args.hidden_cls_dim, use_adv=args.use_adv, xi_var=args.xi_var, n_class=n_class, args=args) model.train_vocab_size = t_vocab model.vocab_size = n_vocab model.logging = logging if args.pretrained_model != '': # load pretrained LM model pretrain_model = lm_nets.RNNForLM( n_vocab, 1024, args.n_layers, 0.50, share_embedding=False, adaptive_softmax=args.adaptive_softmax) serializers.load_npz(args.pretrained_model, pretrain_model) pretrain_model.lstm = pretrain_model.rnn model.set_pretrained_lstm(pretrain_model, word_only=args.word_only) all_nn_flag = args.use_attn_d if all_nn_flag and args.online_nn == 0: word_embs = model.word_embed.W.data model.norm_word_embs = word_embs / np.linalg.norm( word_embs, axis=1).reshape(-1, 1) model.norm_word_embs = np.array(model.norm_word_embs, dtype=np.float32) if args.load_trained_lstm != '': serializers.load_hdf5(args.load_trained_lstm, model) if args.gpu >= 0: model.to_gpu() if all_nn_flag and args.online_nn == 0: model.compute_all_nearest_words(top_k=args.nn_k) # check nearest words def most_sims(word): if word not in vocab: logging.info('[not found]:{}'.format(word)) return False idx = vocab[word] idx_gpu = xp.array([idx], dtype=xp.int32) top_idx = model.get_nearest_words(idx_gpu) sim_ids = top_idx[0] words = [vocab_inv[int(i)] for i in sim_ids] word_line = ','.join(words) logging.info('{}\t\t{}'.format(word, word_line)) most_sims(u'good') most_sims(u'this') most_sims(u'that') most_sims(u'awesome') most_sims(u'bad') most_sims(u'wrong') def evaluate(x_set, x_length_set, y_set): chainer.config.train = False chainer.config.enable_backprop = False iteration_list = range(0, len(x_set), batchsize) correct_cnt = 0 total_cnt = 0.0 predicted_np = [] for i_index, index in enumerate(iteration_list): x = [to_gpu(_x) for _x in x_set[index:index + batchsize]] x_length = x_length_set[index:index + batchsize] y = to_gpu(y_set[index:index + batchsize]) output = model(x, x_length) predict = xp.argmax(output.data, axis=1) correct_cnt += xp.sum(predict == y) total_cnt += len(y) accuracy = (correct_cnt / total_cnt) * 100.0 chainer.config.enable_backprop = True return accuracy def get_unlabled(perm_semi, i_index): index = i_index * batchsize_semi sample_idx = perm_semi[index:index + batchsize_semi] x = [to_gpu(semi_train_x[_i]) for _i in sample_idx] x_length = [semi_train_x_len[_i] for _i in sample_idx] return x, x_length base_alpha = args.alpha opt = optimizers.Adam(alpha=base_alpha) opt.setup(model) opt.add_hook(chainer.optimizer.GradientClipping(args.clip)) if args.freeze_word_emb: model.freeze_word_emb() prev_dev_accuracy = 0.0 global_step = 0.0 adv_rep_num_statics = {} adv_rep_pos_statics = {} if args.eval: dev_accuracy = evaluate(dev_x, dev_x_len, dev_y) log_str = ' [dev] accuracy:{}, length:{}'.format(str(dev_accuracy)) logging.info(log_str) # test test_accuracy = evaluate(test_x, test_x_len, test_y) log_str = ' [test] accuracy:{}, length:{}'.format(str(test_accuracy)) logging.info(log_str) for epoch in range(args.n_epoch): logging.info('epoch:' + str(epoch)) # train model.cleargrads() chainer.config.train = True iteration_list = range(0, len(train_x), batchsize) perm = np.random.permutation(len(train_x)) if args.use_semi_data: perm_semi = [ np.random.permutation(len(semi_train_x)) for _ in range(2) ] perm_semi = np.concatenate(perm_semi, axis=0) def idx_func(shape): return xp.arange(shape).astype(xp.int32) sum_loss = 0.0 sum_loss_z = 0.0 sum_loss_z_sparse = 0.0 sum_loss_label = 0.0 avg_rate = 0.0 avg_rate_num = 0.0 correct_cnt = 0 total_cnt = 0.0 N = len(iteration_list) is_adv_example_list = [] is_adv_example_disc_list = [] is_adv_example_disc_craft_list = [] y_np = [] predicted_np = [] save_items = [] for i_index, index in enumerate(iteration_list): global_step += 1.0 model.set_train(True) sample_idx = perm[index:index + batchsize] x = [to_gpu(train_x[_i]) for _i in sample_idx] x_length = [train_x_len[_i] for _i in sample_idx] y = to_gpu(train_y[sample_idx]) d = None # Classification loss output = model(x, x_length) output_original = output loss = F.softmax_cross_entropy(output, y, normalize=True) if args.use_adv or args.use_semi_data: # Adversarial Training if args.use_adv: output = model(x, x_length, first_step=True, d=None) # Adversarial loss (First step) loss_adv_first = F.softmax_cross_entropy(output, y, normalize=True) model.cleargrads() loss_adv_first.backward() if args.use_attn_d: # iAdv attn_d_grad = model.attention_d_var.grad attn_d_grad = F.normalize(attn_d_grad, axis=1) # Get directional vector dir_normed = model.dir_normed.data attn_d = F.broadcast_to(attn_d_grad, dir_normed.shape).data d = xp.sum(attn_d * dir_normed, axis=1) else: # Adv d = model.d_var.grad output = model(x, x_length, d=d) # Adversarial loss loss_adv = F.softmax_cross_entropy(output, y, normalize=True) loss += loss_adv * args.nl_factor # Virtual Adversarial Training if args.use_semi_data: x, length = get_unlabled(perm_semi, i_index) output_original = model(x, length) output_vat = model(x, length, first_step=True, d=None) loss_vat_first = nets.kl_loss(xp, output_original.data, output_vat) model.cleargrads() loss_vat_first.backward() if args.use_attn_d: # iVAT (ours) attn_d_grad = model.attention_d_var.grad attn_d_grad = F.normalize(attn_d_grad, axis=1) # Get directional vector dir_normed = model.dir_normed.data attn_d = F.broadcast_to(attn_d_grad, dir_normed.shape).data d_vat = xp.sum(attn_d * dir_normed, axis=1) else: # VAT d_vat = model.d_var.grad output_vat = model(x, length, d=d_vat) loss_vat = nets.kl_loss(xp, output_original.data, output_vat) loss += loss_vat predict = xp.argmax(output.data, axis=1) correct_cnt += xp.sum(predict == y) total_cnt += len(y) # update model.cleargrads() loss.backward() opt.update() if args.alpha_decay > 0.0: if args.use_exp_decay: opt.hyperparam.alpha = (base_alpha) * (args.alpha_decay** global_step) else: opt.hyperparam.alpha *= args.alpha_decay # 0.9999 sum_loss += loss.data accuracy = (correct_cnt / total_cnt) * 100.0 logging.info(' [train] sum_loss: {}'.format(sum_loss / N)) logging.info(' [train] apha:{}, global_step:{}'.format( opt.hyperparam.alpha, global_step)) logging.info(' [train] accuracy:{}'.format(accuracy)) model.set_train(False) # dev dev_accuracy = evaluate(dev_x, dev_x_len, dev_y) log_str = ' [dev] accuracy:{}'.format(str(dev_accuracy)) logging.info(log_str) # test test_accuracy = evaluate(test_x, test_x_len, test_y) log_str = ' [test] accuracy:{}'.format(str(test_accuracy)) logging.info(log_str) last_epoch_flag = args.n_epoch - 1 == epoch if prev_dev_accuracy < dev_accuracy: logging.info(' => '.join( [str(prev_dev_accuracy), str(dev_accuracy)])) result_str = 'dev_acc_' + str(dev_accuracy) result_str += '_test_acc_' + str(test_accuracy) model_filename = './models/' + '_'.join( [args.save_name, str(epoch), result_str]) # if len(sentences_train_list) == 1: serializers.save_hdf5(model_filename + '.model', model) prev_dev_accuracy = dev_accuracy nn_update_flag = args.update_nearest_epoch > 0 and ( epoch % args.update_nearest_epoch == 0) if all_nn_flag and nn_update_flag and args.online_nn == 0: model.cleargrads() x = None x_length = None y = None model.compute_all_nearest_words(top_k=args.nn_k)
def _sqnorm(x): with cuda.get_device(x): x = x.ravel() return float(x.dot(x))
import numpy as np import chainer from chainer import Chain, Variable, cuda, functions, links, optimizer, optimizers, serializers import MeCab import pickle import os from utils import * # アテンション機構の追加 # 状態の初期化にバッチサイズを与えることでデータ数の調整を不要に if FLAG_GPU: cuda.check_cuda_available() cuda.get_device(0).use() ARR = cuda.cupy else: ARR = np class LSTM_Encoder(Chain): def __init__(self, vocab_size, embed_size, hidden_size): super(LSTM_Encoder, self).__init__(xe=links.EmbedID(vocab_size, embed_size, ignore_label=-1), eh=links.Linear(embed_size, 4 * hidden_size), hh=links.Linear(hidden_size, 4 * hidden_size)) def __call__(self, x, c, h): e = functions.tanh(self.xe(x)) return functions.lstm(c, self.eh(e) + self.hh(h))
def train(self): params = self.params use_gpu = params['gpu'] >= 0 if use_gpu: cuda.get_device(params['gpu']).use() self.dcgan.to_gpu() xp = cp if use_gpu else np nz = params['nz'] train = params['train'] batchsize = params['batchsize'] output_interval = train // 2 xp.random.seed(0) z_vis = xp.random.uniform(-1, 1, (100, nz)).astype(np.float32) xp.random.seed() loader = ImageLoader(self._image_dir, batchsize) for epoch in range(params['current_epoch'] + 1, params['epoch']): start = time.time() perm = np.random.permutation(train) sum_loss_gen = 0 sum_loss_dis = 0 for i in range(train): x = np.zeros((batchsize, 3, DCGAN.SIZE[0], DCGAN.SIZE[1]), np.float32) for j, image in loader: x[j] = image if use_gpu: x = cuda.to_gpu(x) x = Variable(x) z = Variable( xp.random.uniform(-1, 1, (batchsize, nz)).astype(np.float32)) loss_gen, loss_dis = self.dcgan(z, x) self.opt_gen.zero_grads() loss_gen.backward() self.opt_gen.update() self.opt_dis.zero_grads() loss_dis.backward() self.opt_dis.update() sum_loss_gen += loss_gen.data.get( ) if use_gpu else loss_gen.data sum_loss_dis += loss_dis.data.get( ) if use_gpu else loss_dis.data if i % output_interval == 0: pylab.rcParams['figure.figsize'] = (16.0, 16.0) pylab.clf() z = z_vis z[50:, :] = xp.random.uniform(-1, 1, (50, nz)).astype(np.float32) z = Variable(z) x = self.dcgan.generate(z) for j in range(x.shape[0]): image = x[j] pylab.subplot(10, 10, j + 1) pylab.imshow(image) pylab.axis('off') image_path = os.path.join( self._output_dir, '{}_{}_{}.png'.format(self.name, epoch, i)) pylab.savefig(image_path) sum_loss_gen /= train sum_loss_dis /= train params['current_epoch'] = epoch params['losses'][epoch] = { 'loss_gen': sum_loss_gen, 'loss_dis': sum_loss_dis } self.save() elapsed = time.time() - start print('epoch {} ({:.2f} sec) / loss_gen: {:.8f}, loss_dis: {:.8f}'. format(epoch, elapsed, sum_loss_gen, sum_loss_dis))
def calculate_loss(model, dataset, position): # use random window size in the same way as the original word2vec # implementation. w = np.random.randint(args.window - 1) + 1 # offset is [-w, ..., -1, 1, ..., w] offset = np.concatenate([np.arange(-w, 0), np.arange(1, w + 1)]) pos = np.expand_dims(position, 0) + np.expand_dims(offset, 1) d = xp.asarray(dataset.take(pos)) context = chainer.Variable(d) x_data = xp.asarray(dataset.take(position)) x = chainer.Variable(x_data) return model(x, context) if args.gpu >= 0: cuda.get_device(args.gpu).use() train, _, _ = chainer.datasets.get_ptb_words() if args.test: train = train[:100] vocab = chainer.datasets.get_ptb_words_vocabulary() index2word = {wid: word for word, wid in six.iteritems(vocab)} counts = collections.Counter(train) n_vocab = max(train) + 1 print('n_vocab: %d' % n_vocab) print('data length: %d' % len(train)) if args.out_type == 'hsm':
def main(): parser = argparse.ArgumentParser() parser.add_argument("--batchsize", "-b", type=int, default=64) parser.add_argument("--total-epochs", "-e", type=int, default=5000) parser.add_argument("--num-labeled-data", "-nl", type=int, default=100) parser.add_argument("--gpu-device", "-g", type=int, default=0) parser.add_argument("--grad-clip", "-gc", type=float, default=5) parser.add_argument("--seed", type=int, default=0) parser.add_argument("--model", "-m", type=str, default="model.hdf5") args = parser.parse_args() np.random.seed(args.seed) model = Model() model.load(args.model) mnist_train, mnist_test = chainer.datasets.get_mnist() images_train, labels_train = mnist_train._datasets images_test, labels_test = mnist_test._datasets # normalize images_train = (images_train - 0.5) * 2 images_test = (images_test - 0.5) * 2 dataset = Dataset(train=(images_train, labels_train), test=(images_test, labels_test), num_labeled_data=args.num_labeled_data, num_classes=model.ndim_y) print("#labeled: {}".format(dataset.get_num_labeled_data())) print("#unlabeled: {}".format(dataset.get_num_unlabeled_data())) _, labels = dataset.get_labeled_data() print("labeled data:", labels) total_iterations_train = len(images_train) // args.batchsize # optimizers optimizer_encoder = Optimizer("msgd", 0.01, 0.9) optimizer_encoder.setup(model.encoder) if args.grad_clip > 0: optimizer_encoder.add_hook(GradientClipping(args.grad_clip)) optimizer_semi_supervised = Optimizer("msgd", 0.1, 0.9) optimizer_semi_supervised.setup(model.encoder) if args.grad_clip > 0: optimizer_semi_supervised.add_hook(GradientClipping(args.grad_clip)) optimizer_generator = Optimizer("msgd", 0.1, 0.1) optimizer_generator.setup(model.encoder) if args.grad_clip > 0: optimizer_generator.add_hook(GradientClipping(args.grad_clip)) optimizer_decoder = Optimizer("msgd", 0.01, 0.9) optimizer_decoder.setup(model.decoder) if args.grad_clip > 0: optimizer_decoder.add_hook(GradientClipping(args.grad_clip)) optimizer_discriminator_z = Optimizer("msgd", 0.1, 0.1) optimizer_discriminator_z.setup(model.discriminator_z) if args.grad_clip > 0: optimizer_discriminator_z.add_hook(GradientClipping(args.grad_clip)) optimizer_discriminator_y = Optimizer("msgd", 0.1, 0.1) optimizer_discriminator_y.setup(model.discriminator_y) if args.grad_clip > 0: optimizer_discriminator_y.add_hook(GradientClipping(args.grad_clip)) optimizer_linear_transformation = Optimizer("msgd", 0.01, 0.9) optimizer_linear_transformation.setup(model.linear_transformation) if args.grad_clip > 0: optimizer_linear_transformation.add_hook( GradientClipping(args.grad_clip)) using_gpu = False if args.gpu_device >= 0: cuda.get_device(args.gpu_device).use() model.to_gpu() using_gpu = True xp = model.xp # 0 -> true sample # 1 -> generated sample class_true = np.zeros(args.batchsize, dtype=np.int32) class_fake = np.ones(args.batchsize, dtype=np.int32) if using_gpu: class_true = cuda.to_gpu(class_true) class_fake = cuda.to_gpu(class_fake) # 2D circle # we use a linear transformation to map the 10D representation to a 2D space such that # the cluster heads are mapped to the points that are uniformly placed on a 2D circle. rad = math.radians(360 / model.ndim_y) radius = 5 mapped_cluster_head_2d_target = np.zeros((10, 2), dtype=np.float32) for n in range(model.ndim_y): x = math.cos(rad * n) * radius y = math.sin(rad * n) * radius mapped_cluster_head_2d_target[n] = (x, y) if using_gpu: mapped_cluster_head_2d_target = cuda.to_gpu( mapped_cluster_head_2d_target) # training loop training_start_time = time.time() for epoch in range(args.total_epochs): sum_loss_generator = 0 sum_loss_discriminator = 0 sum_loss_autoencoder = 0 sum_loss_supervised = 0 sum_loss_linear_transformation = 0 sum_discriminator_z_confidence_true = 0 sum_discriminator_z_confidence_fake = 0 sum_discriminator_y_confidence_true = 0 sum_discriminator_y_confidence_fake = 0 epoch_start_time = time.time() dataset.shuffle() # training for itr in range(total_iterations_train): # update model parameters with chainer.using_config("train", True): # sample minibatch x_u = dataset.sample_unlabeled_minibatch(args.batchsize, gpu=using_gpu) x_l, y_l, _ = dataset.sample_labeled_minibatch(args.batchsize, gpu=using_gpu) ### reconstruction phase ### if True: y_onehot_u, z_u = model.encode_x_yz(x_u, apply_softmax_y=True) repr_u = model.encode_yz_representation(y_onehot_u, z_u) x_reconstruction_u = model.decode_representation_x(repr_u) loss_reconstruction_u = F.mean_squared_error( x_u, x_reconstruction_u) y_onehot_l, z_l = model.encode_x_yz(x_l, apply_softmax_y=True) repr_l = model.encode_yz_representation(y_onehot_l, z_l) x_reconstruction_l = model.decode_representation_x(repr_l) loss_reconstruction_l = F.mean_squared_error( x_l, x_reconstruction_l) loss_reconstruction = loss_reconstruction_u + loss_reconstruction_l model.cleargrads() loss_reconstruction.backward() optimizer_encoder.update() optimizer_decoder.update() sum_loss_autoencoder += float(loss_reconstruction.data) ### adversarial phase ### if True: y_onehot_fake_u, z_fake_u = model.encode_x_yz( x_u, apply_softmax_y=True) z_true = sampler.gaussian(args.batchsize, model.ndim_y, mean=0, var=1) y_onehot_true = sampler.onehot_categorical( args.batchsize, model.ndim_y) if using_gpu: z_true = cuda.to_gpu(z_true) y_onehot_true = cuda.to_gpu(y_onehot_true) dz_true = model.discriminate_z(z_true, apply_softmax=False) dz_fake = model.discriminate_z(z_fake_u, apply_softmax=False) dy_true = model.discriminate_y(y_onehot_true, apply_softmax=False) dy_fake = model.discriminate_y(y_onehot_fake_u, apply_softmax=False) discriminator_z_confidence_true = float( xp.mean(F.softmax(dz_true).data[:, 0])) discriminator_z_confidence_fake = float( xp.mean(F.softmax(dz_fake).data[:, 1])) discriminator_y_confidence_true = float( xp.mean(F.softmax(dy_true).data[:, 0])) discriminator_y_confidence_fake = float( xp.mean(F.softmax(dy_fake).data[:, 1])) loss_discriminator_z = F.softmax_cross_entropy( dz_true, class_true) + F.softmax_cross_entropy( dz_fake, class_fake) loss_discriminator_y = F.softmax_cross_entropy( dy_true, class_true) + F.softmax_cross_entropy( dy_fake, class_fake) loss_discriminator = loss_discriminator_z + loss_discriminator_y model.cleargrads() loss_discriminator.backward() optimizer_discriminator_z.update() optimizer_discriminator_y.update() sum_loss_discriminator += float(loss_discriminator.data) sum_discriminator_z_confidence_true += discriminator_z_confidence_true sum_discriminator_z_confidence_fake += discriminator_z_confidence_fake sum_discriminator_y_confidence_true += discriminator_y_confidence_true sum_discriminator_y_confidence_fake += discriminator_y_confidence_fake ### generator phase ### if True: y_onehot_fake_u, z_fake_u = model.encode_x_yz( x_u, apply_softmax_y=True) dz_fake = model.discriminate_z(z_fake_u, apply_softmax=False) dy_fake = model.discriminate_y(y_onehot_fake_u, apply_softmax=False) loss_generator = F.softmax_cross_entropy( dz_fake, class_true) + F.softmax_cross_entropy( dy_fake, class_true) model.cleargrads() loss_generator.backward() optimizer_generator.update() sum_loss_generator += float(loss_generator.data) ### supervised phase ### if True: logit_l, _ = model.encode_x_yz(x_l, apply_softmax_y=False) loss_supervised = F.softmax_cross_entropy(logit_l, y_l) model.cleargrads() loss_supervised.backward() optimizer_semi_supervised.update() sum_loss_supervised += float(loss_supervised.data) ### additional cost ### if True: identity = np.identity(model.ndim_y, dtype=np.float32) if using_gpu: identity = cuda.to_gpu(identity) mapped_head = model.linear_transformation(identity) loss_linear_transformation = F.mean_squared_error( mapped_cluster_head_2d_target, mapped_head) model.cleargrads() loss_linear_transformation.backward() optimizer_linear_transformation.update() sum_loss_linear_transformation += float( loss_linear_transformation.data) printr("Training ... {:3.0f}% ({}/{})".format( (itr + 1) / total_iterations_train * 100, itr + 1, total_iterations_train)) model.save(args.model) labeled_iter_train = dataset.get_iterator(args.batchsize * 20, train=True, labeled=True, gpu=using_gpu) unlabeled_iter_train = dataset.get_iterator(args.batchsize * 20, train=True, unlabeled=True, gpu=using_gpu) average_accuracy_l = 0 average_accuracy_u = 0 for x_l, true_label in labeled_iter_train: with chainer.no_backprop_mode() and chainer.using_config( "train", False): y_onehot_l, _ = model.encode_x_yz(x_l, apply_softmax_y=True) accuracy = F.accuracy(y_onehot_l, true_label) average_accuracy_l += float(accuracy.data) for x_u, true_label in unlabeled_iter_train: with chainer.no_backprop_mode() and chainer.using_config( "train", False): y_onehot_u, _ = model.encode_x_yz(x_u, apply_softmax_y=True) accuracy = F.accuracy(y_onehot_u, true_label) average_accuracy_u += float(accuracy.data) average_accuracy_l /= labeled_iter_train.get_total_iterations() average_accuracy_u /= unlabeled_iter_train.get_total_iterations() clear_console() print( "Epoch {} done in {} sec - loss: g={:.5g}, d={:.5g}, a={:.5g}, s={:.5g}, l={:.5g} - disc_z: true={:.1f}%, fake={:.1f}% - disc_y: true={:.1f}%, fake={:.1f}% - acc: l={:.2f}%, u={:.2f}% - total {} min" .format( epoch + 1, int(time.time() - epoch_start_time), sum_loss_generator / total_iterations_train, sum_loss_discriminator / total_iterations_train, sum_loss_autoencoder / total_iterations_train, sum_loss_supervised / total_iterations_train, sum_loss_linear_transformation / total_iterations_train, sum_discriminator_z_confidence_true / total_iterations_train * 100, sum_discriminator_z_confidence_fake / total_iterations_train * 100, sum_discriminator_y_confidence_true / total_iterations_train * 100, sum_discriminator_y_confidence_fake / total_iterations_train * 100, average_accuracy_l * 100, average_accuracy_u * 100, int((time.time() - training_start_time) // 60))) if epoch == 50: optimizer_encoder.set_learning_rate(0.001) optimizer_decoder.set_learning_rate(0.001) optimizer_semi_supervised.set_learning_rate(0.01) optimizer_generator.set_learning_rate(0.01) optimizer_discriminator_y.set_learning_rate(0.01) optimizer_discriminator_z.set_learning_rate(0.01) if epoch == 1000: optimizer_encoder.set_learning_rate(0.0001) optimizer_decoder.set_learning_rate(0.0001) optimizer_semi_supervised.set_learning_rate(0.001) optimizer_generator.set_learning_rate(0.001) optimizer_discriminator_y.set_learning_rate(0.001) optimizer_discriminator_z.set_learning_rate(0.001)
def main(): current_id = datetime.datetime.today().isoformat( "-") + "-" + os.path.splitext(os.path.basename(__file__))[0] parser = argparse.ArgumentParser(description='I-Maze with Block obs') parser.add_argument( "-modelpath", type=str, help="modelpath without extension(eg .model, .optimizer)") parser.add_argument("-vertical", type=int, default=2, help="vertical corridor length") parser.add_argument("-horizontal", type=int, default=0, help="horizontal corridor length") parser.add_argument("-validation", type=int, default=0, help="validation flag, default:0") parser.add_argument("-outdir", type=str, default="log", help="output dir for loggin, default:'log'") parser.add_argument("-epsdelta", type=float, default=10**-6, help="delta of epsilon, default:10**-6") parser.add_argument("-initexp", type=int, default=10**4, help="initial exproration, default:10**4") parser.add_argument("-eps", type=float, default=1.0, help="epsilon, default:1.0") parser.add_argument("-lr", type=float, default=k_default_lr, help="epsilon, default:" + str(k_default_lr)) parser.add_argument("-modeltype", type=str, default=k_default_modeltype, help="ModelType, default:'" + k_default_modeltype + "'") parser.add_argument("-batchsize", type=int, default=k_default_replay_batch_size, help="replay batch size, default:" + str(k_default_replay_batch_size)) parser.add_argument("-updatefreq", type=int, default=k_default_update_freq, help="update frequency, default:" + str(k_default_update_freq)) parser.add_argument("-gpu", type=int, default=0, help="gpu id, default:0 (cpu is -1)") parser.add_argument("-testoutput", type=int, default=0, help="output only at test, default:0") parser.add_argument("-y", type=int, default=0, help="OK?, default:0") parser.add_argument("-framehistnum", type=int, default=12, help="frame history num, default:12") args = parser.parse_args() print(args) if args.y == 0: input("OK?") ## Make directory and write setting log if not os.path.exists(args.outdir): os.makedirs(args.outdir) with open(os.path.join(args.outdir, current_id + ".args"), "w") as argsf: argsf.write(str(args)) env = I_MazeEnv(horizontal=args.horizontal, vertical=args.vertical, max_step=k_max_step) ## Init model input_dim = k_ob_shape[0] output_dim = len(env.action_set) if args.modeltype == "DQN": model = DQN(input_dim * args.framehistnum, output_dim) elif args.modeltype == "DRQN": model = DRQN(input_dim, output_dim) elif args.modeltype == "MQN": model = MQN(input_dim, output_dim, max_buff_size=args.framehistnum - 1, m=256, e=256) elif args.modeltype == "RMQN": model = RMQN(input_dim, output_dim, max_buff_size=args.framehistnum - 1, m=256, e=256) elif args.modeltype == "FRMQN": model = FRMQN(input_dim, output_dim, max_buff_size=args.framehistnum - 1, m=256, e=256) else: print("not implemented", args.modeltype) exit(0) ## Use GPU if args.gpu >= 0: cuda.get_device(args.gpu).use() model.to_gpu() ## Init agent agent = Agent(k_ob_shape, len(env.action_set), args.framehistnum, model, lr=args.lr, eps_delta=args.epsdelta, eps=args.eps, batch_size=args.batchsize) if args.modelpath: print("load model from ", args.modelpath + ".model and " + args.modelpath + ".optimizer") agent.load(os.path.expanduser(args.modelpath)) train_total_step = 0 if args.validation: ## Run validation mode = run_mode.validation for vertical in [4, 5, 6, 8, 10, 15, 20, 25, 30, 35, 40]: env.vertical = vertical for _ in range(1): run_episode(current_id, args, env, agent, mode, vertical, train_total_step) exit(0) for episode_id in range(k_max_episode): try: if args.validation: assert (not "!!!") else: if episode_id % 100 == 0 and episode_id != 0: ## Run test mode = run_mode.test for j in range(10): run_episode(current_id, args, env, agent, mode, episode_id + j, train_total_step) ## Save model agent.save( os.path.join(args.outdir, current_id + "_episode" + str(episode_id))) ## Run train mode = run_mode.train train_total_step \ = run_episode(current_id, args, env, agent, mode, episode_id, train_total_step) except: ark = {} ark["args"] = vars(args) ark["episode_id"] = episode_id ark["train_total_step"] = train_total_step ark["eps"] = current_eps with open( os.path.join( args.outdir, current_id + "_episode" + str(episode_id) + "_ark.json"), "w") as arkf: ark_str = json.dumps(ark, indent=4, sort_keys=True) arkf.write(ark_str) with open( os.path.join( args.outdir, current_id + "_episode" + str(episode_id) + "_dataset.pkl"), "wb") as datasetf: pickle.dump(agent.dqn.dataset, datasetf) exit(0)
def main(args): start_time = datetime.now().strftime('%Y%m%d_%H_%M_%S') dest = "../result/" + start_time os.makedirs(dest) abs_dest = os.path.abspath(dest) with open(os.path.join(dest, "settings.json"), "w") as fo: fo.write(json.dumps(vars(args), sort_keys=True, indent=4)) # load data data_processor = DataProcessor(args.data, args.vocab, args.test) data_processor.prepare_dataset() train_data = data_processor.train_data dev_data = data_processor.dev_data # create model vocab = data_processor.vocab embed_dim = args.dim cnn = BCNN(n_vocab=len(vocab), n_layer=args.layer, embed_dim=embed_dim, input_channel=1, output_channel=50) # ABCNNはoutput = 50固定らしいが. if args.glove: cnn.load_glove_embeddings(args.glove_path, data_processor.vocab) if args.word2vec: cnn.load_word2vec_embeddings(args.word2vec_path, data_processor.vocab) model = Classifier(cnn, lossfun=sigmoid_cross_entropy, accfun=binary_accuracy) if args.gpu >= 0: cuda.get_device(args.gpu).use() model.to_gpu() # setup optimizer optimizer = O.AdaGrad(args.lr) optimizer.setup(model) # do not use weight decay for embeddings decay_params = { name: 1 for name, variable in model.namedparams() if "embed" not in name } optimizer.add_hook( SelectiveWeightDecay(rate=args.decay, decay_params=decay_params)) train_iter = chainer.iterators.SerialIterator(train_data, args.batchsize) dev_train_iter = chainer.iterators.SerialIterator(train_data, args.batchsize, repeat=False) dev_iter = DevIterator(dev_data, data_processor.n_dev) updater = training.StandardUpdater(train_iter, optimizer, converter=concat_examples, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=abs_dest) # setup evaluation eval_predictor = model.copy().predictor eval_predictor.train = False iters = {"train": dev_train_iter, "dev": dev_iter} trainer.extend( WikiQAEvaluator(iters, eval_predictor, converter=concat_examples, device=args.gpu)) # extentions... trainer.extend(extensions.LogReport()) trainer.extend( extensions.PrintReport([ 'epoch', 'main/loss', 'validation/main/loss', 'validation/main/map', 'validation/main/mrr', 'validation/main/svm_map', 'validation/main/svm_mrr' ])) trainer.extend(extensions.ProgressBar(update_interval=10)) # take a shapshot when the model achieves highest accuracy in dev set trainer.extend( extensions.snapshot_object( model, 'model_epoch_{.updater.epoch}', trigger=chainer.training.triggers.MaxValueTrigger( 'validation/main/map'))) trainer.extend(extensions.ExponentialShift("lr", 0.5, optimizer=optimizer), trigger=chainer.training.triggers.MaxValueTrigger( "validation/main/map")) trainer.run()
def main(): parser = argparse.ArgumentParser(description='') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU device ID') parser.add_argument('--epoch', '-e', type=int, default=50, help='# of epoch') parser.add_argument('--batch_size', type=int, default=128, help='size of mini-batch') parser.add_argument('--density', type=int, default=1, help='density of cnn kernel') parser.add_argument('--small', dest='small', action='store_true', default=False) parser.add_argument('--no_bn', dest='use_bn', action='store_false', default=True) parser.add_argument('--out', default='') parser.set_defaults(test=False) args = parser.parse_args() model = ValueNet(use_bn=args.use_bn) # model = RolloutValueNet(use_bn=args.use_bn, output=41) # log directory out = datetime.datetime.now().strftime('%m%d') if args.out: out = out + '_' + args.out out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs_value", out)) os.makedirs(os.path.join(out_dir, 'models'), exist_ok=True) # gpu if args.gpu >= 0: cuda.get_device(args.gpu).use() model.to_gpu() # setting with open(os.path.join(out_dir, 'setting.txt'), 'w') as f: for k, v in args._get_kwargs(): print('{} = {}'.format(k, v)) f.write('{} = {}\n'.format(k, v)) # prepare for dataset if args.small: train = PreprocessedDataset(train_small_path) else: train = PreprocessedDataset(train_path) test = PreprocessedDataset(test_path) train_iter = iterators.SerialIterator(train, args.batch_size) val_iter = iterators.SerialIterator(test, args.batch_size, repeat=False) # optimizer optimizer = chainer.optimizers.Adam(eps=1e-2) optimizer.setup(model) # start training start = time.time() train_count = 0 for epoch in range(args.epoch): # train train_loss = [] train_accuracy = [] for i in range(len(train) // args.batch_size): batch = train_iter.next() x = chainer.Variable( model.xp.array([b[0] for b in batch], 'float32')) y = chainer.Variable(model.xp.array([b[1] for b in batch], 'int32')) optimizer.update(model, x, y) train_count += 1 progress_report(train_count, start, args.batch_size) train_loss.append(cuda.to_cpu(model.loss.data)) train_accuracy.append(cuda.to_cpu(model.accuracy.data)) # test test_loss = [] test_accuracy = [] it = copy.copy(val_iter) for batch in it: x = chainer.Variable(model.xp.array([b[0] for b in batch], 'float32'), volatile=True) y = chainer.Variable(model.xp.array([b[1] for b in batch], 'int32'), volatile=True) model(x, y, train=False) test_loss.append(cuda.to_cpu(model.loss.data)) test_accuracy.append(cuda.to_cpu(model.accuracy.data)) print('\nepoch {} train_loss {:.5f} train_accuracy {:.3f} \n' ' test_loss {:.5f} test_accuracy {:.3f}'.format( epoch, np.mean(train_loss), np.mean(train_accuracy), np.mean(test_loss), np.mean(test_accuracy))) with open(os.path.join(out_dir, "log"), 'a+') as f: f.write( 'epoch {} train_loss {:.5f} train_accuracy {:.3f} \n' ' test_loss {:.5f} test_accuracy {:.3f} \n'.format( epoch, np.mean(train_loss), np.mean(train_accuracy), np.mean(test_loss), np.mean(test_accuracy))) if epoch % 5 == 0: serializers.save_hdf5( os.path.join(out_dir, "models", "value_net_{}.model".format(epoch)), model)
def experiment(): ite = 100 pdata = 1000 epoch = 100 batchsize = 1000 seed = 2018 gpu = True loss_pu = np.zeros((ite, epoch)) est_error_pu = np.zeros((ite, epoch)) est_error_pubp = np.zeros((ite, epoch)) est_precision_pu = np.zeros((ite, epoch)) est_recall_pu = np.zeros((ite, epoch)) est_precision_pubp = np.zeros((ite, epoch)) est_recall_pubp = np.zeros((ite, epoch)) for i in range(ite): np.random.seed(seed) #PN classification x_train, t_train, x_test, t_test = load_dataset("mnist") t_train[t_train == -1] = 0 t_test[t_test == -1] = 0 pi = np.mean(t_train) x = np.concatenate([x_train, x_test], axis=0) t = np.concatenate([t_train, t_test], axis=0) x = x.reshape(x.shape[0], x.shape[2]*x.shape[3]) dim = x.shape[1] print(x.shape) model = MultiLayerPerceptron(dim) optimizer = optimizers.Adam(1e-5) optimizer.setup(model) if gpu: gpu_device = 0 cuda.get_device(gpu_device).use() model.to_gpu(gpu_device) xp = cuda.cupy else: xp = np model, optimizer = train(x, t, epoch, model, optimizer, batchsize, xp) x_p = x_train[t_train==1] xp_prob = np.array([]) for j in six.moves.range(0, len(x_p), batchsize): X = Variable(xp.array(x_p[j:j + batchsize], xp.float32)) g = chainer.cuda.to_cpu(model(X).data).T[0] xp_prob = np.append(xp_prob, 1/(1+np.exp(-g)), axis=0) xp_prob /= np.mean(xp_prob) xp_prob = xp_prob xp_prob /= np.max(xp_prob) print(xp_prob) rand = np.random.uniform(size=len(x_p)) x_p = x_p[xp_prob > rand] perm = np.random.permutation(len(x_p)) x_p = x_p[perm[:pdata]] tp = np.ones(len(x_p)) tu = np.zeros(len(x_train)) t_train = np.concatenate([tp, tu], axis=0) x_train = np.concatenate([x_p, x_train], axis=0) print(x_train.shape) print(t_train.shape) print(x_test.shape) print(t_test.shape) model = MultiLayerPerceptron(dim) optimizer = optimizers.Adam(alpha=1e-5) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(0.005)) if gpu: gpu_device = 0 cuda.get_device(gpu_device).use() model.to_gpu(gpu_device) xp = cuda.cupy else: xp = np model, optimizer, loss_list, acc1, acc2, pre1, rec1, pre2, rec2 = train_pu(x_train, t_train, x_test, t_test, pi, epoch, model, optimizer, batchsize, xp) loss_pu[i] = loss_list est_error_pu[i] = acc1 est_error_pubp[i] = acc2 est_precision_pu[i] = pre1 est_recall_pu[i] = rec1 est_precision_pubp[i] = pre2 est_recall_pubp[i] = rec2 print(acc1[-1]) print(acc2[-1]) seed += 1 np.savetxt('loss_pu_mnist_%d.csv'%seed, loss_pu, delimiter=',') np.savetxt('est_error_pu_mnist_%d.csv'%seed, est_error_pu, delimiter=',') np.savetxt('est_error_pubp_mnist__%d.csv'%seed, est_error_pubp, delimiter=',') np.savetxt('est_precision_pu_mnist_%d.csv'%seed, est_precision_pu, delimiter=',') np.savetxt('est_recall_pu_mnist_%d.csv'%seed, est_recall_pu, delimiter=',') np.savetxt('est_precision_pubp_mnist_%d.csv'%seed, est_precision_pubp, delimiter=',') np.savetxt('est_recall_pubp_mnist_%d.csv'%seed, est_recall_pubp, delimiter=',') loss_pu_mean = np.mean(loss_pu, axis=1) est_error_pu_mean = np.mean(est_error_pu, axis=1) est_error_pubp_mean = np.mean(est_error_pubp, axis=1) est_error_pu_std = np.std(est_error_pu, axis=1) est_error_pubp_std = np.std(est_error_pubp, axis=1) return loss_pu_mean, est_error_pu_mean, est_error_pubp_mean, est_error_pu_std, est_error_pubp_std
def main(): parser = argparse.ArgumentParser() parser.add_argument('model_file') parser.add_argument('-g', '--gpu', default=0, type=int, help='if -1, use cpu only (default: 0)') args = parser.parse_args() alpha = 0.3 image_alpha = 1 render_eps = 0.015 data_path = '../../train_data/linemodSIXD2017' ## delta for visibility correspondance delta = 0.015 # [m] objs = np.arange(15) + 1 n_class = len(objs) + 1 distance_sanity = 0.02 min_distance = 0.005 output_scale = 0.14 prob_eps = 0.4 eps = 0.02 im_size = (640, 480) interval = 15 ## load object models obj_model_fpath_mask = os.path.join(data_path, 'models', 'obj_{0:0>2}.ply') obj_models = [] for obj in objs: if obj != 'background': print 'Loading data: obj_{0}'.format(obj) obj_model_fpath = obj_model_fpath_mask.format(obj) obj_models.append(inout.load_ply(obj_model_fpath)) ## load network model model = DualCenterProposalNetworkRes50_predict7(n_class=n_class) chainer.serializers.load_npz(args.model_file, model) if args.gpu >= 0: cuda.get_device(args.gpu).use() model.to_gpu() # pose estimator instance # pei = SimplePoseEstimationInterface(distance_sanity=distance_sanity, # min_distance=min_distance, eps=prob_eps) pei = PoseEstimationInterface(objs=['obj_{0:0>2}'.format(i) for i in objs], base_path=data_path, n_ransac=500, distance_sanity=distance_sanity, model_scale=1000.0, model_partial=1, min_distance=min_distance, eps=prob_eps, im_size=im_size) scores_pos = [] scores_rot = [] ids_arr = [] scores_5cm5deg = [] scores_6dpose = [] test_objs = np.delete(np.arange(15) + 1, [2, 6]) for obj_id in test_objs: test = LinemodSIXDSingleInstanceDataset(data_path, obj_id, mode='test', interval=interval, metric_filter=output_scale + eps) im_ids = test.__len__() # im_ids = 50 detect_cnt = 0 cnt_5cm5deg = 0 cnt_6dpose = 0 sum_pos = 0 sum_rot = 0 for im_id in tqdm.trange(im_ids): # print "executing {0} / {1}".format(im_id, test.__len__()) img_rgb, img_depth, pos, rot, K = test.get_example(im_id) x_data = np.expand_dims(img_rgb, axis=0) with chainer.no_backprop_mode(): if args.gpu >= 0: x_data = cuda.to_gpu(x_data) x = chainer.Variable(x_data) with chainer.using_config('train', False): y_cls_d, y_cp_d, y_ocp_d = model(x) y_cls = chainer.cuda.to_cpu(y_cls_d.data)[0] y_cp = chainer.cuda.to_cpu(y_cp_d.data)[0] y_ocp = chainer.cuda.to_cpu(y_ocp_d.data)[0] y_pos, y_rot = pei.execute(y_cls, y_cp * output_scale, y_ocp * output_scale, img_depth, K, estimate_idx=[obj_id - 1]) for i in six.moves.range(n_class - 1): if np.sum(pos[i]) != 0 and np.sum(rot[i]) != 0: # 5cm 5deg metric pos_diff = np.linalg.norm(y_pos[i] - pos[i]) quat = quaternion.from_rotation_matrix( np.dot(y_rot[i].T, rot[i])) quat_w = min(1, abs(quat.w)) diff_angle = np.rad2deg(np.arccos(quat_w)) * 2 # print "obj_{0:0>2} : position_diff = {1}, rotation_diff = {2}".format(i + 1, pos_diff, diff_angle) if i + 1 == obj_id and pos_diff < 1.0: sum_pos += pos_diff sum_rot += diff_angle detect_cnt += 1 if pos_diff < 0.05 and diff_angle < 5: cnt_5cm5deg += 1 # 6d pose metric model_pts = obj_models[i]['pts'].transpose(1, 0) gt_proj = np.dot(rot[i].T, model_pts) + pos[i, :, np.newaxis] pred_proj = np.dot(y_rot[i].T, model_pts) + pos[i, :, np.newaxis] diff_mean = np.mean( np.linalg.norm((gt_proj - pred_proj), axis=0)) if i + 1 == obj_id and pos_diff < 1.0 and diff_mean < 0.1 * object_diameters[ i]: cnt_6dpose += 1 print "obj_{0:0>2} : detection_rate = {1},\n position_diff = {2}, rotation_diff = {3}, 5cm5deg = {4}, 6d pose = {5}" \ .format(obj_id, detect_cnt / (1.0 * im_ids), sum_pos / detect_cnt, sum_rot / detect_cnt, cnt_5cm5deg / (1.0 * im_ids), cnt_6dpose / (1.0 * im_ids)) scores_pos.append(sum_pos / im_ids) scores_rot.append(sum_rot / im_ids) scores_5cm5deg.append(cnt_5cm5deg) scores_6dpose.append(cnt_6dpose) ids_arr.append(im_ids) print "-- results --" print scores_pos print scores_rot print "-- total results --" ids_arr = np.asarray(ids_arr) scores_5cm5deg = np.asarray(scores_5cm5deg) ave_5cm5deg = np.sum(scores_5cm5deg) / (1.0 * np.sum(ids_arr)) ave_6dpose = np.sum(scores_6dpose) / (1.0 * np.sum(ids_arr)) print "5cm5deg metric : {}".format(ave_5cm5deg) print "6d pose metric : {}".format(ave_6dpose) # scores_pos = scores_pos * 1000 # m -> mm scores_5cm5deg = scores_5cm5deg / ids_arr scores_6dpose = scores_6dpose / ids_arr scores_ave = np.array([ave_5cm5deg, ave_6dpose]) if not os.path.exists('eval_results'): os.makedirs('eval_results') np.save('eval_results/scores_pos.npy', scores_pos) np.save('eval_results/scores_rot.npy', scores_rot) np.save('eval_results/scores_5cm5deg.npy', scores_5cm5deg) np.save('eval_results/scores_6dpose.npy', scores_6dpose) np.save('eval_results/scores_ave.npy', scores_ave)
def test_forward_multi_gpu(self): with cuda.get_device(1): self.link.to_gpu() x = cuda.to_gpu(self.x) with cuda.get_device(0): self.check_forward(x)
def init_state(self, param, state): xp = cuda.get_array_module(param.data) with cuda.get_device(param.data): state['mem'] = xp.ones_like(param.data) state['g'] = xp.zeros_like(param.data) state['g2'] = xp.zeros_like(param.data)
def sim_generator(params): global sim_params, cells, stims_ext, stims_mem global i_ion, phie, i_ext_e, i_ext_i, rhs_phie, rhs_vmem, vmem sim_params = params assert sim_params is not None print "elecpy simulation start!" cuda.get_device(0).use() # Constants Sv = 1400 # Surface-to-volume ratio (cm^-1) Cm = 1.0 # Membrane capacitance (uF/cm^2) sigma_l_i = 1.74 # (mS/cm) sigma_t_i = 0.19 # (mS/cm) sigma_l_e = 6.25 # (mS/cm) sigma_t_e = 2.36 # (mS/cm) # Geometory settings im_h = sim_params['geometory']['height'] im_w = sim_params['geometory']['width'] ds = sim_params['geometory']['ds'] # Spatial discretization step (cm) N = im_h * im_w # Time settings udt = sim_params['time']['udt'] # Universal time step (ms) time_end = sim_params['time']['end'] # Logging settings cnt_log = sim_params['log']['cnt'] # num of udt for logging savepath = sim_params['log']['path'] # Cell model settings if sim_params['cell_type'] == 'ohararudy': cells = cell_model_ohararudy((N)) if sim_params['cell_type'] == 'luorudy': cells = cell_model_luorudy((N)) if sim_params['cell_type'] == 'mahajan': cells = cell_model_mahajan((N)) assert cells is not None print "Stimulation settings", stims_ext = [] stims_mem = [] if 'stimulation' in sim_params.keys(): stim_param = sim_params['stimulation'] if 'extracellular' in stim_param: for param in stim_param['extracellular']: stim = ExtracellularStimulator(**param) assert tuple(stim.shape) == (im_h, im_w) stims_ext.append(stim) if 'membrane' in stim_param: for param in stim_param['membrane']: stim = MembraneStimulator(**param) assert tuple(stim.shape) == (im_h, im_w) stims_mem.append(stim) print "...done" print "Allocating data...", cells.create() i_ion = np.zeros((N), dtype=np.float64) phie = np.zeros((N), dtype=np.float64) i_ext_e = np.zeros((N), dtype=np.float64) i_ext_i = np.zeros((N), dtype=np.float64) rhs_phie = np.zeros((N), dtype=np.float64) rhs_vmem = np.zeros((N), dtype=np.float64) tone = np.zeros((N), dtype=np.float64) vmem = np.copy(cells.get_param('v')) mask_ion = np.ones((im_h, im_w), dtype=np.float64) for h in range(im_h): for w in range(im_w): distance = (h - im_h // 2)**2 + (w - im_w // 2)**2 if distance < ((5**2) * 2): mask_ion[h, w] = 0. mask_ion = mask_ion.flatten() print "...done" print "Initializing data...", if 'restart' in sim_params.keys(): cnt_restart = sim_params['restart']['count'] srcpath = sim_params['restart']['source'] pfx = '_{0:0>4}'.format(cnt_restart) phie = np.load('{0}/phie{1}.npy'.format(srcpath, pfx)).flatten() vmem = np.load('{0}/vmem{1}.npy'.format(srcpath, pfx)).flatten() cells.load('{0}/cell{1}'.format(srcpath, pfx)) cnt_udt = cnt_restart * cnt_log print "...done" print 'Building PDE system ...', sigma_l = sigma_l_e + sigma_l_i sigma_t = sigma_t_e + sigma_t_i pde_i = PDE(im_h, im_w, sigma_l_i, sigma_t_i, ds) pde_m = PDE(im_h, im_w, sigma_l, sigma_t, ds) print '...done' # Initialization t = 0. # Time (ms) cnt_udt = 0 # Count of udt dstep = 1 # Time step (# of udt) cnt_save = -1 run_udt = True # Flag of running sim in udt flg_st = False # Flaf of stimulation cnt_st_off = 0 print 'Main loop start!' while t < time_end: t = conv_cntUdt2time(cnt_udt) dt = dstep * udt # Stimulation control i_ext_e[:] = 0.0 flg_st_temp = False for s in stims_ext: i_ext_e += s.get_current(t) * Sv flg_st_temp = flg_st_temp or s.get_flag(t) for s in stims_mem: cells.set_param('st', s.get_current(t)) # step.1 cell state transition cells.set_param('dt', dt) cells.set_param('v', cuda.to_gpu(vmem)) cells.update() i_ion = cells.get_param('it') i_ion = i_ion * mask_ion # step.2 phie rhs_phie = i_ext_e - i_ext_i - pde_i.forward(vmem) pde_cnt, phie = pde_m.solve(phie, rhs_phie, tol=1e-2, maxcnt=1e5) phie -= phie[0] # step.3 vmem rhs_vmem = pde_i.forward(vmem) rhs_vmem += pde_i.forward(phie) tone = (rhs_vmem * dt) / (Cm * Sv) rhs_vmem -= i_ion * Sv rhs_vmem += i_ext_i rhs_vmem *= 1 / (Cm * Sv) vmem += dt * rhs_vmem # Logging & error check cnt_save_now = conv_time2cntSave(t) if cnt_save_now != cnt_save: cnt_save = cnt_save_now sys.stdout.write('\r------------------{0}/{1}ms'.format( t, time_end)) sys.stdout.flush() np.save('{0}/phie_{1:0>4}'.format(savepath, cnt_save), phie.reshape((im_h, im_w))) np.save('{0}/vmem_{1:0>4}'.format(savepath, cnt_save), vmem.reshape((im_h, im_w))) np.save('{0}/tone_{1:0>4}'.format(savepath, cnt_save), tone.reshape((im_h, im_w))) cells.save('{0}/cell_{1:0>4}'.format(savepath, cnt_save)) yield vmem flg = False for i, v in enumerate(vmem): if v != v: print "error : invalid value {1} @ {0} ms, index {2}".format( t, v, i) flg = True break if flg is True: break # Stim off count if flg_st_temp is False: if flg_st is True: cnt_st_off = 0 else: cnt_st_off += 1 flg_st = flg_st_temp # Time step control if run_udt: if cnt_st_off >= 3 and cnt_udt % 10 == 0: dstep = 2 run_udt = False else: if pde_cnt > 5: dstep = 1 run_udt = True cnt_udt += dstep print "elecpy done" yield False
print("Train data loaded: %d" % len(train_vis)) print("Test data loaded: %d" % len(test_vis)) print("num of labels: %d" % num_labels) logging.info("Train data loaded: %d" % len(train_vis)) logging.info("Test data loaded: %d" % len(test_vis)) logging.info("num of labels: %d" % num_labels) # prepare model model = syuwa_cnn(num_labels) if args.restart_from is not None: model = pickle.load(open(args.restart_from, 'rb')) if args.gpu >= 0: import cupy cuda.check_cuda_available() cuda.get_device(args.device_num).use() model.to_gpu() def xparray(data): if args.gpu >= 0: return cupy.asnumpy(data) else: return data opt = get_optimizer(args.opt) opt.setup(model) train_vis = [np.asarray(x).astype(np.float32) for x in train_vis] train_dep = [np.asarray(x).astype(np.float32) for x in train_dep] test_vis = [np.asarray(x).astype(np.float32) for x in test_vis] test_dep = [np.asarray(x).astype(np.float32) for x in test_dep]
def gan_test(args, model_path): # Prepare Flow and Texture GAN model, defined in net.py gen_flow = net.FlowGenerator() serializers.load_npz(model_path["gen_flow"], gen_flow) gen_tex = net.Generator(dimz=100) serializers.load_npz(model_path["gen_tex"], gen_tex) if args.gpu >= 0: cuda.get_device(args.gpu).use() gen_flow.to_gpu() gen_tex.to_gpu() xp = np if args.gpu < 0 else cuda.cupy rows = 5 cols = 5 ### generate videos from Z np.random.seed(0) for i in range(10): print(i) z_flow = Variable(xp.asarray(gen_flow.make_hidden(rows * cols))) z_tex = Variable(xp.asarray(gen_tex.make_hidden(rows * cols))) ### generate flow with chainer.using_config('train', False): flow_fake, _, _ = gen_flow(z_flow) flow_fake_tmp = chainer.cuda.to_cpu(flow_fake.data) ### generate video with chainer.using_config('train', False): y, fore_vid, back_img, h_mask = gen_tex(z_tex, flow_fake) y = chainer.cuda.to_cpu(y.data) fore_vid = chainer.cuda.to_cpu(fore_vid.data) back_img = chainer.cuda.to_cpu(back_img.data) y_mask = chainer.cuda.to_cpu(h_mask.data) flow = flow_fake_tmp preview_dir = '{}/{:03}/'.format(args.out, i) if not os.path.exists(preview_dir): os.makedirs(preview_dir) ## save video y = np.asarray(np.clip((y + 1.) * (255. / 2.), 0.0, 255.0), dtype=np.uint8) B, CH, T, H, W = y.shape Y = y.reshape((rows, cols, CH, T, H, W)) Y = Y.transpose(3, 0, 4, 1, 5, 2) ### T, rows, H, cols, W, ch Y = Y.reshape((T, rows * H, cols * W, CH)) # T, H, W, ch for j in range(0, T): preview_path = preview_dir + 'img_{:03}.jpg'.format(j + 1) Image.fromarray(Y[j]).save(preview_path) ### save fore video y = np.asarray(np.clip((fore_vid + 1.) * (255. / 2.), 0.0, 255.0), dtype=np.uint8) B, CH, T, H, W = y.shape Y = y.reshape((rows, cols, CH, T, H, W)) Y = Y.transpose(3, 0, 4, 1, 5, 2) ### T, rows, H, cols, W, ch Y = Y.reshape((T, rows * H, cols * W, CH)) # T, H, W, ch for j in range(0, T): preview_path = preview_dir + 'fore_{:03}.jpg'.format(j + 1) Image.fromarray(Y[j]).save(preview_path) ### save mask video y = np.asarray(np.clip(y_mask * 255., 0.0, 255.0), dtype=np.uint8) B, CH, T, H, W = y.shape Y = y.reshape((rows, cols, CH, T, H, W)) Y = Y.transpose(3, 0, 4, 1, 5, 2) ### T, rows, H, cols, W, ch Y = Y.reshape((T, rows * H, cols * W, CH)) # T, H, W, ch for j in range(0, T): preview_path = preview_dir + 'mask_{:03}.jpg'.format(j + 1) Image.fromarray(Y[j]).save(preview_path) ### save back img y = np.asarray(np.clip((back_img + 1.) * (255. / 2.), 0.0, 255.0), dtype=np.uint8) B, CH, T, H, W = y.shape y = y[:, :, 0] Y = y.reshape((rows, cols, CH, H, W)) Y = Y.transpose(0, 3, 1, 4, 2) ### rows, H, cols, W, ch Y = Y.reshape((rows * H, cols * W, CH)) # T, H, W, ch preview_path = preview_dir + 'back.jpg' Image.fromarray(Y).save(preview_path) ### save flow y = np.asarray(np.clip((flow + 1.) * (255. / 2.), 0.0, 255.0), dtype=np.uint8) B, CH, T, H, W = y.shape Y = y.reshape((rows, cols, CH, T, H, W)) Y = Y.transpose(3, 0, 4, 1, 5, 2) ### T, rows, H, cols, W, ch Y = Y.reshape((T, rows * H, cols * W, CH)) # T, H, W, ch for j in range(0, T): preview_path = preview_dir + 'flow_{:03}.jpg'.format(j + 1) flow_img = np.hstack((Y[j, :, :, 0], Y[j, :, :, 1])) Image.fromarray(flow_img).save(preview_path)
def roll_out(self, args): """ compute expected rewards :param samples: generated_sample :param given: use x_0 ~ x_given as generated (state) :param dis: discriminator :param pool: multiprocess.Pool :param rollout_num: num of roll out :return: rewards (batch_size) """ tag = None if len(args) == 4: samples, given, dis, rollout_num = args elif len(args) == 5: samples, given, dis, rollout_num, gpu = args cuda.get_device(gpu).use() dis.to_gpu() self.to_gpu() elif len(args) == 6: samples, given, dis, rollout_num, gpu, tag = args else: raise AssertionError('undesired argument') batch_size = len(samples) self.reset_state() if tag is not None: self.lstm1.h = self.tag_embed( chainer.Variable(self.xp.array(tag, 'int32'))) gen_x = np.zeros((batch_size, self.sequence_length), 'int32') x = chainer.Variable(self.xp.asanyarray([self.start_token] * batch_size, 'int32'), volatile=True) self.decode_one_step(x, False) for i in range(given): gen_x[:, i] = samples[:, i] x = chainer.Variable(self.xp.asanyarray(samples[:, i], 'int32'), volatile=True) scores = self.decode_one_step(x, False) scores_ = scores self.save_state() rewards = [] for _ in range(rollout_num): self.set_state() scores = chainer.Variable(scores_.data.copy(), volatile=True) for i in range(given, self.sequence_length): pred = F.softmax(scores) pred = cuda.to_cpu(pred.data) generated = [ np.random.choice(self.vocab_size, p=pred[j]) for j in range(batch_size) ] # pred = cuda.to_cpu(pred.data) - np.finfo(np.float32).epsneg # generated = [] # for j in range(batch_size): # histogram = np.random.multinomial(1, pred[j]) # generated.append(int(np.nonzero(histogram)[0])) gen_x[:, i] = generated x = chainer.Variable(self.xp.asanyarray(generated, 'int32'), volatile=True) scores = self.decode_one_step(x, False) rewards.append(dis.get_reward(gen_x)) return np.mean(rewards, axis=0)
def main(): parser = argparse.ArgumentParser() # logging parser.add_argument('--logfile', '-l', default='', type=str, help='write log data into a file') parser.add_argument('--debug', '-d', action='store_true', help='run in debug mode') parser.add_argument('--silent', '-s', action='store_true', help='run in silent mode') parser.add_argument('--no-progress-bar', action='store_true', help='hide progress bar') # train and validate data parser.add_argument('--train', default='train.txt', type=str, help='set filename of training data') parser.add_argument('--validate', default='dev.txt', type=str, help='set filename of validation data') parser.add_argument('--vocab-size', '-V', default=0, type=int, help='set vocabulary size (0 means no limitation)') parser.add_argument( '--target-speaker', '-T', default='S', help='set target speaker name to be learned for system output') # file settings parser.add_argument('--initial-model', '-i', help='start training from an initial model') parser.add_argument('--model', '-m', required=True, help='set prefix of output model files') parser.add_argument( '--resume', action='store_true', help='resume training from a previously saved snapshot') parser.add_argument('--snapshot', type=str, help='dump a snapshot to a file after each epoch') # Model structure parser.add_argument('--enc-layer', default=2, type=int, help='number of encoder layers') parser.add_argument('--enc-esize', default=100, type=int, help='number of encoder input-embedding units') parser.add_argument('--enc-hsize', default=512, type=int, help='number of encoder hidden units') parser.add_argument('--dec-layer', default=2, type=int, help='number of decoder layers') parser.add_argument('--dec-esize', default=100, type=int, help='number of decoder input-embedding units') parser.add_argument('--dec-hsize', default=512, type=int, help='number of decoder hidden units') parser.add_argument('--dec-psize', default=100, type=int, help='number of decoder pre-output projection units') # training conditions parser.add_argument( '--optimizer', default='Adam', type=str, help="set optimizer (SGD, Adam, AdaDelta, RMSprop, ...)") parser.add_argument('--L2-weight', default=0.0, type=float, help="set weight for L2-regularization term") parser.add_argument('--clip-grads', default=5., type=float, help="set gradient clipping threshold") parser.add_argument('--dropout-rate', default=0.5, type=float, help="set dropout rate in training") parser.add_argument('--num-epochs', '-e', default=20, type=int, help='number of epochs to be trained') parser.add_argument('--learn-rate', '-R', default=1.0, type=float, help='set initial learning rate for SGD') parser.add_argument('--learn-decay', default=1.0, type=float, help='set decaying ratio of learning rate or epsilon') parser.add_argument( '--lower-bound', default=1e-16, type=float, help='set threshold of learning rate or epsilon for early stopping') parser.add_argument('--batch-size', '-b', default=50, type=int, help='set batch size for training and validation') parser.add_argument( '--max-batch-length', default=20, type=int, help='set maximum sequence length to control batch size') parser.add_argument('--seed', default=99, type=int, help='set a seed for random numbers') # select a GPU device parser.add_argument('--gpu', '-g', default=0, type=int, help='GPU ID (negative value indicates CPU)') args = parser.parse_args() # flush stdout if six.PY2: sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) # set up the logger tqdm_logging.config(logger, args.logfile, mode=('a' if args.resume else 'w'), silent=args.silent, debug=args.debug) # gpu setup if args.gpu >= 0: cuda.check_cuda_available() cuda.get_device(args.gpu).use() xp = cuda.cupy xp.random.seed(args.seed) else: xp = np # randomize np.random.seed(args.seed) random.seed(args.seed) logger.info('----------------------------------') logger.info('Train a neural conversation model') logger.info('----------------------------------') if args.resume: if not args.snapshot: logger.error('snapshot file is not spacified.') sys.exit() with open(args.snapshot, 'rb') as f: vocab, optimizer, status, args = pickle.load(f) logger.info('Resume training from epoch %d' % status.epoch) logger.info('Args ' + str(args)) model = optimizer.target else: logger.info('Args ' + str(args)) # Prepare RNN model and load data if args.initial_model: logger.info('Loading a model from ' + args.initial_model) with open(args.initial_model, 'rb') as f: vocab, model, tmp_args = pickle.load(f) status.cur_at = time.time() else: logger.info('Making vocabulary from ' + args.train) vocab = dialog_corpus.get_vocabulary(args.train, vocabsize=args.vocab_size) model = Sequence2SequenceModel( LSTMEncoder(args.enc_layer, len(vocab), args.enc_hsize, args.enc_esize, dropout=args.dropout_rate), LSTMDecoder(args.dec_layer, len(vocab), len(vocab), args.dec_esize, args.dec_hsize, args.dec_psize, dropout=args.dropout_rate)) # Setup optimizer optimizer = vars(optimizers)[args.optimizer]() if args.optimizer == 'SGD': optimizer.lr = args.learn_rate optimizer.use_cleargrads() optimizer.setup(model) optimizer.add_hook(chainer.optimizer.GradientClipping(args.clip_grads)) if args.L2_weight > 0.: optimizer.add_hook(chainer.optimizer.WeightDecay(args.L2_weight)) status = None logger.info('Loading text data from ' + args.train) train_set = dialog_corpus.load(args.train, vocab, args.target_speaker) logger.info('Loading validation data from ' + args.validate) validate_set = dialog_corpus.load(args.validate, vocab, args.target_speaker) logger.info('Making mini batches') train_batchset = dialog_corpus.make_minibatches( train_set, batchsize=args.batch_size, max_length=args.max_batch_length) validate_batchset = dialog_corpus.make_minibatches( validate_set, batchsize=args.batch_size, max_length=args.max_batch_length) # report data summary logger.info('vocabulary size = %d' % len(vocab)) logger.info('#train sample = %d #mini-batch = %d' % (len(train_set), len(train_batchset))) logger.info('#validate sample = %d #mini-batch = %d' % (len(validate_set), len(validate_batchset))) random.shuffle(train_batchset, random.random) # initialize status parameters if status is None: status = Status(max(round(len(train_batchset), -3) / 50, 500), progress_bar=not args.no_progress_bar) else: status.progress_bar = not args.no_progress_bar # move model to gpu if args.gpu >= 0: model.to_gpu() while status.epoch <= args.num_epochs: logger.info('---------------------training--------------------------') if args.optimizer == 'SGD': logger.info('Epoch %d/%d : SGD learning rate = %g' % (status.epoch, args.num_epochs, optimizer.lr)) else: logger.info( 'Epoch %d/%d : %s eps = %g' % (status.epoch, args.num_epochs, args.optimizer, optimizer.eps)) train_ppl = train_step(model, optimizer, train_set, train_batchset, status, xp) logger.info("epoch %d training perplexity: %f" % (status.epoch, train_ppl)) # write the model params modelfile = args.model + '.' + str(status.epoch) logger.info('writing model params to ' + modelfile) model.to_cpu() with open(modelfile, 'wb') as f: pickle.dump((vocab, model, args), f, -1) if args.gpu >= 0: model.to_gpu() # start validation step logger.info('---------------------validation------------------------') start_at = time.time() validate_ppl = validate_step(model, validate_set, validate_batchset, status, xp) logger.info('epoch %d validation perplexity: %.4f' % (status.epoch, validate_ppl)) # update best model with the minimum perplexity if status.min_validate_ppl >= validate_ppl: status.bestmodel_num = status.epoch logger.info('validation perplexity reduced: %.4f -> %.4f' % (status.min_validate_ppl, validate_ppl)) status.min_validate_ppl = validate_ppl elif args.optimizer == 'SGD': modelfile = args.model + '.' + str(status.bestmodel_num) logger.info('reloading model params from ' + modelfile) with open(modelfile, 'rb') as f: vocab, model, tmp_args = pickle.load(f) if args.gpu >= 0: model.to_gpu() optimizer.lr *= args.learn_decay if optimizer.lr < args.lower_bound: break optimizer.setup(model) else: optimizer.eps *= args.learn_decay if optimizer.eps < args.lower_bound: break status.new_epoch(validate_time=time.time() - start_at) # dump snapshot if args.snapshot: logger.info('writing snapshot to ' + args.snapshot) model.to_cpu() with open(args.snapshot, 'wb') as f: pickle.dump((vocab, optimizer, status, args), f, -1) if args.gpu >= 0: model.to_gpu() logger.info('----------------') # make a symbolic link to the best model logger.info('the best model is %s.%d.' % (args.model, status.bestmodel_num)) logger.info('a symbolic link is made as ' + args.model + '.best') if os.path.exists(args.model + '.best'): os.remove(args.model + '.best') os.symlink(os.path.basename(args.model + '.' + str(status.bestmodel_num)), args.model + '.best') logger.info('done')
def main(): parser = argparse.ArgumentParser() parser.add_argument("--batchsize", "-b", type=int, default=64) parser.add_argument("--total-epochs", "-e", type=int, default=5000) parser.add_argument("--num-labeled-data", "-nl", type=int, default=10000) parser.add_argument("--gpu-device", "-g", type=int, default=0) parser.add_argument("--grad-clip", "-gc", type=float, default=5) parser.add_argument("--learning-rate", "-lr", type=float, default=0.0001) parser.add_argument("--momentum", "-mo", type=float, default=0.5) parser.add_argument("--optimizer", "-opt", type=str, default="adam") parser.add_argument("--seed", type=int, default=0) parser.add_argument("--model", "-m", type=str, default="model.hdf5") args = parser.parse_args() np.random.seed(args.seed) model = Model() model.load(args.model) mnist_train, mnist_test = chainer.datasets.get_mnist() images_train, labels_train = mnist_train._datasets images_test, labels_test = mnist_test._datasets # normalize images_train = (images_train - 0.5) * 2 images_test = (images_test - 0.5) * 2 dataset = Dataset(train=(images_train, labels_train), test=(images_test, labels_test), num_labeled_data=args.num_labeled_data, num_classes=model.ndim_y - 1, num_extra_classes=1) print("#labeled: {}".format(dataset.get_num_labeled_data())) print("#unlabeled: {}".format(dataset.get_num_unlabeled_data())) _, labels = dataset.get_labeled_data() total_iterations_train = len(images_train) // args.batchsize # optimizers optimizer_encoder = Optimizer(args.optimizer, args.learning_rate, args.momentum) optimizer_encoder.setup(model.encoder) if args.grad_clip > 0: optimizer_encoder.add_hook(GradientClipping(args.grad_clip)) optimizer_decoder = Optimizer(args.optimizer, args.learning_rate, args.momentum) optimizer_decoder.setup(model.decoder) if args.grad_clip > 0: optimizer_decoder.add_hook(GradientClipping(args.grad_clip)) optimizer_discriminator = Optimizer(args.optimizer, args.learning_rate, args.momentum) optimizer_discriminator.setup(model.discriminator) if args.grad_clip > 0: optimizer_discriminator.add_hook(GradientClipping(args.grad_clip)) using_gpu = False if args.gpu_device >= 0: cuda.get_device(args.gpu_device).use() model.to_gpu() using_gpu = True xp = model.xp # 0 -> true sample # 1 -> generated sample class_true = np.zeros(args.batchsize, dtype=np.int32) class_fake = np.ones(args.batchsize, dtype=np.int32) if using_gpu: class_true = cuda.to_gpu(class_true) class_fake = cuda.to_gpu(class_fake) y_onehot_u = xp.zeros((1, model.ndim_y), dtype=xp.float32) y_onehot_u[0, -1] = 1 # turn on the extra class y_onehot_u = xp.repeat(y_onehot_u, args.batchsize, axis=0) training_start_time = time.time() for epoch in range(args.total_epochs): sum_loss_generator = 0 sum_loss_discriminator = 0 sum_loss_autoencoder = 0 sum_discriminator_confidence_true_l = 0 sum_discriminator_confidence_fake_l = 0 sum_discriminator_confidence_true_u = 0 sum_discriminator_confidence_fake_u = 0 epoch_start_time = time.time() dataset.shuffle() # training for itr in range(total_iterations_train): # update model parameters with chainer.using_config("train", True): # sample minibatch x_u = dataset.sample_unlabeled_minibatch(args.batchsize, gpu=using_gpu) x_l, y_l, y_onehot_l = dataset.sample_labeled_minibatch(args.batchsize, gpu=using_gpu) ### reconstruction phase ### if True: z_u = model.encode_x_z(x_u) x_reconstruction_u = model.decode_z_x(z_u) loss_reconstruction_u = F.mean_squared_error(x_u, x_reconstruction_u) z_l = model.encode_x_z(x_l) x_reconstruction_l = model.decode_z_x(z_l) loss_reconstruction_l = F.mean_squared_error(x_l, x_reconstruction_l) loss_reconstruction = loss_reconstruction_u + loss_reconstruction_l model.cleargrads() loss_reconstruction.backward() optimizer_encoder.update() optimizer_decoder.update() ### adversarial phase ### if True: z_fake_u = model.encode_x_z(x_u) z_fake_l = model.encode_x_z(x_l) if False: z_true_l = sampler.supervised_swiss_roll(args.batchsize, model.ndim_z, y_l, model.ndim_y - 1) z_true_u = sampler.swiss_roll(args.batchsize, model.ndim_z, model.ndim_y - 1) else: z_true_l = sampler.supervised_gaussian_mixture(args.batchsize, model.ndim_z, y_l, model.ndim_y - 1) z_true_u = sampler.gaussian_mixture(args.batchsize, model.ndim_z, model.ndim_y - 1) if using_gpu: z_true_u = cuda.to_gpu(z_true_u) z_true_l = cuda.to_gpu(z_true_l) dz_true_u = model.discriminate(y_onehot_u, z_true_u, apply_softmax=False) dz_fake_u = model.discriminate(y_onehot_u, z_fake_u, apply_softmax=False) dz_true_l = model.discriminate(y_onehot_l, z_true_l, apply_softmax=False) dz_fake_l = model.discriminate(y_onehot_l, z_fake_l, apply_softmax=False) discriminator_confidence_true_u = float(xp.mean(F.softmax(dz_true_u).data[:, 0])) discriminator_confidence_fake_u = float(xp.mean(F.softmax(dz_fake_u).data[:, 1])) discriminator_confidence_true_l = float(xp.mean(F.softmax(dz_true_l).data[:, 0])) discriminator_confidence_fake_l = float(xp.mean(F.softmax(dz_fake_l).data[:, 1])) loss_discriminator = (F.softmax_cross_entropy(dz_true_u, class_true) + F.softmax_cross_entropy(dz_fake_u, class_fake) + F.softmax_cross_entropy(dz_true_l, class_true) + F.softmax_cross_entropy(dz_fake_l, class_fake)) model.cleargrads() loss_discriminator.backward() optimizer_discriminator.update() ### generator phase ### if True: z_fake_u = model.encode_x_z(x_u) z_fake_l = model.encode_x_z(x_l) dz_fake_u = model.discriminate(y_onehot_u, z_fake_u, apply_softmax=False) dz_fake_l = model.discriminate(y_onehot_l, z_fake_l, apply_softmax=False) loss_generator = F.softmax_cross_entropy(dz_fake_u, class_true) + F.softmax_cross_entropy(dz_fake_l, class_true) model.cleargrads() loss_generator.backward() optimizer_encoder.update() sum_loss_discriminator += float(loss_discriminator.data) sum_loss_generator += float(loss_generator.data) sum_loss_autoencoder += float(loss_reconstruction.data) sum_discriminator_confidence_true_u += discriminator_confidence_true_u sum_discriminator_confidence_fake_u += discriminator_confidence_fake_u sum_discriminator_confidence_true_l += discriminator_confidence_true_l sum_discriminator_confidence_fake_l += discriminator_confidence_fake_l printr("Training ... {:3.0f}% ({}/{})".format((itr + 1) / total_iterations_train * 100, itr + 1, total_iterations_train)) model.save(args.model) clear_console() print("Epoch {} done in {} sec - loss: g={:.5g}, d={:.5g}, a={:.5g} - disc_u: true={:.1f}%, fake={:.1f}% - disc_l: true={:.1f}%, fake={:.1f}% - total {} min".format( epoch + 1, int(time.time() - epoch_start_time), sum_loss_generator / total_iterations_train, sum_loss_discriminator / total_iterations_train, sum_loss_autoencoder / total_iterations_train, sum_discriminator_confidence_true_u / total_iterations_train * 100, sum_discriminator_confidence_fake_u / total_iterations_train * 100, sum_discriminator_confidence_true_l / total_iterations_train * 100, sum_discriminator_confidence_fake_l / total_iterations_train * 100, int((time.time() - training_start_time) // 60)))
# and train an LDA-like model on it import os.path import pickle import time from chainer import cuda from chainer import serializers import chainer.optimizers as O import numpy as np from lda2vec import utils from lda2vec import prepare_topics, print_top_words_per_topic from lda2vec_model import LDA2Vec gpu_id = int(os.getenv('CUDA_GPU', 0)) cuda.get_device(gpu_id).use() print "Using GPU " + str(gpu_id) # You must run preprocess.py before this data becomes available vocab = pickle.load(open('vocab', 'r')) corpus = pickle.load(open('corpus', 'r')) data = np.load(open('data.npz', 'r')) flattened = data['flattened'] story_id = data['story_id'] author_id = data['author_id'] time_id = data['time_id'] ranking = data['ranking'].astype('float32') score = data['score'].astype('float32') # Model Parameters # Number of documents
default='0', type=int, help='whether to show the graph of loss values or not') parser.add_argument( '--lang', '-l', default='ja', type=str, help='the choice of a language (Japanese "ja" or English "en" )') args = parser.parse_args() # GPU settings gpu_device = args.gpu if args.gpu >= 0: cuda.check_cuda_available() cuda.get_device(gpu_device).use() def parse_ja_text(text): """ Function to parse Japanese text. :param text: string: sentence written by Japanese :return: list: parsed text """ import MeCab mecab = MeCab.Tagger("mecabrc") mecab.parse('') # list up noun mecab_result = mecab.parseToNode(text) parse_list = []
def experiment(): ite = 100 pdata = 1000 epoch = 100 batchsize = 1000 seed = 2018 gpu = True loss_pu = np.zeros((ite, epoch)) est_error_pu = np.zeros((ite, epoch)) est_error_pubp = np.zeros((ite, epoch)) est_precision_pu = np.zeros((ite, epoch)) est_recall_pu = np.zeros((ite, epoch)) est_precision1_pubp = np.zeros((ite, epoch)) est_recall1_pubp = np.zeros((ite, epoch)) est_precision2_pubp = np.zeros((ite, epoch)) est_recall2_pubp = np.zeros((ite, epoch)) est_precision3_pubp = np.zeros((ite, epoch)) est_recall3_pubp = np.zeros((ite, epoch)) est_precision4_pubp = np.zeros((ite, epoch)) est_recall4_pubp = np.zeros((ite, epoch)) est_precision5_pubp = np.zeros((ite, epoch)) est_recall5_pubp = np.zeros((ite, epoch)) xp = np.loadtxt('xp.csv') xq = np.loadtxt('xq.csv') xn = np.loadtxt('xn.csv') x_train = np.concatenate([xp, xq, xn], axis=0) t_train = np.concatenate([np.ones(len(xp)), np.zeros(len(xq)+len(xn))], axis=0) x_test = np.concatenate([xq, xn], axis=0) t_test = np.concatenate([np.ones(len(xq)), np.zeros(len(xn))], axis=0) for i in range(ite): np.random.seed(seed) #PN classification (KM1, KM2) = wrapper(x_train[t_train==0],x_train[t_train==1]) pi = KM2 #pi = np.mean(t_test) print(x_train.shape) print(t_train.shape) print(x_test.shape) print(t_test.shape) print(pi) dim = x_train.shape[1] model = MultiLayerPerceptron(dim) optimizer = optimizers.Adam(alpha=1e-5) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(0.005)) if gpu: gpu_device = 0 cuda.get_device(gpu_device).use() model.to_gpu(gpu_device) xp = cuda.cupy else: xp = np model, optimizer, loss_list, acc1, acc2, pre0, rec0, pre1, rec1, pre2, rec2, pre3, rec3, pre4, rec4, pre5, rec5, pre, rec = train_pu(x_train, t_train, x_test, t_test, pi, epoch, model, optimizer, batchsize, xp) if i == 0: pres = pre recs = rec else: pres += pre recs += rec loss_pu[i] = loss_list est_error_pu[i] = acc1 est_error_pubp[i] = acc2 est_precision_pu[i] = pre0 est_recall_pu[i] = rec0 est_precision1_pubp[i] = pre1 est_recall1_pubp[i] = rec1 est_precision2_pubp[i] = pre2 est_recall2_pubp[i] = rec2 est_precision3_pubp[i] = pre3 est_recall3_pubp[i] = rec3 est_precision4_pubp[i] = pre4 est_recall4_pubp[i] = rec4 est_precision5_pubp[i] = pre5 est_recall5_pubp[i] = rec5 print(acc1[-1]) print(acc2[-1]) seed += 1 np.savetxt('loss_pu_elkan_%d.csv'%seed, loss_pu, delimiter=',') np.savetxt('est_error_pu_elkan_%d.csv'%seed, est_error_pu, delimiter=',') np.savetxt('est_error_pubp_elkan_%d.csv'%seed, est_error_pubp, delimiter=',') np.savetxt('est_precision_pu_elkan_%d.csv'%seed, est_precision_pu, delimiter=',') np.savetxt('est_recal-l_pu_elkan_%d.csv'%seed, est_recall_pu, delimiter=',') np.savetxt('est_precision-1_pubp_elkan_%d.csv'%seed, est_precision1_pubp, delimiter=',') np.savetxt('est_recall-1_pubp_elkan_%d.csv'%seed, est_recall1_pubp, delimiter=',') np.savetxt('est_precision-2_pubp_elkan_%d.csv'%seed, est_precision2_pubp, delimiter=',') np.savetxt('est_recall-2_pubp_elkan_%d.csv'%seed, est_recall2_pubp, delimiter=',') np.savetxt('est_precision-3_pubp_elkan_%d.csv'%seed, est_precision3_pubp, delimiter=',') np.savetxt('est_recall-3_pubp_elkan_%d.csv'%seed, est_recall3_pubp, delimiter=',') np.savetxt('est_precision-4_pubp_elkan_%d.csv'%seed, est_precision4_pubp, delimiter=',') np.savetxt('est_recall-4_pubp_elkan_%d.csv'%seed, est_recall4_pubp, delimiter=',') np.savetxt('est_precision-5_pubp_elkan_%d.csv'%seed, est_precision5_pubp, delimiter=',') np.savetxt('est_recall-5_pubp_elkan_%d.csv'%seed, est_recall5_pubp, delimiter=',') np.savetxt('precisions_pubp_elkan_%d.csv'%seed, pres/(i+1), delimiter=',') np.savetxt('recalls_pubp_elkan_%d.csv'%seed, recs/(i+1), delimiter=',') loss_pu_mean = np.mean(loss_pu, axis=1) est_error_pu_mean = np.mean(est_error_pu, axis=1) est_error_pubp_mean = np.mean(est_error_pubp, axis=1) est_error_pu_std = np.std(est_error_pu, axis=1) est_error_pubp_std = np.std(est_error_pubp, axis=1) return loss_pu_mean, est_error_pu_mean, est_error_pubp_mean, est_error_pu_std, est_error_pubp_std
def test_get_dummy_device(self): if not cuda.available: self.assertIs(cuda.get_device(), cuda.DummyDevice)
def CifarAnalysis(folderName=None, n_epoch=1, batchsize=1000, **kwd): id_gpu = 0 OutStr = "" OutStr += 'GPU: {}\n'.format(id_gpu) OutStr += 'Minibatch-size: {}\n'.format(batchsize) OutStr += 'epoch: {}\n'.format(n_epoch) OutStr += 'kwd: {}\n'.format(kwd) OutStr += '' print OutStr fOutput = None fInfo = None if folderName: if not os.path.exists(folderName): os.makedirs(folderName) fOutput = open(os.path.join(folderName, "output.dat"), "w") fInfo = open(os.path.join(folderName, "info.dat"), "w") shutil.copyfile(__file__, os.path.join(folderName, os.path.basename(__file__))) if fInfo: fInfo.write(OutStr) # Prepare dataset InDataBatch = [] data_tr = np.zeros((50000, 3 * 32 * 32), dtype=np.float32) data_ev = np.zeros((10000, 3 * 32 * 32), dtype=np.float32) label_tr = np.zeros((50000), dtype=np.int32) label_ev = np.zeros((10000), dtype=np.int32) for i in range(1, 5 + 1): with open("data_cifar10/data_batch_%d" % i, "r") as f: tmp = pickle.load(f) data_tr[(i - 1) * 10000:i * 10000] = tmp["data"] label_tr[(i - 1) * 10000:i * 10000] = tmp["labels"] with open("data_cifar10/test_batch", "r") as f: tmp = pickle.load(f) data_ev[:] = tmp["data"] label_ev[:] = tmp["labels"] ## Prep print "Normalizing data ..." def Normalize(x): avg = np.average(x, axis=1).reshape((len(x), 1)) std = np.sqrt(np.sum(x * x, axis=1) - np.sum(x, axis=1)).reshape( (len(x), 1)) y = (x - avg) / std return y data_tr = Normalize(data_tr) data_ev = Normalize(data_ev) print "done" x_tr = data_tr.reshape((len(data_tr), 3, 32, 32)) x_ev = data_ev.reshape((len(data_ev), 3, 32, 32)) y_tr = label_tr y_ev = label_ev N_tr = len(data_tr) # 50000 N_ev = len(data_ev) # 10000 ## Define analisis Resume = None if "Resume" in kwd: Resume = kwd["Resume"] del kwd["Resume"] model = L.Classifier( ImageProcessNetwork(I_colors=3, I_Xunit=32, I_Yunit=32, F_unit=10, **kwd)) if id_gpu >= 0: cuda.get_device(id_gpu).use() model.to_gpu() xp = np if id_gpu < 0 else cuda.cupy # Setup optimizer optimizer = optimizers.Adam() optimizer.setup(model) # Init/Resume if Resume: print('Load optimizer state from', Resume) serializers.load_hdf5(Resume + ".state", optimizer) serializers.load_hdf5(Resume + ".model", model) # Learning loop if fOutput: fOutput.write("epoch,mode,loss,accuracy\n") for epoch in six.moves.range(1, n_epoch + 1): print 'epoch %d' % epoch # training perm = np.random.permutation(N_tr) sum_accuracy = 0 sum_loss = 0 start = time.time() for i in six.moves.range(0, N_tr, batchsize): x = chainer.Variable(xp.asarray(x_tr[perm[i:i + batchsize]])) t = chainer.Variable(xp.asarray(y_tr[perm[i:i + batchsize]])) # Pass the loss function (Classifier defines it) and its arguments model.predictor.setTrainMode(True) optimizer.update(model, x, t) if (epoch == 1 and i == 0) and folderName: with open(os.path.join(folderName, 'graph.dot'), 'w') as o: g = computational_graph.build_computational_graph( (model.loss, )) o.write(g.dump()) print 'graph generated' sum_loss += float(model.loss.data) * len(t.data) sum_accuracy += float(model.accuracy.data) * len(t.data) end = time.time() elapsed_time = end - start throughput = N_tr / elapsed_time print 'train mean loss=%.3f, accuracy=%.1f%%, throughput=%.0f images/sec' % ( sum_loss / N_tr, sum_accuracy / N_tr * 100., throughput) if fOutput: fOutput.write("%d,Train,%e,%e\n" % (epoch, sum_loss / N_tr, sum_accuracy / N_tr)) # evaluation sum_accuracy = 0 sum_loss = 0 for i in six.moves.range(0, N_ev, batchsize): x = chainer.Variable(xp.asarray(x_ev[i:i + batchsize]), volatile='on') t = chainer.Variable(xp.asarray(y_ev[i:i + batchsize]), volatile='on') model.predictor.setTrainMode(False) loss = model(x, t) sum_loss += float(loss.data) * len(t.data) sum_accuracy += float(model.accuracy.data) * len(t.data) print 'test mean loss=%.3f, accuracy=%.1f%%' % ( sum_loss / N_ev, sum_accuracy / N_ev * 100, ) if fOutput: fOutput.write("%d,Test,%e,%e\n" % (epoch, sum_loss / N_ev, sum_accuracy / N_ev)) if folderName and (epoch % 10 == 0 or epoch == n_epoch): # Save the model and the optimizer if epoch == n_epoch: myFname = os.path.join(folderName, 'mlp_final') else: myFname = os.path.join(folderName, 'mlp_%d' % n_epoch) #print 'save the model' serializers.save_hdf5(myFname + ".model", model) serializers.save_hdf5(myFname + ".state", optimizer) if fOutput: fOutput.close() if fInfo: fInfo.close()