def check_generic(comm, length, bs): assert bs > 0 assert length > 0 a = list(range(comm.rank, length, comm.size)) b = list(range(comm.rank, length, comm.size)) c = list(range(comm.rank, length, comm.size)) model = ExampleModel() dataset = TupleDataset(a, b, c) iterator = SerialIterator(dataset, bs, shuffle=False, repeat=False) evaluator = GenericMultiNodeEvaluator(comm, iterator, model) results = evaluator(None) # Make expected answer iterator.reset() s = [ [ aa + bb + cc # Same calculation as model for aa, bb, cc in batch ] for batch in iterator ] s = comm.gather_obj(s) if comm.rank == 0: # flatten list of lists gathered expected = [] for e in zip(*s): expected.extend(e) for e, r in zip(expected, results): chainer.testing.assert_allclose(e, r) else: assert results is None
class UnpairedIterator(iterator.Iterator): """An iterator for unpaired dataset which wraps two SerialIterator. """ def __init__(self, dataset1, dataset2, batch_size, repeat=True): if len(dataset2) < len(dataset1): self._main_iter = SerialIterator(dataset1, batch_size=batch_size, repeat=repeat, shuffle=True) self._sub_iter = SerialIterator(dataset2, batch_size=batch_size, repeat=True, shuffle=True) self._rev = False else: self._main_iter = SerialIterator(dataset2, batch_size=batch_size, repeat=repeat, shuffle=True) self._sub_iter = SerialIterator(dataset1, batch_size=batch_size, repeat=True, shuffle=True) self._rev = True def __next__(self): if self._rev: return [ x for x in zip(self._sub_iter.next(), self._main_iter.next()) ] else: return [ x for x in zip(self._main_iter.next(), self._sub_iter.next()) ] next = __next__ @property def epoch(self): return self._main_iter.epoch @property def epoch_detail(self): return self._main_iter.epoch_detail @property def previous_epoch_detail(self): return self._main_iter.previous_epoch_detail def reset(self): self._main_iter.reset() self._sub_iter.reset() @property def repeat(self): return self._main_iter.repeat @property def is_new_epoch(self): return self._main_iter.is_new_epoch
def get_trainer_and_reporter( trial:Trial, model:CbLossClassifier, iter_test:iterators.SerialIterator, iter_train:iterators.SerialIterator, batch_converter, args, device=0, best_params={}): if best_params != {}:# 過去の best_param 使う場合 learning_rate = best_params['learning_rate'] else: learning_rate = trial.suggest_loguniform('learning_rate', 1e-5, 1e-1) grad_clipping = trial.suggest_uniform('grad_clipping',0,1.0) optimizer = optimizers.SGD(lr=learning_rate) optimizer.setup(model) optimizer.add_hook(optimizer_hooks.GradientClipping(threshold=grad_clipping)) updater = training.StandardUpdater( iter_train, optimizer, device=device, converter=batch_converter ) early_trigger = training.triggers.EarlyStoppingTrigger( check_trigger=(1, "epoch"), monitor="validation/main/accuracy", patients=3, mode="max", max_trigger=(args.epoch, "epoch") ) trainer = training.Trainer(updater,early_trigger,out='optuna') trainer.extend(extensions.Evaluator(iter_test, model,device=device,converter=batch_converter)) snapshot_writer = training.extensions.snapshot_writers.ThreadQueueWriter() trainer.extend(training.extensions.snapshot_object( target=model, filename='model_{}.npz'.format(args.desc), writer=snapshot_writer),trigger=(10,'epoch')) reporter = extensions.LogReport() trainer.extend(reporter) trainer.extend(integration.ChainerPruningExtension( trial,args.pruning_key,(args.pruning_trigger_epoch,'epoch'))) iter_test.reset() iter_train.reset() return trainer,reporter
def train(): batchsize = 128 max_epoch = 10 device = 0 train_data, test_data = mnist.get_mnist(withlabel=True, ndim=1) train_iter = SerialIterator(train_data, batchsize) test_iter = SerialIterator(test_data, batchsize, repeat=False, shuffle=False) model = MyNetwork() if chainer.cuda.available and device >= 0: model.to_gpu(device) else: device = -1 optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9) optimizer.setup(model) while train_iter.epoch < max_epoch: train_batch = train_iter.next() image_train, target_train = concat_examples(train_batch, device) prediction_train = model(image_train) loss = F.softmax_cross_entropy(prediction_train, target_train) model.cleargrads() loss.backward() optimizer.update() if train_iter.is_new_epoch: loss_array = float(chainer.backends.cuda.to_cpu(loss.array)) print("epoch{:2d} train_loss:{:.04f}".format(train_iter.epoch, loss_array)) test_losses = [] test_accs = [] while True: test_batch = test_iter.next() image_test, target_test = concat_examples(test_batch, device) prediction_test = model(image_test) loss_test = F.softmax_cross_entropy(prediction_test, target_test) test_losses.append(chainer.backends.cuda.to_cpu(loss_test.array)) acc = F.accuracy(prediction_test, target_test) test_accs.append(chainer.backends.cuda.to_cpu(acc.array)) if test_iter.is_new_epoch: test_iter.reset() break mean_loss = np.mean(test_losses) mean_acc = np.mean(test_accs) print("val_loss:{:.04f} val_accuracy:{:.04f}".format(mean_loss, mean_acc)) chainer.serializers.save_npz("model.npz", model)
def check_custom(comm, length, bs): assert bs > 0 assert length > 0 a = list(range(comm.rank, length, comm.size)) b = list(range(comm.rank, length, comm.size)) c = list(range(comm.rank, length, comm.size)) model = ExampleModel() dataset = TupleDataset(a, b, c) iterator = SerialIterator(dataset, bs, shuffle=False, repeat=False) evaluator = CustomMultiNodeEvaluator(comm, iterator, model) result = evaluator(None) iterator.reset() expected = comm.allreduce_obj(sum(2 for batch in iterator)) if comm.rank == 0: assert expected == result else: assert result is None
optimizer.setup(net) for param in net.params(): if param.name != 'b': # バイアス以外だったら param.update_rule.add_hook(WeightDecay(0.0001)) # 重み減衰 ## ネットワークを訓練 gpu_id = 0 # 仕様するGPU番号 n_batch = 64 # バッチサイズ n_epoch = 50 # エポック数 ### ネットワークをGPUメモリ上に転送 net.to_gpu(gpu_id) ### ログ results_train, results_valid = {}, {} results_train['loss'], results_train['accuracy'] = [], [] results_valid['loss'], results_valid['accuracy'] = [], [] train_iter.reset() # 上で一度next()が呼ばれているため count = 1 for epoch in range(n_epoch): while True: #### ミニバッチの取得 train_batch = train_iter.next() #### xとtに分割 ##### データをGPUに転送するために,concat_examplesにgpu_idを渡す x_train, t_train = chainer.dataset.concat_examples(train_batch, gpu_id) ##### 予測値と目的関数を計算 y_train = net(x_train) loss_train = F.softmax_cross_entropy(y_train, t_train) acc_train = F.accuracy(y_train, t_train) ##### 勾配の初期化と勾配の計算 net.cleargrads() loss_train.backward()
def train(batch_size, epoch_count, lamda, datasetA_folder_path, datasetB_folder_path, output_path): dataset_A = data_io.dataset_load(datasetA_folder_path) train_iter_A = SerialIterator(dataset_A, batch_size, repeat=True, shuffle=True) dataset_B = data_io.dataset_load(datasetB_folder_path) train_iter_B = SerialIterator(dataset_B, batch_size, repeat=True, shuffle=True) g_ab = Generator() g_ba = Generator() d_a = Discriminator() d_b = Discriminator() g_ab.to_gpu(0) g_ba.to_gpu(0) d_a.to_gpu(0) d_b.to_gpu(0) opt_g_ab = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5) opt_g_ab.setup(g_ab) opt_g_ba = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5) opt_g_ba.setup(g_ba) opt_d_a = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5) opt_d_a.setup(d_a) opt_d_b = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5) opt_d_b.setup(d_b) iteration = 0 train_iter_A.reset() train_iter_B.reset() log_list = [] image_path = output_path + "image/" disA_model_path = output_path + "dis_A/" disB_model_path = output_path + "dis_B/" genAB_model_path = output_path + "gen_AB/" genBA_model_path = output_path + "gen_BA/" os.mkdir(output_path) os.mkdir(image_path) os.mkdir(disA_model_path) os.mkdir(disB_model_path) os.mkdir(genAB_model_path) os.mkdir(genBA_model_path) for epoch in range(epoch_count): d_a_loss_list = [] d_b_loss_list = [] g_AB_loss_list = [] g_BA_loss_list = [] while True: mini_batch_images_A = train_iter_A.next() mini_batch_images_A = np.array(mini_batch_images_A) mini_batch_images_A = (mini_batch_images_A - 128.0) / 128.0 real_a = Variable(np.array(mini_batch_images_A)) real_a.to_gpu(0) mini_batch_images_B = train_iter_B.next() mini_batch_images_B = np.array(mini_batch_images_B) mini_batch_images_B = (mini_batch_images_B - 128.0) / 128.0 real_b = Variable(np.array(mini_batch_images_B)) real_b.to_gpu(0) fake_b = g_ab(real_a) fake_a = g_ba(real_b) reconstr_a = g_ba(fake_b) reconstr_b = g_ab(fake_a) d_a_real_result = d_a(real_a) d_a_fake_result = d_a(fake_a) loss_d_a = loss_dis(batch_size, d_a_real_result, d_a_fake_result) d_b_real_result = d_b(real_b) d_b_fake_result = d_b(fake_b) loss_d_b = loss_dis(batch_size, d_b_real_result, d_b_fake_result) d_a.cleargrads() loss_d_a.backward() opt_d_a.update() d_b.cleargrads() loss_d_b.backward() opt_d_b.update() """generatorのloss計算""" loss_g_ab = loss_gen(batch_size, d_b_fake_result, real_a, reconstr_a, lamda) loss_g_ba = loss_gen(batch_size, d_a_fake_result, real_b, reconstr_b, lamda) g_ab.cleargrads() loss_g_ab.backward() opt_g_ab.update() g_ba.cleargrads() loss_g_ba.backward() opt_g_ba.update() loss_d_a.to_cpu() loss_d_b.to_cpu() loss_g_ab.to_cpu() loss_g_ba.to_cpu() iteration += batch_size d_a_loss_list.append(loss_d_a.array) d_b_loss_list.append(loss_d_b.array) g_AB_loss_list.append(loss_g_ab.array) g_BA_loss_list.append(loss_g_ba.array) if train_iter_A.is_new_epoch or train_iter_B.is_new_epoch: break real_a.to_cpu() fake_b.to_cpu() reconstr_a.to_cpu() real_b.to_cpu() fake_a.to_cpu() reconstr_b.to_cpu() real_a_images = real_a.array.transpose(0, 2, 3, 1) fake_b_images = fake_b.array.transpose(0, 2, 3, 1) reconstr_a_images = reconstr_a.array.transpose(0, 2, 3, 1) real_b_images = real_b.array.transpose(0, 2, 3, 1) fake_a_images = fake_a.array.transpose(0, 2, 3, 1) reconstr_b_images = reconstr_b.array.transpose(0, 2, 3, 1) data_io.output_images(image_path + str(epoch), real_a_images, fake_b_images, reconstr_a_images, real_b_images, fake_a_images, reconstr_b_images) print("epoch: " + str(epoch) + ", interation: " + str(iteration) + \ ", d_A_loss: " + str(np.mean(d_a_loss_list)) + ", d_B_loss: " + str(np.mean(d_b_loss_list)) + \ ", g_AB_loss: " + str(np.mean(g_AB_loss_list)) + ", g_BA_loss: " + str(np.mean(g_BA_loss_list))) log_json = {"epoch": str(epoch), "interation": str(iteration), \ "d_A_loss": str(np.mean(d_a_loss_list)), "d_B_loss": str(np.mean(d_b_loss_list)), \ "g_AB_loss": str(np.mean(g_AB_loss_list)), "g_BA_loss": str(np.mean(g_BA_loss_list))} log_list.append(log_json) with open(output_path + 'log.json', 'w') as log_file: json.dump(log_list, log_file, indent=4) if (epoch % 100 == 0): g_ab.to_cpu() g_ba.to_cpu() d_a.to_cpu() d_b.to_cpu() save_npz(genAB_model_path + str(epoch) + '.npz', g_ab) save_npz(genBA_model_path + str(epoch) + '.npz', g_ba) save_npz(disA_model_path + str(epoch) + '.npz', d_a) save_npz(disB_model_path + str(epoch) + '.npz', d_b) g_ab.to_gpu(0) g_ba.to_gpu(0) d_a.to_gpu(0) d_b.to_gpu(0) g_ab.to_cpu() g_ba.to_cpu() d_a.to_cpu() d_b.to_cpu() save_npz(genAB_model_path + 'last.npz', g_ab) save_npz(genBA_model_path + 'last.npz', g_ba) save_npz(disA_model_path + 'last.npz', d_a) save_npz(disB_model_path + 'last.npz', d_b)
def train(batch_size, epoch_count, dataset_folder_path, n_hidden, output_path): dataset = data_io.dataset_load(dataset_folder_path) train_iter = SerialIterator(dataset, batch_size, repeat=True, shuffle=True) gen = Generator(n_hidden=n_hidden) dis = Discriminator() gen.to_gpu(0) dis.to_gpu(0) opt_gen = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5) opt_gen.setup(gen) opt_dis = chainer.optimizers.Adam(alpha=0.0002, beta1=0.5) opt_dis.setup(dis) iteration = 0 train_iter.reset() log_list = [] image_path = output_path + "image/" dis_model_path = output_path + "dis/" gen_model_path = output_path + "gen/" os.mkdir(output_path) os.mkdir(image_path) os.mkdir(dis_model_path) os.mkdir(gen_model_path) for epoch in range(epoch_count): d_loss_list = [] g_loss_list = [] while True: mini_batch_images = train_iter.next() mini_batch_images = np.array(mini_batch_images) mini_batch_images = (mini_batch_images - 128.0) / 128.0 x_real = Variable(np.array(mini_batch_images)) x_real.to_gpu(0) y_real = dis(x_real) noise = xp.random.uniform(-1, 1, (batch_size, n_hidden), dtype=np.float32) z = Variable(noise) x_fake = gen(z, batch_size) y_fake = dis(x_fake) d_loss = loss_dis(batch_size, y_real, y_fake) g_loss = loss_gen(batch_size, y_fake) dis.cleargrads() d_loss.backward() opt_dis.update() gen.cleargrads() g_loss.backward() opt_gen.update() d_loss.to_cpu() g_loss.to_cpu() iteration += batch_size d_loss_list.append(d_loss.array) g_loss_list.append(g_loss.array) if train_iter.is_new_epoch: break x_fake.to_cpu() generated_images = x_fake.array generated_images = generated_images.transpose(0, 2, 3, 1) Image.fromarray( np.clip(generated_images[0] * 255, 0.0, 255.0).astype( np.uint8)).save(image_path + str(epoch) + ".png") print("epoch: " + str(epoch) + ", interation: " + str(iteration) + ", d_loss: " + str(np.mean(d_loss_list)) + ", g_loss: " + str(np.mean(g_loss_list))) log_json = { "epoch": str(epoch), "iteration": str(iteration), "d_loss": str(np.mean(d_loss_list)), "g_loss": str(np.mean(g_loss_list)) } log_list.append(log_json) with open(output_path + 'log.json', 'w') as log_file: json.dump(log_list, log_file, indent=4) if (epoch % 100 == 0): dis.to_cpu() save_npz(dis_model_path + str(epoch) + '.npz', dis) gen.to_cpu() save_npz(gen_model_path + str(epoch) + '.npz', gen) gen.to_gpu(0) dis.to_gpu(0) logGraph.save_log_graph(output_path + 'log.json', output_path + "lossGraph.png") dis.to_cpu() save_npz(dis_model_path + 'last.npz', dis) gen.to_cpu() save_npz(gen_model_path + 'last.npz', gen)
f'train mean loss: {sum_loss / train_count}, accuracy: {sum_accuracy / train_count}' ) # evaluation (每轮学习之后,进行一次测试数据集的测试) sum_accuracy = 0 sum_loss = 0 # enable evaluation mode with configuration.using_config('train', False): # this is optional but can reduce computational with chainer.useing_config('enable_backprop', False): for batch in test_iter: x, t = conver.concat_examples(batch, gpuid) x = chainer.Variable(x) t = chainer.Variable(t) loss = model(x, t) sum_loss += float(loss.data) * len(t.data) sum_accuracy += (float(model.accuracy.data) * len(t.data)) # 迭代器状态还原 test_iter.reset() print( f'test mean loss: {sum_loss / test_count}, accuracy: {sum_accuracy / test_count}' ) sum_accuracy = 0 sum_loss = 0 # save the model and the optimizer (保存模型和优化参数) print('save the model') serializers.save_npz('{}/mlp.model'.format(outdir), model) # 加载模型 # serializers.load_npz('{}/mlp.model'.format(outdir),model)
def main(args): random.seed(0) np.random.seed(0) if args.gpu >= 0: cuda.get_device_from_id(args.gpu).use() cuda.cupy.random.seed(0) dataset, id2ene = load_dataset(args.dataset, args.features, args.redirects) print(f'# of examples in dataset: {len(dataset)}') def batch2tensors(batch, device): xp = cuda.cupy if device >= 0 else np xf = xp.zeros((len(batch), args.n_feature), dtype='f') xe = xp.zeros((len(batch), args.embed_size), dtype='f') t = xp.zeros((len(batch), len(id2ene)), dtype='i') for i, item in enumerate(batch): for feature_id in item['feature_ids']: if feature_id < args.n_feature: xf[i, feature_id] = 1.0 if item['embedding']: xe[i] = xp.array(item['embedding'], dtype='f') for ene_id in item['ene_ids']: t[i, ene_id] = 1 x = xp.concatenate((xf, xe), axis=1) return x, t cv_datasets = get_cross_validation_datasets(dataset, args.cv) ys = [] ts = [] for split_idx, cv_dataset in enumerate(cv_datasets): print(f'cross validation ({split_idx + 1}/{len(cv_datasets)})') train, test = cv_dataset train_iter = SerialIterator(train, batch_size=args.batch) test_iter = SerialIterator(test, batch_size=args.batch, repeat=False, shuffle=False) model = ENEClassifier(in_size=args.n_feature + args.embed_size, hidden_size=args.hidden_size, out_size=len(id2ene)) if args.gpu >= 0: model.to_gpu(args.gpu) optimizer = optimizers.Adam() optimizer.setup(model) updater = StandardUpdater(train_iter, optimizer, converter=batch2tensors, device=args.gpu) trainer = Trainer(updater, (args.epoch, 'epoch'), out=args.out_dir) trainer.extend(extensions.LogReport()) trainer.extend( extensions.snapshot_object( model, filename='epoch_{.updater.epoch}.model')) trainer.extend( extensions.Evaluator(test_iter, model, converter=batch2tensors, device=args.gpu)) trainer.extend( extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'elapsed_time'])) trainer.extend(extensions.ProgressBar(update_interval=1)) trainer.run() test_iter.reset() for batch in test_iter: x, t = batch2tensors(batch, device=args.gpu) with chainer.using_config('train', False): y = model.predict(x) ys.append(y) ts.append(t) y_all = F.concat(ys, axis=0) t_all = F.concat(ts, axis=0) prediction_matrix = (y_all.data >= 0.5).astype('f') reference_matrix = (t_all.data == 1).astype('f') accuracy_matrix = prediction_matrix * reference_matrix eb_pred = prediction_matrix.sum( axis=1) # entity-based num. of predicted classes eb_ref = reference_matrix.sum( axis=1) # entity-based num. of reference classes eb_acc = accuracy_matrix.sum( axis=1) # entity-based num. of accurate classes eb_nopred = (eb_pred == 0.).astype('f') # for avoiding zero-division eb_precision = (eb_acc / (eb_pred + eb_nopred)).mean() eb_recall = (eb_acc / eb_ref).mean() eb_f1 = (2 * eb_acc / (eb_pred + eb_ref)).mean() cb_pred = prediction_matrix.sum( axis=0) # class-based num. of predicted examples cb_ref = reference_matrix.sum( axis=0) # class-based num. of reference examples cb_acc = accuracy_matrix.sum( axis=0) # class-based num. of accurate examples cb_nopred = (cb_pred == 0.).astype('f') # for avoiding zero-division cb_macro_precision = (cb_acc / (cb_pred + cb_nopred)).mean() cb_macro_recall = (cb_acc / cb_ref).mean() cb_macro_f1 = (2 * cb_acc / (cb_pred + cb_ref)).mean() cb_micro_precision = cb_acc.sum() / cb_pred.sum() cb_micro_recall = cb_acc.sum() / cb_ref.sum() cb_micro_f1 = (2 * cb_acc.sum()) / (cb_pred.sum() + cb_ref.sum()) print(f'Entity-based Precision: {float(eb_precision):.2%}') print(f'Entity-based Recall: {float(eb_recall):.2%}') print(f'Entity-based F1 score: {float(eb_f1):.2%}') print(f'Class-based macro Precision: {float(cb_macro_precision):.2%}') print(f'Class-based macro Recall: {float(cb_macro_recall):.2%}') print(f'Class-based macro F1 score: {float(cb_macro_f1):.2%}') print(f'Class-based micro Precision: {float(cb_micro_precision):.2%}') print(f'Class-based micro Recall: {float(cb_micro_recall):.2%}') print(f'Class-based micro F1 score: {float(cb_micro_f1):.2%}') print(f'writing out classification results') with open(Path(args.out_dir) / 'classification_result.json', 'w') as fo: for i, item in tqdm(enumerate(dataset)): title = item['title'] predicted_classes = [ id2ene[j] for j, v in enumerate(prediction_matrix[i]) if v == 1.0 ] reference_classes = [ id2ene[j] for j, v in enumerate(reference_matrix[i]) if v == 1.0 ] out = { 'title': title, 'prediction': predicted_classes, 'reference': reference_classes } print(json.dumps(out, ensure_ascii=False), file=fo)