def predict(self, div, epoch=-1, model_name='nn'): """ epoch == -1: load from latest saved model """ data_gen = self.get_sample_generator(div, opt.infer_batch_size) total_steps = int( np.ceil(self.h5[div]['pid'].shape[0] / opt.infer_batch_size)) use_n_gpus = set_num_gpus(1) nn = NN(model_name, len(self.char_dict)) with tf.Session() as sess: # 주의: predict를 multi gpu에서 실행 시 batch split 시 문제 발생 nn.build_model(sess, use_n_gpus=use_n_gpus) sess.run(tf.global_variables_initializer()) nn.load(epoch) y_preds = nn.predict(data_gen, total_steps) self.write_prediction_result(div, y_preds)
def train(self, use_n_gpus, resume=-1, model_name='nn', reverse=False): """ resume == 0: not load resume == -1: load from latest saved model (not from last epoch model) resume > 0: load {resume}th epoch model """ train_gen = self.get_sample_generator('train', opt.batch_size, reverse=reverse) train_steps = int(np.ceil(self.train_data_len / opt.batch_size)) val_gen = self.get_sample_generator('val', opt.batch_size) val_steps = int(np.ceil(self.val_data_len / opt.batch_size)) use_n_gpus = set_num_gpus(use_n_gpus) nn = NN(model_name, len(self.char_dict)) with tf.Session() as sess: nn.build_model(sess, use_n_gpus) sess.run(tf.global_variables_initializer()) nn.fit(train_gen, train_steps, val_gen, val_steps, opt.num_epochs, resume)
def ensemble(self, div, *model_epochs): """ model_epoch = ('model_name1', epoch_num, 'model_name2', epoch_num, ...) epoch_num이 -1이면 제일 마지막 epoch 사용 """ softmax_file_names = [] print(model_epochs) for i in range(0, len(model_epochs), 2): model_name = model_epochs[i] epoch = model_epochs[i + 1] self.logger.info('{}: {} epoch'.format(model_name, epoch)) data_gen = self.get_sample_generator(div, opt.infer_batch_size) total_data_len = self.h5[div]['pid'].shape[0] total_steps = int(np.ceil(total_data_len / opt.infer_batch_size)) use_n_gpus = set_num_gpus(1) nn = NN(model_name, len(self.char_dict)) with tf.Session(graph=tf.Graph()) as sess: nn.build_model(sess, use_n_gpus=use_n_gpus) sess.run(tf.global_variables_initializer()) epoch = nn.load(epoch) y_softmaxs = nn.predict(data_gen, total_steps, softmax=True) file_name = '%s_%dep_%s_softmax.tmp' % (model_name, epoch, div) softmax_file_names.append(file_name) with h5py.File(file_name, 'w') as fout: fout.create_dataset('b', shape=(total_data_len, opt.num_bcate + 1)) fout.create_dataset('m', shape=(total_data_len, opt.num_mcate + 1)) fout.create_dataset('s', shape=(total_data_len, opt.num_scate + 1)) fout.create_dataset('d', shape=(total_data_len, opt.num_dcate + 1)) fout['b'][:] = np.array(y_softmaxs['b']) fout['m'][:] = np.array(y_softmaxs['m']) fout['s'][:] = np.array(y_softmaxs['s']) fout['d'][:] = np.array(y_softmaxs['d']) del y_softmaxs # load softmax.tmp, sum, argmax chunk_size = 50000 steps = int(np.ceil(total_data_len / chunk_size)) y_preds = {'b': [], 'm': [], 's': [], 'd': []} softmax_files = [h5py.File(name, 'r') for name in softmax_file_names] for cate in ['b', 'm', 's', 'd']: self.logger.info('%s category processing...' % cate) for i in range(steps): # softmax h5파일을 chunk_size만큼 읽어서 메모리에 올린다. softmax_per_model = [] for softmax_file in softmax_files: softmax = softmax_file[cate][i * chunk_size:(i + 1) * chunk_size] softmax_per_model.append(softmax) for j in range(len(softmax_per_model[0])): softmax_sum = np.zeros_like(softmax_per_model[0][0]) for softmax in softmax_per_model: softmax_sum += softmax[j] y_preds[cate].append(np.argmax(softmax_sum)) del softmax_per_model self.write_prediction_result(div, y_preds)