def predict(self, div, epoch=-1, model_name='nn'):
"""
epoch == -1: load from latest saved model
"""
data_gen = self.get_sample_generator(div, opt.infer_batch_size)
total_steps = int(
np.ceil(self.h5[div]['pid'].shape[0] / opt.infer_batch_size))
use_n_gpus = set_num_gpus(1)
nn = NN(model_name, len(self.char_dict))
with tf.Session() as sess:
# 주의: predict를 multi gpu에서 실행 시 batch split 시 문제 발생
nn.build_model(sess, use_n_gpus=use_n_gpus)
sess.run(tf.global_variables_initializer())
nn.load(epoch)
y_preds = nn.predict(data_gen, total_steps)
self.write_prediction_result(div, y_preds)
def train(self, use_n_gpus, resume=-1, model_name='nn', reverse=False):
"""
resume == 0: not load
resume == -1: load from latest saved model (not from last epoch model)
resume > 0: load {resume}th epoch model
"""
train_gen = self.get_sample_generator('train',
opt.batch_size,
reverse=reverse)
train_steps = int(np.ceil(self.train_data_len / opt.batch_size))
val_gen = self.get_sample_generator('val', opt.batch_size)
val_steps = int(np.ceil(self.val_data_len / opt.batch_size))
use_n_gpus = set_num_gpus(use_n_gpus)
nn = NN(model_name, len(self.char_dict))
with tf.Session() as sess:
nn.build_model(sess, use_n_gpus)
sess.run(tf.global_variables_initializer())
nn.fit(train_gen, train_steps, val_gen, val_steps, opt.num_epochs,
resume)
def ensemble(self, div, *model_epochs):
"""
model_epoch = ('model_name1', epoch_num, 'model_name2', epoch_num, ...)
epoch_num이 -1이면 제일 마지막 epoch 사용
"""
softmax_file_names = []
print(model_epochs)
for i in range(0, len(model_epochs), 2):
model_name = model_epochs[i]
epoch = model_epochs[i + 1]
self.logger.info('{}: {} epoch'.format(model_name, epoch))
data_gen = self.get_sample_generator(div, opt.infer_batch_size)
total_data_len = self.h5[div]['pid'].shape[0]
total_steps = int(np.ceil(total_data_len / opt.infer_batch_size))
use_n_gpus = set_num_gpus(1)
nn = NN(model_name, len(self.char_dict))
with tf.Session(graph=tf.Graph()) as sess:
nn.build_model(sess, use_n_gpus=use_n_gpus)
sess.run(tf.global_variables_initializer())
epoch = nn.load(epoch)
y_softmaxs = nn.predict(data_gen, total_steps, softmax=True)
file_name = '%s_%dep_%s_softmax.tmp' % (model_name, epoch, div)
softmax_file_names.append(file_name)
with h5py.File(file_name, 'w') as fout:
fout.create_dataset('b',
shape=(total_data_len,
opt.num_bcate + 1))
fout.create_dataset('m',
shape=(total_data_len,
opt.num_mcate + 1))
fout.create_dataset('s',
shape=(total_data_len,
opt.num_scate + 1))
fout.create_dataset('d',
shape=(total_data_len,
opt.num_dcate + 1))
fout['b'][:] = np.array(y_softmaxs['b'])
fout['m'][:] = np.array(y_softmaxs['m'])
fout['s'][:] = np.array(y_softmaxs['s'])
fout['d'][:] = np.array(y_softmaxs['d'])
del y_softmaxs
# load softmax.tmp, sum, argmax
chunk_size = 50000
steps = int(np.ceil(total_data_len / chunk_size))
y_preds = {'b': [], 'm': [], 's': [], 'd': []}
softmax_files = [h5py.File(name, 'r') for name in softmax_file_names]
for cate in ['b', 'm', 's', 'd']:
self.logger.info('%s category processing...' % cate)
for i in range(steps):
# softmax h5파일을 chunk_size만큼 읽어서 메모리에 올린다.
softmax_per_model = []
for softmax_file in softmax_files:
softmax = softmax_file[cate][i * chunk_size:(i + 1) *
chunk_size]
softmax_per_model.append(softmax)
for j in range(len(softmax_per_model[0])):
softmax_sum = np.zeros_like(softmax_per_model[0][0])
for softmax in softmax_per_model:
softmax_sum += softmax[j]
y_preds[cate].append(np.argmax(softmax_sum))
del softmax_per_model
self.write_prediction_result(div, y_preds)