log.info('in : "%s"' % s) log.info('out: "%s"' % WordSpacing.spacing(s.replace(' ', ''), labels)) log.info('sample testing OK.\n') if not os.path.exists(model_file + '.index') or not os.path.exists(model_file + '.meta'): if n_train >= int('100,000'.replace(',', '')): SlackUtil.send_message('%s start (max_sentences=%s, left_gram=%s, right_gram=%.1f)' % (sys.argv[0], n_train, left_gram, right_gram)) WordSpacing.learning(total_epoch, n_train, n_valid, n_test, batch_size, left_gram, right_gram, model_file, features_vector, labels_vector, n_hidden1=n_hidden1, learning_rate=learning_rate, early_stop_cost=early_stop_cost) if n_train >= int('100,000'.replace(',', '')): SlackUtil.send_message('%s end (max_sentences=%s, left_gram=%s, right_gram=%.1f)' % (sys.argv[0], n_train, left_gram, right_gram)) log.info('chek result...') watch = WatchUtil() watch.start('read sentences') sentences = [] # '아버지가 방에 들어 가신다.', '가는 말이 고와야 오는 말이 곱다.'] max_test_sentences = 100 if n_train >= int('100,000'.replace(',', '')): sentences_file = test_sentences_file else: sentences_file = train_sentences_file with gzip.open(sentences_file, 'rt', encoding='utf8') as f: for i, line in enumerate(f, 1): if len(sentences) >= max_test_sentences: break s = line.strip()
def learning(cls, total_epoch, n_train, n_valid, n_test, batch_size, left_gram, right_gram, model_file, features_vector, labels_vector, n_hidden1=100, learning_rate=0.01, early_stop_cost=0.001): ngram = left_gram + right_gram n_features = len(features_vector) * ngram # number of features = 17,380 * 4 n_classes = len(labels_vector) if len(labels_vector) >= 3 else 1 # number of classes = 2 but len=1 log.info('load characters list...') log.info('load characters list OK. len: %s\n' % NumUtil.comma_str(len(features_vector))) watch = WatchUtil() train_file = os.path.join(KO_WIKIPEDIA_ORG_DIR, 'datasets', 'word_spacing', 'ko.wikipedia.org.dataset.sentences=%s.left=%d.right=%d.train.gz' % (n_train, left_gram, right_gram)) valid_file = os.path.join(KO_WIKIPEDIA_ORG_DIR, 'datasets', 'word_spacing', 'ko.wikipedia.org.dataset.sentences=%s.left=%d.right=%d.test.gz' % (n_valid, left_gram, right_gram)) test_file = os.path.join(KO_WIKIPEDIA_ORG_DIR, 'datasets', 'word_spacing', 'ko.wikipedia.org.dataset.sentences=%s.left=%d.right=%d.valid.gz' % (n_test, left_gram, right_gram)) if not os.path.exists(train_file) or not os.path.exists(valid_file) or not os.path.exists(test_file): dataset_dir = os.path.dirname(train_file) if not os.path.exists(dataset_dir): os.makedirs(dataset_dir) watch.start('create dataset') log.info('create dataset...') data_files = (('train', KO_WIKIPEDIA_ORG_TRAIN_SENTENCES_FILE, n_train, train_file, False), ('valid', KO_WIKIPEDIA_ORG_VALID_SENTENCES_FILE, n_valid, valid_file, False), ('test', KO_WIKIPEDIA_ORG_TEST_SENTENCES_FILE, n_test, test_file, False)) for name, data_file, total, dataset_file, to_one_hot_vector in data_files: check_interval = 10000 log.info('check_interval: %s' % check_interval) log.info('%s %s total: %s' % (name, os.path.basename(data_file), NumUtil.comma_str(total))) features, labels = [], [] with gzip.open(data_file, 'rt', encoding='utf8') as f: for i, line in enumerate(f, 1): if total < i: break if i % check_interval == 0: time.sleep(0.01) # prevent cpu overload percent = i / total * 100 log.info('create dataset... %.1f%% readed. data len: %s. %s' % (percent, NumUtil.comma_str(len(features)), data_file)) _f, _l = WordSpacing.sentence2features_labels(line.strip(), left_gram=left_gram, right_gram=right_gram) features.extend(_f) labels.extend(_l) dataset = DataSet(features=features, labels=labels, features_vector=features_vector, labels_vector=labels_vector, name=name) log.info('dataset save... %s' % dataset_file) dataset.save(dataset_file, gzip_format=True, verbose=True) log.info('dataset save OK. %s' % dataset_file) log.info('dataset: %s' % dataset) log.info('create dataset OK.') log.info('') watch.stop('create dataset') watch.start('dataset load') log.info('dataset load...') train = DataSet.load(train_file, gzip_format=True, verbose=True) if n_train >= int('100,000'.replace(',', '')): valid = DataSet.load(valid_file, gzip_format=True, verbose=True) else: valid = DataSet.load(train_file, gzip_format=True, verbose=True) log.info('valid.convert_to_one_hot_vector()...') valid = valid.convert_to_one_hot_vector(verbose=True) log.info('valid.convert_to_one_hot_vector() OK.') log.info('train dataset: %s' % train) log.info('valid dataset: %s' % valid) log.info('dataset load OK.') log.info('') watch.stop('dataset load') graph = WordSpacing.build_FFNN(n_features, n_classes, n_hidden1, learning_rate, watch) train_step, X, Y, cost, predicted, accuracy = graph['train_step'], graph['X'], graph['Y'], graph['cost'], graph['predicted'], graph['accuracy'] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) check_interval = 10 # max(1, min(1000, n_train // 10)) nth_train, nth_input, total_input = 0, 0, total_epoch * train.size log.info('learn...') log.info('total: %s' % NumUtil.comma_str(train.size)) watch.start('learn') valid_cost = sys.float_info.max for epoch in range(1, total_epoch + 1): if valid_cost < early_stop_cost: break for step, (features_batch, labels_batch) in enumerate(train.next_batch(batch_size=batch_size), 1): if valid_cost < early_stop_cost: log.info('valid_cost: %s, early_stop_cost: %s, early stopped.' % (valid_cost, early_stop_cost)) break nth_train += 1 nth_input += features_batch.shape[0] sess.run(train_step, feed_dict={X: features_batch, Y: labels_batch}) # if step % check_interval == 1: percent = nth_input / total_input * 100 valid_cost = sess.run(cost, feed_dict={X: valid.features, Y: valid.labels}) log.info('[epoch=%s][%.1f%%] %s cost: %.4f' % (epoch, percent, valid.name, valid_cost)) watch.stop('learn') log.info('learn OK.\n') log.info('model save... %s' % model_file) watch.start('model save...') model_dir = os.path.dirname(model_file) if not os.path.exists(model_dir): os.makedirs(model_dir) saver = tf.train.Saver() saver.save(sess, model_file) watch.stop('model save...') log.info('model save OK. %s' % model_file) log.info('\n') log.info('batch_size: %s' % batch_size) log.info(watch.summary()) log.info('\n')
def learning(cls, total_epoch, n_train, n_valid, n_test, batch_size, window_size, noise_rate, model_file, features_vector, labels_vector, n_hidden1, learning_rate, dropout_keep_rate, early_stop_cost=0.001): n_features = len(features_vector) * window_size # number of features = 17,382 * 10 log.info('load characters list...') log.info('load characters list OK. len: %s' % NumUtil.comma_str(len(features_vector))) watch = WatchUtil() train_file = os.path.join(KO_WIKIPEDIA_ORG_DIR, 'datasets', 'spelling_error_correction', 'ko.wikipedia.org.dataset.sentences=%s.window_size=%d.train.gz' % (n_train, window_size)) valid_file = os.path.join(KO_WIKIPEDIA_ORG_DIR, 'datasets', 'spelling_error_correction', 'ko.wikipedia.org.dataset.sentences=%s.window_size=%d.valid.gz' % (n_valid, window_size)) test_file = os.path.join(KO_WIKIPEDIA_ORG_DIR, 'datasets', 'spelling_error_correction', 'ko.wikipedia.org.dataset.sentences=%s.window_size=%d.test.gz' % (n_test, window_size)) log.info('train_file: %s' % train_file) log.info('valid_file: %s' % valid_file) log.info('test_file: %s' % test_file) if not os.path.exists(train_file) or not os.path.exists(valid_file) or not os.path.exists(test_file): dataset_dir = os.path.dirname(train_file) if not os.path.exists(dataset_dir): os.makedirs(dataset_dir) watch.start('create dataset') # FIXME: out of memory (1M sentences) log.info('create dataset...') data_files = (('train', KO_WIKIPEDIA_ORG_TRAIN_SENTENCES_FILE, n_train, train_file, False), ('valid', KO_WIKIPEDIA_ORG_VALID_SENTENCES_FILE, n_valid, valid_file, False), ('test', KO_WIKIPEDIA_ORG_TEST_SENTENCES_FILE, n_test, test_file, False)) for (name, data_file, total, dataset_file, to_one_hot_vector) in data_files: check_interval = 10000 log.info('check_interval: %s' % check_interval) log.info('%s %s total: %s' % (name, os.path.basename(data_file), NumUtil.comma_str(total))) log.info('noise_rate: %s' % noise_rate) features, labels = [], [] with gzip.open(data_file, 'rt') as f: for i, line in enumerate(f, 1): if total < i: break if i % check_interval == 0: time.sleep(0.01) # prevent cpu overload percent = i / total * 100 log.info('create dataset... %.1f%% readed. data len: %s. %s' % (percent, NumUtil.comma_str(len(features)), data_file)) sentence = line.strip() for start in range(0, len(sentence) - window_size + 1): # 문자 단위로 노이즈(공백) 생성 chars = sentence[start: start + window_size] for idx in range(len(chars)): noised_chars = StringUtil.replace_with_index(chars, ' ', idx) features.append(noised_chars) labels.append(chars) log.debug('create dataset... %s "%s" -> "%s"' % (name, noised_chars, chars)) # log.info('noise_sampling: %s' % noise_sampling) # for nth_sample in range(noise_sampling): # 초성, 중성, 종성 단위로 노이즈 생성 # for start in range(0, len(sentence) - window_size + 1): # chars = sentence[start: start + window_size] # noised_chars = SpellingErrorCorrection.encode_noise(chars, noise_rate=noise_rate, noise_with_blank=True) # if chars == noised_chars: # continue # if i % check_interval == 0 and nth_sample == 0: # log.info('create dataset... %s "%s" -> "%s"' % (name, noised_chars, chars)) # features.append(noised_chars) # labels.append(chars) # print('dataset features:', features) # print('dataset labels:', labels) dataset = DataSet(features=features, labels=labels, features_vector=features_vector, labels_vector=labels_vector, name=name) log.info('dataset save... %s' % dataset_file) dataset.save(dataset_file, gzip_format=True, verbose=True) log.info('dataset save OK. %s' % dataset_file) log.info('dataset: %s' % dataset) log.info('create dataset OK.') log.info('') watch.stop('create dataset') watch.start('dataset load') log.info('dataset load...') train = DataSet.load(train_file, gzip_format=True, verbose=True) if n_train >= int('100,000'.replace(',', '')): valid = DataSet.load(valid_file, gzip_format=True, verbose=True) else: valid = DataSet.load(train_file, gzip_format=True, verbose=True) log.info('valid.convert_to_one_hot_vector()...') valid = valid.convert_to_one_hot_vector(verbose=True) log.info('valid.convert_to_one_hot_vector() OK.') log.info('train dataset: %s' % train) log.info('valid dataset: %s' % valid) log.info('dataset load OK.') log.info('') watch.stop('dataset load') X, Y, dropout_keep_prob, train_step, cost, y_hat, accuracy = SpellingErrorCorrection.build_DAE(n_features, window_size, noise_rate, n_hidden1, learning_rate, watch) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) check_interval = max(1, min(1000, n_train // 10)) nth_train, nth_input, total_input = 0, 0, total_epoch * train.size log.info('') log.info('learn...') log.info('total_epoch: %s' % total_epoch) log.info('train.size (total features): %s' % NumUtil.comma_str(train.size)) log.info('check_interval: %s' % check_interval) log.info('total_epoch: %s' % total_epoch) log.info('batch_size: %s' % batch_size) log.info('total_input: %s (total_epoch * train.size)' % total_input) log.info('') watch.start('learn') valid_cost = sys.float_info.max for epoch in range(1, total_epoch + 1): if valid_cost < early_stop_cost: log.info('valid_cost: %s, early_stop_cost: %s, early stopped.' % (valid_cost, early_stop_cost)) break for step, (features_batch, labels_batch) in enumerate(train.next_batch(batch_size=batch_size, to_one_hot_vector=True), 1): if valid_cost < early_stop_cost: break nth_train += 1 nth_input += features_batch.shape[0] sess.run(train_step, feed_dict={X: features_batch, Y: labels_batch, dropout_keep_prob: dropout_keep_rate}) # if nth_train % check_interval == 1: percent = nth_input / total_input * 100 valid_cost = sess.run(cost, feed_dict={X: valid.features, Y: valid.labels, dropout_keep_prob: 1.0}) log.info('[epoch=%s][%.1f%%] %s cost: %.8f' % (epoch, percent, valid.name, valid_cost)) watch.stop('learn') log.info('learn OK.') log.info('') log.info('model save... %s' % model_file) watch.start('model save...') model_dir = os.path.dirname(model_file) if not os.path.exists(model_dir): os.makedirs(model_dir) saver = tf.train.Saver() saver.save(sess, model_file) watch.stop('model save...') log.info('model save OK. %s' % model_file) log.info('') log.info('total_epoch: %s' % total_epoch) log.info('batch_size: %s' % batch_size) log.info('total_input: %s (total_epoch * train.size)' % total_input) log.info('') log.info(watch.summary()) log.info('')
valid_x_batch, valid_y_batch = input_pipeline([valid_file], batch_size=n_valid, delim='\t', splits=3) sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(max_to_keep=None) train_writer = tf.summary.FileWriter(TENSORBOARD_LOG_DIR + '/train', sess.graph) valid_writer = tf.summary.FileWriter(TENSORBOARD_LOG_DIR + '/valid', sess.graph) coordinator = tf.train.Coordinator() # coordinator for enqueue threads threads = tf.train.start_queue_runners(sess=sess, coord=coordinator) # start filename queue batch_count = math.ceil(n_train / batch_size) # batch count for one epoch try: watch = WatchUtil() stop_timer = TimerUtil(interval_secs=total_train_time) valid_timer = TimerUtil(interval_secs=valid_check_interval) watch.start() stop_timer.start() valid_timer.start() nth_batch, min_valid_epoch, min_valid_cost = 0, 0, 1e10 epoch, running = 0, True while running: epoch += 1 for i in range(1, batch_count + 1): if stop_timer.is_over(): running = False break if coordinator.should_stop(): break
def train(self, iterations: int, batch: int, embedding: Word2VecEmbedding, args: argparse.Namespace) -> str: batches_in_epoch = int(numpy.ceil( len(self.dataloader.dataset) / batch)) total_batches = batches_in_epoch * iterations nth_total_batch = 0 log.info(f'batches_in_epoch: {batches_in_epoch}') log.info(f'total_batches: {total_batches}') watch = WatchUtil(auto_stop=False) watch.start() best_loss = float("inf") first_epoch, last_epoch = self.epoch + 1, self.epoch + iterations + 1 last_embedding_file = None log.info(Word2VecEmbedding.get_filenpath(args)) for self.epoch in range(first_epoch, last_epoch): log.info(f"[e{self.epoch:2d}] {self}") loss_list = [] for nth, (iword, owords) in enumerate(self.dataloader, 1): try: loss = self.sgns(iword, owords) except RuntimeError: loss_list = [float('-inf')] break self.optim.zero_grad() loss.backward() self.optim.step() # if nth_batch == 1 and self.scheduler is not None and self.epoch >= self.decay_start_epoch: # TODO: TEST # self.scheduler.step() if self.learning_decay != 0: PytorchUtil.set_learning_rate(self.optim, self.epoch, gamma=self.learning_decay, base_lr=self.init_lr, min_lr=1e-10, decay_start=2, decay_interval=3) lr = PytorchUtil.get_learning_rate(self.optim) _, negatives = owords.size() real_loss = loss.data[0] / float(negatives) loss_list.append(real_loss) nth_total_batch += 1 progressed = nth_total_batch / total_batches seconds_per_batch = float( watch.elapsed()) / float(nth_total_batch) remain_batches = total_batches - nth_total_batch remain_secs = int(seconds_per_batch * remain_batches) if nth == 1 or nth == batches_in_epoch or nth % 1000 == 0: log.info( f"[e{self.epoch:2d}][b{nth:5d}/{batches_in_epoch:5d}][{progressed*100:.1f}% remain: {DateUtil.secs_to_string(remain_secs)}][window: {self.window}][lr: {lr:.0e}] loss: {real_loss:.7f}" ) total_loss = numpy.mean(loss_list) log.info( f"[e{self.epoch:2d}][window: {self.window}][lr: {lr:.0e}] total_loss: {total_loss:.7f}, best_loss: {best_loss:.7f}" ) if total_loss > best_loss or total_loss == float( 'inf') or total_loss == float( '-inf'): # bad loss than before or diverge log.info('') log.info( f"[e{self.epoch:2d}][window: {self.window}][lr: {lr:.0e}] total_loss > best_loss BREAK" ) log.info('') break else: if best_loss < total_loss: best_loss = total_loss log.info( f"[e{self.epoch:2d}][window: {self.window}][lr: {lr:.0e}] embedding.save()..." ) args.epoch = self.epoch last_embedding_file = embedding.save( idx2vec=trainer.embedding, filepath=Word2VecEmbedding.get_filenpath(args)) log.info( f"[e{self.epoch:2d}][window: {self.window}][lr: {lr:.0e}] embedding.save() OK. {os.path.basename(embedding.filepath)}" ) return last_embedding_file
# log.info('model_file: %s' % model_file) if not os.path.exists(model_dir): # log.info('model_dir: %s' % model_dir) os.makedirs(model_dir) config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) saver = tf.train.Saver() with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) train_writer = tf.summary.FileWriter(TENSORBOARD_LOG_DIR + '/train', sess.graph) valid_writer = tf.summary.FileWriter(TENSORBOARD_LOG_DIR + '/valid', sess.graph) max_cost = 1e10 best_epoch, best_cost = 0, 1e10 watch.start('train') running, epoch = True, 0 stop_timer = TimerUtil(interval_secs=train_time) valid_timer = TimerUtil(interval_secs=valid_check_interval) stop_timer.start() valid_timer.start() while running: if stop_timer.is_over(): break epoch += 1 _, train_rsme, train_summary = sess.run([train_step, rsme, summary], feed_dict={x: x_train, y: y_train}) train_writer.add_summary(train_summary, global_step=epoch) train_writer.flush()
def learning(cls, sentences_file, batch_size, left_gram, right_gram, model_file, features_vector, labels_vector, n_hidden1=100, max_sentences=0, learning_rate=0.01, layers=2): ngram = left_gram + right_gram n_features = len( features_vector) * ngram # number of features = 17,380 * 4 n_classes = len(labels_vector) if len( labels_vector) >= 3 else 1 # number of classes = 2 but len=1 log.info('load characters list...') log.info('load characters list OK. len: %s\n' % NumUtil.comma_str(len(features_vector))) watch = WatchUtil() train_file = os.path.join( KO_WIKIPEDIA_ORG_DATA_DIR, 'datasets', 'ko.wikipedia.org.dataset.sentences=%d.left=%d.right=%d.train.gz' % (max_sentences, left_gram, right_gram)) validation_file = train_file.replace('.train.', '.validation.') test_file = train_file.replace('.train.', '.test.') if not os.path.exists(train_file) or not os.path.exists( validation_file) or not os.path.exists(test_file): watch.start('create dataset') log.info('create dataset...') features, labels = [], [] check_interval = min(10000, math.ceil(max_sentences)) log.info('total: %s' % NumUtil.comma_str(max_sentences)) with gzip.open(sentences_file, 'rt') as f: for i, line in enumerate(f, 1): if max_sentences < i: break if i % check_interval == 0: log.info( 'create dataset... %.1f%% readed. data len: %s' % (i / max_sentences * 100, NumUtil.comma_str(len(features)))) _f, _l = WordSpacing.sentence2features_labels( line.strip(), left_gram=left_gram, right_gram=right_gram) features.extend(_f) labels.extend(_l) dataset = DataSet(features=features, labels=labels, features_vector=features_vector, labels_vector=labels_vector, name='all') log.info('dataset: %s' % dataset) log.info('create dataset OK.\n') watch.stop('create dataset') watch.start('dataset save') log.info('split to train, test, validation...') datasets = DataSets.to_datasets(dataset, test_rate=0.1, valid_rate=0.1, test_max=10000, valid_max=1000, shuffle=True) train, test, validation = datasets.train, datasets.test, datasets.validation log.info(train) log.info(test) log.info(validation) # log.info('%s %s' % (test.features[0], test.labels[0])) log.info('split to train, test, validation OK.\n') log.info('dataset save... %s' % train_file) train.save(train_file, verbose=True) # save as text log.info('dataset save OK.\n') log.info('dataset save... %s' % validation_file) validation = validation.convert_to_one_hot_vector( verbose=True) # save as vector validation.save(validation_file, verbose=True) log.info('dataset save OK.\n') log.info('dataset save... %s' % test_file) test = test.convert_to_one_hot_vector(verbose=True) test.save(test_file, verbose=True) # save as vector log.info('dataset save OK.\n') watch.stop('dataset save') else: watch.start('dataset load') log.info('dataset load...') train = DataSet.load(train_file, verbose=True) validation = DataSet.load(validation_file, verbose=True) test = DataSet.load(test_file, verbose=True) log.info(train) log.info(validation) log.info(test) log.info('dataset load OK.\n') watch.stop('dataset load') log.info('check samples...') for i, (features_batch, labels_batch) in enumerate( train.next_batch(batch_size=5, to_one_hot_vector=True), 1): if i > 2: break for a, b in zip(features_batch, labels_batch): feature, label = a, b _feature = feature.reshape((ngram, len(features_vector))) chars = ''.join(features_vector.to_values(_feature)) has_space = np.argmax(label) log.info('[%s] %s -> %s, %s (len=%s) %s (len=%s)' % (i, chars, has_space, feature, len(feature), label, len(label))) log.info('check samples OK.\n') graph = WordSpacing.build_FFNN(n_features, n_classes, n_hidden1, learning_rate, watch, layers=layers) train_step, X, Y, cost, hypothesis, predicted, accuracy = graph[ 'train_step'], graph['X'], graph['Y'], graph['cost'], graph[ 'hypothesis'], graph['predicted'], graph['accuracy'] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) n_input = 0 log.info('total: %s' % NumUtil.comma_str(train.size)) log.info('learn...') watch.start('learn') for step, (features_batch, labels_batch) in enumerate( train.next_batch(batch_size=batch_size), 1): n_input += batch_size sess.run(train_step, feed_dict={ X: features_batch, Y: labels_batch }) log.info( '[%s][%.1f%%] validation cost: %.4f' % (NumUtil.comma_str(n_input), n_input / train.size * 100, sess.run(cost, feed_dict={ X: validation.features, Y: validation.labels }))) watch.stop('learn') log.info('learn OK.\n') log.info('evaluate...') watch.start('evaluate...') _hypothesis, _correct, _accuracy = sess.run( [hypothesis, predicted, accuracy], feed_dict={ X: test.features, Y: test.labels }) # Accuracy report watch.stop('evaluate...') log.info('evaluate OK.') log.info('model save... %s' % model_file) watch.start('model save...') model_dir = os.path.dirname(model_file) if not os.path.exists(model_dir): os.makedirs(model_dir) saver = tf.train.Saver() saver.save(sess, model_file) watch.stop('model save...') log.info('model save OK. %s' % model_file) log.info('\n') log.info(watch.summary()) # log.info('hypothesis: %s %s' % (_hypothesis.shape, _hypothesis)) # log.info('correct: %s %s' % (_correct.shape, _correct)) log.info('accuracy: %s %s' % (_accuracy.shape, _accuracy)) log.info('\n')