def main(_): model_path = os.path.join('model', FLAGS.name) if os.path.exists(model_path) is False: os.makedirs(model_path) with codecs.open(FLAGS.input_file, encoding='utf-8') as f: text = f.read() converter = TextConverter(text, FLAGS.max_vocab) converter.save_to_file(os.path.join(model_path, 'converter.pkl')) arr = converter.text_to_arr(text) g = batch_generator(arr, FLAGS.num_seqs, FLAGS.num_steps) print(converter.vocab_size) model = CharRNN(converter.vocab_size, num_seqs=FLAGS.num_seqs, num_steps=FLAGS.num_steps, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, learning_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.train( g, FLAGS.max_steps, model_path, FLAGS.save_every_n, FLAGS.log_every_n, )
def sample(model: CharRNN, vocab: Vocab) -> str: model.eval() with torch.no_grad(): hidden = None input = torch.from_numpy( np.array([ convert_idx_to_one_hot(vocab, vocab.char_to_idx(SOS_TOKEN)) ])).unsqueeze(0) input = input.float().to(config.device) pred = '' for _ in range(8): out, hidden = model(input, hidden) if out.view(-1).div(0.8).exp().sum() == 0: continue topi = torch.multinomial(out.view(-1).div(0.8).exp(), 1)[0] if topi.item() == vocab.char_to_idx(EOS_TOKEN): break pred += vocab.idx_to_char(topi.item()) input = torch.from_numpy( np.array([convert_idx_to_one_hot(vocab, topi.item())])).unsqueeze(0) input = input.float().to(config.device) return pred
def main(_): model_path = os.path.join('model', FLAGS.name) if os.path.exists(model_path) is False: os.makedirs(model_path) with codecs.open(FLAGS.input_file, encoding='utf-8') as f: text = f.read() tokenizer = Tokenizer(text, FLAGS.num_words) tokenizer.save_to_file(os.path.join(model_path, 'tokenizer.pkl')) arr = tokenizer.texts_to_sequences(text) batch = batch_generator(arr, FLAGS.batch_size, FLAGS.num_steps) print(tokenizer.vocab_size) model = CharRNN(tokenizer.vocab_size, batch_size=FLAGS.batch_size, num_steps=FLAGS.num_steps, n_neurons=FLAGS.n_neurons, n_layers=FLAGS.n_layers, learning_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, embedding=FLAGS.embedding, embedding_size=FLAGS.embedding_size ) model.train(batch, FLAGS.n_iterations, model_path, FLAGS.save_every_n, FLAGS.log_every_n, )
def main(_): model_path = os.path.join('model', FLAGS.name) if os.path.exists(model_path) is False: os.makedirs(model_path) with codecs.open(FLAGS.input_file, encoding='utf-8') as f: text = f.read() converter = TextConverter(text, FLAGS.max_vocab) converter.save_to_file(os.path.join(model_path, 'converter.pkl')) arr = converter.text_to_arr(text) g = batch_generator(arr, FLAGS.num_seqs, FLAGS.num_steps) print(converter.vocab_size) model = CharRNN(converter.vocab_size, num_seqs=FLAGS.num_seqs, num_steps=FLAGS.num_steps, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, learning_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size ) model.train(g, FLAGS.max_steps, model_path, FLAGS.save_every_n, FLAGS.log_every_n, )
def main(_): script_path = os.path.abspath(os.path.dirname(__file__)) model_path = os.path.join(script_path, 'model', FLAGS.name) if os.path.exists(model_path) is False: os.makedirs(model_path) with codecs.open(FLAGS.input_file, encoding='utf-8') as f: text = f.read() print("corpus size " + str(len(text))) if os.path.exists(FLAGS.whitelist_file): with codecs.open(FLAGS.whitelist_file, encoding='utf-8') as f: whitelist = f.read() text = remove_non_matching_chars(text, whitelist) converter = TextConverter(text, FLAGS.max_vocab) converter.save_to_file(os.path.join(model_path, 'converter.pkl')) arr = converter.text_to_arr(text) g = batch_generator(arr, FLAGS.num_seqs, FLAGS.num_steps) model = CharRNN(converter.vocab_size, num_seqs=FLAGS.num_seqs, num_steps=FLAGS.num_steps, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, learning_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.train( g, FLAGS.max_steps, model_path, FLAGS.save_every_n, FLAGS.log_every_n, )
def main(_): model_path = os.path.join('model', FLAGS.name) if not os.path.exists(model_path): os.makedirs(model_path) with codecs.open(FLAGS.input_file, encoding='utf-8') as f: text = f.read() convert = TextConvert(text, FLAGS.max_vocab) convert.save_vocab(os.path.join(model_path, 'text_convert.pkl')) arr = convert.text2arr(text) g = batch_generate(arr, FLAGS.num_seqs, FLAGS.num_steps) model = CharRNN( convert.vocab_size, num_seqs=FLAGS.num_seqs, num_steps=FLAGS.num_steps, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, learning_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size ) model.train( g, FLAGS.max_steps, model_path, FLAGS.save_n, FLAGS.print_n, )
def train(opt, th): ''' 训练模型 Args: opt -- 参数 th -- TextConverter对象 Returns: None ''' # 1. 训练数据 data_set = TextDataset(opt.train_data_path, th) train_data = DataLoader(data_set, opt.batch_size, shuffle=True, num_workers=opt.num_workers) # 2. 初始化模型 model = CharRNN(th.vocab_size, opt.embed_size, opt.hidden_size, opt.n_layers, opt.dropout_p, opt.bidir) if USE_CUDA: model = model.cuda(DEVICE_ID) # 3. 优化配置 criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=opt.learning_rate) # 4. 训练 for e in range(opt.max_epochs): epoch_loss = 0 hidden = None for input_seqs, labels in train_data: # 都是[b, seq_len],最后一个不足b # 准备input和hidden b = input_seqs.shape[0] if hidden is not None: hidden = hidden[:, :b, :] labels = labels.long().view(-1) input_seqs, labels = get_variable(input_seqs), get_variable(labels) # 前向计算 probs, hidden = model(input_seqs, hidden) probs = probs.view(-1, th.vocab_size) # loss和反向 loss = criterion(probs, labels) optimizer.zero_grad() loss.backward(retain_graph=True) # 优化 nn.utils.clip_grad_norm(model.parameters(), 5) optimizer.step() epoch_loss += loss.data[0] # 交叉熵 entropy_loss = epoch_loss / len(train_data) perplexity = np.exp(entropy_loss) info = "epoch: {}, perp: {:.3f}".format(e + 1, perplexity) print(info) if perplexity <= opt.min_perplexity or e == opt.max_epochs - 1: print("best model") torch.save(model, opt.model_path) break
def main(_): model_path = os.path.join('model', FLAGS.name) if os.path.exists(model_path) is False: os.makedirs(model_path) print( "---------------------------- Reading Corpus ----------------------------" ) start_time = time.time() read_corpus() print("Read Corpus Finished in " + str(time.time() - start_time) + ' Seconds.') converter = TextConverter(corpus, FLAGS.max_vocab) converter.save_to_file(os.path.join(model_path, 'converter.pkl')) arr = converter.text_to_arr(corpus) g = batch_generator(arr, FLAGS.num_seqs, FLAGS.num_steps) model = CharRNN(converter.vocab_size, num_seqs=FLAGS.num_seqs, num_steps=FLAGS.num_steps, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, learning_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.train( g, FLAGS.max_steps, model_path, FLAGS.save_every_n, FLAGS.log_every_n, )
def main(_): model_path = os.path.join('model', FLAGS.name) # 拼接路径model/'name' if os.path.exists(model_path) is False: os.makedirs(model_path) with codecs.open(FLAGS.input_file, encoding='utf-8') as f: text = f.read() # 读取文本 converter = TextConverter(text, FLAGS.max_vocab) # 文本转换为词汇且截取FLAGS.max_vocab个词 converter.save_to_file(os.path.join(model_path, 'converter.pk1')) # 序列化存储词汇 data = converter.text_to_data(text) # 将文本转化为输入(word_to_int) g = batch_generator(data, FLAGS.n_seqs, FLAGS.n_steps) # 获取batch生成器 print(converter.vocab_size) # 模型参数初始化 model = CharRNN(converter.vocab_size, n_seqs=FLAGS.n_seqs, n_steps=FLAGS.n_steps, state_size=FLAGS.state_size, n_layers=FLAGS.n_layers, learning_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.train(g, FLAGS.max_steps, model_path, FLAGS.save_every_n, FLAGS.log_every_n)
def main(_): FLAGS.start_string = FLAGS.start_string #.decode('utf-8') converter = TextConverter(filename=FLAGS.converter_path) if os.path.isdir(FLAGS.checkpoint_path): FLAGS.checkpoint_path =\ tf.train.latest_checkpoint(FLAGS.checkpoint_path) model = CharRNN(converter.vocab_size, sampling=True, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.load(FLAGS.checkpoint_path) start_string = FLAGS.start_string sys.stdout.write("> ") sys.stdout.flush() start_string = sys.stdin.readline() while start_string: start = converter.text_to_arr(start_string) arr = model.sample(FLAGS.max_length, start, converter.vocab_size) print(converter.arr_to_text(arr)) sys.stdout.write("> ") sys.stdout.flush() sentence = sys.stdin.readline()
def main(_): model_path = os.path.join('model', FLAGS.name)#创建路径字符串 if os.path.exists(model_path) is False:#创建文件夹路径 os.makedirs(model_path) with codecs.open(FLAGS.input_file, encoding='utf-8') as f: text = f.read()#读取整个文件作为字符串 converter = TextConverter(text, FLAGS.max_vocab) converter.save_to_file(os.path.join(model_path, 'converter.pkl')) arr = converter.text_to_arr(text)#将文本序列化 g = batch_generator(arr, FLAGS.num_seqs, FLAGS.num_steps)#100,100 print(converter.vocab_size) model = CharRNN(converter.vocab_size,#创建模型,这里num_classes设置为了字典的大小,因为要预测下一个char num_seqs=FLAGS.num_seqs, num_steps=FLAGS.num_steps, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, learning_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size ) model.train(g,#训练模型 FLAGS.max_steps, model_path, FLAGS.save_every_n, FLAGS.log_every_n, )
def main(_): model_path = os.path.join('model', FLAGS.file_type) if not os.path.exists(model_path): os.makedirs(model_path) # Read and Load Corpus for Train and Validation. training_corpus, validating_corpus = read_corpus() # Build Text Converter print( "---------------------------- Initializing Text Converter ----------------------------" ) start_time = time.time() converter = TextConverter(training_corpus, FLAGS.max_vocab) converter.save_to_file(os.path.join(model_path, 'converter.pkl')) print('Initialize Text Converter Finished in %.3f Seconds.\n' % (time.time() - start_time)) # Vectorize Content of Corpus vectroize_corpus(converter) # Build Char RNN Model model = CharRNN(converter.vocab_size, num_seqs=FLAGS.num_seqs, num_steps=FLAGS.num_steps, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, learning_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) # Train Model model.train(FLAGS.max_steps, model_path, FLAGS.validate_every_n_steps, FLAGS.log_every_n_steps)
def main(_): converter = TextConverter(filename=FLAGS.converter_path) if os.path.isdir(FLAGS.checkpoint_path): FLAGS.checkpoint_path = tf.train.latest_checkpoint( FLAGS.checkpoint_path) model = CharRNN(converter.vocab_size, None, sampling=True, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.load(FLAGS.checkpoint_path) # start = converter.text_to_arr(FLAGS.seed_for_generating) seeds = [ 'var a = fun', 'function a(', 'this.', 'document.', 'window.', 'var a = document.g', 'var a;', 'jQuery' ] for seed in seeds: start = converter.text_to_arr(seed) for i in range(0, FLAGS.num_to_generate): print('Generating: ' + seed + ' -> ' + str(i)) file_name = str(uuid.uuid1()) file_path = '../../BrowserFuzzingData/generated/' + FLAGS.file_type + '/' + file_name + '.' + FLAGS.file_type arr = model.sample(FLAGS.max_length_of_generated, start, converter.vocab_size, converter.word_to_int) f = open(file_path, "wb") f.write(converter.arr_to_text(arr).encode('utf-8')) f.close()
def main(_): model_path = os.path.join('model', FLAGS.name) # 保存模型的路径 if os.path.exists(model_path) is False: os.makedirs(model_path) # 用codecs提供的open方法来指定打开的文件的语言编码,它会在读取的时候自动转换为内部unicode with codecs.open(FLAGS.input_file, encoding='utf-8') as f: text = f.read() # 读取训练的文本 converter = TextConverter(text, FLAGS.max_vocab) # 转换text文本格式 converter.save_to_file(os.path.join(model_path, 'converter.pkl')) arr = converter.text_to_arr(text) # 转换text为数组 g = batch_generator(arr, FLAGS.num_seqs, FLAGS.num_steps) # 批生成 print(converter.vocab_size) model = CharRNN(converter.vocab_size, # 读取模型 num_seqs=FLAGS.num_seqs, num_steps=FLAGS.num_steps, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, learning_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size ) model.train(g, # 训练 FLAGS.max_steps, model_path, FLAGS.save_every_n, FLAGS.log_every_n, )
def main(): inputs, token_to_idx, idx_to_token = load_dataset(file_name=sys.argv[2]) #coloredlogs.install(level='DEBUG') num_layers = 2 rnn_type = 'lstm' dropout = 0.5 emb_size = 50 hidden_size = 256 learning_rate = 0.001 n_tokens = len(idx_to_token) model = CharRNN(num_layers=num_layers, rnn_type=rnn_type, dropout=dropout, n_tokens=n_tokens, emb_size=emb_size, hidden_size=hidden_size, pad_id=token_to_idx[PAD_TOKEN]) if torch.cuda.is_available(): model = model.cuda() optimiser = optim.Adam(model.parameters(), lr=learning_rate) try: model, optimiser, epoch, valid_loss_min = load_ckp( checkpoint_fpath=sys.argv[1], model=model, optimiser=optimiser) generate_sample(model, token_to_idx, idx_to_token, n_tokens=20) except KeyboardInterrupt: print('Aborted!')
def main(_): model_path = os.path.join("model", FLAGS.name) if os.path.exists(model_path) is False: os.makedirs(model_path) with open(FLAGS.input_file, encoding='utf-8') as f: text = f.read() converter = TextConverter(text=text, filename=None, max_vocab=FLAGS.max_vocab) converter.save_to_file(os.path.join(model_path, 'converter.pkl')) arr = converter.text_to_arr(text=text) print(converter.vocab_size()) g = batch_generator(arr, FLAGS.num_seqs, FLAGS.num_steps) model = CharRNN(num_class=converter.vocab_size(), num_seqs=FLAGS.num_seqs, num_steps=FLAGS.num_steps, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, learn_rate=FLAGS.learning_rate, train_keep_prob=FLAGS.train_keep_prob, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.train( batch_generate=g, max_steps=FLAGS.max_steps, save_path=model_path, save_per_n=FLAGS.save_per_n, print_per_n=FLAGS.print_per_n, )
def load_model(model_filename): with open(model_filename, 'rb') as f: checkpoint = torch.load(f) n_hidden, n_layers, state_dict, chars = checkpoint['n_hidden'], checkpoint['n_layers'], \ checkpoint['state_dict'], checkpoint['chars'] model = CharRNN(chars=chars, n_hidden=n_hidden, n_layers=n_layers) model.load_state_dict(state_dict=state_dict) return model
def model_built(self):#,vocab_size,sampling,lstm_size,num_layers,use_embedding,embedding_size): FLAGS.start_string = FLAGS.start_string.decode('utf-8') self.converter = TextConverter(filename=FLAGS.converter_path) if os.path.isdir(FLAGS.checkpoint_path): FLAGS.checkpoint_path =\ tf.train.latest_checkpoint(FLAGS.checkpoint_path) self.tfmodel = CharRNN(self.converter.vocab_size, sampling=True, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) self.tfmodel.load(FLAGS.checkpoint_path)
def main(args): # configure GPU datatype use_gpu = torch.cuda.is_available() if not use_gpu: raise Exception('error: CUDA library unavailable') global gpu_dtype gpu_dtype = torch.cuda.FloatTensor # load train, val, test data with open(args.data_dir + '/data.pkl', 'rb') as f: url_array, label_array = pickle.load(f) # partition dataset (this must sum to <50K) num_train = 10000 num_val = 2000 num_test = 8000 data_train = url_array[:, :num_train, :] labels_train = label_array[:num_train] data_val = url_array[:, num_train:num_train + num_val, :] labels_val = label_array[num_train:num_train + num_val] data_test = url_array[:, num_train + num_val:num_train + num_val + num_test, :] labels_test = label_array[num_train + num_val:num_train + num_val + num_test] # initialize model and configure for GPU model = CharRNN() model = model.type(gpu_dtype) # train model on training data, reporting accuracy on held out validation set train(model, (data_train, labels_train), (data_val, labels_val), args.num_epochs, args.batch_size) # convert model to CPU for use on GPU-less AWS instance model = model.type(torch.FloatTensor) # get test accuracy print('Final results on held-out test set: ') check_accuracy(model, (data_test, labels_test), use_gpu=False) check_accuracy(model, (data_train, labels_train), use_gpu=False) # save model to disk for use in prediction path = 'models/char_rnn.pk' print('Saving model to %s' % path) torch.save(model, path)
def main(_): converter = TextConverter(filename=FLAGS.converter_path) if os.path.isdir(FLAGS.checkpoint_path): FLAGS.checkpoint_path =\ tf.train.latest_checkpoint(FLAGS.checkpoint_path) model = CharRNN(converter.vocab_size, sampling=True, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.load(FLAGS.checkpoint_path) start = converter.text_to_arr(FLAGS.start_string) arr = model.sample(FLAGS.max_length, start, converter.vocab_size) print(converter.arr_to_text(arr))
def sample(checkpoint): samples = [c for c in prime] int_to_vocab, vocab_to_int, no_classes = pickle.load( open("./saves/data.p", "rb")) # Initialize the model model = CharRNN(no_classes=no_classes, sampling=True) saver = tf.train.Saver() with tf.Session() as sess: # Load the checkpoint saver.restore(sess, checkpoint) new_state = sess.run(model.initial_state) # Feed the prime word to the model and predict the next character for c in prime: x = np.zeros((1, 1)) x[0, 0] = vocab_to_int[c] feed = {model.inputs: x, model.initial_state: new_state} preds, new_state = sess.run( [model.prediction, model.final_state], feed_dict=feed) c = pick_top_n(preds, no_classes) samples.append(int_to_vocab[c]) # Generate new samples for i in range(n_samples): x[0, 0] = c feed = {model.inputs: x, model.initial_state: new_state} preds, new_state = sess.run( [model.prediction, model.final_state], feed_dict=feed) c = pick_top_n(preds, no_classes) samples.append(int_to_vocab[c]) return ''.join(samples)
def main(_): tc = TextConverter("", -1, byte_file=FLAGS.vocab_path) output_size = tc.vocab_size if os.path.isdir(FLAGS.checkpoint_path): FLAGS.checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path) model = CharRNN(output_size=output_size, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, sampling=True) model.load(FLAGS.checkpoint_path) start = tc.text_to_arr(FLAGS.start_string) generate_arr = model.sample(FLAGS.length, start, output_size) generate_text = tc.arr_to_text(generate_arr) with open(FLAGS.save_path, 'w', encoding='utf-8') as f: f.write(generate_text) print(generate_text)
def main(): parser = argparse.ArgumentParser("Char-RNN on the complete works of Shakespeare") parser.add_argument("--test", type=bool, default=False, help = "if true, keep only a thousand lines from the Shakespeare corpus") args = parser.parse_args() seed(1616) text = extract_shakespeare_data("data/t8.shakespeare.txt") char_encoder = CharEncoder(text) #get sequences of 100 characters sequences = make_sequences(text) #vectorize with numeric labeling #each character gets mapped to an integer & vice versa sequences = char_encoder.label_sequences(sequences) if args.test: print("Test: downsizing data to 1,000 sequences...") sequences = sequences[:1000] shuffle(sequences) n_training_sequences = int(.9 * len(sequences)) #split the dataset into training and validation sets training = sequences[:n_training_sequences] validation = sequences[n_training_sequences:] hidden_size = 128 rnn = CharRNN(char_encoder.n_chars, hidden_size) train(rnn, training, validation, epochs = 4, lr = 0.01, evaluate_per = 2, batch_size = 20) print(sample(rnn, prime_str = "Macbeth", size = 100, encoder = char_encoder, temperature=.9))
def main(_): tokenizer = Tokenizer(vocab_path=FLAGS.tokenizer_path) if os.path.isdir(FLAGS.checkpoint_path): FLAGS.checkpoint_path = \ tf.train.latest_checkpoint(FLAGS.checkpoint_path) model = CharRNN(tokenizer.vocab_size, sampling=True, n_neurons=FLAGS.n_neurons, n_layers=FLAGS.n_layers, embedding=FLAGS.embedding, embedding_size=FLAGS.embedding_size) model.load(FLAGS.checkpoint_path) start = tokenizer.texts_to_sequences(FLAGS.start_string) arr = model.sample(FLAGS.max_length, start, tokenizer.vocab_size) print(tokenizer.sequences_to_texts(arr))
def composePotery(): converter = TextConverter(filename=FLAGS.converter_path) if os.path.isdir(FLAGS.checkpoint_path): FLAGS.checkpoint_path =\ tf.train.latest_checkpoint(FLAGS.checkpoint_path) model = CharRNN(converter.vocab_size, sampling=True, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.load(FLAGS.checkpoint_path) start = [] arr = model.sample(FLAGS.max_length, start, converter.vocab_size) rawText = converter.arr_to_text(arr) return(selectPoetry(rawText))
def sample(checkpoint, n_samples, lstm_size, vocab_size, prime="The "): samples = [c for c in prime] model = CharRNN(vocab_size, lstm_size=lstm_size, sampling=True) saver = tf.train.Saver() with tf.Session() as sess: saver.restore(sess, checkpoint) new_state = sess.run(model.initial_state) for c in prime: x = np.zeros((1, 1)) x[0, 0] = vocab_to_int[c] feed = { model.inputs: x, model.keep_prob: 1., model.initial_state: new_state } preds, new_state = sess.run([model.prediction, model.final_state], feed_dict=feed) c = pick_top_n(preds, vocab_size) samples.append(int_to_vocab[c]) for i in range(n_samples): x[0, 0] = c feed = { model.inputs: x, model.keep_prob: 1., model.initial_state: new_state } preds, new_state = sess.run([model.prediction, model.final_state], feed_dict=feed) c = pick_top_n(preds, vocab_size) samples.append(int_to_vocab[c]) return ''.join(samples)
def main(_): FLAGS.start_string = FLAGS.start_string.decode('utf-8') converter = TextConverter(filename=FLAGS.converter_path) if os.path.isdir(FLAGS.checkpoint_path): FLAGS.checkpoint_path =\ tf.train.latest_checkpoint(FLAGS.checkpoint_path) model = CharRNN(converter.vocab_size, sampling=True, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.load(FLAGS.checkpoint_path) start = converter.text_to_arr(FLAGS.start_string) arr = model.sample(FLAGS.max_length, start, converter.vocab_size) print(converter.arr_to_text(arr))
def train(model: CharRNN, optimizer: optim.Optimizer, criterion, inputs, targets): model.train() optimizer.zero_grad() hidden = None total_loss = 0 for i in range(inputs.shape[0]): output, hidden = model(inputs[i].unsqueeze(0).unsqueeze(0).float(), hidden) loss = criterion(output.squeeze(0), targets[i].unsqueeze(0).long()) total_loss += loss total_loss.backward() optimizer.step() return output, total_loss.item() / inputs.shape[0]
def start_text(start_text, n_words=250): # Here we have loaded in a model that trained over 20 epochs `rnn_20_epoch.net` with open('saved_model/rnn_20_epoch.net', 'rb') as f: checkpoint = torch.load(f) loaded = CharRNN(checkpoint['tokens'], n_hidden=checkpoint['n_hidden'], n_layers=checkpoint['n_layers']) loaded.load_state_dict(checkpoint['state_dict']) generated_text = sample(loaded, n_words, top_k=5, prime='{} '.format(start_text)) generated_text = generated_text.replace('\n', ' ') generated_text = '{}.'.format(generated_text.split('.')[0]) return generated_text
def generate(): tf.compat.v1.disable_eager_execution() converter = TextConverter(filename=FLAGS.converter_path) if os.path.isdir(FLAGS.checkpoint_path): FLAGS.checkpoint_path =\ tf.train.latest_checkpoint(FLAGS.checkpoint_path) model = CharRNN(converter.vocab_size, sampling=True, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.load(FLAGS.checkpoint_path) start = converter.text_to_arr(FLAGS.start_string) arr = model.sample(FLAGS.max_length, start, converter.vocab_size) return converter.arr_to_text(arr)
def main(_): FLAGS.start_string = FLAGS.start_string.decode('utf-8') converter = TextConverter(filename=FLAGS.converter_path) #创建文本转化器 if os.path.isdir(FLAGS.checkpoint_path): FLAGS.checkpoint_path = tf.train.latest_checkpoint( FLAGS.checkpoint_path) #下载最新模型 model = CharRNN(converter.vocab_size, sampling=True, lstm_size=FLAGS.lstm_size, num_layers=FLAGS.num_layers, use_embedding=FLAGS.use_embedding, embedding_size=FLAGS.embedding_size) model.load(FLAGS.checkpoint_path) #加载模型 start = converter.text_to_arr(FLAGS.start_string) #将input text转为id arr = model.sample(FLAGS.max_length, start, converter.vocab_size) #输出为生成的序列 print(converter.arr_to_text(arr))
def _make_estimator(self): params = tf.contrib.training.HParams(**Config.model.to_dict()) run_config = tf.contrib.learn.RunConfig( model_dir=Config.train.model_dir) char_rnn = CharRNN() self.estimator = tf.estimator.Estimator( model_fn=char_rnn.model_fn, model_dir=Config.train.model_dir, params=params, config=run_config)