def _compare_rnn_layer(rnn_layers, x_train, y_train, epochs=40, batch_size=5, input_shape=(499, 1), memory_length=15, activation='tanh', batch_norm=False, optimizer='adam'): """Builds and trains models with different types of RNN layers.""" histories = [] names = [] for rnn_layer in rnn_layers: if rnn_layer.lower() not in ['lstm', 'gru', 'simplernn']: print('{} is not a valid RNN layer. Only LSTM, GRU, or SimpleRNN' ' RNN layers are acceptable.'.format(rnn_layer)) raise ValueError rnn_model = rnn.RNN(input_shape, memory_length, rnn_layer, activation, batch_norm, optimizer) rnn_model_history = rnn_model.train_rnn_model(x_train, y_train, batch_size, epochs) histories.append(rnn_model_history) names.append(rnn_layer) return histories, names
def test_getitem_and_setitem_work(): nn = rnn.RNN(n_a=4, n_x=1) for i in range(len(nn)): nn[i] = float(i) for i in range(len(nn)): assert nn[i] == float(i)
def init_rnn(m): X, char_to_ind = dr.read_data() K = len(X) RNN = rnn.RNN(K, m=m) return RNN, X, char_to_ind
def _compare_batch_norm(batch_norms, x_train, y_train, epochs=40, batch_size=5, input_shape=(499, 1), memory_length=15, rnn_layer='LSTM', activation='tanh', optimizer='adam'): """Builds and trains models with and without Batch Normalization.""" histories = [] names = [] for batch_norm in batch_norms: if not isinstance(batch_norm, bool): print( '{} is not a boolean value. Only input boolean values'.format( batch_norm)) raise ValueError rnn_model = rnn.RNN(input_shape, memory_length, rnn_layer, activation, batch_norm, optimizer) rnn_model_history = rnn_model.train_rnn_model(x_train, y_train, batch_size, epochs) histories.append(rnn_model_history) if batch_norm: names.append('With Batch Normalization') else: names.append('Without Batch Normalization') return histories, names
def initialize_model(self, use_pretrained_embedding=True): """ Get tree data and initialize a model #data: a dictionary; key-value example: "train"-(tree_list, ner_list) data: a dictionary; key-value example: "train"-{"tree_pyramid_list": tree_pyramid_list, "ner_list": ner_list} tree_pyramid_list: a list of (tree, pyramid) tuples ner_list: a list of dictionaries; key-value example: (3,5)-"PERSON" ne_list: a list of distinct string labels, e.g. "PERSON" """ if self.model and self.model.init: return # Load data and determine dataset related hyperparameters config = rnn.Config() config.alphabet_size = self.character_length config.pos_dimension = self.pos_length config.output_dimension = self.entities_length config.lexicons = self.lexicon_length config.vocabulary_size = len(self.word_list) # Initialize a model self.model = rnn.RNN(config) self.model.sess = tf.Session() self.model.sess.run(tf.global_variables_initializer()) if use_pretrained_embedding: self.load_embedding()
def __init__(self, rng, input_data, dim, n_feature_maps, window_sizes, n_hidden, n_out, h_prev, y_prev): self.cnn = cnn.CNN(input_data=input_data, rng=rng, dim=dim, n_feature_maps=n_feature_maps, window_sizes=window_sizes) self.rnn = rnn.RNN(input_data=self.cnn.output, rng=rng, n_in=n_feature_maps * len(window_sizes), n_hidden=n_hidden, n_out=n_out, h_prev=h_prev, y_prev=y_prev, activation=T.nnet.sigmoid) self.h = self.rnn.h self.window_sizes = window_sizes self.dim = dim self.n_out = n_out self.n_hidden = self.rnn.n_hidden self.params = self.cnn.params + self.rnn.params self.output = self.rnn.output self.loss = self.rnn.loss self.error = self.rnn.error return
def new_agent(i): #if i == 0: # print("New 0") #nets[i] = net.Net(input_sz, hidden_sz, output_sz, genes[i]) nets[i] = rnn.RNN(input_sz, state_sz, genes[i]) pos[i] = [random.random(), random.random()] set_mass(i, starting_mass) alive[i] = True
def rnnMethod(area): global rnn_model if rnn_model is None: rnn_model = rnn.RNN() ranked_scores = rnn_model.train(False, area) ranked_scores = ranked_scores[1:] #delete the first element return ranked_scores
def testBuildRNN_inferenceMode(self): batch_size = 10 seq_len = 5 dict_size = 20 model = rnn.RNN(rnn.RNNConfig(), batch_size=batch_size, dict_size=dict_size) init_inputs = tf.zeros(shape=(batch_size, ), dtype=tf.int32) init_state = model.zero_state(batch_size) finished_fn = lambda step, outputs: tf.constant(True, shape=(batch_size, )) t_outputs, t_state = model(init_inputs, init_state, finished_fn, seq_len) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) outputs, state = sess.run([t_outputs, t_state]) self.assertEqual((batch_size, seq_len, dict_size), outputs.shape)
def load_data_and_initialize_model(dataset, split_list=["train", "validate", "test"], use_pretrained_embedding=True): """ Get tree data and initialize a model #data: a dictionary; key-value example: "train"-(tree_list, ner_list) data: a dictionary; key-value example: "train"-{"tree_pyramid_list": tree_pyramid_list, "ner_list": ner_list} tree_pyramid_list: a list of (tree, pyramid) tuples ner_list: a list of dictionaries; key-value example: (3,5)-"PERSON" ne_list: a list of distinct string labels, e.g. "PERSON" """ # Select the implementation of loading data according to dataset if dataset == "ontonotes": import ontonotes as data_utils elif dataset == "ontochinese": import ontochinese as data_utils elif dataset == "conll2003": import conll2003 as data_utils elif dataset == "conll2003dep": import conll2003dep as data_utils # Load data and determine dataset related hyperparameters config = rnn.Config() (data, word_list, ne_list, config.alphabet_size, config.pos_dimension, config.output_dimension, config.lexicons) = data_utils.read_dataset(split_list) config.vocabulary_size = len(word_list) # Initialize a model model = rnn.RNN(config) """ tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True model.sess = tf.Session(config=tf_config) """ model.sess = tf.Session() model.sess.run(tf.global_variables_initializer()) if use_pretrained_embedding: load_embedding(model, word_list, dataset) return data, ne_list, model
def main(): dataset_filepath = '../../data/datasets/ngsim_feature_trajectories.h5' binedges = [10,15,25,50] max_len = 100 data = utils.load_ngsim_trajectory_data( dataset_filepath, binedges=binedges, max_len=max_len, max_samples=None, train_ratio=.9, target_keys=['lidar_10'] ) exp_dir = '../../data/experiments/imputation' utils.maybe_mkdir(exp_dir) model = rnn.RNN( name='supervised_imputation', input_dim=data['train_x'].shape[2], hidden_dim=256, max_len=max_len, output_dim=len(binedges), batch_size=500, learning_rate=.0005, dropout_keep_prob=.75 ) writer = tf.summary.FileWriter(os.path.join(exp_dir, 'train')) val_writer = tf.summary.FileWriter(os.path.join(exp_dir, 'val')) utils.write_baseline_summary(data['train_lengths'], data['train_y'], writer) utils.write_baseline_summary(data['val_lengths'], data['val_y'], val_writer) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) model.train( data, n_epochs=1000, writer=writer, val_writer=val_writer )
def lambda_handler(event, context): language = event['language'] start_letters = event['start_letters'] model_parameter_object_key = event['model_parameter_object_key'] # example : rnn_params.pkl model_object_key = event['model_object_key'] # example : rnn_model.pth model_bucket = event['model_bucket'] # Load pre-processing parameters # Check if model parameters are available parameter_path = tmp + model_parameter_object_key if not os.path.isfile(parameter_path): s3_client.download_file(model_bucket, model_parameter_object_key, parameter_path) with open(parameter_path, 'rb') as pkl: params = pickle.load(pkl) all_categories = params['all_categories'] n_categories = params['n_categories'] all_letters = params['all_letters'] n_letters = params['n_letters'] # Check if models are available # Download model from S3 if model is not already present model_path = tmp + model_object_key if not os.path.isfile(model_path): s3_client.download_file(model_bucket, model_object_key, model_path) rnn_model = rnn.RNN(n_letters, 128, n_letters, all_categories, n_categories, all_letters, n_letters) rnn_model.load_state_dict(torch.load(model_path)) rnn_model.eval() start = time() output_names = list(rnn_model.samples(language, start_letters)) latency = time() - start return {'latency': latency, 'predict': output_names}
def testReal(): tgt_path = './train/' test_cls_name = 'T' trandir = tgt_path + test_cls_name + '/' testdir = tgt_path + test_cls_name + '/' n1 = rf.RNN(trandir, 'REAL') n2 = lgl.FFNN(trandir) filecount = 0 for filename in os.listdir(testdir): filecount += 1 signals = np.zeros((filecount, cfg.INPUT_SIZE), np.int) index = 0 for filename in os.listdir(testdir): signal = np.fromfile(testdir + filename, dtype=np.int) #if len(signal) != cfg.INPUT_SIZE: # signal = utl.resizeDim(signal,cfg.DIM) signal = signal.reshape((cfg.INPUT_SIZE, )) signals[index, :] = signal[:] index += 1 refined = n1.test(signals) for i in range(filecount): inp = cv2.resize( utl.arr2img(signals[i].reshape((cfg.DIM, cfg.FEATURE_SIZE))), (200, 200)) out = cv2.resize( utl.arr2img(refined[i].reshape((cfg.DIM, cfg.FEATURE_SIZE))), (200, 200)) cv2.imshow('in' + str(i), inp) cv2.imshow('out' + str(i), out) cv2.waitKey(100000) final = n2.getresult(refined) print(final)
def main(): # loading data X, y = DataLoader.load_data() text, X_test, y_test = DataLoader.load_test_data() # intializing network network = rnn.RNN() network.fit(X, y, ephocs=3000) # sample predictions label_names = ['ham', 'spam'] tag = np.random.randint(0,10) print('\n\n[main]: Test prediction:', '\nSMS: ',text[tag] , '\n\nactual: ',label_names[np.argmax(y_test[tag])], '\nprediction: ',label_names[np.argmax(network.predict(X_test[tag]))]) tag = np.random.randint(0,10) print('\n\n[main]: Test prediction:', '\nSMS: ',text[tag] , '\n\nactual: ',label_names[np.argmax(y_test[tag])], '\nprediction: ',label_names[np.argmax(network.predict(X_test[tag]))])
def fs(): input_length = 100 hidden_cnt = 50 data = get_test_data(input_length) rnn_nn = nn.NeuralNetwork(nn=rnn.RNN(input_length, hidden_cnt, data.x.shape[2], data.y.shape[1]), validation_split=0.2, batch_size=256, nb_epoch=10, show_accuracy=True) features, results = rnn_nn.feature_selection(data) print("Selected features: {0}".format(features)) print(results) feature_selection = { "features": features, "results": results, "count": data.x.shape[2] } output = open('../../results/RNN_features', 'wb') pickle.dump(feature_selection, output) output.close()
def _compare_optimizer(optimizers, x_train, y_train, epochs=40, batch_size=5, input_shape=(499, 1), memory_length=15, rnn_layer='LSTM', activation='tanh', batch_norm=False): """Builds and trains models with varying optimizer values.""" histories = [] names = [] for optimizer in optimizers: rnn_model = rnn.RNN(input_shape, memory_length, rnn_layer, activation, batch_norm, optimizer) rnn_model_history = rnn_model.train_rnn_model(x_train, y_train, batch_size, epochs) histories.append(rnn_model_history) names.append('{}'.format(optimizer)) return histories, names
def _compare_memory_length(memory_lengths, x_train, y_train, epochs=40, batch_size=5, input_shape=(499, 1), rnn_layer='LSTM', activation='tanh', batch_norm=False, optimizer='adam'): """Builds and trains models with varying memory length values.""" histories = [] names = [] for memory_length in memory_lengths: rnn_model = rnn.RNN(input_shape, memory_length, rnn_layer, activation, batch_norm, optimizer) rnn_model_history = rnn_model.train_rnn_model(x_train, y_train, batch_size, epochs) histories.append(rnn_model_history) names.append('Memory Length: {}'.format(memory_length)) return histories, names
def main(print=print): np.random.seed(1) nn = rnn.RNN(n_a=2, n_x=1) print(f"Training a RNN with {nn.size} parameters on a stupid example set.\n") avg_loss = 0.0 avg_acc = 0.0 for i in range(10000): inputs, outputs = create_stupid_example(9, m=16) loss, acc = nn.calculate_loss_and_accuracy(inputs, outputs) avg_loss = 0.5 * avg_loss + 0.5 * loss avg_acc = 0.5 * avg_acc + 0.5 * acc if i % 50 == 0: print(f"At iteration {i}, average accuracy is {int(avg_acc * 100)}% (loss is {avg_loss}).") if avg_acc > 0.99: break grad = nn.calculate_gradient_very_slowly(inputs, outputs) nn.learn_very_slowly(grad) print("\nDone training!\n") inputs, outputs = create_stupid_example(10) print(f"Example input : {inputs.ravel()}") print(f"Expected output: {outputs.ravel()}\n") print(f"Actual output:") pred = [pred_y for pred_y in nn.forward_prop_seq(inputs)] for i in range(len(outputs)): print(f" At index {i}, we expected {outputs[i][0][0]} and predicted {pred[i][0][0]}.") return (avg_loss, avg_acc)
def train_rnn(dataset, net_settings, train_optimizer=tf.train.AdamOptimizer): batch_size = FLAGS.batch_size num_hidden_last = net_settings[-1]['num_hidden'] input_placeholder = tf.placeholder(tf.float32, shape=[batch_size, net_settings[0]['dim_size']], name="input_placeholder") labels_placeholder = tf.placeholder(tf.int64, shape=[batch_size], name="labels_placeholder") optimizer = train_optimizer(FLAGS.learning_rate) rnn_model = rnn.RNN(net_settings) with tf.name_scope("LSTM"): net = rnn_model.fit_layers(input_placeholder) with tf.variable_scope("Dense1"): dense = tf.reshape(net, [batch_size, -1]) weights = tf.get_variable(name="weights", shape=[num_hidden_last, FLAGS.num_classes], initializer=tf.truncated_normal_initializer()) bias = tf.get_variable(name="bias", shape=[FLAGS.num_classes], initializer=tf.truncated_normal_initializer()) logits = tf.matmul(dense, weights) + bias loss = compute_loss(logits=logits, labels=labels_placeholder) train_op = optimizer.minimize(loss) accuracy = compute_accuracy(logits=logits, labels=labels_placeholder) saver = tf.train.Saver(tf.trainable_variables()) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) current_exec = str(time.time()) train_dir = FLAGS.checkpoint_path model_save_dir = os.path.join(train_dir, current_exec) if not os.path.exists(model_save_dir): os.makedirs(model_save_dir) model_filename = os.path.join(model_save_dir, "lstm_no_spark.model") with open(os.path.join(model_save_dir, "params_settings"), "w+") as f: f.write(params_str) if os.path.isfile(model_filename) and FLAGS.use_pretrained_model: saver.restore(sess, model_filename) merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter(os.path.join(model_save_dir, "train"), sess.graph) # test_writer = tf.summary.FileWriter(os.path.join(model_save_dir, "test"), sess.graph) train_x, train_y = process_batch(dataset) # val_x, val_y = process_batch(dataset[1]) train_batches = next_batch(train_x, train_y, batch_size=FLAGS.batch_size) # val_batches = next_batch(val_x, val_y, batch_size=FLAGS.batch_size) batch_size = FLAGS.batch_size if FLAGS.batch_size else 1 max_steps = FLAGS.epochs * batch_size total_steps = trange(max_steps) start = time.time() t_acc, v_acc, t_loss, v_loss = 0., 0., 0., 0. for step in total_steps: train_input, train_labels = next(train_batches) print(train_input, type(train_input)) _, t_loss = sess.run([train_op, loss], feed_dict={ input_placeholder: train_input, labels_placeholder: train_labels }) t_loss = np.mean(t_loss) total_steps.set_description('Loss: {:.4f} - t_acc {:.3f}' .format(t_loss, t_acc)) if step % FLAGS.evaluate_every == 0 or (step + 1) == max_steps: saver.save(sess, os.path.join(model_save_dir, 'lstm_no_spark'), global_step=step) summary, t_loss, t_acc = sess.run([merged, loss, accuracy], feed_dict={ input_placeholder: train_input, labels_placeholder: train_labels }) train_writer.add_summary(summary, step) # t_loss = np.mean(t_loss) # val_input, val_labels = val_batches.next() # summary, v_loss, v_acc = sess.run([merged, loss, accuracy], feed_dict={ # input_placeholder: val_input, # labels_placeholder: val_labels # }) # test_writer.add_summary(summary, step) # v_loss = np.mean(v_loss) total_steps.set_description('Loss: {:.4f} - t_acc {:.3f}' .format(t_loss, t_acc)) print('RNN-LSTM - Time: {}'.format(time.time() - start)) return []
def main(videoPath): global t_clip global t_feature2rnn # global t_pred_vec global t_cnn_out_avg global t_pred_avg global t_rnn_pred global cnn_model global rnn_model # build graph cnn_graph = tf.Graph() with cnn_graph.as_default(): t_clip = tf.placeholder(tf.float32, [None, 250, 250, 3], name='clip_input') with tf.name_scope('CNN'): cnn_model = cnn.CNN(data_format='NHWC') cnn_out = cnn_model(t_clip) # (batch, 2) print(cnn_out.shape) # pred_vec = tf.argmax(cnn_out, axis=1, output_type=tf.int32) # (batch,) # t_cnn_out_avg = tf.reduce_sum(cnn_out, axis=0) # (classes, ) batch维度上累加 t_cnn_out_avg = tf.reduce_mean(cnn_out, axis=0) # (classes, ) batch维度上均值 # print(cnn_out_avg.shape) t_pred_avg = tf.nn.softmax(t_cnn_out_avg) # print(t_pred_avg.shape) #(2,) # print(t_pred_avg.dtype) <class 'float'> # return # t_pred_vec = tf.argmax(cnn_out, axis=1, output_type=tf.float32) # (batch,) # t_pred_avg = tf.reduce_mean(t_pred_vec) rnn_graph = tf.Graph() with rnn_graph.as_default(): t_feature2rnn = tf.placeholder(tf.float32, [1, None, 128], name='rnn_input') with tf.name_scope('RNN'): rnn_model = rnn.RNN('GRU', n_hidden=10, n_classes=2) logits = rnn_model(t_feature2rnn, batch_sz=1) t_rnn_pred = tf.nn.softmax(logits) # 可信度 (1, 2) # 0:a, 1:1-b, 2:1-a-b # create sess & restore param sess_conf = tf.ConfigProto() sess_conf.gpu_options.allow_growth = True # sess_conf.gpu_options.per_process_gpu_memory_fraction = 0.75 cnn_sess = tf.Session(graph=cnn_graph, config=sess_conf) rnn_sess = tf.Session(graph=rnn_graph, config=sess_conf) with cnn_graph.as_default(): saver = tf.train.Saver() saver.restore(cnn_sess, cnn_ckpt) with rnn_graph.as_default(): saver = tf.train.Saver() saver.restore(rnn_sess, rnn_ckpt) # saver.restore(cnn_sess, cnn_ckpt) # saver.restore(rnn_sess, rnn_ckpt) # work on video videoPath = r'D:\Lab408\cnn_rnn\src_dir\1.mp4' # videoPath = r'D:\Lab408\cnn_rnn\src_dir\2.mp4' # videoPath = r'D:\Lab408\cnn_rnn\src_dir\3.mp4' # videoPath = r'D:\Lab408\cnn_rnn\src_dir\4.mp4' # videoPath = r'D:\Lab408\cnn_rnn\src_dir\5.mp4' # videoPath = r'D:\Lab408\cnn_rnn\src_dir\fire-smoke-small(13).avi' # videoPath = r'D:\Lab408\cnn_rnn\src_dir\NIST Re-creation of The Station Night Club fire without sprinklers (1).mp4' handelVideo(cnn_sess, rnn_sess, videoPath, videoPath + '.log') cnn_sess.close() rnn_sess.close() return
def create_agents(self): """ create agents, may use parameter sharing when self.training = False, this function will load the saved models """ input_shape = self.env.observation_space output_shape_NO = self.env.action_space_NO output_shape_VNO = self.env.action_space_VNO if self.training: # ! create the first unique agent # create NO agent if self.dueling: model = rnn.DuelingRNN(input_shape, output_shape_NO) target_model = rnn.DuelingRNN(input_shape, output_shape_NO) else: model = rnn.RNN(input_shape, output_shape_NO) target_model = rnn.RNN(input_shape, output_shape_NO) target_model.load_state_dict(model.state_dict()) self.agent_NO = SingleNoAgent.SingleNoAgent( model, target_model, self.env.action_space_NO, self.batch_size, self.learning_rate, self.gamma, self.epsilon, self.hysteretic, training=self.training) # create VNO agent if self.dueling: model = rnn.DuelingRNN(input_shape, output_shape_VNO) target_model = rnn.DuelingRNN(input_shape, output_shape_VNO) else: model = rnn.RNN(input_shape, output_shape_VNO) target_model = rnn.RNN(input_shape, output_shape_VNO) target_model.load_state_dict(model.state_dict()) self.agent_VNO = SingleVnoAgent.SingleVnoAgent( model, target_model, self.env.action_space_VNO, self.batch_size, self.learning_rate, self.gamma, self.epsilon, self.hysteretic, training=self.training) else: # ! load the saved (trained) models # load NO agent if self.dueling: model = rnn.DuelingRNN(input_shape, output_shape_NO) else: model = rnn.RNN(input_shape, output_shape_NO) target_model = None path = 'individual_model/agent_{}/model'.format(0) path = os.path.join(self.model_path, path) model.load_state_dict(torch.load(path)) model.eval() self.agent_NO = SingleNoAgent.SingleNoAgent( model, target_model, self.env.action_space_NO, self.batch_size, self.learning_rate, self.gamma, self.epsilon, self.hysteretic, training=self.training) # load VNO agent if self.dueling: model = rnn.DuelingRNN(input_shape, output_shape_VNO) else: model = rnn.RNN(input_shape, output_shape_VNO) target_model = None path = 'individual_model/agent_{}/model'.format(1) path = os.path.join(self.model_path, path) model.load_state_dict(torch.load(path)) model.eval() self.agent_VNO = SingleVnoAgent.SingleVnoAgent( model, target_model, self.env.action_space_VNO, self.batch_size, self.learning_rate, self.gamma, self.epsilon, self.hysteretic, training=self.training)
import wawProcessor as ww import rnn as r import os import tensorflow as tf from tensorflow.python.keras.optimizers import Adadelta processor = ww.Processor() RNN = r.RNN() model = RNN.getRNN() totoOriginal = processor.openWave( "C:\\Users\\NKF786\\PycharmProjects\\musicEncoding\\originals" + os.sep + "africa-toto-8bit.wav", 'rb') totoCover = processor.openWave( "C:\\Users\\NKF786\\PycharmProjects\\musicEncoding\\covers" + os.sep + "toto-metal-cover-cut-8bit.wav", 'rb') x_train, y_train = processor.getRNNTrainSequences(totoOriginal, totoCover, 50) # print(x_train.shape) # print(y_train.shape) originalSequences = x_train x_train, y_train, x_test, y_test = processor.getValidationSet(x_train, y_train) RNN.fitRNN(model, x_train, y_train, (x_test, y_test)) processor.writeRNNCover(originalSequences, model) totoOriginal.close() totoCover.close()
# optimizer_title) # Based on the above comparisons, an optimal RNN would have epochs = 40 batch_size = 3 memory_length = 20 input_shape = (499, 1) rnn_layer = 'GRU' activation = 'tanh' batch_norm = False optimizer = 'nadam' # Load the training and testing data (x_train, y_train), (x_test, y_test) = load_rnn_data.load_data('data.npy', normalize_data=True, train_split=.66) # Build and train the model test_rnn = rnn.RNN(input_shape, memory_length, rnn_layer, activation, batch_norm, optimizer) rnn_model_history = test_rnn.train_rnn_model(x_train, y_train, batch_size, epochs) # Evaluate the final model test_mse = test_rnn.evaluate_rnn_model(x_test, y_test, batch_size) print('Test MSE: {}'.format(test_mse))
model = gan_rnn.GAN_RNN(g_input_step, g_input_size, g_hidden_size, g_output_step, g_batch_size, g_rate, g_epochs, d_input_step, d_input_size, d_hidden_size, d_batch_size, d_rate, d_epochs, num_epochs, print_interval, num_epochs_test, args.attention, args.wgan, args.w_clip, gumbel, data_file) else: if args.feature == 0: model = gan_rnn_gcn.GAN_RNN_GCN( g_input_step, g_input_size, g_hidden_size, g_output_step, g_batch_size, g_rate, g_epochs, d_input_step, d_input_size, d_hidden_size, d_batch_size, d_rate, d_epochs, num_epochs, print_interval, num_epochs_test, args.attention, args.wgan, args.w_clip, num_support, gumbel, graph_file, data_file) else: model = gan_rnn_gcn_feature.GAN_RNN_GCN_Feature( g_input_step, g_input_size, g_hidden_size, g_output_step, g_batch_size, g_rate, g_epochs, d_input_step, d_input_size, d_hidden_size, d_batch_size, d_rate, d_epochs, num_epochs, print_interval, num_epochs_test, args.attention, args.wgan, args.w_clip, num_support, gumbel, graph_file, data_file) if args.baseline == 1: model = rnn.RNN(g_input_step, g_input_size, g_hidden_size, g_output_step, g_batch_size, g_rate, num_epochs, print_interval, num_epochs_test, args.attention, data_file) model.build_model() model.train()
def __init__(self, rnn_type, ntoken, ninp, nhid, nhidlast, nlayers, dropout=0.5, dropouth=0.5, dropouti=0.5, dropoute=0.1, wdrop=0, tie_weights=False, ldropout=0.5, n_experts=10, ndistilstudents=0, unigram_prob_on_zero=False, unigram_frequencies=None, rnd_models=None): super(RNNModel, self).__init__() self.ndistilstudents=ndistilstudents self.unigram_prob_on_zero=unigram_prob_on_zero self.unigram_frequencies=unigram_frequencies self.rnd_models=rnd_models self.use_dropout = True self.lockdrop = LockedDropout() self.encoder = nn.Embedding(ntoken, ninp) rnn_type = rnn_type.lower() self.rnns = [rnn.RNN(rnn_type, ninp if l == 0 else nhid, nhid if l != nlayers - 1 else nhidlast, 1, dropout=0, n_students=ndistilstudents) for l in range(nlayers)] if wdrop: self.rnns = [WeightDrop(rnn, ['_W', '_U'], dropout=wdrop if self.use_dropout else 0) for rnn in self.rnns] self.rnns = torch.nn.ModuleList(self.rnns) self.prior = nn.Linear(nhidlast, n_experts, bias=False) latent_linear = nn.Linear(nhidlast, n_experts*ninp, bias=not unigram_prob_on_zero) #latent_linear = nn.Linear(nhidlast, n_experts*ninp, bias=True) self.latent = nn.Sequential(latent_linear, nn.Tanh()) self.decoder = nn.Linear(ninp, ntoken) self.decoder.bias.data[:] = torch.zeros_like(self.decoder.bias.data) self.decoder_gain = nn.Parameter(torch.ones(ninp), requires_grad=False) #self.decoder_gain = nn.Parameter(torch.scalar_tensor(1.0), requires_grad=True) #self.decoder_gain = nn.Parameter(torch.zeros(ninp)) # Optionally tie weights as in: # "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016) # https://arxiv.org/abs/1608.05859 # and # "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016) # https://arxiv.org/abs/1611.01462 if tie_weights: #if nhid != ninp: # raise ValueError('When using the tied flag, nhid must be equal to emsize') self.decoder.weight = self.encoder.weight if unigram_prob_on_zero: self.decoder.bias.requires_grad=False pass self.init_weights() self.rnn_type = rnn_type self.ninp = ninp self.nhid = nhid self.nhidlast = nhidlast self.nlayers = nlayers self.dropout = dropout self.dropouti = dropouti self.dropouth = dropouth self.dropoute = dropoute self.ldropout = ldropout self.dropoutl = ldropout self.n_experts = n_experts self.ntoken = ntoken size = 0 for p in self.parameters(): size += p.nelement() print('param size: {}'.format(size))
taxonomy_infile = "/dev/shm/b/wiki_taxonomy_lemmatized.pkl" #pickle 2 parent_taxonomy_infile = "data/parent_taxonomy.pkl" vector_model_infile = "/dev/shm/a/wiki_text_20161201_1to4_200d.model" mag_infile = "/dev/shm/b/mag2.pkl" ccs_infile = "/dev/shm/b/acm_ccs.pkl" """ taxonomy_infile = "/dev/shm/a/wiki_taxonomy_lemmatized.pkl" #pickle 3 parent_taxonomy_infile = "data/parent_taxonomy.pkl" vector_model_infile = "/dev/shm/a/wiki_text_20161201_1to4_200d.model" mag_infile = "/dev/shm/a/mag2.pkl" ccs_infile = "/dev/shm/a/acm_ccs.pkl" mlp_infile = "data/mlp_model.h5" expert_filter_infile = "data/expert_annotation.json" #mag_fos_infile = "data/fos_levelname.csv" taxonomy = load_taxonomy(taxonomy_infile) taxonomy = preprocessTaxonomy(taxonomy) w2v_model = load_vector_model(vector_model_infile) mag = get_data_from_pickle(mag_infile) ccs = get_data_from_pickle(ccs_infile) parent_taxonomy = load_parent_taxonomy(parent_taxonomy_infile) expert_filter = load_json(expert_filter_infile) if os.path.exists(mlp_infile): mlp_model = load_model(mlp_infile) rnn_model = rnn.RNN()
def print_emissions(net, fname, i2voc): o = open("Emissions.{}.txt".format(fname),'w') e_list = net.emissions_list() for i in range(net.hidden_dim): listed = [(float(e_list[j][i]), str(i2voc[j])) for j in range(net.vocab_size)] listed.sort() listed.reverse() o.write("\n%d\n" % i) for prob, word in listed[:50]: o.write(" {:10.8f} {:10s}\n".format(100*prob, str(word))) o.close() vocab_size = len(corpus.dict) print("Vocab size: %d" % vocab_size) net = rnn.RNN(vocab_size, args).to(device) net.corpus = corpus # HACK def repackage_hidden(h): """Wraps hidden states in new Tensors, to detach them from their history.""" if isinstance(h, torch.Tensor): return h.detach() else: return tuple(repackage_hidden(v) for v in h) def get_batch(source, i, tag_source=None): seq_len = min(args.max_len, source.size()[1] - 1 - i) data = source[:,i:i+seq_len+1] if tag_source is not None: tags = tag_source[:,i:i+seq_len+1] return data, tags
lines, kmer, 1, rnn_model_name) vocab_len = len(indexed_word) + 1 # get feature and labels features, labels = tk.feature_label_extractor(tokenized_sequence, vocab_len) # split training and test sets split = int(0.8 * len(features)) x_train = features[:split] x_test = features[split:] y_train = labels[:split] y_test = labels[split:] # initialize the models rnn_model = rnn.RNN(vocab_len, 1, rnn_model_name) # train and save outputs rnn_history = rnn_model.train(x_train, y_train, x_test, y_test, e=epochs) pp.save_and_plot(rnn_history, rnn_model_name) print('Trained RNN saved, outputs generated and saved.') # CNN print('Training CNN') cnn_model_name = str(kmer) + '_kmer_CNN' # shuffle the data random.shuffle(lines)
def __init__(self, rnn_type, ntoken, ninp, nhid, nhidlast, nlayers, dropout=0.5, dropouth=0.5, dropouti=0.5, dropoute=0.1, wdrop=0, tie_weights=False, ldropout=0.5, n_experts=10): super(RNNModel, self).__init__() self.use_dropout = True self.lockdrop = LockedDropout() self.encoder = nn.Embedding(ntoken, ninp) self.rnns = [ rnn.RNN(rnn_type, ninp if l == 0 else nhid, nhid if l != nlayers - 1 else nhidlast, 1, dropout=0) for l in range(nlayers) ] if wdrop: self.rnns = [ WeightDrop(rnn, ['_W', '_U'], dropout=wdrop if self.use_dropout else 0) for rnn in self.rnns ] self.rnns = torch.nn.ModuleList(self.rnns) self.prior = nn.Linear(nhidlast, n_experts, bias=False) self.latent = nn.Sequential(nn.Linear(nhidlast, n_experts * ninp), nn.Tanh()) self.decoder = nn.Linear(ninp, ntoken) # Optionally tie weights as in: # "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016) # https://arxiv.org/abs/1608.05859 # and # "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016) # https://arxiv.org/abs/1611.01462 if tie_weights: #if nhid != ninp: # raise ValueError('When using the tied flag, nhid must be equal to emsize') self.decoder.weight = self.encoder.weight self.init_weights() self.rnn_type = rnn_type self.ninp = ninp self.nhid = nhid self.nhidlast = nhidlast self.nlayers = nlayers self.dropout = dropout self.dropouti = dropouti self.dropouth = dropouth self.dropoute = dropoute self.ldropout = ldropout self.dropoutl = ldropout self.n_experts = n_experts self.ntoken = ntoken size = 0 for p in self.parameters(): size += p.nelement() print('param size: {}'.format(size))
alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" # create mapping of characters to integers (0-25) and the reverse char_to_int = dict((c, i) for i, c in enumerate(alphabet)) int_to_char = dict((i, c) for i, c in enumerate(alphabet)) # prepare the dataset of input to output pairs encoded as integers seq_length = 1 dataX = [] dataY = [] for i in range(0, len(alphabet) - seq_length, 1): seq_in = alphabet[i:i + seq_length] seq_out = alphabet[i + seq_length] dataX.append([char_to_int[char] for char in seq_in]) dataY.append(char_to_int[seq_out]) print(seq_in, '->', seq_out) # reshape X to be [samples, time steps, features] X = np.reshape(dataX, (len(dataX), seq_length, 1)) # normalize X = X / float(len(alphabet)) # one hot encode the output variable y = np_utils.to_categorical(dataY) print(dataX) print(y) vocabulary_size = len(char_to_int) model = rnn.RNN(vocabulary_size) o, s = model.forward(X[10]) print(o.shape) print(o)