def forward_pass(model, data_gen, batch_size, quantiles, gpu = True): model.reset(batch_size, gpu = gpu) #Get input and target data, one-hot encode discrete variables, continuous variables have already been normalized in_seq_continuous, in_seq_discrete, future_in_seq_discrete, target_seq = next(data_gen) in_seq_discrete = one_hot(in_seq_discrete, [24, 31, 12]) future_in_seq_discrete = one_hot(future_in_seq_discrete, [24, 31, 12]) #forward pass net_out, vs_weights = model(in_seq_continuous, in_seq_discrete, None, future_in_seq_discrete) loss = torch.mean(QuantileLoss(net_out, target_seq ,quantiles)) return loss, net_out, vs_weights, (in_seq_continuous, in_seq_discrete, future_in_seq_discrete, target_seq)
def predict(data, labels, n, c): x = tf.placeholder(tf.float32, [None, n]) W = tf.Variable(tf.zeros([n, c])) b = tf.Variable(tf.zeros([c])) y = tf.nn.softmax(tf.matmul(x, W) + b) y_ = tf.placeholder(tf.float32, [None, c]) saver = tf.train.Saver() sess = tf.Session() saver.restore(sess, 'model.ckpt') correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) a, b = data, d.one_hot(labels) actual = (sess.run(tf.argmax(y_, 1), feed_dict={y_: b})) predicted = (sess.run(tf.argmax(y, 1), feed_dict={x: a, y_: b})) # print(predicted) # print(actual) display(actual, predicted) print('Accuracy = ', sess.run(accuracy, feed_dict={x: a, y_: b}))
def predict(data, labels, n, c): x = tf.placeholder(tf.float32, [None, n]) W = tf.Variable(tf.zeros([n, c])) b = tf.Variable(tf.zeros([c])) y = tf.nn.softmax(tf.matmul(x, W) + b) y_ = tf.placeholder(tf.float32, [None, c]) saver = tf.train.Saver() sess = tf.Session() saver.restore(sess, "model.ckpt") correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) a, b = data, d.one_hot(labels) actual = sess.run(tf.argmax(y_, 1), feed_dict={y_: b}) predicted = sess.run(tf.argmax(y, 1), feed_dict={x: a, y_: b}) # print(predicted) # print(actual) display(actual, predicted) print("Accuracy = ", sess.run(accuracy, feed_dict={x: a, y_: b}))
print(f'Sigmoid function: {network.y}') operation = lo.LogicalOperations() print(f'NOT: {operation.not_func(1)}') print(f'AND: {operation.and_func(1, 1)}') print(f'OR: {operation.or_func(0, 0)}') print(f'XOR: {operation.xor_func(0, 1)}') ''' ''' layer.py activation.py ''' # INPUT X, y = data.data_spiral() y_onehot = data.one_hot(y) ''' print(f"X: {X}") print(f"y: {y}") print(f"y_onehot: {y_onehot}") ''' # Model Architecture layer1 = layer.Layer_Dense(2, 4) activation1 = activation.Activation_LeakyReLU() layer2 = layer.Layer_Dense(4, 3) activation2 = activation.Activation_Softmax() loss1 = loss.Loss_CategoricalCrossEntropy() metric1 = metric.Metric_Accuracy()
def train(sess, model, optimizer, log_dir, batch_size, num_sweeps_per_summary, num_sweeps_per_save, train_input_seqs, train_reset_seqs, train_label_seqs, test_input_seqs, test_reset_seqs, test_label_seqs): """ Train a model and export summaries. `log_dir` will be *replaced* if it already exists, so it certainly shouldn't be anything generic like `/home/user`. Args: sess: A TensorFlow `Session`. model: An `LSTMModel`. optimizer: An `Optimizer`. log_dir: A string. The full path to the log directory. batch_size: An integer. The number of sequences in a batch. num_sweeps_per_summary: An integer. The number of sweeps between summaries. num_sweeps_per_save: An integer. The number of sweeps between saves. train_input_seqs: A list of 2-D NumPy arrays, each with shape `[duration, input_size]`. train_reset_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. train_label_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. test_input_seqs: A list of 2-D NumPy arrays, each with shape `[duration, input_size]`. test_reset_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. test_label_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. """ ema = tf.train.ExponentialMovingAverage(decay=0.5) update_train_loss_ema = ema.apply([model.loss]) train_loss_ema = ema.average(model.loss) tf.summary.scalar('train_loss_ema', train_loss_ema) train_accuracy = tf.placeholder(tf.float32, name='train_accuracy') train_edit_dist = tf.placeholder(tf.float32, name='train_edit_dist') test_accuracy = tf.placeholder(tf.float32, name='test_accuracy') test_edit_dist = tf.placeholder(tf.float32, name='test_edit_dist') #values = [train_accuracy, train_edit_dist, test_accuracy, test_edit_dist] #tags = [value.op.name for value in values] tf.summary.scalar('learning_rate', optimizer.learning_rate) for value in [ train_accuracy, train_edit_dist, test_accuracy, test_edit_dist ]: tf.summary.scalar(value.op.name, value) #tf.summary.scalar(tags, tf.stack(values)) summary_op = tf.summary.merge_all() if os.path.exists(log_dir): shutil.rmtree(log_dir) summary_writer = tf.summary.FileWriter(logdir=log_dir, graph=sess.graph) saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) num_sweeps_visited = 0 start_time = time.time() train_gen = data.sweep_generator( [train_input_seqs, train_reset_seqs, train_label_seqs], batch_size=batch_size, shuffle=True, num_sweeps=None) while num_sweeps_visited <= optimizer.num_train_sweeps: if num_sweeps_visited % num_sweeps_per_summary == 0: train_prediction_seqs = models.predict(sess, model, train_input_seqs, train_reset_seqs) train_accuracy_, train_edit_dist_ = metrics.compute_metrics( train_prediction_seqs, train_label_seqs) test_prediction_seqs = models.predict(sess, model, test_input_seqs, test_reset_seqs) test_accuracy_, test_edit_dist_ = metrics.compute_metrics( test_prediction_seqs, test_label_seqs) summary = sess.run(summary_op, feed_dict={ train_accuracy: train_accuracy_, train_edit_dist: train_edit_dist_, test_accuracy: test_accuracy_, test_edit_dist: test_edit_dist_ }) summary_writer.add_summary(summary, global_step=num_sweeps_visited) status_path = os.path.join(log_dir, 'status.txt') with open(status_path, 'w') as f: line = '%05.1f ' % ((time.time() - start_time) / 60) line += '%04d ' % num_sweeps_visited line += '%.6f %08.3f ' % (train_accuracy_, train_edit_dist_) line += '%.6f %08.3f ' % (test_accuracy_, test_edit_dist_) print(line, file=f) label_path = os.path.join(log_dir, 'test_label_seqs.pkl') with open(label_path, 'wb') as f: cPickle.dump(test_label_seqs, f) pred_path = os.path.join(log_dir, 'test_prediction_seqs.pkl') with open(pred_path, 'wb') as f: cPickle.dump(test_prediction_seqs, f) vis_filename = 'test_visualizations_%06d.png' % num_sweeps_visited vis_path = os.path.join(log_dir, vis_filename) fig, axes = data.visualize_predictions(test_prediction_seqs, test_label_seqs, model.target_size) axes[0].set_title(line) plt.tight_layout() plt.savefig(vis_path) plt.close(fig) if num_sweeps_visited % num_sweeps_per_save == 0: saver.save(sess, os.path.join(log_dir, 'model.ckpt')) train_inputs, train_resets, train_labels = train_gen.__next__() # We squeeze here because otherwise the targets would have shape # [batch_size, duration, 1, num_classes]. train_targets = data.one_hot(train_labels, model.target_size) train_targets = train_targets.squeeze(axis=2) _, _, num_sweeps_visited = sess.run( [ optimizer.optimize_op, update_train_loss_ema, optimizer.num_sweeps_visited ], feed_dict={ model.inputs: train_inputs, model.resets: train_resets, model.targets: train_targets, model.training: True })
def train(sess, model, optimizer, log_dir, batch_size, num_sweeps_per_summary, num_sweeps_per_save, train_input_seqs, train_reset_seqs, train_label_seqs, test_input_seqs, test_reset_seqs, test_label_seqs, args): """ Train a model and export summaries. `log_dir` will be *replaced* if it already exists, so it certainly shouldn't be anything generic like `/home/user`. Args: sess: A TensorFlow `Session`. model: An `LSTMModel`. optimizer: An `Optimizer`. log_dir: A string. The full path to the log directory. batch_size: An integer. The number of sequences in a batch. num_sweeps_per_summary: An integer. The number of sweeps between summaries. num_sweeps_per_save: An integer. The number of sweeps between saves. train_input_seqs: A list of 2-D NumPy arrays, each with shape `[duration, input_size]`. train_reset_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. train_label_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. test_input_seqs: A list of 2-D NumPy arrays, each with shape `[duration, input_size]`. test_reset_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. test_label_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. args: An object containing processed arguments as attributes. """ #print(test_label_seqs) ema = tf.train.ExponentialMovingAverage(decay=0.5) #0.5 update_train_loss_ema = ema.apply([model.loss]) train_loss_ema = ema.average(model.loss) tf.summary.scalar('train_loss_ema', train_loss_ema) train_accuracy = tf.placeholder(tf.float32, name='train_accuracy') train_edit_dist = tf.placeholder(tf.float32, name='train_edit_dist') test_accuracy = tf.placeholder(tf.float32, name='test_accuracy') test_edit_dist = tf.placeholder(tf.float32, name='test_edit_dist') #values = [train_accuracy, train_edit_dist, test_accuracy, test_edit_dist] #tags = [value.op.name for value in values] tf.summary.scalar('learning_rate', optimizer.learning_rate) for value in [ train_accuracy, train_edit_dist, test_accuracy, test_edit_dist ]: tf.summary.scalar(value.op.name, value) #tf.summary.scalar(tags, tf.stack(values)) summary_op = tf.summary.merge_all() if os.path.exists(log_dir): shutil.rmtree(log_dir) summary_writer = tf.summary.FileWriter(logdir=log_dir, graph=sess.graph) saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) num_sweeps_visited = 0 start_time = time.time() train_gen = data.sweep_generator( [train_input_seqs, train_reset_seqs, train_label_seqs], batch_size=batch_size, shuffle=True, num_sweeps=None) while num_sweeps_visited <= optimizer.num_train_sweeps: if num_sweeps_visited % num_sweeps_per_summary == 0: test_prediction_seqs = [] train_prediction_seqs, logits = models.predict( sess, model, train_input_seqs, train_reset_seqs) train_accuracy_, train_edit_dist_, train_confusion_matrix = metrics.compute_metrics( train_prediction_seqs, train_label_seqs, log_dir, 'train') print('test_input_seqs:', len(test_input_seqs)) err = 0.0 # init empty predictions no_of_samples = sum([len(seq) for seq in test_input_seqs]) entropy_matrix = np.zeros( (args.sample_times, no_of_samples, 10)) #batch_size entropy_matrix_1 = np.zeros( (args.sample_times, no_of_samples, 1)) # batch_size for sample_id in range(50): test_prediction_seqs, softmax_seqs = models.predict( sess, model, test_input_seqs, test_reset_seqs) #print('softmax_dur:', softmax_seqs) entropy_matrix[sample_id] = np.vstack(softmax_seqs) entropy_matrix_1[sample_id] = np.vstack(test_prediction_seqs) ''' #Variation Ratio value,count = mode(entropy_matrix_1,axis=0) print('count.shape:', value[0,200,0], count[0,200,0]) varition_ratio=1-count/50.0 print(varition_ratio.shape) entropy=np.squeeze(varition_ratio) entropy[entropy > 0.5] = 222 entropy[entropy < 0.5] = 111''' #MC Dropout - Entropy #print('entropy_matrix.shape:',entropy_matrix.shape,entropy_matrix[0,0,:]) entropy_matrix_mean = np.mean(entropy_matrix, axis=0) #print('entropy_matrix_mean.shape:', entropy_matrix_mean.shape,entropy_matrix_mean[0,:]) entropy_log = np.log2(entropy_matrix_mean, where=(entropy_matrix_mean != 0.0)) #print('entropy_log:',entropy_log[0,:]) entropy_log_mul = entropy_matrix_mean * entropy_log #print('entropy_log_mul.shape:', entropy_log_mul.shape, entropy_log_mul[0, :]) entropy = np.sum(entropy_log_mul, axis=1) * -1 #print('entropy.shape:', entropy.shape,entropy[0]) normalized_entropy = (entropy - np.mean(entropy, axis=0)) / np.std( entropy, axis=0) test_accuracy_, test_edit_dist_, test_confusion_matrix = metrics.compute_metrics( test_prediction_seqs, test_label_seqs, log_dir, 'test') summary = sess.run(summary_op, feed_dict={ train_accuracy: train_accuracy_, train_edit_dist: train_edit_dist_, test_accuracy: test_accuracy_, test_edit_dist: test_edit_dist_ }) print('kris_po:: num_sweeps_visited:', num_sweeps_visited) summary_writer.add_summary(summary, global_step=num_sweeps_visited) summary_writer.add_summary(train_confusion_matrix, global_step=num_sweeps_visited) summary_writer.add_summary(test_confusion_matrix, global_step=num_sweeps_visited) status_path = os.path.join(log_dir, 'status.txt') with open(status_path, 'w') as f: line = '%05.1f ' % ((time.time() - start_time) / 60) line += '%04d ' % num_sweeps_visited line += '%.6f %08.3f ' % (train_accuracy_, train_edit_dist_) line += '%.6f %08.3f ' % (test_accuracy_, test_edit_dist_) print(line, file=f) label_path = os.path.join(log_dir, 'test_label_seqs.pkl') with open(label_path, 'wb') as f: cPickle.dump(test_label_seqs, f) pred_path = os.path.join(log_dir, 'test_prediction_seqs.pkl') with open(pred_path, 'wb') as f: cPickle.dump(test_prediction_seqs, f) if num_sweeps_visited == 1197: vis_filename = 'test_visualizations_%06d.png' % num_sweeps_visited vis_path = os.path.join(log_dir, vis_filename) fig, axes = data.visualize_predictions(test_prediction_seqs, test_label_seqs, model.target_size, normalized_entropy) axes[0].set_title(line) plt.tight_layout() plt.savefig(vis_path) plt.close(fig) if num_sweeps_visited % num_sweeps_per_save == 0: saver.save(sess, os.path.join(log_dir, 'model.ckpt')) train_inputs, train_resets, train_labels = train_gen.next() # We squeeze here because otherwise the targets would have shape # [batch_size, duration, 1, num_classes]. train_targets = data.one_hot(train_labels, model.target_size) train_targets = train_targets.squeeze(axis=2) _, _, num_sweeps_visited = sess.run( [ optimizer.optimize_op, update_train_loss_ema, optimizer.num_sweeps_visited ], feed_dict={ model.inputs: train_inputs, model.resets: train_resets, model.targets: train_targets, model.training: True })
def fakeDataset(character): x = torch.zeros(batch_size, 1, len(dataset.corpus)) x[0][0] = one_hot(dataset.getLabelFromChar(character), len(dataset.corpus)) y = None return [x,y]
def on_hot_hamming(a, b): return spatial.distance.hamming( one_hot(a, n_cols=VOCAB).flatten(), one_hot(b, n_cols=VOCAB).flatten() )
def train(sess, model, optimizer, log_dir, batch_size, num_sweeps_per_summary, num_sweeps_per_save, train_input_seqs, train_reset_seqs, train_label_seqs, test_input_seqs, test_reset_seqs, test_label_seqs): """ Train a model and export summaries. `log_dir` will be *replaced* if it already exists, so it certainly shouldn't be anything generic like `/home/user`. Args: sess: A TensorFlow `Session`. model: An `LSTMModel`. optimizer: An `Optimizer`. log_dir: A string. The full path to the log directory. batch_size: An integer. The number of sequences in a batch. num_sweeps_per_summary: An integer. The number of sweeps between summaries. num_sweeps_per_save: An integer. The number of sweeps between saves. train_input_seqs: A list of 2-D NumPy arrays, each with shape `[duration, input_size]`. train_reset_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. train_label_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. test_input_seqs: A list of 2-D NumPy arrays, each with shape `[duration, input_size]`. test_reset_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. test_label_seqs: A list of 2-D NumPy arrays, each with shape `[duration, 1]`. """ ema = tf.train.ExponentialMovingAverage(decay=0.5) update_train_loss_ema = ema.apply([model.loss]) train_loss_ema = ema.average(model.loss) tf.scalar_summary('train_loss_ema', train_loss_ema) train_accuracy = tf.placeholder(tf.float32, name='train_accuracy') train_edit_dist = tf.placeholder(tf.float32, name='train_edit_dist') test_accuracy = tf.placeholder(tf.float32, name='test_accuracy') test_edit_dist = tf.placeholder(tf.float32, name='test_edit_dist') values = [train_accuracy, train_edit_dist, test_accuracy, test_edit_dist] tags = [value.op.name for value in values] tf.scalar_summary('learning_rate', optimizer.learning_rate) tf.scalar_summary(tags, tf.pack(values)) summary_op = tf.merge_all_summaries() if os.path.exists(log_dir): shutil.rmtree(log_dir) summary_writer = tf.train.SummaryWriter(logdir=log_dir, graph=sess.graph) saver = tf.train.Saver() sess.run(tf.initialize_all_variables()) num_sweeps_visited = 0 start_time = time.time() train_gen = data.sweep_generator( [train_input_seqs, train_reset_seqs, train_label_seqs], batch_size=batch_size, shuffle=True, num_sweeps=None) while num_sweeps_visited <= optimizer.num_train_sweeps: if num_sweeps_visited % num_sweeps_per_summary == 0: train_prediction_seqs = models.predict( sess, model, train_input_seqs, train_reset_seqs) train_accuracy_, train_edit_dist_ = metrics.compute_metrics( train_prediction_seqs, train_label_seqs) test_prediction_seqs = models.predict( sess, model, test_input_seqs, test_reset_seqs) test_accuracy_, test_edit_dist_ = metrics.compute_metrics( test_prediction_seqs, test_label_seqs) summary = sess.run(summary_op, feed_dict={train_accuracy: train_accuracy_, train_edit_dist: train_edit_dist_, test_accuracy: test_accuracy_, test_edit_dist: test_edit_dist_}) summary_writer.add_summary(summary, global_step=num_sweeps_visited) status_path = os.path.join(log_dir, 'status.txt') with open(status_path, 'w') as f: line = '%05.1f ' % ((time.time() - start_time)/60) line += '%04d ' % num_sweeps_visited line += '%.6f %08.3f ' % (train_accuracy_, train_edit_dist_) line += '%.6f %08.3f ' % (test_accuracy_, test_edit_dist_) print(line, file=f) label_path = os.path.join(log_dir, 'test_label_seqs.pkl') with open(label_path, 'w') as f: cPickle.dump(test_label_seqs, f) pred_path = os.path.join(log_dir, 'test_prediction_seqs.pkl') with open(pred_path, 'w') as f: cPickle.dump(test_prediction_seqs, f) vis_filename = 'test_visualizations_%06d.png' % num_sweeps_visited vis_path = os.path.join(log_dir, vis_filename) fig, axes = data.visualize_predictions(test_prediction_seqs, test_label_seqs, model.target_size) axes[0].set_title(line) plt.tight_layout() plt.savefig(vis_path) plt.close(fig) if num_sweeps_visited % num_sweeps_per_save == 0: saver.save(sess, os.path.join(log_dir, 'model.ckpt')) train_inputs, train_resets, train_labels = train_gen.next() # We squeeze here because otherwise the targets would have shape # [batch_size, duration, 1, num_classes]. train_targets = data.one_hot(train_labels, model.target_size) train_targets = train_targets.squeeze(axis=2) _, _, num_sweeps_visited = sess.run( [optimizer.optimize_op, update_train_loss_ema, optimizer.num_sweeps_visited], feed_dict={model.inputs: train_inputs, model.resets: train_resets, model.targets: train_targets, model.training: True})